imxyy1soope1/nix-js
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: tidy fetcher (partial)

Browse Source 
* shouldn't have used LLM to implement this...
This commit is contained in:
imxyy_soope_
2026-02-13 21:57:44 +08:00
parent 48a43bed55
commit d95a6e509c
12 changed files with 42 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
nix-js/runtime-ts/src/builtins/derivation.ts
View File

@@ -107,6 +107,13 @@ const extractArgs = (attrs: NixAttrs, outContext: NixStringContext): string[] =>
return argsList.map((a) => coerceToString(a, StringCoercionMode.ToString, true, outContext));
};
const outputPathName = (drvName: string, output: string) => {
if (output === "out") {
return drvName
}
return `${drvName}-${output}`
}
const structuredAttrsExcludedKeys = new Set([
"__structuredAttrs",
"__ignoreNulls",
@@ -296,7 +303,7 @@ export const derivationStrict = (args: NixValue): NixAttrs => {
let drvPath: string;
if (fixedOutputInfo) {
const pathName = Deno.core.ops.op_output_path_name(drvName, "out");
const pathName = outputPathName(drvName, "out");
const outPath = Deno.core.ops.op_make_fixed_output_path(
fixedOutputInfo.hashAlgo,
fixedOutputInfo.hash,
@@ -374,7 +381,7 @@ export const derivationStrict = (args: NixValue): NixAttrs => {
outputInfos = new Map<string, OutputInfo>();
for (const outputName of outputs) {
const pathName = Deno.core.ops.op_output_path_name(drvName, outputName);
const pathName = outputPathName(drvName, outputName);
const outPath = Deno.core.ops.op_make_store_path(`output:${outputName}`, drvModuloHash, pathName);
outputInfos.set(outputName, {
path: outPath,

40
nix-js/runtime-ts/src/builtins/io.ts
View File

@@ -19,6 +19,7 @@ import { getPathValue } from "../path";
import type { NixStringContext, StringWithContext } from "../string-context";
import { mkStringWithContext } from "../string-context";
import { isAttrs, isPath } from "./type-check";
import { baseNameOf } from "./path";
const importCache = new Map<string, NixValue>();
@@ -108,14 +109,6 @@ export interface FetchGitResult {
nar_hash: string | null;
}
export interface FetchHgResult {
out_path: string;
branch: string;
rev: string;
short_rev: string;
rev_count: number;
}
const normalizeUrlInput = (
args: NixValue,
): { url: string; hash?: string; name?: string; executable?: boolean } => {
@@ -139,15 +132,25 @@ const normalizeUrlInput = (
const normalizeTarballInput = (args: NixValue): { url: string; sha256?: string; name?: string } => {
const forced = force(args);
if (isAttrs(forced)) {
const url = forceStringNoCtx(forced.url);
const url = resolvePseudoUrl(forceStringNoCtx(forced.url));
const sha256 = "sha256" in forced ? forceStringNoCtx(forced.sha256) : undefined;
const name = "name" in forced ? forceStringNoCtx(forced.name) : undefined;
const nameRaw = "name" in forced ? forceStringNoCtx(forced.name) : undefined;
// FIXME: extract baseNameOfRaw
const name = nameRaw === "" ? baseNameOf(nameRaw) as string : nameRaw;
return { url, sha256, name };
} else {
return { url: forceStringNoCtx(forced) };
}
};
const resolvePseudoUrl = (url: string) => {
if (url.startsWith("channel:")) {
return `https://channels.nixos.org/${url.substring(8)}/nixexprs.tar.xz`
} else {
return url
}
}
export const fetchurl = (args: NixValue): string => {
const { url, hash, name, executable } = normalizeUrlInput(args);
const result: FetchUrlResult = Deno.core.ops.op_fetch_url(
@@ -212,21 +215,8 @@ export const fetchGit = (args: NixValue): NixAttrs => {
};
};
export const fetchMercurial = (args: NixValue): NixAttrs => {
const attrs = forceAttrs(args);
const url = forceStringValue(attrs.url);
const rev = "rev" in attrs ? forceStringValue(attrs.rev) : null;
const name = "name" in attrs ? forceStringValue(attrs.name) : null;
const result: FetchHgResult = Deno.core.ops.op_fetch_hg(url, rev, name);
return {
outPath: result.out_path,
branch: result.branch,
rev: result.rev,
shortRev: result.short_rev,
revCount: BigInt(result.rev_count),
};
export const fetchMercurial = (_args: NixValue): NixAttrs => {
throw new Error("Not implemented: fetchMercurial")
};
export const fetchTree = (args: NixValue): NixAttrs => {

1
nix-js/runtime-ts/src/builtins/misc.ts
View File

@@ -41,6 +41,7 @@ export const hasContext = context.hasContext;
export const hashFile =
(type: NixValue) =>
(p: NixValue): never => {
const ty = forceStringNoCtx(type);
throw new Error("Not implemented: hashFile");
};

30
nix-js/runtime-ts/src/builtins/path.ts
View File

@@ -25,36 +25,6 @@ import { mkStringWithContext, type NixStringContext } from "../string-context";
* - baseNameOf "foo" → "foo"
*/
export const baseNameOf = (s: NixValue): NixString => {
const forced = force(s);
// Path input → string output (no context)
if (isNixPath(forced)) {
const pathStr = forced.value;
if (pathStr.length === 0) {
return "";
}
let last = pathStr.length - 1;
if (pathStr[last] === "/" && last > 0) {
last -= 1;
}
let pos = last;
while (pos >= 0 && pathStr[pos] !== "/") {
pos -= 1;
}
if (pos === -1) {
pos = 0;
} else {
pos += 1;
}
return pathStr.substring(pos, last + 1);
}
// String input → string output (preserve context)
const context: NixStringContext = new Set();
const pathStr = coerceToString(s, StringCoercionMode.Base, false, context);

2
nix-js/runtime-ts/src/types/global.d.ts vendored
View File

@@ -21,7 +21,6 @@ declare global {
column: number | null;
};
function op_make_store_path(ty: string, hash_hex: string, name: string): string;
function op_output_path_name(drv_name: string, output_name: string): string;
function op_parse_hash(hash_str: string, algo: string | null): { hex: string; algo: string };
function op_make_fixed_output_path(
hash_algo: string,
@@ -49,7 +48,6 @@ declare global {
all_refs: boolean,
name: string | null,
): FetchGitResult;
function op_fetch_hg(url: string, rev: string | null, name: string | null): FetchHgResult;
function op_add_path(
path: string,
name: string | null,

31
nix-js/src/fetcher.rs
View File

@@ -13,7 +13,6 @@ mod archive;
pub(crate) mod cache;
mod download;
mod git;
mod hg;
mod metadata_cache;
pub use cache::FetcherCache;
@@ -47,15 +46,6 @@ pub struct FetchGitResult {
pub nar_hash: Option<String>,
}
#[derive(Serialize)]
pub struct FetchHgResult {
pub out_path: String,
pub branch: String,
pub rev: String,
pub short_rev: String,
pub rev_count: u64,
}
#[op2]
#[serde]
pub fn op_fetch_url<Ctx: RuntimeContext>(
@@ -119,7 +109,7 @@ pub fn op_fetch_url<Ctx: RuntimeContext>(
info!(bytes = data.len(), "Download complete");
let hash = crate::nix_utils::sha256_hex(&String::from_utf8_lossy(&data));
let hash = crate::nix_utils::sha256_hex(&data);
if let Some(ref expected) = expected_hash {
let normalized_expected = normalize_hash(expected);
@@ -228,9 +218,7 @@ pub fn op_fetch_tarball<Ctx: RuntimeContext>(
info!(bytes = data.len(), "Download complete");
info!("Extracting tarball");
let cache = FetcherCache::new().map_err(|e| NixRuntimeError::from(e.to_string()))?;
let (extracted_path, _temp_dir) = cache
.extract_tarball_to_temp(&data)
let (extracted_path, _temp_dir) = archive::extract_tarball_to_temp(&data)
.map_err(|e| NixRuntimeError::from(e.to_string()))?;
info!("Computing NAR hash");
@@ -311,20 +299,6 @@ pub fn op_fetch_git<Ctx: RuntimeContext>(
.map_err(|e| NixRuntimeError::from(e.to_string()))
}
#[op2]
#[serde]
pub fn op_fetch_hg(
#[string] url: String,
#[string] rev: Option<String>,
#[string] name: Option<String>,
) -> Result<FetchHgResult, NixRuntimeError> {
let cache = FetcherCache::new().map_err(|e| NixRuntimeError::from(e.to_string()))?;
let dir_name = name.unwrap_or_else(|| "source".to_string());
hg::fetch_hg(&cache, &url, rev.as_deref(), &dir_name)
.map_err(|e| NixRuntimeError::from(e.to_string()))
}
fn normalize_hash(hash: &str) -> String {
use base64::prelude::*;
if hash.starts_with("sha256-")
@@ -341,6 +315,5 @@ pub fn register_ops<Ctx: RuntimeContext>() -> Vec<deno_core::OpDecl> {
op_fetch_url::<Ctx>(),
op_fetch_tarball::<Ctx>(),
op_fetch_git::<Ctx>(),
op_fetch_hg(),
]
}

11
nix-js/src/fetcher/archive.rs
View File

@@ -1,5 +1,6 @@
use std::fs::{self, File};
use std::io::Cursor;
use std::os::unix::ffi::OsStrExt;
use std::path::{Path, PathBuf};
use flate2::read::GzDecoder;
@@ -125,7 +126,7 @@ fn extract_zip(data: &[u8], dest: &Path) -> Result<(), ArchiveError> {
fn strip_single_toplevel(temp_dir: &Path, dest: &Path) -> Result<PathBuf, ArchiveError> {
let entries: Vec<_> = fs::read_dir(temp_dir)?
.filter_map(|e| e.ok())
.filter(|e| !e.file_name().to_string_lossy().starts_with('.'))
.filter(|e| e.file_name().as_os_str().as_bytes()[0] != b'.')
.collect();
let source_dir = if entries.len() == 1 && entries[0].file_type()?.is_dir() {
@@ -182,6 +183,14 @@ fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<(), std::io::Error> {
Ok(())
}
pub fn extract_tarball_to_temp(
data: &[u8],
) -> Result<(PathBuf, tempfile::TempDir), ArchiveError> {
let temp_dir = tempfile::tempdir()?;
let extracted_path = extract_archive(data, temp_dir.path())?;
Ok((extracted_path, temp_dir))
}
#[derive(Debug)]
pub enum ArchiveError {
IoError(std::io::Error),

64
nix-js/src/fetcher/cache.rs
View File

@@ -1,37 +1,6 @@
use std::fs;
use std::path::PathBuf;
use super::archive::ArchiveError;
#[derive(Debug)]
pub enum CacheError {
Io(std::io::Error),
Archive(ArchiveError),
}
impl std::fmt::Display for CacheError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
CacheError::Io(e) => write!(f, "I/O error: {}", e),
CacheError::Archive(e) => write!(f, "Archive error: {}", e),
}
}
}
impl std::error::Error for CacheError {}
impl From<std::io::Error> for CacheError {
fn from(e: std::io::Error) -> Self {
CacheError::Io(e)
}
}
impl From<ArchiveError> for CacheError {
fn from(e: ArchiveError) -> Self {
CacheError::Archive(e)
}
}
#[derive(Debug)]
pub struct FetcherCache {
base_dir: PathBuf,
@@ -49,41 +18,12 @@ impl FetcherCache {
Ok(Self { base_dir })
}
pub fn make_store_path(&self, hash: &str, name: &str) -> PathBuf {
let short_hash = &hash[..32.min(hash.len())];
self.base_dir
.join("store")
.join(format!("{}-{}", short_hash, name))
}
fn git_cache_dir(&self) -> PathBuf {
self.base_dir.join("gitv3")
}
fn hg_cache_dir(&self) -> PathBuf {
self.base_dir.join("hg")
}
fn hash_key(url: &str) -> String {
crate::nix_utils::sha256_hex(url)
self.base_dir.join("git")
}
pub fn get_git_bare(&self, url: &str) -> PathBuf {
let key = Self::hash_key(url);
let key = crate::nix_utils::sha256_hex(url.as_bytes());
self.git_cache_dir().join(key)
}
pub fn get_hg_bare(&self, url: &str) -> PathBuf {
let key = Self::hash_key(url);
self.hg_cache_dir().join(key)
}
pub fn extract_tarball_to_temp(
&self,
data: &[u8],
) -> Result<(PathBuf, tempfile::TempDir), CacheError> {
let temp_dir = tempfile::tempdir()?;
let extracted_path = super::archive::extract_archive(data, temp_dir.path())?;
Ok((extracted_path, temp_dir))
}
}

196
nix-js/src/fetcher/hg.rs
View File

@@ -1,196 +0,0 @@
use std::fs;
use std::path::PathBuf;
use std::process::Command;
use super::FetchHgResult;
use super::cache::FetcherCache;
pub fn fetch_hg(
cache: &FetcherCache,
url: &str,
rev: Option<&str>,
name: &str,
) -> Result<FetchHgResult, HgError> {
let bare_repo = cache.get_hg_bare(url);
if !bare_repo.exists() {
clone_repo(url, &bare_repo)?;
} else {
pull_repo(&bare_repo)?;
}
let target_rev = rev.unwrap_or("tip").to_string();
let resolved_rev = resolve_rev(&bare_repo, &target_rev)?;
let branch = get_branch(&bare_repo, &resolved_rev)?;
let checkout_dir = checkout_rev(&bare_repo, &resolved_rev, name, cache)?;
let rev_count = get_rev_count(&bare_repo, &resolved_rev)?;
let short_rev = if resolved_rev.len() >= 12 {
resolved_rev[..12].to_string()
} else {
resolved_rev.clone()
};
Ok(FetchHgResult {
out_path: checkout_dir.to_string_lossy().to_string(),
branch,
rev: resolved_rev,
short_rev,
rev_count,
})
}
fn clone_repo(url: &str, dest: &PathBuf) -> Result<(), HgError> {
fs::create_dir_all(dest.parent().unwrap_or(dest))?;
let output = Command::new("hg")
.args(["clone", "-U", url])
.arg(dest)
.env("HGPLAIN", "")
.output()?;
if !output.status.success() {
return Err(HgError::CommandFailed {
operation: "clone".to_string(),
message: String::from_utf8_lossy(&output.stderr).to_string(),
});
}
Ok(())
}
fn pull_repo(repo: &PathBuf) -> Result<(), HgError> {
let output = Command::new("hg")
.args(["pull"])
.current_dir(repo)
.env("HGPLAIN", "")
.output()?;
if !output.status.success() {
return Err(HgError::CommandFailed {
operation: "pull".to_string(),
message: String::from_utf8_lossy(&output.stderr).to_string(),
});
}
Ok(())
}
fn resolve_rev(repo: &PathBuf, rev: &str) -> Result<String, HgError> {
let output = Command::new("hg")
.args(["log", "-r", rev, "--template", "{node}"])
.current_dir(repo)
.env("HGPLAIN", "")
.output()?;
if !output.status.success() {
return Err(HgError::CommandFailed {
operation: "log".to_string(),
message: format!(
"Could not resolve rev '{}': {}",
rev,
String::from_utf8_lossy(&output.stderr)
),
});
}
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
}
fn get_branch(repo: &PathBuf, rev: &str) -> Result<String, HgError> {
let output = Command::new("hg")
.args(["log", "-r", rev, "--template", "{branch}"])
.current_dir(repo)
.env("HGPLAIN", "")
.output()?;
if !output.status.success() {
return Ok("default".to_string());
}
let branch = String::from_utf8_lossy(&output.stdout).trim().to_string();
if branch.is_empty() {
Ok("default".to_string())
} else {
Ok(branch)
}
}
fn checkout_rev(
bare_repo: &PathBuf,
rev: &str,
name: &str,
cache: &FetcherCache,
) -> Result<PathBuf, HgError> {
let hash = crate::nix_utils::sha256_hex(&format!("{}:{}", bare_repo.display(), rev));
let checkout_dir = cache.make_store_path(&hash, name);
if checkout_dir.exists() {
return Ok(checkout_dir);
}
fs::create_dir_all(&checkout_dir)?;
let output = Command::new("hg")
.args(["archive", "-r", rev])
.arg(&checkout_dir)
.current_dir(bare_repo)
.env("HGPLAIN", "")
.output()?;
if !output.status.success() {
fs::remove_dir_all(&checkout_dir)?;
return Err(HgError::CommandFailed {
operation: "archive".to_string(),
message: String::from_utf8_lossy(&output.stderr).to_string(),
});
}
let hg_archival = checkout_dir.join(".hg_archival.txt");
if hg_archival.exists() {
fs::remove_file(&hg_archival)?;
}
Ok(checkout_dir)
}
fn get_rev_count(repo: &PathBuf, rev: &str) -> Result<u64, HgError> {
let output = Command::new("hg")
.args(["log", "-r", &format!("0::{}", rev), "--template", "x"])
.current_dir(repo)
.env("HGPLAIN", "")
.output()?;
if !output.status.success() {
return Ok(0);
}
Ok(output.stdout.len() as u64)
}
#[derive(Debug)]
pub enum HgError {
IoError(std::io::Error),
CommandFailed { operation: String, message: String },
}
impl std::fmt::Display for HgError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
HgError::IoError(e) => write!(f, "I/O error: {}", e),
HgError::CommandFailed { operation, message } => {
write!(f, "Mercurial {} failed: {}", operation, message)
}
}
}
}
impl std::error::Error for HgError {}
impl From<std::io::Error> for HgError {
fn from(e: std::io::Error) -> Self {
HgError::IoError(e)
}
}

12
nix-js/src/nix_utils.rs
View File

@@ -1,9 +1,9 @@
use nix_compat::store_path::compress_hash;
use sha2::{Digest as _, Sha256};
pub fn sha256_hex(data: &str) -> String {
pub fn sha256_hex(data: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(data.as_bytes());
hasher.update(data);
hex::encode(hasher.finalize())
}
@@ -19,11 +19,3 @@ pub fn make_store_path(store_dir: &str, ty: &str, hash_hex: &str, name: &str) ->
format!("{}/{}-{}", store_dir, encoded, name)
}
pub fn output_path_name(drv_name: &str, output_name: &str) -> String {
if output_name == "out" {
drv_name.to_string()
} else {
format!("{}-{}", drv_name, output_name)
}
}

1
nix-js/src/runtime.rs
View File

@@ -55,7 +55,6 @@ fn runtime_extension<Ctx: RuntimeContext>() -> Extension {
op_make_placeholder(),
op_decode_span::<Ctx>(),
op_make_store_path::<Ctx>(),
op_output_path_name(),
op_parse_hash(),
op_make_fixed_output_path::<Ctx>(),
op_add_path::<Ctx>(),

11
nix-js/src/runtime/ops.rs
View File

@@ -246,7 +246,7 @@ pub(super) fn op_resolve_path(
#[deno_core::op2]
#[string]
pub(super) fn op_sha256_hex(#[string] data: String) -> String {
crate::nix_utils::sha256_hex(&data)
crate::nix_utils::sha256_hex(data.as_bytes())
}
#[deno_core::op2]
@@ -325,15 +325,6 @@ pub(super) fn op_make_store_path<Ctx: RuntimeContext>(
crate::nix_utils::make_store_path(store_dir, &ty, &hash_hex, &name)
}
#[deno_core::op2]
#[string]
pub(super) fn op_output_path_name(
#[string] drv_name: String,
#[string] output_name: String,
) -> String {
crate::nix_utils::output_path_name(&drv_name, &output_name)
}
#[derive(serde::Serialize)]
pub(super) struct ParsedHash {
hex: String,