imxyy1soope1/nix-js
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: fetchTree & fetchTarball

Browse Source
This commit is contained in:
imxyy_soope_
2026-01-15 12:16:46 +08:00
parent e676d2f9f4
commit 4f8edab795
7 changed files with 249 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
nix-js/runtime-ts/src/builtins/io.ts
View File

@@ -42,6 +42,7 @@ interface FetchUrlResult {
interface FetchTarballResult {
store_path: string;
hash: string;
nar_hash: string;
}
interface FetchGitResult {
@@ -79,6 +80,26 @@ const normalizeUrlInput = (
return { url, hash, name, executable };
};
const normalizeTarballInput = (
args: NixValue,
): { url: string; hash?: string; narHash?: string; name?: string } => {
const forced = force(args);
if (typeof forced === "string") {
return { url: forced };
}
const attrs = forceAttrs(args);
const url = forceString(attrs.url);
const hash = "hash" in attrs ? forceString(attrs.hash) : undefined;
const narHash =
"narHash" in attrs
? forceString(attrs.narHash)
: "sha256" in attrs
? forceString(attrs.sha256)
: undefined;
const name = "name" in attrs ? forceString(attrs.name) : undefined;
return { url, hash, narHash, name };
};
export const fetchurl = (args: NixValue): string => {
const { url, hash, name, executable } = normalizeUrlInput(args);
const result: FetchUrlResult = Deno.core.ops.op_fetch_url(
@@ -91,8 +112,13 @@ export const fetchurl = (args: NixValue): string => {
};
export const fetchTarball = (args: NixValue): string => {
const { url, hash, name } = normalizeUrlInput(args);
const result: FetchTarballResult = Deno.core.ops.op_fetch_tarball(url, hash ?? null, name ?? null);
const { url, hash, narHash, name } = normalizeTarballInput(args);
const result: FetchTarballResult = Deno.core.ops.op_fetch_tarball(
url,
hash ?? null,
narHash ?? null,
name ?? null,
);
return result.store_path;
};
@@ -191,15 +217,36 @@ const fetchGitForge = (forge: string, attrs: NixAttrs): NixAttrs => {
const owner = forceString(attrs.owner);
const repo = forceString(attrs.repo);
const rev = "rev" in attrs ? forceString(attrs.rev) : "ref" in attrs ? forceString(attrs.ref) : "HEAD";
const host = "host" in attrs ? forceString(attrs.host) : undefined;
const baseUrls: Record<string, string> = {
github: "https://github.com",
gitlab: "https://gitlab.com",
sourcehut: "https://git.sr.ht",
let tarballUrl: string;
switch (forge) {
case "github": {
const apiHost = host || "github.com";
tarballUrl = `https://api.${apiHost}/repos/${owner}/${repo}/tarball/${rev}`;
break;
}
case "gitlab": {
const glHost = host || "gitlab.com";
tarballUrl = `https://${glHost}/api/v4/projects/${owner}%2F${repo}/repository/archive.tar.gz?sha=${rev}`;
break;
}
case "sourcehut": {
const shHost = host || "git.sr.ht";
tarballUrl = `https://${shHost}/${owner}/${repo}/archive/${rev}.tar.gz`;
break;
}
default:
throw new Error(`Unknown forge type: ${forge}`);
}
const outPath = fetchTarball({ url: tarballUrl, ...attrs });
return {
outPath,
rev,
shortRev: rev.substring(0, 7),
};
const url = `${baseUrls[forge]}/${owner}/${repo}`;
return fetchGit({ ...attrs, url, rev });
};
const autoDetectAndFetch = (attrs: NixAttrs): NixAttrs => {

2
nix-js/runtime-ts/src/thunk.ts
View File

@@ -81,7 +81,7 @@ export const force = (value: NixValue): NixStrictValue => {
if (value.func === undefined) {
if (value.result === undefined) {
const thunk = value as NixThunk;
let msg = `infinite recursion encountered (blackhole) at ${thunk}\n`;
let msg = `infinite recursion encountered at ${thunk}\n`;
msg += "Force chain (most recent first):\n";
for (let i = forceStack.length - 1; i >= 0; i--) {
const t = forceStack[i];

2
nix-js/runtime-ts/src/types/global.d.ts vendored
View File

@@ -8,6 +8,7 @@ interface FetchUrlResult {
interface FetchTarballResult {
store_path: string;
hash: string;
nar_hash: string;
}
interface FetchGitResult {
@@ -56,6 +57,7 @@ declare global {
function op_fetch_tarball(
url: string,
expected_hash: string | null,
expected_nar_hash: string | null,
name: string | null,
): FetchTarballResult;
function op_fetch_git(

1
nix-js/src/codegen.rs
View File

@@ -195,6 +195,7 @@ fn should_keep_thunk(ir: &Ir) -> bool {
// Builtin references are safe to evaluate eagerly
Ir::Builtin(_) | Ir::Builtins(_) => false,
Ir::ExprRef(_) => false,
// Everything else should remain lazy:
_ => true,
}
}

67
nix-js/src/fetcher.rs
View File

@@ -23,6 +23,7 @@ pub struct FetchUrlResult {
pub struct FetchTarballResult {
pub store_path: String,
pub hash: String,
pub nar_hash: String,
}
#[derive(Serialize)]
@@ -54,6 +55,8 @@ pub fn op_fetch_url(
#[string] name: Option<String>,
executable: bool,
) -> Result<FetchUrlResult, NixError> {
#[cfg(debug_assertions)]
eprintln!("[DEBUG] fetchurl: {}", url);
let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?;
let downloader = Downloader::new();
@@ -100,40 +103,87 @@ pub fn op_fetch_url(
pub fn op_fetch_tarball(
#[string] url: String,
#[string] expected_hash: Option<String>,
#[string] expected_nar_hash: Option<String>,
#[string] name: Option<String>,
) -> Result<FetchTarballResult, NixError> {
#[cfg(debug_assertions)]
eprintln!("[DEBUG] fetchTarball: url={}, expected_hash={:?}, expected_nar_hash={:?}", url, expected_hash, expected_nar_hash);
let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?;
let downloader = Downloader::new();
let dir_name = name.unwrap_or_else(|| "source".to_string());
if let Some(ref hash) = expected_hash {
let normalized = normalize_hash(hash);
if let Some(cached) = cache.get_tarball(&url, &normalized) {
// Try cache lookup with narHash if provided
if let Some(ref nar_hash) = expected_nar_hash {
let normalized = normalize_hash(nar_hash);
#[cfg(debug_assertions)]
eprintln!("[DEBUG] fetchTarball: normalized nar_hash={}", normalized);
if let Some(cached) = cache.get_extracted_tarball(&url, &normalized) {
#[cfg(debug_assertions)]
eprintln!("[DEBUG] fetchTarball: cache HIT (with expected nar_hash)");
// Need to compute tarball hash if not cached
let tarball_hash = expected_hash.as_ref()
.map(|h| normalize_hash(h))
.unwrap_or_else(|| "".to_string());
return Ok(FetchTarballResult {
store_path: cached.to_string_lossy().to_string(),
hash: normalized,
hash: tarball_hash,
nar_hash: normalized,
});
}
#[cfg(debug_assertions)]
eprintln!("[DEBUG] fetchTarball: cache MISS, downloading...");
} else if let Some((cached, cached_nar_hash)) = cache.get_extracted_tarball_by_url(&url) {
#[cfg(debug_assertions)]
eprintln!("[DEBUG] fetchTarball: cache HIT (by URL, nar_hash={})", cached_nar_hash);
let tarball_hash = expected_hash.as_ref()
.map(|h| normalize_hash(h))
.unwrap_or_else(|| "".to_string());
return Ok(FetchTarballResult {
store_path: cached.to_string_lossy().to_string(),
hash: tarball_hash,
nar_hash: cached_nar_hash,
});
}
#[cfg(debug_assertions)]
eprintln!("[DEBUG] fetchTarball: cache MISS, downloading...");
let data = downloader
.download(&url)
.map_err(|e| NixError::from(e.to_string()))?;
// Compute tarball hash (hash of the archive file itself)
let tarball_hash = crate::nix_hash::sha256_hex(&String::from_utf8_lossy(&data));
// Verify tarball hash if provided
if let Some(ref expected) = expected_hash {
let normalized_expected = normalize_hash(expected);
if tarball_hash != normalized_expected {
return Err(NixError::from(format!(
"Tarball hash mismatch for '{}': expected {}, got {}",
url, normalized_expected, tarball_hash
)));
}
}
let temp_dir = tempfile::tempdir().map_err(|e| NixError::from(e.to_string()))?;
let extracted_path = archive::extract_archive(&data, temp_dir.path())
.map_err(|e| NixError::from(e.to_string()))?;
// Compute NAR hash (hash of the extracted content)
let nar_hash =
nar::compute_nar_hash(&extracted_path).map_err(|e| NixError::from(e.to_string()))?;
if let Some(ref expected) = expected_hash {
#[cfg(debug_assertions)]
eprintln!("[DEBUG] fetchTarball: computed tarball_hash={}, nar_hash={}", tarball_hash, nar_hash);
// Verify NAR hash if provided
if let Some(ref expected) = expected_nar_hash {
let normalized_expected = normalize_hash(expected);
if nar_hash != normalized_expected {
return Err(NixError::from(format!(
"hash mismatch for '{}': expected {}, got {}",
"NAR hash mismatch for '{}': expected {}, got {}",
url, normalized_expected, nar_hash
)));
}
@@ -145,7 +195,8 @@ pub fn op_fetch_tarball(
Ok(FetchTarballResult {
store_path: store_path.to_string_lossy().to_string(),
hash: nar_hash,
hash: tarball_hash,
nar_hash,
})
}
@@ -160,6 +211,8 @@ pub fn op_fetch_git(
all_refs: bool,
#[string] name: Option<String>,
) -> Result<FetchGitResult, NixError> {
#[cfg(debug_assertions)]
eprintln!("[DEBUG] fetchGit: {} (ref: {:?}, rev: {:?})", url, git_ref, rev);
let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?;
let dir_name = name.unwrap_or_else(|| "source".to_string());

116
nix-js/src/fetcher/cache.rs
View File

@@ -101,7 +101,12 @@ impl FetcherCache {
serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?;
if meta.hash == expected_hash {
Some(data_path)
let store_path = self.make_store_path(&meta.hash, &meta.name);
if store_path.exists() {
Some(store_path)
} else {
None
}
} else {
None
}
@@ -163,16 +168,37 @@ impl FetcherCache {
let meta_path = cache_dir.join(&key).join(".meta");
let data_dir = cache_dir.join(&key);
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_tarball: url={}, expected_hash={}", url, expected_hash);
if !meta_path.exists() || !data_dir.exists() {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_tarball: cache miss - meta or data dir not found");
return None;
}
let meta: CacheMetadata =
serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?;
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_tarball: cached hash={}, name={}", meta.hash, meta.name);
if meta.hash == expected_hash {
Some(self.make_store_path(&meta.hash, &meta.name))
let store_path = self.make_store_path(&meta.hash, &meta.name);
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_tarball: hash match, checking store_path={}", store_path.display());
if store_path.exists() {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_tarball: HIT - returning store path");
Some(store_path)
} else {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_tarball: store path doesn't exist");
None
}
} else {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_tarball: hash mismatch (cached={}, expected={})", meta.hash, expected_hash);
None
}
}
@@ -208,6 +234,82 @@ impl FetcherCache {
Ok(store_path)
}
pub fn get_extracted_tarball(&self, url: &str, expected_nar_hash: &str) -> Option<PathBuf> {
let cache_dir = self.tarball_cache_dir();
let key = Self::hash_key(url);
let cache_entry_dir = cache_dir.join(&key);
let meta_path = cache_entry_dir.join(".meta");
let cached_content = cache_entry_dir.join("content");
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball: url={}, expected_nar_hash={}", url, expected_nar_hash);
if !meta_path.exists() || !cached_content.exists() {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball: cache miss - meta or content dir not found");
return None;
}
let meta: CacheMetadata =
serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?;
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball: cached hash={}, name={}", meta.hash, meta.name);
if meta.hash == expected_nar_hash {
let store_path = self.make_store_path(&meta.hash, &meta.name);
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball: hash match, checking store_path={}", store_path.display());
if store_path.exists() {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball: HIT - returning store path");
Some(store_path)
} else {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball: store path doesn't exist");
None
}
} else {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball: hash mismatch (cached={}, expected={})", meta.hash, expected_nar_hash);
None
}
}
pub fn get_extracted_tarball_by_url(&self, url: &str) -> Option<(PathBuf, String)> {
let cache_dir = self.tarball_cache_dir();
let key = Self::hash_key(url);
let cache_entry_dir = cache_dir.join(&key);
let meta_path = cache_entry_dir.join(".meta");
let cached_content = cache_entry_dir.join("content");
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball_by_url: url={}", url);
if !meta_path.exists() || !cached_content.exists() {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball_by_url: cache miss - meta or content dir not found");
return None;
}
let meta: CacheMetadata =
serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?;
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball_by_url: cached hash={}, name={}", meta.hash, meta.name);
let store_path = self.make_store_path(&meta.hash, &meta.name);
if store_path.exists() {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball_by_url: HIT - returning store path and hash");
Some((store_path, meta.hash))
} else {
#[cfg(debug_assertions)]
eprintln!("[CACHE] get_extracted_tarball_by_url: store path doesn't exist");
None
}
}
pub fn put_tarball_from_extracted(
&self,
url: &str,
@@ -219,6 +321,9 @@ impl FetcherCache {
let key = Self::hash_key(url);
let cache_entry_dir = cache_dir.join(&key);
#[cfg(debug_assertions)]
eprintln!("[CACHE] put_tarball_from_extracted: url={}, hash={}, name={}", url, hash, name);
fs::create_dir_all(&cache_entry_dir)?;
let cached_content = cache_entry_dir.join("content");
@@ -234,9 +339,16 @@ impl FetcherCache {
fs::write(cache_entry_dir.join(".meta"), serde_json::to_string(&meta)?)?;
let store_path = self.make_store_path(hash, name);
#[cfg(debug_assertions)]
eprintln!("[CACHE] put_tarball_from_extracted: store_path={}", store_path.display());
if !store_path.exists() {
fs::create_dir_all(store_path.parent().unwrap_or(&store_path))?;
copy_dir_recursive(extracted_path, &store_path)?;
#[cfg(debug_assertions)]
eprintln!("[CACHE] put_tarball_from_extracted: copied to store");
} else {
#[cfg(debug_assertions)]
eprintln!("[CACHE] put_tarball_from_extracted: store path already exists");
}
Ok(store_path)

15
shell.nix Normal file
View File

@@ -0,0 +1,15 @@
let
lockFile = builtins.fromJSON (builtins.readFile ./flake.lock);
flake-compat-node = lockFile.nodes.${lockFile.nodes.root.inputs.flake-compat};
flake-compat = builtins.fetchTarball {
inherit (flake-compat-node.locked) url;
sha256 = flake-compat-node.locked.narHash;
};
flake = (
import flake-compat {
src = ./.;
}
);
in
flake.shellNix