diff --git a/nix-js/runtime-ts/src/builtins/io.ts b/nix-js/runtime-ts/src/builtins/io.ts index 986bf00..1af2b0f 100644 --- a/nix-js/runtime-ts/src/builtins/io.ts +++ b/nix-js/runtime-ts/src/builtins/io.ts @@ -42,6 +42,7 @@ interface FetchUrlResult { interface FetchTarballResult { store_path: string; hash: string; + nar_hash: string; } interface FetchGitResult { @@ -79,6 +80,26 @@ const normalizeUrlInput = ( return { url, hash, name, executable }; }; +const normalizeTarballInput = ( + args: NixValue, +): { url: string; hash?: string; narHash?: string; name?: string } => { + const forced = force(args); + if (typeof forced === "string") { + return { url: forced }; + } + const attrs = forceAttrs(args); + const url = forceString(attrs.url); + const hash = "hash" in attrs ? forceString(attrs.hash) : undefined; + const narHash = + "narHash" in attrs + ? forceString(attrs.narHash) + : "sha256" in attrs + ? forceString(attrs.sha256) + : undefined; + const name = "name" in attrs ? forceString(attrs.name) : undefined; + return { url, hash, narHash, name }; +}; + export const fetchurl = (args: NixValue): string => { const { url, hash, name, executable } = normalizeUrlInput(args); const result: FetchUrlResult = Deno.core.ops.op_fetch_url( @@ -91,8 +112,13 @@ export const fetchurl = (args: NixValue): string => { }; export const fetchTarball = (args: NixValue): string => { - const { url, hash, name } = normalizeUrlInput(args); - const result: FetchTarballResult = Deno.core.ops.op_fetch_tarball(url, hash ?? null, name ?? null); + const { url, hash, narHash, name } = normalizeTarballInput(args); + const result: FetchTarballResult = Deno.core.ops.op_fetch_tarball( + url, + hash ?? null, + narHash ?? null, + name ?? null, + ); return result.store_path; }; @@ -191,15 +217,36 @@ const fetchGitForge = (forge: string, attrs: NixAttrs): NixAttrs => { const owner = forceString(attrs.owner); const repo = forceString(attrs.repo); const rev = "rev" in attrs ? forceString(attrs.rev) : "ref" in attrs ? forceString(attrs.ref) : "HEAD"; + const host = "host" in attrs ? forceString(attrs.host) : undefined; - const baseUrls: Record = { - github: "https://github.com", - gitlab: "https://gitlab.com", - sourcehut: "https://git.sr.ht", + let tarballUrl: string; + switch (forge) { + case "github": { + const apiHost = host || "github.com"; + tarballUrl = `https://api.${apiHost}/repos/${owner}/${repo}/tarball/${rev}`; + break; + } + case "gitlab": { + const glHost = host || "gitlab.com"; + tarballUrl = `https://${glHost}/api/v4/projects/${owner}%2F${repo}/repository/archive.tar.gz?sha=${rev}`; + break; + } + case "sourcehut": { + const shHost = host || "git.sr.ht"; + tarballUrl = `https://${shHost}/${owner}/${repo}/archive/${rev}.tar.gz`; + break; + } + default: + throw new Error(`Unknown forge type: ${forge}`); + } + + const outPath = fetchTarball({ url: tarballUrl, ...attrs }); + + return { + outPath, + rev, + shortRev: rev.substring(0, 7), }; - - const url = `${baseUrls[forge]}/${owner}/${repo}`; - return fetchGit({ ...attrs, url, rev }); }; const autoDetectAndFetch = (attrs: NixAttrs): NixAttrs => { diff --git a/nix-js/runtime-ts/src/thunk.ts b/nix-js/runtime-ts/src/thunk.ts index 59e5989..733bfb2 100644 --- a/nix-js/runtime-ts/src/thunk.ts +++ b/nix-js/runtime-ts/src/thunk.ts @@ -81,7 +81,7 @@ export const force = (value: NixValue): NixStrictValue => { if (value.func === undefined) { if (value.result === undefined) { const thunk = value as NixThunk; - let msg = `infinite recursion encountered (blackhole) at ${thunk}\n`; + let msg = `infinite recursion encountered at ${thunk}\n`; msg += "Force chain (most recent first):\n"; for (let i = forceStack.length - 1; i >= 0; i--) { const t = forceStack[i]; diff --git a/nix-js/runtime-ts/src/types/global.d.ts b/nix-js/runtime-ts/src/types/global.d.ts index 123ff7a..29ed97b 100644 --- a/nix-js/runtime-ts/src/types/global.d.ts +++ b/nix-js/runtime-ts/src/types/global.d.ts @@ -8,6 +8,7 @@ interface FetchUrlResult { interface FetchTarballResult { store_path: string; hash: string; + nar_hash: string; } interface FetchGitResult { @@ -56,6 +57,7 @@ declare global { function op_fetch_tarball( url: string, expected_hash: string | null, + expected_nar_hash: string | null, name: string | null, ): FetchTarballResult; function op_fetch_git( diff --git a/nix-js/src/codegen.rs b/nix-js/src/codegen.rs index 7bd8f9b..27b1a9b 100644 --- a/nix-js/src/codegen.rs +++ b/nix-js/src/codegen.rs @@ -195,6 +195,7 @@ fn should_keep_thunk(ir: &Ir) -> bool { // Builtin references are safe to evaluate eagerly Ir::Builtin(_) | Ir::Builtins(_) => false, Ir::ExprRef(_) => false, + // Everything else should remain lazy: _ => true, } } diff --git a/nix-js/src/fetcher.rs b/nix-js/src/fetcher.rs index cc64dfc..a65f346 100644 --- a/nix-js/src/fetcher.rs +++ b/nix-js/src/fetcher.rs @@ -23,6 +23,7 @@ pub struct FetchUrlResult { pub struct FetchTarballResult { pub store_path: String, pub hash: String, + pub nar_hash: String, } #[derive(Serialize)] @@ -54,6 +55,8 @@ pub fn op_fetch_url( #[string] name: Option, executable: bool, ) -> Result { + #[cfg(debug_assertions)] + eprintln!("[DEBUG] fetchurl: {}", url); let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?; let downloader = Downloader::new(); @@ -100,40 +103,87 @@ pub fn op_fetch_url( pub fn op_fetch_tarball( #[string] url: String, #[string] expected_hash: Option, + #[string] expected_nar_hash: Option, #[string] name: Option, ) -> Result { + #[cfg(debug_assertions)] + eprintln!("[DEBUG] fetchTarball: url={}, expected_hash={:?}, expected_nar_hash={:?}", url, expected_hash, expected_nar_hash); let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?; let downloader = Downloader::new(); let dir_name = name.unwrap_or_else(|| "source".to_string()); - if let Some(ref hash) = expected_hash { - let normalized = normalize_hash(hash); - if let Some(cached) = cache.get_tarball(&url, &normalized) { + // Try cache lookup with narHash if provided + if let Some(ref nar_hash) = expected_nar_hash { + let normalized = normalize_hash(nar_hash); + #[cfg(debug_assertions)] + eprintln!("[DEBUG] fetchTarball: normalized nar_hash={}", normalized); + if let Some(cached) = cache.get_extracted_tarball(&url, &normalized) { + #[cfg(debug_assertions)] + eprintln!("[DEBUG] fetchTarball: cache HIT (with expected nar_hash)"); + // Need to compute tarball hash if not cached + let tarball_hash = expected_hash.as_ref() + .map(|h| normalize_hash(h)) + .unwrap_or_else(|| "".to_string()); return Ok(FetchTarballResult { store_path: cached.to_string_lossy().to_string(), - hash: normalized, + hash: tarball_hash, + nar_hash: normalized, }); } + #[cfg(debug_assertions)] + eprintln!("[DEBUG] fetchTarball: cache MISS, downloading..."); + } else if let Some((cached, cached_nar_hash)) = cache.get_extracted_tarball_by_url(&url) { + #[cfg(debug_assertions)] + eprintln!("[DEBUG] fetchTarball: cache HIT (by URL, nar_hash={})", cached_nar_hash); + let tarball_hash = expected_hash.as_ref() + .map(|h| normalize_hash(h)) + .unwrap_or_else(|| "".to_string()); + return Ok(FetchTarballResult { + store_path: cached.to_string_lossy().to_string(), + hash: tarball_hash, + nar_hash: cached_nar_hash, + }); } + #[cfg(debug_assertions)] + eprintln!("[DEBUG] fetchTarball: cache MISS, downloading..."); let data = downloader .download(&url) .map_err(|e| NixError::from(e.to_string()))?; + // Compute tarball hash (hash of the archive file itself) + let tarball_hash = crate::nix_hash::sha256_hex(&String::from_utf8_lossy(&data)); + + // Verify tarball hash if provided + if let Some(ref expected) = expected_hash { + let normalized_expected = normalize_hash(expected); + if tarball_hash != normalized_expected { + return Err(NixError::from(format!( + "Tarball hash mismatch for '{}': expected {}, got {}", + url, normalized_expected, tarball_hash + ))); + } + } + let temp_dir = tempfile::tempdir().map_err(|e| NixError::from(e.to_string()))?; let extracted_path = archive::extract_archive(&data, temp_dir.path()) .map_err(|e| NixError::from(e.to_string()))?; + // Compute NAR hash (hash of the extracted content) let nar_hash = nar::compute_nar_hash(&extracted_path).map_err(|e| NixError::from(e.to_string()))?; - if let Some(ref expected) = expected_hash { + #[cfg(debug_assertions)] + eprintln!("[DEBUG] fetchTarball: computed tarball_hash={}, nar_hash={}", tarball_hash, nar_hash); + + // Verify NAR hash if provided + if let Some(ref expected) = expected_nar_hash { let normalized_expected = normalize_hash(expected); if nar_hash != normalized_expected { return Err(NixError::from(format!( - "hash mismatch for '{}': expected {}, got {}", + "NAR hash mismatch for '{}': expected {}, got {}", url, normalized_expected, nar_hash ))); } @@ -145,7 +195,8 @@ pub fn op_fetch_tarball( Ok(FetchTarballResult { store_path: store_path.to_string_lossy().to_string(), - hash: nar_hash, + hash: tarball_hash, + nar_hash, }) } @@ -160,6 +211,8 @@ pub fn op_fetch_git( all_refs: bool, #[string] name: Option, ) -> Result { + #[cfg(debug_assertions)] + eprintln!("[DEBUG] fetchGit: {} (ref: {:?}, rev: {:?})", url, git_ref, rev); let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?; let dir_name = name.unwrap_or_else(|| "source".to_string()); diff --git a/nix-js/src/fetcher/cache.rs b/nix-js/src/fetcher/cache.rs index 619ccb0..bcb30f7 100644 --- a/nix-js/src/fetcher/cache.rs +++ b/nix-js/src/fetcher/cache.rs @@ -101,7 +101,12 @@ impl FetcherCache { serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?; if meta.hash == expected_hash { - Some(data_path) + let store_path = self.make_store_path(&meta.hash, &meta.name); + if store_path.exists() { + Some(store_path) + } else { + None + } } else { None } @@ -163,16 +168,37 @@ impl FetcherCache { let meta_path = cache_dir.join(&key).join(".meta"); let data_dir = cache_dir.join(&key); + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_tarball: url={}, expected_hash={}", url, expected_hash); + if !meta_path.exists() || !data_dir.exists() { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_tarball: cache miss - meta or data dir not found"); return None; } let meta: CacheMetadata = serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?; + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_tarball: cached hash={}, name={}", meta.hash, meta.name); + if meta.hash == expected_hash { - Some(self.make_store_path(&meta.hash, &meta.name)) + let store_path = self.make_store_path(&meta.hash, &meta.name); + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_tarball: hash match, checking store_path={}", store_path.display()); + if store_path.exists() { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_tarball: HIT - returning store path"); + Some(store_path) + } else { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_tarball: store path doesn't exist"); + None + } } else { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_tarball: hash mismatch (cached={}, expected={})", meta.hash, expected_hash); None } } @@ -208,6 +234,82 @@ impl FetcherCache { Ok(store_path) } + pub fn get_extracted_tarball(&self, url: &str, expected_nar_hash: &str) -> Option { + let cache_dir = self.tarball_cache_dir(); + let key = Self::hash_key(url); + let cache_entry_dir = cache_dir.join(&key); + let meta_path = cache_entry_dir.join(".meta"); + let cached_content = cache_entry_dir.join("content"); + + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball: url={}, expected_nar_hash={}", url, expected_nar_hash); + + if !meta_path.exists() || !cached_content.exists() { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball: cache miss - meta or content dir not found"); + return None; + } + + let meta: CacheMetadata = + serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?; + + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball: cached hash={}, name={}", meta.hash, meta.name); + + if meta.hash == expected_nar_hash { + let store_path = self.make_store_path(&meta.hash, &meta.name); + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball: hash match, checking store_path={}", store_path.display()); + if store_path.exists() { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball: HIT - returning store path"); + Some(store_path) + } else { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball: store path doesn't exist"); + None + } + } else { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball: hash mismatch (cached={}, expected={})", meta.hash, expected_nar_hash); + None + } + } + + pub fn get_extracted_tarball_by_url(&self, url: &str) -> Option<(PathBuf, String)> { + let cache_dir = self.tarball_cache_dir(); + let key = Self::hash_key(url); + let cache_entry_dir = cache_dir.join(&key); + let meta_path = cache_entry_dir.join(".meta"); + let cached_content = cache_entry_dir.join("content"); + + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball_by_url: url={}", url); + + if !meta_path.exists() || !cached_content.exists() { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball_by_url: cache miss - meta or content dir not found"); + return None; + } + + let meta: CacheMetadata = + serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?; + + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball_by_url: cached hash={}, name={}", meta.hash, meta.name); + + let store_path = self.make_store_path(&meta.hash, &meta.name); + if store_path.exists() { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball_by_url: HIT - returning store path and hash"); + Some((store_path, meta.hash)) + } else { + #[cfg(debug_assertions)] + eprintln!("[CACHE] get_extracted_tarball_by_url: store path doesn't exist"); + None + } + } + pub fn put_tarball_from_extracted( &self, url: &str, @@ -219,6 +321,9 @@ impl FetcherCache { let key = Self::hash_key(url); let cache_entry_dir = cache_dir.join(&key); + #[cfg(debug_assertions)] + eprintln!("[CACHE] put_tarball_from_extracted: url={}, hash={}, name={}", url, hash, name); + fs::create_dir_all(&cache_entry_dir)?; let cached_content = cache_entry_dir.join("content"); @@ -234,9 +339,16 @@ impl FetcherCache { fs::write(cache_entry_dir.join(".meta"), serde_json::to_string(&meta)?)?; let store_path = self.make_store_path(hash, name); + #[cfg(debug_assertions)] + eprintln!("[CACHE] put_tarball_from_extracted: store_path={}", store_path.display()); if !store_path.exists() { fs::create_dir_all(store_path.parent().unwrap_or(&store_path))?; copy_dir_recursive(extracted_path, &store_path)?; + #[cfg(debug_assertions)] + eprintln!("[CACHE] put_tarball_from_extracted: copied to store"); + } else { + #[cfg(debug_assertions)] + eprintln!("[CACHE] put_tarball_from_extracted: store path already exists"); } Ok(store_path) diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..658b162 --- /dev/null +++ b/shell.nix @@ -0,0 +1,15 @@ +let + lockFile = builtins.fromJSON (builtins.readFile ./flake.lock); + flake-compat-node = lockFile.nodes.${lockFile.nodes.root.inputs.flake-compat}; + flake-compat = builtins.fetchTarball { + inherit (flake-compat-node.locked) url; + sha256 = flake-compat-node.locked.narHash; + }; + + flake = ( + import flake-compat { + src = ./.; + } + ); +in + flake.shellNix