fix: fetchTree & fetchTarball

nix-js/runtime-ts/src/builtins/io.ts

@@ -42,6 +42,7 @@ interface FetchUrlResult {
 interface FetchTarballResult {
   store_path: string;
   hash: string;
+  nar_hash: string;
 }
 
 interface FetchGitResult {
@@ -79,6 +80,26 @@ const normalizeUrlInput = (
   return { url, hash, name, executable };
 };
 
+const normalizeTarballInput = (
+  args: NixValue,
+): { url: string; hash?: string; narHash?: string; name?: string } => {
+  const forced = force(args);
+  if (typeof forced === "string") {
+    return { url: forced };
+  }
+  const attrs = forceAttrs(args);
+  const url = forceString(attrs.url);
+  const hash = "hash" in attrs ? forceString(attrs.hash) : undefined;
+  const narHash =
+    "narHash" in attrs
+      ? forceString(attrs.narHash)
+      : "sha256" in attrs
+        ? forceString(attrs.sha256)
+        : undefined;
+  const name = "name" in attrs ? forceString(attrs.name) : undefined;
+  return { url, hash, narHash, name };
+};
+
 export const fetchurl = (args: NixValue): string => {
   const { url, hash, name, executable } = normalizeUrlInput(args);
   const result: FetchUrlResult = Deno.core.ops.op_fetch_url(
@@ -91,8 +112,13 @@ export const fetchurl = (args: NixValue): string => {
 };
 
 export const fetchTarball = (args: NixValue): string => {
-  const { url, hash, name } = normalizeUrlInput(args);
+  const { url, hash, narHash, name } = normalizeTarballInput(args);
-  const result: FetchTarballResult = Deno.core.ops.op_fetch_tarball(url, hash ?? null, name ?? null);
+  const result: FetchTarballResult = Deno.core.ops.op_fetch_tarball(
+    url,
+    hash ?? null,
+    narHash ?? null,
+    name ?? null,
+  );
   return result.store_path;
 };
 
@@ -191,15 +217,36 @@ const fetchGitForge = (forge: string, attrs: NixAttrs): NixAttrs => {
   const owner = forceString(attrs.owner);
   const repo = forceString(attrs.repo);
   const rev = "rev" in attrs ? forceString(attrs.rev) : "ref" in attrs ? forceString(attrs.ref) : "HEAD";
+  const host = "host" in attrs ? forceString(attrs.host) : undefined;
 
-  const baseUrls: Record<string, string> = {
+  let tarballUrl: string;
-    github: "https://github.com",
+  switch (forge) {
-    gitlab: "https://gitlab.com",
+    case "github": {
-    sourcehut: "https://git.sr.ht",
+      const apiHost = host || "github.com";
+      tarballUrl = `https://api.${apiHost}/repos/${owner}/${repo}/tarball/${rev}`;
+      break;
+    }
+    case "gitlab": {
+      const glHost = host || "gitlab.com";
+      tarballUrl = `https://${glHost}/api/v4/projects/${owner}%2F${repo}/repository/archive.tar.gz?sha=${rev}`;
+      break;
+    }
+    case "sourcehut": {
+      const shHost = host || "git.sr.ht";
+      tarballUrl = `https://${shHost}/${owner}/${repo}/archive/${rev}.tar.gz`;
+      break;
+    }
+    default:
+      throw new Error(`Unknown forge type: ${forge}`);
+  }
+
+  const outPath = fetchTarball({ url: tarballUrl, ...attrs });
+
+  return {
+    outPath,
+    rev,
+    shortRev: rev.substring(0, 7),
   };
-
-  const url = `${baseUrls[forge]}/${owner}/${repo}`;
-  return fetchGit({ ...attrs, url, rev });
 };
 
 const autoDetectAndFetch = (attrs: NixAttrs): NixAttrs => {

nix-js/runtime-ts/src/thunk.ts

@@ -81,7 +81,7 @@ export const force = (value: NixValue): NixStrictValue => {
   if (value.func === undefined) {
     if (value.result === undefined) {
       const thunk = value as NixThunk;
-      let msg = `infinite recursion encountered (blackhole) at ${thunk}\n`;
+      let msg = `infinite recursion encountered at ${thunk}\n`;
       msg += "Force chain (most recent first):\n";
       for (let i = forceStack.length - 1; i >= 0; i--) {
         const t = forceStack[i];

nix-js/runtime-ts/src/types/global.d.ts

@@ -8,6 +8,7 @@ interface FetchUrlResult {
 interface FetchTarballResult {
   store_path: string;
   hash: string;
+  nar_hash: string;
 }
 
 interface FetchGitResult {
@@ -56,6 +57,7 @@ declare global {
         function op_fetch_tarball(
           url: string,
           expected_hash: string | null,
+          expected_nar_hash: string | null,
           name: string | null,
         ): FetchTarballResult;
         function op_fetch_git(

nix-js/src/codegen.rs

@@ -195,6 +195,7 @@ fn should_keep_thunk(ir: &Ir) -> bool {
         // Builtin references are safe to evaluate eagerly
         Ir::Builtin(_) | Ir::Builtins(_) => false,
         Ir::ExprRef(_) => false,
+        // Everything else should remain lazy:
         _ => true,
     }
 }

nix-js/src/fetcher.rs

@@ -23,6 +23,7 @@ pub struct FetchUrlResult {
 pub struct FetchTarballResult {
     pub store_path: String,
     pub hash: String,
+    pub nar_hash: String,
 }
 
 #[derive(Serialize)]
@@ -54,6 +55,8 @@ pub fn op_fetch_url(
     #[string] name: Option<String>,
     executable: bool,
 ) -> Result<FetchUrlResult, NixError> {
+    #[cfg(debug_assertions)]
+    eprintln!("[DEBUG] fetchurl: {}", url);
     let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?;
     let downloader = Downloader::new();
 
@@ -100,40 +103,87 @@ pub fn op_fetch_url(
 pub fn op_fetch_tarball(
     #[string] url: String,
     #[string] expected_hash: Option<String>,
+    #[string] expected_nar_hash: Option<String>,
     #[string] name: Option<String>,
 ) -> Result<FetchTarballResult, NixError> {
+    #[cfg(debug_assertions)]
+    eprintln!("[DEBUG] fetchTarball: url={}, expected_hash={:?}, expected_nar_hash={:?}", url, expected_hash, expected_nar_hash);
     let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?;
     let downloader = Downloader::new();
 
     let dir_name = name.unwrap_or_else(|| "source".to_string());
 
-    if let Some(ref hash) = expected_hash {
+    // Try cache lookup with narHash if provided
-        let normalized = normalize_hash(hash);
+    if let Some(ref nar_hash) = expected_nar_hash {
-        if let Some(cached) = cache.get_tarball(&url, &normalized) {
+        let normalized = normalize_hash(nar_hash);
+        #[cfg(debug_assertions)]
+        eprintln!("[DEBUG] fetchTarball: normalized nar_hash={}", normalized);
+        if let Some(cached) = cache.get_extracted_tarball(&url, &normalized) {
+            #[cfg(debug_assertions)]
+            eprintln!("[DEBUG] fetchTarball: cache HIT (with expected nar_hash)");
+            // Need to compute tarball hash if not cached
+            let tarball_hash = expected_hash.as_ref()
+                .map(|h| normalize_hash(h))
+                .unwrap_or_else(|| "".to_string());
             return Ok(FetchTarballResult {
                 store_path: cached.to_string_lossy().to_string(),
-                hash: normalized,
+                hash: tarball_hash,
+                nar_hash: normalized,
             });
         }
+        #[cfg(debug_assertions)]
+        eprintln!("[DEBUG] fetchTarball: cache MISS, downloading...");
+    } else if let Some((cached, cached_nar_hash)) = cache.get_extracted_tarball_by_url(&url) {
+        #[cfg(debug_assertions)]
+        eprintln!("[DEBUG] fetchTarball: cache HIT (by URL, nar_hash={})", cached_nar_hash);
+        let tarball_hash = expected_hash.as_ref()
+            .map(|h| normalize_hash(h))
+            .unwrap_or_else(|| "".to_string());
+        return Ok(FetchTarballResult {
+            store_path: cached.to_string_lossy().to_string(),
+            hash: tarball_hash,
+            nar_hash: cached_nar_hash,
+        });
     }
+    #[cfg(debug_assertions)]
+    eprintln!("[DEBUG] fetchTarball: cache MISS, downloading...");
 
     let data = downloader
         .download(&url)
         .map_err(|e| NixError::from(e.to_string()))?;
 
+    // Compute tarball hash (hash of the archive file itself)
+    let tarball_hash = crate::nix_hash::sha256_hex(&String::from_utf8_lossy(&data));
+
+    // Verify tarball hash if provided
+    if let Some(ref expected) = expected_hash {
+        let normalized_expected = normalize_hash(expected);
+        if tarball_hash != normalized_expected {
+            return Err(NixError::from(format!(
+                "Tarball hash mismatch for '{}': expected {}, got {}",
+                url, normalized_expected, tarball_hash
+            )));
+        }
+    }
+
     let temp_dir = tempfile::tempdir().map_err(|e| NixError::from(e.to_string()))?;
     let extracted_path = archive::extract_archive(&data, temp_dir.path())
         .map_err(|e| NixError::from(e.to_string()))?;
 
+    // Compute NAR hash (hash of the extracted content)
     let nar_hash =
         nar::compute_nar_hash(&extracted_path).map_err(|e| NixError::from(e.to_string()))?;
 
-    if let Some(ref expected) = expected_hash {
+    #[cfg(debug_assertions)]
+    eprintln!("[DEBUG] fetchTarball: computed tarball_hash={}, nar_hash={}", tarball_hash, nar_hash);
+
+    // Verify NAR hash if provided
+    if let Some(ref expected) = expected_nar_hash {
         let normalized_expected = normalize_hash(expected);
 
         if nar_hash != normalized_expected {
             return Err(NixError::from(format!(
-                "hash mismatch for '{}': expected {}, got {}",
+                "NAR hash mismatch for '{}': expected {}, got {}",
                 url, normalized_expected, nar_hash
             )));
         }
@@ -145,7 +195,8 @@ pub fn op_fetch_tarball(
 
     Ok(FetchTarballResult {
         store_path: store_path.to_string_lossy().to_string(),
-        hash: nar_hash,
+        hash: tarball_hash,
+        nar_hash,
     })
 }
 
@@ -160,6 +211,8 @@ pub fn op_fetch_git(
     all_refs: bool,
     #[string] name: Option<String>,
 ) -> Result<FetchGitResult, NixError> {
+    #[cfg(debug_assertions)]
+    eprintln!("[DEBUG] fetchGit: {} (ref: {:?}, rev: {:?})", url, git_ref, rev);
     let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?;
     let dir_name = name.unwrap_or_else(|| "source".to_string());
 

nix-js/src/fetcher/cache.rs

@@ -101,7 +101,12 @@ impl FetcherCache {
             serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?;
 
         if meta.hash == expected_hash {
-            Some(data_path)
+            let store_path = self.make_store_path(&meta.hash, &meta.name);
+            if store_path.exists() {
+                Some(store_path)
+            } else {
+                None
+            }
         } else {
             None
         }
@@ -163,16 +168,37 @@ impl FetcherCache {
         let meta_path = cache_dir.join(&key).join(".meta");
         let data_dir = cache_dir.join(&key);
 
+        #[cfg(debug_assertions)]
+        eprintln!("[CACHE] get_tarball: url={}, expected_hash={}", url, expected_hash);
+
         if !meta_path.exists() || !data_dir.exists() {
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] get_tarball: cache miss - meta or data dir not found");
             return None;
         }
 
         let meta: CacheMetadata =
             serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?;
 
+        #[cfg(debug_assertions)]
+        eprintln!("[CACHE] get_tarball: cached hash={}, name={}", meta.hash, meta.name);
+
         if meta.hash == expected_hash {
-            Some(self.make_store_path(&meta.hash, &meta.name))
+            let store_path = self.make_store_path(&meta.hash, &meta.name);
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] get_tarball: hash match, checking store_path={}", store_path.display());
+            if store_path.exists() {
+                #[cfg(debug_assertions)]
+                eprintln!("[CACHE] get_tarball: HIT - returning store path");
+                Some(store_path)
             } else {
+                #[cfg(debug_assertions)]
+                eprintln!("[CACHE] get_tarball: store path doesn't exist");
+                None
+            }
+        } else {
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] get_tarball: hash mismatch (cached={}, expected={})", meta.hash, expected_hash);
             None
         }
     }
@@ -208,6 +234,82 @@ impl FetcherCache {
         Ok(store_path)
     }
 
+    pub fn get_extracted_tarball(&self, url: &str, expected_nar_hash: &str) -> Option<PathBuf> {
+        let cache_dir = self.tarball_cache_dir();
+        let key = Self::hash_key(url);
+        let cache_entry_dir = cache_dir.join(&key);
+        let meta_path = cache_entry_dir.join(".meta");
+        let cached_content = cache_entry_dir.join("content");
+
+        #[cfg(debug_assertions)]
+        eprintln!("[CACHE] get_extracted_tarball: url={}, expected_nar_hash={}", url, expected_nar_hash);
+
+        if !meta_path.exists() || !cached_content.exists() {
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] get_extracted_tarball: cache miss - meta or content dir not found");
+            return None;
+        }
+
+        let meta: CacheMetadata =
+            serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?;
+
+        #[cfg(debug_assertions)]
+        eprintln!("[CACHE] get_extracted_tarball: cached hash={}, name={}", meta.hash, meta.name);
+
+        if meta.hash == expected_nar_hash {
+            let store_path = self.make_store_path(&meta.hash, &meta.name);
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] get_extracted_tarball: hash match, checking store_path={}", store_path.display());
+            if store_path.exists() {
+                #[cfg(debug_assertions)]
+                eprintln!("[CACHE] get_extracted_tarball: HIT - returning store path");
+                Some(store_path)
+            } else {
+                #[cfg(debug_assertions)]
+                eprintln!("[CACHE] get_extracted_tarball: store path doesn't exist");
+                None
+            }
+        } else {
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] get_extracted_tarball: hash mismatch (cached={}, expected={})", meta.hash, expected_nar_hash);
+            None
+        }
+    }
+
+    pub fn get_extracted_tarball_by_url(&self, url: &str) -> Option<(PathBuf, String)> {
+        let cache_dir = self.tarball_cache_dir();
+        let key = Self::hash_key(url);
+        let cache_entry_dir = cache_dir.join(&key);
+        let meta_path = cache_entry_dir.join(".meta");
+        let cached_content = cache_entry_dir.join("content");
+
+        #[cfg(debug_assertions)]
+        eprintln!("[CACHE] get_extracted_tarball_by_url: url={}", url);
+
+        if !meta_path.exists() || !cached_content.exists() {
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] get_extracted_tarball_by_url: cache miss - meta or content dir not found");
+            return None;
+        }
+
+        let meta: CacheMetadata =
+            serde_json::from_str(&fs::read_to_string(&meta_path).ok()?).ok()?;
+
+        #[cfg(debug_assertions)]
+        eprintln!("[CACHE] get_extracted_tarball_by_url: cached hash={}, name={}", meta.hash, meta.name);
+
+        let store_path = self.make_store_path(&meta.hash, &meta.name);
+        if store_path.exists() {
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] get_extracted_tarball_by_url: HIT - returning store path and hash");
+            Some((store_path, meta.hash))
+        } else {
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] get_extracted_tarball_by_url: store path doesn't exist");
+            None
+        }
+    }
+
     pub fn put_tarball_from_extracted(
         &self,
         url: &str,
@@ -219,6 +321,9 @@ impl FetcherCache {
         let key = Self::hash_key(url);
         let cache_entry_dir = cache_dir.join(&key);
 
+        #[cfg(debug_assertions)]
+        eprintln!("[CACHE] put_tarball_from_extracted: url={}, hash={}, name={}", url, hash, name);
+
         fs::create_dir_all(&cache_entry_dir)?;
 
         let cached_content = cache_entry_dir.join("content");
@@ -234,9 +339,16 @@ impl FetcherCache {
         fs::write(cache_entry_dir.join(".meta"), serde_json::to_string(&meta)?)?;
 
         let store_path = self.make_store_path(hash, name);
+        #[cfg(debug_assertions)]
+        eprintln!("[CACHE] put_tarball_from_extracted: store_path={}", store_path.display());
         if !store_path.exists() {
             fs::create_dir_all(store_path.parent().unwrap_or(&store_path))?;
             copy_dir_recursive(extracted_path, &store_path)?;
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] put_tarball_from_extracted: copied to store");
+        } else {
+            #[cfg(debug_assertions)]
+            eprintln!("[CACHE] put_tarball_from_extracted: store path already exists");
         }
 
         Ok(store_path)

shell.nix

@@ -0,0 +1,15 @@
+let
+  lockFile = builtins.fromJSON (builtins.readFile ./flake.lock);
+  flake-compat-node = lockFile.nodes.${lockFile.nodes.root.inputs.flake-compat};
+  flake-compat = builtins.fetchTarball {
+    inherit (flake-compat-node.locked) url;
+    sha256 = flake-compat-node.locked.narHash;
+  };
+
+  flake = (
+    import flake-compat {
+      src = ./.;
+    }
+  );
+in
+  flake.shellNix