feat: nix_nar

Cargo.lock

@@ -126,6 +126,12 @@ dependencies = [
  "vsimd",
 ]
 
+[[package]]
+name = "base64ct"
+version = "1.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
+
 [[package]]
 name = "bincode"
 version = "1.3.3"
@@ -207,6 +213,17 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "bstr"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.19.0"
@@ -254,6 +271,12 @@ dependencies = [
  "displaydoc",
 ]
 
+[[package]]
+name = "camino"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48"
+
 [[package]]
 name = "capacity_builder"
 version = "0.5.0"
@@ -297,7 +320,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
 dependencies = [
- "nom",
+ "nom 7.1.3",
 ]
 
 [[package]]
@@ -413,6 +436,12 @@ dependencies = [
  "error-code",
 ]
 
+[[package]]
+name = "const-oid"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
+
 [[package]]
 name = "constant_time_eq"
 version = "0.3.1"
@@ -574,6 +603,33 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "curve25519-dalek"
+version = "4.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "curve25519-dalek-derive",
+ "digest",
+ "fiat-crypto",
+ "rustc_version",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "curve25519-dalek-derive"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "data-encoding"
 version = "2.9.0"
@@ -707,6 +763,16 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "der"
+version = "0.7.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
+dependencies = [
+ "const-oid",
+ "zeroize",
+]
+
 [[package]]
 name = "deranged"
 version = "0.5.5"
@@ -825,6 +891,30 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "ed25519"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
+dependencies = [
+ "pkcs8",
+ "signature",
+]
+
+[[package]]
+name = "ed25519-dalek"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9"
+dependencies = [
+ "curve25519-dalek",
+ "ed25519",
+ "serde",
+ "sha2",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "either"
 version = "1.15.0"
@@ -876,6 +966,12 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "fiat-crypto"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
+
 [[package]]
 name = "filetime"
 version = "0.2.26"
@@ -1480,6 +1576,15 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "is_executable"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baabb8b4867b26294d818bf3f651a454b6901431711abb96e296245888d6e8c4"
+dependencies = [
+ "windows-sys 0.60.2",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -1729,6 +1834,39 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nix-compat"
+version = "0.1.0"
+source = "git+https://git.snix.dev/snix/snix.git#5b3716a16b64771013c1c6aefaf3c9fd218e4db8"
+dependencies = [
+ "bitflags",
+ "bstr",
+ "bytes",
+ "data-encoding",
+ "ed25519",
+ "ed25519-dalek",
+ "futures",
+ "mimalloc",
+ "nix-compat-derive",
+ "nom 8.0.0",
+ "num_enum",
+ "pin-project-lite",
+ "sha2",
+ "thiserror 2.0.17",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "nix-compat-derive"
+version = "0.1.0"
+source = "git+https://git.snix.dev/snix/snix.git#5b3716a16b64771013c1c6aefaf3c9fd218e4db8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "nix-daemon"
 version = "0.1.1"
@@ -1763,8 +1901,10 @@ dependencies = [
  "hex",
  "itertools 0.14.0",
  "mimalloc",
+ "nix-compat",
  "nix-daemon",
  "nix-js-macros",
+ "nix-nar",
  "petgraph",
  "regex",
  "reqwest",
@@ -1794,6 +1934,18 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "nix-nar"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15dbfa157df89f4283825ff1c21d53344cfe0d222ea8fde0f9514206dc62d9e0"
+dependencies = [
+ "camino",
+ "is_executable",
+ "symlink",
+ "thiserror 1.0.69",
+]
+
 [[package]]
 name = "nom"
 version = "7.1.3"
@@ -1804,6 +1956,15 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "nom"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.50.3"
@@ -1983,6 +2144,16 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkcs8"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
+dependencies = [
+ "der",
+ "spki",
+]
+
 [[package]]
 name = "pkg-config"
 version = "0.3.32"
@@ -2191,6 +2362,9 @@ name = "rand_core"
 version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom 0.2.16",
+]
 
 [[package]]
 name = "rand_core"
@@ -2619,6 +2793,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "signature"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
+dependencies = [
+ "rand_core 0.6.4",
+]
+
 [[package]]
 name = "simd-adler32"
 version = "0.3.8"
@@ -2665,6 +2848,16 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "spki"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
+dependencies = [
+ "base64ct",
+ "der",
+]
+
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.1"
@@ -2729,6 +2922,12 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
+[[package]]
+name = "symlink"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
+
 [[package]]
 name = "syn"
 version = "2.0.104"

nix-js/Cargo.toml

@@ -36,6 +36,7 @@ regex = "1.11"
 deno_core = "0.376"
 deno_error = "0.7"
 
+nix-nar = "0.3"
 sha2 = "0.10"
 hex = "0.4"
 

nix-js/src/fetcher.rs

@@ -1,4 +1,6 @@
-use deno_core::op2;
+use std::sync::Arc;
+
+use deno_core::{OpState, op2};
 use serde::Serialize;
 use tracing::debug;
 
@@ -7,12 +9,13 @@ pub(crate) mod cache;
 mod download;
 mod git;
 mod hg;
-mod nar;
 
 pub use cache::FetcherCache;
 pub use download::Downloader;
 
 use crate::runtime::NixError;
+use crate::store::StoreBackend;
+use crate::nar;
 
 #[derive(Serialize)]
 pub struct FetchUrlResult {
@@ -101,6 +104,7 @@ pub fn op_fetch_url(
 #[op2]
 #[serde]
 pub fn op_fetch_tarball(
+    state: &mut OpState,
     #[string] url: String,
     #[string] expected_hash: Option<String>,
     #[string] expected_nar_hash: Option<String>,
@@ -110,6 +114,7 @@ pub fn op_fetch_tarball(
         "fetchTarball: url={}, expected_hash={:?}, expected_nar_hash={:?}",
         url, expected_hash, expected_nar_hash
     );
+    let store = state.borrow::<Arc<StoreBackend>>();
     let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?;
     let downloader = Downloader::new();
 
@@ -193,12 +198,13 @@ pub fn op_fetch_tarball(
         }
     }
 
-    let store_path = cache
+    // let store_path = cache
-        .put_tarball_from_extracted(&url, &nar_hash, &extracted_path, &dir_name)
+    //     .put_tarball_from_extracted(&url, &nar_hash, &extracted_path, &dir_name)
-        .map_err(|e| NixError::from(e.to_string()))?;
+    //     .map_err(|e| NixError::from(e.to_string()))?;
+    let store_path = store.as_store().put_directory(&dir_name, &extracted_path).unwrap();
 
     Ok(FetchTarballResult {
-        store_path: store_path.to_string_lossy().to_string(),
+        store_path,
         hash: tarball_hash,
         nar_hash,
     })

nix-js/src/fetcher/cache.rs

@@ -1,6 +1,6 @@
 use std::fs::{self, File};
 use std::io::Write;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use serde::{Deserialize, Serialize};
 use tracing::debug;
@@ -317,7 +317,7 @@ impl FetcherCache {
         &self,
         url: &str,
         hash: &str,
-        extracted_path: &PathBuf,
+        extracted_path: &Path,
         name: &str,
     ) -> Result<PathBuf, CacheError> {
         let cache_dir = self.tarball_cache_dir();
@@ -377,7 +377,7 @@ impl FetcherCache {
     }
 }
 
-fn copy_dir_recursive(src: &PathBuf, dst: &PathBuf) -> Result<(), std::io::Error> {
+fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<(), std::io::Error> {
     fs::create_dir_all(dst)?;
 
     for entry in fs::read_dir(src)? {

nix-js/src/fetcher/nar.rs

@@ -1,127 +0,0 @@
-use sha2::{Digest, Sha256};
-use std::fs;
-use std::io::{self, Write};
-use std::path::Path;
-
-pub fn compute_nar_hash(path: &Path) -> Result<String, io::Error> {
-    let mut hasher = Sha256::new();
-    dump_path(&mut hasher, path)?;
-    Ok(hex::encode(hasher.finalize()))
-}
-
-fn dump_path<W: Write>(sink: &mut W, path: &Path) -> io::Result<()> {
-    write_string(sink, "nix-archive-1")?;
-    write_string(sink, "(")?;
-    dump_entry(sink, path)?;
-    write_string(sink, ")")?;
-    Ok(())
-}
-
-fn dump_entry<W: Write>(sink: &mut W, path: &Path) -> io::Result<()> {
-    let metadata = fs::symlink_metadata(path)?;
-
-    if metadata.is_symlink() {
-        let target = fs::read_link(path)?;
-        write_string(sink, "type")?;
-        write_string(sink, "symlink")?;
-        write_string(sink, "target")?;
-        write_string(sink, &target.to_string_lossy())?;
-    } else if metadata.is_file() {
-        write_string(sink, "type")?;
-        write_string(sink, "regular")?;
-
-        #[cfg(unix)]
-        {
-            use std::os::unix::fs::PermissionsExt;
-            if metadata.permissions().mode() & 0o111 != 0 {
-                write_string(sink, "executable")?;
-                write_string(sink, "")?;
-            }
-        }
-
-        let contents = fs::read(path)?;
-        write_string(sink, "contents")?;
-        write_contents(sink, &contents)?;
-    } else if metadata.is_dir() {
-        write_string(sink, "type")?;
-        write_string(sink, "directory")?;
-
-        let mut entries: Vec<_> = fs::read_dir(path)?
-            .filter_map(|e| e.ok())
-            .map(|e| e.file_name().to_string_lossy().to_string())
-            .collect();
-        entries.sort();
-
-        for name in entries {
-            write_string(sink, "entry")?;
-            write_string(sink, "(")?;
-            write_string(sink, "name")?;
-            write_string(sink, &name)?;
-            write_string(sink, "node")?;
-            write_string(sink, "(")?;
-            dump_entry(sink, &path.join(&name))?;
-            write_string(sink, ")")?;
-            write_string(sink, ")")?;
-        }
-    }
-
-    Ok(())
-}
-
-fn write_string<W: Write>(sink: &mut W, s: &str) -> io::Result<()> {
-    let bytes = s.as_bytes();
-    let len = bytes.len() as u64;
-
-    sink.write_all(&len.to_le_bytes())?;
-    sink.write_all(bytes)?;
-
-    let padding = (8 - (len % 8)) % 8;
-    for _ in 0..padding {
-        sink.write_all(&[0])?;
-    }
-
-    Ok(())
-}
-
-fn write_contents<W: Write>(sink: &mut W, contents: &[u8]) -> io::Result<()> {
-    let len = contents.len() as u64;
-
-    sink.write_all(&len.to_le_bytes())?;
-    sink.write_all(contents)?;
-
-    let padding = (8 - (len % 8)) % 8;
-    for _ in 0..padding {
-        sink.write_all(&[0])?;
-    }
-
-    Ok(())
-}
-
-#[cfg(test)]
-#[allow(clippy::unwrap_used)]
-mod tests {
-    use super::*;
-    use tempfile::TempDir;
-
-    #[test]
-    fn test_simple_file() {
-        let temp = TempDir::new().unwrap();
-        let file_path = temp.path().join("test.txt");
-        fs::write(&file_path, "hello").unwrap();
-
-        let hash = compute_nar_hash(&file_path).unwrap();
-        assert!(!hash.is_empty());
-        assert_eq!(hash.len(), 64);
-    }
-
-    #[test]
-    fn test_directory() {
-        let temp = TempDir::new().unwrap();
-        fs::write(temp.path().join("a.txt"), "aaa").unwrap();
-        fs::write(temp.path().join("b.txt"), "bbb").unwrap();
-
-        let hash = compute_nar_hash(temp.path()).unwrap();
-        assert!(!hash.is_empty());
-        assert_eq!(hash.len(), 64);
-    }
-}

nix-js/src/lib.rs

@@ -8,6 +8,7 @@ mod codegen;
 mod fetcher;
 mod ir;
 mod nix_hash;
+mod nar;
 mod runtime;
 mod store;
 

nix-js/src/nar.rs

@@ -0,0 +1,63 @@
+use nix_nar::Encoder;
+use sha2::{Digest, Sha256};
+use std::io::Read;
+use std::path::Path;
+
+use crate::error::{Error, Result};
+
+pub fn compute_nar_hash(path: &Path) -> Result<String> {
+    let mut hasher = Sha256::new();
+    std::io::copy(
+        &mut Encoder::new(path).map_err(|err| Error::internal(err.to_string()))?,
+        &mut hasher,
+    )
+    .map_err(|err| Error::internal(err.to_string()))?;
+    Ok(hex::encode(hasher.finalize()))
+}
+
+pub fn pack_nar(path: &Path) -> Result<Vec<u8>> {
+    let mut buffer = Vec::new();
+    Encoder::new(path)
+        .map_err(|err| Error::internal(err.to_string()))?
+        .read_to_end(&mut buffer)
+        .map_err(|err| Error::internal(err.to_string()))?;
+    Ok(buffer)
+}
+
+#[cfg(test)]
+#[allow(clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_simple_file() {
+        let temp = TempDir::new().unwrap();
+        let file_path = temp.path().join("test.txt");
+        fs::write(&file_path, "hello").unwrap();
+
+        let hash = compute_nar_hash(&file_path).unwrap();
+        assert_eq!(
+            hash,
+            "0a430879c266f8b57f4092a0f935cf3facd48bbccde5760d4748ca405171e969"
+        );
+        assert!(!hash.is_empty());
+        assert_eq!(hash.len(), 64);
+    }
+
+    #[test]
+    fn test_directory() {
+        let temp = TempDir::new().unwrap();
+        fs::write(temp.path().join("a.txt"), "aaa").unwrap();
+        fs::write(temp.path().join("b.txt"), "bbb").unwrap();
+
+        let hash = compute_nar_hash(temp.path()).unwrap();
+        assert_eq!(
+            hash,
+            "0036c14209749bc9b9631e2077b108b701c322ab53853cd26f2746268a86fc0f"
+        );
+        assert!(!hash.is_empty());
+        assert_eq!(hash.len(), 64);
+    }
+}

nix-js/src/store.rs

@@ -2,6 +2,8 @@ mod config;
 mod error;
 mod validation;
 
+use std::path::Path;
+
 pub use config::{StoreConfig, StoreMode};
 pub use validation::validate_store_path;
 
@@ -29,6 +31,8 @@ pub trait Store: Send + Sync {
         references: Vec<String>,
     ) -> Result<String>;
 
+    fn put_directory(&self, name: &str, path: &Path) -> Result<String>;
+
     fn make_fixed_output_path(
         &self,
         hash_algo: &str,

nix-js/src/store/daemon.rs

@@ -6,6 +6,7 @@ use tokio::net::UnixStream;
 use tokio::sync::Mutex;
 
 use crate::error::{Error, Result};
+use crate::nar::pack_nar;
 
 use super::Store;
 
@@ -137,6 +138,21 @@ impl Store for DaemonStore {
         })
     }
 
+    fn put_directory(&self, name: &str, path: &Path) -> Result<String> {
+        self.block_on(async {
+            let mut store = self.store.lock().await;
+            let (store_path, _) = store
+                .add_to_store(name, "text:sha256", std::iter::empty::<&str>(), false, pack_nar(path).unwrap().as_slice())
+                .result()
+                .await
+                .map_err(|e| {
+                    Error::internal(format!("Daemon error in add_text_to_store: {}", e))
+                })?;
+
+            Ok(store_path)
+        })
+    }
+
     fn make_fixed_output_path(
         &self,
         _hash_algo: &str,

nix-js/src/store/simulated.rs

@@ -93,4 +93,8 @@ impl Store for SimulatedStore {
         let store_path = self.cache.make_store_path(hash, name);
         Ok(store_path.to_string_lossy().to_string())
     }
+
+    fn put_directory(&self, name: &str, path: &Path) -> Result<String> {
+        todo!()
+    }
 }

nix-js/tests/builtins_store.rs

@@ -8,7 +8,10 @@ use utils::eval_result;
 fn init() {
     static INIT: Once = Once::new();
     INIT.call_once(|| {
+        #[cfg(not(feature = "daemon"))]
         unsafe { std::env::set_var("NIX_JS_STORE_MODE", "simulated") };
+        #[cfg(feature = "daemon")]
+        unsafe { std::env::set_var("NIX_JS_STORE_MODE", "daemon") };
     });
 }