diff --git a/Cargo.lock b/Cargo.lock index 7e5f23f..982b2fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -126,6 +126,12 @@ dependencies = [ "vsimd", ] +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bincode" version = "1.3.3" @@ -207,6 +213,17 @@ dependencies = [ "syn", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -254,6 +271,12 @@ dependencies = [ "displaydoc", ] +[[package]] +name = "camino" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" + [[package]] name = "capacity_builder" version = "0.5.0" @@ -297,7 +320,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -413,6 +436,12 @@ dependencies = [ "error-code", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -574,6 +603,33 @@ dependencies = [ "typenum", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -707,6 +763,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.5" @@ -825,6 +891,30 @@ dependencies = [ "syn", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.15.0" @@ -876,6 +966,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "filetime" version = "0.2.26" @@ -1480,6 +1576,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "is_executable" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baabb8b4867b26294d818bf3f651a454b6901431711abb96e296245888d6e8c4" +dependencies = [ + "windows-sys 0.60.2", +] + [[package]] name = "itertools" version = "0.10.5" @@ -1729,6 +1834,39 @@ dependencies = [ "libc", ] +[[package]] +name = "nix-compat" +version = "0.1.0" +source = "git+https://git.snix.dev/snix/snix.git#5b3716a16b64771013c1c6aefaf3c9fd218e4db8" +dependencies = [ + "bitflags", + "bstr", + "bytes", + "data-encoding", + "ed25519", + "ed25519-dalek", + "futures", + "mimalloc", + "nix-compat-derive", + "nom 8.0.0", + "num_enum", + "pin-project-lite", + "sha2", + "thiserror 2.0.17", + "tokio", + "tracing", +] + +[[package]] +name = "nix-compat-derive" +version = "0.1.0" +source = "git+https://git.snix.dev/snix/snix.git#5b3716a16b64771013c1c6aefaf3c9fd218e4db8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "nix-daemon" version = "0.1.1" @@ -1763,8 +1901,10 @@ dependencies = [ "hex", "itertools 0.14.0", "mimalloc", + "nix-compat", "nix-daemon", "nix-js-macros", + "nix-nar", "petgraph", "regex", "reqwest", @@ -1794,6 +1934,18 @@ dependencies = [ "syn", ] +[[package]] +name = "nix-nar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15dbfa157df89f4283825ff1c21d53344cfe0d222ea8fde0f9514206dc62d9e0" +dependencies = [ + "camino", + "is_executable", + "symlink", + "thiserror 1.0.69", +] + [[package]] name = "nom" version = "7.1.3" @@ -1804,6 +1956,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1983,6 +2144,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -2191,6 +2362,9 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] [[package]] name = "rand_core" @@ -2619,6 +2793,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -2665,6 +2848,16 @@ dependencies = [ "url", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2729,6 +2922,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "2.0.104" diff --git a/nix-js/Cargo.toml b/nix-js/Cargo.toml index 94ad8b8..b07a26c 100644 --- a/nix-js/Cargo.toml +++ b/nix-js/Cargo.toml @@ -36,6 +36,7 @@ regex = "1.11" deno_core = "0.376" deno_error = "0.7" +nix-nar = "0.3" sha2 = "0.10" hex = "0.4" diff --git a/nix-js/src/fetcher.rs b/nix-js/src/fetcher.rs index 2ad63a7..cf16caa 100644 --- a/nix-js/src/fetcher.rs +++ b/nix-js/src/fetcher.rs @@ -1,4 +1,6 @@ -use deno_core::op2; +use std::sync::Arc; + +use deno_core::{OpState, op2}; use serde::Serialize; use tracing::debug; @@ -7,12 +9,13 @@ pub(crate) mod cache; mod download; mod git; mod hg; -mod nar; pub use cache::FetcherCache; pub use download::Downloader; use crate::runtime::NixError; +use crate::store::StoreBackend; +use crate::nar; #[derive(Serialize)] pub struct FetchUrlResult { @@ -101,6 +104,7 @@ pub fn op_fetch_url( #[op2] #[serde] pub fn op_fetch_tarball( + state: &mut OpState, #[string] url: String, #[string] expected_hash: Option, #[string] expected_nar_hash: Option, @@ -110,6 +114,7 @@ pub fn op_fetch_tarball( "fetchTarball: url={}, expected_hash={:?}, expected_nar_hash={:?}", url, expected_hash, expected_nar_hash ); + let store = state.borrow::>(); let cache = FetcherCache::new().map_err(|e| NixError::from(e.to_string()))?; let downloader = Downloader::new(); @@ -193,12 +198,13 @@ pub fn op_fetch_tarball( } } - let store_path = cache - .put_tarball_from_extracted(&url, &nar_hash, &extracted_path, &dir_name) - .map_err(|e| NixError::from(e.to_string()))?; + // let store_path = cache + // .put_tarball_from_extracted(&url, &nar_hash, &extracted_path, &dir_name) + // .map_err(|e| NixError::from(e.to_string()))?; + let store_path = store.as_store().put_directory(&dir_name, &extracted_path).unwrap(); Ok(FetchTarballResult { - store_path: store_path.to_string_lossy().to_string(), + store_path, hash: tarball_hash, nar_hash, }) diff --git a/nix-js/src/fetcher/cache.rs b/nix-js/src/fetcher/cache.rs index ec547db..9f9e3ce 100644 --- a/nix-js/src/fetcher/cache.rs +++ b/nix-js/src/fetcher/cache.rs @@ -1,6 +1,6 @@ use std::fs::{self, File}; use std::io::Write; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use tracing::debug; @@ -317,7 +317,7 @@ impl FetcherCache { &self, url: &str, hash: &str, - extracted_path: &PathBuf, + extracted_path: &Path, name: &str, ) -> Result { let cache_dir = self.tarball_cache_dir(); @@ -377,7 +377,7 @@ impl FetcherCache { } } -fn copy_dir_recursive(src: &PathBuf, dst: &PathBuf) -> Result<(), std::io::Error> { +fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<(), std::io::Error> { fs::create_dir_all(dst)?; for entry in fs::read_dir(src)? { diff --git a/nix-js/src/fetcher/nar.rs b/nix-js/src/fetcher/nar.rs deleted file mode 100644 index 7b2bebb..0000000 --- a/nix-js/src/fetcher/nar.rs +++ /dev/null @@ -1,127 +0,0 @@ -use sha2::{Digest, Sha256}; -use std::fs; -use std::io::{self, Write}; -use std::path::Path; - -pub fn compute_nar_hash(path: &Path) -> Result { - let mut hasher = Sha256::new(); - dump_path(&mut hasher, path)?; - Ok(hex::encode(hasher.finalize())) -} - -fn dump_path(sink: &mut W, path: &Path) -> io::Result<()> { - write_string(sink, "nix-archive-1")?; - write_string(sink, "(")?; - dump_entry(sink, path)?; - write_string(sink, ")")?; - Ok(()) -} - -fn dump_entry(sink: &mut W, path: &Path) -> io::Result<()> { - let metadata = fs::symlink_metadata(path)?; - - if metadata.is_symlink() { - let target = fs::read_link(path)?; - write_string(sink, "type")?; - write_string(sink, "symlink")?; - write_string(sink, "target")?; - write_string(sink, &target.to_string_lossy())?; - } else if metadata.is_file() { - write_string(sink, "type")?; - write_string(sink, "regular")?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - if metadata.permissions().mode() & 0o111 != 0 { - write_string(sink, "executable")?; - write_string(sink, "")?; - } - } - - let contents = fs::read(path)?; - write_string(sink, "contents")?; - write_contents(sink, &contents)?; - } else if metadata.is_dir() { - write_string(sink, "type")?; - write_string(sink, "directory")?; - - let mut entries: Vec<_> = fs::read_dir(path)? - .filter_map(|e| e.ok()) - .map(|e| e.file_name().to_string_lossy().to_string()) - .collect(); - entries.sort(); - - for name in entries { - write_string(sink, "entry")?; - write_string(sink, "(")?; - write_string(sink, "name")?; - write_string(sink, &name)?; - write_string(sink, "node")?; - write_string(sink, "(")?; - dump_entry(sink, &path.join(&name))?; - write_string(sink, ")")?; - write_string(sink, ")")?; - } - } - - Ok(()) -} - -fn write_string(sink: &mut W, s: &str) -> io::Result<()> { - let bytes = s.as_bytes(); - let len = bytes.len() as u64; - - sink.write_all(&len.to_le_bytes())?; - sink.write_all(bytes)?; - - let padding = (8 - (len % 8)) % 8; - for _ in 0..padding { - sink.write_all(&[0])?; - } - - Ok(()) -} - -fn write_contents(sink: &mut W, contents: &[u8]) -> io::Result<()> { - let len = contents.len() as u64; - - sink.write_all(&len.to_le_bytes())?; - sink.write_all(contents)?; - - let padding = (8 - (len % 8)) % 8; - for _ in 0..padding { - sink.write_all(&[0])?; - } - - Ok(()) -} - -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod tests { - use super::*; - use tempfile::TempDir; - - #[test] - fn test_simple_file() { - let temp = TempDir::new().unwrap(); - let file_path = temp.path().join("test.txt"); - fs::write(&file_path, "hello").unwrap(); - - let hash = compute_nar_hash(&file_path).unwrap(); - assert!(!hash.is_empty()); - assert_eq!(hash.len(), 64); - } - - #[test] - fn test_directory() { - let temp = TempDir::new().unwrap(); - fs::write(temp.path().join("a.txt"), "aaa").unwrap(); - fs::write(temp.path().join("b.txt"), "bbb").unwrap(); - - let hash = compute_nar_hash(temp.path()).unwrap(); - assert!(!hash.is_empty()); - assert_eq!(hash.len(), 64); - } -} diff --git a/nix-js/src/lib.rs b/nix-js/src/lib.rs index 972bedf..76e6396 100644 --- a/nix-js/src/lib.rs +++ b/nix-js/src/lib.rs @@ -8,6 +8,7 @@ mod codegen; mod fetcher; mod ir; mod nix_hash; +mod nar; mod runtime; mod store; diff --git a/nix-js/src/nar.rs b/nix-js/src/nar.rs new file mode 100644 index 0000000..ab51867 --- /dev/null +++ b/nix-js/src/nar.rs @@ -0,0 +1,63 @@ +use nix_nar::Encoder; +use sha2::{Digest, Sha256}; +use std::io::Read; +use std::path::Path; + +use crate::error::{Error, Result}; + +pub fn compute_nar_hash(path: &Path) -> Result { + let mut hasher = Sha256::new(); + std::io::copy( + &mut Encoder::new(path).map_err(|err| Error::internal(err.to_string()))?, + &mut hasher, + ) + .map_err(|err| Error::internal(err.to_string()))?; + Ok(hex::encode(hasher.finalize())) +} + +pub fn pack_nar(path: &Path) -> Result> { + let mut buffer = Vec::new(); + Encoder::new(path) + .map_err(|err| Error::internal(err.to_string()))? + .read_to_end(&mut buffer) + .map_err(|err| Error::internal(err.to_string()))?; + Ok(buffer) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + use std::fs; + use tempfile::TempDir; + + #[test] + fn test_simple_file() { + let temp = TempDir::new().unwrap(); + let file_path = temp.path().join("test.txt"); + fs::write(&file_path, "hello").unwrap(); + + let hash = compute_nar_hash(&file_path).unwrap(); + assert_eq!( + hash, + "0a430879c266f8b57f4092a0f935cf3facd48bbccde5760d4748ca405171e969" + ); + assert!(!hash.is_empty()); + assert_eq!(hash.len(), 64); + } + + #[test] + fn test_directory() { + let temp = TempDir::new().unwrap(); + fs::write(temp.path().join("a.txt"), "aaa").unwrap(); + fs::write(temp.path().join("b.txt"), "bbb").unwrap(); + + let hash = compute_nar_hash(temp.path()).unwrap(); + assert_eq!( + hash, + "0036c14209749bc9b9631e2077b108b701c322ab53853cd26f2746268a86fc0f" + ); + assert!(!hash.is_empty()); + assert_eq!(hash.len(), 64); + } +} diff --git a/nix-js/src/store.rs b/nix-js/src/store.rs index cf47e86..3aee647 100644 --- a/nix-js/src/store.rs +++ b/nix-js/src/store.rs @@ -2,6 +2,8 @@ mod config; mod error; mod validation; +use std::path::Path; + pub use config::{StoreConfig, StoreMode}; pub use validation::validate_store_path; @@ -29,6 +31,8 @@ pub trait Store: Send + Sync { references: Vec, ) -> Result; + fn put_directory(&self, name: &str, path: &Path) -> Result; + fn make_fixed_output_path( &self, hash_algo: &str, diff --git a/nix-js/src/store/daemon.rs b/nix-js/src/store/daemon.rs index 62d217c..0ea36e6 100644 --- a/nix-js/src/store/daemon.rs +++ b/nix-js/src/store/daemon.rs @@ -6,6 +6,7 @@ use tokio::net::UnixStream; use tokio::sync::Mutex; use crate::error::{Error, Result}; +use crate::nar::pack_nar; use super::Store; @@ -137,6 +138,21 @@ impl Store for DaemonStore { }) } + fn put_directory(&self, name: &str, path: &Path) -> Result { + self.block_on(async { + let mut store = self.store.lock().await; + let (store_path, _) = store + .add_to_store(name, "text:sha256", std::iter::empty::<&str>(), false, pack_nar(path).unwrap().as_slice()) + .result() + .await + .map_err(|e| { + Error::internal(format!("Daemon error in add_text_to_store: {}", e)) + })?; + + Ok(store_path) + }) + } + fn make_fixed_output_path( &self, _hash_algo: &str, diff --git a/nix-js/src/store/simulated.rs b/nix-js/src/store/simulated.rs index 9ea40b6..1397774 100644 --- a/nix-js/src/store/simulated.rs +++ b/nix-js/src/store/simulated.rs @@ -93,4 +93,8 @@ impl Store for SimulatedStore { let store_path = self.cache.make_store_path(hash, name); Ok(store_path.to_string_lossy().to_string()) } + + fn put_directory(&self, name: &str, path: &Path) -> Result { + todo!() + } } diff --git a/nix-js/tests/builtins_store.rs b/nix-js/tests/builtins_store.rs index ac9baa3..a4fe364 100644 --- a/nix-js/tests/builtins_store.rs +++ b/nix-js/tests/builtins_store.rs @@ -8,7 +8,10 @@ use utils::eval_result; fn init() { static INIT: Once = Once::new(); INIT.call_once(|| { + #[cfg(not(feature = "daemon"))] unsafe { std::env::set_var("NIX_JS_STORE_MODE", "simulated") }; + #[cfg(feature = "daemon")] + unsafe { std::env::set_var("NIX_JS_STORE_MODE", "daemon") }; }); }