diff --git a/Cargo.lock b/Cargo.lock index 73e6080..214d49d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1781,6 +1781,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.8.0" @@ -1924,6 +1930,7 @@ dependencies = [ "hashbrown 0.16.1", "hex", "itertools 0.14.0", + "md5", "miette", "mimalloc", "nix-compat", @@ -1939,6 +1946,7 @@ dependencies = [ "rustyline", "serde", "serde_json", + "sha1", "sha2", "string-interner", "tap", @@ -2895,6 +2903,17 @@ dependencies = [ "v8", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" diff --git a/flake.nix b/flake.nix index efeb0c3..ac31e64 100644 --- a/flake.nix +++ b/flake.nix @@ -34,6 +34,7 @@ just samply jq + tokei nodejs nodePackages.npm diff --git a/nix-js/Cargo.toml b/nix-js/Cargo.toml index 73aabeb..094781e 100644 --- a/nix-js/Cargo.toml +++ b/nix-js/Cargo.toml @@ -36,6 +36,8 @@ deno_error = "0.7" nix-nar = "0.3" sha2 = "0.10" +sha1 = "0.10" +md5 = "0.7" hex = "0.4" base64 = "0.22" diff --git a/nix-js/runtime-ts/src/builtins/attrs.ts b/nix-js/runtime-ts/src/builtins/attrs.ts index 8e4934d..20789d1 100644 --- a/nix-js/runtime-ts/src/builtins/attrs.ts +++ b/nix-js/runtime-ts/src/builtins/attrs.ts @@ -1,6 +1,7 @@ +import { mkPos } from "../helpers"; import { createThunk } from "../thunk"; import { forceAttrs, forceFunction, forceList, forceStringValue } from "../type-assert"; -import type { NixAttrs, NixList, NixValue } from "../types"; +import { ATTR_POSITIONS, type NixAttrs, type NixList, type NixValue } from "../types"; export const attrNames = (set: NixValue): string[] => Object.keys(forceAttrs(set)).sort(); @@ -139,7 +140,7 @@ export const unsafeGetAttrPos = return null; } - const positions = (attrs as NixAttrs & Record)[Nix.ATTR_POSITIONS] as + const positions = (attrs as NixAttrs & Record)[ATTR_POSITIONS] as | Record | undefined; if (!positions || !(name in positions)) { @@ -147,5 +148,5 @@ export const unsafeGetAttrPos = } const span = positions[name]; - return Nix.mkPos(span); + return mkPos(span); }; diff --git a/nix-js/runtime-ts/src/builtins/derivation.ts b/nix-js/runtime-ts/src/builtins/derivation.ts index 4fa1ee9..386ec49 100644 --- a/nix-js/runtime-ts/src/builtins/derivation.ts +++ b/nix-js/runtime-ts/src/builtins/derivation.ts @@ -1,7 +1,6 @@ import { addBuiltContext, addDrvDeepContext, - extractInputDrvsAndSrcs, mkStringWithContext, type NixStringContext, } from "../string-context"; @@ -10,8 +9,6 @@ import { forceAttrs, forceList, forceStringNoCtx, forceStringValue } from "../ty import type { NixAttrs, NixValue } from "../types"; import { coerceToString, nixValueToJson, StringCoercionMode } from "./conversion"; -const drvHashCache = new Map(); - export interface OutputInfo { path: string; hashAlgo: string; @@ -190,13 +187,6 @@ const extractArgs = (attrs: NixAttrs, outContext: NixStringContext): string[] => return argsList.map((a) => coerceToString(a, StringCoercionMode.ToString, true, outContext)); }; -const outputPathName = (drvName: string, output: string) => { - if (output === "out") { - return drvName; - } - return `${drvName}-${output}`; -}; - const structuredAttrsExcludedKeys = new Set([ "__structuredAttrs", "__ignoreNulls", @@ -369,134 +359,33 @@ export const derivationStrict = (args: NixValue): NixAttrs => { const drvArgs = extractArgs(attrs, collectedContext); const env = extractEnv(attrs, structuredAttrs, ignoreNulls, collectedContext, drvName); - const { inputDrvs, inputSrcs } = extractInputDrvsAndSrcs(collectedContext); + const envEntries: [string, string][] = Array.from(env.entries()); + const contextArray: string[] = Array.from(collectedContext); - const collectDrvReferences = (): string[] => { - const refs = new Set(); - for (const src of inputSrcs) { - refs.add(src); - } - for (const drvPath of inputDrvs.keys()) { - refs.add(drvPath); - } - return Array.from(refs).sort(); - }; - - let outputInfos: Map; - let drvPath: string; - - if (fixedOutputInfo) { - const pathName = outputPathName(drvName, "out"); - const outPath = Deno.core.ops.op_make_fixed_output_path( - fixedOutputInfo.hashAlgo, - fixedOutputInfo.hash, - fixedOutputInfo.hashMode, - pathName, - ); - - const hashAlgoPrefix = fixedOutputInfo.hashMode === "recursive" ? "r:" : ""; - outputInfos = new Map([ - [ - "out", - { - path: outPath, - hashAlgo: hashAlgoPrefix + fixedOutputInfo.hashAlgo, - hash: fixedOutputInfo.hash, - }, - ], - ]); - env.set("out", outPath); - - const finalDrv: DerivationData = { - name: drvName, - outputs: outputInfos, - inputDrvs, - inputSrcs, - platform, - builder, - args: drvArgs, - env, - }; - const finalAterm = generateAterm(finalDrv); - drvPath = Deno.core.ops.op_write_derivation(drvName, finalAterm, collectDrvReferences()); - - const fixedHashFingerprint = `fixed:out:${hashAlgoPrefix}${fixedOutputInfo.hashAlgo}:${fixedOutputInfo.hash}:${outPath}`; - const fixedModuloHash = Deno.core.ops.op_sha256_hex(fixedHashFingerprint); - drvHashCache.set(drvPath, fixedModuloHash); - } else { - const maskedOutputs = new Map( - outputs.map((o) => [ - o, - { - path: "", - hashAlgo: "", - hash: "", - }, - ]), - ); - const maskedEnv = new Map(env); - for (const output of outputs) { - maskedEnv.set(output, ""); - } - - const maskedDrv: DerivationData = { - name: drvName, - outputs: maskedOutputs, - inputDrvs, - inputSrcs, - platform, - builder, - args: drvArgs, - env: maskedEnv, - }; - - const inputDrvHashes = new Map(); - for (const [drvPath, outputNames] of inputDrvs) { - const cachedHash = drvHashCache.get(drvPath); - if (!cachedHash) { - throw new Error(`Missing modulo hash for input derivation: ${drvPath}`); - } - inputDrvHashes.set(cachedHash, Array.from(outputNames).join(",")); - } - - const maskedAterm = generateAtermModulo(maskedDrv, inputDrvHashes); - const drvModuloHash = Deno.core.ops.op_sha256_hex(maskedAterm); - - outputInfos = new Map(); - for (const outputName of outputs) { - const pathName = outputPathName(drvName, outputName); - const outPath = Deno.core.ops.op_make_store_path(`output:${outputName}`, drvModuloHash, pathName); - outputInfos.set(outputName, { - path: outPath, - hashAlgo: "", - hash: "", - }); - env.set(outputName, outPath); - } - - const finalDrv: DerivationData = { - ...maskedDrv, - outputs: outputInfos, - env, - }; - const finalAterm = generateAterm(finalDrv); - drvPath = Deno.core.ops.op_write_derivation(drvName, finalAterm, collectDrvReferences()); - - const finalAtermModulo = generateAtermModulo(finalDrv, inputDrvHashes); - const cachedModuloHash = Deno.core.ops.op_sha256_hex(finalAtermModulo); - drvHashCache.set(drvPath, cachedModuloHash); - } + const rustResult: { + drvPath: string; + outputs: [string, string][]; + } = Deno.core.ops.op_finalize_derivation({ + name: drvName, + builder, + platform, + outputs, + args: drvArgs, + env: envEntries, + context: contextArray, + fixedOutput: fixedOutputInfo, + }); const result: NixAttrs = {}; const drvPathContext = new Set(); - addDrvDeepContext(drvPathContext, drvPath); - result.drvPath = mkStringWithContext(drvPath, drvPathContext); + addDrvDeepContext(drvPathContext, rustResult.drvPath); + result.drvPath = mkStringWithContext(rustResult.drvPath, drvPathContext); - for (const [outputName, outputInfo] of outputInfos.entries()) { + for (const [outputName, outputPath] of rustResult.outputs) { const outputContext = new Set(); - addBuiltContext(outputContext, drvPath, outputName); - result[outputName] = mkStringWithContext(outputInfo.path, outputContext); + addBuiltContext(outputContext, rustResult.drvPath, outputName); + result[outputName] = mkStringWithContext(outputPath, outputContext); } return result; diff --git a/nix-js/runtime-ts/src/builtins/misc.ts b/nix-js/runtime-ts/src/builtins/misc.ts index 40b4c9b..0966c53 100644 --- a/nix-js/runtime-ts/src/builtins/misc.ts +++ b/nix-js/runtime-ts/src/builtins/misc.ts @@ -209,7 +209,7 @@ export const genericClosure = (args: NixValue): NixValue => { export const outputOf = (_drv: NixValue) => (_out: NixValue): never => { - throw new Error("Not implemented: outputOf"); + throw new Error("Not implemented: outputOf (part of dynamic-derivation)"); }; export const parseDrvName = (s: NixValue): NixAttrs => { @@ -320,8 +320,9 @@ export const splitVersion = (s: NixValue): NixValue => { return components; }; -export const traceVerbose = (_e1: NixValue, _e2: NixValue): never => { - throw new Error("Not implemented: traceVerbose"); +export const traceVerbose = (_e1: NixValue, e2: NixValue): NixStrictValue => { + // TODO: implement traceVerbose + return force(e2) }; export const tryEval = (e: NixValue): { success: NixBool; value: NixStrictValue } => { diff --git a/nix-js/runtime-ts/src/string-context.ts b/nix-js/runtime-ts/src/string-context.ts index c488bc7..fdd26a8 100644 --- a/nix-js/runtime-ts/src/string-context.ts +++ b/nix-js/runtime-ts/src/string-context.ts @@ -146,52 +146,3 @@ export const parseContextToInfoMap = (context: NixStringContext): Map>; inputSrcs: Set } => { - const inputDrvs = new Map>(); - const inputSrcs = new Set(); - - for (const encoded of context) { - const elem = decodeContextElem(encoded); - switch (elem.type) { - case "opaque": - inputSrcs.add(elem.path); - break; - case "drvDeep": { - const closure: { - input_drvs: [string, string[]][]; - input_srcs: string[]; - } = Deno.core.ops.op_compute_fs_closure(elem.drvPath); - - for (const src of closure.input_srcs) { - inputSrcs.add(src); - } - - for (const [drvPath, outputs] of closure.input_drvs) { - let existingOutputs = inputDrvs.get(drvPath); - if (!existingOutputs) { - existingOutputs = new Set(); - inputDrvs.set(drvPath, existingOutputs); - } - for (const output of outputs) { - existingOutputs.add(output); - } - } - break; - } - case "built": { - let outputs = inputDrvs.get(elem.drvPath); - if (!outputs) { - outputs = new Set(); - inputDrvs.set(elem.drvPath, outputs); - } - outputs.add(elem.output); - break; - } - } - } - - return { inputDrvs, inputSrcs }; -}; diff --git a/nix-js/runtime-ts/src/types/global.d.ts b/nix-js/runtime-ts/src/types/global.d.ts index 1d45ec6..56580d8 100644 --- a/nix-js/runtime-ts/src/types/global.d.ts +++ b/nix-js/runtime-ts/src/types/global.d.ts @@ -1,33 +1,70 @@ import type { NixRuntime } from ".."; -import type { FetchTarballResult, FetchUrlResult, FetchGitResult } from "../builtins/io"; +import type { FetchGitResult, FetchTarballResult, FetchUrlResult } from "../builtins/io"; declare global { var Nix: NixRuntime; namespace Deno { namespace core { namespace ops { - function op_resolve_path(currentDir: string, path: string): string; function op_import(path: string): string; function op_scoped_import(path: string, scopeKeys: string[]): string; + + function op_resolve_path(currentDir: string, path: string): string; + function op_read_file(path: string): string; function op_read_file_type(path: string): string; function op_read_dir(path: string): Record; function op_path_exists(path: string): boolean; - function op_sha256_hex(data: string): string; + function op_walk_dir(path: string): [string, string][]; + function op_make_placeholder(output: string): string; + function op_store_path(path: string): string; + + function op_parse_hash(hashStr: string, algo: string | null): { hex: string; algo: string }; + + function op_add_path( + path: string, + name: string | null, + recursive: boolean, + sha256: string | null, + ): string; + function op_add_filtered_path( + path: string, + name: string | null, + recursive: boolean, + sha256: string | null, + includePaths: string[], + ): string; + function op_decode_span(span: string): { file: string | null; line: number | null; column: number | null; }; - function op_make_store_path(ty: string, hashHex: string, name: string): string; - function op_parse_hash(hashStr: string, algo: string | null): { hex: string; algo: string }; - function op_make_fixed_output_path( - hashAlgo: string, - hash: string, - hashMode: string, - name: string, - ): string; + + function op_to_file(name: string, contents: string, references: string[]): string; + + function op_copy_path_to_store(path: string): string; + + function op_get_env(key: string): string; + + function op_match(regex: string, text: string): (string | null)[] | null; + function op_split(regex: string, text: string): (string | (string | null)[])[]; + + function op_from_json(json: string): unknown; + function op_from_toml(toml: string): unknown; + + function op_finalize_derivation(input: { + name: string; + builder: string; + platform: string; + outputs: string[]; + args: string[]; + env: [string, string][]; + context: string[]; + fixedOutput: { hashAlgo: string; hash: string; hashMode: string } | null; + }): { drvPath: string; outputs: [string, string][] }; + function op_fetch_url( url: string, expectedHash: string | null, @@ -48,34 +85,6 @@ declare global { allRefs: boolean, name: string | null, ): FetchGitResult; - function op_add_path( - path: string, - name: string | null, - recursive: boolean, - sha256: string | null, - ): string; - function op_store_path(path: string): string; - function op_to_file(name: string, contents: string, references: string[]): string; - function op_write_derivation(drvName: string, aterm: string, references: string[]): string; - function op_read_derivation_outputs(drvPath: string): string[]; - function op_compute_fs_closure(drvPath: string): { - input_drvs: [string, string[]][]; - input_srcs: string[]; - }; - function op_copy_path_to_store(path: string): string; - function op_get_env(key: string): string; - function op_walk_dir(path: string): [string, string][]; - function op_add_filtered_path( - path: string, - name: string | null, - recursive: boolean, - sha256: string | null, - includePaths: string[], - ): string; - function op_match(regex: string, text: string): (string | null)[] | null; - function op_split(regex: string, text: string): (string | (string | null)[])[]; - function op_from_json(json: string): unknown; - function op_from_toml(toml: string): unknown; } } } diff --git a/nix-js/src/derivation.rs b/nix-js/src/derivation.rs new file mode 100644 index 0000000..4f80227 --- /dev/null +++ b/nix-js/src/derivation.rs @@ -0,0 +1,145 @@ +use std::collections::{BTreeMap, BTreeSet}; + +pub struct OutputInfo { + pub path: String, + pub hash_algo: String, + pub hash: String, +} + +pub struct DerivationData { + pub name: String, + pub outputs: BTreeMap, + pub input_drvs: BTreeMap>, + pub input_srcs: BTreeSet, + pub platform: String, + pub builder: String, + pub args: Vec, + pub env: BTreeMap, +} + +fn escape_string(s: &str) -> String { + let mut result = String::with_capacity(s.len() + 2); + result.push('"'); + for c in s.chars() { + match c { + '"' => result.push_str("\\\""), + '\\' => result.push_str("\\\\"), + '\n' => result.push_str("\\n"), + '\r' => result.push_str("\\r"), + '\t' => result.push_str("\\t"), + _ => result.push(c), + } + } + result.push('"'); + result +} + +fn quote_string(s: &str) -> String { + format!("\"{}\"", s) +} + +impl DerivationData { + pub fn generate_aterm(&self) -> String { + let mut output_entries = Vec::new(); + for (name, info) in &self.outputs { + output_entries.push(format!( + "({},{},{},{})", + quote_string(name), + quote_string(&info.path), + quote_string(&info.hash_algo), + quote_string(&info.hash), + )); + } + let outputs = output_entries.join(","); + + let mut input_drv_entries = Vec::new(); + for (drv_path, output_names) in &self.input_drvs { + let sorted_outs: Vec = output_names.iter().map(|s| quote_string(s)).collect(); + let out_list = format!("[{}]", sorted_outs.join(",")); + input_drv_entries.push(format!("({},{})", quote_string(drv_path), out_list)); + } + let input_drvs = input_drv_entries.join(","); + + let input_srcs: Vec = self.input_srcs.iter().map(|s| quote_string(s)).collect(); + let input_srcs = input_srcs.join(","); + + let args: Vec = self.args.iter().map(|s| escape_string(s)).collect(); + let args = args.join(","); + + let mut env_entries: Vec = Vec::new(); + for (k, v) in &self.env { + env_entries.push(format!("({},{})", escape_string(k), escape_string(v))); + } + + format!( + "Derive([{}],[{}],[{}],{},{},[{}],[{}])", + outputs, + input_drvs, + input_srcs, + quote_string(&self.platform), + escape_string(&self.builder), + args, + env_entries.join(","), + ) + } + + pub fn generate_aterm_modulo( + &self, + input_drv_hashes: &BTreeMap, + ) -> String { + let mut output_entries = Vec::new(); + for (name, info) in &self.outputs { + output_entries.push(format!( + "({},{},{},{})", + quote_string(name), + quote_string(&info.path), + quote_string(&info.hash_algo), + quote_string(&info.hash), + )); + } + let outputs = output_entries.join(","); + + let mut input_drv_entries = Vec::new(); + for (drv_hash, outputs_csv) in input_drv_hashes { + let mut sorted_outs: Vec<&str> = outputs_csv.split(',').collect(); + sorted_outs.sort(); + let out_list: Vec = sorted_outs.iter().map(|s| quote_string(s)).collect(); + let out_list = format!("[{}]", out_list.join(",")); + input_drv_entries.push(format!("({},{})", quote_string(drv_hash), out_list)); + } + let input_drvs = input_drv_entries.join(","); + + let input_srcs: Vec = self.input_srcs.iter().map(|s| quote_string(s)).collect(); + let input_srcs = input_srcs.join(","); + + let args: Vec = self.args.iter().map(|s| escape_string(s)).collect(); + let args = args.join(","); + + let mut env_entries: Vec = Vec::new(); + for (k, v) in &self.env { + env_entries.push(format!("({},{})", escape_string(k), escape_string(v))); + } + + format!( + "Derive([{}],[{}],[{}],{},{},[{}],[{}])", + outputs, + input_drvs, + input_srcs, + quote_string(&self.platform), + escape_string(&self.builder), + args, + env_entries.join(","), + ) + } + + pub fn collect_references(&self) -> Vec { + let mut refs = BTreeSet::new(); + for src in &self.input_srcs { + refs.insert(src.clone()); + } + for drv_path in self.input_drvs.keys() { + refs.insert(drv_path.clone()); + } + refs.into_iter().collect() + } +} diff --git a/nix-js/src/lib.rs b/nix-js/src/lib.rs index 3508ac1..d22368a 100644 --- a/nix-js/src/lib.rs +++ b/nix-js/src/lib.rs @@ -6,6 +6,7 @@ pub mod logging; pub mod value; mod codegen; +mod derivation; mod downgrade; mod fetcher; mod ir; @@ -13,6 +14,7 @@ mod nar; mod nix_utils; mod runtime; mod store; +mod string_context; #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; diff --git a/nix-js/src/runtime.rs b/nix-js/src/runtime.rs index 277a9b6..2fbd7f4 100644 --- a/nix-js/src/runtime.rs +++ b/nix-js/src/runtime.rs @@ -46,31 +46,38 @@ fn runtime_extension() -> Extension { let mut ops = vec![ op_import::(), op_scoped_import::(), + + op_resolve_path(), + op_read_file(), op_read_file_type(), op_read_dir(), op_path_exists(), - op_resolve_path(), - op_sha256_hex(), - op_make_placeholder(), - op_decode_span::(), - op_make_store_path::(), - op_parse_hash(), - op_make_fixed_output_path::(), - op_add_path::(), - op_store_path::(), - op_to_file::(), - op_write_derivation::(), - op_read_derivation_outputs(), - op_compute_fs_closure(), - op_copy_path_to_store::(), - op_get_env(), op_walk_dir(), + + op_make_placeholder(), + op_store_path::(), + + op_parse_hash(), + + op_add_path::(), op_add_filtered_path::(), + + op_decode_span::(), + + op_to_file::(), + + op_copy_path_to_store::(), + + op_get_env(), + op_match(), op_split(), + op_from_json(), op_from_toml(), + + op_finalize_derivation::(), ]; ops.extend(crate::fetcher::register_ops::()); @@ -86,7 +93,6 @@ fn runtime_extension() -> Extension { mod private { use deno_error::js_error_wrapper; - #[allow(dead_code)] #[derive(Debug)] pub struct SimpleErrorWrapper(pub(crate) String); impl std::fmt::Display for SimpleErrorWrapper { @@ -142,6 +148,7 @@ impl Runtime { }); js_runtime.op_state().borrow_mut().put(RegexCache::new()); + js_runtime.op_state().borrow_mut().put(DrvHashCache::new()); let ( is_thunk_symbol, diff --git a/nix-js/src/runtime/ops.rs b/nix-js/src/runtime/ops.rs index db8793f..abe3ef1 100644 --- a/nix-js/src/runtime/ops.rs +++ b/nix-js/src/runtime/ops.rs @@ -244,12 +244,6 @@ pub(super) fn op_resolve_path( Ok(normalized.to_string_lossy().to_string()) } -#[deno_core::op2] -#[string] -pub(super) fn op_sha256_hex(#[string] data: String) -> String { - crate::nix_utils::sha256_hex(data.as_bytes()) -} - #[deno_core::op2] #[string] pub(super) fn op_make_placeholder(#[string] output: String) -> String { @@ -312,20 +306,6 @@ fn byte_offset_to_line_col(content: &str, offset: usize) -> (u32, u32) { (line, col) } -#[deno_core::op2] -#[string] -pub(super) fn op_make_store_path( - state: &mut OpState, - #[string] ty: String, - #[string] hash_hex: String, - #[string] name: String, -) -> String { - let ctx: &Ctx = state.get_ctx(); - let store = ctx.get_store(); - let store_dir = store.get_store_dir(); - crate::nix_utils::make_store_path(store_dir, &ty, &hash_hex, &name) -} - #[derive(serde::Serialize)] pub(super) struct ParsedHash { hex: String, @@ -359,34 +339,6 @@ pub(super) fn op_parse_hash( }) } -#[deno_core::op2] -#[string] -pub(super) fn op_make_fixed_output_path( - state: &mut OpState, - #[string] hash_algo: String, - #[string] hash: String, - #[string] hash_mode: String, - #[string] name: String, -) -> String { - use sha2::{Digest, Sha256}; - - let ctx: &Ctx = state.get_ctx(); - let store = ctx.get_store(); - let store_dir = store.get_store_dir(); - - if hash_algo == "sha256" && hash_mode == "recursive" { - crate::nix_utils::make_store_path(store_dir, "source", &hash, &name) - } else { - let prefix = if hash_mode == "recursive" { "r:" } else { "" }; - let inner_input = format!("fixed:out:{}{}:{}:", prefix, hash_algo, hash); - let mut hasher = Sha256::new(); - hasher.update(inner_input.as_bytes()); - let inner_hash = hex::encode(hasher.finalize()); - - crate::nix_utils::make_store_path(store_dir, "output:out", &inner_hash, &name) - } -} - #[deno_core::op2] #[string] pub(super) fn op_add_path( @@ -504,269 +456,6 @@ pub(super) fn op_to_file( Ok(store_path) } -#[deno_core::op2] -#[string] -pub(super) fn op_write_derivation( - state: &mut OpState, - #[string] drv_name: String, - #[string] aterm: String, - #[serde] references: Vec, -) -> std::result::Result { - tracing::debug!( - "op_write_derivation: name={}.drv, references={:?}", - drv_name, - references - ); - let ctx: &Ctx = state.get_ctx(); - let store = ctx.get_store(); - let store_path = store - .add_text_to_store(&format!("{}.drv", drv_name), &aterm, references) - .map_err(|e| NixRuntimeError::from(format!("failed to write derivation: {}", e)))?; - - Ok(store_path) -} - -#[deno_core::op2] -#[serde] -pub(super) fn op_read_derivation_outputs( - #[string] drv_path: String, -) -> std::result::Result, NixRuntimeError> { - let content = std::fs::read_to_string(&drv_path).map_err(|e| { - NixRuntimeError::from(format!("failed to read derivation {}: {}", drv_path, e)) - })?; - - let outputs = parse_derivation_outputs(&content) - .ok_or_else(|| NixRuntimeError::from(format!("failed to parse derivation {}", drv_path)))?; - - Ok(outputs) -} - -fn parse_derivation_outputs(aterm: &str) -> Option> { - let aterm = aterm.strip_prefix("Derive([")?; - let outputs_end = aterm.find("],[")?; - let outputs_section = &aterm[..outputs_end]; - - let mut outputs = Vec::new(); - let mut pos = 0; - let bytes = outputs_section.as_bytes(); - - while pos < bytes.len() { - while pos < bytes.len() && bytes[pos] != b'(' { - pos += 1; - } - if pos >= bytes.len() { - break; - } - pos += 1; - - if pos >= bytes.len() || bytes[pos] != b'"' { - break; - } - pos += 1; - - let name_start = pos; - while pos < bytes.len() && bytes[pos] != b'"' { - pos += 1; - } - let name = std::str::from_utf8(&bytes[name_start..pos]).ok()?; - outputs.push(name.to_string()); - - while pos < bytes.len() && bytes[pos] != b')' { - pos += 1; - } - pos += 1; - } - - Some(outputs) -} - -#[derive(serde::Serialize)] -pub(super) struct DerivationInputs { - input_drvs: Vec<(String, Vec)>, - input_srcs: Vec, -} - -fn parse_derivation_inputs(aterm: &str) -> Option { - let aterm = aterm.strip_prefix("Derive([")?; - - let mut bracket_count = 1; - let mut pos = 0; - let bytes = aterm.as_bytes(); - while pos < bytes.len() && bracket_count > 0 { - match bytes[pos] { - b'[' => bracket_count += 1, - b']' => bracket_count -= 1, - _ => {} - } - pos += 1; - } - if bracket_count != 0 { - return None; - } - - let rest = &aterm[pos..]; - let rest = rest.strip_prefix(",[")?; - - let mut input_drvs = Vec::new(); - let mut bracket_count = 1; - let mut start = 0; - pos = 0; - let bytes = rest.as_bytes(); - - while pos < bytes.len() && bracket_count > 0 { - match bytes[pos] { - b'[' => bracket_count += 1, - b']' => bracket_count -= 1, - b'(' if bracket_count == 1 => { - start = pos; - } - b')' if bracket_count == 1 => { - let entry = &rest[start + 1..pos]; - if let Some((drv_path, outputs)) = parse_input_drv_entry(entry) { - input_drvs.push((drv_path, outputs)); - } - } - _ => {} - } - pos += 1; - } - - let rest = &rest[pos..]; - let rest = rest.strip_prefix(",[")?; - - let mut input_srcs = Vec::new(); - bracket_count = 1; - pos = 0; - let bytes = rest.as_bytes(); - - while pos < bytes.len() && bracket_count > 0 { - match bytes[pos] { - b'[' => bracket_count += 1, - b']' => bracket_count -= 1, - b'"' if bracket_count == 1 => { - pos += 1; - let src_start = pos; - while pos < bytes.len() && bytes[pos] != b'"' { - if bytes[pos] == b'\\' && pos + 1 < bytes.len() { - pos += 2; - } else { - pos += 1; - } - } - let src = std::str::from_utf8(&bytes[src_start..pos]).ok()?; - input_srcs.push(src.to_string()); - } - _ => {} - } - pos += 1; - } - - Some(DerivationInputs { - input_drvs, - input_srcs, - }) -} - -fn parse_input_drv_entry(entry: &str) -> Option<(String, Vec)> { - let entry = entry.strip_prefix('"')?; - let quote_end = entry.find('"')?; - let drv_path = entry[..quote_end].to_string(); - - let rest = &entry[quote_end + 1..]; - let rest = rest.strip_prefix(",[")?; - let rest = rest.strip_suffix(']')?; - - let mut outputs = Vec::new(); - for part in rest.split(',') { - let part = part.trim(); - if let Some(name) = part.strip_prefix('"').and_then(|s| s.strip_suffix('"')) { - outputs.push(name.to_string()); - } - } - - Some((drv_path, outputs)) -} - -#[derive(serde::Serialize)] -pub(super) struct FsClosureResult { - input_drvs: Vec<(String, Vec)>, - input_srcs: Vec, -} - -#[deno_core::op2] -#[serde] -pub(super) fn op_compute_fs_closure( - #[string] drv_path: String, -) -> std::result::Result { - use std::collections::{BTreeMap, BTreeSet, VecDeque}; - - let mut all_input_srcs: BTreeSet = BTreeSet::new(); - let mut all_input_drvs: BTreeMap> = BTreeMap::new(); - - let mut queue: VecDeque = VecDeque::new(); - let mut visited: BTreeSet = BTreeSet::new(); - - queue.push_back(drv_path); - - while let Some(current_path) = queue.pop_front() { - if visited.contains(¤t_path) { - continue; - } - visited.insert(current_path.clone()); - - all_input_srcs.insert(current_path.clone()); - - if !current_path.ends_with(".drv") { - continue; - } - - let content = match std::fs::read_to_string(¤t_path) { - Ok(c) => c, - Err(e) => { - return Err(NixRuntimeError::from(format!( - "failed to read derivation {}: {}", - current_path, e - ))); - } - }; - - let inputs = parse_derivation_inputs(&content).ok_or_else(|| { - NixRuntimeError::from(format!("failed to parse derivation {}", current_path)) - })?; - - for src in inputs.input_srcs { - all_input_srcs.insert(src.clone()); - if !visited.contains(&src) { - queue.push_back(src); - } - } - - for (dep_drv, outputs) in inputs.input_drvs { - all_input_srcs.insert(dep_drv.clone()); - - let entry = all_input_drvs.entry(dep_drv.clone()).or_default(); - for output in outputs { - entry.insert(output); - } - - if !visited.contains(&dep_drv) { - queue.push_back(dep_drv); - } - } - } - - let input_drvs: Vec<(String, Vec)> = all_input_drvs - .into_iter() - .map(|(k, v)| (k, v.into_iter().collect())) - .collect(); - let input_srcs: Vec = all_input_srcs.into_iter().collect(); - - Ok(FsClosureResult { - input_drvs, - input_srcs, - }) -} - #[deno_core::op2] #[string] pub(super) fn op_copy_path_to_store( @@ -1113,6 +802,19 @@ fn json_to_nix(value: serde_json::Value) -> NixJsonValue { } } +#[derive(Debug, Default)] +pub(super) struct DrvHashCache { + cache: HashMap, +} + +impl DrvHashCache { + pub(super) fn new() -> Self { + Self { + cache: HashMap::new(), + } + } +} + fn toml_to_nix(value: toml::Value) -> std::result::Result { match value { toml::Value::String(s) => Ok(NixJsonValue::Str(s)), @@ -1153,3 +855,254 @@ pub(super) fn op_from_toml( .map_err(|e| NixRuntimeError::from(format!("while parsing TOML: {e}")))?; toml_to_nix(parsed) } + +#[derive(serde::Deserialize)] +pub(super) struct FixedOutputInput { + #[serde(rename = "hashAlgo")] + hash_algo: String, + hash: String, + #[serde(rename = "hashMode")] + hash_mode: String, +} + +#[derive(serde::Deserialize)] +pub(super) struct FinalizeDerivationInput { + name: String, + builder: String, + platform: String, + outputs: Vec, + args: Vec, + env: Vec<(String, String)>, + context: Vec, + #[serde(rename = "fixedOutput")] + fixed_output: Option, +} + +#[derive(serde::Serialize)] +pub(super) struct FinalizeDerivationOutput { + #[serde(rename = "drvPath")] + drv_path: String, + outputs: Vec<(String, String)>, +} + +fn output_path_name(drv_name: &str, output: &str) -> String { + if output == "out" { + drv_name.to_string() + } else { + format!("{}-{}", drv_name, output) + } +} + +#[deno_core::op2] +#[serde] +pub(super) fn op_finalize_derivation( + state: &mut OpState, + #[serde] input: FinalizeDerivationInput, +) -> std::result::Result { + use crate::derivation::{DerivationData, OutputInfo}; + use crate::string_context::extract_input_drvs_and_srcs; + + let ctx: &Ctx = state.get_ctx(); + let store = ctx.get_store(); + let store_dir = store.get_store_dir().to_string(); + + let (input_drvs, input_srcs) = + extract_input_drvs_and_srcs(&input.context).map_err(NixRuntimeError::from)?; + + let env: std::collections::BTreeMap = + input.env.into_iter().collect(); + + let drv_path; + let output_paths: Vec<(String, String)>; + + if let Some(fixed) = &input.fixed_output { + let path_name = output_path_name(&input.name, "out"); + let out_path = crate::runtime::ops::op_make_fixed_output_path_impl( + &store_dir, + &fixed.hash_algo, + &fixed.hash, + &fixed.hash_mode, + &path_name, + ); + + let hash_algo_prefix = if fixed.hash_mode == "recursive" { + "r:" + } else { + "" + }; + + let mut final_outputs = std::collections::BTreeMap::new(); + final_outputs.insert( + "out".to_string(), + OutputInfo { + path: out_path.clone(), + hash_algo: format!("{}{}", hash_algo_prefix, fixed.hash_algo), + hash: fixed.hash.clone(), + }, + ); + + let mut final_env = env; + final_env.insert("out".to_string(), out_path.clone()); + + let drv = DerivationData { + name: input.name.clone(), + outputs: final_outputs, + input_drvs: input_drvs.clone(), + input_srcs: input_srcs.clone(), + platform: input.platform, + builder: input.builder, + args: input.args, + env: final_env, + }; + + let final_aterm = drv.generate_aterm(); + let references = drv.collect_references(); + + drv_path = store + .add_text_to_store(&format!("{}.drv", input.name), &final_aterm, references) + .map_err(|e| NixRuntimeError::from(format!("failed to write derivation: {}", e)))?; + + let fixed_hash_fingerprint = format!( + "fixed:out:{}{}:{}:{}", + hash_algo_prefix, fixed.hash_algo, fixed.hash, out_path, + ); + let fixed_modulo_hash = crate::nix_utils::sha256_hex(fixed_hash_fingerprint.as_bytes()); + + let cache = state.borrow_mut::(); + cache.cache.insert(drv_path.clone(), fixed_modulo_hash); + + output_paths = vec![("out".to_string(), out_path)]; + } else { + let masked_outputs: std::collections::BTreeMap = input + .outputs + .iter() + .map(|o| { + ( + o.clone(), + OutputInfo { + path: String::new(), + hash_algo: String::new(), + hash: String::new(), + }, + ) + }) + .collect(); + + let mut masked_env = env.clone(); + for output in &input.outputs { + masked_env.insert(output.clone(), String::new()); + } + + let masked_drv = DerivationData { + name: input.name.clone(), + outputs: masked_outputs, + input_drvs: input_drvs.clone(), + input_srcs: input_srcs.clone(), + platform: input.platform.clone(), + builder: input.builder.clone(), + args: input.args.clone(), + env: masked_env, + }; + + let mut input_drv_hashes = std::collections::BTreeMap::new(); + { + let cache = state.borrow::(); + for (dep_drv_path, output_names) in &input_drvs { + let cached_hash = + cache.cache.get(dep_drv_path).ok_or_else(|| { + NixRuntimeError::from(format!( + "Missing modulo hash for input derivation: {}", + dep_drv_path + )) + })?; + let mut sorted_outs: Vec<&String> = output_names.iter().collect(); + sorted_outs.sort(); + let outputs_csv: Vec<&str> = + sorted_outs.iter().map(|s| s.as_str()).collect(); + input_drv_hashes + .insert(cached_hash.clone(), outputs_csv.join(",")); + } + } + + let masked_aterm = masked_drv.generate_aterm_modulo(&input_drv_hashes); + let drv_modulo_hash = crate::nix_utils::sha256_hex(masked_aterm.as_bytes()); + + let mut final_outputs = std::collections::BTreeMap::new(); + let mut final_env = env; + let mut result_output_paths = Vec::new(); + + for output_name in &input.outputs { + let path_name = output_path_name(&input.name, output_name); + let out_path = crate::nix_utils::make_store_path( + &store_dir, + &format!("output:{}", output_name), + &drv_modulo_hash, + &path_name, + ); + final_outputs.insert( + output_name.clone(), + OutputInfo { + path: out_path.clone(), + hash_algo: String::new(), + hash: String::new(), + }, + ); + final_env.insert(output_name.clone(), out_path.clone()); + result_output_paths.push((output_name.clone(), out_path)); + } + + let final_drv = DerivationData { + name: input.name, + outputs: final_outputs, + input_drvs, + input_srcs, + platform: input.platform, + builder: input.builder, + args: input.args, + env: final_env, + }; + + let final_aterm = final_drv.generate_aterm(); + let references = final_drv.collect_references(); + + drv_path = store + .add_text_to_store(&format!("{}.drv", final_drv.name), &final_aterm, references) + .map_err(|e| NixRuntimeError::from(format!("failed to write derivation: {}", e)))?; + + let final_aterm_modulo = final_drv.generate_aterm_modulo(&input_drv_hashes); + let cached_modulo_hash = + crate::nix_utils::sha256_hex(final_aterm_modulo.as_bytes()); + + let cache = state.borrow_mut::(); + cache.cache.insert(drv_path.clone(), cached_modulo_hash); + + output_paths = result_output_paths; + } + + Ok(FinalizeDerivationOutput { + drv_path, + outputs: output_paths, + }) +} + +fn op_make_fixed_output_path_impl( + store_dir: &str, + hash_algo: &str, + hash: &str, + hash_mode: &str, + name: &str, +) -> String { + use sha2::{Digest, Sha256}; + + if hash_algo == "sha256" && hash_mode == "recursive" { + crate::nix_utils::make_store_path(store_dir, "source", hash, name) + } else { + let prefix = if hash_mode == "recursive" { "r:" } else { "" }; + let inner_input = format!("fixed:out:{}{}:{}:", prefix, hash_algo, hash); + let mut hasher = Sha256::new(); + hasher.update(inner_input.as_bytes()); + let inner_hash = hex::encode(hasher.finalize()); + + crate::nix_utils::make_store_path(store_dir, "output:out", &inner_hash, name) + } +} diff --git a/nix-js/src/store.rs b/nix-js/src/store.rs index f2ed51c..4fbcc00 100644 --- a/nix-js/src/store.rs +++ b/nix-js/src/store.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - use crate::error::Result; mod config; @@ -39,12 +37,4 @@ pub trait Store: Send + Sync { content: &str, references: Vec, ) -> Result; - - fn make_fixed_output_path( - &self, - hash_algo: &str, - hash: &str, - hash_mode: &str, - name: &str, - ) -> Result; } diff --git a/nix-js/src/store/daemon.rs b/nix-js/src/store/daemon.rs index d00c57f..3a58b8f 100644 --- a/nix-js/src/store/daemon.rs +++ b/nix-js/src/store/daemon.rs @@ -304,51 +304,6 @@ impl Store for DaemonStore { Ok(store_path_str) } - - fn make_fixed_output_path( - &self, - hash_algo: &str, - hash: &str, - hash_mode: &str, - name: &str, - ) -> Result { - use nix_compat::nixhash::{CAHash, NixHash}; - use nix_compat::store_path::build_ca_path; - - let nix_hash = match hash_algo { - "sha256" => { - let hash_bytes = hex::decode(hash) - .map_err(|e| Error::internal(format!("Invalid hash hex: {}", e)))?; - if hash_bytes.len() != 32 { - return Err(Error::internal(format!( - "Invalid sha256 hash length: expected 32, got {}", - hash_bytes.len() - ))); - } - let mut arr = [0u8; 32]; - arr.copy_from_slice(&hash_bytes); - NixHash::Sha256(arr) - } - _ => { - return Err(Error::internal(format!( - "Unsupported hash algorithm: {}", - hash_algo - ))); - } - }; - - let ca_hash = if hash_mode == "r" { - CAHash::Nar(nix_hash) - } else { - CAHash::Flat(nix_hash) - }; - - let store_path: nix_compat::store_path::StorePath = - build_ca_path(name, &ca_hash, Vec::::new(), false) - .map_err(|e| Error::internal(format!("Failed to build store path: {}", e)))?; - - Ok(store_path.to_absolute_path()) - } } const PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::from_parts(1, 37); diff --git a/nix-js/src/store/error.rs b/nix-js/src/store/error.rs index 33f1131..4142457 100644 --- a/nix-js/src/store/error.rs +++ b/nix-js/src/store/error.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] + use std::fmt; #[derive(Debug)] diff --git a/nix-js/src/string_context.rs b/nix-js/src/string_context.rs new file mode 100644 index 0000000..5808c6a --- /dev/null +++ b/nix-js/src/string_context.rs @@ -0,0 +1,211 @@ +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +pub enum StringContextElem { + Opaque { path: String }, + DrvDeep { drv_path: String }, + Built { drv_path: String, output: String }, +} + +impl StringContextElem { + pub fn decode(encoded: &str) -> Self { + if let Some(drv_path) = encoded.strip_prefix('=') { + StringContextElem::DrvDeep { + drv_path: drv_path.to_string(), + } + } else if let Some(rest) = encoded.strip_prefix('!') { + if let Some(second_bang) = rest.find('!') { + let output = rest[..second_bang].to_string(); + let drv_path = rest[second_bang + 1..].to_string(); + StringContextElem::Built { drv_path, output } + } else { + StringContextElem::Opaque { + path: encoded.to_string(), + } + } + } else { + StringContextElem::Opaque { + path: encoded.to_string(), + } + } + } +} + +pub type InputDrvs = BTreeMap>; +pub type Srcs = BTreeSet; +pub fn extract_input_drvs_and_srcs( + context: &[String], +) -> Result<(InputDrvs, Srcs), String> { + let mut input_drvs: BTreeMap> = BTreeMap::new(); + let mut input_srcs: BTreeSet = BTreeSet::new(); + + for encoded in context { + match StringContextElem::decode(encoded) { + StringContextElem::Opaque { path } => { + input_srcs.insert(path); + } + StringContextElem::DrvDeep { drv_path } => { + compute_fs_closure(&drv_path, &mut input_drvs, &mut input_srcs)?; + } + StringContextElem::Built { drv_path, output } => { + input_drvs.entry(drv_path).or_default().insert(output); + } + } + } + + Ok((input_drvs, input_srcs)) +} + +fn compute_fs_closure( + drv_path: &str, + input_drvs: &mut BTreeMap>, + input_srcs: &mut BTreeSet, +) -> Result<(), String> { + let mut queue: VecDeque = VecDeque::new(); + let mut visited: BTreeSet = BTreeSet::new(); + + queue.push_back(drv_path.to_string()); + + while let Some(current_path) = queue.pop_front() { + if visited.contains(¤t_path) { + continue; + } + visited.insert(current_path.clone()); + input_srcs.insert(current_path.clone()); + + if !current_path.ends_with(".drv") { + continue; + } + + let content = std::fs::read_to_string(¤t_path) + .map_err(|e| format!("failed to read derivation {}: {}", current_path, e))?; + + let inputs = parse_derivation_inputs(&content) + .ok_or_else(|| format!("failed to parse derivation {}", current_path))?; + + for src in inputs.input_srcs { + input_srcs.insert(src.clone()); + if !visited.contains(&src) { + queue.push_back(src); + } + } + + for (dep_drv, outputs) in inputs.input_drvs { + input_srcs.insert(dep_drv.clone()); + let entry = input_drvs.entry(dep_drv.clone()).or_default(); + for output in outputs { + entry.insert(output); + } + if !visited.contains(&dep_drv) { + queue.push_back(dep_drv); + } + } + } + + Ok(()) +} + +struct DerivationInputs { + input_drvs: Vec<(String, Vec)>, + input_srcs: Vec, +} + +fn parse_derivation_inputs(aterm: &str) -> Option { + let aterm = aterm.strip_prefix("Derive([")?; + + let mut bracket_count: i32 = 1; + let mut pos = 0; + let bytes = aterm.as_bytes(); + while pos < bytes.len() && bracket_count > 0 { + match bytes[pos] { + b'[' => bracket_count += 1, + b']' => bracket_count -= 1, + _ => {} + } + pos += 1; + } + if bracket_count != 0 { + return None; + } + + let rest = &aterm[pos..]; + let rest = rest.strip_prefix(",[")?; + + let mut input_drvs = Vec::new(); + let mut bracket_count: i32 = 1; + let mut start = 0; + pos = 0; + let bytes = rest.as_bytes(); + + while pos < bytes.len() && bracket_count > 0 { + match bytes[pos] { + b'[' => bracket_count += 1, + b']' => bracket_count -= 1, + b'(' if bracket_count == 1 => { + start = pos; + } + b')' if bracket_count == 1 => { + let entry = &rest[start + 1..pos]; + if let Some((drv_path, outputs)) = parse_input_drv_entry(entry) { + input_drvs.push((drv_path, outputs)); + } + } + _ => {} + } + pos += 1; + } + + let rest = &rest[pos..]; + let rest = rest.strip_prefix(",[")?; + + let mut input_srcs = Vec::new(); + bracket_count = 1; + pos = 0; + let bytes = rest.as_bytes(); + + while pos < bytes.len() && bracket_count > 0 { + match bytes[pos] { + b'[' => bracket_count += 1, + b']' => bracket_count -= 1, + b'"' if bracket_count == 1 => { + pos += 1; + let src_start = pos; + while pos < bytes.len() && bytes[pos] != b'"' { + if bytes[pos] == b'\\' && pos + 1 < bytes.len() { + pos += 2; + } else { + pos += 1; + } + } + let src = std::str::from_utf8(&bytes[src_start..pos]).ok()?; + input_srcs.push(src.to_string()); + } + _ => {} + } + pos += 1; + } + + Some(DerivationInputs { + input_drvs, + input_srcs, + }) +} + +fn parse_input_drv_entry(entry: &str) -> Option<(String, Vec)> { + let entry = entry.strip_prefix('"')?; + let quote_end = entry.find('"')?; + let drv_path = entry[..quote_end].to_string(); + + let rest = &entry[quote_end + 1..]; + let rest = rest.strip_prefix(",[")?; + let rest = rest.strip_suffix(']')?; + + let mut outputs = Vec::new(); + for part in rest.split(',') { + let part = part.trim(); + if let Some(name) = part.strip_prefix('"').and_then(|s| s.strip_suffix('"')) { + outputs.push(name.to_string()); + } + } + + Some((drv_path, outputs)) +}