clean up

2026-03-12 17:47:46 +08:00
parent 7a7229d70e
commit 0c9a391618
511 changed files with 234 additions and 12772 deletions
@@ -0,0 +1,305 @@
+use deno_core::OpState;
+use deno_core::ToV8;
+use deno_core::op2;
+use nix_compat::nixhash::HashAlgo;
+use nix_compat::nixhash::NixHash;
+use tracing::{debug, info, warn};
+
+use crate::store::Store as _;
+
+mod archive;
+pub(crate) mod cache;
+mod download;
+mod git;
+mod metadata_cache;
+
+pub use cache::FetcherCache;
+pub use download::Downloader;
+pub use metadata_cache::MetadataCache;
+
+use crate::nar;
+
+#[derive(ToV8)]
+pub struct FetchUrlResult {
+    pub store_path: String,
+    pub hash: String,
+}
+
+#[derive(ToV8)]
+pub struct FetchTarballResult {
+    pub store_path: String,
+    pub nar_hash: String,
+}
+
+#[derive(ToV8)]
+pub struct FetchGitResult {
+    pub out_path: String,
+    pub rev: String,
+    pub short_rev: String,
+    pub rev_count: u64,
+    pub last_modified: u64,
+    pub last_modified_date: String,
+    pub submodules: bool,
+    pub nar_hash: Option<String>,
+}
+
+#[op2]
+pub fn op_fetch_url<Ctx: RuntimeContext>(
+    state: &mut OpState,
+    #[string] url: String,
+    #[string] expected_hash: Option<String>,
+    #[string] name: Option<String>,
+    executable: bool,
+) -> Result<FetchUrlResult, NixRuntimeError> {
+    let _span = tracing::info_span!("op_fetch_url", url = %url).entered();
+    info!("fetchurl started");
+
+    let file_name =
+        name.unwrap_or_else(|| url.rsplit('/').next().unwrap_or("download").to_string());
+
+    let metadata_cache =
+        MetadataCache::new(3600).map_err(|e| NixRuntimeError::from(e.to_string()))?;
+
+    let input = serde_json::json!({
+        "type": "file",
+        "url": url,
+        "name": file_name,
+        "executable": executable,
+    });
+
+    if let Some(cached_entry) = metadata_cache
+        .lookup(&input)
+        .map_err(|e| NixRuntimeError::from(e.to_string()))?
+    {
+        let cached_hash = cached_entry
+            .info
+            .get("hash")
+            .and_then(|v| v.as_str())
+            .unwrap_or("");
+
+        if let Some(ref expected) = expected_hash {
+            let normalized_expected = normalize_hash(expected);
+            if cached_hash != normalized_expected {
+                warn!("Cached hash mismatch, re-fetching");
+            } else {
+                info!("Cache hit");
+                return Ok(FetchUrlResult {
+                    store_path: cached_entry.store_path.clone(),
+                    hash: cached_hash.to_string(),
+                });
+            }
+        } else {
+            info!("Cache hit (no hash check)");
+            return Ok(FetchUrlResult {
+                store_path: cached_entry.store_path.clone(),
+                hash: cached_hash.to_string(),
+            });
+        }
+    }
+
+    info!("Cache miss, downloading");
+    let downloader = Downloader::new();
+    let data = downloader
+        .download(&url)
+        .map_err(|e| NixRuntimeError::from(e.to_string()))?;
+
+    info!(bytes = data.len(), "Download complete");
+
+    let hash = crate::nix_utils::sha256_hex(&data);
+
+    if let Some(ref expected) = expected_hash {
+        let normalized_expected = normalize_hash(expected);
+        if hash != normalized_expected {
+            return Err(NixRuntimeError::from(format!(
+                "hash mismatch for '{}': expected {}, got {}",
+                url, normalized_expected, hash
+            )));
+        }
+    }
+
+    let ctx: &Ctx = state.get_ctx();
+    let store = ctx.get_store();
+    let store_path = store
+        .add_to_store(&file_name, &data, false, vec![])
+        .map_err(|e| NixRuntimeError::from(e.to_string()))?;
+
+    info!(store_path = %store_path, "Added to store");
+
+    #[cfg(unix)]
+    if executable {
+        use std::os::unix::fs::PermissionsExt;
+        if let Ok(metadata) = std::fs::metadata(&store_path) {
+            let mut perms = metadata.permissions();
+            perms.set_mode(0o755);
+            let _ = std::fs::set_permissions(&store_path, perms);
+        }
+    }
+
+    let info = serde_json::json!({
+        "hash": hash,
+        "url": url,
+    });
+
+    metadata_cache
+        .add(&input, &info, &store_path, true)
+        .map_err(|e| NixRuntimeError::from(e.to_string()))?;
+
+    Ok(FetchUrlResult { store_path, hash })
+}
+
+#[op2]
+pub fn op_fetch_tarball<Ctx: RuntimeContext>(
+    state: &mut OpState,
+    #[string] url: String,
+    #[string] name: Option<String>,
+    #[string] sha256: Option<String>,
+) -> Result<FetchTarballResult, NixRuntimeError> {
+    let _span = tracing::info_span!("op_fetch_tarball", url = %url).entered();
+    info!("fetchTarball started");
+
+    let dir_name = name.unwrap_or_else(|| "source".to_string());
+    let metadata_cache =
+        MetadataCache::new(3600).map_err(|e| NixRuntimeError::from(e.to_string()))?;
+
+    let input = serde_json::json!({
+        "type": "tarball",
+        "url": url,
+        "name": dir_name,
+    });
+
+    let expected_sha256 = sha256
+        .map(
+            |ref sha256| match NixHash::from_str(sha256, Some(HashAlgo::Sha256)) {
+                Ok(NixHash::Sha256(digest)) => Ok(digest),
+                _ => Err(format!("fetchTarball: invalid sha256 '{sha256}'")),
+            },
+        )
+        .transpose()?;
+    let expected_hex = expected_sha256.map(hex::encode);
+
+    if let Some(cached_entry) = metadata_cache
+        .lookup(&input)
+        .map_err(|e| NixRuntimeError::from(e.to_string()))?
+    {
+        let cached_nar_hash = cached_entry
+            .info
+            .get("nar_hash")
+            .and_then(|v| v.as_str())
+            .unwrap_or("");
+
+        if let Some(ref hex) = expected_hex {
+            if cached_nar_hash == hex {
+                info!("Cache hit");
+                return Ok(FetchTarballResult {
+                    store_path: cached_entry.store_path.clone(),
+                    nar_hash: cached_nar_hash.to_string(),
+                });
+            }
+        } else if !cached_entry.is_expired(3600) {
+            info!("Cache hit (no hash check)");
+            return Ok(FetchTarballResult {
+                store_path: cached_entry.store_path.clone(),
+                nar_hash: cached_nar_hash.to_string(),
+            });
+        }
+    }
+
+    info!("Cache miss, downloading tarball");
+    let downloader = Downloader::new();
+    let data = downloader
+        .download(&url)
+        .map_err(|e| NixRuntimeError::from(e.to_string()))?;
+
+    info!(bytes = data.len(), "Download complete");
+
+    info!("Extracting tarball");
+    let (extracted_path, _temp_dir) = archive::extract_tarball_to_temp(&data)
+        .map_err(|e| NixRuntimeError::from(e.to_string()))?;
+
+    info!("Computing NAR hash");
+    let nar_hash =
+        nar::compute_nar_hash(&extracted_path).map_err(|e| NixRuntimeError::from(e.to_string()))?;
+    let nar_hash_hex = hex::encode(nar_hash);
+
+    debug!(
+        nar_hash = %nar_hash_hex,
+        "Hash computation complete"
+    );
+
+    if let Some(ref expected) = expected_hex
+        && nar_hash_hex != *expected
+    {
+        return Err(NixRuntimeError::from(format!(
+            "Tarball hash mismatch for '{}': expected {}, got {}",
+            url, expected, nar_hash_hex
+        )));
+    }
+
+    info!("Adding to store");
+    let ctx: &Ctx = state.get_ctx();
+    let store = ctx.get_store();
+    let store_path = store
+        .add_to_store_from_path(&dir_name, &extracted_path, vec![])
+        .map_err(|e| NixRuntimeError::from(e.to_string()))?;
+
+    info!(store_path = %store_path, "Added to store");
+
+    let info = serde_json::json!({
+        "nar_hash": nar_hash_hex,
+        "url": url,
+    });
+
+    let immutable = expected_sha256.is_some();
+    metadata_cache
+        .add(&input, &info, &store_path, immutable)
+        .map_err(|e| NixRuntimeError::from(e.to_string()))?;
+
+    Ok(FetchTarballResult {
+        store_path,
+        nar_hash: nar_hash_hex,
+    })
+}
+
+#[op2]
+pub fn op_fetch_git<Ctx: RuntimeContext>(
+    state: &mut OpState,
+    #[string] url: String,
+    #[string] git_ref: Option<String>,
+    #[string] rev: Option<String>,
+    shallow: bool,
+    submodules: bool,
+    all_refs: bool,
+    #[string] name: Option<String>,
+) -> Result<FetchGitResult, NixRuntimeError> {
+    let _span = tracing::info_span!("op_fetch_git", url = %url).entered();
+    info!("fetchGit started");
+    let cache = FetcherCache::new().map_err(|e| NixRuntimeError::from(e.to_string()))?;
+    let dir_name = name.unwrap_or_else(|| "source".to_string());
+
+    let ctx: &Ctx = state.get_ctx();
+    let store = ctx.get_store();
+
+    git::fetch_git(
+        &cache,
+        store,
+        &url,
+        git_ref.as_deref(),
+        rev.as_deref(),
+        shallow,
+        submodules,
+        all_refs,
+        &dir_name,
+    )
+    .map_err(|e| NixRuntimeError::from(e.to_string()))
+}
+
+fn normalize_hash(hash: &str) -> String {
+    use base64::prelude::*;
+    if hash.starts_with("sha256-")
+        && let Some(b64) = hash.strip_prefix("sha256-")
+        && let Ok(bytes) = BASE64_STANDARD.decode(b64)
+    {
+        return hex::encode(bytes);
+    }
+    hash.to_string()
+}