This commit is contained in:
2026-03-12 17:47:46 +08:00
parent 7a7229d70e
commit 0c9a391618
511 changed files with 234 additions and 12772 deletions

173
fix/src/fetcher/archive.rs Normal file
View File

@@ -0,0 +1,173 @@
use std::fs;
use std::io::Cursor;
use std::os::unix::ffi::OsStrExt;
use std::path::{Path, PathBuf};
use flate2::read::GzDecoder;
#[derive(Debug, Clone, Copy)]
pub enum ArchiveFormat {
TarGz,
TarXz,
TarBz2,
Tar,
}
impl ArchiveFormat {
pub fn detect(url: &str, data: &[u8]) -> Self {
if url.ends_with(".tar.gz") || url.ends_with(".tgz") {
return ArchiveFormat::TarGz;
}
if url.ends_with(".tar.xz") || url.ends_with(".txz") {
return ArchiveFormat::TarXz;
}
if url.ends_with(".tar.bz2") || url.ends_with(".tbz2") {
return ArchiveFormat::TarBz2;
}
if url.ends_with(".tar") {
return ArchiveFormat::Tar;
}
if data.len() >= 2 && data[0] == 0x1f && data[1] == 0x8b {
return ArchiveFormat::TarGz;
}
if data.len() >= 6 && &data[0..6] == b"\xfd7zXZ\x00" {
return ArchiveFormat::TarXz;
}
if data.len() >= 3 && &data[0..3] == b"BZh" {
return ArchiveFormat::TarBz2;
}
ArchiveFormat::TarGz
}
}
pub fn extract_tarball(data: &[u8], dest: &Path) -> Result<PathBuf, ArchiveError> {
let format = ArchiveFormat::detect("", data);
let temp_dir = dest.join("_extract_temp");
fs::create_dir_all(&temp_dir)?;
match format {
ArchiveFormat::TarGz => extract_tar_gz(data, &temp_dir)?,
ArchiveFormat::TarXz => extract_tar_xz(data, &temp_dir)?,
ArchiveFormat::TarBz2 => extract_tar_bz2(data, &temp_dir)?,
ArchiveFormat::Tar => extract_tar(data, &temp_dir)?,
}
strip_single_toplevel(&temp_dir, dest)
}
fn extract_tar_gz(data: &[u8], dest: &Path) -> Result<(), ArchiveError> {
let decoder = GzDecoder::new(Cursor::new(data));
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
Ok(())
}
fn extract_tar_xz(data: &[u8], dest: &Path) -> Result<(), ArchiveError> {
let decoder = xz2::read::XzDecoder::new(Cursor::new(data));
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
Ok(())
}
fn extract_tar_bz2(data: &[u8], dest: &Path) -> Result<(), ArchiveError> {
let decoder = bzip2::read::BzDecoder::new(Cursor::new(data));
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
Ok(())
}
fn extract_tar(data: &[u8], dest: &Path) -> Result<(), ArchiveError> {
let mut archive = tar::Archive::new(Cursor::new(data));
archive.unpack(dest)?;
Ok(())
}
fn strip_single_toplevel(temp_dir: &Path, dest: &Path) -> Result<PathBuf, ArchiveError> {
let entries: Vec<_> = fs::read_dir(temp_dir)?
.filter_map(|e| e.ok())
.filter(|e| e.file_name().as_os_str().as_bytes()[0] != b'.')
.collect();
let source_dir = if entries.len() == 1 && entries[0].file_type()?.is_dir() {
entries[0].path()
} else {
temp_dir.to_path_buf()
};
let final_dest = dest.join("content");
if final_dest.exists() {
fs::remove_dir_all(&final_dest)?;
}
if source_dir == *temp_dir {
fs::rename(temp_dir, &final_dest)?;
} else {
copy_dir_recursive(&source_dir, &final_dest)?;
fs::remove_dir_all(temp_dir)?;
}
Ok(final_dest)
}
fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<(), std::io::Error> {
fs::create_dir_all(dst)?;
for entry in fs::read_dir(src)? {
let entry = entry?;
let path = entry.path();
let dest_path = dst.join(entry.file_name());
let metadata = fs::symlink_metadata(&path)?;
if metadata.is_symlink() {
let target = fs::read_link(&path)?;
#[cfg(unix)]
{
std::os::unix::fs::symlink(&target, &dest_path)?;
}
#[cfg(windows)]
{
if target.is_dir() {
std::os::windows::fs::symlink_dir(&target, &dest_path)?;
} else {
std::os::windows::fs::symlink_file(&target, &dest_path)?;
}
}
} else if metadata.is_dir() {
copy_dir_recursive(&path, &dest_path)?;
} else {
fs::copy(&path, &dest_path)?;
}
}
Ok(())
}
pub fn extract_tarball_to_temp(data: &[u8]) -> Result<(PathBuf, tempfile::TempDir), ArchiveError> {
let temp_dir = tempfile::tempdir()?;
let extracted_path = extract_tarball(data, temp_dir.path())?;
Ok((extracted_path, temp_dir))
}
#[derive(Debug)]
pub enum ArchiveError {
IoError(std::io::Error),
}
impl std::fmt::Display for ArchiveError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ArchiveError::IoError(e) => write!(f, "I/O error: {}", e),
}
}
}
impl std::error::Error for ArchiveError {}
impl From<std::io::Error> for ArchiveError {
fn from(e: std::io::Error) -> Self {
ArchiveError::IoError(e)
}
}

29
fix/src/fetcher/cache.rs Normal file
View File

@@ -0,0 +1,29 @@
use std::fs;
use std::path::PathBuf;
#[derive(Debug)]
pub struct FetcherCache {
base_dir: PathBuf,
}
impl FetcherCache {
pub fn new() -> Result<Self, std::io::Error> {
let base_dir = dirs::cache_dir()
.unwrap_or_else(|| PathBuf::from("/tmp"))
.join("fix")
.join("fetchers");
fs::create_dir_all(&base_dir)?;
Ok(Self { base_dir })
}
fn git_cache_dir(&self) -> PathBuf {
self.base_dir.join("git")
}
pub fn get_git_bare(&self, url: &str) -> PathBuf {
let key = crate::nix_utils::sha256_hex(url.as_bytes());
self.git_cache_dir().join(key)
}
}

View File

@@ -0,0 +1,64 @@
use std::time::Duration;
use reqwest::blocking::Client;
pub struct Downloader {
client: Client,
}
impl Downloader {
pub fn new() -> Self {
let client = Client::builder()
.timeout(Duration::from_secs(300))
.user_agent("nix-js/0.1")
.build()
.expect("Failed to create HTTP client");
Self { client }
}
pub fn download(&self, url: &str) -> Result<Vec<u8>, DownloadError> {
let response = self
.client
.get(url)
.send()
.map_err(|e| DownloadError::NetworkError(e.to_string()))?;
if !response.status().is_success() {
return Err(DownloadError::HttpError {
url: url.to_string(),
status: response.status().as_u16(),
});
}
response
.bytes()
.map(|b| b.to_vec())
.map_err(|e| DownloadError::NetworkError(e.to_string()))
}
}
impl Default for Downloader {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug)]
pub enum DownloadError {
NetworkError(String),
HttpError { url: String, status: u16 },
}
impl std::fmt::Display for DownloadError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DownloadError::NetworkError(msg) => write!(f, "Network error: {}", msg),
DownloadError::HttpError { url, status } => {
write!(f, "HTTP error {} for URL: {}", status, url)
}
}
}
}
impl std::error::Error for DownloadError {}

315
fix/src/fetcher/git.rs Normal file
View File

@@ -0,0 +1,315 @@
use std::fs;
use std::path::PathBuf;
use std::process::Command;
use super::FetchGitResult;
use super::cache::FetcherCache;
use crate::store::Store;
#[allow(clippy::too_many_arguments)]
pub fn fetch_git(
cache: &FetcherCache,
store: &dyn Store,
url: &str,
git_ref: Option<&str>,
rev: Option<&str>,
_shallow: bool,
submodules: bool,
all_refs: bool,
name: &str,
) -> Result<FetchGitResult, GitError> {
let bare_repo = cache.get_git_bare(url);
if !bare_repo.exists() {
clone_bare(url, &bare_repo)?;
} else {
fetch_repo(&bare_repo, all_refs)?;
}
let target_rev = resolve_rev(&bare_repo, git_ref, rev)?;
let temp_dir = tempfile::tempdir()?;
let checkout_dir = checkout_rev_to_temp(&bare_repo, &target_rev, submodules, temp_dir.path())?;
let nar_hash = hex::encode(
crate::nar::compute_nar_hash(&checkout_dir)
.map_err(|e| GitError::NarHashError(e.to_string()))?,
);
let store_path = store
.add_to_store_from_path(name, &checkout_dir, vec![])
.map_err(|e| GitError::StoreError(e.to_string()))?;
let rev_count = get_rev_count(&bare_repo, &target_rev)?;
let last_modified = get_last_modified(&bare_repo, &target_rev)?;
let last_modified_date = format_timestamp(last_modified);
let short_rev = if target_rev.len() >= 7 {
target_rev[..7].to_string()
} else {
target_rev.clone()
};
Ok(FetchGitResult {
out_path: store_path,
rev: target_rev,
short_rev,
rev_count,
last_modified,
last_modified_date,
submodules,
nar_hash: Some(nar_hash),
})
}
fn clone_bare(url: &str, dest: &PathBuf) -> Result<(), GitError> {
fs::create_dir_all(dest.parent().unwrap_or(dest))?;
let output = Command::new("git")
.args(["clone", "--bare", url])
.arg(dest)
.output()?;
if !output.status.success() {
return Err(GitError::CommandFailed {
operation: "clone".to_string(),
message: String::from_utf8_lossy(&output.stderr).to_string(),
});
}
Ok(())
}
fn fetch_repo(repo: &PathBuf, all_refs: bool) -> Result<(), GitError> {
let mut args = vec!["fetch", "--prune"];
if all_refs {
args.push("--all");
}
let output = Command::new("git").args(args).current_dir(repo).output()?;
if !output.status.success() {
return Err(GitError::CommandFailed {
operation: "fetch".to_string(),
message: String::from_utf8_lossy(&output.stderr).to_string(),
});
}
Ok(())
}
fn resolve_rev(
repo: &PathBuf,
git_ref: Option<&str>,
rev: Option<&str>,
) -> Result<String, GitError> {
if let Some(rev) = rev {
return Ok(rev.to_string());
}
let ref_to_resolve = git_ref.unwrap_or("HEAD");
let output = Command::new("git")
.args(["rev-parse", ref_to_resolve])
.current_dir(repo)
.output()?;
if !output.status.success() {
let output = Command::new("git")
.args(["rev-parse", &format!("refs/heads/{}", ref_to_resolve)])
.current_dir(repo)
.output()?;
if !output.status.success() {
let output = Command::new("git")
.args(["rev-parse", &format!("refs/tags/{}", ref_to_resolve)])
.current_dir(repo)
.output()?;
if !output.status.success() {
return Err(GitError::CommandFailed {
operation: "rev-parse".to_string(),
message: format!("Could not resolve ref: {}", ref_to_resolve),
});
}
return Ok(String::from_utf8_lossy(&output.stdout).trim().to_string());
}
return Ok(String::from_utf8_lossy(&output.stdout).trim().to_string());
}
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
}
fn checkout_rev_to_temp(
bare_repo: &PathBuf,
rev: &str,
submodules: bool,
temp_path: &std::path::Path,
) -> Result<PathBuf, GitError> {
let checkout_dir = temp_path.join("checkout");
fs::create_dir_all(&checkout_dir)?;
let output = Command::new("git")
.args(["--work-tree", checkout_dir.to_str().unwrap_or(".")])
.arg("checkout")
.arg(rev)
.arg("--")
.arg(".")
.current_dir(bare_repo)
.output()?;
if !output.status.success() {
fs::remove_dir_all(&checkout_dir)?;
return Err(GitError::CommandFailed {
operation: "checkout".to_string(),
message: String::from_utf8_lossy(&output.stderr).to_string(),
});
}
if submodules {
let output = Command::new("git")
.args(["submodule", "update", "--init", "--recursive"])
.current_dir(&checkout_dir)
.output()?;
if !output.status.success() {
tracing::warn!(
"failed to initialize submodules: {}",
String::from_utf8_lossy(&output.stderr)
);
}
}
let git_dir = checkout_dir.join(".git");
if git_dir.exists() {
fs::remove_dir_all(&git_dir)?;
}
Ok(checkout_dir)
}
fn get_rev_count(repo: &PathBuf, rev: &str) -> Result<u64, GitError> {
let output = Command::new("git")
.args(["rev-list", "--count", rev])
.current_dir(repo)
.output()?;
if !output.status.success() {
return Ok(0);
}
let count_str = String::from_utf8_lossy(&output.stdout);
count_str.trim().parse().unwrap_or(0).pipe(Ok)
}
fn get_last_modified(repo: &PathBuf, rev: &str) -> Result<u64, GitError> {
let output = Command::new("git")
.args(["log", "-1", "--format=%ct", rev])
.current_dir(repo)
.output()?;
if !output.status.success() {
return Ok(0);
}
let ts_str = String::from_utf8_lossy(&output.stdout);
ts_str.trim().parse().unwrap_or(0).pipe(Ok)
}
fn format_timestamp(ts: u64) -> String {
use std::time::{Duration, UNIX_EPOCH};
let datetime = UNIX_EPOCH + Duration::from_secs(ts);
let secs = datetime
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
let days_since_epoch = secs / 86400;
let remaining_secs = secs % 86400;
let hours = remaining_secs / 3600;
let minutes = (remaining_secs % 3600) / 60;
let seconds = remaining_secs % 60;
let (year, month, day) = days_to_ymd(days_since_epoch);
format!(
"{:04}{:02}{:02}{:02}{:02}{:02}",
year, month, day, hours, minutes, seconds
)
}
fn days_to_ymd(days: u64) -> (u64, u64, u64) {
let mut y = 1970;
let mut remaining = days as i64;
loop {
let days_in_year = if is_leap_year(y) { 366 } else { 365 };
if remaining < days_in_year {
break;
}
remaining -= days_in_year;
y += 1;
}
let days_in_months: [i64; 12] = if is_leap_year(y) {
[31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
} else {
[31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
};
let mut m = 1;
for days_in_month in days_in_months.iter() {
if remaining < *days_in_month {
break;
}
remaining -= *days_in_month;
m += 1;
}
(y, m, (remaining + 1) as u64)
}
fn is_leap_year(y: u64) -> bool {
(y.is_multiple_of(4) && !y.is_multiple_of(100)) || y.is_multiple_of(400)
}
trait Pipe: Sized {
fn pipe<F, R>(self, f: F) -> R
where
F: FnOnce(Self) -> R,
{
f(self)
}
}
impl<T> Pipe for T {}
#[derive(Debug)]
pub enum GitError {
IoError(std::io::Error),
CommandFailed { operation: String, message: String },
NarHashError(String),
StoreError(String),
}
impl std::fmt::Display for GitError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
GitError::IoError(e) => write!(f, "I/O error: {}", e),
GitError::CommandFailed { operation, message } => {
write!(f, "Git {} failed: {}", operation, message)
}
GitError::NarHashError(e) => write!(f, "NAR hash error: {}", e),
GitError::StoreError(e) => write!(f, "Store error: {}", e),
}
}
}
impl std::error::Error for GitError {}
impl From<std::io::Error> for GitError {
fn from(e: std::io::Error) -> Self {
GitError::IoError(e)
}
}

View File

@@ -0,0 +1,218 @@
#![allow(dead_code)]
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use rusqlite::{Connection, OptionalExtension, params};
use serde::{Deserialize, Serialize};
#[derive(Debug)]
pub enum CacheError {
Database(rusqlite::Error),
Json(serde_json::Error),
}
impl std::fmt::Display for CacheError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
CacheError::Database(e) => write!(f, "Database error: {}", e),
CacheError::Json(e) => write!(f, "JSON error: {}", e),
}
}
}
impl std::error::Error for CacheError {}
impl From<rusqlite::Error> for CacheError {
fn from(e: rusqlite::Error) -> Self {
CacheError::Database(e)
}
}
impl From<serde_json::Error> for CacheError {
fn from(e: serde_json::Error) -> Self {
CacheError::Json(e)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CacheEntry {
pub input: serde_json::Value,
pub info: serde_json::Value,
pub store_path: String,
pub immutable: bool,
pub timestamp: u64,
}
impl CacheEntry {
pub fn is_expired(&self, ttl_seconds: u64) -> bool {
if self.immutable {
return false;
}
if ttl_seconds == 0 {
return false;
}
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Clock may have gone backwards")
.as_secs();
now > self.timestamp + ttl_seconds
}
}
pub struct MetadataCache {
conn: Connection,
ttl_seconds: u64,
}
impl MetadataCache {
pub fn new(ttl_seconds: u64) -> Result<Self, CacheError> {
let cache_dir = dirs::cache_dir()
.unwrap_or_else(|| PathBuf::from("/tmp"))
.join("nix-js");
std::fs::create_dir_all(&cache_dir).map_err(|e| {
CacheError::Database(rusqlite::Error::ToSqlConversionFailure(Box::new(e)))
})?;
let db_path = cache_dir.join("fetcher-cache.sqlite");
let conn = Connection::open(db_path)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS cache (
input TEXT NOT NULL PRIMARY KEY,
info TEXT NOT NULL,
store_path TEXT NOT NULL,
immutable INTEGER NOT NULL,
timestamp INTEGER NOT NULL
)",
[],
)?;
Ok(Self { conn, ttl_seconds })
}
pub fn lookup(&self, input: &serde_json::Value) -> Result<Option<CacheEntry>, CacheError> {
let input_str = serde_json::to_string(input)?;
let entry: Option<(String, String, String, i64, i64)> = self
.conn
.query_row(
"SELECT input, info, store_path, immutable, timestamp FROM cache WHERE input = ?1",
params![input_str],
|row| {
Ok((
row.get(0)?,
row.get(1)?,
row.get(2)?,
row.get(3)?,
row.get(4)?,
))
},
)
.optional()?;
match entry {
Some((input_json, info_json, store_path, immutable, timestamp)) => {
let entry = CacheEntry {
input: serde_json::from_str(&input_json)?,
info: serde_json::from_str(&info_json)?,
store_path,
immutable: immutable != 0,
timestamp: timestamp as u64,
};
if entry.is_expired(self.ttl_seconds) {
Ok(None)
} else {
Ok(Some(entry))
}
}
None => Ok(None),
}
}
pub fn lookup_expired(
&self,
input: &serde_json::Value,
) -> Result<Option<CacheEntry>, CacheError> {
let input_str = serde_json::to_string(input)?;
let entry: Option<(String, String, String, i64, i64)> = self
.conn
.query_row(
"SELECT input, info, store_path, immutable, timestamp FROM cache WHERE input = ?1",
params![input_str],
|row| {
Ok((
row.get(0)?,
row.get(1)?,
row.get(2)?,
row.get(3)?,
row.get(4)?,
))
},
)
.optional()?;
match entry {
Some((input_json, info_json, store_path, immutable, timestamp)) => {
Ok(Some(CacheEntry {
input: serde_json::from_str(&input_json)?,
info: serde_json::from_str(&info_json)?,
store_path,
immutable: immutable != 0,
timestamp: timestamp as u64,
}))
}
None => Ok(None),
}
}
pub fn add(
&self,
input: &serde_json::Value,
info: &serde_json::Value,
store_path: &str,
immutable: bool,
) -> Result<(), CacheError> {
let input_str = serde_json::to_string(input)?;
let info_str = serde_json::to_string(info)?;
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Clock may have gone backwards")
.as_secs();
self.conn.execute(
"INSERT OR REPLACE INTO cache (input, info, store_path, immutable, timestamp)
VALUES (?1, ?2, ?3, ?4, ?5)",
params![
input_str,
info_str,
store_path,
if immutable { 1 } else { 0 },
timestamp as i64
],
)?;
Ok(())
}
pub fn update_timestamp(&self, input: &serde_json::Value) -> Result<(), CacheError> {
let input_str = serde_json::to_string(input)?;
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Clock may have gone backwards")
.as_secs();
self.conn.execute(
"UPDATE cache SET timestamp = ?1 WHERE input = ?2",
params![timestamp as i64, input_str],
)?;
Ok(())
}
}