diff --git a/upki-mirror/src/bin/intermediates.rs b/upki-mirror/src/bin/intermediates.rs index d4cf38fa..52d0ecf4 100644 --- a/upki-mirror/src/bin/intermediates.rs +++ b/upki-mirror/src/bin/intermediates.rs @@ -10,7 +10,7 @@ use eyre::{Context, Report, anyhow}; use rustls_pki_types::CertificateDer; use rustls_pki_types::pem::PemObject; use serde::Deserialize; -use upki::revocation::{Manifest, ManifestFile}; +use upki::data::{Manifest, ManifestFile}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), Report> { diff --git a/upki-mirror/src/bin/mozilla-crlite.rs b/upki-mirror/src/bin/mozilla-crlite.rs index dca4619f..b493ad27 100644 --- a/upki-mirror/src/bin/mozilla-crlite.rs +++ b/upki-mirror/src/bin/mozilla-crlite.rs @@ -7,7 +7,7 @@ use std::time::SystemTime; use aws_lc_rs::digest::{SHA256, digest}; use clap::{Parser, ValueEnum}; use eyre::{Context, Report, anyhow}; -use upki::revocation::{Manifest, ManifestFile}; +use upki::data::{Manifest, ManifestFile}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), Report> { diff --git a/upki/src/data.rs b/upki/src/data.rs new file mode 100644 index 00000000..c89ae9fa --- /dev/null +++ b/upki/src/data.rs @@ -0,0 +1,470 @@ +use core::fmt; +use core::time::Duration; +use std::collections::HashSet; +use std::env; +use std::fs::{self, File, Permissions}; +use std::io::{self, BufReader, Read, Write}; +#[cfg(target_family = "unix")] +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use aws_lc_rs::digest; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, info}; + +use crate::revocation::{Error, INDEX_BIN, Index}; + +/// The structure contained in a manifest.json +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Manifest { + /// When this file was generated. + /// + /// UNIX timestamp in seconds. + pub generated_at: u64, + + /// Some human-readable text. + pub comment: String, + + /// List of required files. + #[serde(alias = "filters")] + pub files: Vec, +} + +impl Manifest { + /// Logs metadata fields in this manifest. + pub fn introduce(&self) -> Result<(), Error> { + let dt = match DateTime::::from_timestamp(self.generated_at as i64, 0) { + Some(dt) => dt.to_rfc3339(), + None => { + return Err(Error::InvalidTimestamp { + input: self.generated_at.to_string(), + context: "manifest generated (in s)", + }); + } + }; + + info!(comment = self.comment, date = dt, "parsed manifest"); + Ok(()) + } + + /// Load a manifest by reading the named JSON file. + pub(crate) fn from_file(file_name: &Path) -> Result { + let file = match File::open(file_name) { + Ok(f) => f, + Err(error) => { + return Err(Error::FileRead { + error, + path: Some(file_name.to_owned()), + }); + } + }; + + serde_json::from_reader(BufReader::new(file)).map_err(|error| Error::FileDecode { + error: Box::new(error), + path: Some(file_name.to_owned()), + }) + } +} + +/// Manifest data for a single disk file. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct ManifestFile { + /// Relative filename. + /// + /// This is also the suggested local filename. + pub filename: String, + + /// File size, indicative. Allows a fetcher to predict data usage. + pub size: usize, + + /// SHA256 hash of file contents. + #[serde(with = "hex::serde")] + pub hash: Vec, +} + +pub(crate) async fn fetch_inner( + dry_run: bool, + fetch_url: &str, + manifest_url: String, + old_manifest: &Option, + cache_dir: PathBuf, +) -> Result { + info!("fetching {fetch_url} into {cache_dir:?}..."); + let client = reqwest::Client::builder() + .use_rustls_tls() + .timeout(Duration::from_secs(REQUEST_TIMEOUT)) + .user_agent(format!( + "{}/{} ({})", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_VERSION"), + env!("CARGO_PKG_REPOSITORY") + )) + .build() + .map_err(|error| Error::HttpFetch { + error: Box::new(error), + url: manifest_url.clone(), + })?; + + let response = client + .get(&manifest_url) + .send() + .await + .map_err(|error| Error::HttpFetch { + error: Box::new(error), + url: manifest_url.clone(), + })? + .error_for_status() + .map_err(|error| Error::HttpFetch { + error: Box::new(error), + url: manifest_url.clone(), + })?; + + let manifest = response + .json::() + .await + .map_err(|error| Error::FileDecode { + error: Box::new(error), + path: None, + })?; + + manifest.introduce()?; + + let plan = Plan::construct(&manifest, old_manifest, fetch_url, &cache_dir)?; + + if dry_run { + println!( + "{} steps required ({} bytes to download)", + plan.steps.len(), + plan.download_bytes() + ); + for step in plan.steps { + println!("- {step}"); + } + return Ok(ExitCode::SUCCESS); + } + + info!( + "{} steps required ({} bytes to download).", + plan.steps.len(), + plan.download_bytes() + ); + + for step in plan.steps { + step.execute(&client).await?; + } + + info!("success"); + Ok(ExitCode::SUCCESS) +} + +pub(crate) struct Plan { + steps: Vec, +} + +impl Plan { + /// Form a plan of how to synchronize with the remote server. + /// + /// - `manifest` describes the contents of the remote server. + /// - `old_manifest` is an alleged current manifest, whose files are left alone. + /// - `remote_url` is the base URL. + /// - `local` is the path into which files are downloaded. The caller ensures this exists. + pub(crate) fn construct( + manifest: &Manifest, + old_manifest: &Option, + remote_url: &str, + local: &Path, + ) -> Result { + let mut steps = Vec::new(); + + // Collect unwanted files for deletion + let mut unwanted_files = HashSet::new(); + + if local.exists() { + let iter = fs::read_dir(local).map_err(|error| Error::CreateDirectory { + error, + path: local.to_owned(), + })?; + + for entry in iter { + let entry = match entry { + Ok(e) => e, + Err(error) => return Err(Error::FileRead { error, path: None }), + }; + + let path = Path::new(&entry.file_name()).to_owned(); + let name = path.to_string_lossy(); + if name.ends_with(".filter") || name.ends_with(".delta") { + unwanted_files.insert(path); + } + } + } else { + steps.push(PlanStep::CreateDir(local.to_owned())); + } + + for file in &manifest.files { + unwanted_files.remove(Path::new(&file.filename)); + + let path = local.join(&file.filename); + match hash_file(&path) { + Ok(digest) if digest.as_ref() == file.hash => continue, + _ => {} + } + + steps.push(PlanStep::download(file, remote_url, local)); + } + + if let Some(old_manifest) = &old_manifest { + for file in &old_manifest.files { + unwanted_files.remove(Path::new(&file.filename)); + } + } + + steps.push(PlanStep::SaveIndex { + manifest: manifest.clone(), + local_dir: local.to_owned(), + }); + + steps.push(PlanStep::SaveManifest { + manifest: manifest.clone(), + local_dir: local.to_owned(), + }); + + for filename in unwanted_files { + steps.push(PlanStep::Delete(local.join(filename))); + } + + Ok(Self { steps }) + } + + /// How many bytes will we download? + pub(crate) fn download_bytes(&self) -> usize { + self.steps + .iter() + .filter_map(|s| match s { + PlanStep::Download { file, .. } => Some(file.size), + _ => None, + }) + .sum() + } +} + +/// One step moving closer to local sync with the remote contents. +enum PlanStep { + CreateDir(PathBuf), + + /// Download `file` from `remote` to `local` + Download { + file: ManifestFile, + /// URL. + remote_url: String, + /// Full path to output file. + local: PathBuf, + }, + + /// Delete the given single local file. + Delete(PathBuf), + + /// Build and save the index from filter universe metadata. + SaveIndex { + manifest: Manifest, + local_dir: PathBuf, + }, + + /// Save the manifest structure + SaveManifest { + manifest: Manifest, + local_dir: PathBuf, + }, +} + +impl PlanStep { + async fn execute(self, client: &reqwest::Client) -> Result<(), Error> { + match self { + Self::CreateDir(path) => { + fs::create_dir_all(&path).map_err(|error| Error::CreateDirectory { error, path })? + } + Self::Download { + file, + remote_url, + local, + } => { + debug!("downloading {:?}", file); + + let response = client + .get(&remote_url) + .send() + .await + .map_err(|error| Error::HttpFetch { + error: Box::new(error), + url: remote_url.clone(), + })? + .error_for_status() + .map_err(|error| Error::HttpFetch { + error: Box::new(error), + url: remote_url.clone(), + })?; + + let bytes = response + .bytes() + .await + .map_err(|error| Error::HttpFetch { + error: Box::new(error), + url: remote_url.clone(), + })?; + + atomic_write(&local, &bytes).map_err(|error| Error::FileWrite { + error, + path: local.clone(), + })?; + + match hash_file(&local) { + Ok(digest) if digest.as_ref() == file.hash => {} + Ok(_) => return Err(Error::HashMismatch(local)), + Err(error) => { + return Err(Error::FileRead { + error, + path: Some(local), + }); + } + } + + debug!("download successful"); + } + Self::Delete(target) => { + debug!("deleting unreferenced file {target:?}"); + fs::remove_file(&target).map_err(|error| Error::RemoveFile { + error, + path: target, + })?; + } + Self::SaveIndex { + manifest, + local_dir, + } => { + debug!("building index"); + let Some(buf) = Index::write(&manifest, &local_dir) else { + return Ok(()); + }; + + #[cfg(target_family = "unix")] + let temp = tempfile::Builder::new() + .permissions(Permissions::from_mode(0o644)) + .suffix(".new") + .tempfile_in(&local_dir); + #[cfg(not(target_family = "unix"))] + let temp = tempfile::Builder::new() + .suffix(".new") + .tempfile_in(&local_dir); + + let mut local_temp = temp.map_err(|error| Error::FileWrite { + error, + path: local_dir.clone(), + })?; + + local_temp + .as_file_mut() + .write_all(&buf) + .map_err(|error| Error::FileWrite { + error, + path: local_temp.path().to_owned(), + })?; + + let path = local_dir.join(INDEX_BIN); + local_temp + .persist(&path) + .map_err(|error| Error::FileWrite { + error: error.error, + path, + })?; + } + Self::SaveManifest { + manifest, + local_dir, + } => { + debug!("saving manifest"); + let path = local_dir.join(MANIFEST_JSON); + let data = + serde_json::to_vec(&manifest).map_err(|error| Error::ManifestEncode { + error: Box::new(error), + path: path.clone(), + })?; + atomic_write(&path, &data).map_err(|error| Error::FileWrite { error, path })?; + } + } + + Ok(()) + } + + fn download(file: &ManifestFile, remote_url: &str, local: &Path) -> Self { + Self::Download { + file: file.clone(), + remote_url: format!("{remote_url}{}", file.filename), + local: local.join(&file.filename), + } + } +} + +impl fmt::Display for PlanStep { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CreateDir(path) => write!(f, "create directory {path:?}"), + Self::Download { + file, + remote_url, + local, + } => write!( + f, + "download {} bytes from {remote_url} to {local:?}", + file.size + ), + Self::Delete(path) => write!(f, "delete stale file {path:?}"), + Self::SaveIndex { local_dir, .. } => { + write!(f, "build index from filters into {local_dir:?}") + } + Self::SaveManifest { local_dir, .. } => { + write!(f, "save new manifest into {local_dir:?}") + } + } + } +} + +/// Atomically write `data` to `path` via a temporary file and rename. +fn atomic_write(path: &Path, data: &[u8]) -> Result<(), io::Error> { + let dir = path + .parent() + .expect("path must have parent"); + + #[cfg(target_family = "unix")] + let temp = tempfile::Builder::new() + .permissions(Permissions::from_mode(0o644)) + .tempfile_in(dir); + #[cfg(not(target_family = "unix"))] + let temp = tempfile::Builder::new().tempfile_in(dir); + + let mut temp = temp?; + temp.write_all(data)?; + temp.persist(path) + .map_err(|error| error.error)?; + Ok(()) +} + +fn hash_file(path: &Path) -> Result { + let mut file = File::open(path)?; + let mut hasher = digest::Context::new(&digest::SHA256); + let mut buffer = [0; 4096]; + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + + hasher.update(&buffer[..n]); + } + + Ok(hasher.finish()) +} + +pub(crate) const MANIFEST_JSON: &str = "manifest.json"; +const REQUEST_TIMEOUT: u64 = 30; diff --git a/upki/src/intermediates.rs b/upki/src/intermediates.rs new file mode 100644 index 00000000..41a3521c --- /dev/null +++ b/upki/src/intermediates.rs @@ -0,0 +1,49 @@ +use std::process::ExitCode; + +use serde::{Deserialize, Serialize}; + +use crate::Config; +use crate::data::{MANIFEST_JSON, Manifest, fetch_inner}; +use crate::revocation::Error; + +/// Update the local intermediates cache by fetching updates over the network. +/// +/// `dry_run` means this call fetches the new manifest, but does not fetch any +/// required files; but the necessary files are printed to stdout. Therefore +/// such a call is not completely "dry" -- perhaps "moist". +pub async fn fetch(dry_run: bool, config: &Config) -> Result { + let Some(intermediates) = &config.intermediates else { + return Ok(ExitCode::SUCCESS); + }; + let mut manifest_path = config.intermediates_cache_dir(); + manifest_path.push(MANIFEST_JSON); + let old_manifest = Manifest::from_file(&manifest_path).ok(); + let manifest_url = format!("{}{MANIFEST_JSON}", intermediates.fetch_url); + fetch_inner( + dry_run, + &intermediates.fetch_url, + manifest_url, + &old_manifest, + config.intermediates_cache_dir(), + ) + .await +} + +/// Details about intermediate preloading. +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct IntermediatesConfig { + /// Whether to fetch things at all. + enabled: bool, + /// Where to fetch intermediate certificates. + fetch_url: String, +} + +impl Default for IntermediatesConfig { + fn default() -> Self { + Self { + enabled: false, + fetch_url: "https://upki.rustls.dev/intermediates/".into(), + } + } +} diff --git a/upki/src/lib.rs b/upki/src/lib.rs index a1d22979..11e5d858 100644 --- a/upki/src/lib.rs +++ b/upki/src/lib.rs @@ -7,6 +7,7 @@ use std::{fmt, fs, io}; use serde::{Deserialize, Serialize}; +use crate::intermediates::IntermediatesConfig; use crate::revocation::RevocationConfig; /// `upki` configuration. @@ -18,6 +19,9 @@ pub struct Config { /// Configuration for crlite-style revocation. pub revocation: RevocationConfig, + + /// Configuration for intermediate preloading. + pub intermediates: Option, } impl Config { @@ -50,12 +54,17 @@ impl Config { Ok(Self { cache_dir: platform::default_cache_dir()?, revocation: RevocationConfig::default(), + intermediates: Some(IntermediatesConfig::default()), }) } pub(crate) fn revocation_cache_dir(&self) -> PathBuf { self.cache_dir.join("revocation") } + + fn intermediates_cache_dir(&self) -> PathBuf { + self.cache_dir.join("intermediates") + } } /// How the path to a configuration file was decided upon. @@ -199,3 +208,9 @@ const CONFIG_FILE: &str = "config.toml"; /// Determining revocation status of publicly trusted certificates. pub mod revocation; + +/// Fetching intermediate certificates to assist chain building. +pub mod intermediates; + +/// Data storage. +pub mod data; diff --git a/upki/src/main.rs b/upki/src/main.rs index b2e66342..61893857 100644 --- a/upki/src/main.rs +++ b/upki/src/main.rs @@ -8,8 +8,8 @@ use clap::{Parser, Subcommand}; use eyre::{Context, Report}; use rustls_pki_types::CertificateDer; use rustls_pki_types::pem::PemObject; -use upki::revocation::{Index, Manifest, RevocationCheckInput, fetch}; -use upki::{Config, ConfigPath}; +use upki::revocation::{Index, Manifest, RevocationCheckInput}; +use upki::{Config, ConfigPath, intermediates, revocation}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result { @@ -33,7 +33,15 @@ async fn main() -> Result { let config = Config::from_file_or_default(&config_path)?; Ok(match args.command { - Command::Fetch { dry_run } => fetch(dry_run, &config).await?, + Command::Fetch { dry_run } => { + match ( + revocation::fetch(dry_run, &config).await?, + intermediates::fetch(dry_run, &config).await?, + ) { + (ExitCode::SUCCESS, ExitCode::SUCCESS) => ExitCode::SUCCESS, + (..) => ExitCode::FAILURE, + } + } Command::Verify => Manifest::from_config(&config)?.verify(&config)?, Command::ShowConfigPath => unreachable!(), Command::ShowConfig => { diff --git a/upki/src/revocation/fetch.rs b/upki/src/revocation/fetch.rs index be75012d..9442abe7 100644 --- a/upki/src/revocation/fetch.rs +++ b/upki/src/revocation/fetch.rs @@ -6,23 +6,11 @@ //! the local filesystem contents. Finally, the plan is executed. If that succeeds //! the remote server contents matches the local filesystem. -use core::fmt; -use core::time::Duration; -use std::collections::HashSet; -use std::env; -use std::fs::{self, File, Permissions}; -use std::io::{self, Read, Write}; -#[cfg(target_family = "unix")] -use std::os::unix::fs::PermissionsExt; -use std::path::{Path, PathBuf}; use std::process::ExitCode; -use aws_lc_rs::digest; -use tracing::{debug, info}; - -use super::index::INDEX_BIN; -use super::{Error, Index, Manifest, ManifestFile}; +use super::Error; use crate::Config; +use crate::data::{MANIFEST_JSON, fetch_inner}; /// Update the local revocation cache by fetching updates over the network. /// @@ -30,393 +18,16 @@ use crate::Config; /// required files; but the necessary files are printed to stdout. Therefore /// such a call is not completely "dry" -- perhaps "moist". pub async fn fetch(dry_run: bool, config: &Config) -> Result { - let cache_dir = config.revocation_cache_dir(); - info!( - "fetching {} into {:?}...", - &config.revocation.fetch_url, &cache_dir, - ); - let manifest_url = format!("{}{MANIFEST_JSON}", config.revocation.fetch_url); - let client = reqwest::Client::builder() - .use_rustls_tls() - .timeout(Duration::from_secs(REQUEST_TIMEOUT)) - .user_agent(format!( - "{}/{} ({})", - env!("CARGO_PKG_NAME"), - env!("CARGO_PKG_VERSION"), - env!("CARGO_PKG_REPOSITORY") - )) - .build() - .map_err(|error| Error::HttpFetch { - error: Box::new(error), - url: manifest_url.clone(), - })?; - - let response = client - .get(&manifest_url) - .send() - .await - .map_err(|error| Error::HttpFetch { - error: Box::new(error), - url: manifest_url.clone(), - })? - .error_for_status() - .map_err(|error| Error::HttpFetch { - error: Box::new(error), - url: manifest_url.clone(), - })?; - - let manifest = response - .json::() - .await - .map_err(|error| Error::FileDecode { - error: Box::new(error), - path: None, - })?; - - manifest.introduce()?; - - let old_manifest = Manifest::from_config(config).ok(); - - let plan = Plan::construct( - &manifest, - &old_manifest, + let old_manifest = super::Manifest::from_config(config) + .ok() + .map(|m| m.0); + fetch_inner( + dry_run, &config.revocation.fetch_url, - &cache_dir, - )?; - - if dry_run { - println!( - "{} steps required ({} bytes to download)", - plan.steps.len(), - plan.download_bytes() - ); - for step in plan.steps { - println!("- {step}"); - } - return Ok(ExitCode::SUCCESS); - } - - info!( - "{} steps required ({} bytes to download).", - plan.steps.len(), - plan.download_bytes() - ); - - for step in plan.steps { - step.execute(&client).await?; - } - - info!("success"); - Ok(ExitCode::SUCCESS) -} - -pub(crate) struct Plan { - steps: Vec, -} - -impl Plan { - /// Form a plan of how to synchronize with the remote server. - /// - /// - `manifest` describes the contents of the remote server. - /// - `old_manifest` is an alleged current manifest, whose files are left alone. - /// - `remote_url` is the base URL. - /// - `local` is the path into which files are downloaded. The caller ensures this exists. - pub(crate) fn construct( - manifest: &Manifest, - old_manifest: &Option, - remote_url: &str, - local: &Path, - ) -> Result { - let mut steps = Vec::new(); - - // Collect unwanted files for deletion - let mut unwanted_files = HashSet::new(); - - if local.exists() { - let iter = fs::read_dir(local).map_err(|error| Error::CreateDirectory { - error, - path: local.to_owned(), - })?; - - for entry in iter { - let entry = match entry { - Ok(e) => e, - Err(error) => return Err(Error::FileRead { error, path: None }), - }; - - let path = Path::new(&entry.file_name()).to_owned(); - let name = path.to_string_lossy(); - if name.ends_with(".filter") || name.ends_with(".delta") { - unwanted_files.insert(path); - } - } - } else { - steps.push(PlanStep::CreateDir(local.to_owned())); - } - - for file in &manifest.files { - unwanted_files.remove(Path::new(&file.filename)); - - let path = local.join(&file.filename); - match hash_file(&path) { - Ok(digest) if digest.as_ref() == file.hash => continue, - _ => {} - } - - steps.push(PlanStep::download(file, remote_url, local)); - } - - if let Some(old_manifest) = &old_manifest { - for file in &old_manifest.files { - unwanted_files.remove(Path::new(&file.filename)); - } - } - - steps.push(PlanStep::SaveIndex { - manifest: manifest.clone(), - local_dir: local.to_owned(), - }); - - steps.push(PlanStep::SaveManifest { - manifest: manifest.clone(), - local_dir: local.to_owned(), - }); - - for filename in unwanted_files { - steps.push(PlanStep::Delete(local.join(filename))); - } - - Ok(Self { steps }) - } - - /// How many bytes will we download? - pub(crate) fn download_bytes(&self) -> usize { - self.steps - .iter() - .filter_map(|s| match s { - PlanStep::Download { file, .. } => Some(file.size), - _ => None, - }) - .sum() - } -} - -/// One step moving closer to local sync with the remote contents. -enum PlanStep { - CreateDir(PathBuf), - - /// Download `file` from `remote` to `local` - Download { - file: ManifestFile, - /// URL. - remote_url: String, - /// Full path to output file. - local: PathBuf, - }, - - /// Delete the given single local file. - Delete(PathBuf), - - /// Build and save the index from filter universe metadata. - SaveIndex { - manifest: Manifest, - local_dir: PathBuf, - }, - - /// Save the manifest structure - SaveManifest { - manifest: Manifest, - local_dir: PathBuf, - }, -} - -impl PlanStep { - async fn execute(self, client: &reqwest::Client) -> Result<(), Error> { - match self { - Self::CreateDir(path) => { - fs::create_dir_all(&path).map_err(|error| Error::CreateDirectory { error, path })? - } - Self::Download { - file, - remote_url, - local, - } => { - debug!("downloading {:?}", file); - - let response = client - .get(&remote_url) - .send() - .await - .map_err(|error| Error::HttpFetch { - error: Box::new(error), - url: remote_url.clone(), - })? - .error_for_status() - .map_err(|error| Error::HttpFetch { - error: Box::new(error), - url: remote_url.clone(), - })?; - - let bytes = response - .bytes() - .await - .map_err(|error| Error::HttpFetch { - error: Box::new(error), - url: remote_url.clone(), - })?; - - atomic_write(&local, &bytes).map_err(|error| Error::FileWrite { - error, - path: local.clone(), - })?; - - match hash_file(&local) { - Ok(digest) if digest.as_ref() == file.hash => {} - Ok(_) => return Err(Error::HashMismatch(local)), - Err(error) => { - return Err(Error::FileRead { - error, - path: Some(local), - }); - } - } - - debug!("download successful"); - } - Self::Delete(target) => { - debug!("deleting unreferenced file {target:?}"); - fs::remove_file(&target).map_err(|error| Error::RemoveFile { - error, - path: target, - })?; - } - Self::SaveIndex { - manifest, - local_dir, - } => { - debug!("building index"); - let Some(buf) = Index::write(&manifest, &local_dir) else { - return Ok(()); - }; - - #[cfg(target_family = "unix")] - let temp = tempfile::Builder::new() - .permissions(Permissions::from_mode(0o644)) - .suffix(".new") - .tempfile_in(&local_dir); - #[cfg(not(target_family = "unix"))] - let temp = tempfile::Builder::new() - .suffix(".new") - .tempfile_in(&local_dir); - - let mut local_temp = temp.map_err(|error| Error::FileWrite { - error, - path: local_dir.clone(), - })?; - - local_temp - .as_file_mut() - .write_all(&buf) - .map_err(|error| Error::FileWrite { - error, - path: local_temp.path().to_owned(), - })?; - - let path = local_dir.join(INDEX_BIN); - local_temp - .persist(&path) - .map_err(|error| Error::FileWrite { - error: error.error, - path, - })?; - } - Self::SaveManifest { - manifest, - local_dir, - } => { - debug!("saving manifest"); - let path = local_dir.join(MANIFEST_JSON); - let data = - serde_json::to_vec(&manifest).map_err(|error| Error::ManifestEncode { - error: Box::new(error), - path: path.clone(), - })?; - atomic_write(&path, &data).map_err(|error| Error::FileWrite { error, path })?; - } - } - - Ok(()) - } - - fn download(file: &ManifestFile, remote_url: &str, local: &Path) -> Self { - Self::Download { - file: file.clone(), - remote_url: format!("{remote_url}{}", file.filename), - local: local.join(&file.filename), - } - } -} - -impl fmt::Display for PlanStep { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::CreateDir(path) => write!(f, "create directory {path:?}"), - Self::Download { - file, - remote_url, - local, - } => write!( - f, - "download {} bytes from {remote_url} to {local:?}", - file.size - ), - Self::Delete(path) => write!(f, "delete stale file {path:?}"), - Self::SaveIndex { local_dir, .. } => { - write!(f, "build index from filters into {local_dir:?}") - } - Self::SaveManifest { local_dir, .. } => { - write!(f, "save new manifest into {local_dir:?}") - } - } - } -} - -/// Atomically write `data` to `path` via a temporary file and rename. -fn atomic_write(path: &Path, data: &[u8]) -> Result<(), io::Error> { - let dir = path - .parent() - .expect("path must have parent"); - - #[cfg(target_family = "unix")] - let temp = tempfile::Builder::new() - .permissions(Permissions::from_mode(0o644)) - .tempfile_in(dir); - #[cfg(not(target_family = "unix"))] - let temp = tempfile::Builder::new().tempfile_in(dir); - - let mut temp = temp?; - temp.write_all(data)?; - temp.persist(path) - .map_err(|error| error.error)?; - Ok(()) -} - -fn hash_file(path: &Path) -> Result { - let mut file = File::open(path)?; - let mut hasher = digest::Context::new(&digest::SHA256); - let mut buffer = [0; 4096]; - loop { - let n = file.read(&mut buffer)?; - if n == 0 { - break; - } - - hasher.update(&buffer[..n]); - } - - Ok(hasher.finish()) + manifest_url, + &old_manifest, + config.revocation_cache_dir(), + ) + .await } - -const MANIFEST_JSON: &str = "manifest.json"; -const REQUEST_TIMEOUT: u64 = 30; diff --git a/upki/src/revocation/index.rs b/upki/src/revocation/index.rs index d142f8fb..aabcf429 100644 --- a/upki/src/revocation/index.rs +++ b/upki/src/revocation/index.rs @@ -7,8 +7,9 @@ use std::path::{Path, PathBuf}; use clubcard_crlite::{CRLiteClubcard, CRLiteStatus}; -use super::{Error, Manifest, RevocationCheckInput, RevocationStatus}; +use super::{Error, RevocationCheckInput, RevocationStatus}; use crate::Config; +use crate::data::Manifest; /// Binary-encoded index of universe metadata for all filters in a manifest. /// @@ -95,7 +96,7 @@ impl Index { /// Build index bytes by reading filter files from `dir` and extracting universe metadata. /// /// Returns `None` if any filter file cannot be read or decoded. - pub(super) fn write(manifest: &Manifest, dir: &Path) -> Option> { + pub(crate) fn write(manifest: &Manifest, dir: &Path) -> Option> { let mut by_log_id: BTreeMap<[u8; 32], Vec<(u8, u64, u64)>> = BTreeMap::new(); for (filter_idx, filter) in manifest.files.iter().enumerate() { @@ -344,7 +345,7 @@ const FILENAME_SIZE: usize = 32; const LOG_DIR_ENTRY_SIZE: usize = 32 + 8 + 2; const ENTRY_SIZE: usize = 1 + 8 + 8; -pub(super) const INDEX_BIN: &str = "index.bin"; +pub(crate) const INDEX_BIN: &str = "index.bin"; const INDEX_MAGIC: &[u8; 8] = b"upkiidx0"; #[cfg(test)] @@ -437,6 +438,7 @@ mod tests { Config { cache_dir: dir.to_owned(), revocation: RevocationConfig::default(), + intermediates: None, } } diff --git a/upki/src/revocation/mod.rs b/upki/src/revocation/mod.rs index 3f8b1f08..d093696f 100644 --- a/upki/src/revocation/mod.rs +++ b/upki/src/revocation/mod.rs @@ -1,65 +1,37 @@ use core::error::Error as StdError; +use core::ops::Deref; use core::str::FromStr; use core::{fmt, str}; -use std::fs::File; -use std::io::{self, BufReader}; +use std::io; use std::path::PathBuf; use std::process::ExitCode; use aws_lc_rs::digest; use base64::Engine; use base64::prelude::BASE64_STANDARD; -use chrono::{DateTime, Utc}; use clubcard_crlite::CRLiteKey; use rustls_pki_types::{CertificateDer, TrustAnchor}; use serde::{Deserialize, Serialize}; -use tracing::info; -use crate::Config; +use crate::{Config, data}; mod fetch; -use fetch::Plan; pub use fetch::fetch; mod index; +pub(crate) use index::INDEX_BIN; pub use index::Index; /// The structure contained in a manifest.json #[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Manifest { - /// When this file was generated. - /// - /// UNIX timestamp in seconds. - pub generated_at: u64, - - /// Some human-readable text. - pub comment: String, - - /// List of required files. - #[serde(alias = "filters")] - pub files: Vec, -} +pub struct Manifest(data::Manifest); impl Manifest { /// Load the revocation manifest from the cache directory specified in the configuration. pub fn from_config(config: &Config) -> Result { let mut file_name = config.revocation_cache_dir(); - file_name.push("manifest.json"); - - let file = match File::open(&file_name) { - Ok(f) => f, - Err(error) => { - return Err(Error::FileRead { - error, - path: Some(file_name), - }); - } - }; - - serde_json::from_reader(BufReader::new(file)).map_err(|error| Error::FileDecode { - error: Box::new(error), - path: Some(file_name), - }) + file_name.push(data::MANIFEST_JSON); + data::Manifest::from_file(&file_name).map(Manifest) } /// Verify the current contents of the cache against this manifest. @@ -67,44 +39,21 @@ impl Manifest { /// This performs disk IO but does not perform network IO. pub fn verify(&self, config: &Config) -> Result { self.introduce()?; - let plan = Plan::construct(self, &None, "https://.../", &config.revocation_cache_dir())?; + let plan = + data::Plan::construct(self, &None, "https://.../", &config.revocation_cache_dir())?; match plan.download_bytes() { 0 => Ok(ExitCode::SUCCESS), bytes => Err(Error::Outdated(bytes)), } } - - /// Logs metadata fields in this manifest. - pub fn introduce(&self) -> Result<(), Error> { - let dt = match DateTime::::from_timestamp(self.generated_at as i64, 0) { - Some(dt) => dt.to_rfc3339(), - None => { - return Err(Error::InvalidTimestamp { - input: self.generated_at.to_string(), - context: "manifest generated (in s)", - }); - } - }; - - info!(comment = self.comment, date = dt, "parsed manifest"); - Ok(()) - } } -/// Manifest data for a single crlite filter file. -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct ManifestFile { - /// Relative filename. - /// - /// This is also the suggested local filename. - pub filename: String, +impl Deref for Manifest { + type Target = data::Manifest; - /// File size, indicative. Allows a fetcher to predict data usage. - pub size: usize, - - /// SHA256 hash of file contents. - #[serde(with = "hex::serde")] - pub hash: Vec, + fn deref(&self) -> &Self::Target { + &self.0 + } } /// Input parameters for a revocation check. diff --git a/upki/tests/integration.rs b/upki/tests/integration.rs index 55e17918..78717397 100644 --- a/upki/tests/integration.rs +++ b/upki/tests/integration.rs @@ -36,7 +36,7 @@ fn config_unknown_fields() { .arg("--config-file") .arg("tests/data/config_unknown_fields/config.toml") .arg("show-config"), - @r###" + @r#" success: false exit_code: 1 ----- stdout ----- @@ -49,12 +49,12 @@ fn config_unknown_fields() { | 1 | cache_dir = "tests/data/config_unknown_fields/" | ^^^^^^^^^ - unknown field `cache_dir`, expected `cache-dir` or `revocation` + unknown field `cache_dir`, expected one of `cache-dir`, `revocation`, `intermediates` Location: upki/src/main.rs:[LINE]:[COLUMN] - "###); + "#); } #[test]