Skip to content

Commit d8443c7

Browse files
Copilotszmyty
andauthored
feat: add incremental build system with explicit dependency tracking
Introduce src/incremental.rs with: - FileDependency: records (path, SHA-256 hash) pairs for input files - DependencyMap: maps output paths → their file-level dependencies, with record(), dependencies_for(), is_output_up_to_date(), and outputs_affected_by() methods - hash_file(): hashes a file's contents - build_output_dependencies(): collects deps (input, config, template) - load_dependency_map() / save_dependency_map(): JSON persistence Integrate into build.rs: - Load .renderflow-deps.json at build start - Compute file-level deps for each output in the render loop - Log incremental dependency status at DEBUG level (is_output_up_to_date, dependencies_for) - Record file deps in the map for every successful render (real or skipped) - Persist .renderflow-deps.json alongside the existing output cache Integrate into watch.rs: - Use outputs_affected_by() to log which outputs are affected by each changed file (visible with --verbose / --debug) Introduce RenderResult type alias in build.rs to satisfy clippy::type_complexity. Agent-Logs-Url: https://github.com/egohygiene/renderflow/sessions/18886f34-978b-4397-a9a2-313ba41e7d00 Co-authored-by: szmyty <14865041+szmyty@users.noreply.github.com>
1 parent 8ea8bf6 commit d8443c7

4 files changed

Lines changed: 565 additions & 11 deletions

File tree

src/commands/build.rs

Lines changed: 55 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use crate::cache::{compute_input_hash, compute_output_hash, load_cache, load_out
1212
use crate::config::{load_config, OutputType};
1313
use crate::deps::validate_dependencies;
1414
use crate::files::{ensure_output_dir, validate_input};
15+
use crate::incremental::{build_output_dependencies, load_dependency_map, save_dependency_map, FileDependency};
1516
use crate::optimization::OptimizationMode;
1617
use crate::pipeline::{Pipeline, StrategyStep};
1718
use crate::strategies::select_strategy;
@@ -36,6 +37,14 @@ pub fn run_resilient(config_path: &str) -> Result<()> {
3637
run_impl(config_path, false, true, None)
3738
}
3839

40+
/// Each element produced by the parallel render loop:
41+
/// (format_name, output_path, render_result, optional_output_hash, optional_file_deps).
42+
///
43+
/// The optional hash and deps are `Some` only for successful renders (including
44+
/// outputs that were skipped as up-to-date) and are used to update the output
45+
/// cache and dependency map after all formats have finished.
46+
type RenderResult = (String, String, Result<()>, Option<String>, Option<Vec<FileDependency>>);
47+
3948
fn run_impl(config_path: &str, dry_run: bool, resilient: bool, optimization: Option<OptimizationMode>) -> Result<()> {
4049
if dry_run {
4150
info!("Dry-run mode enabled — no files will be created and no commands will be executed");
@@ -192,14 +201,21 @@ fn run_impl(config_path: &str, dry_run: bool, resilient: bool, optimization: Opt
192201
let output_cache_path = output_dir.join(".renderflow-output-cache.json");
193202
let mut output_cache = load_output_cache(&output_cache_path);
194203

204+
// Load the dependency map so that file-level dependencies are tracked across
205+
// builds. This records which specific input files (source document, config,
206+
// templates) produced each output, enabling precise change attribution.
207+
let dep_map_path = output_dir.join(".renderflow-deps.json");
208+
let mut dep_map = load_dependency_map(&dep_map_path);
209+
195210
// Output formats are rendered concurrently via rayon. Progress bar updates
196211
// and log messages may interleave across formats; this is expected and
197212
// acceptable for parallel execution.
198213
//
199-
// Each element is (format_name, output_path, result, Option<new_output_hash>).
200-
// The optional hash is Some only when the render succeeded (or was skipped as
201-
// up-to-date), and is used to update the output cache after all formats finish.
202-
let render_results: Vec<(String, String, Result<()>, Option<String>)> = config
214+
// Each element is (format_name, output_path, result, Option<new_output_hash>,
215+
// Option<file_deps>). The optional hash and deps are Some only when the
216+
// render succeeded (or was skipped as up-to-date), and are used to update
217+
// the output cache and dependency map after all formats finish.
218+
let render_results: Vec<RenderResult> = config
203219
.outputs
204220
.par_iter()
205221
.map(|output| {
@@ -219,13 +235,34 @@ fn run_impl(config_path: &str, dry_run: bool, resilient: bool, optimization: Opt
219235
pb.set_message(format!("[DRY RUN] [{format}] Would render output"));
220236
pb.inc(1);
221237
pb.println(format!("[DRY RUN] Would write output to: {}", output_path));
222-
(format_str, output_path, Ok(()), None)
238+
(format_str, output_path, Ok(()), None, None)
223239
} else {
240+
// Build the list of file dependencies for this output. This is
241+
// used to populate the dependency map after the render completes.
242+
let template_path = output.template.as_deref().map(|name| {
243+
Path::new("templates").join(name)
244+
});
245+
let file_deps = build_output_dependencies(
246+
&canonical_input,
247+
Path::new(config_path),
248+
template_path.as_deref(),
249+
);
250+
224251
// Compute a hash of all inputs that determine this output's content.
225252
// If the stored hash matches and the output file already exists, pandoc
226253
// can be skipped entirely. The template file content (not just its
227254
// name) is included so that edits to a template file invalidate the
228255
// output cache even when the template path is unchanged.
256+
//
257+
// Additionally, log the file-level dependency status (from the
258+
// dependency map) at DEBUG level so that the precise reason a rebuild
259+
// was or was not triggered is visible in verbose output.
260+
debug!(
261+
output = %output_path,
262+
dep_map_up_to_date = dep_map.is_output_up_to_date(&output_path, &file_deps),
263+
recorded_deps = ?dep_map.dependencies_for(&output_path),
264+
"Incremental dependency check"
265+
);
229266
let template_content = output.template.as_deref().and_then(|name| {
230267
let path = Path::new("templates").join(name);
231268
match fs::read_to_string(&path) {
@@ -255,7 +292,7 @@ fn run_impl(config_path: &str, dry_run: bool, resilient: bool, optimization: Opt
255292
info!("Skipping {} render (unchanged)", format);
256293
pb.inc(1);
257294
pb.println(format!("↩ Skipping {} output (unchanged): {}", format, output_path));
258-
return (format_str, output_path, Ok(()), Some(output_hash));
295+
return (format_str, output_path, Ok(()), Some(output_hash), Some(file_deps));
259296
}
260297

261298
let result = (|| -> Result<()> {
@@ -270,6 +307,7 @@ fn run_impl(config_path: &str, dry_run: bool, resilient: bool, optimization: Opt
270307
})();
271308

272309
let new_hash = if result.is_ok() { Some(output_hash) } else { None };
310+
let new_deps = if result.is_ok() { Some(file_deps) } else { None };
273311

274312
match &result {
275313
Ok(_) => {
@@ -283,28 +321,35 @@ fn run_impl(config_path: &str, dry_run: bool, resilient: bool, optimization: Opt
283321
pb.println(format!("✘ Failed to render {} output: {:#}", format, e));
284322
}
285323
}
286-
(format_str, output_path, result, new_hash)
324+
(format_str, output_path, result, new_hash, new_deps)
287325
}
288326
})
289327
.collect();
290328

291-
// Persist updated output cache for all successful renders (including skipped ones).
329+
// Persist updated output cache and dependency map for all successful renders
330+
// (including skipped ones).
292331
if !dry_run {
293-
for (_, output_path, result, new_hash) in &render_results {
332+
for (_, output_path, result, new_hash, new_deps) in &render_results {
294333
if result.is_ok() {
295334
if let Some(hash) = new_hash {
296335
output_cache.insert(output_path.clone(), hash.clone());
297336
}
337+
if let Some(deps) = new_deps {
338+
dep_map.record(output_path.clone(), deps.clone());
339+
}
298340
}
299341
}
300342
if let Err(e) = save_output_cache(&output_cache, &output_cache_path) {
301343
warn!(error = %e, "Failed to save output cache");
302344
}
345+
if let Err(e) = save_dependency_map(&dep_map, &dep_map_path) {
346+
warn!(error = %e, "Failed to save dependency map");
347+
}
303348
}
304349

305350
let failed_outputs: Vec<(String, anyhow::Error)> = render_results
306351
.into_iter()
307-
.filter_map(|(fmt, _, r, _)| r.err().map(|e| (fmt, e)))
352+
.filter_map(|(fmt, _, r, _, _)| r.err().map(|e| (fmt, e)))
308353
.collect();
309354

310355
if dry_run {

src/commands/watch.rs

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@ use notify_debouncer_mini::{new_debouncer, notify::RecursiveMode};
33
use std::path::{Path, PathBuf};
44
use std::sync::mpsc;
55
use std::time::Duration;
6-
use tracing::{error, info, warn};
6+
use tracing::{debug, error, info, warn};
77

88
use crate::config::load_config;
9+
use crate::incremental::{hash_file, load_dependency_map};
910

1011
use super::build;
1112

@@ -47,6 +48,12 @@ pub fn run(config_path: &str, debounce_ms: u64) -> Result<()> {
4748
Ok(events) => {
4849
for event in &events {
4950
info!("File changed → rebuilding... ({})", event.path.display());
51+
52+
// Use the dependency map to log which outputs are affected by
53+
// this specific file change. This gives the user (and
54+
// developers) visibility into the incremental build decisions
55+
// without changing the current full-rebuild strategy.
56+
log_affected_outputs(config_path, &event.path);
5057
}
5158
if let Err(e) = build::run_resilient(config_path) {
5259
error!("Build failed: {:#}", e);
@@ -61,6 +68,37 @@ pub fn run(config_path: &str, debounce_ms: u64) -> Result<()> {
6168
Ok(())
6269
}
6370

71+
/// Log which outputs in the dependency map are affected by a change to `changed_path`.
72+
///
73+
/// This is best-effort: if the config cannot be loaded or the dependency map
74+
/// cannot be found, the function silently returns.
75+
fn log_affected_outputs(config_path: &str, changed_path: &Path) {
76+
let Ok(config) = load_config(config_path) else { return };
77+
let output_dir = PathBuf::from(&config.output_dir);
78+
let dep_map_path = output_dir.join(".renderflow-deps.json");
79+
let dep_map = load_dependency_map(&dep_map_path);
80+
81+
let changed_str = changed_path.to_string_lossy();
82+
// Hash the changed file to compare with recorded hashes. If the file
83+
// cannot be read (e.g. it was deleted) we use an empty string so that
84+
// every recorded hash will differ, correctly marking all dependents stale.
85+
let current_hash = hash_file(changed_path).unwrap_or_default();
86+
let affected = dep_map.outputs_affected_by(&changed_str, &current_hash);
87+
88+
if affected.is_empty() {
89+
debug!(
90+
changed = %changed_str,
91+
"No tracked outputs depend on this file (or dependency map is empty)"
92+
);
93+
} else {
94+
debug!(
95+
changed = %changed_str,
96+
affected_outputs = ?affected,
97+
"Outputs affected by this file change (per dependency map)"
98+
);
99+
}
100+
}
101+
64102
/// Collect extra paths to watch beyond the config file itself.
65103
///
66104
/// Tries to load the config so that the actual input file is watched.

0 commit comments

Comments
 (0)