Skip to content

Commit 59565c6

Browse files
ndbroadbentndbroadbent
committed
added support for submodule hashes
1 parent 1e0075f commit 59565c6

File tree

11 files changed

+338
-2
lines changed

11 files changed

+338
-2
lines changed

‎schemas/v1/job-schema.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,22 @@
4848
}
4949
]
5050
},
51+
"source_submodules": {
52+
"oneOf": [
53+
{
54+
"type": "string",
55+
"description": "Single git submodule path to include when hashing source files"
56+
},
57+
{
58+
"type": "array",
59+
"description": "List of git submodule paths whose commit hashes are tracked",
60+
"items": {
61+
"type": "string"
62+
},
63+
"minItems": 1
64+
}
65+
]
66+
},
5167
"parallelism": {
5268
"type": "integer",
5369
"description": "Number of parallel instances of this job",

‎src/graph/dependency_graph.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ mod tests {
184184
architectures: None,
185185
resource_class: None,
186186
source_files: None,
187+
source_submodules: None,
187188
parallelism: None,
188189
requires: requires.map(|deps| {
189190
if deps.len() == 1 {

‎src/models/job.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,13 @@ pub struct Job {
3636
)]
3737
pub source_files: Option<Vec<String>>,
3838

39+
#[serde(
40+
skip_serializing_if = "Option::is_none",
41+
default,
42+
deserialize_with = "deserialize_string_or_vec"
43+
)]
44+
pub source_submodules: Option<Vec<String>>,
45+
3946
#[serde(skip_serializing_if = "Option::is_none")]
4047
pub parallelism: Option<u32>,
4148

‎src/packages/deduplicator.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ impl PackageDeduplicator {
9595
architectures: None,
9696
resource_class: None,
9797
source_files: None,
98+
source_submodules: None,
9899
parallelism: None,
99100
requires: None,
100101
cache: None,
@@ -249,6 +250,7 @@ mod tests {
249250
architectures: None,
250251
resource_class: None,
251252
source_files: None,
253+
source_submodules: None,
252254
parallelism: None,
253255
requires: None,
254256
cache: None,
@@ -270,6 +272,7 @@ mod tests {
270272
architectures: None,
271273
resource_class: None,
272274
source_files: None,
275+
source_submodules: None,
273276
parallelism: None,
274277
requires: None,
275278
cache: None,
@@ -291,6 +294,7 @@ mod tests {
291294
architectures: None,
292295
resource_class: None,
293296
source_files: None,
297+
source_submodules: None,
294298
parallelism: None,
295299
requires: None,
296300
cache: None,
@@ -380,6 +384,7 @@ mod tests {
380384
architectures: None,
381385
resource_class: None,
382386
source_files: None,
387+
source_submodules: None,
383388
parallelism: None,
384389
requires: None,
385390
cache: None,

‎src/providers/circleci/generator.rs‎

Lines changed: 126 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,7 @@ impl CircleCIGenerator {
373373
architectures: None,
374374
resource_class: None,
375375
source_files: None,
376+
source_submodules: None,
376377
parallelism: None,
377378
requires: None,
378379
cache: None,
@@ -462,7 +463,47 @@ fi"#,
462463
job_name.clone()
463464
};
464465
// Hash calculation (reuse existing function)
465-
let hash_step = self.build_hash_calculation_step(config, source_files)?;
466+
let mut extra_patterns: Vec<String> = Vec::new();
467+
if let Some(submodules) = &job_def.source_submodules {
468+
for submodule in submodules {
469+
let output_file =
470+
Self::submodule_hash_output_path(&variant, submodule);
471+
472+
let mut params = serde_yaml::Mapping::new();
473+
params.insert(
474+
serde_yaml::Value::String("submodule".to_string()),
475+
serde_yaml::Value::String(submodule.clone()),
476+
);
477+
params.insert(
478+
serde_yaml::Value::String("output_file".to_string()),
479+
serde_yaml::Value::String(output_file.clone()),
480+
);
481+
482+
let mut step_map = serde_yaml::Mapping::new();
483+
step_map.insert(
484+
serde_yaml::Value::String(
485+
"cigen_write_submodule_commit_hash".to_string(),
486+
),
487+
serde_yaml::Value::Mapping(params),
488+
);
489+
490+
setup_job.steps.push(cc::CircleCIStep::new(
491+
serde_yaml::Value::Mapping(step_map),
492+
));
493+
494+
extra_patterns.push(output_file);
495+
}
496+
}
497+
498+
let hash_step = self.build_hash_calculation_step(
499+
config,
500+
source_files,
501+
if extra_patterns.is_empty() {
502+
None
503+
} else {
504+
Some(&extra_patterns)
505+
},
506+
)?;
466507
setup_job.steps.push(cc::CircleCIStep::new(hash_step));
467508
// Restore exists cache with OS + variant + job hash
468509
let mut restore_cfg = serde_yaml::Mapping::new();
@@ -933,6 +974,7 @@ cat /tmp/continuation.json
933974
architectures: Some(archs.iter().cloned().collect()),
934975
resource_class: None,
935976
source_files: None,
977+
source_submodules: None,
936978
parallelism: None,
937979
requires: None,
938980
cache: None,
@@ -1588,6 +1630,8 @@ cat /tmp/continuation.json
15881630
&mut circleci_job,
15891631
config,
15901632
source_files,
1633+
job.source_submodules.as_ref(),
1634+
job_name,
15911635
architecture,
15921636
)?;
15931637

@@ -2423,10 +2467,50 @@ echo "export JOB_STATUS_KEY=\"${{OS_LABEL}}-job_status-v1-{job}-{arch}-${{JOB_HA
24232467
circleci_job: &mut CircleCIJob,
24242468
config: &Config,
24252469
source_files: &Vec<String>,
2470+
submodules: Option<&Vec<String>>,
2471+
job_name: &str,
24262472
architecture: &str,
24272473
) -> Result<()> {
2474+
let mut unfiltered_patterns: Vec<String> = Vec::new();
2475+
2476+
if let Some(submodules) = submodules {
2477+
for submodule in submodules {
2478+
let output_file = Self::submodule_hash_output_path(job_name, submodule);
2479+
2480+
let mut params = serde_yaml::Mapping::new();
2481+
params.insert(
2482+
serde_yaml::Value::String("submodule".to_string()),
2483+
serde_yaml::Value::String(submodule.clone()),
2484+
);
2485+
params.insert(
2486+
serde_yaml::Value::String("output_file".to_string()),
2487+
serde_yaml::Value::String(output_file.clone()),
2488+
);
2489+
2490+
let mut step_map = serde_yaml::Mapping::new();
2491+
step_map.insert(
2492+
serde_yaml::Value::String("cigen_write_submodule_commit_hash".to_string()),
2493+
serde_yaml::Value::Mapping(params),
2494+
);
2495+
2496+
circleci_job
2497+
.steps
2498+
.push(CircleCIStep::new(serde_yaml::Value::Mapping(step_map)));
2499+
2500+
unfiltered_patterns.push(output_file);
2501+
}
2502+
}
2503+
24282504
// Add hash calculation step
2429-
let hash_step = self.build_hash_calculation_step(config, source_files)?;
2505+
let hash_step = self.build_hash_calculation_step(
2506+
config,
2507+
source_files,
2508+
if unfiltered_patterns.is_empty() {
2509+
None
2510+
} else {
2511+
Some(&unfiltered_patterns)
2512+
},
2513+
)?;
24302514
circleci_job.steps.push(CircleCIStep::new(hash_step));
24312515

24322516
// Add skip check step
@@ -2440,6 +2524,7 @@ echo "export JOB_STATUS_KEY=\"${{OS_LABEL}}-job_status-v1-{job}-{arch}-${{JOB_HA
24402524
&self,
24412525
config: &Config,
24422526
source_files: &Vec<String>,
2527+
unfiltered_patterns: Option<&Vec<String>>,
24432528
) -> Result<serde_yaml::Value> {
24442529
let source_file_groups = config
24452530
.source_file_groups
@@ -2476,6 +2561,14 @@ echo "export JOB_STATUS_KEY=\"${{OS_LABEL}}-job_status-v1-{job}-{arch}-${{JOB_HA
24762561
serde_yaml::Value::String("patterns".to_string()),
24772562
serde_yaml::Value::String(pat_str),
24782563
);
2564+
if let Some(extra_patterns) = unfiltered_patterns
2565+
&& !extra_patterns.is_empty()
2566+
{
2567+
params.insert(
2568+
serde_yaml::Value::String("unfiltered_patterns".to_string()),
2569+
serde_yaml::Value::String(extra_patterns.join(" ")),
2570+
);
2571+
}
24792572
let mut step_map = serde_yaml::Mapping::new();
24802573
step_map.insert(
24812574
serde_yaml::Value::String("cigen_calculate_sha256".to_string()),
@@ -2484,6 +2577,37 @@ echo "export JOB_STATUS_KEY=\"${{OS_LABEL}}-job_status-v1-{job}-{arch}-${{JOB_HA
24842577
Ok(serde_yaml::Value::Mapping(step_map))
24852578
}
24862579

2580+
fn sanitize_identifier(input: &str) -> String {
2581+
let mut sanitized = String::new();
2582+
let mut last_was_dash = false;
2583+
2584+
for ch in input.chars() {
2585+
if ch.is_ascii_alphanumeric() {
2586+
sanitized.push(ch);
2587+
last_was_dash = false;
2588+
} else if !last_was_dash {
2589+
sanitized.push('-');
2590+
last_was_dash = true;
2591+
}
2592+
}
2593+
2594+
let trimmed = sanitized.trim_matches('-');
2595+
if trimmed.is_empty() {
2596+
"entry".to_string()
2597+
} else {
2598+
trimmed.to_string()
2599+
}
2600+
}
2601+
2602+
fn submodule_hash_output_path(job_identifier: &str, submodule: &str) -> String {
2603+
let job_part = Self::sanitize_identifier(job_identifier);
2604+
let module_part = Self::sanitize_identifier(submodule);
2605+
format!(
2606+
"/tmp/cigen/submodule-hashes/{}-{}.commit",
2607+
job_part, module_part
2608+
)
2609+
}
2610+
24872611
fn build_skip_check_step(
24882612
&self,
24892613
_config: &Config,

‎src/providers/circleci/template_commands.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,15 @@ steps:
6262
.expect("Failed to parse shallow_checkout template");
6363
commands.insert("cigen_shallow_checkout".to_string(), shallow_checkout);
6464

65+
// write_submodule_commit_hash command - ensure submodule changes trigger job re-runs
66+
let write_submodule_commit_hash =
67+
serde_yaml::from_str(include_str!("templates/write_submodule_commit_hash.yml"))
68+
.expect("Failed to parse write_submodule_commit_hash template");
69+
commands.insert(
70+
"cigen_write_submodule_commit_hash".to_string(),
71+
write_submodule_commit_hash,
72+
);
73+
6574
// cigen_calculate_sha256 command - efficient hashing with per-pattern caching
6675
let calculate_hash = serde_yaml::from_str(
6776
r#"
@@ -245,6 +254,7 @@ mod tests {
245254
fn test_template_commands_loaded() {
246255
assert!(is_template_command("continue_circleci_pipeline"));
247256
assert!(is_template_command("cigen_shallow_checkout"));
257+
assert!(is_template_command("cigen_write_submodule_commit_hash"));
248258
assert!(!is_template_command("unknown_command"));
249259
}
250260

‎src/providers/circleci/templates/write_submodule_commit_hash.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
description: "Write git submodule commit hash to a file"
2+
parameters:
3+
submodule:
4+
type: string
5+
description: "Path to the git submodule"
6+
output_file:
7+
type: string
8+
description: "File path to write the commit hash"
9+
steps:
10+
- run:
11+
name: Record commit hash for << parameters.submodule >>
12+
command: |
13+
set -eu
14+
SUBMODULE_PATH="<< parameters.submodule >>"
15+
OUTPUT_FILE="<< parameters.output_file >>"
16+
17+
if [ -z "$SUBMODULE_PATH" ]; then
18+
echo "Submodule path is required" >&2
19+
exit 1
20+
fi
21+
22+
TMP_FILE="$(mktemp /tmp/cigen-submodule-hash.XXXXXX)"
23+
if ! git ls-tree HEAD -- "$SUBMODULE_PATH" > "$TMP_FILE" 2>"${TMP_FILE}.err"; then
24+
echo "Failed to read commit hash for submodule $SUBMODULE_PATH" >&2
25+
cat "${TMP_FILE}.err" >&2 || true
26+
rm -f "$TMP_FILE" "${TMP_FILE}.err"
27+
exit 1
28+
fi
29+
30+
COMMIT_HASH=$(awk 'NR==1 {print $3}' "$TMP_FILE")
31+
rm -f "$TMP_FILE" "${TMP_FILE}.err"
32+
33+
if [ -z "$COMMIT_HASH" ]; then
34+
echo "Failed to parse commit hash for submodule $SUBMODULE_PATH" >&2
35+
exit 1
36+
fi
37+
38+
OUTPUT_DIR=$(dirname "$OUTPUT_FILE")
39+
if [ -n "$OUTPUT_DIR" ] && [ ! -d "$OUTPUT_DIR" ]; then
40+
mkdir -p "$OUTPUT_DIR"
41+
fi
42+
43+
printf '%s' "$COMMIT_HASH" > "$OUTPUT_FILE"

0 commit comments

Comments
 (0)