implement more required features

Author: ndbroadbent

plugins/provider-circleci/src/main.rs

@@ -128,6 +128,12 @@ message JobDefinition {
   repeated string source_files = 13;   // Expanded source file patterns
   repeated PackageSpec package_specs = 14;
   repeated string services = 15;       // Declared service containers
+  repeated MatrixRow matrix_rows = 16; // Explicit matrix rows (alternative to dimensions)
+  string stage = 17;                   // Stage this job belongs to
+}
+
+message MatrixRow {
+  map<string, string> values = 1;
 }
 
 message MatrixValue {

proto/plugin.proto

@@ -154,43 +154,66 @@ fn load_jobs_and_workflows(config_dir: &Path, config: &mut CigenConfig) -> Resul
 
             let workflow_id = workflow_name.to_string();
             let workflow_config = load_workflow_config(&workflow_path)?;
-            if !config.workflows.contains_key(&workflow_id) {
-                config
-                    .workflows
-                    .insert(workflow_id.clone(), workflow_config);
-            }
+            // Insert workflow config if not already present (load_workflow_config returns default if not found)
+            // We want to insert it even if default to track the workflow existence.
+            // But we should check if we already loaded it from a file in the loop below?
+            // Actually, the loop handles directories first, then files.
+            // If we have a directory, we load config from inside it.
+            // If we check config.workflows first, we might skip loading/overwriting?
+            // Let's just insert/update.
+            config
+                .workflows
+                .insert(workflow_id.clone(), workflow_config.clone());
 
             let jobs_dir = workflow_path.join("jobs");
             if !jobs_dir.exists() {
                 continue;
             }
 
-            for job_file in fs::read_dir(&jobs_dir)? {
-                let job_file = job_file?;
-                let job_path = job_file.path();
-
-                if !matches!(
-                    job_path.extension().and_then(|s| s.to_str()),
-                    Some("yml" | "yaml")
-                ) {
-                    continue;
+            // Stack for recursive traversal: (current_path, stage_name)
+            // If stage_name is None, we are at root jobs dir.
+            let mut stack = vec![(jobs_dir.clone(), None::<String>)];
+
+            while let Some((dir, current_stage)) = stack.pop() {
+                for entry in fs::read_dir(&dir)? {
+                    let entry = entry?;
+                    let path = entry.path();
+
+                    if path.is_dir() {
+                        let dir_name = path
+                            .file_name()
+                            .and_then(|s| s.to_str())
+                            .unwrap_or_default()
+                            .to_string();
+                        
+                        let next_stage = current_stage.clone().or(Some(dir_name));
+                        stack.push((path, next_stage));
+                    } else if matches!(
+                        path.extension().and_then(|s| s.to_str()),
+                        Some("yml" | "yaml")
+                    ) {
+                        let stage = current_stage.clone().unwrap_or_else(|| "default".to_string());
+                        
+                        // Use path relative to jobs_dir as job_id
+                        let relative_path = path.strip_prefix(&jobs_dir).unwrap_or(&path);
+                        let job_id = relative_path.with_extension("").to_string_lossy().to_string();
+
+                        let job_yaml = fs::read_to_string(&path)?;
+                        let mut job: Job = serde_yaml::from_str(&job_yaml)
+                            .with_context(|| format!("Failed to parse {}", path.display()))?;
+
+                        job.workflow = Some(workflow_name.to_string());
+                        job.stage = Some(stage.clone());
+                        migrate_requires_to_needs(&mut job);
+
+                        config.jobs.insert(job_id, job);
+                    }
                 }
-
-                let job_name = job_path
-                    .file_stem()
-                    .and_then(|s| s.to_str())
-                    .context("Invalid job filename")?
-                    .to_string();
-
-                let job_yaml = fs::read_to_string(&job_path)?;
-                let mut job: Job = serde_yaml::from_str(&job_yaml)
-                    .with_context(|| format!("Failed to parse {}", job_path.display()))?;
-
-                job.workflow = Some(workflow_name.to_string());
-                migrate_requires_to_needs(&mut job);
-
-                config.jobs.insert(job_name, job);
             }
+            
+            // Resolve dependencies (siblings)
+            resolve_job_dependencies(&mut config.jobs);
+
         } else if matches!(
             workflow_path.extension().and_then(|s| s.to_str()),
             Some("yml" | "yaml")
@@ -212,6 +235,40 @@ fn load_jobs_and_workflows(config_dir: &Path, config: &mut CigenConfig) -> Resul
     Ok(())
 }
 
+fn resolve_job_dependencies(jobs: &mut HashMap<String, Job>) {
+    let job_keys: Vec<String> = jobs.keys().cloned().collect();
+    
+    for (job_id, job) in jobs.iter_mut() {
+        for need in &mut job.needs {
+            // If need is already a valid key, skip
+            if job_keys.contains(need) {
+                continue;
+            }
+            
+            // Try to resolve as sibling
+            let parent_dir = Path::new(job_id).parent().unwrap_or(Path::new(""));
+            let sibling_path = parent_dir.join(need.as_str());
+            let sibling_key = sibling_path.to_string_lossy().to_string();
+            
+            if job_keys.contains(&sibling_key) {
+                *need = sibling_key;
+                continue;
+            }
+            
+            // Check if it's a simple name that exists elsewhere (ambiguous but maybe intended?)
+            // For now, only support siblings or full paths.
+        }
+    }
+}
+
+fn should_prefix_job(config: &WorkflowConfig, stage: &str) -> bool {
+    if stage == "default" {
+        config.default_stage_prefix
+    } else {
+        config.stage_prefix
+    }
+}
+
 fn load_workflow_config(workflow_path: &Path) -> Result<WorkflowConfig> {
     for candidate in ["config.yml", "config.yaml"] {
         let candidate_path = workflow_path.join(candidate);

src/loader.rs

@@ -2,11 +2,11 @@ use std::collections::HashMap;
 
 use crate::plugin::protocol::{
     self, CacheDefinition, CigenSchema, CommandDefinition as ProtoCommandDefinition,
-    CommandParameter as ProtoCommandParameter, CustomStep, JobDefinition, MatrixValue,
+    CommandParameter as ProtoCommandParameter, CustomStep, JobDefinition, MatrixRow, MatrixValue,
     PackageSpec as ProtoPackageSpec, ProjectConfig, RestoreCacheStep, RunStep, RunnerDefinition,
-    SaveCacheStep, SkipConfig, Step, StringList, UsesStep, WorkflowDefinition,
+    SaveCacheStep, SkipConfig, Step, StringList, UsesStep, WorkflowConditionKind as ProtoWorkflowConditionKind, WorkflowDefinition,
 };
-use crate::schema;
+use crate::schema::{self, JobMatrix};
 use serde_yaml::Value;
 
 /// Convert schema::CigenConfig to protobuf CigenSchema
@@ -100,14 +100,27 @@ fn command_parameter_to_proto(parameter: &schema::CommandParameter) -> ProtoComm
 }
 
 fn job_to_proto(id: &str, job: &schema::Job) -> JobDefinition {
+    let (matrix_dimensions_map, matrix_rows_vec) = match &job.matrix {
+        Some(JobMatrix::Dimensions(dims)) => (
+            dims.iter()
+                .map(|(key, value)| (key.clone(), MatrixValue { values: value.clone() }))
+                .collect(),
+            Vec::new(),
+        ),
+        Some(JobMatrix::Explicit(rows)) => (
+            HashMap::new(),
+            rows.iter()
+                .map(|row| MatrixRow { values: row.clone() })
+                .collect(),
+        ),
+        None => (HashMap::new(), Vec::new()),
+    };
+
     JobDefinition {
         id: id.to_string(),
         needs: job.needs.clone(),
-        matrix: job
-            .matrix
-            .iter()
-            .map(|(key, value)| (key.clone(), matrix_value_to_proto(value)))
-            .collect(),
+        matrix: matrix_dimensions_map,
+        matrix_rows: matrix_rows_vec,
         packages: job.packages.iter().map(|pkg| pkg.name.clone()).collect(),
         steps: job.steps.iter().map(step_to_proto).collect(),
         skip_if: job.skip_if.as_ref().map(skip_config_to_proto),
@@ -133,6 +146,7 @@ fn job_to_proto(id: &str, job: &schema::Job) -> JobDefinition {
         source_files: job.source_files.clone(),
         package_specs: job.packages.iter().map(package_to_proto).collect(),
         services: job.services.clone(),
+        stage: job.stage.clone().unwrap_or_default(),
     }
 }
 
@@ -156,12 +170,12 @@ fn workflow_condition_to_proto(
         .unwrap_or(schema::WorkflowConditionKind::Parameter)
     {
         schema::WorkflowConditionKind::Parameter => {
-            protocol::WorkflowConditionKind::Parameter as i32
+            ProtoWorkflowConditionKind::Parameter as i32
         }
-        schema::WorkflowConditionKind::Variable => protocol::WorkflowConditionKind::Variable as i32,
-        schema::WorkflowConditionKind::Env => protocol::WorkflowConditionKind::Env as i32,
+        schema::WorkflowConditionKind::Variable => ProtoWorkflowConditionKind::Variable as i32,
+        schema::WorkflowConditionKind::Env => ProtoWorkflowConditionKind::Env as i32,
         schema::WorkflowConditionKind::Expression => {
-            protocol::WorkflowConditionKind::Expression as i32
+            ProtoWorkflowConditionKind::Expression as i32
         }
     };
 
@@ -188,13 +202,7 @@ fn package_to_proto(package: &schema::PackageSpec) -> ProtoPackageSpec {
     }
 }
 
-fn matrix_value_to_proto(value: &schema::MatrixDimension) -> MatrixValue {
-    match value {
-        schema::MatrixDimension::List(values) => MatrixValue {
-            values: values.clone(),
-        },
-    }
-}
+
 
 fn step_to_proto(step: &schema::Step) -> Step {
     match step {
@@ -345,7 +353,7 @@ mod tests {
             "test".to_string(),
             schema::Job {
                 needs: vec![],
-                matrix: HashMap::new(),
+                matrix: None, // Updated for Option<JobMatrix>
                 packages: vec![schema::PackageSpec::from_name("ruby".to_string())],
                 services: vec![],
                 environment: HashMap::new(),