Merge pull request #63 from foundersandcoders/claude/extract-utility-library-UTRy9

Refactor ILR mapping logic into dedicated modules

Author: JasonWarrenUK

.github/workflows/sync-engine.yml

@@ -0,0 +1,90 @@
+name: Sync engine to schema-forge
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "src/lib/schema/**"
+      - "src/lib/types/interpreterTypes.ts"
+      - "src/lib/types/schemaTypes.ts"
+      - "src/lib/transforms/**"
+      - "src/lib/utils/schema/**"
+      - "src/lib/utils/csv/**"
+      - "src/lib/utils/xml/**"
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Iris
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Check for engine changes
+        id: changes
+        run: |
+          ENGINE_PATHS=(
+            "src/lib/schema"
+            "src/lib/types/interpreterTypes.ts"
+            "src/lib/types/schemaTypes.ts"
+            "src/lib/transforms"
+            "src/lib/utils/schema"
+            "src/lib/utils/csv"
+            "src/lib/utils/xml"
+          )
+
+          CHANGED=false
+          for path in "${ENGINE_PATHS[@]}"; do
+            if git diff --name-only HEAD~1 HEAD -- "$path" | grep -q .; then
+              CHANGED=true
+              break
+            fi
+          done
+
+          echo "changed=$CHANGED" >> "$GITHUB_OUTPUT"
+
+      - name: Clone schema-forge
+        if: steps.changes.outputs.changed == 'true'
+        run: |
+          git clone https://x-access-token:${{ secrets.SCHEMA_FORGE_PAT }}@github.com/JasonWarrenUK/schema-forge.git /tmp/schema-forge
+
+      - name: Sync engine files
+        if: steps.changes.outputs.changed == 'true'
+        run: |
+          TARGET=/tmp/schema-forge/src
+
+          # Clear previous engine content (full replace, not merge)
+          rm -rf "$TARGET"
+          mkdir -p "$TARGET"
+
+          # Copy engine modules preserving structure
+          cp -r src/lib/schema      "$TARGET/schema"
+          cp -r src/lib/transforms  "$TARGET/transforms"
+          cp -r src/lib/utils/schema "$TARGET/utils/schema"
+          cp -r src/lib/utils/csv   "$TARGET/utils/csv"
+          cp -r src/lib/utils/xml   "$TARGET/utils/xml"
+
+          mkdir -p "$TARGET/types"
+          cp src/lib/types/interpreterTypes.ts "$TARGET/types/"
+          cp src/lib/types/schemaTypes.ts      "$TARGET/types/"
+
+      - name: Commit and push
+        if: steps.changes.outputs.changed == 'true'
+        working-directory: /tmp/schema-forge
+        run: |
+          git config user.name "iris-sync[bot]"
+          git config user.email "iris-sync[bot]@users.noreply.github.com"
+
+          git add -A
+          if git diff --cached --quiet; then
+            echo "No changes to commit"
+            exit 0
+          fi
+
+          COMMIT_SHA="${{ github.sha }}"
+          git commit -m "sync: update engine from iris@${COMMIT_SHA:0:7}
+
+          Source: ${{ github.server_url }}/${{ github.repository }}/commit/${COMMIT_SHA}"
+          git push

bun.lock

@@ -7,6 +7,7 @@ import type { MappingConfig } from '../types/schemaTypes';
 import type { IrisStorage } from '../storage';
 import { buildSchemaRegistry } from '../schema/registryBuilder';
 import { validateSchemaCompatibility, type CompatibilityResult } from '../schema/schemaCompatibility';
+import { FAM_PATHS, APP_FIN_PATHS, LLDD_PATHS, EMPLOYMENT_PATHS } from './builderPaths';
 
 export interface MappingCompatibilityResult {
 	success: boolean;
@@ -45,8 +46,16 @@ export async function validateMappingCompatibility(params: {
 		};
 	}
 
+	// Compute ILR-specific builder paths to validate
+	const builderPaths = [
+		...(mapping.famTemplates?.length ? FAM_PATHS : []),
+		...(mapping.appFinTemplates?.length ? APP_FIN_PATHS : []),
+		...(mapping.employmentStatuses?.length ? EMPLOYMENT_PATHS : []),
+		...LLDD_PATHS,
+	];
+
 	// Validate compatibility
-	const compatibility = validateSchemaCompatibility(mapping, registry);
+	const compatibility = validateSchemaCompatibility(mapping, registry, builderPaths);
 
 	return {
 		success: true,

src/lib/utils/config/mapping.ts src/lib/mappings/aimUtils.tssrc/lib/utils/config/mapping.ts renamed to src/lib/mappings/aimUtils.ts

@@ -5,7 +5,7 @@
  */
 
 import type { MappingConfig } from '../types/schemaTypes';
-import { generateAimMappings } from '../utils/config/mapping';
+import { generateAimMappings } from './aimUtils';
 
 export const facAirtableMapping: MappingConfig = {
 	id: 'fac-airtable-2025',

src/lib/mappings/compatibility.ts

@@ -0,0 +1,123 @@
+import type { ColumnMapping, MappingConfig } from '../types/schemaTypes';
+import type { SchemaRegistry } from '../types/interpreterTypes';
+import { mapCsvToSchema } from '../schema/columnMapper';
+import { getTransform } from '../transforms/registry';
+import { hasAimData } from './aimUtils';
+import {
+	buildFamEntries,
+	buildAppFinRecords,
+	buildEmploymentStatuses,
+	buildLLDDFields,
+} from './builders';
+import { generateUUID } from '../utils/uuid';
+
+/**
+ * Maps a single CSV row to ILR structure with support for multiple LearningDelivery aims
+ *
+ * Detects which aims (1-5) have data and creates separate LearningDelivery elements for each.
+ * Auto-generates AimSeqNumber based on the aim group number.
+ *
+ * @param csvRow - Object with CSV column headers as keys
+ * @param config - Mapping configuration with aim-grouped mappings
+ * @param registry - Schema registry for path validation
+ * @returns Nested object structure with multiple LearningDelivery elements
+ */
+export function mapCsvToSchemaWithAims(
+	csvRow: Record<string, string>,
+	config: MappingConfig,
+	registry: SchemaRegistry
+): Record<string, unknown> {
+	// Separate learner-level and aim-specific mappings
+	const learnerMappings = config.mappings.filter((m) => !m.aimNumber);
+	const aimMappings = config.mappings.filter((m) => m.aimNumber);
+
+	// Group aim mappings by aim number
+	const aimGroups = new Map<number, ColumnMapping[]>();
+	for (const mapping of aimMappings) {
+		if (!mapping.aimNumber) continue;
+		if (!aimGroups.has(mapping.aimNumber)) {
+			aimGroups.set(mapping.aimNumber, []);
+		}
+		aimGroups.get(mapping.aimNumber)!.push(mapping);
+	}
+
+	// Apply learner-level mappings using generic mapper
+	const result = mapCsvToSchema(csvRow, learnerMappings, registry);
+
+	// Build LLDD fields with conditional logic
+	const llddFields = buildLLDDFields(csvRow, 'Primary additional needs');
+	const messagePart = result.Message as Record<string, unknown>;
+	if (messagePart && messagePart.Learner) {
+		const learnerArray = messagePart.Learner as Record<string, unknown>[];
+		if (learnerArray.length > 0) {
+			learnerArray[0].LLDDHealthProb = llddFields.LLDDHealthProb;
+			if (llddFields.LLDDandHealthProblem) {
+				learnerArray[0].LLDDandHealthProblem = llddFields.LLDDandHealthProblem;
+			}
+		}
+	}
+
+	// Build employment status entries
+	const statuses = buildEmploymentStatuses(csvRow, config.employmentStatuses);
+	if (statuses.length > 0) {
+		const messagePart = result.Message as Record<string, unknown>;
+		if (messagePart && messagePart.Learner) {
+			const learnerArray = messagePart.Learner as Record<string, unknown>[];
+			if (learnerArray.length > 0) learnerArray[0].LearnerEmploymentStatus = statuses;
+		}
+	}
+
+	// Detect which aims have data and build LearningDelivery elements
+	const detectionField = config.aimDetectionField || 'Programme aim {n} Learning ref';
+	const learningDeliveries: Record<string, unknown>[] = [];
+
+	for (const [aimNumber, mappings] of aimGroups.entries()) {
+		if (!hasAimData(csvRow, aimNumber, detectionField)) continue;
+
+		const delivery: Record<string, unknown> = { AimSeqNumber: aimNumber };
+
+		for (const mapping of mappings) {
+			const columnKey = Object.keys(csvRow).find(
+				(key) => key.trim().toLowerCase() === mapping.csvColumn.trim().toLowerCase()
+			);
+
+			if (!columnKey) continue;
+
+			const rawValue = csvRow[columnKey];
+			const value = mapping.transform ? getTransform(mapping.transform)(rawValue) : rawValue;
+
+			// Extract just the field name from the full path
+			// e.g., "Message.Learner.LearningDelivery.LearnAimRef" -> "LearnAimRef"
+			const pathParts = mapping.xsdPath.split('.');
+			const fieldName = pathParts[pathParts.length - 1];
+
+			delivery[fieldName] = value;
+		}
+
+		// Generate SWSupAimId (software-generated UUID for tracking)
+		delivery.SWSupAimId = generateUUID();
+
+		// Build FAM entries for this aim
+		const fams = buildFamEntries(csvRow, config.famTemplates, aimNumber);
+		if (fams.length > 0) delivery.LearningDeliveryFAM = fams;
+
+		// Build AppFinRecord entries for this aim
+		const fins = buildAppFinRecords(csvRow, config.appFinTemplates, aimNumber);
+		if (fins.length > 0) delivery.AppFinRecord = fins;
+
+		learningDeliveries.push(delivery);
+	}
+
+	// Inject LearningDelivery array into result structure
+	if (learningDeliveries.length > 0) {
+		const messagePart = result.Message as Record<string, unknown>;
+		if (messagePart && messagePart.Learner) {
+			const learnerArray = messagePart.Learner as Record<string, unknown>[];
+			if (learnerArray.length > 0) {
+				learnerArray[0].LearningDelivery = learningDeliveries;
+			}
+		}
+	}
+
+	return result;
+}

src/lib/mappings/fac-airtable-2025.ts

@@ -0,0 +1,18 @@
+import type { ColumnMapping } from '../types/schemaTypes';
+import { hasAimData } from './aimUtils';
+
+/**
+ * Creates a skip filter for aim-specific mappings during CSV validation.
+ *
+ * - At header level (no row): skips all aim-specific mappings (validated per-row instead)
+ * - At row level: skips aim mappings where the aim has no data
+ *
+ * @param detectionField - Field template with {n} placeholder for aim detection
+ */
+export function createAimSkipFilter(detectionField: string = 'Programme aim {n} Learning ref') {
+	return (mapping: ColumnMapping, row?: Record<string, string>): boolean => {
+		if (!mapping.aimNumber) return false;
+		if (!row) return true; // header-level: skip all aim mappings
+		return !hasAimData(row, mapping.aimNumber, detectionField);
+	};
+}