feat: add target catalog with CRUD scripts and targetRefs resolution

.github/workflows/ci.yml

@@ -80,12 +80,20 @@ jobs:
           sys.modules['f.shared'] = fake_f.shared
           sys.modules['f.shared.otel_logging'] = fake_f.shared.otel_logging
 
+          fake_controller_shared = FakeControllerModule()
+          fake_controller_otel = MagicMock()
+          fake_controller_otel.get_logger = lambda name: MagicMock()
+          fake_controller_shared.otel_logging = fake_controller_otel
+          sys.modules['f.controller.shared'] = fake_controller_shared
+          sys.modules['f.controller.shared.otel_logging'] = fake_controller_otel
+
           # Pre-populate all f.controller.xxx modules BEFORE any imports
           for module_name in ['config', 'database', 'breeder_service', 'breeder_create',
                               'breeder_get', 'breeder_delete', 'breeders_get',
                               'breeder_stop', 'breeder_start',
                               'credential_create', 'credential_get', 'credential_delete',
-                              'credentials_get']:
+                              'credentials_get',
+                              'target_create', 'target_get', 'targets_get', 'target_delete']:
               stub = FakeControllerModule()
               sys.modules[f'f.controller.{module_name}'] = stub
 
@@ -134,6 +142,18 @@ jobs:
           import controller.credentials_get as credentials_get
           populate_stub_module(sys.modules['f.controller.credentials_get'], credentials_get)
 
+          import controller.target_create as target_create
+          populate_stub_module(sys.modules['f.controller.target_create'], target_create)
+
+          import controller.target_get as target_get
+          populate_stub_module(sys.modules['f.controller.target_get'], target_get)
+
+          import controller.targets_get as targets_get
+          populate_stub_module(sys.modules['f.controller.targets_get'], targets_get)
+
+          import controller.target_delete as target_delete
+          populate_stub_module(sys.modules['f.controller.target_delete'], target_delete)
+
           # Test imports
           from controller.config import DatabaseConfig
           from controller.breeder_service import BreederService
@@ -144,6 +164,8 @@ jobs:
           from controller.breeders_get import main as list_breeders
           from controller.breeder_stop import main as stop_breeder
           from controller.breeder_start import main as start_breeder
+          from controller.target_create import main as create_target
+          from controller.target_get import main as get_target
 
           # Setup test config - use the actual database names
           # Meta DB connection
@@ -185,6 +207,18 @@ jobs:
           execute_ddl_query(admin_config, 'CREATE DATABASE archive_db;')
           print('✓ Created archive_db')
 
+          # Create a target first (breeder configs now require targetRefs)
+          print('Creating test target for breeder config...')
+          target_result = create_target(request_data={
+              'name': 'test-target',
+              'targetType': 'ssh',
+              'address': 'test.local',
+              'username': 'test_user'
+          })
+          assert target_result['result'] == 'SUCCESS', f'Target create failed: {target_result}'
+          test_target_id = target_result['data']['id']
+          print(f'✓ Created test target: {test_target_id}')
+
           # Test 1: Create breeder (tests schema creation + data insertion)
           print('Testing breeder creation (schema + data)...')
           breeder_config = {
@@ -203,12 +237,7 @@ jobs:
                       }
                   },
                   'effectuation': {
-                      'targets': [{
-                          'type': 'ssh',
-                          'address': 'test.local',
-                          'username': 'test_user',
-                          'credentialName': 'test-ssh-key'
-                      }]
+                      'targetRefs': [test_target_id]
                   },
                   'cooperation': {'active': False},
                   'objectives': [{'name': 'test_metric', 'goal': 'MINIMIZE'}],
@@ -395,12 +424,20 @@ jobs:
           sys.modules['f.shared'] = fake_f.shared
           sys.modules['f.shared.otel_logging'] = fake_f.shared.otel_logging
 
+          fake_controller_shared = FakeControllerModule()
+          fake_controller_otel = MagicMock()
+          fake_controller_otel.get_logger = lambda name: MagicMock()
+          fake_controller_shared.otel_logging = fake_controller_otel
+          sys.modules['f.controller.shared'] = fake_controller_shared
+          sys.modules['f.controller.shared.otel_logging'] = fake_controller_otel
+
           # Pre-populate all f.controller.xxx modules BEFORE any imports
           for module_name in ['config', 'database', 'breeder_service', 'breeder_create',
                               'breeder_get', 'breeder_delete', 'breeders_get',
                               'breeder_stop', 'breeder_start',
                               'credential_create', 'credential_get', 'credential_delete',
-                              'credentials_get']:
+                              'credentials_get',
+                              'target_create', 'target_get', 'targets_get', 'target_delete']:
               stub = FakeControllerModule()
               sys.modules[f'f.controller.{module_name}'] = stub
 
@@ -450,6 +487,18 @@ jobs:
           import controller.credentials_get as credentials_get
           populate_stub_module(sys.modules['f.controller.credentials_get'], credentials_get)
 
+          import controller.target_create as target_create
+          populate_stub_module(sys.modules['f.controller.target_create'], target_create)
+
+          import controller.target_get as target_get
+          populate_stub_module(sys.modules['f.controller.target_get'], target_get)
+
+          import controller.targets_get as targets_get
+          populate_stub_module(sys.modules['f.controller.targets_get'], targets_get)
+
+          import controller.target_delete as target_delete
+          populate_stub_module(sys.modules['f.controller.target_delete'], target_delete)
+
           # Test imports
           from controller.config import DatabaseConfig
           from controller.database import execute_query, execute_ddl_query

controller/breeder_service.py

@@ -218,6 +218,61 @@ def normalize_dict(obj):
         if 'settings' in config:
             config['settings'] = normalize_dict(config['settings'])
 
+    def _resolve_target_refs(self, breeder_config):
+        """Resolve targetRefs to inline targets from the targets catalog
+
+        If effectuation.targetRefs is present, fetches each target from
+        the metadata DB and populates effectuation.targets with the resolved data.
+        Backward compatible: if targetRefs is absent, targets are used as-is.
+
+        Args:
+            breeder_config: Breeder configuration dict (modified in place)
+
+        Raises:
+            ValueError: If any target ref cannot be resolved
+        """
+        target_refs = breeder_config.get('effectuation', {}).get('targetRefs', [])
+        if not target_refs:
+            return
+
+        logger.info(f"Resolving {len(target_refs)} target references")
+
+        effectuation = breeder_config.get('effectuation', {})
+        effectuation_type = effectuation.get('type', 'ssh')
+
+        resolved_targets = []
+        meta_db = MetadataDatabaseRepository(DatabaseConfig.META_DB)
+        meta_db.create_targets_table()
+
+        for ref_id in target_refs:
+            target_row = meta_db.fetch_target_by_id(ref_id)
+            if not target_row:
+                target_row = meta_db.fetch_target_by_name(ref_id)
+            if not target_row:
+                raise ValueError(
+                    f"Cannot resolve target reference '{ref_id}'. "
+                    f"Target not found in catalog by ID or name."
+                )
+
+            target_entry = {
+                'id': str(target_row[0]),
+                'type': target_row[2],
+                'address': target_row[3],
+            }
+
+            if target_row[4]:
+                target_entry['username'] = target_row[4]
+            if target_row[5]:
+                target_entry['credentialId'] = target_row[5]
+            if target_row[6]:
+                target_entry['description'] = target_row[6]
+
+            resolved_targets.append(target_entry)
+            logger.info(f"Resolved target ref '{ref_id}' -> {target_row[1]} ({target_row[3]})")
+
+        breeder_config['effectuation']['targets'] = resolved_targets
+        logger.info(f"Resolved {len(resolved_targets)} targets from {len(target_refs)} references")
+
     def create_breeder(self, breeder_config, name):
         """Create a new breeder instance
 
@@ -232,6 +287,8 @@ def create_breeder(self, breeder_config, name):
         breeder_id = None
 
         try:
+            self._resolve_target_refs(breeder_config)
+
             BreederConfig.validate_minimal(breeder_config)
 
             breeder_instance_name = name

controller/config.py

@@ -451,10 +451,12 @@ def validate_minimal(breeder_config, strict_mode=True):
                 "Example: objectives: [{name: 'latency', goal: 'MINIMIZE', reconnaissance: {...}}]"
             )
 
-        if not breeder_config.get('effectuation', {}).get('targets') or len(breeder_config.get('effectuation', {}).get('targets', [])) == 0:
+        target_refs = breeder_config.get('effectuation', {}).get('targetRefs')
+        if not target_refs or not isinstance(target_refs, list) or len(target_refs) == 0:
             errors.append(
-                "Missing or empty effectuation.targets array. "
-                "Example: effectuation: {targets: [{type: 'ssh', address: '1.2.3.4', ...}]}"
+                "Missing or empty effectuation.targetRefs. "
+                "Targets must be registered via the API first, then referenced by ID or name. "
+                "Example: effectuation: {targetRefs: ['my-server', '550e8400-...']}"
             )
 
         # Support multiple settings categories (sysctl, sysfs, cpufreq, ethtool)
@@ -559,64 +561,14 @@ def validate_minimal(breeder_config, strict_mode=True):
                     "Cooperation requires parallel > 1 for multiple workers to share trials."
                 )
 
-        # Validate target type compatibility and required fields
+        # Validate targetRefs
         breeder_type = breeder_config.get('breeder', {}).get('type')
-        if breeder_type in BREEDER_CAPABILITIES:
-            supported_types = BREEDER_CAPABILITIES[breeder_type]['supported_target_types']
+        target_refs = breeder_config.get('effectuation', {}).get('targetRefs', [])
 
-            for idx, target in enumerate(breeder_config.get('effectuation', {}).get('targets', [])):
-                target_type = target.get('type')
-
-                if not target_type:
-                    errors.append(f"Target {idx}: Missing required 'type' field. Example: type: 'ssh'")
-                elif target_type not in supported_types:
-                    errors.append(
-                        f"Target {idx} ({target.get('address', 'unknown')}): "
-                        f"Type '{target_type}' not supported by breeder '{breeder_type}'. "
-                        f"Supported types: {supported_types}"
-                    )
-
-                # Validate SSH-specific target fields
-                if target_type == 'ssh':
-                    # Check address
-                    if 'address' not in target:
-                        errors.append(
-                            f"Target {idx}: Missing required field 'address' for SSH target. "
-                            f"Example: address: '192.168.1.10'"
-                        )
-                    elif not isinstance(target['address'], str) or target['address'].strip() == "":
-                        errors.append(
-                            f"Target {idx}: 'address' must be a non-empty string"
-                        )
-
-                    # Check username
-                    if 'username' not in target:
-                        errors.append(
-                            f"Target {idx}: Missing required field 'username' for SSH target. "
-                            f"Example: username: 'admin'"
-                        )
-                    elif not isinstance(target['username'], str) or target['username'].strip() == "":
-                        errors.append(
-                            f"Target {idx}: 'username' must be a non-empty string"
-                        )
-
-                    # Check credential (either credentialName or credentialId)
-                    has_credential_name = 'credentialName' in target
-                    has_credential_id = 'credentialId' in target
-
-                    if not has_credential_name and not has_credential_id:
-                        errors.append(
-                            f"Target {idx}: SSH target requires either 'credentialName' or 'credentialId'. "
-                            f"Example: credentialName: 'my-ssh-key'"
-                        )
-                    elif has_credential_name and (not isinstance(target['credentialName'], str) or target['credentialName'].strip() == ""):
-                        errors.append(
-                            f"Target {idx}: 'credentialName' must be a non-empty string"
-                        )
-                    elif has_credential_id and (not isinstance(target['credentialId'], str) or target['credentialId'].strip() == ""):
-                        errors.append(
-                            f"Target {idx}: 'credentialId' must be a non-empty string"
-                        )
+        if target_refs and isinstance(target_refs, list):
+            for idx, ref in enumerate(target_refs):
+                if not isinstance(ref, str) or ref.strip() == "":
+                    errors.append(f"targetRefs[{idx}]: must be a non-empty string (target ID or name)")
 
         # Validate objectives reconnaissance configuration
         for idx, objective in enumerate(breeder_config.get('objectives', [])):

controller/database.py

@@ -142,6 +142,7 @@ def __init__(self, base_config):
         self.base_config = base_config.copy()
         self.breeder_table_name = 'breeder_meta_data'
         self.credentials_table_name = 'credentials'
+        self.targets_table_name = 'targets'
 
     def _get_db_config(self):
         """Get database config with metadata database name"""
@@ -311,6 +312,95 @@ def delete_credential(self, credential_id):
         execute_query(db_config, query)
         logger.info(f"Deleted credential catalog entry: {credential_id}")
 
+    def create_targets_table(self):
+        """Create the targets catalog table"""
+        db_config = self._get_db_config()
+
+        query = f"""
+        CREATE TABLE IF NOT EXISTS {self.targets_table_name}
+        (
+        id uuid PRIMARY KEY,
+        name VARCHAR(255) UNIQUE NOT NULL,
+        target_type VARCHAR(50) NOT NULL,
+        address VARCHAR(255) NOT NULL,
+        username VARCHAR(255),
+        credential_id VARCHAR(255),
+        description TEXT,
+        allows_downtime BOOLEAN DEFAULT FALSE,
+        metadata JSONB,
+        created_at TIMESTAMPTZ DEFAULT NOW(),
+        last_used_at TIMESTAMPTZ
+        );
+        """
+
+        execute_query(db_config, query)
+        logger.info(f"Ensured targets table exists: {self.targets_table_name}")
+
+    def insert_target(self, target_id, name, target_type, address, username=None, credential_id=None, description=None, allows_downtime=False, metadata=None):
+        """Insert target catalog entry"""
+        db_config = self._get_db_config()
+        metadata_json = json.dumps(metadata) if metadata else 'NULL'
+        description_escaped = "'" + description.replace("'", "''") + "'" if description else 'NULL'
+        username_escaped = "'" + username.replace("'", "''") + "'" if username else 'NULL'
+        credential_id_escaped = "'" + credential_id.replace("'", "''") + "'" if credential_id else 'NULL'
+
+        query = f"""
+        INSERT INTO {self.targets_table_name}
+        (id, name, target_type, address, username, credential_id, description, allows_downtime, metadata)
+        VALUES('{target_id}', '{name}', '{target_type}', '{address}', {username_escaped},
+                {credential_id_escaped}, {description_escaped}, {allows_downtime}, {metadata_json}::jsonb);
+        """
+
+        execute_query(db_config, query)
+        logger.info(f"Inserted target catalog entry: {name}")
+
+    def fetch_targets_list(self):
+        """Fetch list of all targets"""
+        db_config = self._get_db_config()
+
+        query = f"""
+        SELECT id, name, target_type, address, username, credential_id, description, allows_downtime, created_at, last_used_at
+        FROM {self.targets_table_name}
+        ORDER BY created_at DESC;
+        """
+
+        result = execute_query(db_config, query, with_result=True)
+        return result if result else []
+
+    def fetch_target_by_id(self, target_id):
+        """Fetch target by ID"""
+        db_config = self._get_db_config()
+
+        query = f"""
+        SELECT id, name, target_type, address, username, credential_id, description, allows_downtime, metadata, created_at, last_used_at
+        FROM {self.targets_table_name}
+        WHERE id = '{target_id}';
+        """
+
+        result = execute_query(db_config, query, with_result=True)
+        return result[0] if result else None
+
+    def fetch_target_by_name(self, name):
+        """Fetch target by name"""
+        db_config = self._get_db_config()
+
+        query = f"""
+        SELECT id, name, target_type, address, username, credential_id, description, allows_downtime, metadata, created_at, last_used_at
+        FROM {self.targets_table_name}
+        WHERE name = '{name}';
+        """
+
+        result = execute_query(db_config, query, with_result=True)
+        return result[0] if result else None
+
+    def delete_target(self, target_id):
+        """Delete target from catalog"""
+        db_config = self._get_db_config()
+
+        query = f"DELETE FROM {self.targets_table_name} WHERE id = '{target_id}';"
+        execute_query(db_config, query)
+        logger.info(f"Deleted target catalog entry: {target_id}")
+
     def update_credential_last_used(self, credential_id):
         """Update the last_used_at timestamp for a credential"""
         db_config = self._get_db_config()