feat: import license tags from licenses-catalog and associate them with licenses (#1614)

Co-authored-by: davidgamez <1192523+davidgamez@users.noreply.github.com>

Author: Copilot

functions-python/tasks_executor/README.md

@@ -68,17 +68,6 @@ To update the geolocation files precision:
 }
 ```
 
-To populate license rules:
-
-```json
-{
-  "task": "populate_license_rules",
-  "payload": {
-    "dry_run": true
-  }
-}
-```
-
 To populate licenses:
 
 ```json

functions-python/tasks_executor/src/main.py

@@ -47,14 +47,12 @@
 )
 from tasks.data_import.jbda.import_jbda_feeds import import_jbda_handler
 
-from tasks.licenses.populate_license_rules import (
-    populate_license_rules_handler,
-)
 
 from tasks.licenses.populate_licenses import (
     populate_licenses_handler,
 )
 
+
 init_logger()
 LIST_COMMAND: Final[str] = "list"
 tasks = {
@@ -97,12 +95,9 @@
         "description": "Imports JBDA data into the system.",
         "handler": import_jbda_handler,
     },
-    "populate_license_rules": {
-        "description": "Populates license rules in the database from a predefined JSON source.",
-        "handler": populate_license_rules_handler,
-    },
     "populate_licenses": {
-        "description": "Populates licenses and license-rules in the database from a predefined JSON source.",
+        "description": "Populates licenses, license-rules and license-tags "
+        "in the database from a predefined JSON source.",
         "handler": populate_licenses_handler,
     },
     "match_licenses": {

functions-python/tasks_executor/src/tasks/licenses/populate_license_rules.py

@@ -26,28 +26,16 @@
 )
 
 
-def populate_license_rules_handler(payload):
-    """
-    Handler for populating license rules.
-
-    Args:
-        payload (dict): Incoming payload data.
-
-    """
-    (dry_run) = get_parameters(payload)
-    return populate_license_rules_task(dry_run)
-
-
 @with_db_session
-def populate_license_rules_task(dry_run, db_session):
+def populate_license_rules(dry_run, db_session):
     """
     Populates license rules in the database. This function is triggered by a Cloud Task.
 
     Args:
         dry_run (bool): If True, the function will simulate the operation without making changes.
         db_session: Database session for executing queries.
     """
-    logging.info("Starting populate_license_rules_task with dry_run=%s", dry_run)
+    logging.info("Starting populate_license_rules with dry_run=%s", dry_run)
 
     try:
         logging.info("Downloading rules from %s", RULES_JSON_URL)
@@ -72,9 +60,10 @@ def populate_license_rules_task(dry_run, db_session):
         logging.info(
             "Loaded %d rules from %d categories.", len(rules_data), len(rules_json)
         )
-
+        result = ""
         if dry_run:
-            logging.info("Dry run: would insert/update %d rules.", len(rules_data))
+            result = f"Dry run: would insert/update {len(rules_data)} rules."
+            logging.info(result)
         else:
             for rule_data in rules_data:
                 rule_object = Rule(
@@ -85,9 +74,9 @@ def populate_license_rules_task(dry_run, db_session):
                 )
                 db_session.merge(rule_object)
 
-            logging.info(
-                "Successfully upserted %d rules into the database.", len(rules_data)
-            )
+            result = f"Successfully upserted {len(rules_data)} rules into the database."
+            logging.info(result)
+        return result
 
     except requests.exceptions.RequestException as e:
         logging.error("Failed to download rules JSON file: %s", e)
@@ -96,16 +85,3 @@ def populate_license_rules_task(dry_run, db_session):
         logging.error("An error occurred while populating license rules: %s", e)
         db_session.rollback()
         raise
-
-
-def get_parameters(payload):
-    """
-    Get parameters from the payload and environment variables.
-
-    Args:
-        payload (dict): dictionary containing the payload data.
-    Returns:
-        tuple: (dry_run, after_date)
-    """
-    dry_run = payload.get("dry_run", False)
-    return dry_run

functions-python/tasks_executor/src/tasks/licenses/populate_license_tags.py

@@ -0,0 +1,111 @@
+# This script defines a task to populate the 'license_tag' table in the database from a canonical
+# JSON file. It is designed to be triggered as a background task.
+#
+# The script performs the following steps:
+# 1. Fetches the 'tags.json' file from the MobilityData/licenses-aas GitHub repository.
+# 2. The JSON file categorises tags into groups (e.g., 'spdx', 'license', 'domain', 'copyleft').
+#    Each group contains a '_group' metadata entry and individual tag entries.
+# 3. For each group and each tag (skipping the '_group' metadata key) it builds a record with:
+#    a. id: composite key of the form "group:tag" (e.g., "spdx:osi-approved")
+#    b. group: the group name (e.g., "spdx")
+#    c. tag: the tag name (e.g., "osi-approved")
+#    d. description: human-readable description of the tag
+# 4. For each tag record, it performs an "upsert" operation using SQLAlchemy's `merge` method,
+#    with the tag's composite 'id' acting as the primary key.
+# 5. Supports a 'dry_run' mode, which simulates the process and logs intended actions
+#    without committing any changes to the database.
+# 6. Includes error handling for network issues and database transactions.
+import logging
+
+import requests
+from shared.database.database import with_db_session
+from shared.database_gen.sqlacodegen_models import LicenseTag, LicenseTagGroup
+
+
+TAGS_JSON_URL = "https://raw.githubusercontent.com/MobilityData/licenses-catalog/main/data/tags.json"
+
+
+@with_db_session
+def populate_license_tags(dry_run, db_session):
+    """
+    Populates license tags in the database from a canonical JSON source.
+
+    Args:
+        dry_run (bool): If True, simulates the operation without making changes.
+        db_session: Database session for executing queries.
+    """
+    logging.info("Starting populate_license_tags with dry_run=%s", dry_run)
+
+    try:
+        logging.info("Downloading tags from %s", TAGS_JSON_URL)
+        response = requests.get(TAGS_JSON_URL, timeout=10)
+        response.raise_for_status()
+        tags_json = response.json()
+
+        tags_data = []
+        groups_data = {}
+
+        for group_name, group_entries in tags_json.items():
+            group_meta = group_entries.get("_group", {}) or {}
+            groups_data[group_name] = {
+                "id": group_name,
+                "short_name": group_meta.get("short"),
+                "description": group_meta.get("description") or group_name,
+            }
+
+            for tag_name, tag_entry in group_entries.items():
+                if tag_name == "_group":
+                    # Skip the group-level metadata entry
+                    continue
+                tag_id = f"{group_name}:{tag_name}"
+                tags_data.append(
+                    {
+                        "id": tag_id,
+                        "group": group_name,
+                        "tag": tag_name,
+                        "description": tag_entry.get("description"),
+                        "url": tag_entry.get("url"),
+                    }
+                )
+
+        logging.info(
+            "Loaded %d groups and %d tags from tags.json.",
+            len(groups_data),
+            len(tags_data),
+        )
+
+        result = ""
+        if dry_run:
+            result = f"Dry run: would insert/update {len(groups_data)} groups and {len(tags_data)} tags."
+            logging.info(result)
+        else:
+            # Upsert groups first so FK from license_tag.group is satisfied
+            for group in groups_data.values():
+                group_object = LicenseTagGroup(
+                    id=group["id"],
+                    short_name=group["short_name"],
+                    description=group["description"],
+                )
+                db_session.merge(group_object)
+
+            # Then upsert tags that reference those groups
+            for tag_data in tags_data:
+                tag_object = LicenseTag(
+                    id=tag_data["id"],
+                    group=tag_data["group"],
+                    tag=tag_data["tag"],
+                    description=tag_data["description"],
+                    url=tag_data["url"],
+                )
+                db_session.merge(tag_object)
+            result = f"Successfully upserted {len(groups_data)} groups and {len(tags_data)} tags into the database."
+            logging.info(result)
+        return result
+
+    except requests.exceptions.RequestException as e:
+        logging.error("Failed to download tags JSON file: %s", e)
+        raise
+    except Exception as e:
+        logging.error("An error occurred while populating license tags: %s", e)
+        db_session.rollback()
+        raise

functions-python/tasks_executor/src/tasks/licenses/populate_licenses.py

@@ -1,5 +1,6 @@
-# This script defines a task to populate the 'licenses' table and the 'license_rules'
-# association table in the database. It is designed to be triggered as a background task.
+# This script defines a task to populate the 'licenses' table, the 'license_rules'
+# association table, and the 'license_license_tags' association table in the database.
+# It is designed to be triggered as a background task.
 #
 # The script performs the following steps:
 # 1. Fetches a list of license definition files from the MobilityData/licenses-aas GitHub repository
@@ -15,6 +16,9 @@
 #    e. Associates the found rules with the license. The SQLAlchemy ORM automatically
 #       manages the creation of records in the 'license_rules' join table to establish
 #       the many-to-many relationship.
+#    f. Extracts the associated tag IDs from the 'tags' list at the top level of the JSON.
+#    g. Queries the 'license_tag' table to find the corresponding LicenseTag objects.
+#    h. Associates the found tags with the license via the 'license_license_tags' join table.
 # 4. Supports a 'dry_run' mode, which simulates the process and logs intended
 #    actions without committing any changes to the database.
 # 5. Includes error handling for network issues and database transactions.
@@ -23,22 +27,33 @@
 
 import requests
 from shared.database.database import with_db_session
-from shared.database_gen.sqlacodegen_models import License, Rule
+from shared.database_gen.sqlacodegen_models import License, LicenseTag, Rule
+from .populate_license_rules import populate_license_rules
+from .populate_license_tags import populate_license_tags
 
 LICENSES_API_URL = (
-    "https://api.github.com/repos/MobilityData/licenses-aas/contents/data/licenses"
+    "https://api.github.com/repos/MobilityData/licenses-catalog/contents/data/licenses"
 )
 
 
 def populate_licenses_handler(payload):
     """
-    Handler for populating licenses.
+    Handler function for the populate licenses task.
+    This function imports the license rules and tags population tasks to ensure that the necessary data is available
+    before populating the licenses.
 
     Args:
         payload (dict): Incoming payload data.
     """
     dry_run = get_parameters(payload)
-    return populate_licenses_task(dry_run)
+    rules_result = populate_license_rules(dry_run)
+    tags_result = populate_license_tags(dry_run)
+    license_result = populate_licenses_task(dry_run)
+    return {
+        "rules": rules_result,
+        "tags": tags_result,
+        "licenses": license_result,
+    }
 
 
 @with_db_session
@@ -69,7 +84,8 @@ def populate_licenses_task(dry_run, db_session):
         logging.info("Loaded %d licenses.", len(licenses_data))
 
         if dry_run:
-            logging.info("Dry run: would process %d licenses.", len(licenses_data))
+            result = f"Dry run: would process {len(licenses_data)} licenses."
+            logging.info(result)
         else:
             for license_data in licenses_data:
                 spdx_data = license_data.get("spdx")
@@ -132,14 +148,33 @@ def populate_licenses_task(dry_run, db_session):
                             len(rules),
                             len(all_rule_names),
                         )
+
+                # Clear existing tags and assign updated ones
+                license_object.tags = []
+
+                tag_ids = license_data.get("tags", [])
+                if tag_ids:
+                    tags = (
+                        db_session.query(LicenseTag)
+                        .filter(LicenseTag.id.in_(tag_ids))
+                        .all()
+                    )
+                    license_object.tags.extend(tags)
+                    if len(tags) != len(tag_ids):
+                        logging.warning(
+                            "License '%s': Found %d of %d tags in the database.",
+                            license_id,
+                            len(tags),
+                            len(tag_ids),
+                        )
+
                 # Merge the license object into the session. This handles updating existing licenses (upsert),
-                # including their rule associations.
+                # including their rule and tag associations.
                 if not is_new:
                     db_session.merge(license_object)
-
-            logging.info(
-                "Successfully upserted licenses into the database.",
-            )
+            result = "Successfully upserted licenses into the database."
+            logging.info(result)
+        return result
 
     except requests.exceptions.RequestException as e:
         logging.error("Failed to download licenses JSON file: %s", e)

functions-python/tasks_executor/tests/tasks/populate_licenses_and_rules/test_populate_license_rules.py

@@ -4,7 +4,7 @@
 from sqlalchemy.exc import SQLAlchemyError
 
 from tasks.licenses.populate_license_rules import (
-    populate_license_rules_task,
+    populate_license_rules,
     RULES_JSON_URL,
 )
 from shared.database_gen.sqlacodegen_models import Rule
@@ -43,7 +43,7 @@ def test_populate_rules_success(self, mock_requests_get):
         mock_db_session = MagicMock()
 
         # Act
-        populate_license_rules_task(dry_run=False, db_session=mock_db_session)
+        populate_license_rules(dry_run=False, db_session=mock_db_session)
 
         # Assert
         mock_requests_get.assert_called_once_with(RULES_JSON_URL, timeout=10)
@@ -78,7 +78,7 @@ def test_populate_rules_dry_run(self, mock_requests_get):
         mock_db_session = MagicMock()
 
         # Act
-        populate_license_rules_task(dry_run=True, db_session=mock_db_session)
+        populate_license_rules(dry_run=True, db_session=mock_db_session)
 
         # Assert
         mock_requests_get.assert_called_once_with(RULES_JSON_URL, timeout=10)
@@ -96,7 +96,7 @@ def test_request_exception_handling(self, mock_requests_get):
 
         # Act & Assert
         with self.assertRaises(requests.exceptions.RequestException):
-            populate_license_rules_task(dry_run=False, db_session=mock_db_session)
+            populate_license_rules(dry_run=False, db_session=mock_db_session)
 
         mock_db_session.merge.assert_not_called()
         mock_db_session.rollback.assert_not_called()
@@ -115,7 +115,7 @@ def test_database_exception_handling(self, mock_requests_get):
 
         # Act & Assert
         with self.assertRaises(SQLAlchemyError):
-            populate_license_rules_task(dry_run=False, db_session=mock_db_session)
+            populate_license_rules(dry_run=False, db_session=mock_db_session)
 
         self.assertTrue(mock_db_session.merge.called)
         mock_db_session.rollback.assert_called_once()