Merge branch 'main' into user_feature_flag

Author: davidgamez

api/.openapi-generator/FILES

@@ -26,6 +26,8 @@ src/feeds_gen/models/gbfs_version.py
 src/feeds_gen/models/get_matching_licenses_request.py
 src/feeds_gen/models/gtfs_dataset.py
 src/feeds_gen/models/gtfs_feed.py
+src/feeds_gen/models/gtfs_feed_availability_check.py
+src/feeds_gen/models/gtfs_feed_availability_response.py
 src/feeds_gen/models/gtfs_rt_feed.py
 src/feeds_gen/models/latest_dataset.py
 src/feeds_gen/models/latest_dataset_validation_report.py

api/src/feeds/impl/feeds_api_impl.py

@@ -9,13 +9,15 @@
 from feeds.impl.error_handling import raise_http_error, raise_http_validation_error, convert_exception
 from shared.db_models.feed_impl import FeedImpl
 from shared.db_models.gbfs_feed_impl import GbfsFeedImpl
+from shared.db_models.gtfs_feed_availability_check_impl import GtfsFeedAvailabilityCheckImpl
 from shared.db_models.gtfs_feed_impl import GtfsFeedImpl
 from shared.db_models.gtfs_rt_feed_impl import GtfsRTFeedImpl
 from feeds_gen.apis.feeds_api_base import BaseFeedsApi
 from feeds_gen.models.feed import Feed
 from feeds_gen.models.gbfs_feed import GbfsFeed
 from feeds_gen.models.gtfs_dataset import GtfsDataset
 from feeds_gen.models.gtfs_feed import GtfsFeed
+from feeds_gen.models.gtfs_feed_availability_response import GtfsFeedAvailabilityResponse
 from feeds_gen.models.gtfs_rt_feed import GtfsRTFeed
 from middleware.request_context import is_user_email_restricted
 from shared.common.db_utils import (
@@ -26,6 +28,7 @@
     get_gbfs_feeds_query,
 )
 from shared.common.error_handling import (
+    availability_from_after_to,
     invalid_date_message,
     feed_not_found,
     gtfs_feed_not_found,
@@ -38,6 +41,7 @@
     Feed as FeedOrm,
     Gtfsdataset,
     Gtfsfeed,
+    GtfsFeedAvailabilityCheck,
     Gtfsrealtimefeed,
 )
 from shared.feed_filters.feed_filter import FeedFilter
@@ -308,6 +312,53 @@ def get_gtfs_feed_gtfs_rt_feeds(self, id: str, db_session: Session) -> List[Gtfs
         else:
             raise_http_error(404, gtfs_feed_not_found.format(id))
 
+    @with_db_session
+    def get_gtfs_feed_availability(
+        self,
+        id: str,
+        _from: str,
+        to: str,
+        limit: int,
+        offset: int,
+        sort: str,
+        db_session: Session,
+    ) -> GtfsFeedAvailabilityResponse:
+        """Returns historical availability checks for a GTFS feed."""
+        if _from and not valid_iso_date(_from):
+            raise_http_validation_error(invalid_date_message.format("from"))
+        if to and not valid_iso_date(to):
+            raise_http_validation_error(invalid_date_message.format("to"))
+
+        from_dt = datetime.fromisoformat(_from.replace("Z", "+00:00")) if _from else None
+        to_dt = datetime.fromisoformat(to.replace("Z", "+00:00")) if to else None
+
+        if from_dt and to_dt and from_dt > to_dt:
+            raise_http_validation_error(availability_from_after_to)
+
+        feed = self._get_gtfs_feed(id, db_session, include_options_for_joinedload=False)
+        if not feed:
+            raise_http_error(404, gtfs_feed_not_found.format(id))
+
+        query = db_session.query(GtfsFeedAvailabilityCheck).filter(GtfsFeedAvailabilityCheck.feed_id == feed.id)
+        if from_dt:
+            query = query.filter(GtfsFeedAvailabilityCheck.checked_at >= from_dt)
+        if to_dt:
+            query = query.filter(GtfsFeedAvailabilityCheck.checked_at <= to_dt)
+
+        total = query.count()
+        order = (
+            GtfsFeedAvailabilityCheck.checked_at.asc() if sort == "asc" else GtfsFeedAvailabilityCheck.checked_at.desc()
+        )
+        checks = query.order_by(order).offset(offset).limit(limit).all()
+
+        return GtfsFeedAvailabilityResponse(
+            feed_id=id,
+            total=total,
+            offset=offset,
+            limit=limit,
+            checks=[GtfsFeedAvailabilityCheckImpl.from_orm(c) for c in checks],
+        )
+
     @with_db_session
     def get_gbfs_feed(
         self,

api/src/scripts/populate_db_test_data.py

@@ -10,6 +10,7 @@
     Gtfsdataset,
     Validationreport,
     Gtfsfeed,
+    GtfsFeedAvailabilityCheck,
     Notice,
     Feature,
     License,
@@ -165,6 +166,11 @@ def populate_test_datasets(self, filepath, db_session: "Session"):
         if "validation_reports" in data:
             validation_report_dict = {}
             for report in data["validation_reports"]:
+                if report["dataset_id"] not in dataset_dict:
+                    self.logger.error(
+                        f"No dataset found with id: {report['dataset_id']}; skipping validation report {report['id']}"
+                    )
+                    continue
                 validation_report = Validationreport(
                     id=report["id"],
                     validator_version=report["validator_version"],
@@ -305,6 +311,29 @@ def populate_test_datasets(self, filepath, db_session: "Session"):
                         gbfs_version.gbfsendpoints.append(gbfs_endpoint)
                 gbfs_feed.gbfsversions.append(gbfs_version)
 
+        # GTFS feed availability checks
+        if "gtfs_availability_checks" in data:
+            for check in data["gtfs_availability_checks"]:
+                feed_stable_id = check.get("feed_stable_id")
+                gtfs_feed = db_session.query(Gtfsfeed).filter(Gtfsfeed.stable_id == feed_stable_id).one_or_none()
+                if not gtfs_feed:
+                    self.logger.error(
+                        f"No GTFS feed found with stable_id: {feed_stable_id}; skipping availability check"
+                    )
+                    continue
+                availability_check = GtfsFeedAvailabilityCheck(
+                    id=uuid4(),
+                    feed_id=gtfs_feed.id,
+                    checked_at=check["checked_at"],
+                    request_url=check.get("request_url", "https://example.com/feed.zip"),
+                    request_type=check["request_type"],
+                    success=check["success"],
+                    status_code=check.get("status_code"),
+                    latency_ms=check.get("latency_ms"),
+                    error_type=check.get("error_type"),
+                )
+                db_session.add(availability_check)
+
         db_session.commit()
         db_session.execute(text(f"REFRESH MATERIALIZED VIEW CONCURRENTLY {t_feedsearch.name}"))
 
@@ -325,6 +354,10 @@ def populate(self):
 
     def populate_test_feeds(self, feeds_data, db_session: "Session"):
         for feed_data in feeds_data:
+            existing = db_session.query(Gtfsfeed).filter(Gtfsfeed.stable_id == feed_data["id"]).one_or_none()
+            if existing:
+                logger.info(f"Feed {feed_data['id']} already exists, skipping")
+                continue
             feed = Gtfsfeed(
                 id=str(uuid4()),
                 stable_id=feed_data["id"],

api/src/shared/common/error_handling.py

@@ -3,6 +3,7 @@
 invalid_date_message: Final[str] = (
     "Invalid date format for '{}'. Expected ISO 8601 format, example: '2021-01-01T00:00:00Z'"
 )
+availability_from_after_to: Final[str] = "'from' timestamp must be before 'to' timestamp"
 invalid_bounding_coordinates: Final[str] = "Invalid bounding coordinates {} {}"
 invalid_bounding_method: Final[str] = "Invalid bounding_filter_method {}"
 feed_not_found: Final[str] = "Feed '{}' not found"

api/src/shared/common/license_utils.py

@@ -415,18 +415,27 @@ def resolve_license(
     # 4) Generic heuristics
     heuristic_match = heuristic_spdx(url_str)
     if heuristic_match:
-        return [
-            MatchingLicense(
-                license_id=heuristic_match,
-                license_url=url_str,
-                normalized_url=url_normalized,
-                spdx_id=heuristic_match,
-                match_type="heuristic",
-                confidence=0.95,
-                matched_name=heuristic_match,
-                matched_source="pattern-heuristics",
+        if db_session is not None:
+            # Check if the license found is actually in the DB
+            db_lic = (
+                db_session.query(License).filter(func.lower(License.id) == func.lower(heuristic_match)).one_or_none()
             )
-        ]
+            if db_lic is None:
+                logging.warning("Heuristic SPDX ID %s not found in DB, skipping assignment", heuristic_match)
+                heuristic_match = None
+        if heuristic_match:
+            return [
+                MatchingLicense(
+                    license_id=heuristic_match,
+                    license_url=url_str,
+                    normalized_url=url_normalized,
+                    spdx_id=heuristic_match,
+                    match_type="heuristic",
+                    confidence=0.95,
+                    matched_name=heuristic_match,
+                    matched_source="pattern-heuristics",
+                )
+            ]
 
     # 5) Fuzzy (same host candidates only)
     if allow_fuzzy and url_host and db_session is not None:

api/src/shared/db_models/gtfs_feed_availability_check_impl.py

@@ -0,0 +1,30 @@
+from shared.database_gen.sqlacodegen_models import GtfsFeedAvailabilityCheck as GtfsFeedAvailabilityCheckOrm
+from feeds_gen.models.gtfs_feed_availability_check import GtfsFeedAvailabilityCheck
+
+_REQUEST_TYPE_TO_METHOD = {"http_head": "HEAD", "http_get": "GET"}
+
+
+class GtfsFeedAvailabilityCheckImpl(GtfsFeedAvailabilityCheck):
+    """Implementation of the `GtfsFeedAvailabilityCheck` model.
+    This class converts a SQLAlchemy row DB object to a Pydantic model.
+    """
+
+    class Config:
+        """Pydantic configuration.
+        Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object."""
+
+        from_attributes = True
+
+    @classmethod
+    def from_orm(cls, check: GtfsFeedAvailabilityCheckOrm | None) -> GtfsFeedAvailabilityCheck | None:
+        """Create a model instance from a SQLAlchemy GtfsFeedAvailabilityCheck row object."""
+        if not check:
+            return None
+        return cls(
+            checked_at=check.checked_at,
+            success=check.success,
+            request_method=_REQUEST_TYPE_TO_METHOD.get(check.request_type, check.request_type),
+            status_code=check.status_code,
+            latency_ms=float(check.latency_ms) if check.latency_ms is not None else None,
+            error_type=check.error_type,
+        )

api/tests/integration/test_data/extra_test_data.json

@@ -11,7 +11,6 @@
       "content_html": "",
       "created_at": "2024-01-01T00:00:00Z",
       "updated_at": "2024-01-02T00:00:00Z"
-
     },
     {
       "id": "license-2",
@@ -24,7 +23,6 @@
       "content_html": "",
       "created_at": "2024-01-01T00:00:00Z",
       "updated_at": "2024-01-02T00:00:00Z"
-
     },
     {
       "id": "license-3",
@@ -39,7 +37,6 @@
       "updated_at": "2024-01-02T00:00:00Z"
     }
   ],
-
   "rules": [
     {
       "name": "license-rule-1",
@@ -60,7 +57,6 @@
       "type": "limitation"
     }
   ],
-
   "license_rules": [
     {
       "license_id": "license-1",
@@ -79,7 +75,6 @@
       "rule_id": "license-rule-3"
     }
   ],
-
   "license_tag_groups": [
     {
       "id": "family",
@@ -92,7 +87,6 @@
       "description": "License type taxonomy"
     }
   ],
-
   "license_tags": [
     {
       "id": "family:ODC",
@@ -107,7 +101,6 @@
       "description": "Open Data Commons license"
     }
   ],
-
   "license_license_tags": [
     {
       "license_id": "license-1",
@@ -118,7 +111,6 @@
       "tag_id": "license:open-data-commons"
     }
   ],
-
   "feeds": [
     {
       "id": "mdb-60",
@@ -237,5 +229,57 @@
         }
       ]
     }
+  ],
+  "gtfs_availability_checks": [
+    {
+      "feed_stable_id": "mdb-1",
+      "checked_at": "2025-01-10T10:00:00+00:00",
+      "request_url": "https://example.com/gtfs.zip",
+      "request_type": "http_head",
+      "success": true,
+      "status_code": 200,
+      "latency_ms": 120,
+      "error_type": null
+    },
+    {
+      "feed_stable_id": "mdb-1",
+      "checked_at": "2025-02-10T10:00:00+00:00",
+      "request_url": "https://example.com/gtfs.zip",
+      "request_type": "http_head",
+      "success": false,
+      "status_code": 503,
+      "latency_ms": 450,
+      "error_type": "http_error"
+    },
+    {
+      "feed_stable_id": "mdb-1",
+      "checked_at": "2025-03-10T10:00:00+00:00",
+      "request_url": "https://example.com/gtfs.zip",
+      "request_type": "http_get",
+      "success": true,
+      "status_code": 206,
+      "latency_ms": 200,
+      "error_type": null
+    },
+    {
+      "feed_stable_id": "mdb-1",
+      "checked_at": "2025-04-10T10:00:00+00:00",
+      "request_url": "https://example.com/gtfs.zip",
+      "request_type": "http_head",
+      "success": true,
+      "status_code": 200,
+      "latency_ms": 95,
+      "error_type": null
+    },
+    {
+      "feed_stable_id": "mdb-1",
+      "checked_at": "2025-05-10T10:00:00+00:00",
+      "request_url": "https://example.com/gtfs.zip",
+      "request_type": "http_head",
+      "success": false,
+      "status_code": null,
+      "latency_ms": null,
+      "error_type": "connection_timeout"
+    }
   ]
 }