diff --git a/builder/requirements.txt b/builder/requirements.txt index 71f9d61a..da679be0 100644 --- a/builder/requirements.txt +++ b/builder/requirements.txt @@ -1,3 +1,4 @@ jsonschema redis dill==0.3.8 +beautifulsoup4==4.12.3 diff --git a/metadata_manager/ap_src_meta_fetcher.py b/metadata_manager/ap_src_meta_fetcher.py index 33bfff0a..7757ce3b 100644 --- a/metadata_manager/ap_src_meta_fetcher.py +++ b/metadata_manager/ap_src_meta_fetcher.py @@ -6,6 +6,18 @@ import ap_git import os +from utils.apache_dir_listing import parse_apache_dir_listing + + +class FirmwareServerUnavailableError(Exception): + """Raised when the firmware server cannot be reached or returns an error.""" + + +def _board_artifacts_subdir(board_id: str, vehicle_id: str = None) -> str: + if vehicle_id == "heli": + return f"{board_id}-heli" + return board_id + class APSourceMetadataFetcher: """ @@ -503,11 +515,7 @@ def get_board_defaults_from_fw_server( """ import requests - # Heli builds are stored under a separate folder - artifacts_subdir = board_id - if vehicle_id == "Heli": - artifacts_subdir += "-heli" - + artifacts_subdir = _board_artifacts_subdir(board_id, vehicle_id) features_txt_url = f"{artifacts_url}/{artifacts_subdir}/features.txt" try: @@ -551,6 +559,50 @@ def get_board_defaults_from_fw_server( ) return None + def get_board_standard_artifacts_from_fw_server( + self, + version_artifacts_url: str, + board_id: str, + vehicle_id: str = None, + ) -> list | None: + """ + Fetch standard build artifact file listings from firmware.ardupilot.org. + + Parameters: + version_artifacts_url (str): Base URL for build artifacts for a version. + board_id (str): Board identifier + vehicle_id (str): Vehicle identifier (for special handling like Heli) + + Returns: + list: File entries with name, url, size, and modified fields. + None: If the board directory does not exist on the firmware server. + + Raises: + FirmwareServerUnavailableError: If the firmware server is unreachable + or returns a non-404 error. + """ + import requests + + board_artifacts_subdir = _board_artifacts_subdir(board_id, vehicle_id) + listing_url = f"{version_artifacts_url.rstrip('/')}/{board_artifacts_subdir}/" + + try: + response = requests.get(listing_url, timeout=30) + if response.status_code == 404: + return None + response.raise_for_status() + return parse_apache_dir_listing(response.text, base_url=listing_url) + except requests.HTTPError as e: + self.logger.warning( + f"Failed to fetch standard artifacts from {listing_url}: {e}" + ) + raise FirmwareServerUnavailableError(str(e)) from e + except requests.RequestException as e: + self.logger.warning( + f"Failed to fetch standard artifacts from {listing_url}: {e}" + ) + raise FirmwareServerUnavailableError(str(e)) from e + @staticmethod def get_singleton(): return APSourceMetadataFetcher.__singleton diff --git a/tests/utils/fixtures/apache_dir_listing_sample.html b/tests/utils/fixtures/apache_dir_listing_sample.html new file mode 100644 index 00000000..fe41c634 --- /dev/null +++ b/tests/utils/fixtures/apache_dir_listing_sample.html @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + +
TypeFilenameDateSize
Parent Directory ----
arducopter.abinTue Apr 2 05:11:12 20241814971
arducopter.apjTue Apr 15 05:11:12 20241640045
features.txtTue Apr 2 05:11:12 20248294
+ + diff --git a/tests/utils/test_apache_dir_listing.py b/tests/utils/test_apache_dir_listing.py new file mode 100644 index 00000000..371e105c --- /dev/null +++ b/tests/utils/test_apache_dir_listing.py @@ -0,0 +1,60 @@ +""" +Tests for Apache directory listing parser. +""" +from datetime import datetime, timezone +from pathlib import Path + +from utils.apache_dir_listing import parse_apache_dir_listing, _parse_apache_date + + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +SAMPLE_LISTING_HTML = (FIXTURES_DIR / "apache_dir_listing_sample.html").read_text() +BASE_URL = "https://firmware.ardupilot.org/Copter/stable-4.5.0/CubeOrange/" + + +class TestParseApacheDate: + def test_parses_single_digit_day(self): + result = _parse_apache_date("Tue Apr 2 05:11:12 2024") + assert result == datetime(2024, 4, 2, 5, 11, 12, tzinfo=timezone.utc) + + def test_parses_double_digit_day(self): + result = _parse_apache_date("Tue Apr 15 05:11:12 2024") + assert result == datetime(2024, 4, 15, 5, 11, 12, tzinfo=timezone.utc) + + def test_returns_none_for_dash(self): + assert _parse_apache_date("--") is None + + def test_returns_none_for_invalid(self): + assert _parse_apache_date("not a date") is None + + +class TestParseApacheDirListing: + def test_parses_files_and_skips_parent_directory(self): + entries = parse_apache_dir_listing(SAMPLE_LISTING_HTML, BASE_URL) + + assert len(entries) == 3 + assert entries[0]["name"] == "arducopter.abin" + assert entries[1]["name"] == "arducopter.apj" + assert entries[2]["name"] == "features.txt" + + def test_resolves_absolute_urls(self): + entries = parse_apache_dir_listing(SAMPLE_LISTING_HTML, BASE_URL) + + assert entries[0]["url"] == ( + "https://firmware.ardupilot.org/Copter/stable-4.5.0/" + "CubeOrange/arducopter.abin" + ) + + def test_parses_size_and_modified(self): + entries = parse_apache_dir_listing(SAMPLE_LISTING_HTML, BASE_URL) + + assert entries[0]["size"] == 1814971 + assert entries[0]["modified"] == datetime( + 2024, 4, 2, 5, 11, 12, tzinfo=timezone.utc + ) + assert entries[1]["modified"] == datetime( + 2024, 4, 15, 5, 11, 12, tzinfo=timezone.utc + ) + + def test_returns_empty_list_when_no_table(self): + assert parse_apache_dir_listing("", BASE_URL) == [] diff --git a/tests/web/test_vehicles_api.py b/tests/web/test_vehicles_api.py index c935654c..e51a6abf 100644 --- a/tests/web/test_vehicles_api.py +++ b/tests/web/test_vehicles_api.py @@ -9,6 +9,7 @@ VehicleBase, VersionOut, BoardOut, + StandardArtifactOut, FeatureOut, CategoryBase, FeatureDefault, @@ -78,6 +79,18 @@ def dummy_feature( dependencies=[], ) + @staticmethod + def dummy_standard_artifact( + name="arducopter.apj", + url="https://firmware.ardupilot.org/Copter/stable-4.5.0/CubeOrange/arducopter.apj", + ): + return StandardArtifactOut( + name=name, + url=url, + size=1640045, + modified=None, + ) + # GET /vehicles def test_list_vehicles_returns_200_with_vehicle_list(self, client): @@ -442,6 +455,75 @@ def test_get_board_method_not_allowed(self, client): response = method("/api/v1/vehicles/copter/versions/v1/boards/b1") assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED + # GET /vehicles/{vehicle_id}/versions/{version_id}/boards/{board_id}/standard_artifacts + + _STANDARD_ARTIFACTS_URL = ( + "/api/v1/vehicles/copter/versions/copter-4.5.0-stable/" + "boards/MatekH743/standard_artifacts" + ) + + def test_list_board_standard_artifacts_returns_200(self, client): + mock_vehicles_service = Mock() + mock_vehicles_service.get_board_standard_artifacts.return_value = [ + self.dummy_standard_artifact() + ] + with self.override_vehicles_service(client, mock_vehicles_service): + response = client.get(self._STANDARD_ARTIFACTS_URL) + + assert response.status_code == status.HTTP_200_OK + assert "application/json" in response.headers["content-type"] + + def test_list_board_standard_artifacts_returns_404_when_not_found(self, client): + mock_vehicles_service = Mock() + mock_vehicles_service.get_board_standard_artifacts.return_value = None + with self.override_vehicles_service(client, mock_vehicles_service): + response = client.get(self._STANDARD_ARTIFACTS_URL) + + assert response.status_code == status.HTTP_404_NOT_FOUND + assert "MatekH743" in response.json()["detail"] + + def test_list_board_standard_artifacts_returns_502_on_fw_server_error(self, client): + from metadata_manager.ap_src_meta_fetcher import FirmwareServerUnavailableError + + mock_vehicles_service = Mock() + mock_vehicles_service.get_board_standard_artifacts.side_effect = ( + FirmwareServerUnavailableError("connection failed") + ) + with self.override_vehicles_service(client, mock_vehicles_service): + response = client.get(self._STANDARD_ARTIFACTS_URL) + + assert response.status_code == status.HTTP_502_BAD_GATEWAY + + def test_list_board_standard_artifacts_response_schema(self, client): + mock_vehicles_service = Mock() + mock_vehicles_service.get_board_standard_artifacts.return_value = [ + self.dummy_standard_artifact() + ] + with self.override_vehicles_service(client, mock_vehicles_service): + response = client.get(self._STANDARD_ARTIFACTS_URL) + + data = response.json() + assert len(data) == 1 + for field in ["name", "url", "size", "modified"]: + assert field in data[0] + + def test_list_board_standard_artifacts_service_called_with_correct_ids(self, client): + mock_vehicles_service = Mock() + mock_vehicles_service.get_board_standard_artifacts.return_value = [ + self.dummy_standard_artifact() + ] + with self.override_vehicles_service(client, mock_vehicles_service): + client.get(self._STANDARD_ARTIFACTS_URL) + + mock_vehicles_service.get_board_standard_artifacts.assert_called_once_with( + "copter", "copter-4.5.0-stable", "MatekH743" + ) + + def test_list_board_standard_artifacts_method_not_allowed(self, client): + for method in [client.post, client.put, client.patch, client.delete]: + response = method(self._STANDARD_ARTIFACTS_URL) + assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED + # GET /vehicles/{vehicle_id}/versions/{version_id}/boards/{board_id}/features _FEATURES_URL = "/api/v1/vehicles/copter/versions/copter-4.5.0-stable/boards/MatekH743/features" diff --git a/tests/web/test_vehicles_service.py b/tests/web/test_vehicles_service.py index 35eb60b2..66772b64 100644 --- a/tests/web/test_vehicles_service.py +++ b/tests/web/test_vehicles_service.py @@ -1145,3 +1145,149 @@ def test_get_feature_returns_correct_match_among_many( assert result is not None assert result.id == "FEATURE_B" assert result.default.enabled is False + + # Tests for get_board_standard_artifacts + + def test_get_board_standard_artifacts_version_not_found_returns_none( + self, service, mock_versions_fetcher + ): + mock_versions_fetcher.get_version_info.return_value = None + + result = service.get_board_standard_artifacts( + "copter", "nonexistent-version-id", "CubeRed" + ) + + assert result is None + + def test_get_board_standard_artifacts_board_not_found_returns_none( + self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + ): + version_info = VersionInfo( + remote_info=RemoteInfo( + name="ardupilot", + url="https://github.com/ArduPilot/ardupilot.git", + ), + commit_ref="refs/tags/Copter-4.5.0", + release_type="stable", + version_number="4.5.0", + ap_build_artifacts_url="https://firmware.ardupilot.org/Copter/stable-4.5.0", + ) + mock_versions_fetcher.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeOrange"] + + result = service.get_board_standard_artifacts( + "copter", version_info.version_id, "UnknownBoard" + ) + + assert result is None + + def test_get_board_standard_artifacts_no_artifacts_url_returns_none( + self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + ): + version_info = VersionInfo( + remote_info=RemoteInfo( + name="ardupilot", + url="https://github.com/ArduPilot/ardupilot.git", + ), + commit_ref="refs/tags/Copter-4.5.0", + release_type="stable", + version_number="4.5.0", + ap_build_artifacts_url=None, + ) + mock_versions_fetcher.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed"] + + result = service.get_board_standard_artifacts( + "copter", version_info.version_id, "CubeRed" + ) + + assert result is None + mock_ap_src_metadata_fetcher.get_board_standard_artifacts_from_fw_server.assert_not_called() + + def test_get_board_standard_artifacts_success( + self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + ): + from datetime import datetime, timezone + + version_info = VersionInfo( + remote_info=RemoteInfo( + name="ardupilot", + url="https://github.com/ArduPilot/ardupilot.git", + ), + commit_ref="refs/tags/Copter-4.5.0", + release_type="stable", + version_number="4.5.0", + ap_build_artifacts_url="https://firmware.ardupilot.org/Copter/stable-4.5.0", + ) + mock_versions_fetcher.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed"] + mock_ap_src_metadata_fetcher.get_board_standard_artifacts_from_fw_server.return_value = [ + { + "name": "arducopter.apj", + "url": "https://firmware.ardupilot.org/Copter/stable-4.5.0/CubeRed/arducopter.apj", + "size": 1640045, + "modified": datetime(2024, 4, 2, 5, 11, 12, tzinfo=timezone.utc), + } + ] + + result = service.get_board_standard_artifacts( + "copter", version_info.version_id, "CubeRed" + ) + + assert len(result) == 1 + assert result[0].name == "arducopter.apj" + assert result[0].size == 1640045 + mock_ap_src_metadata_fetcher.get_board_standard_artifacts_from_fw_server.assert_called_once_with( + version_artifacts_url="https://firmware.ardupilot.org/Copter/stable-4.5.0", + board_id="CubeRed", + vehicle_id="copter", + ) + + def test_get_board_standard_artifacts_fw_server_404_returns_none( + self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + ): + version_info = VersionInfo( + remote_info=RemoteInfo( + name="ardupilot", + url="https://github.com/ArduPilot/ardupilot.git", + ), + commit_ref="refs/tags/Copter-4.5.0", + release_type="stable", + version_number="4.5.0", + ap_build_artifacts_url="https://firmware.ardupilot.org/Copter/stable-4.5.0", + ) + mock_versions_fetcher.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed"] + mock_ap_src_metadata_fetcher.get_board_standard_artifacts_from_fw_server.return_value = None + + result = service.get_board_standard_artifacts( + "copter", version_info.version_id, "CubeRed" + ) + + assert result is None + + def test_get_board_standard_artifacts_fw_server_error_propagates( + self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + ): + from metadata_manager.ap_src_meta_fetcher import FirmwareServerUnavailableError + + version_info = VersionInfo( + remote_info=RemoteInfo( + name="ardupilot", + url="https://github.com/ArduPilot/ardupilot.git", + ), + commit_ref="refs/tags/Copter-4.5.0", + release_type="stable", + version_number="4.5.0", + ap_build_artifacts_url="https://firmware.ardupilot.org/Copter/stable-4.5.0", + ) + mock_versions_fetcher.get_version_info.return_value = version_info + mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed"] + mock_ap_src_metadata_fetcher.get_board_standard_artifacts_from_fw_server.side_effect = ( + FirmwareServerUnavailableError("connection failed") + ) + + with pytest.raises(FirmwareServerUnavailableError): + service.get_board_standard_artifacts( + "copter", version_info.version_id, "CubeRed" + ) diff --git a/utils/apache_dir_listing.py b/utils/apache_dir_listing.py new file mode 100644 index 00000000..879dfc01 --- /dev/null +++ b/utils/apache_dir_listing.py @@ -0,0 +1,88 @@ +""" +Parse Apache HTML directory listings. +""" +from datetime import datetime, timezone +from typing import Optional +from urllib.parse import urljoin + +from bs4 import BeautifulSoup + + +def _parse_apache_date(raw: str) -> Optional[datetime]: + """ + Parse an Apache directory listing date string into UTC datetime. + + Example input: "Tue Apr 2 05:11:12 2024" + """ + raw = raw.strip() + if not raw or raw == "--": + return None + + try: + parsed = datetime.strptime(raw, "%a %b %d %H:%M:%S %Y") + return parsed.replace(tzinfo=timezone.utc) + except ValueError: + return None + + +def _parse_size(raw: str) -> Optional[int]: + raw = raw.strip() + if not raw or raw == "--": + return None + + try: + return int(raw) + except ValueError: + return None + + +def _is_parent_directory_row(link_text: str, icon_src: Optional[str]) -> bool: + if icon_src and "back.gif" in icon_src: + return True + return "parent directory" in link_text.lower() + + +def parse_apache_dir_listing(html: str, base_url: str) -> list[dict]: + """ + Parse an Apache HTML directory listing into file entries. + + Parameters: + html: Raw HTML from the directory listing page. + base_url: Base URL of the listing (used to resolve relative links). + + Returns: + List of dicts with keys: name, url, size, modified. + """ + soup = BeautifulSoup(html, "html.parser") + table = soup.find("table") + if not table: + return [] + + entries = [] + for row in table.find_all("tr"): + cells = row.find_all("td") + if len(cells) < 4: + continue + + link = cells[1].find("a") + if not link: + continue + + name = link.get_text(strip=True) + href = link.get("href") + if not href or not name: + continue + + icon = cells[0].find("img") + icon_src = icon.get("src") if icon else None + if _is_parent_directory_row(name, icon_src): + continue + + entries.append({ + "name": name, + "url": urljoin(base_url, href), + "modified": _parse_apache_date(cells[2].get_text()), + "size": _parse_size(cells[3].get_text()), + }) + + return entries diff --git a/web/api/v1/vehicles.py b/web/api/v1/vehicles.py index 0fe97ee8..a9cb7550 100644 --- a/web/api/v1/vehicles.py +++ b/web/api/v1/vehicles.py @@ -5,9 +5,11 @@ VehicleBase, VersionOut, BoardOut, + StandardArtifactOut, FeatureOut, ) from web.services.vehicles import get_vehicles_service, VehiclesService +from metadata_manager.ap_src_meta_fetcher import FirmwareServerUnavailableError router = APIRouter(prefix="/vehicles", tags=["vehicles"]) @@ -183,6 +185,55 @@ async def get_board( return board +@router.get( + "/{vehicle_id}/versions/{version_id}/boards/{board_id}/standard_artifacts", + response_model=List[StandardArtifactOut], + responses={ + 404: {"description": "Standard artifacts not found"}, + 502: {"description": "Firmware server unavailable"}, + } +) +async def list_board_standard_artifacts( + vehicle_id: str = Path(..., description="Vehicle identifier"), + version_id: str = Path(..., description="Version identifier"), + board_id: str = Path(..., description="Board identifier"), + service: VehiclesService = Depends(get_vehicles_service) +): + """ + Get standard build artifact files from firmware.ardupilot.org for a board. + + Args: + vehicle_id: The vehicle identifier + version_id: The version identifier + board_id: The board identifier + + Returns: + List of artifact files with download URLs + """ + try: + artifacts = service.get_board_standard_artifacts( + vehicle_id, version_id, board_id + ) + except FirmwareServerUnavailableError: + raise HTTPException( + status_code=502, + detail=( + "Failed to fetch standard artifacts from firmware server" + ) + ) + + if artifacts is None: + raise HTTPException( + status_code=404, + detail=( + f"Standard artifacts not found for board '{board_id}' " + f"in vehicle '{vehicle_id}' version '{version_id}'" + ) + ) + + return artifacts + + # --- Feature Endpoints --- @router.get( "/{vehicle_id}/versions/{version_id}/boards/{board_id}/features", diff --git a/web/requirements.txt b/web/requirements.txt index 1862bbda..b79ecc28 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -3,6 +3,7 @@ uvicorn==0.40.0 pydantic==2.5.0 redis==5.2.1 requests==2.31.0 +beautifulsoup4==4.12.3 jsonschema==4.20.0 dill==0.3.8 packaging==25.0 diff --git a/web/schemas/__init__.py b/web/schemas/__init__.py index 22f58bad..ed1a13da 100644 --- a/web/schemas/__init__.py +++ b/web/schemas/__init__.py @@ -27,6 +27,7 @@ VersionOut, BoardBase, BoardOut, + StandardArtifactOut, CategoryBase, FeatureDefault, FeatureBase, @@ -49,6 +50,7 @@ "VersionOut", "BoardBase", "BoardOut", + "StandardArtifactOut", "CategoryBase", "FeatureDefault", "FeatureBase", diff --git a/web/schemas/vehicles.py b/web/schemas/vehicles.py index 64ac43c9..ef8725be 100644 --- a/web/schemas/vehicles.py +++ b/web/schemas/vehicles.py @@ -1,4 +1,5 @@ # app/schemas/vehicles.py +from datetime import datetime from typing import Literal, Optional from pydantic import BaseModel, Field @@ -49,6 +50,15 @@ class BoardOut(BoardBase): version_id: str = Field(..., description="Associated version identifier") +class StandardArtifactOut(BaseModel): + name: str = Field(..., description="Artifact filename") + url: str = Field(..., description="Download URL on firmware.ardupilot.org") + size: Optional[int] = Field(None, description="File size in bytes") + modified: Optional[datetime] = Field( + None, description="Last modified time (UTC, ISO 8601)" + ) + + # --- Features --- class CategoryBase(BaseModel): id: str = Field(..., description="Unique category identifier") diff --git a/web/services/vehicles.py b/web/services/vehicles.py index 1ebe6662..1c258874 100644 --- a/web/services/vehicles.py +++ b/web/services/vehicles.py @@ -10,6 +10,7 @@ RemoteInfo, VersionOut, BoardOut, + StandardArtifactOut, FeatureOut, CategoryBase, FeatureDefault, @@ -150,6 +151,52 @@ def get_board( return board return None + def get_board_standard_artifacts( + self, + vehicle_id: str, + version_id: str, + board_id: str, + ) -> Optional[List[StandardArtifactOut]]: + """Get standard build artifacts from firmware.ardupilot.org for a board.""" + version_info = self.versions_fetcher.get_version_info( + vehicle_id=vehicle_id, + version_id=version_id, + ) + if not version_info: + return None + + if not self.get_board(vehicle_id, version_id, board_id): + return None + + if version_info.ap_build_artifacts_url is None: + return None + + logger.info( + f'Standard artifacts requested for {vehicle_id} ' + f'{version_info.remote_info.name} {version_info.commit_ref} ' + f'board {board_id}' + ) + + artifacts = ( + self.ap_src_metadata_fetcher.get_board_standard_artifacts_from_fw_server( + version_artifacts_url=version_info.ap_build_artifacts_url, + board_id=board_id, + vehicle_id=vehicle_id, + ) + ) + if artifacts is None: + return None + + return [ + StandardArtifactOut( + name=entry["name"], + url=entry["url"], + size=entry.get("size"), + modified=entry.get("modified"), + ) + for entry in artifacts + ] + def get_features( self, vehicle_id: str,