Add support for retrieving raw file content from GitLab (#124)

* distro: introduce raw URL formatters

Refactor GitHub formatting code and add GitLab formatter.

* distro: add manifest download helper

Return either cached version or attempt download and return result.

* distro: download GitHub raw URL using helpers

* distro: support GitLab hosted pkgs and manifests

* Update GitHub Actions checkout and Pixi setup versions

* Delete .github/workflows/enforce-sha.yml

---------

Co-authored-by: Tobias Fischer <info@tobiasfischer.info>

Author: gavanderhoorn

.github/workflows/ci.yml

@@ -15,10 +15,10 @@ jobs:
       fail-fast: false
 
     steps:
-    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+    - uses: actions/checkout@v7
 
     - name: Setup Pixi
-      uses: prefix-dev/setup-pixi@28eb668aafebd9dede9d97c4ba1cd9989a4d0004 # v0.9.2
+      uses: prefix-dev/setup-pixi@v0.10.0
       with:
         pixi-version: latest
         cache: true

.github/workflows/enforce-sha.yml

@@ -187,9 +187,30 @@ def get_python_version(self):
     def get_package_names(self):
         return self._distro.release_packages.keys()
 
+    def get_package_xml_for_additional_package(self, pkg_info):
+        raw_url_base = pkg_info.get("url")
+        if "github.com" in raw_url_base:
+            raw_url = self._construct_raw_url_github(pkg_info)
+            return self._download_raw_pkg_xml_or_cached(url=raw_url)
+        if "gitlab.com" in raw_url_base:
+            raw_url = self._construct_raw_url_gitlab(pkg_info)
+            return self._download_raw_pkg_xml_or_cached(url=raw_url)
+        raise RuntimeError(f"Cannot handle unknown repository hoster: {raw_url_base}")
+
+    def _download_raw_pkg_xml_or_cached(self, url):
+        if url in self._additional_xml_cache:
+            return self._additional_xml_cache[url]
+        try:
+            with urllib.request.urlopen(url) as resp:
+                xml_content = resp.read().decode("utf-8")
+                self._additional_xml_cache[url] = xml_content
+                return xml_content
+        except Exception as e:
+            raise RuntimeError(f"Failed to fetch package.xml from {url}: {e}")
+
     # Based on https://github.com/ros-infrastructure/rosdistro/blob/fad8d9f647631945847cb18bc1d1f43008d7a282/src/rosdistro/manifest_provider/github.py#L51C1-L69C29
     # But with the option to specify the name of the package.xml file in case the repo uses a non-standard name
-    def get_package_xml_for_additional_package(self, pkg_info):
+    def _construct_raw_url_github(self, pkg_info):
         # Build raw GitHub URL for package.xml
         raw_url_base = pkg_info.get("url")
         if raw_url_base.endswith(".git"):
@@ -205,13 +226,20 @@ def get_package_xml_for_additional_package(self, pkg_info):
         if additional_folder != "":
             additional_folder = additional_folder + "/"
         raw_url = f"https://raw.githubusercontent.com/{owner_repo}/{ref}/{additional_folder}{xml_name}"
-        if raw_url in self._additional_xml_cache:
-            return self._additional_xml_cache[raw_url]
+        return raw_url
 
-        try:
-            with urllib.request.urlopen(raw_url) as resp:
-                xml_content = resp.read().decode("utf-8")
-                self._additional_xml_cache[raw_url] = xml_content
-                return xml_content
-        except Exception as e:
-            raise RuntimeError(f"Failed to fetch package.xml from {raw_url}: {e}")
+    # format (checked against GitLab 19.x): https://gitlab.com/<NAMESPACE>/-/raw/<REV>/<PATH>
+    def _construct_raw_url_gitlab(self, pkg_info):
+        raw_url_base = pkg_info.get("url")
+        if raw_url_base.endswith(".git"):
+            raw_url_base = raw_url_base[:-4]
+        if "gitlab.com" not in raw_url_base:
+            raise RuntimeError(f"Cannot handle non-GitLab URL: {raw_url_base}")
+        # Use rev if available, otherwise fallback to tag
+        ref = pkg_info.get("rev") or pkg_info.get("tag")
+        xml_name = pkg_info.get("package_xml_name", "package.xml")
+        additional_folder = pkg_info.get("additional_folder", "")
+        if additional_folder != "":
+            additional_folder = additional_folder + "/"
+        raw_url = f"{raw_url_base}/-/raw/{ref}/{additional_folder}{xml_name}"
+        return raw_url