feat: add GitLab token support to avoid API rate limits

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2018 The Python Packaging Authority
+Copyright (c) 2019 Daniel Garijo, Ontology Engineering Group, Universidad Politécnica de Madrid
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -16,4 +16,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

src/somef/__main__.py

@@ -1,7 +1,11 @@
 # -*- coding: utf-8 -*-
 
 import click
-from click_option_group import optgroup, RequiredMutuallyExclusiveOptionGroup, RequiredAnyOptionGroup
+from click_option_group import (
+    optgroup,
+    RequiredMutuallyExclusiveOptionGroup,
+    RequiredAnyOptionGroup,
+)
 
 import logging
 from . import configuration
@@ -13,33 +17,53 @@ class URLParamType(click.types.StringParamType):
     name = "url"
 
 
-@click.group(context_settings={'help_option_names': ['-h', '--help']})
+@click.group(context_settings={"help_option_names": ["-h", "--help"]})
 @click.version_option(__version__)
 def cli():
     click.echo("SOftware Metadata Extraction Framework (SOMEF) Command Line Interface")
 
 
-@cli.command(help="Configure GitHub credentials and classifiers file path")
-@click.option('-a', '--auto', help="Automatically configure SOMEF", is_flag=True, default=False)
-@click.option('-b', '--base_uri', type=URLParamType(), help="Base URI for somef transformations",
-              default=constants.CONF_DEFAULT_BASE_URI)
+@cli.command(help="Configure GitHub/GitLab credentials and classifiers file path")
+@click.option("-a", "--auto", help="Automatically configure SOMEF", is_flag=True, default=False)
+@click.option(
+    "-b",
+    "--base_uri",
+    type=URLParamType(),
+    help="Base URI for somef transformations",
+    default=constants.CONF_DEFAULT_BASE_URI,
+)
 def configure(auto, base_uri):
     if auto:
-        click.echo(
-            "Configuring SOMEF automatically. To assign credentials edit the configuration file or run "
-            "the interactive mode")
+        click.echo("Configuring SOMEF automatically. To assign credentials edit the configuration file or run the interactive mode")
         configuration.configure()
     elif base_uri is not constants.CONF_DEFAULT_BASE_URI:
         configuration.update_base_uri(base_uri)
     else:
-        authorization = click.prompt("Authorization", default="")
-        description = click.prompt("Documentation classifier model file", default=configuration.default_description)
+        github_authorization = click.prompt("GitHub Authorization token (leave blank to skip)", default="")
+        gitlab_authorization = click.prompt(
+            "GitLab Authorization token (leave blank to skip; works for gitlab.com and self-hosted instances)",
+            default="",
+        )
+        description = click.prompt(
+            "Documentation classifier model file",
+            default=configuration.default_description,
+        )
         invocation = click.prompt("Invocation classifier model file", default=configuration.default_invocation)
-        installation = click.prompt("Installation classifier model file", default=configuration.default_installation)
+        installation = click.prompt(
+            "Installation classifier model file",
+            default=configuration.default_installation,
+        )
         citation = click.prompt("Citation classifier model file", default=configuration.default_citation)
         base_uri = click.prompt("Base URI for RDF generation", default=base_uri)
-        # configuration.configure()
-        configuration.configure(authorization, description, invocation, installation, citation, base_uri)
+        configuration.configure(
+            github_authorization,
+            gitlab_authorization,
+            description,
+            invocation,
+            installation,
+            citation,
+            base_uri,
+        )
     click.secho(f"Success", fg="green")
 
 
@@ -57,24 +81,24 @@ def configure(auto, base_uri):
     "-ic",
     is_flag=True,
     default=False,
-    help="Flag to ignore running the classifiers (by default False)"
+    help="Flag to ignore running the classifiers (by default False)",
 )
 @click.option(
     "--ignore_github_metadata",
     "-igm",
     is_flag=True,
     default=False,
-    help="Flag to ignore Github Metadata (by default False)"
+    help="Flag to ignore Github Metadata (by default False)",
 )
 @click.option(
     "--readme_only",
     "-ro",
     is_flag=True,
     default=False,
     help="Flag to retrieve only the README.md file from the Github/Gitlab Repository URL. If such file does not exist, "
-         "no metadata will be retrieved (by default False)"
+    "no metadata will be retrieved (by default False)",
 )
-@optgroup.group('Input', cls=RequiredMutuallyExclusiveOptionGroup)
+@optgroup.group("Input", cls=RequiredMutuallyExclusiveOptionGroup)
 @optgroup.option(
     "--repo_url",
     "-r",
@@ -85,23 +109,23 @@ def configure(auto, base_uri):
     "--doc_src",
     "-d",
     type=click.Path(exists=True),
-    help="Path to the README file source"
+    help="Path to the README file source",
 )
 @optgroup.option(
     "--local_repo",
     "-l",
     type=click.Path(exists=True),
-    help="Path to local repository"
+    help="Path to local repository",
 )
 @optgroup.option(
     "--in_file",
     "-i",
     type=click.Path(exists=True),
     help=""""A file of newline separated links to GitHub/Gitlab repositories to process in bulk. Each repository will be 
          stored in a different file called $out_$url.json where $out is the name selected as out file and $url is the
-         url of the target repository (url encoded)"""
+         url of the target repository (url encoded)""",
 )
-@optgroup.group('Output', cls=RequiredAnyOptionGroup)
+@optgroup.group("Output", cls=RequiredAnyOptionGroup)
 @optgroup.option(
     "--output",
     "-o",
@@ -112,90 +136,98 @@ def configure(auto, base_uri):
     "--codemeta_out",
     "-c",
     type=click.Path(),
-    help="Path to an output codemeta file"
+    help="Path to an output codemeta file",
 )
 @optgroup.option(
     "--google_codemeta_out",
     "-gc",
     type=click.Path(),
-    help="Path to an output Google-compliant codemeta file"
+    help="Path to an output Google-compliant codemeta file",
 )
 @optgroup.option(
     "--graph_out",
     "-g",
     type=click.Path(),
     help="""Path to the output Knowledge Graph export file. If supplied, the output will be a Knowledge Graph,
-            in the format given in the --format option chosen (turtle, json-ld)"""
+            in the format given in the --format option chosen (turtle, json-ld)""",
 )
 @click.option(
     "--graph_format",
     "-f",
     type=click.Choice(["turtle", "json-ld"]),
     default="turtle",
-    help="""If the --graph_out option is given, this is the format that the graph will be stored in"""
+    help="""If the --graph_out option is given, this is the format that the graph will be stored in""",
 )
 @click.option(
     "--pretty",
     "-p",
     is_flag=True,
     default=False,
-    help="""Pretty print the JSON output file so that it is easy to compare to another JSON output file."""
+    help="""Pretty print the JSON output file so that it is easy to compare to another JSON output file.""",
 )
 @click.option(
     "--missing",
     "-m",
     is_flag=True,
     default=False,
     help="""The JSON will include a field missing_categories to report with the missing metadata fields SOMEF was not 
-    able to find. """
+    able to find. """,
 )
 @click.option(
     "--keep_tmp",
     "-kt",
     type=click.Path(),
     help="""SOMEF will NOT delete the temporary folder where files are stored for analysis. Files will be stored at the
-    desired path"""
+    desired path""",
 )
 @click.option(
     "--ignore_test_folder",
     "-itf",
     is_flag=True,
     default=True,
-    help="""SOMEF will ignore the contents of all files within folders named test (True by default)"""
+    help="""SOMEF will ignore the contents of all files within folders named test (True by default)""",
 )
 @click.option(
     "--requirements_all",
     "-all",
     is_flag=True,
     default=False,
-    help="Export all detected requirements, including text and libraries (default)."
+    help="Export all detected requirements, including text and libraries (default).",
 )
 @click.option(
     "--requirements_v",
     "-v",
     is_flag=True,
     default=False,
-    help="Export only requirements from structured sources (pom.xml, requirements.txt, etc.)"
+    help="Export only requirements from structured sources (pom.xml, requirements.txt, etc.)",
 )
 @click.option(
     "--reconcile_authors",
     "-ra",
     is_flag=True,
     default=False,
-    help="""SOMEF will extract additional information from certain files like CODEOWNERS, etc."""
+    help="""SOMEF will extract additional information from certain files like CODEOWNERS, etc.""",
 )
 @click.option(
     "--branch",
     "-b",
     type=str,
     default=None,
-    help="Branch of the repository to analyze. Overrides the default branch."
+    help="Branch of the repository to analyze. Overrides the default branch.",
+)
+@click.option(
+    "--gitlab_token",
+    "-gt",
+    type=str,
+    default=None,
+    help="GitLab personal access token to avoid API rate limits. Works for gitlab.com and self-hosted GitLab instances. "
+    "Can also be set via the SOMEF_GITLAB_TOKEN environment variable or with 'somef configure'.",
 )
 @click.option(
     "--tag",
     type=str,
     default=None,
-    help="Tag of the repository to analyze. Incompatible with --branch"
+    help="Tag of the repository to analyze. Incompatible with --branch",
 )
 def describe(requirements_v, requirements_all, **kwargs):
     # import so missing packages get installed when appropriate
@@ -204,9 +236,9 @@ def describe(requirements_v, requirements_all, **kwargs):
     elif requirements_all:
         kwargs["requirements_mode"] = "all"
     else:
-        kwargs["requirements_mode"] = "all" 
+        kwargs["requirements_mode"] = "all"
 
     from . import somef_cli
+
     somef_cli.run_cli(**kwargs)
     click.secho(f"Success", fg="green")
-

src/somef/configuration.py

@@ -48,12 +48,15 @@ def update_base_uri(base_uri):
             json.dump(data, fh)
 
 
-def configure(authorization="",
-              description=default_description,
-              invocation=default_invocation,
-              installation=default_installation,
-              citation=default_citation,
-              base_uri=constants.CONF_DEFAULT_BASE_URI):
+def configure(
+    github_authorization="",
+    gitlab_authorization="",
+    description=default_description,
+    invocation=default_invocation,
+    installation=default_installation,
+    citation=default_citation,
+    base_uri=constants.CONF_DEFAULT_BASE_URI,
+):
     """ Function to configure the main program"""
     import nltk
     nltk.download('wordnet')
@@ -72,16 +75,21 @@ def configure(authorization="",
     #         data = json.load(fh)
     # else:
     data = {
-        constants.CONF_AUTHORIZATION: "token " + authorization,
         constants.CONF_DESCRIPTION: description,
         constants.CONF_INVOCATION: invocation,
         constants.CONF_INSTALLATION: installation,
         constants.CONF_CITATION: citation,
-        constants.CONF_BASE_URI: base_uri
+        constants.CONF_BASE_URI: base_uri,
     }
 
-    if data[constants.CONF_AUTHORIZATION] == "token ":
-        del data[constants.CONF_AUTHORIZATION]
+    if github_authorization:
+        data[constants.CONF_GITHUB_AUTHORIZATION] = "token " + github_authorization
+
+    if gitlab_authorization:
+        token = gitlab_authorization
+        if not token.lower().startswith("bearer "):
+            token = "Bearer " + token
+        data[constants.CONF_GITLAB_AUTHORIZATION] = token
 
     with credentials_file.open("w") as fh:
         credentials_file.parent.chmod(0o700)

src/somef/parser/codeowners_parser.py

@@ -105,7 +105,7 @@ def parse_codeowners_file(file_path, metadata_result: Result, source, reconcile_
 #         return None
 
 
-def enrich_user(username, repo_type, server_url=None):
+def enrich_user(username, repo_type, server_url=None, gitlab_authorization=None):
     """
     Enrich user metadata using the appropriate platform API.
 
@@ -115,6 +115,8 @@ def enrich_user(username, repo_type, server_url=None):
     repo_type : str "GITHUB" or "GITLAB"
     server_url : str, optional
         Base URL of GitLab instance if repo_type is "GITLAB"
+    gitlab_authorization : str, optional
+        GitLab personal access token (works for gitlab.com and self-hosted instances)
 
     Returns
     -------
@@ -146,7 +148,14 @@ def enrich_user(username, repo_type, server_url=None):
             if not server_url.startswith("http"):
                 server_url = "https://" + server_url
             api_url = f"{server_url.rstrip('/')}/api/v4/users?username={username}"
-            response = requests.get(api_url, timeout=5)
+            # Build auth header — same token works for gitlab.com and self-hosted instances
+            gl_headers = {}
+            if gitlab_authorization:
+                token = gitlab_authorization
+                if not token.startswith("Bearer "):
+                    token = "Bearer " + token
+                gl_headers["Authorization"] = token
+            response = requests.get(api_url, headers=gl_headers, timeout=5)
             if response.status_code != 200:
                 logging.warning(f"GitLab API request failed for {username}: {response.status_code}")
                 return None