github: export more fields

Author: Gerit Wagner

colrev/constants.py

@@ -193,11 +193,31 @@ class Fields:
     EUROPE_PMC_ID = "colrev.europe_pmc.europe_pmc_id"
     PROSPERO_ID = "colrev.prospero.id"
 
+    # --- GitHub fields ---
     GITHUB_VERSION = "colrev.github.version"
     GITHUB_LICENSE = "colrev.github.license"
     GITHUB_LANGUAGE = "colrev.github.language"
     GITHUB_NR_CONTRIBUTORS = "colrev.github.nr_contributors"
     GITHUB_NR_COMMITS = "colrev.github.nr_commits"
+    GITHUB_STAR_COUNT = "colrev.github.star_count"
+    GITHUB_WATCHER_COUNT = "colrev.github.watcher_count"
+    GITHUB_FORK_COUNT = "colrev.github.fork_count"
+    GITHUB_ISSUE_COUNT_OPEN = "colrev.github.issue_count_open"
+    GITHUB_ISSUE_COUNT_CLOSED = "colrev.github.issue_count_closed"
+    GITHUB_ISSUE_COUNT_TOTAL = "colrev.github.issue_count_total"
+    GITHUB_COMMIT_LAST_DATE = "colrev.github.commit_last_date"  # YYYY-MM-DD
+    GITHUB_LATEST_RELEASE_DATE = "colrev.github.latest_release_date"  # YYYY-MM-DD
+    GITHUB_REPO_IS_ARCHIVED = "colrev.github.repo_is_archived"  # bool
+    GITHUB_REPO_IS_ARCHIVED_DATE = (
+        "colrev.github.repo_is_archived_date"  # YYYY-MM-DD (proxy)
+    )
+    GITHUB_LANGUAGES_PCT = "colrev.github.languages_pct"  # {lang: pct}
+    GITHUB_BRANCH_COUNT_ACTIVE = "colrev.github.branch_count_active"
+    GITHUB_BRANCH_COUNT_STALE = "colrev.github.branch_count_stale"
+    GITHUB_BRANCH_COUNT_ALL = "colrev.github.branch_count_all"
+    GITHUB_BASIC_INFO_OWNER = "colrev.github.basic_info_owner"
+    GITHUB_BASIC_INFO_REPO = "colrev.github.basic_info_repo"
+    GITHUB_BASIC_INFO_URL = "colrev.github.basic_info_url"
 
 
 class FieldsRegex:

colrev/packages/github/src/record_transformer.py

@@ -3,9 +3,14 @@
 from __future__ import annotations
 
 import re
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+from typing import Dict
 
 from github import Github
 from github.GithubException import GithubException
+from github.GithubException import UnknownObjectException
 
 import colrev.record.record
 import colrev.record.record_prep
@@ -84,6 +89,122 @@ def _set_nr_commits(record_dict: dict, repo: Github.Repository.Repository) -> No
         pass
 
 
+def _set_issue_counts(record_dict: dict, repo: Github.Repository.Repository) -> None:
+    try:
+        open_ = repo.get_issues(state="open").totalCount
+        closed = repo.get_issues(state="closed").totalCount
+        record_dict[Fields.GITHUB_ISSUE_COUNT_OPEN] = open_
+        record_dict[Fields.GITHUB_ISSUE_COUNT_CLOSED] = closed
+        record_dict[Fields.GITHUB_ISSUE_COUNT_TOTAL] = open_ + closed
+    except GithubException:
+        record_dict[Fields.GITHUB_ISSUE_COUNT_OPEN] = 0
+        record_dict[Fields.GITHUB_ISSUE_COUNT_CLOSED] = 0
+        record_dict[Fields.GITHUB_ISSUE_COUNT_TOTAL] = 0
+
+
+def _set_stars_watch_forks(
+    record_dict: dict, repo: Github.Repository.Repository
+) -> None:
+    try:
+        record_dict[Fields.GITHUB_STAR_COUNT] = (
+            getattr(repo, "stargazers_count", 0) or 0
+        )
+    except GithubException:
+        record_dict[Fields.GITHUB_STAR_COUNT] = 0
+    try:
+        record_dict[Fields.GITHUB_WATCHER_COUNT] = (
+            getattr(repo, "subscribers_count", 0) or 0
+        )
+    except GithubException:
+        record_dict[Fields.GITHUB_WATCHER_COUNT] = 0
+    try:
+        record_dict[Fields.GITHUB_FORK_COUNT] = getattr(repo, "forks_count", 0) or 0
+    except GithubException:
+        record_dict[Fields.GITHUB_FORK_COUNT] = 0
+
+
+def _set_archive_status(record_dict: dict, repo: Github.Repository.Repository) -> None:
+    try:
+        is_archived = bool(getattr(repo, "archived", False))
+        record_dict[Fields.GITHUB_REPO_IS_ARCHIVED] = is_archived
+        if is_archived:
+            pushed_at = getattr(repo, "pushed_at", None)
+            if pushed_at:
+                record_dict[Fields.GITHUB_REPO_IS_ARCHIVED_DATE] = pushed_at.strftime(
+                    "%Y-%m-%d"
+                )
+    except GithubException:
+        record_dict[Fields.GITHUB_REPO_IS_ARCHIVED] = False
+
+
+def _set_last_commit_date(
+    record_dict: dict, repo: Github.Repository.Repository
+) -> None:
+    try:
+        c = repo.get_commits()[0]
+        dt = c.commit.author.date
+        record_dict[Fields.GITHUB_COMMIT_LAST_DATE] = dt.strftime("%Y-%m-%d")
+    except (GithubException, IndexError):
+        pass
+
+
+def _set_latest_release_date(
+    record_dict: dict, repo: Github.Repository.Repository
+) -> None:
+    try:
+        rel = repo.get_latest_release()
+        if rel and getattr(rel, "published_at", None):
+            record_dict[Fields.GITHUB_LATEST_RELEASE_DATE] = rel.published_at.strftime(
+                "%Y-%m-%d"
+            )
+    except (UnknownObjectException, GithubException):
+        pass
+
+
+def _calc_language_percentages(repo: Github.Repository.Repository) -> Dict[str, float]:
+    try:
+        lang_bytes = repo.get_languages()
+        total = sum(lang_bytes.values()) or 0
+        if total == 0:
+            return {}
+        return {k: round((v / total) * 100.0, 2) for k, v in lang_bytes.items()}
+    except GithubException:
+        return {}
+
+
+def _set_languages_pct(record_dict: dict, repo: Github.Repository.Repository) -> None:
+    lang_pct = _calc_language_percentages(repo)
+    if lang_pct:
+        record_dict[Fields.GITHUB_LANGUAGES_PCT] = lang_pct
+
+
+def _set_branch_statuses(record_dict: dict, repo: Github.Repository.Repository) -> None:
+    try:
+        branches = list(repo.get_branches())
+    except GithubException:
+        return
+
+    active = stale = 0
+    now = datetime.now(timezone.utc)
+    threshold = now - timedelta(days=90)
+
+    for br in branches:
+        try:
+            sha = br.commit.sha
+            commit = repo.get_commit(sha)
+            dt = commit.commit.author.date
+            if dt >= threshold:
+                active += 1
+            else:
+                stale += 1
+        except GithubException:
+            stale += 1
+
+    record_dict[Fields.GITHUB_BRANCH_COUNT_ACTIVE] = active
+    record_dict[Fields.GITHUB_BRANCH_COUNT_STALE] = stale
+    record_dict[Fields.GITHUB_BRANCH_COUNT_ALL] = active + stale
+
+
 def _update_record_based_on_citation_cff(
     record_dict: dict, repo: Github.Repository.Repository
 ) -> dict:
@@ -114,11 +235,22 @@ def repo_to_record(
         Fields.URL: repo.html_url,
         Fields.ABSTRACT: repo.description,
         Fields.GITHUB_LANGUAGE: repo.language,
+        # Basic info triplet
+        Fields.GITHUB_BASIC_INFO_OWNER: repo.owner.login,
+        Fields.GITHUB_BASIC_INFO_REPO: repo.name,
+        Fields.GITHUB_BASIC_INFO_URL: repo.html_url,
     }
 
     _set_license(record_dict, repo)
     _set_nr_contributors(record_dict, repo)
     _set_nr_commits(record_dict, repo)
+    _set_issue_counts(record_dict, repo)
+    _set_stars_watch_forks(record_dict, repo)
+    _set_archive_status(record_dict, repo)
+    _set_last_commit_date(record_dict, repo)
+    _set_latest_release_date(record_dict, repo)
+    _set_languages_pct(record_dict, repo)
+    _set_branch_statuses(record_dict, repo)
 
     # Use data from CITATION.cff file (if available)
     _update_record_based_on_citation_cff(record_dict, repo)