chore(ci): fix release script

Author: ivan-aksamentov

.github/workflows/build-and-deploy.yml

@@ -19,12 +19,11 @@ defaults:
     shell: bash -euxo pipefail {0}
 
 env:
-  GITHUB_REPOSITORY_URL: ${{ github.server_url }}/${{ github.repository }}
   VERBOSE: 1
 
 jobs:
   build-and-deploy-datasets:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
 
     environment:
       name: ${{ github.ref }}
@@ -37,6 +36,7 @@ jobs:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AWS_DEFAULT_REGION: us-east-2
+      GH_TOKEN: ${{ github.token }}
 
     steps:
       - name: "Checkout code"
@@ -45,31 +45,6 @@ jobs:
           fetch-depth: 0
           submodules: true
 
-      - name: "Authenticate git"
-        run: |
-          export GITHUB_TOKEN="${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_REPO }}"
-          export BOT_GIT_USER_EMAIL="${{ secrets.BOT_GIT_USER_EMAIL }}"
-          export BOT_GIT_USER_NAME="${{ secrets.BOT_GIT_USER_NAME }}"
-
-          : ${GITHUB_TOKEN?"The env var GITHUB_TOKEN is required"}
-          : ${BOT_GIT_USER_EMAIL?"The env var BOT_GIT_USER_EMAIL is required"}
-          : ${BOT_GIT_USER_NAME?"The env var BOT_GIT_USER_NAME is required"}
-
-          git config --global user.email "${BOT_GIT_USER_EMAIL}"
-          git config --global user.name "${BOT_GIT_USER_NAME}"
-
-          gh auth setup-git >/dev/null
-
-      - name: "Extract branch name"
-        run: |
-          echo "current_branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
-        id: branch-name
-
-      - name: "Checkout branch"
-        run: |
-          git fetch --all
-          git switch ${{ steps.branch-name.outputs.current_branch }}
-
       - name: "Install system dependencies"
         run: |
           sudo apt-get install brotli pigz parallel python3 rename --yes -qq >/dev/null
@@ -90,11 +65,17 @@ jobs:
       - name: "Rebuild, commit and push datasets"
         if: github.ref != 'refs/heads/release'
         run: |
+          git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}"
+          git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}"
+
           ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --push --repo="${GITHUB_REPOSITORY}"
 
       - name: "Rebuild, commit, push and make a release"
         if: github.ref == 'refs/heads/release'
         run: |
+          git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}"
+          git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}"
+
           ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --release --repo="${GITHUB_REPOSITORY}"
 
       - name: "Deploy dataset server"

scripts/lib/changelog.py

@@ -3,7 +3,8 @@
 
 
 def changelog_prepare(dataset, updated_at, changelog_path):
-  path = dataset["path"]
+  path = dict_get(dataset, ["path"])
+  name = dict_get(dataset, ["attributes", "name"])
   release_notes = changelog_get_unreleased_section(changelog_path)
   if len(release_notes) == 0:
     raise ValueError(
@@ -13,10 +14,10 @@ def changelog_prepare(dataset, updated_at, changelog_path):
   full_changelog = file_read(changelog_path).replace(f"## Unreleased", f"## {updated_at}")
   file_write(full_changelog, changelog_path)
 
-  attr_table = format_dataset_attributes_md_table(dict_get_required(dataset, ["attributes"]))
+  # attr_table = format_dataset_attributes_md_table(dict_get_required(dataset, ["attributes"]))
   release_notes = release_notes.replace(
     "## Unreleased",
-    f"""### {path}\n\n{attr_table}""".strip("\n ")
+    f"""## {name} ({path})""".strip("\n ")
   )
 
   return release_notes
@@ -39,10 +40,8 @@ def changelog_get_unreleased_section(changelog_path: str):
 
 
 def format_dataset_attributes_md_table(attributes):
-  attr_table = f"| {'attribute':20} | {'value':20} | {'value friendly':40} |\n"
-  attr_table += f"| {'-' * 20} | {'-' * 20} | {'-' * 40} |\n"
+  attr_table = f"| {'attribute':20} | {'value':40} |\n"
+  attr_table += f"| {'-' * 20} | {'-' * 40} |\n"
   for attr_name, attr_val in attributes.items():
-    value = attr_val["value"]
-    value_friendly = dict_get(attr_val, ["valueFriendly"]) or ""
-    attr_table += f'| {attr_name:20} | {value:20} | {value_friendly:40} |\n'
+    attr_table += f'| {attr_name:20} | {attr_val:40} |\n'
   return attr_table

scripts/lib/container.py

@@ -1,6 +1,6 @@
 from collections import namedtuple, Counter
 from functools import reduce
-from typing import List, Iterable, TypeVar, Callable, Union, Dict, Any
+from typing import List, Iterable, TypeVar, Callable, Union, Dict, Any, Hashable, Optional, Sequence
 
 T = TypeVar('T')
 
@@ -100,6 +100,15 @@ def unique(it: Iterable[T]):
   return iter(set(it))
 
 
+# https://stackoverflow.com/a/49168973
+def unique_by(values: Sequence[T], key: Optional[Callable[[T], Hashable]] = None) -> List[T]:
+  return list(
+    dict.fromkeys(values)
+    if key is None
+    else dict((key(value), value) for value in reversed(values)).values()
+  )
+
+
 def find_duplicates(it: Iterable[T]) -> List[T]:
   return [x for x, occurrences in Counter(it).items() if occurrences > 1]
 

scripts/rebuild

@@ -5,13 +5,14 @@ Builds a fresh data repo from source data
 """
 import argparse
 import json
+import time
 from collections import defaultdict
 from copy import deepcopy
 from os import getcwd
 from os.path import dirname, realpath, join, relpath, isfile
 
 from lib.changelog import changelog_prepare, changelog_get_unreleased_section
-from lib.container import dict_get, dict_get_required, dict_set, unique, find_index_by, first, format_list, \
+from lib.container import dict_get, dict_get_required, dict_set, find_index_by, first, format_list, \
   dict_remove_many, find_duplicates, dict_cleanup
 from lib.date import now_iso, iso_to_iso_safe
 from lib.fasta import fasta_read_exactly_one_seq
@@ -117,14 +118,18 @@ def get_new_dataset_order(datasets, dataset_order):
   return dataset_order
 
 
-def sort_collections(collections, dataset_order):
-  return [collection for x in dataset_order for collection in collections if collection["meta"]["id"] == x]
+def sort_collections(collections, collections_order):
+  return [collection for x in collections_order for collection in collections if collection["meta"]["id"] == x]
 
 
 def sort_datasets(datasets, dataset_order):
   return [dataset for x in dataset_order for dataset in datasets if dataset["path"] == x]
 
 
+def sort_release_infos(release_infos, dataset_order):
+  return [release_info for x in dataset_order for release_info in release_infos if release_info["dataset"]["path"] == x]
+
+
 def parse_args():
   parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
@@ -209,15 +214,15 @@ def main():
   tag = iso_to_iso_safe(updated_at) if args.release else "unreleased"
   git_check_tag(tag)
 
-  collection_json_paths = find_files("collection.json", args.input_dir)
+  collection_json_paths = list(find_files("collection.json", args.input_dir))
 
   collections = []
   release_infos = []
   all_refs = {}
   for collection_json_path in collection_json_paths:
-    collection, release_infos, refs = process_one_collection(collection_json_path, args, tag, updated_at)
+    collection, release_infos_for_dataset, refs = process_one_collection(collection_json_path, args, tag, updated_at)
     collections.append(collection)
-    release_infos.extend(release_infos)
+    release_infos.extend(release_infos_for_dataset)
     all_refs.update(refs)
 
   collections = sort_collections(collections, ["nextstrain", "community"])
@@ -254,10 +259,14 @@ def main():
     commit_hash = commit_changes(args, tag, release_infos)
 
     if args.push:
+      l.info("Pushing commited changes to GitHub")
       git_push()
 
       if args.release:
+        l.info("Releasing to GitHub")
         release_notes = aggregate_release_notes(release_infos)
+        l.info(f"Release notes:\n-------\n{release_notes}\n-------\nEnd of release notes\n")
+        time.sleep(5)
         publish_to_github_releases(args, tag, commit_hash, release_notes)
 
 
@@ -285,6 +294,7 @@ def process_one_collection(collection_json_path, args, tag, updated_at):
   json_write(collection_json, collection_json_path, no_sort_keys=True)
 
   release_infos = prepare_dataset_release_infos(args, datasets, collection_dir, tag, updated_at)
+  release_infos = sort_release_infos(release_infos, dataset_order)
 
   collection_info = deepcopy(collection_json)
 
@@ -351,26 +361,33 @@ def prepare_dataset_release_infos(args, datasets, collection_dir, tag, updated_a
 
 
 def aggregate_release_notes(release_infos):
-  dataset_names_friendly = format_list(
-    unique([get_dataset_name_friendly(release_info["dataset"]) for release_info in release_infos]),
-    sep="\n", marker="- ", quote=False
-  )
-  release_notes = f"This release contains changes for datasets:\n\n{dataset_names_friendly}\n\n\n"
+  dataset_list = format_dataset_list(release_infos)
+  release_notes = f"This release contains changes for datasets:\n\n{dataset_list}\n\n\n"
   for release_info in release_infos:
     release_notes += f'\n{release_info["release_notes"]}\n\n'
   return release_notes
 
 
+def format_dataset_list(release_infos):
+  entries = [format_dataset_list_entry(release_info) for release_info in release_infos]
+  return format_list(entries, sep="\n", marker="- ", quote=False)
+
+
+def format_dataset_list_entry(release_info):
+  path = dict_get_required(release_info, ['dataset', 'path'])
+  name = get_dataset_name_friendly(release_info['dataset'])
+  return f"{path} ({name})"
+
+
 def commit_changes(args, tag, release_infos):
-  l.info(f"Commiting changes for '{tag}'")
+  l.info(f"Committing changes for '{tag}'")
 
   commit_message = "chore: rebuild [skip ci]"
   if args.release:
-    dataset_names = format_list(
-      unique([get_dataset_name(release_info["dataset"]) for release_info in release_infos]),
-      sep="\n", marker="- ", quote=False
-    )
-    commit_message = f"chore: release '{tag}'\n\nUpdated datasets:\n\n{dataset_names}"
+    dataset_list = format_dataset_list(release_infos)
+    commit_message = f"chore: release '{tag}'\n\nUpdated datasets:\n\n{dataset_list}"
+
+  l.info(f"Commit message:\n--------\n{commit_message}\n--------\nEnd of commit message\n\n")
 
   return git_commit_all(commit_message)
 
@@ -438,12 +455,8 @@ def create_dataset_package(args, dataset, pathogen_json, tag, dataset_dir):
   file_write("User-agent: *\nDisallow: /\n", join(args.output_dir, "robots.txt"))
 
 
-def get_dataset_name(dataset):
-  return dict_get_required(dataset, ["attributes", "name", "value"])
-
-
 def get_dataset_name_friendly(dataset):
-  return dict_get(dataset, ["attributes", "name", "valueFriendly"]) or get_dataset_name(dataset)
+  return dict_get_required(dataset, ["attributes", "name"])
 
 
 if __name__ == '__main__':