Add support for 'latest' tags where applicable

Author: camilbancioiu

erdpy/config.py

@@ -146,28 +146,28 @@ def get_defaults() -> Dict[str, Any]:
         "proxy": "https://testnet-gateway.elrond.com",
         "chainID": "T",
         "txVersion": "1",
-        "dependencies.arwentools.tag": "v1.1.2",
-        "dependencies.elrond_wasm_rs.tag": "v0.12.0",
+        "dependencies.arwentools.tag": "latest",
+        "dependencies.elrond_wasm_rs.tag": "latest",
         "dependencies.arwentools.urlTemplate.linux": "https://github.com/ElrondNetwork/arwen-wasm-vm/archive/{TAG}.tar.gz",
         "dependencies.arwentools.urlTemplate.osx": "https://github.com/ElrondNetwork/arwen-wasm-vm/archive/{TAG}.tar.gz",
         "dependencies.llvm.tag": "v9-19feb",
         "dependencies.llvm.urlTemplate.linux": "https://ide.elrond.com/vendor-llvm/{TAG}/linux-amd64.tar.gz?t=19feb",
         "dependencies.llvm.urlTemplate.osx": "https://ide.elrond.com/vendor-llvm/{TAG}/darwin-amd64.tar.gz?t=19feb",
-        "dependencies.rust.tag": "",
+        "dependencies.rust.tag": "nightly",
         "dependencies.nodejs.tag": "v12.18.3",
         "dependencies.nodejs.urlTemplate.linux": "https://nodejs.org/dist/{TAG}/node-{TAG}-linux-x64.tar.gz",
         "dependencies.nodejs.urlTemplate.osx": "https://nodejs.org/dist/{TAG}/node-{TAG}-darwin-x64.tar.gz",
-        "dependencies.elrond_go.tag": "master",
+        "dependencies.elrond_go.tag": "latest",
         "dependencies.elrond_go.urlTemplate.linux": "https://github.com/ElrondNetwork/elrond-go/archive/{TAG}.tar.gz",
         "dependencies.elrond_go.urlTemplate.osx": "https://github.com/ElrondNetwork/elrond-go/archive/{TAG}.tar.gz",
         "dependencies.elrond_go.url": "https://github.com/ElrondNetwork/elrond-go/archive/{TAG}.tar.gz",
-        "dependencies.elrond_proxy_go.tag": "master",
+        "dependencies.elrond_proxy_go.tag": "latest",
         "dependencies.elrond_proxy_go.urlTemplate.linux": "https://github.com/ElrondNetwork/elrond-proxy-go/archive/{TAG}.tar.gz",
         "dependencies.elrond_proxy_go.urlTemplate.osx": "https://github.com/ElrondNetwork/elrond-proxy-go/archive/{TAG}.tar.gz",
         "dependencies.golang.tag": "go1.15.2",
         "dependencies.golang.urlTemplate.linux": "https://golang.org/dl/{TAG}.linux-amd64.tar.gz",
         "dependencies.golang.urlTemplate.osx": "https://golang.org/dl/{TAG}.darwin-amd64.tar.gz",
-        "dependencies.mcl_signer.tag": "v1.0.0",
+        "dependencies.mcl_signer.tag": "latest",
         "dependencies.mcl_signer.urlTemplate.linux": "https://github.com/ElrondNetwork/elrond-sdk-go-tools/releases/download/{TAG}/mcl_signer_{TAG}_ubuntu-latest.tar.gz",
         "dependencies.mcl_signer.urlTemplate.osx": "https://github.com/ElrondNetwork/elrond-sdk-go-tools/releases/download/{TAG}/mcl_signer_{TAG}_macos-latest.tar.gz",
     }

erdpy/dependencies/install.py

@@ -44,8 +44,8 @@ def get_all_deps() -> List[DependencyModule]:
         ArwenToolsModule(key="arwentools"),
         Rust(key="rust"),
         NodejsModule(key="nodejs", aliases=[]),
-        StandaloneModule(key="elrond_go"),
-        StandaloneModule(key="elrond_proxy_go"),
+        StandaloneModule(key="elrond_go", repo_name="elrond-go", organisation="ElrondNetwork"),
+        StandaloneModule(key="elrond_proxy_go", repo_name="elrond-proxy-go", organisation="ElrondNetwork"),
         GolangModule(key="golang"),
         MclSignerModule(key="mcl_signer")
     ]

erdpy/dependencies/modules.py

@@ -2,7 +2,7 @@
 import os
 import shutil
 from os import path
-from typing import Dict, List
+from typing import Dict, List, Optional
 from pathlib import Path
 
 from erdpy import config, dependencies, downloader, errors, myprocess, utils, workstation
@@ -22,6 +22,9 @@ def install(self, tag: str, overwrite: bool) -> None:
         # Fallback to default tag if not provided
         tag = tag or config.get_dependency_tag(self.key)
 
+        if tag == 'latest':
+            tag = self.get_latest_release()
+
         logger.debug(f"install: key={self.key}, tag={tag}")
 
         if self._should_skip(tag, overwrite):
@@ -31,8 +34,6 @@ def install(self, tag: str, overwrite: bool) -> None:
         self.uninstall(tag)
         self._do_install(tag)
 
-        # Upon installation we update the default tag
-        config.set_dependency_tag(self.key, tag)
         self._post_install(tag)
 
     def _do_install(self, tag: str) -> None:
@@ -55,15 +56,23 @@ def is_installed(self, tag: str) -> bool:
     def get_env(self) -> Dict[str, str]:
         raise NotImplementedError()
 
+    def get_latest_release(self) -> str:
+        raise NotImplementedError()
+
 
 class StandaloneModule(DependencyModule):
-    def __init__(self, key: str, aliases: List[str] = None, repo_name=None):
+    def __init__(self,
+                 key: str,
+                 aliases: List[str] = None,
+                 repo_name: Optional[str] = None,
+                 organisation: Optional[str] = None):
         if aliases is None:
             aliases = list()
 
         super().__init__(key, aliases)
         self.archive_type = "tar.gz"
         self.repo_name = repo_name
+        self.organisation = organisation
 
     def _do_install(self, tag: str):
         self._download(tag)
@@ -73,7 +82,7 @@ def uninstall(self, tag: str):
         if os.path.isdir(self.get_directory(tag)):
             shutil.rmtree(self.get_directory(tag))
 
-    def is_installed(self, tag: str):
+    def is_installed(self, tag: str) -> bool:
         return path.isdir(self.get_directory(tag))
 
     def _download(self, tag: str):
@@ -105,6 +114,7 @@ def get_source_directory(self, tag: str):
         # the initial 'v'.
         if tag.startswith("v"):
             tag = tag[1:]
+        assert isinstance(self.repo_name, str)
         source_folder = folder / (self.repo_name + '-' + tag)
         return source_folder
 
@@ -114,13 +124,22 @@ def get_parent_directory(self):
 
     def _get_download_url(self, tag: str) -> str:
         platform = workstation.get_platform()
+
         url = config.get_dependency_url(self.key, tag, platform)
         if url is None:
             raise errors.PlatformNotSupported(self.key, platform)
 
         url = url.replace("{TAG}", tag)
         return url
 
+    def get_latest_release(self) -> str:
+        if self.repo_name is None or self.organisation is None:
+            raise ValueError(f'{self.key}: repo_name or organisation not specified')
+
+        org_repo = f'{self.organisation}/{self.repo_name}'
+        tag = utils.query_latest_release_tag(org_repo)
+        return tag
+
     def _get_archive_path(self, tag: str) -> str:
         tools_folder = workstation.get_tools_folder()
         archive = path.join(tools_folder, f"{self.key}.{tag}.{self.archive_type}")
@@ -134,6 +153,7 @@ def __init__(self, key: str, aliases: List[str] = None):
 
         super().__init__(key, aliases)
         self.repo_name = 'arwen-wasm-vm'
+        self.organisation = 'ElrondNetwork'
 
     def _post_install(self, tag: str):
         dependencies.install_module('golang')
@@ -195,6 +215,9 @@ def get_env(self):
     def get_gopath(self):
         return path.join(self.get_parent_directory(), "GOPATH")
 
+    def get_latest_release(self) -> str:
+        raise errors.UnsupportedConfigurationValue("Golang tag must always be explicit, not latest")
+
 
 class NodejsModule(StandaloneModule):
     def __init__(self, key: str, aliases: List[str]):
@@ -216,6 +239,9 @@ def get_env(self):
             "PATH": f"{bin_folder}:{os.environ['PATH']}",
         }
 
+    def get_latest_release(self) -> str:
+        raise errors.UnsupportedConfigurationValue("Nodejs tag must always be explicit, not latest")
+
 
 class Rust(DependencyModule):
     def __init__(self, key: str, aliases: List[str] = None):
@@ -242,7 +268,7 @@ def uninstall(self, tag: str):
         if os.path.isdir(directory):
             shutil.rmtree(directory)
 
-    def is_installed(self, tag: str):
+    def is_installed(self, tag: str) -> bool:
         try:
             myprocess.run_process(["rustc", "--version"], env=self.get_env())
             return True
@@ -266,13 +292,18 @@ def get_env(self):
             "CARGO_HOME": directory
         }
 
+    def get_latest_release(self) -> str:
+        raise errors.UnsupportedConfigurationValue("Rust tag must either be explicit, empty or 'nightly'")
+
 
 class MclSignerModule(StandaloneModule):
     def __init__(self, key: str, aliases: List[str] = None):
         if aliases is None:
             aliases = list()
 
         super().__init__(key, aliases)
+        self.organisation = 'ElrondNetwork'
+        self.repo_name = 'elrond-sdk-go-tools'
 
     def _post_install(self, tag: str):
         directory = self.get_directory(tag)

erdpy/errors.py

@@ -130,6 +130,10 @@ def __init__(self, name: str):
         super().__init__(f"This configuration name is protected: {name}.")
 
 
+class UnsupportedConfigurationValue(KnownError):
+    pass
+
+
 class UnknownDerivationFunction(KnownError):
     def __init__(self):
         super().__init__("Unknown key derivation function.")

erdpy/projects/templates_config.py

@@ -2,11 +2,16 @@
 
 from erdpy import config
 from erdpy.projects.templates_repository import TemplatesRepository
+from erdpy.utils import query_latest_release_tag
 
 
 def get_templates_repositories():
     timestamp = int(time.time())
     examples_rs_tag = config.get_dependency_tag('elrond_wasm_rs')
+
+    if examples_rs_tag == 'latest':
+        examples_rs_tag = query_latest_release_tag('ElrondNetwork/elrond-wasm-rs')
+
     examples_rs_tag_no_v = remove_initial_v_from_version(examples_rs_tag)
 
     return [

erdpy/utils.py

@@ -6,10 +6,11 @@
 import shutil
 import stat
 import sys
+import requests
 import tarfile
 import zipfile
 from pathlib import Path
-from typing import Any, AnyStr, List, Union, Optional, cast, IO, Dict
+from typing import Any, List, Union, Optional, cast, IO, Dict
 
 import toml
 
@@ -225,6 +226,18 @@ def parse_keys(bls_public_keys):
     return parsed_keys, len(keys)
 
 
+def query_latest_release_tag(repo: str) -> str:
+    """
+    Queries the Github API to retrieve the latest released tag of the specified
+    repository. The repository must be of the form 'organisation/project'.
+    """
+    url = f'https://api.github.com/repos/{repo}/releases/latest'
+    response = requests.get(url)
+    response.raise_for_status()
+    latest_release_tag = str(response.json()['tag_name'])
+    return latest_release_tag
+
+
 # https://code.visualstudio.com/docs/python/debugging
 def breakpoint():
     import debugpy