Merge pull request #50 from ElrondNetwork/testnet-fixes

Testnet and other fixes

Author: camilbancioiu

erdpy/cli.py

@@ -19,7 +19,7 @@
 import erdpy.cli_transactions
 import erdpy.cli_validators
 import erdpy.cli_wallet
-import erdpy.cli_delagation
+import erdpy.cli_delegation
 import erdpy.cli_dns
 from erdpy import config, errors, scope
 from erdpy._version import __version__
@@ -99,7 +99,7 @@ def setup_parser(args: List[str] = sys.argv[1:]):
     commands.append(erdpy.cli_block.setup_parser(subparsers))
     commands.append(erdpy.cli_testnet.setup_parser(args, subparsers))
     commands.append(erdpy.cli_data.setup_parser(subparsers))
-    commands.append(erdpy.cli_delagation.setup_parser(args, subparsers))
+    commands.append(erdpy.cli_delegation.setup_parser(args, subparsers))
     commands.append(erdpy.cli_dns.setup_parser(args, subparsers))
 
     parser.epilog = """

erdpy/cli_delagation.py erdpy/cli_delegation.pyerdpy/cli_delagation.py renamed to erdpy/cli_delegation.py

@@ -11,7 +11,7 @@ def setup_parser(subparsers: Any) -> Any:
     parser = cli_shared.add_group_subparser(subparsers, "deps", "Manage dependencies or elrond-sdk modules")
     subparsers = parser.add_subparsers()
 
-    choices = list(get_deps_dict().keys())
+    choices = ['all'] + list(get_deps_dict().keys())
 
     sub = cli_shared.add_command_subparser(subparsers, "deps", "install", "Install dependencies or elrond-sdk modules.")
     sub.add_argument("name", choices=choices, help="the dependency to install")

erdpy/cli_deps.py

@@ -1,7 +1,9 @@
 import os.path
+import semver
 from typing import Any, Dict, List
+from pathlib import Path
 
-from erdpy import errors, utils
+from erdpy import errors, utils, workstation
 
 ROOT_FOLDER_NAME = "elrondsdk"
 LOCAL_CONFIG_PATH = os.path.join(os.getcwd(), "erdpy.json")
@@ -170,6 +172,8 @@ def get_defaults() -> Dict[str, Any]:
         "dependencies.mcl_signer.tag": "latest",
         "dependencies.mcl_signer.urlTemplate.linux": "https://github.com/ElrondNetwork/elrond-sdk-go-tools/releases/download/{TAG}/mcl_signer_{TAG}_ubuntu-latest.tar.gz",
         "dependencies.mcl_signer.urlTemplate.osx": "https://github.com/ElrondNetwork/elrond-sdk-go-tools/releases/download/{TAG}/mcl_signer_{TAG}_macos-latest.tar.gz",
+        "testnet.validate_expected_keys": "false",
+        "github_api_token": "",
     }
 
 
@@ -236,3 +240,44 @@ def determine_final_args(argv: List[str], config_args: Dict[str, Any]) -> List[s
         pre_args = [verbose_flag]
 
     return pre_args + argv + extra_args
+
+
+def get_dependency_directory(key: str, tag: str) -> Path:
+    parent_directory = get_dependency_parent_directory(key)
+    if tag == 'latest':
+        tag = get_latest_semver_from_directory(parent_directory)
+
+    return parent_directory / tag
+
+
+def get_dependency_parent_directory(key: str) -> Path:
+    tools_folder = Path(workstation.get_tools_folder())
+    return tools_folder / key
+
+
+def get_latest_semver_from_directory(directory: Path) -> str:
+    subdirs = [subdir.name for subdir in directory.iterdir()]
+    versions = parse_strings_to_semver(subdirs)
+    if len(versions) == 0:
+        raise Exception(f'no versions found in {directory}')
+
+    if len(versions) == 1:
+        latest_version = versions[0]
+    else:
+        latest_version = sorted(versions).pop()
+    return 'v' + str(latest_version)
+
+
+def parse_strings_to_semver(version_strings: List[str]) -> List[semver.VersionInfo]:
+    versions = []
+    for version_string in version_strings:
+        try:
+            # Omit the 'v' prefix of the version string
+            version_string = version_string[1:]
+            version = semver.VersionInfo.parse(version_string)
+        except ValueError:
+            continue
+
+        versions.append(version)
+
+    return versions

erdpy/config.py

@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 from typing import Dict, List
 
 from erdpy import config, errors
@@ -10,11 +11,16 @@
 
 
 def install_module(key: str, tag: str = "", overwrite: bool = False):
-    module = get_module_by_key(key)
-    module.install(tag, overwrite)
+    if key == 'all':
+        modules = get_all_deps()
+    else:
+        modules = [get_module_by_key(key)]
+
+    for module in modules:
+        module.install(tag, overwrite)
 
 
-def get_module_directory(key: str) -> str:
+def get_module_directory(key: str) -> Path:
     module = get_module_by_key(key)
     default_tag = config.get_dependency_tag(key)
     directory = module.get_directory(default_tag)

erdpy/dependencies/install.py

@@ -15,7 +15,7 @@ def __init__(self, key: str, aliases: List[str]):
         self.key = key
         self.aliases = aliases
 
-    def get_directory(self, tag: str) -> str:
+    def get_directory(self, tag: str) -> Path:
         raise NotImplementedError()
 
     def install(self, tag: str, overwrite: bool) -> None:
@@ -88,7 +88,7 @@ def is_installed(self, tag: str) -> bool:
     def _download(self, tag: str):
         url = self._get_download_url(tag)
         archive_path = self._get_archive_path(tag)
-        downloader.download(url, archive_path)
+        downloader.download(url, str(archive_path))
 
     def _extract(self, tag: str):
         archive_path = self._get_archive_path(tag)
@@ -101,26 +101,23 @@ def _extract(self, tag: str):
         else:
             raise errors.UnknownArchiveType(self.archive_type)
 
-    def get_directory(self, tag: str):
-        folder = path.join(self.get_parent_directory(), tag)
-        return folder
+    def get_directory(self, tag: str) -> Path:
+        return config.get_dependency_directory(self.key, tag)
 
     def get_source_directory(self, tag: str):
-        folder = Path(self.get_directory(tag))
-
         # Due to how the GitHub creates archives for repository releases, the
         # path will contain the tag in two variants: with the 'v' prefix (e.g.
         # "v1.1.0"), but also without (e.g. "1.1.0"), hence the need to remove
         # the initial 'v'.
-        if tag.startswith("v"):
-            tag = tag[1:]
+        tag_no_v = tag
+        if tag_no_v.startswith("v"):
+            tag_no_v = tag_no_v[1:]
         assert isinstance(self.repo_name, str)
-        source_folder = folder / (self.repo_name + '-' + tag)
+        source_folder = self.get_directory(tag) / f'{self.repo_name}-{tag_no_v}'
         return source_folder
 
-    def get_parent_directory(self):
-        tools_folder = workstation.get_tools_folder()
-        return path.join(tools_folder, self.key)
+    def get_parent_directory(self) -> Path:
+        return config.get_dependency_parent_directory(self.key)
 
     def _get_download_url(self, tag: str) -> str:
         platform = workstation.get_platform()
@@ -140,9 +137,9 @@ def get_latest_release(self) -> str:
         tag = utils.query_latest_release_tag(org_repo)
         return tag
 
-    def _get_archive_path(self, tag: str) -> str:
-        tools_folder = workstation.get_tools_folder()
-        archive = path.join(tools_folder, f"{self.key}.{tag}.{self.archive_type}")
+    def _get_archive_path(self, tag: str) -> Path:
+        tools_folder = Path(workstation.get_tools_folder())
+        archive = tools_folder / f"{self.key}.{tag}.{self.archive_type}"
         return archive
 
 
@@ -163,6 +160,7 @@ def _post_install(self, tag: str):
 
         self.make_binary_symlink_in_parent_folder(tag, 'arwendebug', 'arwendebug')
         self.make_binary_symlink_in_parent_folder(tag, 'test', 'mandos-test')
+        self.copy_libwasmer_in_parent_directory(tag)
 
     def build_binary(self, tag, binary_name):
         source_folder = self.binary_source_folder(tag, binary_name)
@@ -178,12 +176,19 @@ def make_binary_symlink_in_parent_folder(self, tag, binary_name, symlink_name):
         source_folder = self.binary_source_folder(tag, binary_name)
         binary = source_folder / binary_name
 
-        parent = Path(self.get_parent_directory())
+        parent = self.get_parent_directory()
         symlink = parent / symlink_name
 
         symlink.unlink(missing_ok=True)
         symlink.symlink_to(binary)
 
+    def copy_libwasmer_in_parent_directory(self, tag):
+        libwasmer_directory = self.get_source_directory(tag) / 'wasmer'
+        parent_directory = self.get_parent_directory()
+        for f in libwasmer_directory.iterdir():
+            if f.suffix in ['.dylib', '.so', '.dll']:
+                shutil.copy(f, parent_directory)
+
     def get_env(self):
         return {
         }

erdpy/dependencies/modules.py

@@ -19,9 +19,13 @@ def initialize():
     if os.path.exists(testnet_toml):
         global chain_id, proxy
 
-        testnet_config = TestnetConfiguration.from_file(testnet_toml)
         chain_id = CHAIN_ID
-        proxy = f"http://localhost:{testnet_config.proxy_port()}"
+
+        try:
+            testnet_config = TestnetConfiguration.from_file(testnet_toml)
+            proxy = f"http://localhost:{testnet_config.proxy_port()}"
+        except FileNotFoundError:
+            logger.warn("components of the testnet may be missing")
 
 
 def get_chain_id():

erdpy/scope.py

@@ -60,6 +60,10 @@ def __init__(self, config):
             folder_path = folder_path.replace('{ELRONDSDK}', str(sdk_folder))
 
             default_tag = erdpy.config.get_dependency_tag(folder_key)
+            if default_tag == 'latest':
+                parent_folder = erdpy.config.get_dependency_parent_directory(folder_key)
+                default_tag = erdpy.config.get_latest_semver_from_directory(parent_folder)
+
             folder_path = folder_path.replace('{TAG}', default_tag)
 
             # If the user has not specified a custom source repository, the

erdpy/testnet/config.py

@@ -1,5 +1,7 @@
 from typing import Dict, Any
 
+import erdpy.config
+
 from erdpy.testnet.config import TestnetConfiguration
 from erdpy.testnet.nodes_setup_json import CHAIN_ID
 
@@ -84,29 +86,34 @@ def patch_enable_epochs(data: ConfigDict, testnet_config: TestnetConfiguration):
     enable_epochs['BackwardCompSaveKeyValueEnableEpoch'] = 5
     enable_epochs['SCRSizeInvariantCheckEnableEpoch'] = 5
 
-    enable_epochs['MultiESDTTransferFixOnCallBackOnEnableEpoch'] = 5
-    enable_epochs['ESDTNFTCreateOnMultiShard'] = 5
-    enable_epochs['RemoveNonUpdatedStorageEnableEpoch'] = 5
-    enable_epochs['FixOOGReturnCodeEnableEpoch'] = 5
-    enable_epochs['AddTokensToDelegationEnableEpoch'] = 5
-    enable_epochs['CorrectFirstQueuedEpoch'] = 5
-    enable_epochs['MetaESDTSetEnableEpoch'] = 5
-    enable_epochs['OptimizeGasUsedInCrossMiniBlocksEnableEpoch'] = 5
-    enable_epochs['DeleteDelegatorAfterClaimRewardsEnableEpoch'] = 5
+    enable_epochs['MultiESDTTransferFixOnCallBackOnEnableEpoch'] = 0
+    enable_epochs['ESDTNFTCreateOnMultiShard'] = 0
+    enable_epochs['RemoveNonUpdatedStorageEnableEpoch'] = 0
+    enable_epochs['FixOOGReturnCodeEnableEpoch'] = 0
+    enable_epochs['AddTokensToDelegationEnableEpoch'] = 0
+    enable_epochs['CorrectFirstQueuedEpoch'] = 0
+    enable_epochs['MetaESDTSetEnableEpoch'] = 0
+    enable_epochs['OptimizeGasUsedInCrossMiniBlocksEnableEpoch'] = 0
+    enable_epochs['DeleteDelegatorAfterClaimRewardsEnableEpoch'] = 0
 
     enable_epochs['MaxNodesChangeEnableEpoch'] = [
         {'EpochEnable': 0, 'MaxNumNodes': 36, 'NodesToShufflePerShard': 4},
         {'EpochEnable': 1, 'MaxNumNodes': 56, 'NodesToShufflePerShard': 2}
     ]
 
-    validate_expected_keys(data['EnableEpochs'], enable_epochs)
+    validate = erdpy.config.get_value('testnet.validate_expected_keys') == 'true'
+
+    if validate:
+        validate_expected_keys(data['EnableEpochs'], enable_epochs)
     data['EnableEpochs'].update(enable_epochs)
 
     gas_schedule = dict()
     gas_schedule['GasScheduleByEpochs'] = [
         {'StartEpoch': 0, 'FileName': 'gasScheduleV3.toml'}
     ]
-    validate_expected_keys(data['GasSchedule'], gas_schedule)
+
+    if validate:
+        validate_expected_keys(data['GasSchedule'], gas_schedule)
     data['GasSchedule'].update(gas_schedule)
 
 

erdpy/testnet/node_config_toml.py

@@ -0,0 +1,17 @@
+import semver
+from erdpy.dependencies.modules import StandaloneModule
+
+
+def test_semver_parsing():
+    v = semver.VersionInfo.parse('1.2.3')
+    assert (v.major, v.minor, v.patch) == (1, 2, 3)
+
+    v = semver.VersionInfo.parse('1.2.0-beta.3')
+    assert (v.major, v.minor, v.patch) == (1, 2, 0)
+    assert v.prerelease == 'beta.3'
+
+
+def test_semver_sorting():
+    folders = ['master', 'development', 'v1.2.3', 'v1.3.19', 'v0.1.1-beta.2']
+    latest = StandaloneModule.get_latest_semver_from_folder_list(folders)
+    assert latest == 'v1.3.19'