From 03a909d80ab8119153826779351daa9b1a0a644a Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:15:49 +0200 Subject: [PATCH 01/25] Update goth --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b7a83d8c3..c6088f9aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,7 @@ tests_unit = {cmd = "pytest --cov=yapapi --cov-report html --cov-report term -sv tests_integration_init = { sequence = ["_gothv_env", "_gothv_requirements", "_gothv_assets"], help="Initialize the integration test environment"} tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError --only-rerun requests.exceptions.ConnectionError --only-rerun OSError --only-rerun requests.exceptions.ReadTimeout", help = "Run the integration tests"} _gothv_env = "python -m venv .envs/yapapi-goth" -_gothv_requirements = ".envs/yapapi-goth/bin/pip install -U --extra-index-url https://test.pypi.org/simple/ goth==0.17.0 pip pytest pytest-asyncio pytest-rerunfailures pexpect" +_gothv_requirements = ".envs/yapapi-goth/bin/pip install -U --extra-index-url https://test.pypi.org/simple/ goth==0.21.1 pip pytest pytest-asyncio pytest-rerunfailures pexpect" _gothv_assets = ".envs/yapapi-goth/bin/python -m goth create-assets tests/goth_tests/assets" clean = {cmd = "rm -rf .coverage .requirements.txt dist md handbook build", help = "Clean all development related files" } From f7808ae614472ab1c737c3e4793b056d743c27ac Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:24:36 +0200 Subject: [PATCH 02/25] Update goth --- .github/workflows/build_publish.yml | 71 ++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_publish.yml b/.github/workflows/build_publish.yml index f6489250c..84564ad2a 100644 --- a/.github/workflows/build_publish.yml +++ b/.github/workflows/build_publish.yml @@ -10,16 +10,28 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v6 with: - python-version: '3.9' - - uses: Gr1N/setup-poetry@v8 - - run: poetry install - - run: poetry run poe tests_unit - - run: poetry run poe checks_codestyle - - run: poetry run poe checks_typing - - run: poetry run poe checks_license + python-version: '3.14' + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 + + - name: Install dependencies + run: poetry install + + - name: Unit tests + run: poetry run poe tests_unit + + - name: Code style, typing and license checks + run: poetry run poe checks_codestyle + + - name: Check typings + run: poetry run poe checks_typing + + - name: Check licenses + run: poetry run poe checks_license build: needs: [test] @@ -27,24 +39,32 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v6 with: - python-version: '3.9' - - uses: Gr1N/setup-poetry@v8 + python-version: '3.14' + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 + - name: Get git release tag run: echo "git-release-tag=yapapi $(git describe --tags)" >> $GITHUB_OUTPUT id: git_describe + - name: Get package version run: echo "poetry-version=$(poetry version)" >> $GITHUB_OUTPUT id: poetry_version + - name: Fail on version mismatch run: exit 1 if: ${{ steps.git_describe.outputs.git-release-tag != steps.poetry_version.outputs.poetry-version }} + - name: Build the release run: poetry build + - name: Store the built package uses: actions/upload-artifact@v4 with: @@ -58,16 +78,21 @@ jobs: if: ${{ github.event.action == 'prereleased' }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v6 with: - python-version: '3.9' - - uses: Gr1N/setup-poetry@v8 + python-version: '3.14' + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 + - name: Retrieve the built package uses: actions/download-artifact@v4 with: name: dist path: dist + - name: Publish to pypi run: | poetry config repositories.testpypi https://test.pypi.org/legacy/ @@ -80,11 +105,15 @@ jobs: if: ${{ github.event.action == 'released' }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v6 with: - python-version: '3.9' - - uses: Gr1N/setup-poetry@v8 + python-version: '3.14' + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 + - name: Retrieve the built package uses: actions/download-artifact@v4 with: From 8e26cb0314fad2915652707edbeaff0b86b8ded1 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:26:22 +0200 Subject: [PATCH 03/25] Update goth --- .github/workflows/goth-nightly.yml | 10 +++++----- .github/workflows/goth.yml | 6 +++--- .github/workflows/tests.yml | 13 +++---------- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index 752adb4ea..e96317c8e 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -47,19 +47,19 @@ jobs: run: sudo apt-get install -y libffi-dev build-essential - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: - python-version: '3.9' + python-version: '3.14' - name: Install and configure Poetry - run: python -m pip install -U pip setuptools poetry==1.3.2 + run: python -m pip install -U pip setuptools poetry==2.2.1 - name: Install dependencies run: | - poetry env use python3.9 + poetry env use python3.14 poetry install - name: Disconnect Docker containers from default network diff --git a/.github/workflows/goth.yml b/.github/workflows/goth.yml index a153091c2..baa4a0d7f 100644 --- a/.github/workflows/goth.yml +++ b/.github/workflows/goth.yml @@ -20,12 +20,12 @@ jobs: uses: actions/checkout@v4 - name: Configure python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: '3.10' + python-version: '3.14' - name: Install Poetry - run: curl -sSL https://install.python-poetry.org | python3 - --version 1.8.2 + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 - name: Install dependencies run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 470ed5025..fbb960b6e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,21 +15,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] os: - ubuntu-latest - - macos-latest - - windows-latest - exclude: - - os: windows-latest - python-version: "3.10" - - os: macos-latest - python-version: "3.10" fail-fast: false steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - uses: Gr1N/setup-poetry@v8 From 13506836cf49388dee312e79e0faaf719431f300 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:40:52 +0200 Subject: [PATCH 04/25] remove license check --- .github/workflows/build_publish.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build_publish.yml b/.github/workflows/build_publish.yml index 84564ad2a..67f0653dc 100644 --- a/.github/workflows/build_publish.yml +++ b/.github/workflows/build_publish.yml @@ -30,9 +30,6 @@ jobs: - name: Check typings run: poetry run poe checks_typing - - name: Check licenses - run: poetry run poe checks_license - build: needs: [test] name: Build the release From d82c1cba3eab668514e56a18dee6ec77fd4c1737 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:41:32 +0200 Subject: [PATCH 05/25] remove license check --- .github/workflows/tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fbb960b6e..e7a78b8cc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,7 +45,5 @@ jobs: - run: poetry run poe checks_codestyle - run: poetry run poe checks_typing if: ${{ steps.extended-checks.outputs.ENABLE }} - - run: poetry run poe checks_license - if: ${{ steps.extended-checks.outputs.ENABLE }} - run: poetry run poe sphinx -W if: ${{ steps.extended-checks-sphinx.outputs.ENABLE }} From 42e55f08e76613db352694f04f48218d2ca2bca9 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:42:25 +0200 Subject: [PATCH 06/25] remove license check --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e7a78b8cc..6ade4d8e1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,7 +32,7 @@ jobs: name: Enable extended checks id: extended-checks - run: echo "ENABLE=1" >> $GITHUB_OUTPUT - if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.9' }} + if: ${{ matrix.os == 'ubuntu-latest' }} name: Enable sphinx check id: extended-checks-sphinx From 5a2bb93ffe3e967c0a524f54805e1cf6cf52ecf6 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:48:04 +0200 Subject: [PATCH 07/25] update deps --- pyproject.toml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c6088f9aa..284db0f10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ requires = ["poetry_core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] -python = "^3.8" +python = "^3.10" aiohttp = "^3.8" aiohttp-sse-client = "^0.2.1" @@ -51,19 +51,19 @@ docs = ['sphinx', 'sphinx-autodoc-typehints', 'sphinx-rtd-theme'] [tool.poetry.group.dev.dependencies] black = "^24.4.2" -factory-boy = "^3.2.0" -isort = "^5.10.1" -liccheck = "^0.4.7" -mypy = "^1.10.0" -poethepoet = "^0.8.0" -pytest = "^6.2" -pytest-asyncio = "^0.14" -pytest-cov = "^2.11" -pytest-rerunfailures = "^10.1" +factory-boy = "^3.3.3" +isort = "^7.0" +liccheck = "^0.9.2" +mypy = "^1.18.2" +poethepoet = "^0.37" +pytest = "^8.4" +pytest-asyncio = "^1.2" +pytest-cov = "^7.0" +pytest-rerunfailures = "^16.1" autoflake = "^1" -flake8 = "^5" -flake8-docstrings = "^1.6" -Flake8-pyproject = "^1.2.2" +flake8 = "^7.3" +flake8-docstrings = "^1.7" +Flake8-pyproject = "^1.2.3" pyproject-autoflake = "^1.0.2" From 6702c63bec0f0231e6c34f8842150e4bb6aae70a Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:50:16 +0200 Subject: [PATCH 08/25] format --- examples/http-auth/utils.py | 114 ++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 examples/http-auth/utils.py diff --git a/examples/http-auth/utils.py b/examples/http-auth/utils.py new file mode 100644 index 000000000..a92810fc2 --- /dev/null +++ b/examples/http-auth/utils.py @@ -0,0 +1,114 @@ +"""Utilities for yapapi example scripts.""" + +import argparse +import asyncio +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +import colorama # type: ignore + +from yapapi import ( + Golem, + NoPaymentAccountError, +) +from yapapi import __version__ as yapapi_version +from yapapi import ( + windows_event_loop_fix, +) +from yapapi.log import enable_default_logger + +TEXT_COLOR_RED = "\033[31;1m" +TEXT_COLOR_GREEN = "\033[32;1m" +TEXT_COLOR_YELLOW = "\033[33;1m" +TEXT_COLOR_BLUE = "\033[34;1m" +TEXT_COLOR_MAGENTA = "\033[35;1m" +TEXT_COLOR_CYAN = "\033[36;1m" +TEXT_COLOR_WHITE = "\033[37;1m" + +TEXT_COLOR_DEFAULT = "\033[0m" + +colorama.init() + + +def build_parser(description: str) -> argparse.ArgumentParser: + current_time_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z") + default_log_path = Path(tempfile.gettempdir()) / f"yapapi_{current_time_str}.log" + + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + "--payment-driver", "--driver", help="Payment driver name, for example `erc20`" + ) + parser.add_argument( + "--payment-network", "--network", help="Payment network name, for example `rinkeby`" + ) + parser.add_argument("--subnet-tag", help="Subnet name, for example `devnet-beta`") + parser.add_argument( + "--log-file", + default=str(default_log_path), + help="Log file for YAPAPI; default: %(default)s", + ) + return parser + + +def format_usage(usage): + return { + "current_usage": usage.current_usage, + "timestamp": usage.timestamp.isoformat(sep=" ") if usage.timestamp else None, + } + + +def print_env_info(golem: Golem): + print( + f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" + f"Using subnet: {TEXT_COLOR_YELLOW}{golem.subnet_tag}{TEXT_COLOR_DEFAULT}, " + f"payment driver: {TEXT_COLOR_YELLOW}{golem.payment_driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{golem.payment_network}{TEXT_COLOR_DEFAULT}\n" + ) + + +def run_golem_example(example_main, log_file=None): + # This is only required when running on Windows with Python prior to 3.8: + windows_event_loop_fix() + + if log_file: + enable_default_logger( + log_file=log_file, + debug_activity_api=True, + debug_market_api=True, + debug_payment_api=True, + debug_net_api=True, + ) + + loop = asyncio.get_event_loop() + task = loop.create_task(example_main) + + try: + loop.run_until_complete(task) + except NoPaymentAccountError as e: + handbook_url = ( + "https://handbook.golem.network/requestor-tutorials/" + "flash-tutorial-of-requestor-development" + ) + print( + f"{TEXT_COLOR_RED}" + f"No payment account initialized for driver `{e.required_driver}` " + f"and network `{e.required_network}`.\n\n" + f"See {handbook_url} on how to initialize payment accounts for a requestor node." + f"{TEXT_COLOR_DEFAULT}" + ) + except KeyboardInterrupt: + print( + f"{TEXT_COLOR_YELLOW}" + "Shutting down gracefully, please wait a short while " + "or press Ctrl+C to exit immediately..." + f"{TEXT_COLOR_DEFAULT}" + ) + task.cancel() + try: + loop.run_until_complete(task) + print( + f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}" + ) + except (asyncio.CancelledError, KeyboardInterrupt): + pass From 978af7b9a5c38a70a299bef83b3126872ebfc335 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:51:43 +0200 Subject: [PATCH 09/25] format --- .github/workflows/tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6ade4d8e1..057663092 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,10 +22,13 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: Gr1N/setup-poetry@v8 + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 - run: echo "ENABLE=1" >> $GITHUB_OUTPUT if: ${{ matrix.os == 'ubuntu-latest' }} From 5e3a09a3d5fa5d9e768f4cd35946e5134fd9ef59 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 15:52:43 +0200 Subject: [PATCH 10/25] fix --- yapapi/props/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 44c91c99d..d95e6b07b 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -223,7 +223,7 @@ def constraint( ``` """ # the default / default_factory exception is resolved by the `field` function - return field( # type: ignore + return field( default=default, default_factory=default_factory, metadata={ From b42fe60d916fb049e73805d48afe0c283b7636f9 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:10:42 +0200 Subject: [PATCH 11/25] fix get_distribution --- yapapi/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yapapi/__init__.py b/yapapi/__init__.py index 51a60d71c..63ca146c1 100644 --- a/yapapi/__init__.py +++ b/yapapi/__init__.py @@ -2,10 +2,10 @@ import asyncio import sys +from importlib.metadata import version from pathlib import Path import toml -from pkg_resources import get_distribution from yapapi.ctx import ExecOptions, WorkContext from yapapi.engine import NoPaymentAccountError @@ -22,7 +22,7 @@ def get_version() -> str: return pyproject["tool"]["poetry"]["version"] - return get_distribution("yapapi").version + return version("yapapi") def windows_event_loop_fix(): From 55649a6e081a2b0b81260e82a9a604991e4bb9a3 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:14:32 +0200 Subject: [PATCH 12/25] fix get_distribution --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 284db0f10..c5cd79630 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ requires = ["poetry_core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] -python = "^3.10" +python = "^3.11" aiohttp = "^3.8" aiohttp-sse-client = "^0.2.1" @@ -42,9 +42,9 @@ pip = "*" alive_progress = "3.1" # Docs -sphinx = { version = "^4.0.1", optional = true } -sphinx-autodoc-typehints = { version = "^1.12.0", optional = true } -sphinx-rtd-theme = { version = "^1.0.0", optional = true} +sphinx = { version = "^8.2", optional = true } +sphinx-autodoc-typehints = { version = "^3.5.1", optional = true } +sphinx-rtd-theme = { version = "^3.0.2", optional = true} [tool.poetry.extras] docs = ['sphinx', 'sphinx-autodoc-typehints', 'sphinx-rtd-theme'] From 7043da8f2d861fd5de0e17e59f63c38f1a391a58 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:19:42 +0200 Subject: [PATCH 13/25] update deps --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c5cd79630..2cda5e993 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ requires = ["poetry_core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] -python = "^3.11" +python = "^3.10" aiohttp = "^3.8" aiohttp-sse-client = "^0.2.1" @@ -42,8 +42,8 @@ pip = "*" alive_progress = "3.1" # Docs -sphinx = { version = "^8.2", optional = true } -sphinx-autodoc-typehints = { version = "^3.5.1", optional = true } +sphinx = { version = "^8.0", optional = true } +sphinx-autodoc-typehints = { version = "^3.0", optional = true } sphinx-rtd-theme = { version = "^3.0.2", optional = true} [tool.poetry.extras] From cb4299de30e701bd5cd9684ba2323c916c72b95d Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:27:36 +0200 Subject: [PATCH 14/25] update deps --- yapapi/payload/vm.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 7bd1b70f1..11d3d1cda 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -40,9 +40,6 @@ class VmRequest(ExeUnitRequest): package_format: VmPackageFormat = prop_base.prop("golem.srv.comp.vm.package_format") -import json - - @dataclass class VmManifestRequest(ExeUnitManifestRequest): def __init__(self, **kwargs): From 87ed13259308e1fed5315926077275674adf6c7d Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:28:11 +0200 Subject: [PATCH 15/25] update deps --- yapapi/payload/vm.py | 96 ++++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 11d3d1cda..0281a0da3 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -148,54 +148,54 @@ async def manifest( capabilities: Optional[List[VmCaps]] = None, ) -> Package: """ - Build a reference to application payload. - - There are two approaches to handle outbound network access in Golem: - - 1. Recommended: Partner Scheme (using node_descriptor) - Uses a signed node descriptor along with a manifest to grant access to either whitelisted - domains or unrestricted access. Providers only need to trust the certificate once. - - example usage:: - - package = await vm.manifest( - manifest = open("manifest_partner_unrestricted.json", "rb").read(), - node_descriptor = json.loads(open("node-descriptor.signed.json", "r").read()), - capabilities = ["inet"], - ) - - 2. Alternative: Pure Manifest Scheme - Requires providers to manually trust each domain listed in the manifest. More complex to set up - and maintain. - - example usage:: - - package = await vm.manifest( - manifest = open("manifest_whitelist.json", "rb").read(), - manifest_sig = open("manifest.json.sha256.sig", "rb").read(), - manifest_sig_algorithm = "sha256", - manifest_cert = open("cert.der", "rb").read(), - capabilities = ["inet", "manifest-support"], - ) - - For more information about outbound access schemes, see: - https://handbook.golem.network/requestor-tutorials/vm-runtime/accessing-internet - - Parameters: - :param manifest: Computation Payload Manifest as raw data or base64 encoded string - :param manifest_sig: Optional signature of manifest (required for pure manifest scheme) - :param manifest_sig_algorithm: Optional signature algorithm, e.g. "sha256" (required for pure manifest scheme) - :param manifest_cert: Optional public certificate for manifest verification (required for pure manifest scheme) - :param node_descriptor: Optional signed node descriptor (recommended for partner scheme) - :param min_mem_gib: Minimal memory required to execute application code - :param min_storage_gib: Minimal disk storage to execute tasks - :param min_cpu_threads: Minimal available logical CPU cores - :param capabilities: Optional list of required VM capabilities. Use ["inet"] for partner scheme - or ["inet", "manifest-support"] for pure manifest scheme - :return: The payload definition for the given VM image - - The manifest, manifest_sig, and manifest_cert parameters can be provided either as raw data - or already base64 encoded. The function will automatically handle the encoding if needed. +Build a reference to application payload. + +There are two approaches to handle outbound network access in Golem: + +1. Recommended: Partner Scheme (using node_descriptor) + Uses a signed node descriptor along with a manifest to grant access to either whitelisted + domains or unrestricted access. Providers only need to trust the certificate once. + + example usage:: + + package = await vm.manifest( + manifest = open("manifest_partner_unrestricted.json", "rb").read(), + node_descriptor = json.loads(open("node-descriptor.signed.json", "r").read()), + capabilities = ["inet"], + ) + +2. Alternative: Pure Manifest Scheme + Requires providers to manually trust each domain listed in the manifest. More complex to set up + and maintain. + + example usage:: + + package = await vm.manifest( + manifest = open("manifest_whitelist.json", "rb").read(), + manifest_sig = open("manifest.json.sha256.sig", "rb").read(), + manifest_sig_algorithm = "sha256", + manifest_cert = open("cert.der", "rb").read(), + capabilities = ["inet", "manifest-support"], + ) + +For more information about outbound access schemes, see: +https://handbook.golem.network/requestor-tutorials/vm-runtime/accessing-internet + +Parameters: +:param manifest: Computation Payload Manifest as raw data or base64 encoded string +:param manifest_sig: Optional signature of manifest (required for pure manifest scheme) +:param manifest_sig_algorithm: Optional signature algorithm, e.g. "sha256" (required for pure manifest scheme) +:param manifest_cert: Optional public certificate for manifest verification (required for pure manifest scheme) +:param node_descriptor: Optional signed node descriptor (recommended for partner scheme) +:param min_mem_gib: Minimal memory required to execute application code +:param min_storage_gib: Minimal disk storage to execute tasks +:param min_cpu_threads: Minimal available logical CPU cores +:param capabilities: Optional list of required VM capabilities. Use ["inet"] for partner scheme + or ["inet", "manifest-support"] for pure manifest scheme +:return: The payload definition for the given VM image + +The manifest, manifest_sig, and manifest_cert parameters can be provided either as raw data +or already base64 encoded. The function will automatically handle the encoding if needed. """ # Helper function to handle encoding From 1e49b9f8c6212759b86ca28182adaac1e67641bb Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:29:02 +0200 Subject: [PATCH 16/25] update deps --- yapapi/payload/vm.py | 98 ++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 0281a0da3..0e98e10f0 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -148,55 +148,55 @@ async def manifest( capabilities: Optional[List[VmCaps]] = None, ) -> Package: """ -Build a reference to application payload. - -There are two approaches to handle outbound network access in Golem: - -1. Recommended: Partner Scheme (using node_descriptor) - Uses a signed node descriptor along with a manifest to grant access to either whitelisted - domains or unrestricted access. Providers only need to trust the certificate once. - - example usage:: - - package = await vm.manifest( - manifest = open("manifest_partner_unrestricted.json", "rb").read(), - node_descriptor = json.loads(open("node-descriptor.signed.json", "r").read()), - capabilities = ["inet"], - ) - -2. Alternative: Pure Manifest Scheme - Requires providers to manually trust each domain listed in the manifest. More complex to set up - and maintain. - - example usage:: - - package = await vm.manifest( - manifest = open("manifest_whitelist.json", "rb").read(), - manifest_sig = open("manifest.json.sha256.sig", "rb").read(), - manifest_sig_algorithm = "sha256", - manifest_cert = open("cert.der", "rb").read(), - capabilities = ["inet", "manifest-support"], - ) - -For more information about outbound access schemes, see: -https://handbook.golem.network/requestor-tutorials/vm-runtime/accessing-internet - -Parameters: -:param manifest: Computation Payload Manifest as raw data or base64 encoded string -:param manifest_sig: Optional signature of manifest (required for pure manifest scheme) -:param manifest_sig_algorithm: Optional signature algorithm, e.g. "sha256" (required for pure manifest scheme) -:param manifest_cert: Optional public certificate for manifest verification (required for pure manifest scheme) -:param node_descriptor: Optional signed node descriptor (recommended for partner scheme) -:param min_mem_gib: Minimal memory required to execute application code -:param min_storage_gib: Minimal disk storage to execute tasks -:param min_cpu_threads: Minimal available logical CPU cores -:param capabilities: Optional list of required VM capabilities. Use ["inet"] for partner scheme - or ["inet", "manifest-support"] for pure manifest scheme -:return: The payload definition for the given VM image - -The manifest, manifest_sig, and manifest_cert parameters can be provided either as raw data -or already base64 encoded. The function will automatically handle the encoding if needed. - """ + Build a reference to application payload. + + There are two approaches to handle outbound network access in Golem: + + 1. Recommended: Partner Scheme (using node_descriptor) + Uses a signed node descriptor along with a manifest to grant access to either whitelisted + domains or unrestricted access. Providers only need to trust the certificate once. + + example usage:: + + package = await vm.manifest( + manifest = open("manifest_partner_unrestricted.json", "rb").read(), + node_descriptor = json.loads(open("node-descriptor.signed.json", "r").read()), + capabilities = ["inet"], + ) + + 2. Alternative: Pure Manifest Scheme + Requires providers to manually trust each domain listed in the manifest. More complex to set up + and maintain. + + example usage:: + + package = await vm.manifest( + manifest = open("manifest_whitelist.json", "rb").read(), + manifest_sig = open("manifest.json.sha256.sig", "rb").read(), + manifest_sig_algorithm = "sha256", + manifest_cert = open("cert.der", "rb").read(), + capabilities = ["inet", "manifest-support"], + ) + + For more information about outbound access schemes, see: + https://handbook.golem.network/requestor-tutorials/vm-runtime/accessing-internet + + Parameters: + :param manifest: Computation Payload Manifest as raw data or base64 encoded string + :param manifest_sig: Optional signature of manifest (required for pure manifest scheme) + :param manifest_sig_algorithm: Optional signature algorithm, e.g. "sha256" (required for pure manifest scheme) + :param manifest_cert: Optional public certificate for manifest verification (required for pure manifest scheme) + :param node_descriptor: Optional signed node descriptor (recommended for partner scheme) + :param min_mem_gib: Minimal memory required to execute application code + :param min_storage_gib: Minimal disk storage to execute tasks + :param min_cpu_threads: Minimal available logical CPU cores + :param capabilities: Optional list of required VM capabilities. Use ["inet"] for partner scheme + or ["inet", "manifest-support"] for pure manifest scheme + :return: The payload definition for the given VM image + + The manifest, manifest_sig, and manifest_cert parameters can be provided either as raw data + or already base64 encoded. The function will automatically handle the encoding if needed. +""" # Helper function to handle encoding def ensure_base64(data: Optional[Union[str, bytes]]) -> Optional[str]: From 0d68f1b67ed9e4c2a9112c18489e7571f70c92c7 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:33:16 +0200 Subject: [PATCH 17/25] update deps --- yapapi/payload/vm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 0e98e10f0..0f5eb663d 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -196,7 +196,8 @@ async def manifest( The manifest, manifest_sig, and manifest_cert parameters can be provided either as raw data or already base64 encoded. The function will automatically handle the encoding if needed. -""" + + """ # Helper function to handle encoding def ensure_base64(data: Optional[Union[str, bytes]]) -> Optional[str]: From 110fe1305a6dbb668514dc5bb3450a9615dd0977 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:33:42 +0200 Subject: [PATCH 18/25] update deps --- yapapi/payload/vm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 0f5eb663d..84225ddbd 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -150,6 +150,7 @@ async def manifest( """ Build a reference to application payload. + There are two approaches to handle outbound network access in Golem: 1. Recommended: Partner Scheme (using node_descriptor) From 286982364157133c383f62beaac9b59ee30c4960 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:36:08 +0200 Subject: [PATCH 19/25] update deps --- yapapi/payload/vm.py | 64 +++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 84225ddbd..8d0e13bef 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -150,54 +150,68 @@ async def manifest( """ Build a reference to application payload. - There are two approaches to handle outbound network access in Golem: 1. Recommended: Partner Scheme (using node_descriptor) Uses a signed node descriptor along with a manifest to grant access to either whitelisted domains or unrestricted access. Providers only need to trust the certificate once. - example usage:: + Example usage:: package = await vm.manifest( - manifest = open("manifest_partner_unrestricted.json", "rb").read(), - node_descriptor = json.loads(open("node-descriptor.signed.json", "r").read()), - capabilities = ["inet"], + manifest=open("manifest_partner_unrestricted.json", "rb").read(), + node_descriptor=json.loads(open("node-descriptor.signed.json", "r").read()), + capabilities=["inet"], ) 2. Alternative: Pure Manifest Scheme Requires providers to manually trust each domain listed in the manifest. More complex to set up and maintain. - example usage:: + Example usage:: package = await vm.manifest( - manifest = open("manifest_whitelist.json", "rb").read(), - manifest_sig = open("manifest.json.sha256.sig", "rb").read(), - manifest_sig_algorithm = "sha256", - manifest_cert = open("cert.der", "rb").read(), - capabilities = ["inet", "manifest-support"], + manifest=open("manifest_whitelist.json", "rb").read(), + manifest_sig=open("manifest.json.sha256.sig", "rb").read(), + manifest_sig_algorithm="sha256", + manifest_cert=open("cert.der", "rb").read(), + capabilities=["inet", "manifest-support"], ) For more information about outbound access schemes, see: https://handbook.golem.network/requestor-tutorials/vm-runtime/accessing-internet - Parameters: - :param manifest: Computation Payload Manifest as raw data or base64 encoded string - :param manifest_sig: Optional signature of manifest (required for pure manifest scheme) - :param manifest_sig_algorithm: Optional signature algorithm, e.g. "sha256" (required for pure manifest scheme) - :param manifest_cert: Optional public certificate for manifest verification (required for pure manifest scheme) - :param node_descriptor: Optional signed node descriptor (recommended for partner scheme) - :param min_mem_gib: Minimal memory required to execute application code - :param min_storage_gib: Minimal disk storage to execute tasks - :param min_cpu_threads: Minimal available logical CPU cores - :param capabilities: Optional list of required VM capabilities. Use ["inet"] for partner scheme - or ["inet", "manifest-support"] for pure manifest scheme - :return: The payload definition for the given VM image - + Parameters + ---------- + manifest : Union[str, bytes] + Computation Payload Manifest as raw data or base64 encoded string. + manifest_sig : Optional[Union[str, bytes]] + Optional signature of manifest (required for pure manifest scheme). + manifest_sig_algorithm : Optional[str] + Optional signature algorithm, e.g. "sha256" (required for pure manifest scheme). + manifest_cert : Optional[Union[str, bytes]] + Optional public certificate for manifest verification (required for pure manifest scheme). + node_descriptor : Optional[Dict[str, Any]] + Optional signed node descriptor (recommended for partner scheme). + min_mem_gib : float + Minimal memory required to execute application code. + min_storage_gib : float + Minimal disk storage to execute tasks. + min_cpu_threads : int + Minimal available logical CPU cores. + capabilities : Optional[List[VmCaps]] + Optional list of required VM capabilities. Use ["inet"] for partner scheme + or ["inet", "manifest-support"] for pure manifest scheme. + + Returns + ------- + Package + The payload definition for the given VM image. + + Notes + ----- The manifest, manifest_sig, and manifest_cert parameters can be provided either as raw data or already base64 encoded. The function will automatically handle the encoding if needed. - """ # Helper function to handle encoding From eb9f833cafa002cb078ab5df9cb2430836f0b212 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:39:05 +0200 Subject: [PATCH 20/25] update python version --- .github/workflows/build_publish.yml | 8 ++++---- .github/workflows/goth-nightly.yml | 4 ++-- .github/workflows/goth.yml | 2 +- .github/workflows/tests.yml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_publish.yml b/.github/workflows/build_publish.yml index 67f0653dc..d68f84631 100644 --- a/.github/workflows/build_publish.yml +++ b/.github/workflows/build_publish.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v6 with: - python-version: '3.14' + python-version: '3.13' - name: Install Poetry run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 @@ -40,7 +40,7 @@ jobs: - uses: actions/setup-python@v6 with: - python-version: '3.14' + python-version: '3.13' - name: Install Poetry run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 @@ -79,7 +79,7 @@ jobs: - uses: actions/setup-python@v6 with: - python-version: '3.14' + python-version: '3.13' - name: Install Poetry run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 @@ -106,7 +106,7 @@ jobs: - uses: actions/setup-python@v6 with: - python-version: '3.14' + python-version: '3.13' - name: Install Poetry run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index e96317c8e..6750b9bf1 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -52,14 +52,14 @@ jobs: - name: Configure python uses: actions/setup-python@v6 with: - python-version: '3.14' + python-version: '3.13' - name: Install and configure Poetry run: python -m pip install -U pip setuptools poetry==2.2.1 - name: Install dependencies run: | - poetry env use python3.14 + poetry env use python3.13 poetry install - name: Disconnect Docker containers from default network diff --git a/.github/workflows/goth.yml b/.github/workflows/goth.yml index baa4a0d7f..422532541 100644 --- a/.github/workflows/goth.yml +++ b/.github/workflows/goth.yml @@ -22,7 +22,7 @@ jobs: - name: Configure python uses: actions/setup-python@v6 with: - python-version: '3.14' + python-version: '3.13' - name: Install Poetry run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 057663092..d3514cbf2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13"] os: - ubuntu-latest fail-fast: false From 39122194546d0434c2ef3e5e4970a95a471640e1 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:43:30 +0200 Subject: [PATCH 21/25] fix goth version --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2cda5e993..705a107fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,9 +85,9 @@ _format_black = "black ." tests_unit = {cmd = "pytest --cov=yapapi --cov-report html --cov-report term -sv --ignore tests/goth_tests", help = "Run only unit tests"} tests_integration_init = { sequence = ["_gothv_env", "_gothv_requirements", "_gothv_assets"], help="Initialize the integration test environment"} -tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError --only-rerun requests.exceptions.ConnectionError --only-rerun OSError --only-rerun requests.exceptions.ReadTimeout", help = "Run the integration tests"} +tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection", help = "Run the integration tests"} _gothv_env = "python -m venv .envs/yapapi-goth" -_gothv_requirements = ".envs/yapapi-goth/bin/pip install -U --extra-index-url https://test.pypi.org/simple/ goth==0.21.1 pip pytest pytest-asyncio pytest-rerunfailures pexpect" +_gothv_requirements = ".envs/yapapi-goth/bin/pip install -U --extra-index-url https://test.pypi.org/simple/ goth==0.22.1 pip pytest pytest-asyncio pytest-rerunfailures pexpect" _gothv_assets = ".envs/yapapi-goth/bin/python -m goth create-assets tests/goth_tests/assets" clean = {cmd = "rm -rf .coverage .requirements.txt dist md handbook build", help = "Clean all development related files" } From 5dcd8661a8ee0d4543ff0eab384766ff1e77b9a0 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 14 Oct 2025 16:47:53 +0200 Subject: [PATCH 22/25] bump yapapi version --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 705a107fa..340e39d75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "yapapi" -version = "0.13.1" +version = "0.14.0" description = "High-level Python API for the New Golem" authors = ["GolemFactory "] license = "LGPL-3.0-or-later" @@ -177,7 +177,6 @@ line_length = 100 [tool.black] line-length = 100 -target-version = ['py38'] [tool.pytest.ini_options] asyncio_mode = "auto" From 42e2d2954ee19cc352398e3a47779c502307215c Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 15 Oct 2025 19:11:52 +0200 Subject: [PATCH 23/25] Fix log path --- tests/goth_tests/conftest.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/goth_tests/conftest.py b/tests/goth_tests/conftest.py index 35e88a76f..b915c0c5f 100644 --- a/tests/goth_tests/conftest.py +++ b/tests/goth_tests/conftest.py @@ -100,10 +100,22 @@ def project_dir() -> Path: @pytest.fixture(scope="session") def log_dir() -> Path: - base_dir = Path("/", "tmp", "goth-tests") - date_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z") + """Fixture providing unique directory for logs from a test run.""" + base_dir = Path("logs") + date_str = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d_%H-%M-%S") log_dir = base_dir / f"goth_{date_str}" - log_dir.mkdir(parents=True) + log_dir.mkdir(parents=True, exist_ok=True) + + # Create symlink to latest on Linux + if os.name != "nt": + latest_link = base_dir / "latest" + try: + if latest_link.is_symlink() or latest_link.exists(): + latest_link.unlink() + latest_link.symlink_to(f"goth_{date_str}", target_is_directory=True) + except OSError as e: + print(f"Warning: could not create symlink {latest_link} -> {log_dir}: {e}") + return log_dir From 84f77bc7510cafe338840ab03e189fd490eeb774 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 15 Oct 2025 19:12:30 +0200 Subject: [PATCH 24/25] Fix log path --- tests/goth_tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/goth_tests/conftest.py b/tests/goth_tests/conftest.py index b915c0c5f..72b2924c8 100644 --- a/tests/goth_tests/conftest.py +++ b/tests/goth_tests/conftest.py @@ -1,3 +1,4 @@ +import os import asyncio from datetime import datetime, timezone from pathlib import Path From e5734ebe6a0c3519e745f8f7f53e31df49fc5c2b Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 15 Oct 2025 19:32:34 +0200 Subject: [PATCH 25/25] Fix log path --- .../test_agreement_termination/test_agreement_termination.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/goth_tests/test_agreement_termination/test_agreement_termination.py b/tests/goth_tests/test_agreement_termination/test_agreement_termination.py index 58ec70820..76661b27c 100644 --- a/tests/goth_tests/test_agreement_termination/test_agreement_termination.py +++ b/tests/goth_tests/test_agreement_termination/test_agreement_termination.py @@ -48,6 +48,7 @@ async def assert_all_tasks_computed(stream): remaining_ids = {1, 2, 3, 4, 5, 6} async for line in stream: + logger.log(logging.INFO, line) m = re.search(r"TaskAccepted\(.*task=Task\(id=([0-9]+)", line) if m: task_id = int(m.group(1))