Merge pull request #230 from fairagro/feature/ARC_as_string

Zalfsten · web-flow · commit bbbfac5ef75f · 2026-04-27T14:43:43.000+02:00
Feature/arc as string
diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml
@@ -74,16 +74,10 @@ jobs:
             NEW_VERSION="${BASE_VERSION}-rc.${BRANCH_LABEL}.${{ github.run_number }}"
 
             # PEP 440 compliant variant for Python package builds (hatch-vcs / setuptools-scm).
-            # Format: X.Y.Z.devN+branch.name
-            PEP440_BRANCH=$(echo "$GITHUB_REF_NAME" \
-              | sed 's|feature/||' \
-              | tr '[:upper:]' '[:lower:]' \
-              | sed -E 's/[^a-z0-9]+/./g; s/^\.+//; s/\.+$//; s/\.{2,}/./g' \
-              | cut -c1-30)
-            if [[ -z "$PEP440_BRANCH" ]]; then
-              PEP440_BRANCH="feature"
-            fi
-            PEP440_VERSION="${BASE_VERSION}.dev${{ github.run_number }}+${PEP440_BRANCH}"
+            # For pre-releases, use simpler format that hatchling can parse: X.Y.ZaN.branch
+            # This is PEP 440 compliant and works with hatchling's version parser
+            # Simple .devN format without branch information for maximum compatibility
+            PEP440_VERSION="${BASE_VERSION}.dev${{ github.run_number }}"
           else
             NEW_VERSION="${BASE_VERSION}"
             PEP440_VERSION="${BASE_VERSION}"
diff --git a/.github/workflows/reusable-release.yml b/.github/workflows/reusable-release.yml
@@ -147,11 +147,11 @@ jobs:
           UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
         run: uv publish --publish-url https://upload.pypi.org/legacy/ dist/*
 
-      - name: Publish to TestPyPI (pre-release)
+      - name: Publish to PyPI (pre-release)
         if: inputs.release_type == 'feature'
         env:
-          UV_PUBLISH_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }}
-        run: uv publish --publish-url https://test.pypi.org/legacy/ dist/*
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: uv publish --publish-url https://upload.pypi.org/legacy/ dist/*
 
   create-release-tag:
     name: CreateReleaseTag/Release
diff --git a/middleware/api_client/README.md b/middleware/api_client/README.md
@@ -51,13 +51,13 @@ async def main():
         # Send a single ARC
         response = await client.create_or_update_arc(
             rdi="my-rdi",
-            arc=arc,
+            arc=arc,  # Can be ARC object, dict, or JSON string
         )
         print(f"ARC status: {response.status}")
 
         # Or run a harvest workflow
         async def arc_stream():
-            yield arc
+            yield arc  # Can yield ARC objects, dicts, or JSON strings
 
         harvest = await client.harvest_arcs(
             rdi="my-rdi",
@@ -85,22 +85,22 @@ asyncio.run(main())
 
 ## API Methods
 
-### `create_or_update_arc(rdi: str, arc: ARC | dict) -> ArcResult`
+### `create_or_update_arc(rdi: str, arc: ARC | dict | str) -> ArcResult`
 
 Create or update one ARC in the Middleware API.
 
 **Parameters:**
 
 - `rdi` (str): The RDI identifier (e.g., "edaphobase").
-- `arc` (ARC | dict): ARC object from arctrl or pre-serialised RO-Crate dict.
+- `arc` (ARC | dict | str): ARC object from arctrl, pre-serialised RO-Crate dict, or JSON string.
 
 **Returns:**
 
 - `ArcResult`: Contains the result of the operation.
 
 **Raises:**
 
-- `ApiClientError`: If the request fails due to HTTP errors or network issues.
+- `ApiClientError`: If the request fails due to HTTP errors, network issues, or invalid JSON.
 
 **Example:**
 
@@ -112,17 +112,18 @@ arc = ARC.from_arc_investigation(inv)
 
 response = await client.create_or_update_arc(
     rdi="edaphobase",
-    arc=arc,
+    arc=arc,  # Can also be dict or JSON string
 )
 ```
 
-### `harvest_arcs(rdi: str, arcs: AsyncIterator[ARC | dict], expected_datasets: int | None = None) -> HarvestResult`
+### `harvest_arcs(rdi: str, arcs: AsyncIterator[ARC | dict | str], expected_datasets: int | None = None) -> HarvestResult`
 
 Convenience workflow to create a harvest, upload all ARCs from an async iterator, and complete the harvest.
 
 - Uses `config.max_concurrency` by default.
 - Continues on item-level submission errors and skips failed items.
 - Cancels the harvest only for catastrophic errors.
+- Supports ARC objects, pre-serialised RO-Crate dicts, and JSON strings.
 
 All errors are raised as `ApiClientError` exceptions:
 
@@ -132,24 +133,7 @@ from middleware.api_client import ApiClientError
 try:
     response = await client.create_or_update_arc(
         rdi="my-rdi",
-        arc=arc,
+        arc=arc,  # Can be ARC object, dict, or JSON string
     )
 except ApiClientError as e:
     print(f"API Error: {e}")
-```
-
-## Configuration via Environment Variables
-
-You can override configuration values using environment variables:
-
-```bash
-export API_URL="https://production-api:8000"
-export CLIENT_CERT_PATH="/secure/certs/prod-cert.pem"
-export CLIENT_KEY_PATH="/secure/certs/prod-key.pem"
-```
-
-Or use Docker secrets in `/run/secrets/`.
-
-## License
-
-This is part of the FAIRagro Advanced Middleware project.
diff --git a/middleware/api_client/src/middleware/api_client/api_client.py b/middleware/api_client/src/middleware/api_client/api_client.py
@@ -262,13 +262,13 @@ def _process_completed_arc_tasks(
     async def _submit_arcs_parallel(
         self,
         harvest_id: str,
-        arcs: "AsyncGenerator[ARC | dict[str, Any], None] | AsyncIterator[ARC | dict[str, Any]]",
+        arcs: "AsyncGenerator[ARC | dict[str, Any] | str, None] | AsyncIterator[ARC | dict[str, Any] | str]",
     ) -> int:
         """Submit all ARCs in bounded parallelism and return number of skipped ARC submissions."""
         pending_tasks: set[asyncio.Task[None]] = set()
         failed_submissions = 0
 
-        async def submit_one(arc_item: "ARC | dict[str, Any]") -> None:
+        async def submit_one(arc_item: "ARC | dict[str, Any] | str") -> None:
             await self.submit_arc_in_harvest(harvest_id, arc_item)
 
         async for arc in arcs:
@@ -446,10 +446,18 @@ async def _delete(self, path: str) -> None:
     # ------------------------------------------------------------------
 
     @classmethod
-    def _serialize_arc(cls, arc: "ARC | dict[str, Any]") -> dict[str, Any]:
-        """Serialize an ARC object to a plain RO-Crate JSON dict."""
+    def _serialize_arc(cls, arc: "ARC | dict[str, Any] | str") -> dict[str, Any]:
+        """Serialize an ARC object, dict, or JSON string to a plain RO-Crate JSON dict."""
         if isinstance(arc, dict):
             return arc
+        if isinstance(arc, str):
+            try:
+                data = json.loads(arc)
+                if not isinstance(data, dict):
+                    raise ApiClientError(f"JSON string must represent a dictionary, got {type(data).__name__}")
+                return cast(dict[str, Any], data)
+            except json.JSONDecodeError as e:
+                raise ApiClientError(f"Invalid JSON string provided for ARC: {e}") from e
         return cast(dict[str, Any], json.loads(arc.ToROCrateJsonString()))
 
     @classmethod
@@ -473,7 +481,7 @@ def _parse_harvest_response(cls, data: Any) -> HarvestResult:
     async def create_or_update_arc(
         self,
         rdi: str,
-        arc: "ARC | dict[str, Any]",
+        arc: "ARC | dict[str, Any] | str",
     ) -> ArcResult:
         """Create or update an ARC.
 
@@ -483,7 +491,7 @@ async def create_or_update_arc(
 
         Args:
             rdi: RDI identifier.
-            arc: ARC object or a pre-serialised RO-Crate JSON dict.
+            arc: ARC object, a pre-serialised RO-Crate JSON dict, or a JSON string.
 
         Returns:
             :class:`ArcResult` with the result of the operation.
@@ -579,7 +587,7 @@ async def cancel_harvest(self, harvest_id: str) -> None:
     async def submit_arc_in_harvest(
         self,
         harvest_id: str,
-        arc: "ARC | dict[str, Any]",
+        arc: "ARC | dict[str, Any] | str",
     ) -> ArcResult:
         """Submit an ARC within an active harvest run.
 
@@ -588,7 +596,7 @@ async def submit_arc_in_harvest(
 
         Args:
             harvest_id: Harvest identifier.
-            arc: ARC object or a pre-serialised RO-Crate JSON dict.
+            arc: ARC object, a pre-serialised RO-Crate JSON dict, or a JSON string.
 
         Returns:
             :class:`ArcResult` with the result of the operation.
@@ -601,7 +609,7 @@ async def submit_arc_in_harvest(
     async def harvest_arcs(
         self,
         rdi: str,
-        arcs: "AsyncGenerator[ARC | dict[str, Any], None] | AsyncIterator[ARC | dict[str, Any]]",
+        arcs: "AsyncGenerator[ARC | dict[str, Any] | str, None] | AsyncIterator[ARC | dict[str, Any] | str]",
         expected_datasets: int | None = None,
     ) -> HarvestResult:
         """Create a harvest, upload all ARCs from an async generator, then complete it.
@@ -618,8 +626,8 @@ async def harvest_arcs(
 
         Args:
             rdi: RDI identifier for the harvest.
-            arcs: Async generator or async iterator yielding ARC objects or
-                pre-serialised RO-Crate dicts.
+            arcs: Async generator or async iterator yielding ARC objects,
+                pre-serialised RO-Crate dicts, or JSON strings.
             expected_datasets: Optional hint about the total number of ARCs.
 
         Returns:
@@ -631,7 +639,7 @@ async def harvest_arcs(
 
         Example::
 
-            async def my_arcs() -> AsyncGenerator[dict, None]:
+            async def my_arcs() -> AsyncGenerator[dict | str, None]:
                 for arc in source:
                     yield arc
 
diff --git a/middleware/api_client/tests/unit/test_client.py b/middleware/api_client/tests/unit/test_client.py
@@ -226,6 +226,28 @@ async def test_create_or_update_arc_with_dict(client_config: Config) -> None:
     assert isinstance(response, ArcResult)
 
 
+@pytest.mark.asyncio
+@respx.mock
+async def test_create_or_update_arc_with_json_string(client_config: Config) -> None:
+    """Test create_or_update_arc with a JSON string."""
+    route = respx.post(f"{client_config.api_url}v3/arcs").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_ARC_RESPONSE)
+    )
+    async with ApiClient(client_config) as client:
+        response = await client.create_or_update_arc(rdi="test-rdi", arc='{"id": "mock-arc"}')
+    assert route.called
+    assert isinstance(response, ArcResult)
+    assert response.arc_id == "arc-123"
+
+
+@pytest.mark.asyncio
+async def test_create_or_update_arc_with_invalid_json_string(client_config: Config) -> None:
+    """Test create_or_update_arc with an invalid JSON string."""
+    async with ApiClient(client_config) as client:
+        with pytest.raises(ApiClientError, match="Invalid JSON string provided for ARC"):
+            await client.create_or_update_arc(rdi="test-rdi", arc='{"id": "mock-arc"')
+
+
 @pytest.mark.asyncio
 @respx.mock
 async def test_create_or_update_arc_http_error(client_config: Config) -> None:
@@ -495,13 +517,35 @@ async def test_submit_arc_in_harvest_invalid_response(client_config: Config) ->
             await client.submit_arc_in_harvest("harvest-456", arc={"id": "mock"})
 
 
+@pytest.mark.asyncio
+@respx.mock
+async def test_submit_arc_in_harvest_with_json_string(client_config: Config) -> None:
+    """Test submit_arc_in_harvest with a JSON string."""
+    route = respx.post(f"{client_config.api_url}v3/harvests/harvest-456/arcs").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_ARC_RESPONSE)
+    )
+    async with ApiClient(client_config) as client:
+        response = await client.submit_arc_in_harvest("harvest-456", arc='{"id": "mock-arc"}')
+    assert route.called
+    assert isinstance(response, ArcResult)
+    assert response.arc_id == "arc-123"
+
+
+@pytest.mark.asyncio
+async def test_submit_arc_in_harvest_with_invalid_json_string(client_config: Config) -> None:
+    """Test submit_arc_in_harvest with an invalid JSON string."""
+    async with ApiClient(client_config) as client:
+        with pytest.raises(ApiClientError, match="Invalid JSON string provided for ARC"):
+            await client.submit_arc_in_harvest("harvest-456", arc='{"id": "mock-arc"')
+
+
 # ---------------------------------------------------------------------------
 # harvest_arcs
 # ---------------------------------------------------------------------------
 
 
-async def _arc_gen(*arcs: "dict[str, Any]") -> AsyncGenerator["dict[str, Any]", None]:
-    """Yield the provided arc dicts as an async generator."""
+async def _arc_gen(*arcs: "dict[str, Any] | str | ARC") -> AsyncGenerator["dict[str, Any] | str | ARC", None]:
+    """Yield the provided arc dicts, JSON strings, or ARC objects as an async generator."""
     for arc in arcs:
         yield arc
 
@@ -650,6 +694,52 @@ async def test_harvest_arcs_cancels_on_catastrophic_error(client_config: Config)
     assert cancel_route.called
 
 
+@pytest.mark.asyncio
+@respx.mock
+async def test_harvest_arcs_with_json_string(client_config: Config) -> None:
+    """harvest_arcs supports JSON strings in async generator."""
+    completed_response = {**_HARVEST_RESPONSE, "status": "COMPLETED", "completed_at": "2024-01-01T01:00:00Z"}
+    respx.post(f"{client_config.api_url}v3/harvests").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_HARVEST_RESPONSE)
+    )
+    respx.post(f"{client_config.api_url}v3/harvests/harvest-456/arcs").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_ARC_RESPONSE)
+    )
+    respx.post(f"{client_config.api_url}v3/harvests/harvest-456/complete").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=completed_response)
+    )
+
+    arcs = _arc_gen(
+        '{"id": "arc-1-string"}',
+        {"id": "arc-2-dict"},
+        ARC.from_arc_investigation(ArcInvestigation.create(identifier="test", title="Test")),
+    )
+    async with ApiClient(client_config) as client:
+        result = await client.harvest_arcs("test-rdi", arcs, expected_datasets=3)
+
+    assert isinstance(result, HarvestResult)
+    assert result.status == "COMPLETED"
+
+
+@pytest.mark.asyncio
+@respx.mock
+async def test_harvest_arcs_with_invalid_json_string(client_config: Config) -> None:
+    """harvest_arcs raises ApiClientError when JSON string is invalid."""
+    # Mock the harvest creation endpoint to prevent actual HTTP requests
+    respx.post(f"{client_config.api_url}v3/harvests").mock(
+        return_value=httpx.Response(http.HTTPStatus.OK, json=_HARVEST_RESPONSE)
+    )
+    # Mock the harvest cancellation endpoint
+    respx.delete(f"{client_config.api_url}v3/harvests/harvest-456").mock(
+        return_value=httpx.Response(http.HTTPStatus.NO_CONTENT)
+    )
+
+    async with ApiClient(client_config) as client:
+        arcs = _arc_gen('{"id": "arc-1"')  # Single invalid JSON string
+        with pytest.raises(ApiClientError, match="Invalid JSON string provided for ARC"):
+            await client.harvest_arcs("test-rdi", arcs)
+
+
 @pytest.mark.asyncio
 @respx.mock
 async def test_harvest_arcs_cancel_failure_does_not_mask_original_error(client_config: Config) -> None:
diff --git a/spec/ci-cd/design.md b/spec/ci-cd/design.md
@@ -138,17 +138,13 @@ On push feature/* or schedule:
 14. **PEP 440 parallel version for Python packages**
     — Docker semver pre-release format (`1.2.3-rc.branch.42`) is not valid
     PEP 440. The build phase computes a parallel `pep440_version` in the format
-    `1.2.3.dev42+branch.name` and injects it via
+    `1.2.3.dev42` and injects it via
     `SETUPTOOLS_SCM_PRETEND_VERSION` to override hatch-vcs version discovery,
     so Docker and Python packages share the same numeric baseline.
+    — This simple `.devN` format was chosen for maximum compatibility with both
+    hatchling and PyPI, using a global run number for uniqueness across all branches.
 
-15. **Registry selection via `release_type` input**
-    — The `publish-pypi` job selects `https://upload.pypi.org/legacy/` for
-    `release_type == 'final'` and `https://test.pypi.org/legacy/` for
-    `release_type == 'feature'`. Separate secrets (`PYPI_TOKEN`,
-    `TEST_PYPI_TOKEN`) are used for each registry.
-
-16. **Python packages built once in the build phase, reused in release**
+15. **Python packages built once in the build phase, reused in release**
     — `reusable-build.yml` includes a `python-build` job that produces wheels
     and sdists for both publishable packages and uploads them as the artifact
     `python-packages-{version}`. This mirrors the Docker transfer-artifact
diff --git a/spec/ci-cd/spec.md b/spec/ci-cd/spec.md
@@ -97,8 +97,7 @@ pipelines run on GitHub Actions.
 
 - [ ] Publish Python packages to PyPI for `middleware/api_client` and `middleware/shared` components.
 - [ ] PyPI packages must be published whenever a Docker image is successfully pushed to a registry.
-- [ ] Final releases from `main` branch must publish packages to **PyPI** (<https://pypi.org>).
-- [ ] Feature branch pre-releases must publish packages to **TestPyPI** (<https://test.pypi.org/>).
+- [ ] Both final releases from `main` and feature branch pre-releases must publish packages to **PyPI** (<https://pypi.org>).
 - [ ] Packages must be published only after the `reusable-check.yml` security scans have passed.
 - [ ] Publish the `middleware/api_client` component under the name `fairagro-middleware-api-client`.
 - [ ] Publish the `middleware/shared` component under the name `fairagro-middleware-shared`.
@@ -109,7 +108,7 @@ pipelines run on GitHub Actions.
 - [ ] Each package must include required dependencies from `pyproject.toml`.
 - [ ] PyPI packages must use the exact same semantic version as the Docker image.
 - [ ] Final release from `main`: `MAJOR.MINOR.PATCH`.
-- [ ] Feature branch pre-release: `MAJOR.MINOR.PATCH-rc.{branch-label}.{run_number}`.
+- [ ] Feature branch pre-release: `MAJOR.MINOR.PATCH.dev{RUN_NUMBER}` (PEP 440 compliant format using global run number for uniqueness).
 - [ ] If a GitHub release is created, the packages must be added to the artifact list.
 - [ ] If a github release is created, include `pip install` commands for each package with exact version information.
 - [ ] If a GitHub release is created, provide fallback instructions for local installation from source.
