Skip to content

Commit d01e123

Browse files
committed
Merge remote-tracking branch 'origin/master' into fix-rq-state-when-marking-ashandled-fails
2 parents 425093b + 9293706 commit d01e123

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+729
-375
lines changed

.github/workflows/_release_docs.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ jobs:
6868
run: uv run poe build-docs
6969
env:
7070
APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
71+
SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
7172

7273
- name: Set up GitHub Pages
7374
uses: actions/configure-pages@v5

.github/workflows/_tests.yaml

Lines changed: 116 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,119 @@ jobs:
2020
tests_concurrency: "1"
2121

2222
integration_tests:
23-
name: Integration tests
24-
uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main
25-
secrets: inherit
26-
with:
27-
python_versions: '["3.10", "3.14"]'
28-
operating_systems: '["ubuntu-latest"]'
29-
python_version_for_codecov: "3.14"
30-
operating_system_for_codecov: ubuntu-latest
31-
tests_concurrency: "16"
23+
name: Integration tests (${{ matrix.python-version }}, ${{ matrix.os }})
24+
25+
if: >-
26+
${{
27+
(github.event_name == 'pull_request' && github.event.pull_request.head.repo.owner.login == 'apify') ||
28+
(github.event_name == 'push' && github.ref == 'refs/heads/master')
29+
}}
30+
31+
strategy:
32+
matrix:
33+
os: ["ubuntu-latest"]
34+
python-version: ["3.10", "3.14"]
35+
36+
runs-on: ${{ matrix.os }}
37+
38+
env:
39+
TESTS_CONCURRENCY: "16"
40+
41+
steps:
42+
- name: Checkout repository
43+
uses: actions/checkout@v6
44+
45+
- name: Set up Python ${{ matrix.python-version }}
46+
uses: actions/setup-python@v6
47+
with:
48+
python-version: ${{ matrix.python-version }}
49+
50+
- name: Set up uv package manager
51+
uses: astral-sh/setup-uv@v7
52+
with:
53+
python-version: ${{ matrix.python-version }}
54+
55+
- name: Install Python dependencies
56+
run: uv run poe install-dev
57+
58+
- name: Run integration tests
59+
run: uv run poe integration-tests-cov
60+
env:
61+
APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_PYTHON_SDK_API_TOKEN }}
62+
APIFY_TEST_USER_2_API_TOKEN: ${{ secrets.APIFY_TEST_USER_2_API_TOKEN }}
63+
64+
- name: Upload integration test coverage
65+
if: >-
66+
${{
67+
matrix.os == 'ubuntu-latest' &&
68+
matrix.python-version == '3.14' &&
69+
env.CODECOV_TOKEN != ''
70+
}}
71+
uses: codecov/codecov-action@v5
72+
env:
73+
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
74+
with:
75+
token: ${{ env.CODECOV_TOKEN }}
76+
files: coverage-integration.xml
77+
flags: integration
78+
79+
e2e_tests:
80+
name: E2E tests (${{ matrix.python-version }}, ${{ matrix.os }})
81+
82+
if: >-
83+
${{
84+
(github.event_name == 'pull_request' && github.event.pull_request.head.repo.owner.login == 'apify') ||
85+
(github.event_name == 'push' && github.ref == 'refs/heads/master')
86+
}}
87+
88+
strategy:
89+
# E2E tests build and run Actors on the platform. Limit parallel workflows to 1 to avoid exceeding
90+
# the platform's memory limits. A single workflow with 16 pytest workers provides good test
91+
# parallelization while staying within platform constraints.
92+
max-parallel: 1
93+
matrix:
94+
os: ["ubuntu-latest"]
95+
python-version: ["3.10", "3.14"]
96+
97+
runs-on: ${{ matrix.os }}
98+
99+
env:
100+
TESTS_CONCURRENCY: "16"
101+
102+
steps:
103+
- name: Checkout repository
104+
uses: actions/checkout@v6
105+
106+
- name: Set up Python ${{ matrix.python-version }}
107+
uses: actions/setup-python@v6
108+
with:
109+
python-version: ${{ matrix.python-version }}
110+
111+
- name: Set up uv package manager
112+
uses: astral-sh/setup-uv@v7
113+
with:
114+
python-version: ${{ matrix.python-version }}
115+
116+
- name: Install Python dependencies
117+
run: uv run poe install-dev
118+
119+
- name: Run E2E tests
120+
run: uv run poe e2e-tests-cov
121+
env:
122+
APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_PYTHON_SDK_API_TOKEN }}
123+
APIFY_TEST_USER_2_API_TOKEN: ${{ secrets.APIFY_TEST_USER_2_API_TOKEN }}
124+
125+
- name: Upload E2E test coverage
126+
if: >-
127+
${{
128+
matrix.os == 'ubuntu-latest' &&
129+
matrix.python-version == '3.14' &&
130+
env.CODECOV_TOKEN != ''
131+
}}
132+
uses: codecov/codecov-action@v5
133+
env:
134+
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
135+
with:
136+
token: ${{ env.CODECOV_TOKEN }}
137+
files: coverage-e2e.xml
138+
flags: e2e

CHANGELOG.md

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,19 @@
22

33
All notable changes to this project will be documented in this file.
44

5-
<!-- git-cliff-unreleased-start -->
6-
## 3.1.1 - **not yet released**
5+
## [3.2.0](https://github.com/apify/apify-sdk-python/releases/tag/v3.2.0) (2026-02-11)
76

87
### 🚀 Features
98

109
- Add `Actor` method `use_state` ([#738](https://github.com/apify/apify-sdk-python/pull/738)) ([8a78b6f](https://github.com/apify/apify-sdk-python/commit/8a78b6fc9324e948b3481b07a3582215d3c966fe)) by [@Mantisus](https://github.com/Mantisus), closes [#735](https://github.com/apify/apify-sdk-python/issues/735)
1110

11+
### 🐛 Bug Fixes
12+
13+
- Stop silently swallowing exceptions in request queue ([#777](https://github.com/apify/apify-sdk-python/pull/777)) ([6358d66](https://github.com/apify/apify-sdk-python/commit/6358d66aeb83484845b17f7c8632b6c763cef368)) by [@vdusek](https://github.com/vdusek)
14+
- Handle TimeoutError in Actor __aexit__ to prevent resource leaks ([#776](https://github.com/apify/apify-sdk-python/pull/776)) ([fb13765](https://github.com/apify/apify-sdk-python/commit/fb13765448a2a6e2b776de819ece68f90abff1e3)) by [@vdusek](https://github.com/vdusek)
15+
- Pass name instead of id for name param in SmartApifyStorageClient ([#775](https://github.com/apify/apify-sdk-python/pull/775)) ([56cfc38](https://github.com/apify/apify-sdk-python/commit/56cfc38aa98a2a6689dd077e9d5c5d8729872413)) by [@vdusek](https://github.com/vdusek)
16+
1217

13-
<!-- git-cliff-unreleased-end -->
1418
## [3.1.0](https://github.com/apify/apify-sdk-python/releases/tag/v3.1.0) (2025-12-08)
1519

1620
### 🚀 Features

CONTRIBUTING.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,26 @@ To run the documentation locally (requires Node.js):
118118
uv run poe run-docs
119119
```
120120

121+
## Commits
122+
123+
We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages. This convention is used to automatically determine version bumps during the release process.
124+
125+
### Available commit types
126+
127+
| Type | Description |
128+
| ---- | ----------- |
129+
| `feat` | A new feature |
130+
| `fix` | A bug fix |
131+
| `docs` | Documentation only changes |
132+
| `style` | Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc) |
133+
| `refactor` | A code change that neither fixes a bug nor adds a feature |
134+
| `perf` | A code change that improves performance |
135+
| `test` | Adding missing tests or correcting existing tests |
136+
| `build` | Changes that affect the build system or external dependencies (example scopes: gulp, broccoli, npm) |
137+
| `ci` | Changes to our CI configuration files and scripts (example scopes: Travis, Circle, BrowserStack, SauceLabs) |
138+
| `chore` | Other changes that don't modify src or test files |
139+
| `revert` | Reverts a previous commit |
140+
121141
## Release process
122142

123143
Publishing new versions to [PyPI](https://pypi.org/project/apify) is automated through GitHub Actions.

pyproject.toml

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
44

55
[project]
66
name = "apify"
7-
version = "3.1.1"
7+
version = "3.2.0"
88
description = "Apify SDK for Python"
99
authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
1010
license = { file = "LICENSE" }
@@ -64,6 +64,11 @@ scrapy = ["scrapy>=2.11.0"]
6464

6565
[dependency-groups]
6666
dev = [
67+
# TODO: Remove this constraint once pydoc-markdown updates its dependencies.
68+
# Package pydoc-markdown is unmaintained and pins old docspec-python with vulnerable black.
69+
# See https://github.com/apify/apify-client-python/pull/582/ for more details.
70+
# We explicitly constrain black>=24.3.0 to override the transitive dependency.
71+
"black>=24.3.0",
6772
"build<2.0.0",
6873
"crawlee[parsel]",
6974
"dycw-pytest-only<3.0.0",
@@ -140,15 +145,13 @@ indent-style = "space"
140145
"**/{tests}/*" = [
141146
"D", # Everything from the pydocstyle
142147
"INP001", # File {filename} is part of an implicit namespace package, add an __init__.py
148+
"PLC0415", # `import` should be at the top-level of a file
143149
"PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
144150
"S101", # Use of assert detected
145151
"SLF001", # Private member accessed: `{name}`
146152
"T20", # flake8-print
147-
"TRY301", # Abstract `raise` to an inner function
148153
"TID252", # Prefer absolute imports over relative imports from parent modules
149-
]
150-
"**/{tests}/{integration}/*" = [
151-
"PLC0415", # `import` should be at the top-level of a file
154+
"TRY301", # Abstract `raise` to an inner function
152155
]
153156
"**/{docs,website}/**" = [
154157
"D", # Everything from the pydocstyle
@@ -229,6 +232,8 @@ unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit
229232
unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify --cov-report=xml:coverage-unit.xml tests/unit"
230233
integration-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/integration"
231234
integration-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify --cov-report=xml:coverage-integration.xml tests/integration"
235+
e2e-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/e2e"
236+
e2e-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify --cov-report=xml:coverage-e2e.xml tests/e2e"
232237
check-code = ["lint", "type-check", "unit-tests"]
233238

234239
[tool.poe.tasks.install-dev]

src/apify/_actor.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,8 +252,12 @@ async def finalize() -> None:
252252
# Persist Actor state
253253
await self._save_actor_state()
254254

255-
await asyncio.wait_for(finalize(), self._cleanup_timeout.total_seconds())
256-
self._is_initialized = False
255+
try:
256+
await asyncio.wait_for(finalize(), self._cleanup_timeout.total_seconds())
257+
except TimeoutError:
258+
self.log.exception('Actor cleanup timed out')
259+
finally:
260+
self._is_initialized = False
257261

258262
if self._exit_process:
259263
sys.exit(self.exit_code)

src/apify/storage_clients/_apify/_request_queue_shared_client.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -279,8 +279,8 @@ async def reclaim_request(
279279
if forefront:
280280
self._should_check_for_forefront_requests = True
281281

282-
except Exception as exc:
283-
logger.debug(f'Error reclaiming request {request.unique_key}: {exc!s}')
282+
except Exception:
283+
logger.exception(f'Error reclaiming request {request.unique_key}')
284284
return None
285285
else:
286286
return processed_request

src/apify/storage_clients/_apify/_request_queue_single_client.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -263,8 +263,8 @@ async def reclaim_request(
263263
self.metadata.handled_request_count -= 1
264264
self.metadata.pending_request_count += 1
265265

266-
except Exception as exc:
267-
logger.debug(f'Error reclaiming request {request.unique_key}: {exc!s}')
266+
except Exception:
267+
logger.exception(f'Error reclaiming request {request.unique_key}')
268268
return None
269269
else:
270270
return processed_request

src/apify/storage_clients/_smart_apify/_storage_client.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ async def create_dataset_client(
7474
configuration: CrawleeConfiguration | None = None,
7575
) -> DatasetClient:
7676
return await self.get_suitable_storage_client().create_dataset_client(
77-
id=id, name=id, alias=alias, configuration=configuration
77+
id=id, name=name, alias=alias, configuration=configuration
7878
)
7979

8080
@override
@@ -87,7 +87,7 @@ async def create_kvs_client(
8787
configuration: CrawleeConfiguration | None = None,
8888
) -> KeyValueStoreClient:
8989
return await self.get_suitable_storage_client().create_kvs_client(
90-
id=id, name=id, alias=alias, configuration=configuration
90+
id=id, name=name, alias=alias, configuration=configuration
9191
)
9292

9393
@override
@@ -100,7 +100,7 @@ async def create_rq_client(
100100
configuration: CrawleeConfiguration | None = None,
101101
) -> RequestQueueClient:
102102
return await self.get_suitable_storage_client().create_rq_client(
103-
id=id, name=id, alias=alias, configuration=configuration
103+
id=id, name=name, alias=alias, configuration=configuration
104104
)
105105

106106
def get_suitable_storage_client(self, *, force_cloud: bool = False) -> StorageClient:

tests/e2e/README.md

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
# E2E tests
2+
3+
These tests build and run Actors using the Python SDK on the Apify platform. They are slower than integration tests (see [`tests/integration/`](../integration/)) because they need to build and deploy Actors.
4+
5+
When writing new tests, prefer integration tests if possible. Only write E2E tests when you need to test something that requires building and running an Actor on the platform.
6+
7+
## Running
8+
9+
```bash
10+
export APIFY_TEST_USER_API_TOKEN=<your-token>
11+
uv run poe e2e-tests
12+
```
13+
14+
To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`.
15+
16+
## Key fixtures
17+
18+
- **`apify_client_async`** — A session-scoped `ApifyClientAsync` instance configured with the test token and API URL.
19+
- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test.
20+
- **`make_actor`** — Factory for creating temporary Actors on the Apify platform (built, then auto-deleted after the test).
21+
- **`run_actor`** — Starts an Actor run and waits for completion (10 min timeout).
22+
23+
## How to write tests
24+
25+
### Creating an Actor from a Python function
26+
27+
You can create Actors straight from a Python function. This is great because the test Actor source code gets checked by the linter.
28+
29+
```python
30+
async def test_something(
31+
make_actor: MakeActorFunction,
32+
run_actor: RunActorFunction,
33+
) -> None:
34+
async def main() -> None:
35+
async with Actor:
36+
print('Hello!')
37+
38+
actor = await make_actor(label='something', main_func=main)
39+
run_result = await run_actor(actor)
40+
41+
assert run_result.status == 'SUCCEEDED'
42+
```
43+
44+
The `src/main.py` file will be set to the function definition, prepended with `import asyncio` and `from apify import Actor`. You can add extra imports directly inside the function body.
45+
46+
### Creating an Actor from source files
47+
48+
Pass the `main_py` argument for a single-file Actor:
49+
50+
```python
51+
async def test_something(
52+
make_actor: MakeActorFunction,
53+
run_actor: RunActorFunction,
54+
) -> None:
55+
expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}'
56+
main_py_source = f"""
57+
import asyncio
58+
from datetime import datetime
59+
from apify import Actor
60+
async def main():
61+
async with Actor:
62+
await Actor.set_value('OUTPUT', '{expected_output}')
63+
"""
64+
65+
actor = await make_actor(label='something', main_py=main_py_source)
66+
await run_actor(actor)
67+
68+
output_record = await actor.last_run().key_value_store().get_record('OUTPUT')
69+
assert output_record is not None
70+
assert output_record['value'] == expected_output
71+
```
72+
73+
Or pass `source_files` for multi-file Actors:
74+
75+
```python
76+
actor_source_files = {
77+
'src/utils.py': """
78+
from datetime import datetime, timezone
79+
def get_current_datetime():
80+
return datetime.now(timezone.utc)
81+
""",
82+
'src/main.py': """
83+
import asyncio
84+
from apify import Actor
85+
from .utils import get_current_datetime
86+
async def main():
87+
async with Actor:
88+
print('Hello! It is ' + str(get_current_datetime()))
89+
""",
90+
}
91+
actor = await make_actor(label='something', source_files=actor_source_files)
92+
```
93+
94+
### Assertions inside Actors
95+
96+
Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown. Always include explicit assertion messages:
97+
98+
```python
99+
assert is_finished is False, f'is_finished={is_finished}'
100+
```

0 commit comments

Comments
 (0)