Skip to content

Commit c3fd6e8

Browse files
MantisusMantisus
committed
Merge branch 'master' into sql-metadata-buffer
2 parents eae231d + 0a5426d commit c3fd6e8

File tree

149 files changed

+7153
-3653
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

149 files changed

+7153
-3653
lines changed

‎.github/workflows/_check_code.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Code checks
2+
3+
on:
4+
# Runs when manually triggered from the GitHub UI.
5+
workflow_dispatch:
6+
7+
# Runs when invoked by another workflow.
8+
workflow_call:
9+
10+
jobs:
11+
actions_lint_check:
12+
name: Actions lint check
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout repository
16+
uses: actions/checkout@v6
17+
- name: Run actionlint
18+
uses: rhysd/actionlint@v1.7.10
19+
20+
spell_check:
21+
name: Spell check
22+
runs-on: ubuntu-latest
23+
steps:
24+
- name: Checkout repository
25+
uses: actions/checkout@v6
26+
- name: Check spelling with typos
27+
uses: crate-ci/typos@v1.42.1
28+
29+
lint_check:
30+
name: Lint check
31+
uses: apify/workflows/.github/workflows/python_lint_check.yaml@main
32+
with:
33+
python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
34+
35+
type_check:
36+
name: Type check
37+
uses: apify/workflows/.github/workflows/python_type_check.yaml@main
38+
with:
39+
python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'

‎.github/workflows/_check_docs.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
name: Doc checks
2+
3+
on:
4+
# Runs when manually triggered from the GitHub UI.
5+
workflow_dispatch:
6+
7+
# Runs when invoked by another workflow.
8+
workflow_call:
9+
10+
jobs:
11+
doc_checks:
12+
name: Doc checks
13+
uses: apify/workflows/.github/workflows/python_docs_check.yaml@main

.github/workflows/build_and_deploy_docs.yaml renamed to .github/workflows/_release_docs.yaml

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,24 @@
1-
name: Build and deploy docs
1+
name: Doc release
22

33
on:
4+
# Runs when manually triggered from the GitHub UI.
45
workflow_dispatch:
6+
7+
# Runs when invoked by another workflow.
58
workflow_call:
69
inputs:
710
ref:
811
required: true
912
type: string
1013

1114
env:
12-
NODE_VERSION: 20
15+
NODE_VERSION: 22
1316
PYTHON_VERSION: 3.14
17+
CHECKOUT_REF: ${{ github.event_name == 'workflow_call' && inputs.ref || github.ref }}
1418

1519
jobs:
16-
build_and_deploy_docs:
20+
release_docs:
21+
name: Doc release
1722
environment:
1823
name: github-pages
1924
permissions:
@@ -24,10 +29,10 @@ jobs:
2429

2530
steps:
2631
- name: Checkout repository
27-
uses: actions/checkout@v5
32+
uses: actions/checkout@v6
2833
with:
2934
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
30-
ref: ${{ github.event_name == 'workflow_call' && inputs.ref || github.ref }}
35+
ref: ${{ env.CHECKOUT_REF }}
3136

3237
- name: Set up Node
3338
uses: actions/setup-node@v6
@@ -45,15 +50,13 @@ jobs:
4550
python-version: ${{ env.PYTHON_VERSION }}
4651

4752
- name: Install Python dependencies
48-
run: make install-dev
49-
50-
- name: Build generated API reference
51-
run: make build-api-reference
53+
run: uv run poe install-dev
5254

5355
- name: Build Docusaurus docs
54-
run: make build-docs
56+
run: uv run poe build-docs
5557
env:
5658
APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
59+
SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
5760

5861
- name: Set up GitHub Pages
5962
uses: actions/configure-pages@v5
@@ -67,6 +70,10 @@ jobs:
6770
uses: actions/deploy-pages@v4
6871

6972
- name: Invalidate CloudFront cache
70-
run: gh workflow run invalidate.yaml --repo apify/apify-docs-private
73+
run: |
74+
gh workflow run invalidate-cloudfront.yml \
75+
--repo apify/apify-docs-private \
76+
--field deployment=crawlee-web
77+
echo "✅ CloudFront cache invalidation workflow triggered successfully"
7178
env:
7279
GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

‎.github/workflows/_tests.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
name: Tests
2+
3+
on:
4+
# Runs when manually triggered from the GitHub UI.
5+
workflow_dispatch:
6+
7+
# Runs when invoked by another workflow.
8+
workflow_call:
9+
10+
jobs:
11+
unit_tests:
12+
name: Unit tests
13+
uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main
14+
secrets: inherit
15+
with:
16+
python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
17+
operating_systems: '["ubuntu-latest", "windows-latest", "macos-latest"]'
18+
python_version_for_codecov: "3.14"
19+
operating_system_for_codecov: ubuntu-latest
20+
tests_concurrency: "2"

‎.github/workflows/check_pr_title.yaml‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎.github/workflows/manual_release_stable.yaml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
name: Stable release
2+
3+
on:
4+
# Runs when manually triggered from the GitHub UI, with options to specify the type of release.
5+
workflow_dispatch:
6+
inputs:
7+
release_type:
8+
description: Release type
9+
required: true
10+
type: choice
11+
default: auto
12+
options:
13+
- auto
14+
- custom
15+
- patch
16+
- minor
17+
- major
18+
custom_version:
19+
description: The custom version to bump to (only for "custom" type)
20+
required: false
21+
type: string
22+
default: ""
23+
24+
concurrency:
25+
group: release
26+
cancel-in-progress: false
27+
28+
jobs:
29+
code_checks:
30+
name: Code checks
31+
uses: ./.github/workflows/_check_code.yaml
32+
33+
release_prepare:
34+
name: Release prepare
35+
needs: [code_checks]
36+
runs-on: ubuntu-latest
37+
outputs:
38+
version_number: ${{ steps.release_prepare.outputs.version_number }}
39+
tag_name: ${{ steps.release_prepare.outputs.tag_name }}
40+
changelog: ${{ steps.release_prepare.outputs.changelog }}
41+
release_notes: ${{ steps.release_prepare.outputs.release_notes }}
42+
steps:
43+
- uses: apify/workflows/git-cliff-release@main
44+
name: Release prepare
45+
id: release_prepare
46+
with:
47+
release_type: ${{ inputs.release_type }}
48+
custom_version: ${{ inputs.custom_version }}
49+
existing_changelog_path: CHANGELOG.md
50+
51+
changelog_update:
52+
name: Changelog update
53+
needs: [release_prepare]
54+
uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main
55+
with:
56+
version_number: ${{ needs.release_prepare.outputs.version_number }}
57+
changelog: ${{ needs.release_prepare.outputs.changelog }}
58+
secrets: inherit
59+
60+
github_release:
61+
name: GitHub release
62+
needs: [release_prepare, changelog_update]
63+
runs-on: ubuntu-latest
64+
env:
65+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
66+
steps:
67+
- name: GitHub release
68+
uses: softprops/action-gh-release@v2
69+
with:
70+
tag_name: ${{ needs.release_prepare.outputs.tag_name }}
71+
name: ${{ needs.release_prepare.outputs.version_number }}
72+
target_commitish: ${{ needs.changelog_update.outputs.changelog_commitish }}
73+
body: ${{ needs.release_prepare.outputs.release_notes }}
74+
75+
pypi_publish:
76+
name: PyPI publish
77+
needs: [release_prepare, changelog_update]
78+
runs-on: ubuntu-latest
79+
permissions:
80+
contents: write
81+
id-token: write # Required for OIDC authentication.
82+
environment:
83+
name: pypi
84+
url: https://pypi.org/project/crawlee
85+
steps:
86+
- name: Prepare distribution
87+
uses: apify/workflows/prepare-pypi-distribution@main
88+
with:
89+
package_name: crawlee
90+
is_prerelease: ""
91+
version_number: ${{ needs.release_prepare.outputs.version_number }}
92+
ref: ${{ needs.changelog_update.outputs.changelog_commitish }}
93+
# Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication.
94+
- name: Publish package to PyPI
95+
uses: pypa/gh-action-pypi-publish@release/v1
96+
97+
# TODO: add job for publish package to Conda
98+
# https://github.com/apify/crawlee-python/issues/104

.github/workflows/update_new_issue.yaml renamed to .github/workflows/on_issue.yaml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
1-
name: Update new issue
1+
name: CI (issue)
22

33
on:
4+
# Runs when a new issue is opened.
45
issues:
56
types:
67
- opened
78

89
jobs:
910
label_issues:
10-
name: Label issues
11+
name: Add labels
1112
runs-on: ubuntu-latest
1213
permissions:
1314
issues: write

‎.github/workflows/on_master.yaml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: CI (master)
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
tags-ignore:
8+
- "**" # Ignore all tags to avoid duplicate executions triggered by tag pushes.
9+
10+
concurrency:
11+
group: release
12+
cancel-in-progress: false
13+
14+
jobs:
15+
doc_checks:
16+
name: Doc checks
17+
uses: ./.github/workflows/_check_docs.yaml
18+
19+
doc_release:
20+
# Skip this for non-docs commits and forks.
21+
if: "startsWith(github.event.head_commit.message, 'docs') && startsWith(github.repository, 'apify/')"
22+
name: Doc release
23+
needs: [doc_checks]
24+
uses: ./.github/workflows/_release_docs.yaml
25+
with:
26+
# Use the same ref as the one that triggered the workflow.
27+
ref: ${{ github.ref }}
28+
secrets: inherit
29+
30+
code_checks:
31+
name: Code checks
32+
uses: ./.github/workflows/_check_code.yaml
33+
34+
tests:
35+
# Skip this for "ci" and "docs" commits.
36+
if: "!startsWith(github.event.head_commit.message, 'ci') && !startsWith(github.event.head_commit.message, 'docs')"
37+
name: Tests
38+
uses: ./.github/workflows/_tests.yaml
39+
secrets: inherit
40+
41+
release_prepare:
42+
# Skip this for "ci", "docs" and "test" commits and for forks.
43+
if: "!startsWith(github.event.head_commit.message, 'ci') && !startsWith(github.event.head_commit.message, 'docs') && !startsWith(github.event.head_commit.message, 'test') && startsWith(github.repository, 'apify/')"
44+
name: Release prepare
45+
needs: [code_checks, tests]
46+
runs-on: ubuntu-latest
47+
outputs:
48+
version_number: ${{ steps.release_prepare.outputs.version_number }}
49+
tag_name: ${{ steps.release_prepare.outputs.tag_name }}
50+
changelog: ${{ steps.release_prepare.outputs.changelog }}
51+
steps:
52+
- uses: apify/workflows/git-cliff-release@main
53+
id: release_prepare
54+
name: Release prepare
55+
with:
56+
release_type: prerelease
57+
existing_changelog_path: CHANGELOG.md
58+
59+
changelog_update:
60+
name: Changelog update
61+
needs: [release_prepare]
62+
uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main
63+
with:
64+
version_number: ${{ needs.release_prepare.outputs.version_number }}
65+
changelog: ${{ needs.release_prepare.outputs.changelog }}
66+
secrets: inherit
67+
68+
pypi_publish:
69+
name: PyPI publish
70+
needs: [release_prepare, changelog_update]
71+
runs-on: ubuntu-latest
72+
permissions:
73+
contents: write
74+
id-token: write # Required for OIDC authentication.
75+
environment:
76+
name: pypi
77+
url: https://pypi.org/project/crawlee
78+
steps:
79+
- name: Prepare distribution
80+
uses: apify/workflows/prepare-pypi-distribution@main
81+
with:
82+
package_name: crawlee
83+
is_prerelease: "yes"
84+
version_number: ${{ needs.release_prepare.outputs.version_number }}
85+
ref: ${{ needs.changelog_update.outputs.changelog_commitish }}
86+
87+
- name: Publish package to PyPI
88+
uses: pypa/gh-action-pypi-publish@release/v1
89+
90+
doc_release_post_publish:
91+
name: Doc release post publish
92+
needs: [changelog_update, pypi_publish]
93+
uses: ./.github/workflows/_release_docs.yaml
94+
with:
95+
# Use the ref from the changelog update to include the updated changelog.
96+
ref: ${{ needs.changelog_update.outputs.changelog_commitish }}
97+
secrets: inherit

0 commit comments

Comments
 (0)