diff --git a/.github/workflows/release-prepare-branch.yml b/.github/workflows/release-prepare-branch.yml index 7742d9abf2..713d485b0b 100644 --- a/.github/workflows/release-prepare-branch.yml +++ b/.github/workflows/release-prepare-branch.yml @@ -10,6 +10,33 @@ # This workflow automates the preparation of patch release branches. # It validates the release branch, updates gradle.properties to the next version # generates the THIRD-PARTY file, and creates a pull request with all changes. +# +# The OpenSearch project does not permit this repository's workflows to push +# branches and open pull requests against itself. Instead, the pull request +# branch is pushed to a fork and the PR is opened from that fork against this +# repository. This requires two settings to be configured: +# +# vars.RELEASE_FORK_REPOSITORY - the fork the PR branch is pushed to, as +# 'owner/repo'. It must be a real GitHub fork +# of this repository (same fork network), as +# cross-repository PRs are only allowed within +# a fork network. +# secrets.RELEASE_FORK_TOKEN - a classic personal access token with the +# 'public_repo' (or 'repo' for private +# repositories) and 'workflow' scopes. The +# action uses it to push the branch to the +# fork and to open the PR against this +# repository, so its owner must have write +# access to the fork. The 'workflow' scope is +# required because the pushed branch may modify +# files under .github/workflows; GitHub rejects +# such pushes from a token without it. A +# fine-grained token does not work here because +# the single token must act on two repositories. +# +# The PR base is always this repository (the one running the workflow). To test +# on your own fork, run this workflow from your fork and point +# RELEASE_FORK_REPOSITORY at a fork of your fork within the same network. name: Prepare Release Branch @@ -27,6 +54,12 @@ jobs: steps: - name: Checkout Data Prepper uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + # Do not persist the checkout credentials in the local git config. + # When pushing to a fork, peter-evans/create-pull-request adds its own + # authentication, and a persisted Authorization header would collide + # with it, causing 'Duplicate header: "Authorization"' (HTTP 400). + persist-credentials: false - name: Validate release branch id: validate_branch @@ -50,6 +83,20 @@ jobs: echo "✅ Valid release branch: $BRANCH_NAME" echo "branch=$BRANCH_NAME" >> $GITHUB_OUTPUT + - name: Validate fork configuration + run: | + if [ -z "${{ vars.RELEASE_FORK_REPOSITORY }}" ]; then + echo "::error::The RELEASE_FORK_REPOSITORY variable is not set." + echo "Set it to the 'owner/repo' of a fork of this repository that the release PR branch will be pushed to." + exit 1 + fi + if [ -z "${{ secrets.RELEASE_FORK_TOKEN }}" ]; then + echo "::error::The RELEASE_FORK_TOKEN secret is not set." + echo "Set it to a token with write access to ${{ vars.RELEASE_FORK_REPOSITORY }} and permission to create pull requests against this repository." + exit 1 + fi + echo "Pull request branch will be pushed to fork: ${{ vars.RELEASE_FORK_REPOSITORY }}" + - name: Set up JDK uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 with: @@ -91,18 +138,12 @@ jobs: echo "Successfully generated THIRD-PARTY file" - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - - name: Create Pull Request id: create_pr uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6 with: - token: ${{ steps.github_app_token.outputs.token }} + token: ${{ secrets.RELEASE_FORK_TOKEN }} + push-to-fork: ${{ vars.RELEASE_FORK_REPOSITORY }} add-paths: | gradle.properties THIRD-PARTY @@ -129,6 +170,7 @@ jobs: After merging this PR: - [ ] Prepare release notes (see [release notes script](release/script/release-notes/README.md)) + - [ ] Open an issue in the [opensearch-project/.github](https://github.com/opensearch-project/.github) repository requesting the release tag `${{ steps.version.outputs.version }}` be created and pushed. - [ ] Run the [Release Artifacts workflow](https://github.com/opensearch-project/data-prepper/actions/workflows/release.yml) - [ ] Approve the release issue (requires 2 maintainer approvals) - [ ] Update the draft release with release notes @@ -153,8 +195,9 @@ jobs: run: | echo "::error::Failed to create pull request" echo "This could be due to:" - echo " - Missing or invalid GitHub App credentials" - echo " - Insufficient permissions" + echo " - An invalid or expired RELEASE_FORK_TOKEN" + echo " - The token lacking write access to ${{ vars.RELEASE_FORK_REPOSITORY }} or permission to create pull requests" + echo " - RELEASE_FORK_REPOSITORY (${{ vars.RELEASE_FORK_REPOSITORY }}) not being a fork of this repository in the same fork network" echo " - Network issues" echo "Please check the logs above for more details" exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b13472db5c..c63db9b711 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,32 @@ # compatible open source license. # +# This workflow builds and validates the release artifacts, drafts the GitHub +# release, and opens the changelog pull request. +# +# The OpenSearch project does not permit this repository's workflows to push +# branches and open pull requests against itself. Instead, the changelog PR +# branch is pushed to a fork and the PR is opened from that fork against this +# repository. This requires two settings to be configured: +# +# vars.RELEASE_FORK_REPOSITORY - the fork the PR branch is pushed to, as +# 'owner/repo'. It must be a real GitHub fork +# of this repository (same fork network), as +# cross-repository PRs are only allowed within +# a fork network. +# secrets.RELEASE_FORK_TOKEN - a classic personal access token with the +# 'public_repo' (or 'repo' for private +# repositories) and 'workflow' scopes. The +# action uses it to push the branch to the +# fork and to open the PR against this +# repository, so its owner must have write +# access to the fork. The 'workflow' scope is +# required because the pushed branch may modify +# files under .github/workflows; GitHub rejects +# such pushes from a token without it. A +# fine-grained token does not work here because +# the single token must act on two repositories. + name: Release Artifacts on: @@ -30,9 +56,60 @@ permissions: pull-requests: write jobs: + # Verifies the release preconditions + # 1. The release tag must already exist. A maintainer requests the tag via an issue in the + # opensearch-project/.github repository before running this workflow. + # 2. The fork settings used to open the changelog pull request must be set. + verify-preconditions: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout Data Prepper + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Get Version + run: grep '^version=' gradle.properties >> $GITHUB_ENV + - name: Verify release tag exists + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + github-token: ${{ github.TOKEN }} + script: | + const tag = '${{ env.version }}'; + try { + await github.rest.git.getRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `tags/${tag}` + }); + core.info(`Found tag ${tag}.`); + } catch (error) { + if (error.status === 404) { + core.setFailed( + `Release tag '${tag}' does not exist. Request the tag by opening an issue in the ` + + `opensearch-project/.github repository before running this workflow.` + ); + } else { + throw error; + } + } + - name: Validate fork configuration + run: | + if [ -z "${{ vars.RELEASE_FORK_REPOSITORY }}" ]; then + echo "::error::The RELEASE_FORK_REPOSITORY variable is not set." + echo "Set it to the 'owner/repo' of a fork of this repository that the changelog PR branch will be pushed to." + exit 1 + fi + if [ -z "${{ secrets.RELEASE_FORK_TOKEN }}" ]; then + echo "::error::The RELEASE_FORK_TOKEN secret is not set." + echo "Set it to a token with write access to ${{ vars.RELEASE_FORK_REPOSITORY }} and permission to create pull requests against this repository." + exit 1 + fi + echo "Changelog pull request branch will be pushed to fork: ${{ vars.RELEASE_FORK_REPOSITORY }}" + build: runs-on: ubuntu-latest timeout-minutes: 50 + needs: verify-preconditions steps: - name: Set up JDK @@ -181,24 +258,12 @@ jobs: echo 'release_major_tag: ${{ github.event.inputs.release-major-tag }}' >> release-description.yaml echo 'release_latest_tag: ${{ github.event.inputs.release-latest-tag }}' >> release-description.yaml - - name: Create tag - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - github-token: ${{ github.TOKEN }} - script: | - github.rest.git.createRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: 'refs/tags/${{ env.version }}', - sha: context.sha - }) - - name: Draft release uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v1 with: draft: true name: '${{ env.version }}' - tag_name: 'refs/tags/${{ env.version }}' + tag_name: '${{ env.version }}' files: | release-description.yaml @@ -227,17 +292,11 @@ jobs: - name: Generate Changelog run: release/script/changelog/generate-changelog.sh ${{ needs.promote.outputs.version }} - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - - name: Create Pull Request uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6 with: - token: ${{ steps.github_app_token.outputs.token }} + token: ${{ secrets.RELEASE_FORK_TOKEN }} + push-to-fork: ${{ vars.RELEASE_FORK_REPOSITORY }} add-paths: | release/release-notes/data-prepper.change-log-${{ needs.promote.outputs.version }}.md commit-message: 'Add Data Prepper ${{ needs.promote.outputs.version }} changelog.' diff --git a/release/README.md b/release/README.md index d8fedb3abe..51a824b6d1 100644 --- a/release/README.md +++ b/release/README.md @@ -9,3 +9,82 @@ It is not a project and instead holds release notes and change logs for Data Pre ## Performing a Release See [RELEASING.md](../RELEASING.md) for instructions to follow for any release. + +## Configuring the GitHub repo + +Running a release requires both AWS staging resources and a number of GitHub Actions secrets and variables. +The release workflows ([Prepare Release Branch](../.github/workflows/release-prepare-branch.yml) and +[Release Artifacts](../.github/workflows/release.yml)) read these to build and upload the artifacts and to +open their pull requests (PRs). + +### Deploy the AWS staging resources + +The release build uploads archives and Maven artifacts to AWS S3 and publishes Docker images to a staging ECR +repository. These resources, and the IAM role that GitHub Actions assumes to access them, are provisioned by +the [staging-resources-cdk](staging-resources-cdk/README.md) project. Follow that project's README to install +the CDK and deploy the stacks before running a release. + +The CDK deployment provides the values you will use for the `RELEASE_IAM_ROLE`, `ARCHIVES_BUCKET_NAME`, +`ARCHIVES_PUBLIC_URL`, and `ECR_REPOSITORY_URL` secrets described below. + +### Configure a fork for pull requests + +The OpenSearch project does not permit this repository's workflows to push branches and open PRs +against itself. Instead, the release workflows push their PR branches to a fork and open the PR from that fork +back against this repository. This applies to both the release preparation PR and the changelog PR. + +To run a release build, you must have a fork of Data Prepper used for staging these changes and PRs. +This must be an actual GitHub fork of the repository you are releasing, because GitHub only permits +cross-repository PRs between repositories in the same fork network. + +The fork is identified by the `RELEASE_FORK_REPOSITORY` variable, and the token used to push to it and open +the PR is the `RELEASE_FORK_TOKEN` secret. `RELEASE_FORK_TOKEN` must be a **classic** personal access token +(PAT) meeting the following requirements. + +* It is a classic PAT. A fine-grained token does not work because the single token must act on two + repositories (the fork it pushes to and this repository it opens the PR against). +* Its owner has write access to the fork named in `RELEASE_FORK_REPOSITORY`. +* It has the `public_repo` scope (or `repo` if the repository is private). +* It has the `workflow` scope. This is required because the pushed branch may include changes to files + under `.github/workflows`, and GitHub rejects such pushes from a token without it. + +Because a classic PAT acts as its owner across all of their repositories, use a dedicated release or bot +account to own this token rather than a maintainer's personal account, and set a short expiration so the +token is rotated regularly. + +### Secrets and variables + +Configure the following on the repository that runs the release workflows. Create variables as +[repository variables](https://docs.github.com/en/actions/learn-github-actions/variables) and secrets as +[repository secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets). + +| Name | Type | Source | Description | +| ---- | ---- | ------ | ----------- | +| `RELEASE_FORK_REPOSITORY` | Variable | You | The `owner/repo` of the fork the PR branches are pushed to, for example `opensearch-ci-bot/data-prepper`. | +| `RELEASE_FORK_TOKEN` | Secret | You | A classic PAT used to push the PR branch to the fork and open the PR. See the requirements above. | +| `RELEASE_IAM_ROLE` | Secret | CDK | The ARN of the IAM role GitHub Actions assumes to access the staging resources. | +| `ARCHIVES_BUCKET_NAME` | Secret | CDK | The name of the S3 bucket the release archives and Maven artifacts are uploaded to. | +| `ARCHIVES_PUBLIC_URL` | Secret | CDK | The public base URL the uploaded archives are served from, used by the tarball smoke tests. | +| `ECR_REPOSITORY_URL` | Secret | CDK | The URL of the staging ECR repository the Docker images are pushed to. | + +## Testing release changes + +You can test release changes by running the release workflows from your own fork of Data Prepper. +The release workflows always open their PRs against the repository that runs them, so when you run them +from your fork, the PRs target your fork. + +To set this up: +* Run the workflows from your fork of Data Prepper (for example `your-user/data-prepper`). +* Create a second fork of your fork in an account or organization you control (for example + `your-test-org/data-prepper`). This second fork is the head repository that the PR branches are pushed to. + It is required because GitHub does not allow a PR where the head and base are the same repository. + Note that GitHub permits only one fork per account, so the second fork must live in a different account + or organization; both forks remain in the same fork network, which is what allows the cross-repository PR. +* Set `RELEASE_FORK_REPOSITORY` and `RELEASE_FORK_TOKEN` on your fork as described in + [Configuring the GitHub repo](#configuring-the-github-repo), pointing `RELEASE_FORK_REPOSITORY` at the + second fork. + +To test, you must also make a few manual modifications that you will not check in. +* Update the `get_approvers` step with `minimum-approvals: 1`. Otherwise, you will be blocked in the promote step. +* Change your `CODEOWNERS` file to have only your user. Without this, the promote step will try to assign this to users who are not in your fork and then fail. +* You should consider updating the "Build Jar Files" step to only `assemble` instead of `build` to speed up your tests. `./gradlew --parallel --max-workers 2 assemble`