Skip to content

Commit 8dfb51f

Browse files
mikheillomidzeqmikheillomidzeq
authored
[CP-48] Automate javascript package release (#85)
* Add package.json. Add testing release job. * Add package-lock json * Only pack necessary files * Fix the npmrc file creation * Test one more time. * Lets commit npmrc and do not create if via workflow * Correct the step name. * Correct the step name. * Remove testing comments and make PR ready for review. * Move shared part into separate file. Update release drafter and tag creation job to wait for both package push. * Fix the docs. * Generate .d.ts and .js files using buf plugin. * add .js extension to imports. * Update readme.md * Merge main and regenerate js code. * Fix tag in release step. * Address code review feedback. * Address code review feedback v2. * Merge main in.
1 parent 891303e commit 8dfb51f

36 files changed

Lines changed: 2587 additions & 812 deletions

‎.github/actions/shared-setup/action.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: "Shared Setup"
2+
description: "GCP auth and version environment setup"
3+
inputs:
4+
workload_identity_provider:
5+
required: true
6+
description: "The Workload Identity Provider to use for authentication"
7+
service_account:
8+
required: true
9+
description: "The Service Account to impersonate"
10+
version_source:
11+
required: true
12+
description: "The source of the version to set, typically the branch name"
13+
target:
14+
required: true
15+
description: "The target language for the versioning, either 'python' or 'node'"
16+
runs:
17+
using: "composite"
18+
steps:
19+
- name: Authenticate to Google Cloud
20+
uses: google-github-actions/auth@v2
21+
with:
22+
workload_identity_provider: ${{ inputs.workload_identity_provider }}
23+
service_account: ${{ inputs.service_account }}
24+
25+
- name: Extract version from branch name
26+
shell: bash
27+
run: |
28+
VERSION="${{ inputs.version_source##*/ }}"
29+
STRIPPED_VERSION="${VERSION#v}"
30+
31+
echo "RELEASE_VERSION=$VERSION" >> $GITHUB_ENV
32+
echo "PACKAGE_VERSION=$STRIPPED_VERSION" >> $GITHUB_ENV
33+
34+
if [ "${{ inputs.target }}" = "python" ]; then
35+
sed -i "s/^version = .*/version = \"$STRIPPED_VERSION\"/" pyproject.toml
36+
elif [ "${{ inputs.target }}" = "node" ]; then
37+
jq --arg version "$STRIPPED_VERSION" '.version = $version' package.json > tmp.$$.json && mv tmp.$$.json package.json
38+
fi

‎.github/workflows/release-workflow.yaml‎

Lines changed: 81 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,19 @@
1-
name: Upload Python Package to GCP Artifact Registry
1+
name: Upload Python and JS Packages to GCP Artifact Registry
22

33
on:
44
push:
55
branches:
66
- "releases/v*"
77

8-
98
jobs:
10-
Release:
11-
name: Release Workflow
9+
python-release:
10+
name: Python Release
1211
runs-on: ubuntu-latest
1312

1413
permissions:
15-
pull-requests: write # write permission is required to create a github release
16-
contents: write # allow pushing tags
17-
id-token: write # needed for authentication since we are using Workload Identity Federation
14+
pull-requests: write
15+
contents: write
16+
id-token: write
1817

1918
steps:
2019
- name: Checkout repository
@@ -26,53 +25,100 @@ jobs:
2625
# Install a specific version of uv.
2726
version: "0.7.0"
2827

29-
- name: "Set up Python"
28+
- name: Set up Python
3029
uses: actions/setup-python@v5
3130
with:
3231
python-version-file: "pyproject.toml"
33-
- run: |
34-
make python/dev-install
3532

36-
- name: Authenticate to Google Cloud
37-
uses: google-github-actions/auth@v2
33+
- run: make python/dev-install
34+
35+
- name: "Shared: Auth + Version Info"
36+
uses: ./.github/actions/shared-setup
3837
with:
3938
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
4039
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
40+
version_source: ${{ github.ref }}
41+
target: python
4142

42-
- name: Configure gcloud
43+
- name: Configure gcloud for Python
4344
run: |
4445
gcloud config set project qdrant-cloud
45-
gcloud config set artifacts/repository python
4646
gcloud config set artifacts/location us
47+
gcloud config set artifacts/repository python
4748
48-
- name: Set version in pyproject.toml from branch name
49-
run: |
50-
VERSION="${GITHUB_REF##*/}" # e.g., v1.1.1
51-
STRIPPED_VERSION="${VERSION#v}" # e.g., 1.1.1
52-
53-
echo "Detected version: $VERSION"
54-
echo "RELEASE_VERSION=$VERSION" >> $GITHUB_ENV
55-
echo "PACKAGE_VERSION=$STRIPPED_VERSION" >> $GITHUB_ENV
56-
57-
sed -i "s/^version = .*/version = \"$STRIPPED_VERSION\"/" pyproject.toml
58-
59-
- name: Build the package
60-
run: |
61-
uv build
62-
63-
- name: Upload to Artifact Registry
49+
- name: Build and upload Python package
6450
run: |
51+
uv build
6552
uv run twine upload \
6653
--repository-url https://us-python.pkg.dev/qdrant-cloud/python/ \
6754
--username oauth2accesstoken \
6855
--password "$(gcloud auth print-access-token)" \
6956
dist/*
70-
57+
58+
js-release:
59+
name: Node.js Release
60+
runs-on: ubuntu-latest
61+
62+
permissions:
63+
pull-requests: write
64+
contents: write
65+
id-token: write
66+
67+
steps:
68+
- name: Checkout repository
69+
uses: actions/checkout@v4
70+
71+
- name: Set up Node.js
72+
uses: actions/setup-node@v4
73+
with:
74+
node-version: "22"
75+
76+
- name: "Shared: Auth + Version Info"
77+
uses: ./.github/actions/shared-setup
78+
with:
79+
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
80+
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
81+
version_source: ${{ github.ref }}
82+
target: node
83+
84+
- name: Move build output to publish root
85+
run: |
86+
mkdir -p npm_publish
87+
cp -r gen/typescript/* npm_publish/
88+
cp package.example.json npm_publish/package.json
89+
cp .npmrc npm_publish/
90+
cp README.md npm_publish/ || true
91+
92+
- name: Refresh the access token for connecting to the repository
93+
working-directory: npm_publish
94+
run: npm run artifactregistry-login
95+
96+
- name: Publish JS package
97+
working-directory: npm_publish
98+
run: npm publish
99+
finalize-release:
100+
name: Finalize Release
101+
runs-on: ubuntu-latest
102+
103+
needs:
104+
- python-release
105+
- js-release
106+
107+
permissions:
108+
pull-requests: write
109+
contents: write
110+
id-token: write
111+
112+
steps:
113+
- name: Checkout repository
114+
uses: actions/checkout@v4
115+
71116
- name: Tag Release
72-
shell: bash
73117
run: |
74-
git tag ${{ env.RELEASE_VERSION }}
75-
git push origin ${{ env.RELEASE_VERSION }}
118+
VERSION="${GITHUB_REF##*/}" # from 'refs/heads/releases/v1.2.1' → 'v1.2.1'
119+
echo "RELEASE_VERSION=$VERSION" >> $GITHUB_ENV
120+
git tag "$VERSION"
121+
git push origin tag "$VERSION"
76122
77123
- name: Publish Release Notes
78124
uses: release-drafter/release-drafter@v6
@@ -82,4 +128,4 @@ jobs:
82128
tag: ${{ env.RELEASE_VERSION }}
83129
publish: true
84130
env:
85-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
131+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,5 @@ tmp/
55
__pycache__/
66
*.pyc
77
/gen-dummy
8+
node_modules/
9+
var/

‎.npmrc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
@qdrant:registry=https://us-npm.pkg.dev/qdrant-cloud/npm/
2+
//us-npm.pkg.dev/qdrant-cloud/npm/:always-auth=true

‎README.md‎

Lines changed: 62 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -47,26 +47,67 @@ import (
4747

4848
### Python projects
4949

50-
First you need to install the repository as a package:
51-
52-
``` sh
53-
uv add "git+https://github.com/qdrant/qdrant-cloud-public-api"
54-
```
55-
56-
Then you can import the generated code in your python project. Ex:
57-
58-
```python
59-
from qdrant.cloud.booking.v2.booking_pb2_grpc import BookingServiceServicer
60-
from qdrant.cloud.booking.v2.booking_pb2 import ListPackagesRequest
61-
62-
63-
def main():
64-
print(BookingServiceServicer)
65-
print(ListPackagesRequest)
66-
67-
68-
if __name__ == "__main__":
69-
main()
70-
```
50+
- Make sure you are authorised to gcloud. `gcloud auth application-default login`
51+
52+
- Install the package:
53+
``` sh
54+
# install auth tool
55+
uv tool install keyring --with keyrings.google-artifactregistry-auth --force
56+
# install
57+
uv add qdrant-cloud-public-api --keyring-provider subprocess --index https://oauth2accesstoken@us-python.pkg.dev/qdrant-cloud/python/simple
58+
```
59+
60+
61+
- Now you can import the generated code in your python project. Ex:
62+
63+
```python
64+
from qdrant.cloud.booking.v2.booking_pb2_grpc import BookingServiceServicer
65+
from qdrant.cloud.booking.v2.booking_pb2 import ListPackagesRequest
66+
67+
68+
def main():
69+
print(BookingServiceServicer)
70+
print(ListPackagesRequest)
71+
72+
73+
if __name__ == "__main__":
74+
main()
75+
```
76+
77+
78+
### Typescript projects
79+
80+
- Make sure you are authorised to gcloud. `gcloud auth application-default login`
81+
- Create `.npmrc` file in the root of your project with the following content:
82+
```text
83+
@qdrant:registry=https://us-npm.pkg.dev/qdrant-cloud/npm/
84+
//us-npm.pkg.dev/qdrant-cloud/npm/:always-auth=true
85+
```
86+
- Add auth script to your package.json file:
87+
```json
88+
"scripts": {
89+
...
90+
"artifactregistry-login": "npx --registry=https://registry.npmjs.org google-artifactregistry-auth"
91+
...
92+
},
93+
```
94+
- Run the auth script:
95+
```shell
96+
npm run artifactregistry-login
97+
```
98+
- Install the package:
99+
``` sh
100+
npm install @qdrant/qdrant-cloud-public-api
101+
```
102+
103+
- Now you can import the generated code in your typescript project. Ex:
104+
105+
```typescript
106+
import * as grpc from '@qdrant/qdrant-cloud-public-api/qdrant/cloud/cluster/v1/cluster_pb.js';
107+
import { DatabaseApiKeyService } from "@qdrant/qdrant-cloud-public-api/qdrant/cloud/cluster/auth/v1/database_api_key_pb.js";
108+
109+
console.log('Loaded service definition keys:', Object.keys(grpc));
110+
console.log(DatabaseApiKeyService);
111+
```
71112
72113
## [Release Process](docs/release.md)

‎buf.gen.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,10 @@ plugins:
5353
# Generate base types and services for Typescript
5454
- remote: buf.build/bufbuild/es:v2.2.3
5555
include_imports: true
56-
opt: target=ts
56+
opt:
57+
- target=js+dts
58+
- import_extension=js
59+
5760
out: gen/typescript
5861

5962
inputs:

‎docs/release.md‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
This document explains how to release a new version of the generated libraries (Python, JavaScript, etc.) from the proto repository.
44
> **Note:**
5-
> Only python is supported for now, but the process is designed to be extensible for JavaScript or other languages in the future.
5+
> Only python and typescript is supported for now, but the process is designed to be extensible for other languages in the future.
66
77
---
88
## Release Steps
@@ -26,23 +26,30 @@ Once the branch is pushed:
2626
- A GitHub Actions workflow will trigger automatically.
2727
- The workflow will:
2828
- Inject the version into the generated client(s) dynamically.
29-
- Build the Python package (qdrant-cloud-public-api).
30-
- Upload the Python package to the Google Artifact Registry.
31-
- (Future extension: Publish JavaScript packages, if needed.)
29+
- Build the Python and Typescript package (qdrant-cloud-public-api).
30+
- Upload the Python and Typescript package to the Google Artifact Registry.
3231

3332
---
3433

3534
## How Versioning Works
36-
- The version in pyproject.toml is automatically replaced by the version from Git branch name during the build.
37-
- Example: if the pushed branch is releases/v1.3.0, the final Python package version will be 1.3.0.
38-
- Developers should not manually edit the version field in pyproject.toml.
35+
- The version in pyproject.toml and package.json is automatically replaced by the version from Git branch name during the build.
36+
- Example: if the pushed branch is releases/v1.3.0, the final Python/Typescript package version will be 1.3.0.
37+
- Developers should not manually edit the version field in pyproject.toml or package.json.
3938

4039
Inside pyproject.toml, you will see:
4140
```toml
4241
# The version is replaced by the version from Git branch name during the release process
4342
version = "0.0.0"
4443
```
4544
This is normal.
45+
Same for package.json:
46+
```json
47+
{
48+
"name": "@qdrant/qdrant-cloud-public-api",
49+
"version": "0.0.0",
50+
...
51+
}
52+
```
4653

4754
---
4855

gen/typescript/buf/validate/validate_pb.ts renamed to gen/typescript/buf/validate/validate_pb.d.ts

Lines changed: 69 additions & 108 deletions
Large diffs are not rendered by default.

‎gen/typescript/buf/validate/validate_pb.js‎

Lines changed: 316 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎gen/typescript/google/api/annotations_pb.d.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// @generated by protoc-gen-es v2.2.3 with parameter "target=js+dts,import_extension=js"
16+
// @generated from file google/api/annotations.proto (package google.api, syntax proto3)
17+
/* eslint-disable */
18+
19+
import type { GenExtension, GenFile } from "@bufbuild/protobuf/codegenv1";
20+
import type { MethodOptions } from "@bufbuild/protobuf/wkt";
21+
import type { HttpRule } from "./http_pb.js";
22+
23+
/**
24+
* Describes the file google/api/annotations.proto.
25+
*/
26+
export declare const file_google_api_annotations: GenFile;
27+
28+
/**
29+
* See `HttpRule`.
30+
*
31+
* @generated from extension: google.api.HttpRule http = 72295728;
32+
*/
33+
export declare const http: GenExtension<MethodOptions, HttpRule>;
34+

0 commit comments

Comments
 (0)