Skip to content

Commit bd36ee8

Browse files
dcodeIOdcodeIO
committed
ci: Track protobuf conformance in CI
1 parent 1904533 commit bd36ee8

9 files changed

Lines changed: 775 additions & 19 deletions

File tree

‎.github/actions/conformance/action.yml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
name: "Run protobuf conformance"
2+
description: "Build and run the upstream Protocol Buffers conformance suite for protobuf.js."
3+
inputs:
4+
upstream-version:
5+
description: "Upstream protocolbuffers/protobuf tag or branch to test against."
6+
default: "v33.0"
7+
upstream-dir:
8+
description: "Directory where upstream protobuf is checked out."
9+
default: "tests/conformance/upstream"
10+
runner-cache:
11+
description: "Directory used to cache the built upstream conformance runner."
12+
default: "tests/conformance/runner-cache"
13+
output-dir:
14+
description: "Directory where conformance logs and generated reports are written."
15+
default: "tests/conformance/out"
16+
maximum-edition:
17+
description: "Maximum protobuf edition passed to the conformance runner."
18+
default: "2024"
19+
outputs:
20+
exit-code:
21+
description: "Exit code returned by the upstream conformance runner."
22+
value: ${{ steps.run-conformance.outputs.exit_code }}
23+
runs:
24+
using: "composite"
25+
steps:
26+
- name: "Prepare conformance runner cache"
27+
shell: bash
28+
run: mkdir -p "${{ inputs.runner-cache }}"
29+
30+
- uses: actions/cache@v5
31+
id: conformance-runner-cache
32+
with:
33+
path: ${{ inputs.runner-cache }}
34+
key: protobuf-conformance-runner-${{ runner.os }}-${{ inputs.upstream-version }}
35+
36+
- name: "Check out upstream protobuf"
37+
shell: bash
38+
run: git clone --depth 1 --branch "${{ inputs.upstream-version }}" https://github.com/protocolbuffers/protobuf.git "${{ inputs.upstream-dir }}"
39+
40+
- name: "Install build dependencies"
41+
if: steps.conformance-runner-cache.outputs.cache-hit != 'true'
42+
shell: bash
43+
run: |
44+
sudo apt-get update
45+
sudo apt-get install -y cmake ninja-build
46+
47+
- name: "Build upstream conformance runner"
48+
if: steps.conformance-runner-cache.outputs.cache-hit != 'true'
49+
shell: bash
50+
run: |
51+
cmake -S "${{ inputs.upstream-dir }}" -B "${{ inputs.runner-cache }}/build" -G Ninja \
52+
-DCMAKE_BUILD_TYPE=Release \
53+
-Dprotobuf_BUILD_CONFORMANCE=ON \
54+
-Dprotobuf_BUILD_TESTS=OFF \
55+
-Dprotobuf_BUILD_EXAMPLES=OFF \
56+
-Dprotobuf_INSTALL=OFF \
57+
-Dprotobuf_FORCE_FETCH_DEPENDENCIES=ON
58+
cmake --build "${{ inputs.runner-cache }}/build" --target conformance_test_runner --parallel 2
59+
60+
- name: "Generate protobuf.js conformance target"
61+
shell: bash
62+
env:
63+
PROTOBUF_UPSTREAM: ${{ inputs.upstream-dir }}
64+
run: node tests/conformance/generate.js
65+
66+
- name: "Locate Node.js"
67+
id: node
68+
shell: bash
69+
run: echo "path=$(command -v node)" >> "$GITHUB_OUTPUT"
70+
71+
- name: "List conformance tests"
72+
shell: bash
73+
run: |
74+
mkdir -p "${{ inputs.output-dir }}"
75+
: > "${{ inputs.output-dir }}/failure_list.txt"
76+
"${{ inputs.runner-cache }}/build/conformance_test_runner" \
77+
--maximum_edition "${{ inputs.maximum-edition }}" \
78+
--enforce_recommended \
79+
--verbose \
80+
--failure_list "${{ inputs.output-dir }}/failure_list.txt" \
81+
--output_dir "${{ inputs.output-dir }}" \
82+
"${{ steps.node.outputs.path }}" tests/conformance/testee.js --list \
83+
> "${{ inputs.output-dir }}/conformance-tests.log" 2>&1
84+
85+
- name: "Run conformance suite"
86+
id: run-conformance
87+
shell: bash
88+
run: |
89+
set +e
90+
"${{ inputs.runner-cache }}/build/conformance_test_runner" \
91+
--maximum_edition "${{ inputs.maximum-edition }}" \
92+
--enforce_recommended \
93+
--failure_list "${{ inputs.output-dir }}/failure_list.txt" \
94+
--output_dir "${{ inputs.output-dir }}" \
95+
"${{ steps.node.outputs.path }}" tests/conformance/testee.js \
96+
> "${{ inputs.output-dir }}/conformance.log" 2>&1
97+
status=$?
98+
set -e
99+
echo "exit_code=$status" >> "$GITHUB_OUTPUT"
100+
tail -n 80 "${{ inputs.output-dir }}/conformance.log"
101+
if [ -s "${{ inputs.output-dir }}/failing_tests.txt" ]; then
102+
echo ""
103+
echo "Failing conformance tests:"
104+
cat "${{ inputs.output-dir }}/failing_tests.txt"
105+
fi
106+
107+
- name: "Summarize conformance results"
108+
if: always()
109+
shell: bash
110+
run: node tests/conformance/report.js "${{ inputs.output-dir }}/conformance.log" "${{ inputs.output-dir }}/conformance-tests.log" --json "${{ inputs.output-dir }}/conformance.json" | tee -a "$GITHUB_STEP_SUMMARY"

‎.github/workflows/test.yml‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
run: npm run test:sources
4242
- name: "Test types"
4343
run: npm run test:types
44-
bench:
44+
benchmark:
4545
runs-on: ubuntu-latest
4646
steps:
4747
- uses: actions/checkout@v6
@@ -51,4 +51,33 @@ jobs:
5151
- name: "Install dependencies"
5252
run: npm install
5353
- name: "Run benchmark"
54-
run: npm run bench
54+
run: |
55+
set -o pipefail
56+
npm run bench 2>&1 | tee "$RUNNER_TEMP/bench.log"
57+
{
58+
echo '```'
59+
sed -r 's/\x1b\[[0-9;]*m//g' "$RUNNER_TEMP/bench.log"
60+
echo '```'
61+
} >> "$GITHUB_STEP_SUMMARY"
62+
conformance:
63+
runs-on: ubuntu-latest
64+
steps:
65+
- uses: actions/checkout@v6
66+
- uses: actions/setup-node@v6
67+
with:
68+
node-version: "lts/*"
69+
- name: "Install dependencies"
70+
run: npm install --ignore-scripts
71+
- name: "Install CLI dependencies"
72+
run: npm --prefix cli install --ignore-scripts
73+
- uses: ./.github/actions/conformance
74+
- uses: actions/upload-artifact@v7
75+
if: always()
76+
with:
77+
name: conformance-results
78+
path: |
79+
tests/conformance/out/conformance.json
80+
tests/conformance/out/conformance.log
81+
tests/conformance/out/conformance-tests.log
82+
tests/conformance/out/failing_tests.txt
83+
if-no-files-found: ignore

‎README.md‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
**Protocol Buffers** are a language-neutral, platform-neutral, extensible way of serializing structured data for use in communications protocols, data storage, and more, originally designed at Google ([see](https://protobuf.dev/)).
1111

12-
**protobuf.js** is a freestanding JavaScript implementation of Protocol Buffers with TypeScript support for Node.js and the browser. It works with `.proto` files out of the box and can generate optimized encoders and decoders at runtime or emit them statically.
12+
**protobuf.js** is a standalone JavaScript implementation of Protocol Buffers with TypeScript support for Node.js and the browser. It works with `.proto` files out of the box, is optimized for fast binary I/O, and supports runtime reflection as well as static code generation.
1313

1414
## Getting started
1515

@@ -25,7 +25,7 @@ The [command line utility](./cli/) for generating reflection bundles, static cod
2525
npm install --save-dev protobufjs-cli
2626
```
2727

28-
The CLI is a small but capable standalone protobuf.js toolchain. It does not require `protoc`.
28+
The CLI is a small but capable standalone protobuf.js toolchain. It does not require `protoc` or a language plugin.
2929

3030
### Choose a runtime
3131

@@ -35,7 +35,7 @@ The CLI is a small but capable standalone protobuf.js toolchain. It does not req
3535
| `protobufjs/light.js` | Reflection | You load JSON bundles or build schemas programmatically
3636
| `protobufjs/minimal.js` | Static runtime | You only use generated static code
3737

38-
Builds with reflection include just-in-time code generation. Use the CLI to emit the same optimized code ahead of time and run it on the minimal runtime. The full build includes the light build, and the light build includes the minimal runtime.
38+
Reflection builds generate specialized code at runtime. The CLI can emit the same optimized code ahead of time for the minimal runtime. The full build includes the light build, and the light build includes the minimal runtime.
3939

4040
### Browser builds
4141

@@ -112,6 +112,19 @@ const object = AwesomeMessage.toObject(message, {
112112
});
113113
```
114114

115+
Common `ConversionOptions` are:
116+
117+
| Option | Effect |
118+
|--------|--------|
119+
| `longs: String` | Converts 64-bit values to decimal strings |
120+
| `longs: Number` | Converts 64-bit values to JS numbers (may lose precision) |
121+
| `enums: String` | Converts enum values to names |
122+
| `bytes: String` | Converts bytes to base64 strings |
123+
| `defaults: true` | Includes default values for unset fields |
124+
| `arrays: true` | Includes empty arrays for repeated fields |
125+
| `objects: true` | Includes empty objects for map fields |
126+
| `oneofs: true` | Includes virtual oneof discriminator properties |
127+
115128
## Message API
116129

117130
Message types expose focused methods for validation, conversion, and binary I/O.
@@ -126,18 +139,7 @@ Message types expose focused methods for validation, conversion, and binary I/O.
126139
Converts broader JavaScript input into a message instance.
127140

128141
* **toObject**(message: `Message`, options?: `ConversionOptions`): `object`
129-
Converts a message instance to a plain object for JSON or interoperability. Common options:
130-
131-
| Option | Effect |
132-
|--------|--------|
133-
| `longs: String` | Converts 64-bit values to decimal strings |
134-
| `longs: Number` | Converts 64-bit values to JS numbers (may lose precision) |
135-
| `enums: String` | Converts enum values to names |
136-
| `bytes: String` | Converts bytes to base64 strings |
137-
| `defaults: true` | Includes default values for unset fields |
138-
| `arrays: true` | Includes empty arrays for repeated fields |
139-
| `objects: true` | Includes empty objects for map fields |
140-
| `oneofs: true` | Includes virtual oneof discriminator properties |
142+
Converts a message instance to a plain object for JSON or interoperability.
141143

142144
* **encode**(message: `Message | object`, writer?: `Writer`): `Writer`
143145
Encodes a message or equivalent plain object. Call `.finish()` on the returned writer to obtain a buffer.
@@ -298,11 +300,15 @@ For protobuf descriptor interoperability, see [ext/descriptor](./ext/descriptor)
298300

299301
In [CSP](https://w3c.github.io/webappsec-csp/)-restricted environments that disallow unsafe-eval, use generated static code instead of runtime code generation.
300302

303+
## Conformance
304+
305+
protobuf.js targets full binary wire-format conformance for **Proto2**, **Proto3** and **Editions**. CI runs the official Protocol Buffers conformance suite, with logs uploaded as artifacts.
306+
301307
## Performance
302308

303-
Both protobuf.js reflection and static modes execute specialized encoder and decoder functions generated for each message type instead of a generic descriptor-walking interpreter.
309+
In both reflection and static modes, protobuf.js builds specialized encoders and decoders for each message type instead of interpreting descriptors at runtime.
304310

305-
The repository includes a small benchmark for the bundled fixture in [`bench/`](./bench/). It compares protobuf.js reflection and static code against native `JSON.stringify`/`JSON.parse` and [google-protobuf](https://www.npmjs.com/package/google-protobuf). Results depend on hardware, Node.js version, and the message shape, so they should be treated as indicative rather than absolute.
311+
The repository includes a small benchmark for the bundled fixture in [`bench/`](./bench/). It compares protobuf.js reflection and static code against native `JSON.stringify`/`JSON.parse` and [google-protobuf](https://www.npmjs.com/package/google-protobuf) (`protoc-gen-js`). Results depend on hardware, Node.js version, and the message shape, so they should be treated as indicative rather than absolute.
306312

307313
One run on an AMD Ryzen 9 9950X3D with Node.js 24.13.0 and google-protobuf 4.0.2 produced:
308314

‎tests/conformance/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
generated/
2+
out/
3+
upstream/

‎tests/conformance/check.js‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
"use strict";
2+
3+
var fs = require("fs");
4+
5+
var args = process.argv.slice(2),
6+
file = args[0],
7+
binaryThreshold = 100,
8+
report,
9+
binary,
10+
binaryPercent;
11+
12+
for (var i = 1; i < args.length; ++i) {
13+
if (args[i] === "--binary")
14+
binaryThreshold = Number(args[++i]);
15+
}
16+
17+
if (!file || !isFinite(binaryThreshold)) {
18+
console.error("usage: node tests/conformance/check.js <conformance-json> [--binary <percent>]");
19+
process.exit(1);
20+
}
21+
22+
if (!fs.existsSync(file)) {
23+
console.error("missing conformance summary: " + file);
24+
process.exit(1);
25+
}
26+
27+
report = JSON.parse(fs.readFileSync(file, "utf8"));
28+
binary = report.totals && report.totals.byFormat && report.totals.byFormat.binary;
29+
if (!binary || !binary.total) {
30+
console.error("missing Binary conformance results in " + file);
31+
process.exit(1);
32+
}
33+
34+
binaryPercent = binary.passPercent * 100;
35+
if (binaryPercent + 1e-9 < binaryThreshold) {
36+
console.error("Binary conformance below " + binaryThreshold.toFixed(2) + "%: "
37+
+ binaryPercent.toFixed(2) + "% (" + binary.passed + "/" + binary.total + ")");
38+
if (binary.failed)
39+
console.error("Binary failures: " + binary.failed);
40+
if (binary.skipped)
41+
console.error("Binary skipped: " + binary.skipped);
42+
process.exit(1);
43+
}
44+
45+
console.log("Binary conformance: " + binaryPercent.toFixed(2) + "% (" + binary.passed + "/" + binary.total + ")");

‎tests/conformance/generate.js‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
"use strict";
2+
3+
var child_process = require("child_process"),
4+
fs = require("fs"),
5+
path = require("path");
6+
7+
var rootDir = path.resolve(__dirname, "../.."),
8+
upstreamDir = process.env.PROTOBUF_UPSTREAM || path.join(__dirname, "upstream"),
9+
outputDir = path.join(__dirname, "generated"),
10+
outputFile = path.join(outputDir, "messages.js"),
11+
upstreamUnstableSchemaFile = "conformance/test_protos/test_messages_edition_unstable.proto",
12+
unstableSchemaFile = path.join(outputDir, "test_messages_edition_unstable.proto"),
13+
importRoots = [
14+
"src",
15+
"conformance",
16+
"conformance/test_protos",
17+
"editions/golden"
18+
],
19+
schemaFiles = [
20+
"conformance/conformance.proto",
21+
"src/google/protobuf/test_messages_proto2.proto",
22+
"src/google/protobuf/test_messages_proto3.proto",
23+
"conformance/test_protos/test_messages_edition2023.proto",
24+
"editions/golden/test_messages_proto2_editions.proto",
25+
"editions/golden/test_messages_proto3_editions.proto"
26+
];
27+
28+
if (!fs.existsSync(upstreamDir)) {
29+
console.error("missing upstream protobuf checkout: " + upstreamDir);
30+
process.exit(1);
31+
}
32+
33+
upstreamDir = path.resolve(upstreamDir);
34+
35+
fs.mkdirSync(outputDir, { recursive: true });
36+
37+
// Upstream v34+ includes REQUIRED EditionUnstable conformance tests even with
38+
// --maximum_edition 2024. Optionally use a local stable-edition copy because
39+
// the parser intentionally only supports released editions.
40+
if (fs.existsSync(fromUpstream(upstreamUnstableSchemaFile))) {
41+
fs.writeFileSync(
42+
unstableSchemaFile,
43+
fs.readFileSync(fromUpstream(upstreamUnstableSchemaFile), "utf8")
44+
.replace("edition = \"UNSTABLE\";", "edition = \"2024\";")
45+
);
46+
schemaFiles.push(unstableSchemaFile);
47+
}
48+
49+
runPbjs(importRoots.map(fromUpstream), schemaFiles.map(fromSchemaFile));
50+
51+
function fromUpstream(relativePath) {
52+
return path.join(upstreamDir, relativePath);
53+
}
54+
55+
function fromSchemaFile(schemaFile) {
56+
return path.isAbsolute(schemaFile) ? schemaFile : fromUpstream(schemaFile);
57+
}
58+
59+
function runPbjs(importPaths, protoFiles) {
60+
var args = [
61+
path.join(rootDir, "cli/bin/pbjs"),
62+
"-t", "static-module",
63+
"-w", "commonjs",
64+
"--dependency", "../../../minimal",
65+
"-o", outputFile
66+
];
67+
68+
importPaths.forEach(function(importPath) {
69+
args.push("-p", importPath);
70+
});
71+
Array.prototype.push.apply(args, protoFiles);
72+
73+
var result = child_process.spawnSync(process.execPath, args, {
74+
cwd: rootDir,
75+
stdio: "inherit"
76+
});
77+
78+
if (result.error)
79+
throw result.error;
80+
81+
process.exit(result.status || 0);
82+
}

0 commit comments

Comments
 (0)