Skip to content

Commit 155a56f

Browse files
omer-topalomer-topal
committed
test: update load test for deeper paths and newer schema version
1 parent cb0c9af commit 155a56f

2 files changed

Lines changed: 141 additions & 42 deletions

File tree

‎README.md‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -128,15 +128,15 @@ We conducted a load test on **Permify** using **1000 VUs (Virtual Users)** and *
128128

129129
| **Metric** | **Value / Stats** |
130130
|-------------------------------|--------------------------------------------------------------------------------|
131-
| **Total Checks** | ✅ 100.00% (75,369 out of 75,369) |
132-
| **Data Received** | 15 MB (145 kB/s) |
133-
| **Data Sent** | 21 MB (203 kB/s) |
134-
| **Dropped Iterations** | 271,664 (2,688.45/s) |
135-
| **HTTP Request Duration** | avg = 10.14ms · p(90) = 14.3ms · p(95) = 26.34ms · max = 295.29ms |
136-
| **HTTP Request Waiting Time** | avg = 9.96ms · p(90) = 14.1ms · p(95) = 26.17ms · max = 295.21ms |
137-
| **HTTP Request Failed** | ❌ 0.00% (0 out of 75,369) |
138-
| **Total HTTP Requests** | 75,369 (745.87/s) |
139-
| **Virtual Users (VUs)** | 46 avg (min = 13, max = 1000) |
131+
| **Total Checks** | ✅ 100.00% (74614 out of 74614) |
132+
| **Data Received** | 17 MB (168 kB/s) |
133+
| **Data Sent** | 27 MB (268 kB/s) |
134+
| **Dropped Iterations** | 272433 (2696.482348/s) |
135+
| **HTTP Request Duration** | avg=21.3ms min=428µs med=15.38ms max=617.85ms p(90)=45.7ms p(95)=58.99ms |
136+
| **HTTP Request Waiting Time** | avg=21.27ms min=399µs med=15.35ms max=617.83ms p(90)=45.67ms p(95)=58.96ms |
137+
| **HTTP Request Failed** | ❌ 0.00% (0 out of 74614) |
138+
| **Total HTTP Requests** | 74614 (738.51308/s) |
139+
| **Virtual Users (VUs)** | 114 avg (min=14, max=1000) |
140140

141141
📄 **[Full Performance Test Report →](/docs/performance-test/README.md)**
142142

‎docs/performance-test/README.md‎

Lines changed: 132 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,9 @@ For guideline purposes we perform load test on **Permify** with 1000 VU and 1000
1111

1212
## 1. Test Environment
1313

14-
- Permify version **1.3.0**
14+
- Permify version **1.6.9**
1515
- Google Kubernetes Engine general-purpose machines for clusters
1616
- Postgres 15 on Google Cloud SQL
17-
- pgcat for server side pooling
1817
- **[values.yaml](./values.yaml)**: This file holds the default configuration (e.g. Helm values, resource settings, etc.) used for the performance test environment.
1918

2019
## 2. Schema
@@ -48,17 +47,115 @@ entity interaction {
4847
```
4948

5049
## 3. K6 Test Script
51-
Below is the k6test.js script used to measure load on Permify by performing both write and check requests:
50+
51+
### Write Script
52+
Seeds data for the schema used in our load tests.
53+
```javascript
54+
import http from "k6/http";
55+
import { fail } from "k6";
56+
57+
export const options = { vus: 1, iterations: 1 };
58+
59+
const TENANT_ID = "<tenant-id>";
60+
const url = "<your-api-endpoint>";
61+
62+
const TOTAL_IDS = 100000;
63+
const BATCH_SIZE = 1000;
64+
65+
function writeInBatches(items, key) {
66+
for (let start = 0; start < items.length; start += BATCH_SIZE) {
67+
const chunk = items.slice(start, start + BATCH_SIZE);
68+
const batchNumber = start / BATCH_SIZE + 1;
69+
const res = http.post(
70+
`${url}/v1/tenants/${TENANT_ID}/data/write`,
71+
JSON.stringify({
72+
metadata: { schema_version: "<schema-version>" },
73+
[key]: chunk,
74+
}),
75+
{
76+
headers: { "Content-Type": "application/json" },
77+
}
78+
);
79+
80+
if (res.status < 200 || res.status >= 300) {
81+
console.error(`\nPOST /data/write ${key} batch ${batchNumber} FAILED status=${res.status}\nbody=\n${res.body}\n`);
82+
fail(`POST /data/write ${key} batch ${batchNumber} non-2xx`);
83+
}
84+
}
85+
}
86+
87+
export default function () {
88+
const tuples = [];
89+
const attributes = [];
90+
91+
for (let i = 0; i < TOTAL_IDS; i++) {
92+
const id = String(i);
93+
const nextId = String((i + 1) % TOTAL_IDS);
94+
const blockedId = String((i + 2) % TOTAL_IDS);
95+
const isPublic = i % 4 === 0;
96+
97+
tuples.push(
98+
{
99+
entity: { type: "user", id },
100+
relation: "self",
101+
subject: { type: "user", id, relation: "" },
102+
},
103+
{
104+
entity: { type: "user", id },
105+
relation: "follower",
106+
subject: { type: "user", id: nextId, relation: "" },
107+
},
108+
{
109+
entity: { type: "user", id },
110+
relation: "blocked",
111+
subject: { type: "user", id: blockedId, relation: "" },
112+
},
113+
{
114+
entity: { type: "content", id },
115+
relation: "owner",
116+
subject: { type: "user", id, relation: "" },
117+
},
118+
{
119+
entity: { type: "interaction", id },
120+
relation: "creator",
121+
subject: { type: "user", id, relation: "" },
122+
},
123+
{
124+
entity: { type: "interaction", id },
125+
relation: "parent",
126+
subject: { type: "content", id, relation: "" },
127+
}
128+
);
129+
130+
attributes.push(
131+
{
132+
entity: { type: "user", id },
133+
attribute: "is_public",
134+
value: { boolean: isPublic },
135+
},
136+
{
137+
entity: { type: "content", id },
138+
attribute: "is_public",
139+
value: { boolean: isPublic },
140+
}
141+
);
142+
}
143+
144+
writeInBatches(tuples, "tuples");
145+
writeInBatches(attributes, "attributes");
146+
147+
console.log(`Seeded ${TOTAL_IDS} users, ${TOTAL_IDS} contents, and ${TOTAL_IDS} interactions`);
148+
}
149+
```
150+
151+
### Test Script
152+
153+
Below is the k6test.js script used to measure load on Permify by performing check requests:
52154
```javascript
53155
import http from 'k6/http';
54156
import {check, sleep} from 'k6';
55157

56158
export let options = {
57-
cloud: {
58-
distribution: {
59-
distributionLabel1: {loadZone: 'amazon:de:frankfurt', percent: 100},
60-
},
61-
},
62159
scenarios: {
63160
contacts_load: {
64161
executor: 'ramping-arrival-rate',
@@ -79,26 +176,28 @@ function getRandomId() {
79176
return Math.floor(Math.random() * 100000).toString();
80177
}
81178

82-
let reuseIdProbability = 0.3;
83-
let currentId = getRandomId();
179+
let reuseIdProbability = 0.1;
180+
let currentEntityId = getRandomId();
181+
let currentSubjectId = getRandomId();
84182

85183
export default function () {
86184
let entityId, subjectId;
185+
const entityType = Math.random() < 0.5 ? 'content' : 'interaction';
87186

88187
// Decide whether to reuse the current ID for the entity
89188
if (Math.random() < reuseIdProbability) {
90-
entityId = currentId; // Reuse the existing ID
189+
entityId = currentEntityId; // Reuse the existing entity ID
91190
} else {
92-
entityId = getRandomId(); // Generate a new ID
93-
currentId = entityId; // Update current ID to the new one
191+
entityId = getRandomId(); // Generate a new entity ID
192+
currentEntityId = entityId; // Update current entity ID
94193
}
95194

96195
// Decide whether to reuse the current ID for the subject
97196
if (Math.random() < reuseIdProbability) {
98-
subjectId = currentId; // Reuse the existing ID
197+
subjectId = currentSubjectId; // Reuse the existing subject ID
99198
} else {
100-
subjectId = getRandomId(); // Generate a new ID
101-
currentId = subjectId; // Update current ID to the new one
199+
subjectId = getRandomId(); // Generate a new subject ID
200+
currentSubjectId = subjectId; // Update current subject ID
102201
}
103202

104203
const url = '<your-api-endpoint>';
@@ -109,7 +208,7 @@ export default function () {
109208
depth: 20
110209
},
111210
entity: {
112-
type: "content",
211+
type: entityType,
113212
id: entityId,
114213
},
115214
permission: 'view',
@@ -160,21 +259,21 @@ export default function () {
160259

161260
| **Metric** | **Value/Stats** |
162261
|--------------------------------|-------------------------------------------------------------------------------|
163-
| **checks** | 100.00% (75369 out of 75369) |
164-
| **data_received** | 15 MB (145 kB/s) |
165-
| **data_sent** | 21 MB (203 kB/s) |
166-
| **dropped_iterations** | 271664 (2688.447952/s) |
167-
| **http_req_blocked** | avg=78.61µs min=208ns med=537ns max=45.53ms p(90)=775ns p(95)=885ns |
168-
| **http_req_connecting** | avg=16.27µs min=0s med=0s max=15.27ms p(90)=0s p(95)=0s |
169-
| **http_req_duration** | avg=10.14ms min=1.61ms med=7.44ms max=295.29ms p(90)=14.3ms p(95)=26.34ms |
170-
| **expected_response** | avg=10.14ms min=1.61ms med=7.44ms max=295.29ms p(90)=14.3ms p(95)=26.34ms |
171-
| **http_req_failed** | 0.00% (0 out of 75369) |
172-
| **http_req_receiving** | avg=94.63µs min=15.55µs med=70.42µs max=37.13ms p(90)=168.34µs p(95)=234.35µs |
173-
| **http_req_sending** | avg=86.07µs min=25.5µs med=69.44µs max=10.25ms p(90)=121.44µs p(95)=171.32µs |
262+
| **checks** | 100.00% (74614 out of 74614) |
263+
| **data_received** | 17 MB (168 kB/s) |
264+
| **data_sent** | 27 MB (268 kB/s) |
265+
| **dropped_iterations** | 272433 (2696.482348/s) |
266+
| **http_req_blocked** | avg=5.59µs min=0s med=2µs max=2.08ms p(90)=4µs p(95)=5µs |
267+
| **http_req_connecting** | avg=2.57µs min=0s med=0s max=2.04ms p(90)=0s p(95=0s |
268+
| **http_req_duration** | avg=21.3ms min=428µs med=15.38ms max=617.85ms p(90)=45.7ms p(95)=58.99ms |
269+
| **expected_response** | avg=21.3ms min=428µs med=15.38ms max=617.85ms p(90)=45.7ms p(95)=58.99ms |
270+
| **http_req_failed** | 0.00% (0 out of 74614) |
271+
| **http_req_receiving** | avg=20.27µs min=4µs med=17µs max=2.86ms p(90)=37µs p(95)=44µs |
272+
| **http_req_sending** | avg=11.16µs min=1µs med=8µs max=2.51ms p(90)=18µs p(95)=22µs |
174273
| **http_req_tls_handshaking** | avg=59.38µs min=0s med=0s max=44.21ms p(90)=0s p(95)=0s |
175-
| **http_req_waiting** | avg=9.96ms min=1.41ms med=7.27ms max=295.21ms p(90)=14.1ms p(95)=26.17ms |
176-
| **http_reqs** | 75369 (745.86855/s) |
177-
| **iteration_duration** | avg=1.01s min=1s med=1s max=1.29s p(90)=1.01s p(95)=1.02s |
178-
| **iterations** | 75369 (745.86855/s) |
179-
| **vus** | 46 (min=13, max=1000) |
274+
| **http_req_waiting** | avg=21.27ms min=399µs med=15.35ms max=617.83ms p(90)=45.67ms p(95)=58.96ms |
275+
| **http_reqs** | 74614 (738.51308/s) |
276+
| **iteration_duration** | avg=1.02s min=1s med=1.01s max=1.61s p(90)=1.04s p(95)=1.05s |
277+
| **iterations** | 74614 (738.51308/s) |
278+
| **vus** | 114 (min=14, max=1000) |
180279
| **vus_max** | 1000 (min=50, max=1000) |

0 commit comments

Comments
 (0)