Skip to content

Commit 3b88322

Browse files
tolgaozentolgaozen
authored
Merge pull request #2575 from Permify/feature/race-condition
feat(integration-test): add race condition tests and update API versi…
2 parents 00bea4c + 31f5528 commit 3b88322

16 files changed

Lines changed: 500 additions & 70 deletions

File tree

‎docs/api-reference/apidocs.swagger.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"info": {
44
"title": "Permify API",
55
"description": "Permify is an open source authorization service for creating fine-grained and scalable authorization systems.",
6-
"version": "v1.4.6",
6+
"version": "v1.4.7",
77
"contact": {
88
"name": "API Support",
99
"url": "https://github.com/Permify/permify/issues",

‎docs/api-reference/openapiv2/apidocs.swagger.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"info": {
44
"title": "Permify API",
55
"description": "Permify is an open source authorization service for creating fine-grained and scalable authorization systems.",
6-
"version": "v1.4.6",
6+
"version": "v1.4.7",
77
"contact": {
88
"name": "API Support",
99
"url": "https://github.com/Permify/permify/issues",

‎integration-test/race_condition/README.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Race Condition Integration Test
2+
3+
This test validates the fix for concurrent transaction race condition in Permify.
4+
5+
## 🚀 Quick Start
6+
7+
### 1. Start Test Environment
8+
9+
```bash
10+
cd integration-test/race_condition
11+
docker compose -f docker-compose-test.yml up --build -d
12+
```
13+
14+
### 2. Run Tests
15+
16+
```bash
17+
# Simple test
18+
./test_race_condition.sh
19+
20+
# Parallel test (more aggressive)
21+
./test_race_condition_parallel.sh
22+
```
23+
24+
## 🧹 Cleanup
25+
26+
```bash
27+
docker compose -f docker-compose-test.yml down -v
28+
```
29+
30+
## 📁 Files
31+
32+
- `schema.perm` - Test schema
33+
- `test_race_condition.sh` - Simple test
34+
- `test_race_condition_parallel.sh` - Parallel test
35+
- `docker-compose-test.yml` - Test environment

‎integration-test/race_condition/docker-compose-test.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
services:
2+
postgres:
3+
image: postgres:17-alpine
4+
environment:
5+
POSTGRES_USER: pg
6+
POSTGRES_PASSWORD: pg
7+
POSTGRES_DB: permify
8+
ports: [ "5434:5432" ]
9+
10+
permify:
11+
build: ../../
12+
environment:
13+
PERMIFY_DATABASE_ENGINE: postgres
14+
PERMIFY_DATABASE_URI: postgres://pg:pg@postgres:5432/permify?sslmode=disable
15+
ports: [ "3477:3476" ]
16+
depends_on: [ postgres ]
17+
command: ["serve", "--http-enabled", "--log-level=debug"]

‎integration-test/race_condition/schema.perm‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
entity user {}
2+
3+
entity doc {
4+
relation can_read @user
5+
relation can_comment @user
6+
relation can_edit @user
7+
8+
permission read = can_read
9+
permission comment = can_comment
10+
permission edit = can_edit
11+
}

‎integration-test/race_condition/test_race_condition.sh‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Write schema
5+
curl -sS -w '\n' 'http://localhost:3477/v1/tenants/t1/schemas/write' \
6+
-H "Content-Type: application/json" \
7+
-d "$(cat "schema.perm" | jq -Rs '{schema: . }')"
8+
9+
# Grant permissions
10+
curl -sS -w '\n' 'http://localhost:3477/v1/tenants/t1/data/write' \
11+
-H "Content-Type: application/json" \
12+
-d '{
13+
"metadata": {},
14+
"tuples": [
15+
{"entity": {"type": "doc", "id": "d1"}, "relation": "can_read", "subject": {"type": "user", "id": "u1"}},
16+
{"entity": {"type": "doc", "id": "d1"}, "relation": "can_comment", "subject": {"type": "user", "id": "u1"}},
17+
{"entity": {"type": "doc", "id": "d1"}, "relation": "can_edit", "subject": {"type": "user", "id": "u1"}}
18+
]
19+
}'
20+
21+
# Define functions
22+
function check_access() {
23+
ACTION=$1
24+
25+
curl -sS -w '\n' 'http://localhost:3477/v1/tenants/t1/permissions/check' \
26+
-H "Content-Type: application/json" \
27+
-d "$(jq -Rn --arg action "$ACTION" '{
28+
metadata: {depth: 20},
29+
entity: {type: "doc", id: "d1"},
30+
permission: $action,
31+
subject: {type: "user", id: "u1"}
32+
}')"
33+
}
34+
35+
function drop_access() {
36+
RELATION=$1
37+
38+
curl -sS -w '\n' 'http://localhost:3477/v1/tenants/t1/data/delete' \
39+
-H "Content-Type: application/json" \
40+
-d "$(jq -Rn --arg relation "$RELATION" '{
41+
tuple_filter: {
42+
entity: {type: "doc", id: "d1"},
43+
relation: $relation,
44+
subject: {type: "user", "ids": ["u1"]}
45+
},
46+
attribute_filter: {}
47+
}')"
48+
}
49+
50+
echo "=== BEFORE DELETE ==="
51+
echo "READ = $(check_access 'read')"
52+
echo "COMMENT = $(check_access 'comment')"
53+
echo "EDIT = $(check_access 'edit')"
54+
55+
echo ""
56+
echo "=== CONCURRENT DELETE ==="
57+
drop_access 'can_comment' &
58+
PID1=$!
59+
drop_access 'can_edit' &
60+
PID2=$!
61+
wait $PID1
62+
wait $PID2
63+
64+
echo ""
65+
echo "=== AFTER DELETE ==="
66+
echo "READ = $(check_access 'read')"
67+
echo "COMMENT = $(check_access 'comment')"
68+
echo "EDIT = $(check_access 'edit')"
69+
70+
sleep 10
71+
72+
echo ""
73+
echo "=== AFTER 10s WAIT ==="
74+
echo "READ = $(check_access 'read')"
75+
echo "COMMENT = $(check_access 'comment')"
76+
echo "EDIT = $(check_access 'edit')"

‎integration-test/race_condition/test_race_condition_parallel.sh‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Write schema
5+
curl -sS -w '\n' 'http://localhost:3477/v1/tenants/t1/schemas/write' \
6+
-H "Content-Type: application/json" \
7+
-d "$(cat "schema.perm" | jq -Rs '{schema: . }')"
8+
9+
# Grant permissions
10+
curl -sS -w '\n' 'http://localhost:3477/v1/tenants/t1/data/write' \
11+
-H "Content-Type: application/json" \
12+
-d '{
13+
"metadata": {},
14+
"tuples": [
15+
{"entity": {"type": "doc", "id": "d1"}, "relation": "can_read", "subject": {"type": "user", "id": "u1"}},
16+
{"entity": {"type": "doc", "id": "d1"}, "relation": "can_comment", "subject": {"type": "user", "id": "u1"}},
17+
{"entity": {"type": "doc", "id": "d1"}, "relation": "can_edit", "subject": {"type": "user", "id": "u1"}}
18+
]
19+
}'
20+
21+
# Define functions
22+
function check_access() {
23+
ACTION=$1
24+
curl -sS -w '\n' 'http://localhost:3477/v1/tenants/t1/permissions/check' \
25+
-H "Content-Type: application/json" \
26+
-d "$(jq -Rn --arg action "$ACTION" '{
27+
metadata: {depth: 20},
28+
entity: {type: "doc", id: "d1"},
29+
permission: $action,
30+
subject: {type: "user", id: "u1"}
31+
}')"
32+
}
33+
34+
function drop_access() {
35+
RELATION=$1
36+
curl -sS -w '\n' 'http://localhost:3477/v1/tenants/t1/data/delete' \
37+
-H "Content-Type: application/json" \
38+
-d "$(jq -Rn --arg relation "$RELATION" '{
39+
tuple_filter: {
40+
entity: {type: "doc", id: "d1"},
41+
relation: $relation,
42+
subject: {type: "user", "ids": ["u1"]}
43+
},
44+
attribute_filter: {}
45+
}')"
46+
}
47+
48+
echo "=== BEFORE DELETE ==="
49+
echo "READ = $(check_access 'read')"
50+
echo "COMMENT = $(check_access 'comment')"
51+
echo "EDIT = $(check_access 'edit')"
52+
53+
echo ""
54+
echo "=== CONCURRENT DELETE ==="
55+
drop_access 'can_comment' &
56+
PID1=$!
57+
drop_access 'can_edit' &
58+
PID2=$!
59+
wait $PID1
60+
wait $PID2
61+
62+
echo ""
63+
echo "=== AFTER DELETE (PARALLEL CHECKS) ==="
64+
check_access 'read' &
65+
CHECK_PID1=$!
66+
check_access 'comment' &
67+
CHECK_PID2=$!
68+
check_access 'edit' &
69+
CHECK_PID3=$!
70+
wait $CHECK_PID1
71+
wait $CHECK_PID2
72+
wait $CHECK_PID3
73+
74+
echo ""
75+
echo "=== AFTER 1s WAIT (PARALLEL CHECKS) ==="
76+
sleep 1
77+
check_access 'read' &
78+
CHECK_PID4=$!
79+
check_access 'comment' &
80+
CHECK_PID5=$!
81+
check_access 'edit' &
82+
CHECK_PID6=$!
83+
wait $CHECK_PID4
84+
wait $CHECK_PID5
85+
wait $CHECK_PID6
86+
87+
echo ""
88+
echo "=== AFTER 5s WAIT (PARALLEL CHECKS) ==="
89+
sleep 5
90+
check_access 'read' &
91+
CHECK_PID7=$!
92+
check_access 'comment' &
93+
CHECK_PID8=$!
94+
check_access 'edit' &
95+
CHECK_PID9=$!
96+
wait $CHECK_PID7
97+
wait $CHECK_PID8
98+
wait $CHECK_PID9

‎internal/info.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ var Identifier = ""
2323
*/
2424
const (
2525
// Version is the last release of the Permify (e.g. v0.1.0)
26-
Version = "v1.4.6"
26+
Version = "v1.4.7"
2727
)
2828

2929
// Function to create a single line of the ASCII art with centered content and color

‎internal/storage/postgres/data_reader.go‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ type DataReader struct {
3333
// It initializes a new DataReader with a given database, a logger, and sets transaction options to be read-only with Repeatable Read isolation level.
3434
func NewDataReader(database *db.Postgres) *DataReader {
3535
return &DataReader{
36-
database: database, // Set the database to the passed in PostgreSQL instance
37-
txOptions: pgx.TxOptions{IsoLevel: pgx.ReadCommitted, AccessMode: pgx.ReadOnly}, // Set the transaction options
36+
database: database, // Set the database to the passed in PostgreSQL instance
37+
txOptions: pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}, // Set the transaction options
3838
}
3939
}
4040

@@ -57,7 +57,7 @@ func (r *DataReader) QueryRelationships(ctx context.Context, tenantID string, fi
5757
var args []interface{}
5858
builder := r.database.Builder.Select("entity_type, entity_id, relation, subject_type, subject_id, subject_relation").From(RelationTuplesTable).Where(squirrel.Eq{"tenant_id": tenantID})
5959
builder = utils.TuplesFilterQueryForSelectBuilder(builder, filter)
60-
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint)
60+
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint, st.(snapshot.Token).Snapshot)
6161

6262
if pagination.Cursor() != "" {
6363
var t database.ContinuousToken
@@ -132,7 +132,7 @@ func (r *DataReader) ReadRelationships(ctx context.Context, tenantID string, fil
132132
// Build the relationships query based on the provided filter, snapshot value, and pagination settings.
133133
builder := r.database.Builder.Select("id, entity_type, entity_id, relation, subject_type, subject_id, subject_relation").From(RelationTuplesTable).Where(squirrel.Eq{"tenant_id": tenantID})
134134
builder = utils.TuplesFilterQueryForSelectBuilder(builder, filter)
135-
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint)
135+
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint, st.(snapshot.Token).Snapshot)
136136

137137
// Apply the pagination token and limit to the query.
138138
if pagination.Token() != "" {
@@ -219,7 +219,7 @@ func (r *DataReader) QuerySingleAttribute(ctx context.Context, tenantID string,
219219
var args []interface{}
220220
builder := r.database.Builder.Select("entity_type, entity_id, attribute, value").From(AttributesTable).Where(squirrel.Eq{"tenant_id": tenantID})
221221
builder = utils.AttributesFilterQueryForSelectBuilder(builder, filter)
222-
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint)
222+
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint, st.(snapshot.Token).Snapshot)
223223

224224
// Generate the SQL query and arguments.
225225
var query string
@@ -278,7 +278,7 @@ func (r *DataReader) QueryAttributes(ctx context.Context, tenantID string, filte
278278
var args []interface{}
279279
builder := r.database.Builder.Select("entity_type, entity_id, attribute, value").From(AttributesTable).Where(squirrel.Eq{"tenant_id": tenantID})
280280
builder = utils.AttributesFilterQueryForSelectBuilder(builder, filter)
281-
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint)
281+
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint, st.(snapshot.Token).Snapshot)
282282

283283
if pagination.Cursor() != "" {
284284
var t database.ContinuousToken
@@ -366,7 +366,7 @@ func (r *DataReader) ReadAttributes(ctx context.Context, tenantID string, filter
366366
// Build the relationships query based on the provided filter, snapshot value, and pagination settings.
367367
builder := r.database.Builder.Select("id, entity_type, entity_id, attribute, value").From(AttributesTable).Where(squirrel.Eq{"tenant_id": tenantID})
368368
builder = utils.AttributesFilterQueryForSelectBuilder(builder, filter)
369-
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint)
369+
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint, st.(snapshot.Token).Snapshot)
370370

371371
// Apply the pagination token and limit to the query.
372372
if pagination.Token() != "" {
@@ -479,7 +479,7 @@ func (r *DataReader) QueryUniqueSubjectReferences(ctx context.Context, tenantID
479479
})
480480

481481
// Apply snapshot filter
482-
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint)
482+
builder = utils.SnapshotQuery(builder, st.(snapshot.Token).Value.Uint, st.(snapshot.Token).Snapshot)
483483

484484
// Apply exclusion if the list is not empty
485485
if len(excluded) > 0 {
@@ -556,11 +556,12 @@ func (r *DataReader) HeadSnapshot(ctx context.Context, tenantID string) (token.S
556556
defer span.End()
557557
// Log snapshot operation
558558
slog.DebugContext(ctx, "getting head snapshot for tenant_id", slog.String("tenant_id", tenantID))
559-
// Declare transaction ID variable
559+
// Declare transaction ID and snapshot variables
560560
var xid db.XID8
561+
var snapshotValue string
561562

562-
// Build the query to find the highest transaction ID associated with the tenant.
563-
builder := r.database.Builder.Select("id").From(TransactionsTable).Where(squirrel.Eq{"tenant_id": tenantID}).OrderBy("id DESC").Limit(1)
563+
// Build the query to find the highest transaction ID and snapshot associated with the tenant.
564+
builder := r.database.Builder.Select("id", "snapshot").From(TransactionsTable).Where(squirrel.Eq{"tenant_id": tenantID}).OrderBy("id DESC").Limit(1)
564565
query, args, err := builder.ToSql()
565566
if err != nil {
566567
return nil, utils.HandleError(ctx, span, err, base.ErrorCode_ERROR_CODE_SQL_BUILDER)
@@ -569,18 +570,18 @@ func (r *DataReader) HeadSnapshot(ctx context.Context, tenantID string) (token.S
569570
// TODO: To optimize this query, create the following index concurrently to avoid table locks:
570571
// CREATE INDEX CONCURRENTLY idx_transactions_tenant_id_id ON transactions(tenant_id, id DESC);
571572

572-
// Execute the query and retrieve the highest transaction ID.
573-
err = r.database.ReadPool.QueryRow(ctx, query, args...).Scan(&xid)
573+
// Execute the query and retrieve the highest transaction ID and snapshot.
574+
err = r.database.ReadPool.QueryRow(ctx, query, args...).Scan(&xid, &snapshotValue)
574575
if err != nil {
575576
// If no rows are found, return a snapshot token with a value of 0.
576577
if errors.Is(err, pgx.ErrNoRows) {
577-
return snapshot.Token{Value: db.XID8{Uint: 0}}, nil
578+
return snapshot.Token{Value: db.XID8{Uint: 0}, Snapshot: ""}, nil
578579
}
579580
return nil, utils.HandleError(ctx, span, err, base.ErrorCode_ERROR_CODE_SCAN)
580581
}
581582

582583
slog.DebugContext(ctx, "successfully retrieved latest snapshot token")
583584
// Return snapshot token
584585
// Return the latest snapshot token associated with the tenant.
585-
return snapshot.Token{Value: xid}, nil
586+
return snapshot.Token{Value: xid, Snapshot: snapshotValue}, nil
586587
}

0 commit comments

Comments
 (0)