[server] Retry rev-mismatched code-sync inserts server-side (#21319)

When multiple workspaces sync globalState concurrently, the If-Match
rev becomes stale between the client's GET and POST. The VS Code client
retries 412 responses with no recursion limit, exhausting its
100-request/5-min budget and triggering 'Settings sync is suspended'.

Retry the insert up to 3 times server-side with the current latest rev,
absorbing transient concurrency conflicts before they reach the client.

Co-authored-by: Ona <no-reply@ona.com>

Author: geropl

components/gitpod-db/src/typeorm/code-sync-resource-db.spec.db.ts

@@ -61,6 +61,30 @@ export class CodeSyncResourceDBSpec {
         expect(inserted).to.be.true;
     }
 
+    @test()
+    async getLatestRevision(): Promise<void> {
+        const kind = "machines";
+
+        // Returns "0" when no resources exist
+        let latestRev = await this.db.getLatestRevision(this.userId, kind, undefined);
+        expect(latestRev).to.equal("0");
+
+        // Returns the rev of the most recent insert
+        const rev1 = await this.db.insert(this.userId, kind, undefined, undefined, async () => {});
+        latestRev = await this.db.getLatestRevision(this.userId, kind, undefined);
+        expect(latestRev).to.equal(rev1);
+
+        // Returns the newest rev after a second insert
+        const rev2 = await this.db.insert(this.userId, kind, undefined, undefined, async () => {});
+        latestRev = await this.db.getLatestRevision(this.userId, kind, undefined);
+        expect(latestRev).to.equal(rev2);
+        expect(latestRev).not.to.equal(rev1);
+
+        // Can be used to successfully insert with the correct rev
+        const rev3 = await this.db.insert(this.userId, kind, undefined, latestRev, async () => {});
+        expect(rev3).not.to.be.undefined;
+    }
+
     @test()
     async getDeleteResources(): Promise<void> {
         const kind = "machines";

components/gitpod-db/src/typeorm/code-sync-resource-db.ts

@@ -127,6 +127,25 @@ export class CodeSyncResourceDB {
         return this.doGetResources(connection.manager, userId, kind, collection);
     }
 
+    /**
+     * Returns the rev of the most recent resource entry, or "0" if none exists.
+     * Used by the code-sync service to resolve the current rev for server-side
+     * retry of rev-mismatched inserts.
+     */
+    async getLatestRevision(userId: string, kind: ServerResource, collection: string | undefined): Promise<string> {
+        const connection = await this.typeORM.getConnection();
+        const latest = await connection.manager
+            .createQueryBuilder(DBCodeSyncResource, "resource")
+            .where("resource.userId = :userId AND resource.kind = :kind AND resource.collection = :collection", {
+                userId,
+                kind,
+                collection: collection || uuid.NIL,
+            })
+            .orderBy("resource.created", "DESC")
+            .getOne();
+        return latest?.rev ?? "0";
+    }
+
     async deleteSettingsSyncResources(userId: string, doDelete: () => Promise<void>): Promise<void> {
         const connection = await this.typeORM.getConnection();
         await connection.transaction(async (manager) => {

components/server/src/code-sync/code-sync-service.ts

@@ -35,6 +35,11 @@ import { SubjectId } from "../auth/subject-id";
 const defaultRevLimit = 20;
 // It should keep it aligned with client_max_body_size for /code-sync location.
 const defaultContentLimit = "1Mb";
+// Max server-side retries when a rev mismatch (412) occurs during insert.
+// Absorbs concurrency conflicts from multiple workspaces writing the same resource,
+// preventing the VS Code client from entering an unbounded retry loop that exhausts
+// its 100-request/5-min budget and triggers "Settings sync is suspended" banners.
+const maxInsertRetries = 3;
 export type CodeSyncConfig = Partial<{
     revLimit: number;
     contentLimit: number;
@@ -330,46 +335,55 @@ export class CodeSyncService {
         const isEditSessionsResource = resourceKey === "editSessions";
         const userId = req.user!.id;
         const contentType = req.headers["content-type"] || "*/*";
-        const newRev = await this.db.insert(
-            userId,
-            resourceKey,
-            collection,
-            latestRev,
-            async (rev, oldRevs) => {
-                const request = new UploadUrlRequest();
-                request.setOwnerId(userId);
-                request.setName(toObjectName(resourceKey, rev, collection));
-                request.setContentType(contentType);
-                const blobsClient = this.blobsProvider.getDefault();
-                const urlResponse = await util.promisify<UploadUrlRequest, UploadUrlResponse>(
-                    blobsClient.uploadUrl.bind(blobsClient),
-                )(request);
-                const url = urlResponse.getUrl();
-                const content = req.body as string;
-                const response = await fetch(url, {
-                    timeout: 10000,
-                    method: "PUT",
-                    body: content,
-                    headers: {
-                        "content-length": req.headers["content-length"] || String(content.length),
-                        "content-type": contentType,
-                    },
-                });
-                if (response.status !== 200) {
-                    throw new Error(
-                        `code sync: blob service: upload failed with ${response.status} ${response.statusText}`,
-                    );
-                }
 
-                if (oldRevs.length) {
-                    // Asynchonously delete old revs from storage
-                    Promise.allSettled(
-                        oldRevs.map((rev) => this.doDeleteResource(userId, resourceKey, rev, collection)),
-                    ).catch(() => {});
-                }
-            },
-            { revLimit, overwrite: !isEditSessionsResource },
-        );
+        const doInsert = async (rev: string, oldRevs: string[]) => {
+            const request = new UploadUrlRequest();
+            request.setOwnerId(userId);
+            request.setName(toObjectName(resourceKey, rev, collection));
+            request.setContentType(contentType);
+            const blobsClient = this.blobsProvider.getDefault();
+            const urlResponse = await util.promisify<UploadUrlRequest, UploadUrlResponse>(
+                blobsClient.uploadUrl.bind(blobsClient),
+            )(request);
+            const url = urlResponse.getUrl();
+            const content = req.body as string;
+            const response = await fetch(url, {
+                timeout: 10000,
+                method: "PUT",
+                body: content,
+                headers: {
+                    "content-length": req.headers["content-length"] || String(content.length),
+                    "content-type": contentType,
+                },
+            });
+            if (response.status !== 200) {
+                throw new Error(
+                    `code sync: blob service: upload failed with ${response.status} ${response.statusText}`,
+                );
+            }
+
+            if (oldRevs.length) {
+                // Asynchronously delete old revs from storage
+                Promise.allSettled(
+                    oldRevs.map((rev) => this.doDeleteResource(userId, resourceKey, rev, collection)),
+                ).catch(() => {});
+            }
+        };
+
+        const insertOptions = { revLimit, overwrite: !isEditSessionsResource };
+
+        // Try the insert with the client-provided rev first.
+        // On rev mismatch, retry with the server's current latest rev. This absorbs
+        // concurrency conflicts (e.g. multiple workspaces syncing globalState) server-side,
+        // preventing the VS Code client from entering an unbounded retry loop that
+        // exhausts its request budget and triggers "Settings sync is suspended" banners.
+        let newRev = await this.db.insert(userId, resourceKey, collection, latestRev, doInsert, insertOptions);
+        if (!newRev && !isEditSessionsResource && latestRev) {
+            for (let attempt = 0; attempt < maxInsertRetries && !newRev; attempt++) {
+                const currentLatest = await this.db.getLatestRevision(userId, resourceKey, collection);
+                newRev = await this.db.insert(userId, resourceKey, collection, currentLatest, doInsert, insertOptions);
+            }
+        }
 
         if (!newRev) {
             res.sendStatus(isEditSessionsResource ? 400 : 412);