fix(adoption-insights): partition overlap error and auto correct the invalid partitions (#790)

* fix partition overlap error and auto correct the invalid partitions

* add changeset

Author: karthikjeeyar

workspaces/adoption-insights/.changeset/tough-paws-invent.md

@@ -0,0 +1,5 @@
+---
+'@red-hat-developer-hub/backstage-plugin-adoption-insights-backend': patch
+---
+
+Fix partition overlap error

workspaces/adoption-insights/plugins/adoption-insights-backend/src/database/partition.test.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { createPartition } from './partition';
+import { Knex } from 'knex';
+
+const mockRaw = jest.fn();
+const knex = {
+  schema: { raw: mockRaw },
+} as unknown as Knex;
+
+beforeEach(() => {
+  jest.clearAllMocks();
+});
+
+describe('createPartition', () => {
+  it('should create a partition without errors', async () => {
+    mockRaw.mockResolvedValueOnce(undefined);
+
+    await createPartition(knex, 2025, 5);
+
+    expect(mockRaw).toHaveBeenCalledWith(
+      expect.stringContaining(`CREATE TABLE IF NOT EXISTS events_2025_05`),
+    );
+  });
+
+  it('should handle overlapping partition error and retry', async () => {
+    const overlapError = new Error(
+      'partition "events_2025_05" would overlap partition "events_2025_04"',
+    );
+
+    mockRaw
+      .mockRejectedValueOnce(overlapError) // create fails
+      .mockResolvedValueOnce(undefined) // drop overlap partition
+      .mockResolvedValueOnce(undefined) // recreate dropped partition
+      .mockResolvedValueOnce(undefined); // retry create current partition
+
+    await createPartition(knex, 2025, 5);
+
+    expect(mockRaw).toHaveBeenCalledWith(
+      expect.stringContaining(`DROP TABLE IF EXISTS events_2025_04 CASCADE`),
+    );
+    expect(mockRaw).toHaveBeenCalledWith(
+      expect.stringContaining(`CREATE TABLE IF NOT EXISTS events_2025_04`),
+    );
+    expect(mockRaw).toHaveBeenCalledWith(
+      expect.stringContaining(`CREATE TABLE IF NOT EXISTS events_2025_05`),
+    );
+  });
+
+  it('should handle max retry when recreating dropped partition', async () => {
+    const overlapError = new Error(
+      'partition "events_2025_05" would overlap partition "events_2025_04"',
+    );
+
+    mockRaw
+      .mockRejectedValueOnce(overlapError) // create fails for events_2025_05
+      .mockResolvedValueOnce(undefined) // drop overlap partition events_2025_04
+      .mockRejectedValueOnce(overlapError) // create fails again for events_2025_04
+      .mockResolvedValueOnce(undefined); // retry create current partition -  events_2025_05
+
+    await expect(createPartition(knex, 2025, 5)).rejects.toThrow(
+      'Exceeded max retries for partition 2025_4',
+    );
+  });
+
+  it('should not retry for non-overlap errors', async () => {
+    const otherError = new Error('some other SQL error');
+    mockRaw.mockRejectedValueOnce(otherError);
+
+    await expect(createPartition(knex, 2025, 6)).rejects.toThrow(
+      'some other SQL error',
+    );
+
+    expect(mockRaw).toHaveBeenCalledTimes(1);
+  });
+});

workspaces/adoption-insights/plugins/adoption-insights-backend/src/database/partition.ts

@@ -18,26 +18,61 @@ import {
   LoggerService,
   SchedulerService,
 } from '@backstage/backend-plugin-api/index';
+import {
+  extractOverlappingPartition,
+  isPartitionOverlapError,
+  parsePartitionDate,
+} from '../utils/partition';
+
+type AttemptTracker = Map<string, number>;
 
 export const createPartition = async (
   knex: Knex,
   year: number,
   month: number,
+  attempts: AttemptTracker = new Map(),
+  maxRetries = 1,
 ) => {
-  const startDate = `${year}-${month.toString().padStart(2, '0')}-01`;
-  const nextMonth = new Date(year, month, 1);
-  nextMonth.setMonth(nextMonth.getMonth() + 1);
-  const endDate = `${nextMonth.getFullYear()}-${(nextMonth.getMonth() + 1)
-    .toString()
-    .padStart(2, '0')}-01`;
+  const start = new Date(year, month - 1, 1);
+  const end = new Date(year, month, 1);
+
+  const startDate = start.toISOString().slice(0, 10);
+  const endDate = end.toISOString().slice(0, 10);
+
+  const partitionName = `events_${year}_${month.toString().padStart(2, '0')}`;
+  const key = `${year}_${month}`;
 
-  const partitionName = `events_${year}_${month}`;
+  // track max attempts
+  const currentAttempt = attempts.get(key) ?? 0;
+  if (currentAttempt > maxRetries) {
+    throw new Error(`Exceeded max retries for partition ${key}`);
+  }
+  attempts.set(key, currentAttempt + 1);
 
-  await knex.schema.raw(`
+  try {
+    await knex.schema.raw(`
       CREATE TABLE IF NOT EXISTS ${partitionName} 
       PARTITION OF events
       FOR VALUES FROM ('${startDate}') TO ('${endDate}');
     `);
+  } catch (error) {
+    if (isPartitionOverlapError(error)) {
+      const overlappingPartition = extractOverlappingPartition(error.message);
+      const { year: y, month: m } = parsePartitionDate(overlappingPartition);
+
+      await knex.schema.raw(
+        `DROP TABLE IF EXISTS ${overlappingPartition} CASCADE`,
+      );
+
+      // Recreate the dropped overlapping partition
+      await createPartition(knex, y, m, attempts, maxRetries);
+
+      // Retry the current one
+      await createPartition(knex, year, month, attempts, maxRetries);
+    } else {
+      throw error;
+    }
+  }
 };
 
 export const schedulePartition = async (

workspaces/adoption-insights/plugins/adoption-insights-backend/src/utils/partition.test.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  extractOverlappingPartition,
+  parsePartitionDate,
+  isPartitionOverlapError,
+} from './partition';
+
+describe('extractOverlappingPartition', () => {
+  it('should extract the overlapping partition name', () => {
+    const msg =
+      'partition "events_2025_05" would overlap partition "events_2025_04"';
+    expect(extractOverlappingPartition(msg)).toBe('events_2025_04');
+  });
+
+  it('should return empty string if pattern does not match', () => {
+    const msg = 'some other error message';
+    expect(extractOverlappingPartition(msg)).toBe('');
+  });
+});
+
+describe('parsePartitionDate', () => {
+  it('should parse valid partition name', () => {
+    const input = 'events_2025_04';
+    expect(parsePartitionDate(input)).toEqual({ year: 2025, month: 4 });
+  });
+
+  it('should parse single-digit month correctly', () => {
+    const input = 'events_2025_9';
+    expect(parsePartitionDate(input)).toEqual({ year: 2025, month: 9 });
+  });
+
+  it('should throw error for invalid format', () => {
+    const input = 'invalid_partition_name';
+    expect(() => parsePartitionDate(input)).toThrow(
+      'Cannot parse partition name: invalid_partition_name',
+    );
+  });
+});
+
+describe('isPartitionOverlapError', () => {
+  it('should return true for overlap error', () => {
+    const err = {
+      message:
+        'partition "events_2025_05" would overlap partition "events_2025_04"',
+    };
+    expect(isPartitionOverlapError(err)).toBe(true);
+  });
+
+  it('should return false for non-overlap error', () => {
+    const err = { message: 'some other error' };
+    expect(isPartitionOverlapError(err)).toBe(false);
+  });
+
+  it('should return false if message is missing', () => {
+    const err = { msg: 'missing message property' };
+    expect(isPartitionOverlapError(err)).toBe(false);
+  });
+
+  it('should return false for null error', () => {
+    expect(isPartitionOverlapError(null)).toBe(false);
+  });
+
+  it('should return false for non-object error', () => {
+    expect(isPartitionOverlapError('error string')).toBe(false);
+  });
+});

workspaces/adoption-insights/plugins/adoption-insights-backend/src/utils/partition.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export const extractOverlappingPartition = (message: string): string => {
+  const match = message.match(/would overlap partition "(.*?)"$/);
+  return match?.[1] || '';
+};
+
+export const parsePartitionDate = (
+  name: string,
+): { year: number; month: number } => {
+  const match = name.match(/events_(\d{4})_(\d{1,2})/);
+  if (!match) throw new Error(`Cannot parse partition name: ${name}`);
+  return { year: parseInt(match[1], 10), month: parseInt(match[2], 10) };
+};
+
+export const isPartitionOverlapError = (error: unknown): boolean => {
+  return (
+    error !== null &&
+    typeof (error as any).message === 'string' &&
+    (error as any).message.includes('would overlap partition')
+  );
+};