hotfix: remove external id check for now, it's too expensive (hatchet-dev#4048)

Author: mrkaye97

pkg/repository/olap.go

@@ -3673,31 +3673,31 @@ func (p *OLAPRepositoryImpl) processSinglePartition(ctx context.Context, process
 		return nil
 	}
 
-	connStatementTimeout := 5 * 60 * 1000 // 5 minutes
+	// connStatementTimeout := 5 * 60 * 1000 // 5 minutes
 
-	conn, release, err := sqlchelpers.AcquireConnectionWithStatementTimeout(ctx, p.pool, p.l, connStatementTimeout)
+	// conn, release, err := sqlchelpers.AcquireConnectionWithStatementTimeout(ctx, p.pool, p.l, connStatementTimeout)
 
-	if err != nil {
-		return fmt.Errorf("failed to acquire connection with statement timeout: %w", err)
-	}
+	// if err != nil {
+	// 	return fmt.Errorf("failed to acquire connection with statement timeout: %w", err)
+	// }
 
-	defer release()
+	// defer release()
 
-	duplicatedExternalIds, err := p.ValidateNoDuplicateOLAPExternalIds(ctx, conn, partitionDate)
+	// duplicatedExternalIds, err := p.ValidateNoDuplicateOLAPExternalIds(ctx, conn, partitionDate)
 
-	if err != nil {
-		return fmt.Errorf("failed to validate no duplicate external ids: %w", err)
-	}
+	// if err != nil {
+	// 	return fmt.Errorf("failed to validate no duplicate external ids: %w", err)
+	// }
 
-	if len(duplicatedExternalIds) > 0 {
-		var duplicatedIds []string
+	// if len(duplicatedExternalIds) > 0 {
+	// 	var duplicatedIds []string
 
-		for _, row := range duplicatedExternalIds {
-			duplicatedIds = append(duplicatedIds, row.ExternalId.String())
-		}
+	// 	for _, row := range duplicatedExternalIds {
+	// 		duplicatedIds = append(duplicatedIds, row.ExternalId.String())
+	// 	}
 
-		return fmt.Errorf("found duplicate external ids in partition %s. Sampled ids: %s", partitionDate.String(), strings.Join(duplicatedIds, ", "))
-	}
+	// 	return fmt.Errorf("found duplicate external ids in partition %s. Sampled ids: %s", partitionDate.String(), strings.Join(duplicatedIds, ", "))
+	// }
 
 	lastExternalId := jobMeta.LastExternalId
 

pkg/repository/payloadstore.go

@@ -353,6 +353,9 @@ func (p *payloadStoreRepositoryImpl) retrieve(ctx context.Context, tx sqlcv1.DBT
 			key := ExternalPayloadLocationKey(payload.ExternalLocationKey.String)
 			var retrieveFromExternalOpt RetrieveFromExternalOpts
 
+			// todo: this doesn't actually make sense, since
+			// we'd never write the index file into the payload location directly
+			// so we can remove this if block
 			if strings.HasSuffix(string(key), ".index") {
 				retrieveFromExternalOpt = RetrieveFromExternalOpts{
 					Method: RetrieveFromExternalByIndexFile,
@@ -854,31 +857,31 @@ func (p *payloadStoreRepositoryImpl) processSinglePartition(ctx context.Context,
 		return nil
 	}
 
-	connStatementTimeout := 5 * 60 * 1000 // 5 minutes
+	// connStatementTimeout := 5 * 60 * 1000 // 5 minutes
 
-	conn, release, err := sqlchelpers.AcquireConnectionWithStatementTimeout(ctx, p.pool, p.l, connStatementTimeout)
+	// conn, release, err := sqlchelpers.AcquireConnectionWithStatementTimeout(ctx, p.pool, p.l, connStatementTimeout)
 
-	if err != nil {
-		return fmt.Errorf("failed to acquire connection with statement timeout: %w", err)
-	}
+	// if err != nil {
+	// 	return fmt.Errorf("failed to acquire connection with statement timeout: %w", err)
+	// }
 
-	defer release()
+	// defer release()
 
-	duplicatedExternalIds, err := p.ValidateNoDuplicateExternalIds(ctx, conn, partitionDate)
+	// duplicatedExternalIds, err := p.ValidateNoDuplicateExternalIds(ctx, conn, partitionDate)
 
-	if err != nil {
-		return fmt.Errorf("failed to validate no duplicate external ids: %w", err)
-	}
+	// if err != nil {
+	// 	return fmt.Errorf("failed to validate no duplicate external ids: %w", err)
+	// }
 
-	if len(duplicatedExternalIds) > 0 {
-		var duplicatedIds []string
+	// if len(duplicatedExternalIds) > 0 {
+	// 	var duplicatedIds []string
 
-		for _, row := range duplicatedExternalIds {
-			duplicatedIds = append(duplicatedIds, row.ExternalId.String())
-		}
+	// 	for _, row := range duplicatedExternalIds {
+	// 		duplicatedIds = append(duplicatedIds, row.ExternalId.String())
+	// 	}
 
-		return fmt.Errorf("found duplicate external ids in partition %s. Sampled ids: %s", partitionDate.String(), strings.Join(duplicatedIds, ", "))
-	}
+	// 	return fmt.Errorf("found duplicate external ids in partition %s. Sampled ids: %s", partitionDate.String(), strings.Join(duplicatedIds, ", "))
+	// }
 
 	lastExternalId := jobMeta.LastExternalId