Merge branch 'main' into jme/LFXV2-1371

Author: bramwelt

charts/lfx-v1-sync-helper/templates/app-deployment.yaml

@@ -61,6 +61,10 @@ spec:
           ports:
             - containerPort: 8080
               name: web
+          {{- with .Values.app.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           livenessProbe:
             httpGet:
               path: /livez

charts/lfx-v1-sync-helper/values.yaml

@@ -205,6 +205,15 @@ app:
       # key in the secret which contains the signing certificate
       privateKeyKey: signer.pem
 
+  # resources is the resource configuration for the app container
+  resources:
+    requests:
+      memory: "32Mi"
+      cpu: "10m"
+    limits:
+      memory: "256Mi"
+      cpu: "500m"
+
   # auth0 is the configuration for Auth0 authentication for v1 API calls
   auth0:
     # secret contains the configuration for Auth0 authentication
@@ -252,7 +261,7 @@ dynamodbStreamConsumer:
     # DYNAMODB_TABLES is a comma-separated list of DynamoDB table names to consume.
     # Defaults to the full set of tables used by the tap-dynamodb Meltano extractor.
     DYNAMODB_TABLES:
-      value: "itx-poll,itx-poll-vote,itx-surveys,itx-survey-responses,surveymonkey-surveys,itx-zoom-meetings-mappings-v2,itx-zoom-meetings-v2,itx-zoom-past-meetings-mappings,itx-zoom-past-meetings,itx-zoom-past-meetings-attendees,itx-zoom-past-meetings-invitees,itx-zoom-past-meetings-recordings,itx-zoom-past-meetings-summaries,itx-zoom-meetings-registrants-v2,itx-zoom-meetings-invite-responses-v2,itx-zoom-meetings-attachments-v2,itx-zoom-past-meetings-attachments,itx-groupsio-v2-service,itx-groupsio-v2-subgroup,itx-groupsio-v2-member"
+      value: "itx-poll,itx-poll-vote,itx-surveys,itx-survey-responses,surveymonkey-surveys,itx-zoom-meetings-mappings-v2,itx-zoom-meetings-v2,itx-zoom-past-meetings-mappings,itx-zoom-past-meetings,itx-zoom-past-meetings-attendees,itx-zoom-past-meetings-invitees,itx-zoom-past-meetings-recordings,itx-zoom-past-meetings-summaries,itx-zoom-meetings-registrants-v2,itx-zoom-meetings-invite-responses-v2,itx-zoom-meetings-attachments-v2,itx-zoom-past-meetings-attachments,itx-groupsio-v2-service,itx-groupsio-v2-subgroup,itx-groupsio-v2-member,itx-groupsio-v2-artifact"
     # START_FROM_LATEST controls the iterator start position for new shards with no checkpoint.
     # Set to "true" to only receive new records; "false" (default) replays all available records.
     START_FROM_LATEST:

cmd/lfx-v1-sync-helper/handlers.go

@@ -157,7 +157,8 @@ func handleKVPut(ctx context.Context, entry jetstream.KeyValueEntry) bool {
 		return false
 	case "itx-groupsio-v2-service",
 		"itx-groupsio-v2-subgroup",
-		"itx-groupsio-v2-member":
+		"itx-groupsio-v2-member",
+		"itx-groupsio-v2-artifact":
 		// Groups.io records are processed by lfx-v2-mailing-list-service eventing processor.
 		logger.With("key", key).DebugContext(ctx, "groupsio record updated, processed by lfx-v2-mailing-list-service")
 		return false
@@ -271,7 +272,8 @@ func handleResourceDelete(ctx context.Context, key string, v1Principal string, v
 		return false
 	case "itx-groupsio-v2-service",
 		"itx-groupsio-v2-subgroup",
-		"itx-groupsio-v2-member":
+		"itx-groupsio-v2-member",
+		"itx-groupsio-v2-artifact":
 		// Groups.io records are processed by lfx-v2-mailing-list-service eventing processor.
 		logger.With("key", key).DebugContext(ctx, "groupsio record deleted, processed by lfx-v2-mailing-list-service")
 		return false

manifests/dynamodb-list-tables-job.yaml

@@ -68,7 +68,7 @@ spec:
               # Check specific ITX tables we need access to
               echo ""
               echo "=== Checking specific ITX tables ==="
-              for table in "itx-poll" "itx-poll-vote" "itx-surveys" "itx-survey-responses" "surveymonkey-surveys" "itx-zoom-meetings-mappings-v2" "itx-zoom-meetings-v2" "itx-zoom-past-meetings-mappings" "itx-zoom-past-meetings" "itx-zoom-past-meetings-attendees" "itx-zoom-past-meetings-invitees" "itx-zoom-past-meetings-recordings" "itx-zoom-past-meetings-summaries" "itx-zoom-meetings-registrants-v2" "itx-zoom-meetings-invite-responses-v2" "itx-zoom-meetings-attachments-v2" "itx-zoom-past-meetings-attachments" "itx-groupsio-v2-service" "itx-groupsio-v2-subgroup" "itx-groupsio-v2-member"; do
+              for table in "itx-poll" "itx-poll-vote" "itx-surveys" "itx-survey-responses" "surveymonkey-surveys" "itx-zoom-meetings-mappings-v2" "itx-zoom-meetings-v2" "itx-zoom-past-meetings-mappings" "itx-zoom-past-meetings" "itx-zoom-past-meetings-attendees" "itx-zoom-past-meetings-invitees" "itx-zoom-past-meetings-recordings" "itx-zoom-past-meetings-summaries" "itx-zoom-meetings-registrants-v2" "itx-zoom-meetings-invite-responses-v2" "itx-zoom-meetings-attachments-v2" "itx-zoom-past-meetings-attachments" "itx-groupsio-v2-service" "itx-groupsio-v2-subgroup" "itx-groupsio-v2-member" "itx-groupsio-v2-artifact"; do
                 echo "Checking table: $table"
                 aws dynamodb describe-table --table-name "$table" --region ${AWS_DEFAULT_REGION} --query 'Table.{TableName:TableName,Status:TableStatus,ItemCount:ItemCount}' --output table || echo "Failed to access table: $table"
               done

meltano/meltano.yml

@@ -80,6 +80,7 @@ plugins:
           - itx-groupsio-v2-service
           - itx-groupsio-v2-subgroup
           - itx-groupsio-v2-member
+          - itx-groupsio-v2-artifact
         metadata:
           "*":
             # Setting the replication key allows conditional loads for
@@ -115,6 +116,8 @@ plugins:
             replication-key: last_modified_at
           itx-groupsio-v2-member:
             replication-key: last_modified_at
+          itx-groupsio-v2-artifact:
+            replication-key: last_modified_at
         use_aws_env_vars: true
       schema:
         # Override auto-detected schema for alternate columns and datatypes that