Upgrade 0.15 (#8)

Update to use data caterer 0.15.2, add in HTTP example using OpenAPI spec, update JSON schema with new options

Author: pflooky

README.md

@@ -271,7 +271,7 @@ can be found below.
 | configuration_file   | File path to configuration file                                                                                                       | `insta-integration.yaml`                 |
 | insta_infra_folder   | Folder path to insta-infra ([this repository](https://github.com/data-catering/insta-infra))                                          | `${HOME}/.insta-integration/insta-infra` |
 | base_folder          | Folder path to use for execution files                                                                                                | `${HOME}/.insta-integration`             |
-| data_caterer_version | Version of data-caterer Docker image                                                                                                  | `0.14.5`                                 |
+| data_caterer_version | Version of data-caterer Docker image                                                                                                  | `0.15.2`                                 |
 | data_caterer_user    | User for data-caterer. If you don't have one yet, [create one here](https://data.catering/latest/get-started/quick-start/#get-token)  | <empty>                                  |
 | data_caterer_token   | Token for data-caterer. If you don't have one yet, [create one here](https://data.catering/latest/get-started/quick-start/#get-token) | <empty>                                  |
 
@@ -295,7 +295,7 @@ jobs:
           configuration_file: my/custom/folder/insta-integration.yaml
           insta_infra_folder: insta-infra/folder
           base_folder: execution/folder
-          data_caterer_version: 0.14.5
+          data_caterer_version: 0.15.2
           data_caterer_user: ${{ secrets.DATA_CATERER_USER }}
           data_caterer_token: ${{ secrets.DATA_CATERER_TOKEN }}
 ```

__tests__/insta-integration.test.js

@@ -373,14 +373,14 @@ describe('extractRelationships', () => {
         currentPlan
       )
     }).toThrow(
-      'Relationship should follow pattern: <generation name>.<field name>'
+      'Relationship should follow pattern: <generation name>||<fields> or <data source>||<generation name>||<fields>, relationship=c'
     )
   })
 
   it('should throw an error when relationship is defined without generation', () => {
     testConfig = {
       relationship: {
-        'service.field': ['child.field']
+        'service||field': ['child||field']
       }
     }
     expect(() => {
@@ -395,7 +395,7 @@ describe('extractRelationships', () => {
   it('should correctly extract relationships and update currentPlan', () => {
     testConfig = {
       relationship: {
-        'parentTask.id': ['child1Task.parent_id', 'child2Task.parent_id']
+        'parentTask||id': ['child1Task||parent_id', 'child2Task||parent_id']
       },
       generation: {
         parent: [{ name: 'parentTask' }],
@@ -438,11 +438,56 @@ describe('extractRelationships', () => {
     ])
   })
 
+  it('should correctly extract relationships with 3 part format and update currentPlan', () => {
+    testConfig = {
+      relationship: {
+        'http||POST/pet||body.id': [
+          'http||GET/pet/{id}||pathParamid',
+          'http||DELETE/pet/{id}||pathParamid'
+        ]
+      },
+      generation: {
+        http: [{ name: 'parentTask' }]
+      }
+    }
+    generationTaskToServiceMapping = {
+      parentTask: 'http'
+    }
+
+    extractRelationships(
+      testConfig,
+      generationTaskToServiceMapping,
+      currentPlan
+    )
+
+    expect(currentPlan.sinkOptions.foreignKeys).toEqual([
+      {
+        source: {
+          dataSource: 'http',
+          step: 'POST/pet',
+          fields: ['body.id']
+        },
+        generate: [
+          {
+            dataSource: 'http',
+            step: 'GET/pet/{id}',
+            fields: ['pathParamid']
+          },
+          {
+            dataSource: 'http',
+            step: 'DELETE/pet/{id}',
+            fields: ['pathParamid']
+          }
+        ]
+      }
+    ])
+  })
+
   it('should handle multiple relationships', () => {
     testConfig = {
       relationship: {
-        'userTask.id': ['orderTask.user_id'],
-        'productTask.id': ['orderTask.product_id']
+        'userTask||id': ['orderTask||user_id'],
+        'productTask||id': ['orderTask||product_id']
       },
       generation: {
         user: [{ name: 'userTask' }],

__tests__/main.test.js

@@ -36,7 +36,7 @@ describe('getDataCatererVersion', () => {
   })
 
   it('returns the default data caterer version if not provided', () => {
-    expect(getDataCatererVersion('')).toBe('0.14.5')
+    expect(getDataCatererVersion('')).toBe('0.15.2')
   })
 })
 

action.yml

@@ -18,7 +18,7 @@ inputs:
     default: '/home/runner/work'
   data_caterer_version:
     description: 'Version of data-caterer Docker image'
-    default: '0.14.5'
+    default: '0.15.2'
   data_caterer_user:
     description: 'User for data-caterer'
     default: ''

badges/coverage.svg

@@ -0,0 +1,18 @@
+services:
+  - name: httpbin
+run:
+  - command: echo hello
+    test:
+      env:
+        ENABLE_GENERATE_PLAN_AND_TASKS: 'true'
+      relationship:
+        http||POST/pets||body.id:
+          - http||GET/pets/{id}||pathParamid
+          - http||DELETE/pets/{id}||pathParamid
+      generation:
+        http:
+          - options:
+              metadataSourceType: openApi
+              schemaLocation: https://raw.githubusercontent.com/data-catering/data-caterer-example/refs/heads/main/docker/mount/http/petstore.json
+            count:
+              records: 5

dist/index.js

@@ -9,8 +9,8 @@ run:
       mount:
         - ${PWD}/example/my-app/shared/generated:/opt/app/shared/generated
       relationship:
-        postgres_balance.account_number:
-          - postgres_transaction.account_number
+        postgres_balance||account_number:
+          - postgres_transaction||account_number
       generation:
         postgres:
           - name: postgres_transaction
@@ -48,6 +48,10 @@ run:
               path: /opt/app/shared/generated/balances.csv
               header: true
             validations:
+              - field: account_number
+                validation:
+                  - type: 'null'
+                    negate: true
               - expr: ISNOTNULL(account_number)
               - aggType: count
                 aggExpr: count == 1000

dist/index.js.map

@@ -10,8 +10,8 @@ run: #how to run your app/job, can run multiple, run in order
       APP_VERSION: 1.3.1
     test: #using data-caterer, generate and validate data
       relationship:
-        kafka_accounts.account_id:
-          - kafka_transactions.account_id
+        kafka_accounts||account_id:
+          - kafka_transactions||account_id
       generation:
         kafka: #name matches with service from above or could point to application (i.e. REST API)
           - name: kafka_accounts