Add in script to validate example YAML files against JSON schema, update README

Author: pflooky

README.md

@@ -45,7 +45,7 @@ To use this GitHub Action, you need to get a username and token from
        runs-on: ubuntu-latest
        steps:
          - name: Run integration tests
-           uses: data-catering/insta-integration@v2
+           uses: data-catering/insta-integration@v3
            with:
              data_caterer_user: ${{ secrets.DATA_CATERER_USER }}
              data_caterer_token: ${{ secrets.DATA_CATERER_TOKEN }}
@@ -129,10 +129,11 @@ The following data sources are available to generate/validate data.
 <summary>Click here</summary>
 
 | Data Source Type | Data Source                        | Support |
-| ---------------- | ---------------------------------- | ------- |
+| ---------------- |------------------------------------| ------- |
 | Cloud Storage    | AWS S3                             | ✅      |
 | Cloud Storage    | Azure Blob Storage                 | ✅      |
 | Cloud Storage    | GCP Cloud Storage                  | ✅      |
+| Database         | BigQuery                           | ✅      |
 | Database         | Cassandra                          | ✅      |
 | Database         | MySQL                              | ✅      |
 | Database         | Postgres                           | ✅      |
@@ -148,10 +149,10 @@ The following data sources are available to generate/validate data.
 | File             | Hudi                               | ❌      |
 | HTTP             | REST API                           | ✅      |
 | Messaging        | Kafka                              | ✅      |
+| Messaging        | RabbitMQ                           | ✅      |
 | Messaging        | Solace                             | ✅      |
 | Messaging        | ActiveMQ                           | ❌      |
 | Messaging        | Pulsar                             | ❌      |
-| Messaging        | RabbitMQ                           | ❌      |
 | Metadata         | Data Contract CLI                  | ✅      |
 | Metadata         | Great Expectations                 | ✅      |
 | Metadata         | Marquez                            | ✅      |
@@ -349,7 +350,11 @@ Validate the JSON Schema:
 ajv compile --spec=draft2019 -s schema/insta-integration-config-latest.json
 ```
 
-Validate a `insta-integration.yaml` file:
+#### Validate YAML file
+
+You can run `npm run validate-yaml` and it will validate all the YAML files under the `examples` directory.
+
+Otherwise, if you have a different pathway, validate via `ajv`:
 
 ```shell
 ajv validate --spec=draft2019 -s schema/insta-integration-config-latest.json -d example/postgres-to-csv.yaml

example/spring-postgres.yaml

@@ -27,12 +27,14 @@ run:
               validationIdentifier: GET/account/{accountNumber}/transactions
             fields:
               - name: httpUrl
+                type: struct
                 fields:
                   - name: url
                     static: http://localhost:8080/account/{accountNumber}/transactions
                   - name: method
                     static: GET
                   - name: pathParams
+                    type: struct
                     fields:
                       - name: accountNumber
                         options:
@@ -46,6 +48,6 @@ run:
               validationIdentifier: GET/account/{accountNumber}/transactions
             validations:
               - expr: request.method == 'GET'
-              - expr: response.timeTaken < 100
+              - expr: response.timeTakenMs < 100
               - expr: response.headers.Content-Type == 'application/json'
               - expr: response.body.account_number matches 'ACC[0-9]{8}'

insta-integration.yaml

@@ -3,8 +3,8 @@ run:
   - command: echo 'hello world'
     test:
       relationship:
-        my_parquet.account_id:
-          - my_csv.account_id
+        my_parquet||account_id:
+          - my_csv||account_id
       generation:
         parquet:
           - name: my_parquet