Skip to content

Commit 9f2d70d

Browse files
committed
Pin airbyte-mysql and fix tests
A newer version has changed its schema for the replication mode and also flat out refuses to connect to our test MySQL instance. Pin it to a known working version and update the test fixtures (seems like datetime parsing finally works now).
1 parent 266d723 commit 9f2d70d

File tree

1 file changed

+18
-14
lines changed

1 file changed

+18
-14
lines changed

test/splitgraph/ingestion/test_airbyte.py

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525

2626
class MySQLAirbyteDataSource(AirbyteDataSource):
27-
docker_image = "airbyte/source-mysql:latest"
27+
docker_image = "airbyte/source-mysql:0.6.8"
2828
airbyte_name = "airbyte-mysql"
2929

3030
credentials_schema = merge_jsonschema(
@@ -84,7 +84,11 @@ def _source(local_engine_empty, table_params=None, extra_params=None, extra_cred
8484
json_schema={
8585
"type": "object",
8686
"properties": {
87-
"discovery": {"type": "string"},
87+
"discovery": {
88+
"type": "string",
89+
"format": "date-time",
90+
"airbyte_type": "timestamp_without_timezone",
91+
},
8892
"friendly": {"type": "boolean"},
8993
"binary_data": {"type": "string", "contentEncoding": "base64"},
9094
"name": {"type": "string"},
@@ -183,7 +187,7 @@ def test_airbyte_mysql_source_introspection_end_to_end(local_engine_empty):
183187
TableColumn(
184188
ordinal=0,
185189
name="discovery",
186-
pg_type="character varying",
190+
pg_type="timestamp without time zone",
187191
is_pk=False,
188192
comment=None,
189193
),
@@ -512,13 +516,13 @@ def _assert_scd_data(repo):
512516
"_airbyte_active_row": 1,
513517
"_airbyte_emitted_at": mock.ANY,
514518
"_airbyte_end_at": None,
515-
"_airbyte_mushrooms_hashid": "882da3c55d7481c75a8f8919fc8441e1",
519+
"_airbyte_mushrooms_hashid": "876c0065527b987205cf16cf7eb2f3fc",
516520
"_airbyte_normalized_at": mock.ANY,
517521
"_airbyte_start_at": 1,
518522
"_airbyte_unique_key": "c4ca4238a0b923820dcc509a6f75849b",
519523
"_airbyte_unique_key_scd": mock.ANY,
520524
"binary_data": "YmludHN0AA==",
521-
"discovery": "2012-11-11T08:06:26.000000Z",
525+
"discovery": "2012-11-11T08:06:26+00:00",
522526
"friendly": True,
523527
"mushroom_id": 1,
524528
"name": "portobello",
@@ -529,13 +533,13 @@ def _assert_scd_data(repo):
529533
"_airbyte_active_row": 1,
530534
"_airbyte_emitted_at": mock.ANY,
531535
"_airbyte_end_at": None,
532-
"_airbyte_mushrooms_hashid": "7c835aeba7c53e3a13acb6f953820bb6",
536+
"_airbyte_mushrooms_hashid": "084b6a38ffaece6b803256a6c56b1c68",
533537
"_airbyte_normalized_at": mock.ANY,
534538
"_airbyte_start_at": 2,
535539
"_airbyte_unique_key": "c81e728d9d4c2f636f067f89cc14862c",
536540
"_airbyte_unique_key_scd": mock.ANY,
537541
"binary_data": "AAAxMjMAAA==",
538-
"discovery": "2018-03-17T08:06:26.000000Z",
542+
"discovery": "2018-03-17T08:06:26+00:00",
539543
"friendly": False,
540544
"mushroom_id": 2,
541545
"name": "deathcap",
@@ -547,7 +551,7 @@ def _assert_scd_data(repo):
547551
def _assert_normalized_data(repo, unique_key=False):
548552
expected = [
549553
{
550-
"discovery": "2012-11-11T08:06:26.000000Z",
554+
"discovery": "2012-11-11T08:06:26+00:00",
551555
"friendly": True,
552556
"binary_data": "YmludHN0AA==",
553557
"name": "portobello",
@@ -556,10 +560,10 @@ def _assert_normalized_data(repo, unique_key=False):
556560
"_airbyte_ab_id": mock.ANY,
557561
"_airbyte_emitted_at": mock.ANY,
558562
"_airbyte_normalized_at": mock.ANY,
559-
"_airbyte_mushrooms_hashid": "882da3c55d7481c75a8f8919fc8441e1",
563+
"_airbyte_mushrooms_hashid": "876c0065527b987205cf16cf7eb2f3fc",
560564
},
561565
{
562-
"discovery": "2018-03-17T08:06:26.000000Z",
566+
"discovery": "2018-03-17T08:06:26+00:00",
563567
"friendly": False,
564568
"binary_data": "AAAxMjMAAA==",
565569
"name": "deathcap",
@@ -568,7 +572,7 @@ def _assert_normalized_data(repo, unique_key=False):
568572
"_airbyte_ab_id": mock.ANY,
569573
"_airbyte_emitted_at": mock.ANY,
570574
"_airbyte_normalized_at": mock.ANY,
571-
"_airbyte_mushrooms_hashid": "7c835aeba7c53e3a13acb6f953820bb6",
575+
"_airbyte_mushrooms_hashid": "084b6a38ffaece6b803256a6c56b1c68",
572576
},
573577
]
574578

@@ -577,7 +581,7 @@ def _assert_normalized_data(repo, unique_key=False):
577581
TableColumn(
578582
ordinal=1,
579583
name="discovery",
580-
pg_type="character varying",
584+
pg_type="timestamp with time zone",
581585
is_pk=False,
582586
comment=None,
583587
),
@@ -671,7 +675,7 @@ def _assert_raw_data(repo):
671675
"_airbyte_data": {
672676
"name": "portobello",
673677
"friendly": True,
674-
"discovery": "2012-11-11T08:06:26.000000Z",
678+
"discovery": "2012-11-11T08:06:26.000000",
675679
"binary_data": "YmludHN0AA==",
676680
"mushroom_id": 1,
677681
"varbinary_data": "fwAAAQ==",
@@ -683,7 +687,7 @@ def _assert_raw_data(repo):
683687
"_airbyte_data": {
684688
"name": "deathcap",
685689
"friendly": False,
686-
"discovery": "2018-03-17T08:06:26.000000Z",
690+
"discovery": "2018-03-17T08:06:26.000000",
687691
"binary_data": "AAAxMjMAAA==",
688692
"mushroom_id": 2,
689693
"varbinary_data": "fwAAAQ==",

0 commit comments

Comments
 (0)