Skip to content

Commit 45f2c7c

Browse files
amotlamotl
committed
fixes
1 parent 4fc5127 commit 45f2c7c

2 files changed

Lines changed: 30 additions & 14 deletions

File tree

‎Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ deps-ci:
2222
uv pip install --system -r requirements-dev.txt
2323

2424
test-ci:
25-
set -a; source test.env; set +a; TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto -x -rP -vv --tb=short --durations=10 --cov=ingestr --no-cov-on-fail
25+
set -a; source test.env; set +a; TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto -x -rP -vv --tb=short --durations=10 --cov=ingestr --no-cov-on-fail -k "not cratedb"
26+
set -a; source test.env; set +a; TESTCONTAINERS_RYUK_DISABLED=true pytest -rP -vv --tb=short --durations=10 --cov=ingestr --no-cov-on-fail -k "cratedb"
2627

2728
test : venv lock-deps
2829
. venv/bin/activate; $(MAKE) test-ci

‎ingestr/main_test.py‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -680,12 +680,19 @@ def get_uri_read(url: str, image: DockerImage) -> str:
680680
)
681681
@pytest.mark.parametrize("source", list(SOURCES.values()), ids=list(SOURCES.keys()))
682682
def test_create_replace(source, dest):
683+
if isinstance(source.container, SqlServerContainer) and isinstance(
684+
dest, CrateDbDockerImage
685+
):
686+
pytest.skip(
687+
"CrateDB type mapping does not support `DATE` yet, "
688+
"see https://github.com/crate-workbench/ingestr/issues/4"
689+
)
683690
with ThreadPoolExecutor() as executor:
684691
source_future = executor.submit(source.start)
685692
dest_future = executor.submit(dest.start)
686693
source_uri = source_future.result()
687694
dest_uri = dest_future.result()
688-
dest_uri_read = get_uri_read(dest_uri, dest)
695+
dest_uri_read = get_uri_read(dest_uri, dest)
689696
db_to_db_create_replace(source_uri, dest_uri, dest_uri_read)
690697
source.stop()
691698
dest.stop()
@@ -696,7 +703,7 @@ def test_create_replace(source, dest):
696703
)
697704
@pytest.mark.parametrize("source", list(SOURCES.values()), ids=list(SOURCES.keys()))
698705
def test_append(source, dest):
699-
if isinstance(dest, CrateDbDockerImage) and (source in [pgDocker, mysqlDocker]):
706+
if isinstance(dest, CrateDbDockerImage):
700707
pytest.skip(
701708
"CrateDB support for 'append' strategy pending, "
702709
"see https://github.com/crate-workbench/ingestr/issues/6"
@@ -706,7 +713,7 @@ def test_append(source, dest):
706713
dest_future = executor.submit(dest.start)
707714
source_uri = source_future.result()
708715
dest_uri = dest_future.result()
709-
dest_uri_read = get_uri_read(dest_uri, dest)
716+
dest_uri_read = get_uri_read(dest_uri, dest)
710717
db_to_db_append(source_uri, dest_uri, dest_uri_read)
711718
source.stop()
712719
dest.stop()
@@ -727,7 +734,7 @@ def test_merge_with_primary_key(source, dest):
727734
dest_future = executor.submit(dest.start)
728735
source_uri = source_future.result()
729736
dest_uri = dest_future.result()
730-
dest_uri_read = get_uri_read(dest_uri, dest)
737+
dest_uri_read = get_uri_read(dest_uri, dest)
731738
db_to_db_merge_with_primary_key(source_uri, dest_uri, dest_uri_read)
732739
source.stop()
733740
dest.stop()
@@ -748,7 +755,7 @@ def test_delete_insert_without_primary_key(source, dest):
748755
dest_future = executor.submit(dest.start)
749756
source_uri = source_future.result()
750757
dest_uri = dest_future.result()
751-
dest_uri_read = get_uri_read(dest_uri, dest)
758+
dest_uri_read = get_uri_read(dest_uri, dest)
752759
db_to_db_delete_insert_without_primary_key(source_uri, dest_uri, dest_uri_read)
753760
source.stop()
754761
dest.stop()
@@ -769,7 +776,7 @@ def test_delete_insert_with_time_range(source, dest):
769776
dest_future = executor.submit(dest.start)
770777
source_uri = source_future.result()
771778
dest_uri = dest_future.result()
772-
dest_uri_read = get_uri_read(dest_uri, dest)
779+
dest_uri_read = get_uri_read(dest_uri, dest)
773780
db_to_db_delete_insert_with_timerange(source_uri, dest_uri, dest_uri_read)
774781
source.stop()
775782
dest.stop()
@@ -1531,7 +1538,6 @@ def test_kafka_to_db(dest):
15311538
)
15321539
dest_uri = dest_future.result()
15331540
kafka = source_future.result()
1534-
dest_uri_read = get_uri_read(dest_uri, dest)
15351541

15361542
# kafka = KafkaContainer("confluentinc/cp-kafka:7.6.0").start(timeout=60)
15371543

@@ -1556,6 +1562,7 @@ def run():
15561562
assert res.exit_code == 0
15571563

15581564
def get_output_table():
1565+
dest_uri_read = get_uri_read(dest_uri, dest)
15591566
dest_engine = sqlalchemy.create_engine(dest_uri_read)
15601567
# CrateDB needs an explicit flush to make data available for reads immediately.
15611568
if dest_engine.dialect.name == "crate":
@@ -1625,7 +1632,7 @@ def get_output_table():
16251632
def test_arrow_mmap_to_db_create_replace(dest):
16261633
if isinstance(dest, CrateDbDockerImage):
16271634
pytest.skip(
1628-
"CrateDB is not supported for this test, "
1635+
"CrateDB type mapping does not support `DATE` yet, "
16291636
"see https://github.com/crate-workbench/ingestr/issues/4"
16301637
)
16311638

@@ -1741,7 +1748,7 @@ def run_command(df: pd.DataFrame, incremental_key: Optional[str] = None):
17411748
pytest.skip("clickhouse is not supported for this test")
17421749
if dest_uri.startswith("cratedb://"):
17431750
pytest.skip(
1744-
"CrateDB is not supported for this test, "
1751+
"CrateDB type mapping does not support `DATE` yet, "
17451752
"see https://github.com/crate-workbench/ingestr/issues/4"
17461753
)
17471754

@@ -1851,7 +1858,7 @@ def build_datetime(ds: str):
18511858
def test_arrow_mmap_to_db_merge_without_incremental(dest):
18521859
if isinstance(dest, CrateDbDockerImage):
18531860
pytest.skip(
1854-
"CrateDB is not supported for this test, "
1861+
"CrateDB type mapping does not support `DATE` yet, "
18551862
"see https://github.com/crate-workbench/ingestr/issues/4"
18561863
)
18571864
schema = f"testschema_arrow_mmap_{get_random_string(5)}"
@@ -2457,6 +2464,14 @@ def replace(
24572464
dest_connection_url,
24582465
dest_connection_url_read: str,
24592466
):
2467+
if source_connection_url.startswith(
2468+
"sqlserver://"
2469+
) and dest_connection_url.startswith("cratedb://"):
2470+
pytest.skip(
2471+
"CrateDB type mapping does not support `DATE` yet, "
2472+
"see https://github.com/crate-workbench/ingestr/issues/4"
2473+
)
2474+
24602475
# CrateDB: Compensate for "Type `date` does not support storage".
24612476
updated_at_type = "DATE"
24622477
if dest_connection_url.startswith("cratedb://"):
@@ -2705,7 +2720,7 @@ def test_custom_query(testcase, source, dest):
27052720
dest_future = executor.submit(dest.start)
27062721
source_uri = source_future.result()
27072722
dest_uri = dest_future.result()
2708-
dest_uri_read = get_uri_read(dest_uri, dest)
2723+
dest_uri_read = get_uri_read(dest_uri, dest)
27092724
testcase(source_uri, dest_uri, dest_uri_read)
27102725
source.stop()
27112726
dest.stop()
@@ -2923,7 +2938,7 @@ def test_successful_ingestion(dest_uri, dest_uri_read):
29232938

29242939
if dest_uri.startswith("cratedb://"):
29252940
pytest.skip(
2926-
"CrateDB is not supported for this test, "
2941+
"CrateDB type mapping does not support `DATE` yet, "
29272942
"see https://github.com/crate-workbench/ingestr/issues/4"
29282943
)
29292944

@@ -3028,7 +3043,7 @@ def test_incremental_ingestion(dest_uri, dest_uri_read):
30283043

30293044
if dest_uri.startswith("cratedb://"):
30303045
pytest.skip(
3031-
"CrateDB is not supported for this test, "
3046+
"CrateDB type mapping does not support `DATE` yet, "
30323047
"see https://github.com/crate-workbench/ingestr/issues/4"
30333048
)
30343049

0 commit comments

Comments
 (0)