From c00bb465a14d6714d2fc32bb60b43f205ed95644 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 28 May 2026 15:04:57 +0200 Subject: [PATCH] I/O: Migrate from `ingestr` to `omniload` ingestr v1 started using the Functional Source License [1], which prevents commercial use for two years. omniload keeps using the MIT license. [1] https://fsl.software/ --- .github/workflows/postgresql.yml | 8 +-- .gitignore | 2 +- CHANGES.md | 1 + cratedb_toolkit/io/ingestr/boot.py | 41 ---------------- .../io/{ingestr => omniload}/__init__.py | 0 .../io/{ingestr => omniload}/api.py | 49 +++++++++---------- cratedb_toolkit/io/omniload/boot.py | 36 ++++++++++++++ cratedb_toolkit/io/router.py | 6 +-- cratedb_toolkit/model.py | 4 +- doc/.gitignore | 2 +- doc/backlog/io.md | 8 +-- doc/backlog/main.md | 2 - doc/install.md | 2 +- doc/io/_db-options.md | 2 +- doc/io/data-warehouse/databricks/index.md | 2 +- doc/io/database/postgresql/research.md | 2 +- doc/io/file/index.md | 4 +- doc/io/index.md | 2 +- doc/io/search-engine/elasticsearch/index.md | 2 +- doc/io/service/index.md | 32 ++++++------ doc/io/stream/kafka.md | 4 +- pyproject.toml | 6 +-- release/oci-ingest/Dockerfile | 2 +- tests/io/{ingestr => omniload}/__init__.py | 0 .../{ingestr => omniload}/test_postgresql.py | 8 +-- 25 files changed, 110 insertions(+), 117 deletions(-) delete mode 100644 cratedb_toolkit/io/ingestr/boot.py rename cratedb_toolkit/io/{ingestr => omniload}/__init__.py (100%) rename cratedb_toolkit/io/{ingestr => omniload}/api.py (70%) create mode 100644 cratedb_toolkit/io/omniload/boot.py rename tests/io/{ingestr => omniload}/__init__.py (100%) rename tests/io/{ingestr => omniload}/test_postgresql.py (85%) diff --git a/.github/workflows/postgresql.yml b/.github/workflows/postgresql.yml index 6a753ebc..187fec7e 100644 --- a/.github/workflows/postgresql.yml +++ b/.github/workflows/postgresql.yml @@ -5,19 +5,19 @@ on: pull_request: paths: - '.github/workflows/postgresql.yml' - - 'cratedb_toolkit/io/ingestr/**' + - 'cratedb_toolkit/io/omniload/**' - 'cratedb_toolkit/testing/testcontainers/postgresql.py' - 'cratedb_toolkit/testing/testcontainers/util.py' - - 'tests/io/ingestr/*postgresql*' + - 'tests/io/omniload/*postgresql*' - 'pyproject.toml' push: branches: [ main ] paths: - '.github/workflows/postgresql.yml' - - 'cratedb_toolkit/io/ingestr/**' + - 'cratedb_toolkit/io/omniload/**' - 'cratedb_toolkit/testing/testcontainers/postgresql.py' - 'cratedb_toolkit/testing/testcontainers/util.py' - - 'tests/io/ingestr/*postgresql*' + - 'tests/io/omniload/*postgresql*' - 'pyproject.toml' # Allow job to be triggered manually. diff --git a/.gitignore b/.gitignore index b3c3fdf0..8056c48f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ build coverage.xml node_modules /cfr -/doc/_build +/doc/_build/* /tmp /var /DOWNLOAD diff --git a/CHANGES.md b/CHANGES.md index 75cfd9b9..e36df04a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ - Dependency: Update sqlalchemy-crate version to 0.43.1 - Removed support for Rockset - Fixed a bug in `cfr jobstats ui` where slider will pick an initial value too large +- I/O: Migrated from `ingestr` to `omniload` ## 2026/06/17 v0.0.49 - Stopped leaking password to log output in `ctk cfr jobstats collect`. diff --git a/cratedb_toolkit/io/ingestr/boot.py b/cratedb_toolkit/io/ingestr/boot.py deleted file mode 100644 index 9f3058c0..00000000 --- a/cratedb_toolkit/io/ingestr/boot.py +++ /dev/null @@ -1,41 +0,0 @@ -import logging -from unittest import mock - -from boltons.urlutils import URL - -logger = logging.getLogger(__name__) - - -def import_ingestr(): - """Import ingestr with CrateDB destination adapter.""" - try: - with ( - mock.patch("ingestr.src.telemetry.event.track"), - mock.patch("dlt.common.runtime.telemetry._TELEMETRY_STARTED", True), - ): - import dlt_cratedb # noqa: F401 - import ingestr.main - import ingestr.src.factory - from dlt.common.configuration import ConfigFieldMissingException - from ingestr.src.destinations import GenericSqlDestination - - class CrateDBDestination(GenericSqlDestination): - def dlt_dest(self, uri: str, **kwargs): - uri = self._replace_url(uri) - import dlt_cratedb.impl.cratedb.factory - - return dlt_cratedb.impl.cratedb.factory.cratedb(credentials=uri, **kwargs) - - @staticmethod - def _replace_url(uri: str) -> str: - url_obj = URL(uri) - if url_obj.scheme == "cratedb": - url_obj.scheme = "postgres" - return str(url_obj) - - ingestr.src.factory.SourceDestinationFactory.destinations["cratedb"] = CrateDBDestination - - return True, ingestr, ConfigFieldMissingException - except ImportError: - logger.error("Could not import ingestr") - return False, None, None diff --git a/cratedb_toolkit/io/ingestr/__init__.py b/cratedb_toolkit/io/omniload/__init__.py similarity index 100% rename from cratedb_toolkit/io/ingestr/__init__.py rename to cratedb_toolkit/io/omniload/__init__.py diff --git a/cratedb_toolkit/io/ingestr/api.py b/cratedb_toolkit/io/omniload/api.py similarity index 70% rename from cratedb_toolkit/io/ingestr/api.py rename to cratedb_toolkit/io/omniload/api.py index 461b5e53..c51bc1f8 100644 --- a/cratedb_toolkit/io/ingestr/api.py +++ b/cratedb_toolkit/io/omniload/api.py @@ -3,51 +3,51 @@ from yarl import URL -from cratedb_toolkit.io.ingestr.boot import import_ingestr +from cratedb_toolkit.io.omniload.boot import import_omniload from cratedb_toolkit.model import DatabaseAddress logger = logging.getLogger(__name__) @lru_cache(maxsize=1) -def _get_ingestr(): +def _get_omniload(): """ - Get an ingestr API handle, cached. + Get an omniload API handle, cached. """ - return import_ingestr() + return import_omniload() -def ingestr_select(source_url: str) -> bool: +def omniload_select(source_url: str) -> bool: """ - Whether to select `ingestr` for this data source. + Whether to select `omniload` for this data source. """ - ingestr_available, ingestr, ConfigFieldMissingException = _get_ingestr() + omniload_available, omniload, ConfigFieldMissingException = _get_omniload() - if not ingestr_available: - logger.debug("ingestr is not installed") + if not omniload_available: + logger.debug("omniload is not installed") return False try: - factory = ingestr.src.factory.SourceDestinationFactory(source_url, "csv:////tmp/foobar.csv") + factory = omniload.src.factory.SourceDestinationFactory(source_url, "csv:////tmp/foobar.csv") factory.get_source() - scheme = ingestr.src.factory.parse_scheme_from_uri(source_url) - logger.info(f"Selecting ingestr for source scheme: {scheme}") + scheme = omniload.src.factory.parse_scheme_from_uri(source_url) + logger.info(f"Selecting omniload for source scheme: {scheme}") return True except (ImportError, ValueError, AttributeError) as ex: if "Unsupported source scheme" in str(ex): - logger.debug(f"Failed to select ingestr for source url '{source_url}': {ex}") + logger.debug(f"Failed to select omniload for source url '{source_url}': {ex}") else: - logger.exception(f"Unexpected error with ingestr for source url: {source_url}") + logger.exception(f"Unexpected error with omniload for source url: {source_url}") return False except Exception: - logger.exception(f"Unexpected error with ingestr for source url: {source_url}") + logger.exception(f"Unexpected error with omniload for source url: {source_url}") return False -def ingestr_copy(source_url: str, target_address: DatabaseAddress, progress: bool = False) -> bool: +def omniload_copy(source_url: str, target_address: DatabaseAddress, progress: bool = False) -> bool: """ - Invoke data transfer to CrateDB from any source provided by `ingestr`. + Invoke data transfer to CrateDB from any source provided by `omniload`. - https://cratedb-toolkit.readthedocs.io/io/ingestr/ + https://cratedb-toolkit.readthedocs.io/io/omniload/ Synopsis: @@ -63,11 +63,11 @@ def ingestr_copy(source_url: str, target_address: DatabaseAddress, progress: boo "postgresql://pguser:secret11@postgresql.example.org:5432/postgres?table=public.diamonds" \ "crate://crate:na@localhost:4200/testdrive/ibis_diamonds" """ - ingestr_available, ingestr, ConfigFieldMissingException = _get_ingestr() + omniload_available, omniload, ConfigFieldMissingException = _get_omniload() # Sanity checks. - if not ingestr_available: - raise ModuleNotFoundError("ingestr subsystem not installed") + if not omniload_available: + raise ModuleNotFoundError("omniload subsystem not installed") # Compute source and target URLs and table names. # Table names use dotted notation `.`. @@ -91,7 +91,7 @@ def ingestr_copy(source_url: str, target_address: DatabaseAddress, progress: boo target_uri, target_table_address = target_address.decode() target_table = target_table_address.fullname - target_url = target_address.to_ingestr_url() + target_url = target_address.to_omniload_url() if not source_table: raise ValueError("Source table is required") @@ -101,7 +101,7 @@ def ingestr_copy(source_url: str, target_address: DatabaseAddress, progress: boo if source_fragment: source_table += f"#{source_fragment}" - logger.info("Invoking ingestr") + logger.info("Invoking omniload") logger.info(f"Source URL: {source_url_obj}") logger.info(f"Target URL: {target_url}") logger.info(f"Source Table: {source_table}") @@ -114,7 +114,6 @@ def ingestr_copy(source_url: str, target_address: DatabaseAddress, progress: boo dest_uri=str(target_url), source_table=source_table, dest_table=target_table, - yes=True, ) if start_date is not None: ingest_kwargs["interval_start"] = start_date @@ -122,7 +121,7 @@ def ingestr_copy(source_url: str, target_address: DatabaseAddress, progress: boo ingest_kwargs["page_size"] = batch_size try: - ingestr.main.ingest(**ingest_kwargs) + omniload.main.ingest(**ingest_kwargs) return True except ConfigFieldMissingException: logger.error( diff --git a/cratedb_toolkit/io/omniload/boot.py b/cratedb_toolkit/io/omniload/boot.py new file mode 100644 index 00000000..59f5a440 --- /dev/null +++ b/cratedb_toolkit/io/omniload/boot.py @@ -0,0 +1,36 @@ +import logging + +from boltons.urlutils import URL + +logger = logging.getLogger(__name__) + + +def import_omniload(): + """Import omniload with CrateDB destination adapter.""" + try: + import dlt_cratedb # noqa: F401 + import omniload.main + import omniload.src.factory + from dlt.common.configuration import ConfigFieldMissingException + from omniload.src.destinations import GenericSqlDestination + + class CrateDBDestination(GenericSqlDestination): + def dlt_dest(self, uri: str, **kwargs): + uri = self._replace_url(uri) + import dlt_cratedb.impl.cratedb.factory + + return dlt_cratedb.impl.cratedb.factory.cratedb(credentials=uri, **kwargs) + + @staticmethod + def _replace_url(uri: str) -> str: + url_obj = URL(uri) + if url_obj.scheme == "cratedb": + url_obj.scheme = "postgres" + return str(url_obj) + + omniload.src.factory.SourceDestinationFactory.destinations["cratedb"] = CrateDBDestination + + return True, omniload, ConfigFieldMissingException + except ImportError: + logger.error("Could not import omniload") + return False, None, None diff --git a/cratedb_toolkit/io/router.py b/cratedb_toolkit/io/router.py index 6ac4ab97..eb495754 100644 --- a/cratedb_toolkit/io/router.py +++ b/cratedb_toolkit/io/router.py @@ -119,12 +119,12 @@ def load_table( return from_iceberg(str(source_url_obj), target_url) - from cratedb_toolkit.io.ingestr.api import ingestr_copy, ingestr_select + from cratedb_toolkit.io.omniload.api import omniload_copy, omniload_select source_url = str(source_url_obj) - if ingestr_select(source_url): - return ingestr_copy(source_url, target, progress=True) + if omniload_select(source_url): + return omniload_copy(source_url, target, progress=True) else: raise NotImplementedError(f"Importing resource not implemented yet: {source_url_obj}") diff --git a/cratedb_toolkit/model.py b/cratedb_toolkit/model.py index 6479e5b4..a6e9a712 100644 --- a/cratedb_toolkit/model.py +++ b/cratedb_toolkit/model.py @@ -141,9 +141,9 @@ def to_postgresql_url(self, port: int = 5432) -> URL: uri.port = port return uri - def to_ingestr_url(self, port: int = 5432) -> URL: + def to_omniload_url(self, port: int = 5432) -> URL: """ - Return the `cratedb://` variant of the database URI, suitable for `ingestr`. + Return the `cratedb://` variant of the database URI, suitable for `omniload`. """ uri = deepcopy(self.to_postgresql_url(port)) uri.scheme = "cratedb" diff --git a/doc/.gitignore b/doc/.gitignore index a485625d..88f9974b 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1 +1 @@ -/_build +_build/* diff --git a/doc/backlog/io.md b/doc/backlog/io.md index cbc0cdfb..d1734060 100644 --- a/doc/backlog/io.md +++ b/doc/backlog/io.md @@ -1,10 +1,10 @@ # Backlog for `ctk {load,save}` ## General -- ingestr: Educate about batch size (`page_size`) -- ingestr: More parameters for Kinesis - - https://getbruin.com/docs/ingestr/commands/ingest.html - - https://getbruin.com/docs/ingestr/supported-sources/kinesis.html +- omniload: Educate about batch size (`page_size`) +- omniload: More parameters for Kinesis + - https://omniload.readthedocs.io/commands/ingest.html + - https://omniload.readthedocs.io/supported-sources/kinesis.html - ```python # incremental_strategy="delete+insert", # mask=["temperature:noise:0.35", "temperature:round:100"], diff --git a/doc/backlog/main.md b/doc/backlog/main.md index ddbf4e2a..c5db51ee 100644 --- a/doc/backlog/main.md +++ b/doc/backlog/main.md @@ -1,8 +1,6 @@ # Main Backlog ## Iteration +0 -- Exercise data imports from AWS S3 and other Object Storage providers. - Evaluate the new `ingestr` subsystem. - A new item for the datasets API https://github.com/dr5hn/countries-states-cities-database diff --git a/doc/install.md b/doc/install.md index 253fb466..79876fed 100644 --- a/doc/install.md +++ b/doc/install.md @@ -30,7 +30,7 @@ ctk --version Alternatively, use a container image from GHCR. Use `ghcr.io/crate/cratedb-toolkit` for the default toolset. To use the "ingest" image (which includes the `io-ingest` -extras and the ingestr-based I/O adapters), use `ghcr.io/crate/cratedb-toolkit-ingest`. +extras and the omniload-based I/O adapters), use `ghcr.io/crate/cratedb-toolkit-ingest`. The default image does not include these extras or drivers. Run with Docker or Podman. diff --git a/doc/io/_db-options.md b/doc/io/_db-options.md index 90018544..c77555de 100644 --- a/doc/io/_db-options.md +++ b/doc/io/_db-options.md @@ -22,4 +22,4 @@ ctk load "protocol://?table=query:SELECT * FROM sys.summits WHERE height > 4000" ``` -[custom queries for SQL sources]: https://bruin-data.github.io/ingestr/supported-sources/custom_queries.html +[custom queries for SQL sources]: https://omniload.readthedocs.io/supported-sources/custom_queries.html diff --git a/doc/io/data-warehouse/databricks/index.md b/doc/io/data-warehouse/databricks/index.md index ad9ba085..d4a43645 100644 --- a/doc/io/data-warehouse/databricks/index.md +++ b/doc/io/data-warehouse/databricks/index.md @@ -152,5 +152,5 @@ See also the documentation about [SQLAlchemy's Databricks dialect]. [cratedb-toolkit]: https://pypi.org/project/cratedb-toolkit/ [Databricks]: https://www.databricks.com/ -[Databricks OAuth M2M authentication]: https://getbruin.com/docs/ingestr/supported-sources/databricks.html +[Databricks OAuth M2M authentication]: https://omniload.readthedocs.io/supported-sources/databricks.html [SQLAlchemy's Databricks dialect]: https://docs.databricks.com/aws/en/dev-tools/sqlalchemy diff --git a/doc/io/database/postgresql/research.md b/doc/io/database/postgresql/research.md index bcb305f9..e4336511 100644 --- a/doc/io/database/postgresql/research.md +++ b/doc/io/database/postgresql/research.md @@ -2,7 +2,7 @@ ## Full-load -This subsystem is already covered by ingestr/dlt. +This subsystem is already covered by omniload/dlt (ex. ingestr/dlt). ## CDC diff --git a/doc/io/file/index.md b/doc/io/file/index.md index 5f7a5539..56cbabb1 100644 --- a/doc/io/file/index.md +++ b/doc/io/file/index.md @@ -17,7 +17,7 @@ ctk load \ "s3://?access_key_id=${AWS_ACCESS_KEY_ID}&secret_access_key=${AWS_SECRET_ACCESS_KEY}&table=openaq-fetches/realtime/2023-02-25/1677351953_eea_2aa299a7-b688-4200-864a-8df7bac3af5b.ndjson#jsonl" \ "crate://crate:na@localhost:4200/testdrive/s3_ndjson" ``` -See documentation about [ingestr and Amazon S3] about details of the URI format, +See documentation about [omniload and Amazon S3] about details of the URI format, file globbing patterns, compression options, and file type hinting options. :::{div} :style: font-size: small @@ -43,4 +43,4 @@ ctk load \ ``` -[ingestr and Amazon S3]: https://bruin-data.github.io/ingestr/supported-sources/s3.html +[omniload and Amazon S3]: https://omniload.readthedocs.io/supported-sources/s3.html diff --git a/doc/io/index.md b/doc/io/index.md index 1a884983..b834ac95 100644 --- a/doc/io/index.md +++ b/doc/io/index.md @@ -262,7 +262,7 @@ managed/index [Databricks]: https://www.databricks.com/ [DuckDB]: https://github.com/duckdb/duckdb [DynamoDB]: https://aws.amazon.com/dynamodb/ -[incremental loading]: https://bruin-data.github.io/ingestr/getting-started/incremental-loading.html +[incremental loading]: https://omniload.readthedocs.io/getting-started/incremental-loading.html [InfluxDB]: https://github.com/influxdata/influxdb [MongoDB]: https://github.com/mongodb/mongo [MongoDB Atlas]: https://www.mongodb.com/atlas diff --git a/doc/io/search-engine/elasticsearch/index.md b/doc/io/search-engine/elasticsearch/index.md index 2d860f60..b08ba4ca 100644 --- a/doc/io/search-engine/elasticsearch/index.md +++ b/doc/io/search-engine/elasticsearch/index.md @@ -54,7 +54,7 @@ wget https://cdn.crate.io/downloads/datasets/cratedb-datasets/academy/chicago-da ``` Import data into Elasticsearch. ```shell -ingestr ingest --yes \ +omniload ingest --yes \ --source-uri "csv://taxi_details.csv" \ --source-table "data" \ --dest-uri "elasticsearch://localhost:9200?secure=false" \ diff --git a/doc/io/service/index.md b/doc/io/service/index.md index 01803997..bbafbf92 100644 --- a/doc/io/service/index.md +++ b/doc/io/service/index.md @@ -473,19 +473,19 @@ ctk load \ ``` -[Anthropic entities]: https://bruin-data.github.io/ingestr/supported-sources/anthropic.html#available-tables -[Asana entities]: https://bruin-data.github.io/ingestr/supported-sources/asana.html#tables -[Attio entities]: https://bruin-data.github.io/ingestr/supported-sources/attio.html#tables -[Facebook Ads entities]: https://bruin-data.github.io/ingestr/supported-sources/facebook-ads.html#tables -[GitHub entities]: https://bruin-data.github.io/ingestr/supported-sources/github.html#tables -[Google Ads entities]: https://bruin-data.github.io/ingestr/supported-sources/google-ads.html#tables -[Google Analytics entities]: https://bruin-data.github.io/ingestr/supported-sources/google_analytics.html#available-tables -[HubSpot entities]: https://bruin-data.github.io/ingestr/supported-sources/hubspot.html#tables -[Jira entities]: https://bruin-data.github.io/ingestr/supported-sources/jira.html#tables -[Salesforce entities]: https://bruin-data.github.io/ingestr/supported-sources/salesforce.html#tables -[Shopify entities]: https://bruin-data.github.io/ingestr/supported-sources/shopify.html#tables -[Slack entities]: https://bruin-data.github.io/ingestr/supported-sources/slack.html#tables -[Stripe entities]: https://bruin-data.github.io/ingestr/supported-sources/stripe.html#all-endpoints -[Wise entities]: https://bruin-data.github.io/ingestr/supported-sources/wise.html#tables -[Zendesk entities]: https://bruin-data.github.io/ingestr/supported-sources/zendesk.html#tables -[Zoom entities]: https://bruin-data.github.io/ingestr/supported-sources/zoom.html#uri-format +[Anthropic entities]: https://omniload.readthedocs.io/supported-sources/anthropic.html#available-tables +[Asana entities]: https://omniload.readthedocs.io/supported-sources/asana.html#tables +[Attio entities]: https://omniload.readthedocs.io/supported-sources/attio.html#tables +[Facebook Ads entities]: https://omniload.readthedocs.io/supported-sources/facebook-ads.html#tables +[GitHub entities]: https://omniload.readthedocs.io/supported-sources/github.html#tables +[Google Ads entities]: https://omniload.readthedocs.io/supported-sources/google-ads.html#tables +[Google Analytics entities]: https://omniload.readthedocs.io/supported-sources/google_analytics.html#available-tables +[HubSpot entities]: https://omniload.readthedocs.io/supported-sources/hubspot.html#tables +[Jira entities]: https://omniload.readthedocs.io/supported-sources/jira.html#tables +[Salesforce entities]: https://omniload.readthedocs.io/supported-sources/salesforce.html#tables +[Shopify entities]: https://omniload.readthedocs.io/supported-sources/shopify.html#tables +[Slack entities]: https://omniload.readthedocs.io/supported-sources/slack.html#tables +[Stripe entities]: https://omniload.readthedocs.io/supported-sources/stripe.html#all-endpoints +[Wise entities]: https://omniload.readthedocs.io/supported-sources/wise.html#tables +[Zendesk entities]: https://omniload.readthedocs.io/supported-sources/zendesk.html#tables +[Zoom entities]: https://omniload.readthedocs.io/supported-sources/zoom.html#uri-format diff --git a/doc/io/stream/kafka.md b/doc/io/stream/kafka.md index fe276f6c..cf45baa1 100644 --- a/doc/io/stream/kafka.md +++ b/doc/io/stream/kafka.md @@ -147,6 +147,6 @@ Kafka+CrateDB-in-a-box example rig using {Docker,Podman} Compose. [Apache Kafka]: https://kafka.apache.org/ [cratedb-toolkit]: https://pypi.org/project/cratedb-toolkit/ -[kafka-compose.yml]: https://github.com/crate/cratedb-examples/blob/main/application/ingestr/kafka-compose.yml -[kafka-demo.xsh]: https://github.com/crate/cratedb-examples/blob/main/application/ingestr/kafka-demo.xsh +[kafka-compose.yml]: https://github.com/crate/cratedb-examples/blob/main/application/omniload/kafka-compose.yml +[kafka-demo.xsh]: https://github.com/crate/cratedb-examples/blob/main/application/omniload/kafka-demo.xsh [kcat]: https://github.com/edenhill/kcat diff --git a/pyproject.toml b/pyproject.toml index 3e04b877..d0183b58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -188,8 +188,8 @@ optional-dependencies.io-ingest = [ "cratedb-toolkit[io-base]", "dlt>=1.10,<1.28", "dlt-cratedb>=0.0.1", - "ingestr>=0.13.61,<0.15", - "sqlalchemy>=1,<2", + "omniload<0.2", + "sqlalchemy>=1,<2.1", ] optional-dependencies.io-opentable = [ "cratedb-toolkit[deltalake,iceberg]", @@ -359,13 +359,13 @@ analysis.allowed-unresolved-imports = [ "fastapi.**", "importlib_metadata.**", "importlib_resources.**", - "ingestr.**", "jessiql.**", "kaggle.**", "kinesis.**", "llama_index.**", "lorrystream.**", "mcp.**", + "omniload.**", "pymongo.**", "sqlalchemy.**", "tikray.**", diff --git a/release/oci-ingest/Dockerfile b/release/oci-ingest/Dockerfile index b00bc939..7234b7ab 100644 --- a/release/oci-ingest/Dockerfile +++ b/release/oci-ingest/Dockerfile @@ -122,7 +122,7 @@ RUN \ --mount=type=cache,id=uv-${TARGETARCH}${TARGETVARIANT},target=/root/.cache/uv \ true \ && WHEEL="$(ls /tmp/*.whl)" \ - && uv pip install --upgrade --overrides="/tmp/overrides.txt" "ingestr[oracle,odbc]" "cratedb-toolkit[io-ingest] @ file://${WHEEL}" + && uv pip install --upgrade --overrides="/tmp/overrides.txt" "omniload[full]" "cratedb-toolkit[io-ingest] @ file://${WHEEL}" # Install curl, for running Compose health checks. RUN --mount=type=cache,id=apt-cache-${TARGETARCH}${TARGETVARIANT},target=/var/cache/apt \ diff --git a/tests/io/ingestr/__init__.py b/tests/io/omniload/__init__.py similarity index 100% rename from tests/io/ingestr/__init__.py rename to tests/io/omniload/__init__.py diff --git a/tests/io/ingestr/test_postgresql.py b/tests/io/omniload/test_postgresql.py similarity index 85% rename from tests/io/ingestr/test_postgresql.py rename to tests/io/omniload/test_postgresql.py index 46130cb2..795d5fb7 100644 --- a/tests/io/ingestr/test_postgresql.py +++ b/tests/io/omniload/test_postgresql.py @@ -17,17 +17,17 @@ def test_postgresql_load_table(caplog, cratedb): """ # postgresql_url = f"{postgresql.get_connection_url(driver=None)}?sslmode=disable&table=information_schema.tables" # noqa: ERA001, E501 # cratedb_url = f"{cratedb.get_connection_url()}/testdrive/demo" # noqa: ERA001 - ingestr_postgresql_url = ( + omniload_postgresql_url = ( "postgresql://postgres:postgres@localhost:5433?sslmode=disable&table=information_schema.tables" ) - ingestr_cratedb_url = "cratedb://crate:crate@localhost:5432/testdrive/demo" + omniload_cratedb_url = "cratedb://crate:crate@localhost:5432/testdrive/demo" cratedb_sqlalchemy_url = "crate://crate:crate@localhost:4200/" # Run transfer command. - runner = CliRunner(env={"CRATEDB_CLUSTER_URL": ingestr_cratedb_url}) + runner = CliRunner(env={"CRATEDB_CLUSTER_URL": omniload_cratedb_url}) result = runner.invoke( cli, - args=f"load {ingestr_postgresql_url}", + args=f"load {omniload_postgresql_url}", catch_exceptions=False, ) assert result.exit_code == 0