Skip to content

Commit b82e5ec

Browse files
MaxteabagMaxteabag
committed
Add integration and unit tests for Impala, osquery, and SurrealDB
Adds test coverage for the three providers introduced in this PR: - SurrealDB: docker-compose service, fixture, and integration suite modelled on the Turso tests. Overrides test_query_with_where / test_query_aggregate / test_query_insert with SurrealQL-native versions because record IDs are typed (`test_users:1`, not ints) and aggregate syntax differs (`count()` + `GROUP ALL`). - Impala: full apache/impala:4.5.0-* quickstart (hms, statestored, catalogd, impalad) under the enterprise profile. Mounts hive-site.xml from impala_quickstart_conf/, uses a fixed underscore-free network name so Java's URL parser accepts the DNS suffix, and aliases quickstart-hive-metastore / statestored / catalogd as Impala expects. Healthcheck uses a Python socket probe since /bin/sh lacks /dev/tcp. Overrides test_primary_key_detection (PK is a Kudu-only feature). - osquery: unit tests with the `osquery` SDK mocked via sys.modules. Covers capability flags, default socket path per platform, spawn and socket connect paths, connection wrapper caching, test/query/tables/ columns execution, and error paths. No daemon required.
1 parent 966c5ca commit b82e5ec

9 files changed

Lines changed: 1511 additions & 0 deletions

File tree

‎infra/docker/docker-compose.test.yml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,101 @@ services:
175175
retries: 10
176176
start_period: 10s
177177

178+
# Apache Impala ships no single-container image. The official quickstart runs
179+
# Hive Metastore + statestored + catalogd + impalad together, all pinned to
180+
# apache/impala:4.5.0-* with hive-site.xml mounted from impala_quickstart_conf/.
181+
# Based on https://github.com/apache/impala/blob/master/docker/quickstart.yml
182+
impala-hms:
183+
image: apache/impala:4.5.0-impala_quickstart_hms
184+
container_name: sqlit-test-impala-hms
185+
command: ["hms"]
186+
volumes:
187+
- impala-warehouse:/var/lib/hive
188+
- impala-warehouse:/user/hive/warehouse
189+
- ./impala_quickstart_conf:/opt/hive/conf:ro
190+
# hive-site.xml refers to the metastore as `thrift://quickstart-hive-metastore:9083`
191+
# (matching the upstream quickstart). Alias so catalogd/impalad can resolve it.
192+
networks:
193+
impala-network:
194+
aliases:
195+
- quickstart-hive-metastore
196+
profiles:
197+
- enterprise
198+
199+
impala-statestored:
200+
image: apache/impala:4.5.0-statestored
201+
container_name: sqlit-test-impala-statestored
202+
command: ["-redirect_stdout_stderr=false", "-logtostderr", "-v=1"]
203+
volumes:
204+
- ./impala_quickstart_conf:/opt/impala/conf:ro
205+
# Impala hard-codes peer hostnames `statestored` and `catalogd` when the
206+
# services register — alias so the other containers can reach it there.
207+
networks:
208+
impala-network:
209+
aliases:
210+
- statestored
211+
profiles:
212+
- enterprise
213+
214+
impala-catalogd:
215+
image: apache/impala:4.5.0-catalogd
216+
container_name: sqlit-test-impala-catalogd
217+
depends_on:
218+
- impala-statestored
219+
- impala-hms
220+
command:
221+
- "-redirect_stdout_stderr=false"
222+
- "-logtostderr"
223+
- "-v=1"
224+
- "-hms_event_polling_interval_s=1"
225+
- "-invalidate_tables_timeout_s=999999"
226+
volumes:
227+
- impala-warehouse:/user/hive/warehouse
228+
- ./impala_quickstart_conf:/opt/impala/conf:ro
229+
networks:
230+
impala-network:
231+
aliases:
232+
- catalogd
233+
profiles:
234+
- enterprise
235+
236+
impala:
237+
image: apache/impala:4.5.0-impalad_coord_exec
238+
container_name: sqlit-test-impala
239+
depends_on:
240+
- impala-statestored
241+
- impala-catalogd
242+
command:
243+
- "-v=1"
244+
- "-redirect_stdout_stderr=false"
245+
- "-logtostderr"
246+
- "-mt_dop_auto_fallback=true"
247+
- "-default_query_options=mt_dop=4,default_file_format=parquet,default_transactional_type=insert_only"
248+
- "-mem_limit=4gb"
249+
environment:
250+
- JAVA_TOOL_OPTIONS=-Xmx1g
251+
ports:
252+
- "${IMPALA_PORT:-21050}:21050"
253+
volumes:
254+
- impala-warehouse:/user/hive/warehouse
255+
- ./impala_quickstart_conf:/opt/impala/conf:ro
256+
networks:
257+
- impala-network
258+
healthcheck:
259+
# /bin/sh in this image is dash, which doesn't support bash's /dev/tcp.
260+
# The Impala image ships Python, so use that for a tiny TCP probe.
261+
test:
262+
- "CMD"
263+
- "python"
264+
- "-c"
265+
- "import socket,sys; s=socket.socket(); s.settimeout(2); s.connect(('localhost',21050)); sys.exit(0)"
266+
interval: 10s
267+
timeout: 5s
268+
retries: 30
269+
start_period: 90s
270+
profiles:
271+
- enterprise
272+
178273
mariadb:
179274
image: mariadb:11
180275
container_name: sqlit-test-mariadb
@@ -220,6 +315,20 @@ services:
220315
retries: 10
221316
start_period: 10s
222317

318+
surrealdb:
319+
image: surrealdb/surrealdb:latest
320+
container_name: sqlit-test-surrealdb
321+
command: start --user root --pass root --bind 0.0.0.0:8000 memory
322+
ports:
323+
- "${SURREALDB_PORT:-8000}:8000"
324+
healthcheck:
325+
test: ["CMD", "/surreal", "is-ready", "--endpoint", "http://localhost:8000"]
326+
interval: 5s
327+
timeout: 5s
328+
retries: 10
329+
start_period: 10s
330+
331+
223332
firebird:
224333
image: firebirdsql/firebird:5.0.3-noble
225334
container_name: sqlit-test-firebird
@@ -333,3 +442,12 @@ services:
333442
timeout: 5s
334443
retries: 10
335444
start_period: 10s
445+
446+
volumes:
447+
impala-warehouse:
448+
449+
networks:
450+
impala-network:
451+
# Underscore-free FQDN: Java's URL parser rejects hostnames with underscores,
452+
# which is why we can't let compose auto-prefix the network name.
453+
name: impalanet

‎infra/docker/impala_quickstart_conf/hive-site.xml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
<!--
4+
Licensed to the Apache Software Foundation (ASF) under one or more
5+
contributor license agreements. See the NOTICE file distributed with
6+
this work for additional information regarding copyright ownership.
7+
The ASF licenses this file to You under the Apache License, Version 2.0
8+
(the "License"); you may not use this file except in compliance with
9+
the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
-->
19+
<!--
20+
Hive configuration for Impala quickstart docker cluster.
21+
-->
22+
<configuration>
23+
<property>
24+
<!-- Required for automatic metadata sync. -->
25+
<name>hive.metastore.dml.events</name>
26+
<value>true</value>
27+
</property>
28+
<property>
29+
<!-- User impala is not authorized to consume notifications by default, disable
30+
authentication to work around this. -->
31+
<name>hive.metastore.event.db.notification.api.auth</name>
32+
<value>false</value>
33+
</property>
34+
<property>
35+
<name>hive.metastore.uris</name>
36+
<value>thrift://quickstart-hive-metastore:9083</value>
37+
</property>
38+
<!-- Managed and external tablespaces must live on the Docker volumes that we
39+
configure for the quickstart cluster. -->
40+
<property>
41+
<name>hive.metastore.warehouse.dir</name>
42+
<value>/user/hive/warehouse/managed</value>
43+
</property>
44+
<property>
45+
<name>hive.metastore.warehouse.external.dir</name>
46+
<value>/user/hive/warehouse/external</value>
47+
</property>
48+
<property>
49+
<!-- Required to enable Hive transactions -->
50+
<name>hive.support.concurrency</name>
51+
<value>true</value>
52+
</property>
53+
<property>
54+
<!-- Required to enable Hive transactions -->
55+
<name>hive.txn.manager</name>
56+
<value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
57+
</property>
58+
<property>
59+
<!-- Use embedded Derby database -->
60+
<name>javax.jdo.option.ConnectionDriverName</name>
61+
<value>org.apache.derby.jdbc.EmbeddedDriver</value>
62+
</property>
63+
<property>
64+
<!-- Use embedded Derby database -->
65+
<name>javax.jdo.option.ConnectionURL</name>
66+
<value>jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true</value>
67+
</property>
68+
<!-- Hive stats autogathering negatively affects latency of DDL operations, etc and
69+
is not particularly useful for Impala -->
70+
<property>
71+
<name>hive.stats.autogather</name>
72+
<value>false</value>
73+
</property>
74+
</configuration>

‎tests/conftest.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from tests.fixtures.duckdb import *
1212
from tests.fixtures.firebird import *
1313
from tests.fixtures.flight import *
14+
from tests.fixtures.impala import *
1415
from tests.fixtures.mariadb import *
1516
from tests.fixtures.mssql import *
1617
from tests.fixtures.mysql import *
@@ -23,6 +24,7 @@
2324
from tests.fixtures.ssh import *
2425
from tests.fixtures.spanner import *
2526
from tests.fixtures.sqlite import *
27+
from tests.fixtures.surrealdb import *
2628
from tests.fixtures.turso import *
2729
from tests.fixtures.utils import *
2830

0 commit comments

Comments
 (0)