tests: better testing of the hbase connectors

docs/modules/spark-k8s/partials/supported-versions.adoc

@@ -5,7 +5,7 @@
 
 - 4.1.2 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (LTS)
 - 4.1.1 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (Deprecated)
-- 3.5.8 (Hadoop 3.4.2, Scala 2.12, Python 3.11, Java 17) (Deprecated)
+- 3.5.8 (HBase 2.6.6, Hadoop 3.4.3, Scala 2.12, Python 3.11, Java 17) (Deprecated)
 
 Apache Spark 4.1.1 & 4.1.2 have the following known issues (as of July 2026):
 

tests/templates/kuttl/hbase-connector/10-deploy-spark-app.yaml.j2

@@ -77,7 +77,11 @@ data:
     spark = SparkSession.builder.appName("test-hbase").getOrCreate()
 
     df = spark.createDataFrame(
-      [("row1", "Hello, Stackable!")],
+      [
+        ("row1", "Hello, Stackable!"),
+        ("row2", "Hello, HBase!"),
+        ("row3", "Hello, Spark!"),
+      ],
       "key: string, value: string"
     )
 
@@ -101,3 +105,53 @@ data:
       .option('catalog', catalog)\
       .option('newtable', '5')\
       .save()
+
+    # Read the whole table back and register it as a temporary view so we can
+    # run SQL queries against it.
+    #
+    # hbase.spark.pushdown.columnfilter=false disables server-side predicate
+    # pushdown: the connector would otherwise ship a SparkSQLPushDownFilter to
+    # the RegionServers, which requires the hbase-spark connector classes on
+    # the RegionServer classpath (not present in the Stackable HBase image).
+    # With pushdown off, WHERE clauses are evaluated Spark-side over a full
+    # scan, which is all this test needs.
+    read_df = spark\
+      .read\
+      .format("org.apache.hadoop.hbase.spark")\
+      .option('catalog', catalog)\
+      .option('hbase.spark.pushdown.columnfilter', 'false')\
+      .load()
+    read_df.createOrReplaceTempView("test_hbase")
+
+    # Full scan: all rows that were written must be readable again.
+    all_rows = spark.sql("SELECT key, value FROM test_hbase").collect()
+    actual = {row["key"]: row["value"] for row in all_rows}
+    expected = {
+      "row1": "Hello, Stackable!",
+      "row2": "Hello, HBase!",
+      "row3": "Hello, Spark!",
+    }
+    assert actual == expected, f"full scan mismatch: {actual} != {expected}"
+
+    # Row-key point lookup.
+    point = spark.sql("SELECT value FROM test_hbase WHERE key = 'row2'").collect()
+    assert len(point) == 1, f"point lookup returned {len(point)} rows, expected 1"
+    assert point[0]["value"] == "Hello, HBase!", f"unexpected value: {point[0]['value']}"
+
+    # Range scan on the row key.
+    range_rows = spark.sql(
+      "SELECT key FROM test_hbase WHERE key >= 'row2' ORDER BY key"
+    ).collect()
+    assert [r["key"] for r in range_rows] == ["row2", "row3"], \
+      f"unexpected range scan result: {[r['key'] for r in range_rows]}"
+
+    # Filter on a non-rowkey column value.
+    like_rows = spark.sql(
+      "SELECT key FROM test_hbase WHERE value LIKE 'Hello, S%' ORDER BY key"
+    ).collect()
+    assert [r["key"] for r in like_rows] == ["row1", "row3"], \
+      f"unexpected value filter result: {[r['key'] for r in like_rows]}"
+
+    # Aggregation over the scanned rows.
+    count = spark.sql("SELECT COUNT(*) AS c FROM test_hbase").collect()[0]["c"]
+    assert count == 3, f"expected 3 rows, got {count}"

tests/templates/kuttl/product-config-compat/10-deploy-spark-app.yaml.j2

@@ -5,6 +5,7 @@ metadata:
   name: pyspark-pi
 spec:
   sparkImage:
+    pullPolicy: IfNotPresent
 {% if test_scenario['values']['spark-regression'].find(",") > 0 %}
     custom: "{{ test_scenario['values']['spark-regression'].split(',')[1] }}"
     productVersion: "{{ test_scenario['values']['spark-regression'].split(',')[0] }}"

tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json

@@ -2,5 +2,5 @@
   "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE",
   "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n",
   "spark-env.sh": "",
-  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/role-group: sparkapplication\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    prometheus.io/scrape: 'true'\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: Always\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: config\n"
+  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/role-group: sparkapplication\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    prometheus.io/scrape: 'true'\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: config\n"
 }

tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json

@@ -2,5 +2,5 @@
   "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE",
   "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n",
   "spark-env.sh": "",
-  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/role-group: sparkapplication\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: Always\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: config\n"
+  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/role-group: sparkapplication\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: config\n"
 }

tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json

@@ -31,7 +31,7 @@
         }
       ],
       "image": "oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev",
-      "imagePullPolicy": "Always",
+      "imagePullPolicy": "IfNotPresent",
       "name": "spark-submit",
       "resources": {
         "limits": {

tests/test-definition.yaml

@@ -56,7 +56,7 @@ dimensions:
       - "CLUSTER.LOCAL"
   - name: hbase
     values:
-      - 2.6.4
+      - 2.6.6
   - name: hdfs-latest
     values:
       - 3.5.0