This should now work

Author: Ghislain Fourny

pyproject.toml

@@ -22,6 +22,12 @@ classifiers = [
   "Programming Language :: Python :: 3.11",
 ]
 
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+jsoniq = ["jars/*.jar"]
+
 [project.urls]
 Homepage = "https://rumbledb.org"
 Documentation = "https://rumble.readthedocs.io/en/latest/"

src/jsoniq/jars/rumbledb-1.24.0.jar

@@ -1,13 +1,23 @@
+from pyspark import RDD
 from pyspark.sql import SparkSession
 import json
 
 class SequenceOfItems:
-    def __init__(self, sequence):
+    def __init__(self, sequence, sparkcontext):
         self._jsequence = sequence
+        self._sparkcontext = sparkcontext
 
     def getAsJSONList(self):
         return [json.loads(l.serializeAsJSON()) for l in self._jsequence.getAsList()]
 
+    def getAsJSONRDD(self):
+        rdd = self._jsequence.getAsStringRDD();
+        print("Strings:");
+        for s in rdd.take(10):
+            print(s);
+        rdd = RDD(rdd, self._sparkcontext)
+        return rdd.map(lambda l: json.loads(l))
+
     def nextJSON(self):
         return self._jsequence.next().serializeAsJSON()
 

src/jsoniq/sequence.py

@@ -1,57 +1,53 @@
 from pyspark.sql import SparkSession
 from .sequence import SequenceOfItems
+import importlib.resources as pkg_resources
+
+with pkg_resources.path("jsoniq.jars", "rumbledb-1.24.0.jar") as jar_path:
+    print("Found Rumble JAR at:", jar_path)
+    jar_path_str = "file://" + str(jar_path)
 
 class MetaRumbleSession(type):
     def __getattr__(cls, item):
-        print(f"Dynamically handled: {item}")
         if item == "builder":
             return cls._builder
         else:
             return getattr(SparkSession, item)
 
 class RumbleSession(object, metaclass=MetaRumbleSession):
     def __init__(self, spark_session: SparkSession):
-        print("Initializing RumbleSession object");
         self._sparksession = spark_session
         self._jrumblesession = spark_session._jvm.org.rumbledb.api.Rumble(spark_session._jsparkSession)
 
     class Builder:
         def __init__(self):
-            self._sparkbuilder = SparkSession.builder.config("spark.jars", "file:///Users/ghislain/Code/rumble/target/rumbledb-1.24.0-jar-with-dependencies.jar")
+            self._sparkbuilder = SparkSession.builder.config("spark.jars", jar_path_str)
 
         def getOrCreate(self):
-            print("getOrCreate called");
             return RumbleSession(self._sparkbuilder.getOrCreate())
 
         def appName(self, name):
-            print(f"Setting app name: {name}");
             self._sparkbuilder = self._sparkbuilder.appName(name);
             return self;
 
         def master(self, url):
-            print(f"Setting master URL: {url}");
             self._sparkbuilder = self._sparkbuilder.master(url);
             return self;
 
         def config(self, key, value):
-            print(f"Setting config: {key} = {value}");
             self._sparkbuilder = self._sparkbuilder.config(key, value);   
             return self;
 
         def config(self, conf):
-            print(f"Setting config: {conf}");
             self._sparkbuilder = self._sparkbuilder.config(conf);   
             return self;
 
         def __getattr__(self, name):
-            print(f"Calling attribute: {name}");
             res = getattr(self._sparkbuilder, name);
             return res;
 
     _builder = Builder()
 
     def bindDataFrameAsVariable(self, name: str, df):
-        print(f"Binding DataFrame as variable: {name}");
         conf = self._jrumblesession.getConfiguration();
         if not name.startswith("$"):
             raise ValueError("Variable name must start with a dollar symbol ('$').")
@@ -61,8 +57,7 @@ def bindDataFrameAsVariable(self, name: str, df):
 
     def jsoniq(self, str):
         sequence = self._jrumblesession.runQuery(str);
-        return SequenceOfItems(sequence);
+        return SequenceOfItems(sequence, self._sparksession.sparkContext);
 
     def __getattr__(self, item):
-        print(f"Accessing attribute: {item}")
         return getattr(self._sparksession, item)