VDK-DuckDB: Introducing a new database plugin (#2561)

<VDK-plugin for the DuckDB Database.>


# Why?
This plugin has been created to check the differences how databases are
implemented in a VDK plugin.
The aim of this change is to add DuckDB to the list of database plugins.

# What?
I made a new DuckDB-plugin based on the SQL-plugin.
For making the DuckDB work I had to change several things.
For the Plugin I had to change: Module Imports, Configuration,
Connection Factory Method, Ingester Method and Library Specific Code.
For the Ingestion I had to change: Import Statements, Configuration and
connection, Target Path, Data Type Conversion, Query compatibility,
General naming and references, Connection Handling.
For the Configuration I had to change: Constants, Class Names, Method
for Database File, Configuration Key, Descriptions.
For the Connection I had to change: Classes, Class documentation, File
Name, Import Statement, Connection Method, Isolation Level.

# How has this been tested?
I've made a test program that first relied on the 'auto create feature'
of the plugin. But after several errors I realized
that probably the auto create function wasn't functioning as expected. 
So now the test doesn't rely on this function and instead doesn't create
the table in advance, disables auto create, checks if the table is
missing
and makes sure the ingestion logic remains.

# What type of change are you making?
This is a new feature.
Now the users can make use of the DuckDB database from the list of
database plugins.

---------

Co-authored-by: Maximiliaan <maxsobry@gmai.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

projects/vdk-plugins/vdk-duckdb/.plugin-ci.yml

@@ -0,0 +1,22 @@
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+image: "python:3.7"
+
+.build-vdk-duckdb:
+  variables:
+    PLUGIN_NAME: vdk-duckdb
+  extends: .build-plugin
+
+build-py37-vdk-duckdb:
+  extends: .build-vdk-duckdb
+  image: "python:3.7"
+
+build-py311-vdk-duckdb:
+  extends: .build-vdk-duckdb
+  image: "python:3.11"
+
+release-vdk-duckdb:
+  variables:
+    PLUGIN_NAME: vdk-duckdb
+  extends: .release-plugin

projects/vdk-plugins/vdk-duckdb/README.md

@@ -0,0 +1,45 @@
+# duckdb
+
+Simple description of my project.
+
+TODO: what the project is about, what is its purpose
+
+
+## Usage
+
+```
+pip install vdk-duckdb
+```
+
+### Configuration
+
+(`vdk config-help` is useful command to browse all config options of your installation of vdk)
+
+| Name | Description | (example)  Value |
+|---|---|---|
+| dummy_config_key | Dummy configuration | "Dummy" |
+
+### Example
+
+TODO
+
+### Build and testing
+
+```
+pip install -r requirements.txt
+pip install -e .
+pytest
+```
+
+In VDK repo [../build-plugin.sh](https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins/build-plugin.sh) script can be used also.
+
+
+#### Note about the CICD:
+
+.plugin-ci.yaml is needed only for plugins part of [Versatile Data Kit Plugin repo](https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins).
+
+The CI/CD is separated in two stages, a build stage and a release stage.
+The build stage is made up of a few jobs, all which inherit from the same
+job configuration and only differ in the Python version they use (3.7, 3.8, 3.9 and 3.10).
+They run according to rules, which are ordered in a way such that changes to a
+plugin's directory trigger the plugin CI, but changes to a different plugin does not.

projects/vdk-plugins/vdk-duckdb/requirements.txt

@@ -0,0 +1,12 @@
+# this file is used to provide testing requirements
+# for requirements (dependencies) needed during and after installation of the plugin see (and update) setup.py install_requires section
+
+click
+duckdb
+pytest
+
+pytest
+pytest-cov
+
+vdk-core
+vdk-test-utils

projects/vdk-plugins/vdk-duckdb/setup.py

@@ -0,0 +1,40 @@
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import pathlib
+
+import setuptools
+
+"""
+Builds a package with the help of setuptools in order for this package to be imported in other projects
+"""
+
+__version__ = "0.1.0"
+
+setuptools.setup(
+    name="vdk-duckdb",
+    version=__version__,
+    url="https://github.com/vmware/versatile-data-kit",
+    description="DuckDB Plugin for VDK.",
+    long_description=pathlib.Path("README.md").read_text(),
+    long_description_content_type="text/markdown",
+    install_requires=["vdk-core", "tabulate"],
+    package_dir={"": "src"},
+    packages=setuptools.find_namespace_packages(where="src"),
+    # This is the only vdk plugin specific part
+    # Define entry point called "vdk.plugin.run" with name of plugin and module to act as entry point.
+    entry_points={"vdk.plugin.run": ["vdk-duckdb = vdk.plugin.duckdb.duckdb_plugin"]},
+    classifiers=[
+        "Development Status :: 2 - Pre-Alpha",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+    ],
+    project_urls={
+        "Documentation": "https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins/vdk-duckdb",
+        "Source Code": "https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins/vdk-duckdb",
+        "Bug Tracker": "https://github.com/vmware/versatile-data-kit/issues/new/choose",
+    },
+)

projects/vdk-plugins/vdk-duckdb/src/vdk/plugin/duckdb/duckdb_configuration.py

@@ -0,0 +1,38 @@
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import pathlib
+import tempfile
+
+from vdk.internal.core.config import Configuration
+from vdk.internal.core.config import ConfigurationBuilder
+
+DUCKDB_FILE = "DUCKDB_FILE"
+DUCKDB_INGEST_AUTO_CREATE_TABLE_ENABLED = "DUCKDB_INGEST_AUTO_CREATE_TABLE_ENABLED"
+
+
+class DuckDBConfiguration:
+    def __init__(self, configuration: Configuration):
+        self.__config = configuration
+
+    def get_auto_create_table_enabled(self) -> bool:
+        return self.__config.get_value(DUCKDB_INGEST_AUTO_CREATE_TABLE_ENABLED)
+
+    def get_duckdb_file(self):
+        duckdb_file_path = self.__config.get_value(DUCKDB_FILE) or "default_path.duckdb"
+        return pathlib.Path(duckdb_file_path)
+
+
+def add_definitions(config_builder: ConfigurationBuilder):
+    config_builder.add(
+        key=DUCKDB_FILE,
+        default_value=str(
+            pathlib.Path(tempfile.gettempdir()).joinpath("vdk-duckdb.db")
+        ),
+        description="The file of the DuckDB database.",
+    )
+    config_builder.add(
+        key=DUCKDB_INGEST_AUTO_CREATE_TABLE_ENABLED,
+        default_value=True,
+        description="If set to true, auto create table if it does not exist during ingestion."
+        "This is only applicable when ingesting data into DuckDB (ingest method is DuckDB).",
+    )

projects/vdk-plugins/vdk-duckdb/src/vdk/plugin/duckdb/duckdb_connection.py

@@ -0,0 +1,37 @@
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import pathlib
+import tempfile
+from typing import List
+
+import duckdb
+from vdk.internal.util.decorators import closing_noexcept_on_close
+
+log = logging.getLogger(__name__)
+
+
+class DuckDBConnection:
+    """
+    Create file based DuckDB database.
+    """
+
+    def __init__(
+        self,
+        duckdb_file: pathlib.Path = pathlib.Path(tempfile.gettempdir()).joinpath(
+            "vdk-duckdb.db"
+        ),
+    ):
+        self.__db_file = duckdb_file
+
+    def new_connection(self):
+        log.info(
+            f"Creating new connection against local file database located at: {self.__db_file}"
+        )
+        return duckdb.connect(f"{self.__db_file}")
+
+    def execute_query(self, query: str) -> List[List]:
+        conn = self.new_connection()
+        with closing_noexcept_on_close(conn.cursor()) as cursor:
+            cursor.execute(query)
+            return cursor.fetchall()

projects/vdk-plugins/vdk-duckdb/src/vdk/plugin/duckdb/duckdb_plugin.py

@@ -0,0 +1,61 @@
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import pathlib
+
+import click
+import duckdb
+from tabulate import tabulate
+from vdk.api.plugin.hook_markers import hookimpl
+from vdk.internal.builtin_plugins.run.job_context import JobContext
+from vdk.internal.core.config import ConfigurationBuilder
+from vdk.internal.util.decorators import closing_noexcept_on_close
+
+log = logging.getLogger(__name__)
+"""
+Include the plugins implementation. For example:
+"""
+
+
+@hookimpl
+def vdk_configure(config_builder: ConfigurationBuilder) -> None:
+    """Define the configuration settings needed for duckdb"""
+    config_builder.add("DUCKDB_FILE", default_value="mydb.duckdb")
+
+
+@hookimpl
+def initialize_job(context: JobContext) -> None:
+    conf = context.core_context.configuration
+    duckdb_file = conf.get_value("DUCKDB_FILE")
+
+    context.connections.add_open_connection_factory_method(
+        "DUCKDB", lambda: duckdb.connect(database=duckdb_file)
+    )
+
+
+@click.command(
+    name="duckdb-query", help="Execute a DuckDB query against a local DUCKDB database."
+)
+@click.option("-q", "--query", type=click.STRING, required=True)
+@click.pass_context
+def duckdb_query(ctx: click.Context, query):
+    conf = ctx.obj.configuration
+    duckdb_file = conf.get_value("DUCKDB_FILE")
+    conn = duckdb.connect(database=duckdb_file)
+
+    with closing_noexcept_on_close(conn.cursor()) as cursor:
+        cursor.execute(query)
+        column_names = (
+            [column_info[0] for column_info in cursor.description]
+            if cursor.description
+            else ()  # same as the default value for the headers parameters of the tabulate function
+        )
+        res = cursor.fetchall()
+        click.echo(tabulate(res, headers=column_names))
+
+
+@hookimpl
+def vdk_command_line(root_command: click.Group):
+    """Here we extend the vdk with a new command called "duckdb-query"
+    enabling users to execute"""
+    root_command.add_command(duckdb_query)