diff --git a/airbyte_cdk/utils/docker.py b/airbyte_cdk/utils/docker.py index d8654c39c..5516aa720 100644 --- a/airbyte_cdk/utils/docker.py +++ b/airbyte_cdk/utils/docker.py @@ -16,6 +16,7 @@ from airbyte_cdk.models.connector_metadata import ConnectorLanguage, MetadataFile from airbyte_cdk.utils.docker_image_templates import ( DOCKERIGNORE_TEMPLATE, + JAVA_CONNECTOR_DOCKERFILE_TEMPLATE, MANIFEST_ONLY_DOCKERFILE_TEMPLATE, PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE, ) @@ -197,6 +198,22 @@ def build_connector_image( base_tag = f"{metadata.data.dockerRepository}:{tag}" arch_images: list[str] = [] + + if metadata.data.language == ConnectorLanguage.JAVA: + # This assumes that the repo root ('airbyte') is three levels above the + # connector directory (airbyte/airbyte-integrations/connectors/source-foo). + repo_root = connector_directory.parent.parent.parent + # For Java connectors, we need to build the connector tar file first. + subprocess.run( + [ + "./gradlew", + f":airbyte-integrations:connectors:{connector_name}:distTar", + ], + cwd=repo_root, + text=True, + check=True, + ) + for arch in [ArchEnum.AMD64, ArchEnum.ARM64]: docker_tag = f"{base_tag}-{arch.value}" docker_tag_parts = docker_tag.split("/") @@ -248,10 +265,7 @@ def get_dockerfile_template( return MANIFEST_ONLY_DOCKERFILE_TEMPLATE if metadata.data.language == ConnectorLanguage.JAVA: - raise ValueError( - f"Java and Kotlin connectors are not yet supported. " - "Please use airbyte-ci or gradle to build your image." - ) + return JAVA_CONNECTOR_DOCKERFILE_TEMPLATE raise ValueError( f"Unsupported connector language: {metadata.data.language}. " @@ -322,10 +336,20 @@ def verify_connector_image( ) # check that the output is valid JSON if result.stdout: - try: - json.loads(result.stdout) - except json.JSONDecodeError: - logger.error("Invalid JSON output from spec command.") + found_spec_output = False + for line in result.stdout.split("\n"): + if line.strip(): + try: + # Check if the line is a valid JSON object + msg = json.loads(line) + if isinstance(msg, dict) and "type" in msg and msg["type"] == "SPEC": + found_spec_output = True + + except json.JSONDecodeError as e: + logger.warning(f"Invalid JSON output from spec command: {e}: {line}") + + if not found_spec_output: + logger.error("No valid JSON output found for spec command.") return False else: logger.error("No output from spec command.") diff --git a/airbyte_cdk/utils/docker_image_templates.py b/airbyte_cdk/utils/docker_image_templates.py index 17fff1e56..761b9adeb 100644 --- a/airbyte_cdk/utils/docker_image_templates.py +++ b/airbyte_cdk/utils/docker_image_templates.py @@ -14,8 +14,9 @@ DOCKERIGNORE_TEMPLATE: str = "\n".join( [ "# This file is auto-generated. Do not edit.", - # "*," + "*," # Ignore everything not explicitly allowed below "build/", + "!build/distributions/*.tar", ".venv/", "secrets/", "!setup.py", @@ -36,7 +37,7 @@ # PYTHON CONNECTOR IMAGE ## ########################### -PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE = """ +PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE = r""" # syntax=docker/dockerfile:1 # check=skip=all ARG BASE_IMAGE @@ -99,3 +100,37 @@ ENV AIRBYTE_ENTRYPOINT="python ./main.py" ENTRYPOINT ["python", "./main.py"] """ + +######################### +# JAVA CONNECTOR IMAGE ## +######################### + +JAVA_CONNECTOR_DOCKERFILE_TEMPLATE = """ +# Java connector Dockerfile for Airbyte + +# Build arguments +ARG BASE_IMAGE + +# Base image - using Amazon Corretto (Amazon's distribution of OpenJDK) +FROM ${BASE_IMAGE} +ARG CONNECTOR_KEBAB_NAME + +# Set permissions for downloaded files +RUN chmod +x /airbyte/base.sh /airbyte/javabase.sh && \ + chown airbyte:airbyte /airbyte/base.sh /airbyte/javabase.sh /airbyte/dd-java-agent.jar + +ENV AIRBYTE_ENTRYPOINT="/airbyte/base.sh" +ENV APPLICATION="${CONNECTOR_KEBAB_NAME}" + +# Add the connector TAR file and extract it +COPY ./build/distributions/${CONNECTOR_KEBAB_NAME}.tar /tmp/${CONNECTOR_KEBAB_NAME}.tar +RUN tar xf /tmp/${CONNECTOR_KEBAB_NAME}.tar --strip-components=1 -C /airbyte && \ + rm -rf /tmp/${CONNECTOR_KEBAB_NAME}.tar && \ + chown -R airbyte:airbyte /airbyte + +# Set the non-root user +USER airbyte + +# Set entrypoint +ENTRYPOINT ["/airbyte/base.sh"] +"""