From c29a1ac6f4c9a9eca13b1d5aff482c0cfe9ba9a2 Mon Sep 17 00:00:00 2001 From: Augusto Hidalgo Date: Mon, 27 Apr 2026 08:50:09 +0000 Subject: [PATCH 01/13] Bump requirements in e2e test Change-Id: Id1ab768dfaabb7868713ac238c8840406e2c98c0 GitOrigin-RevId: daf3412c40c0d2509e543470f02608b6c4761fe3 --- tests/e2e/incompatible_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/incompatible_requirements.txt b/tests/e2e/incompatible_requirements.txt index b1d81767..f6cfce96 100644 --- a/tests/e2e/incompatible_requirements.txt +++ b/tests/e2e/incompatible_requirements.txt @@ -1,4 +1,4 @@ protobuf==5.29.6 google-cloud-dlp==3.9.2 -apache-airflow-providers-google==8.6.0 +apache-airflow-providers-google==8.10.0 proto-plus==1.20.5 \ No newline at end of file From cdfb2ba3b3a762aa2a29bb85012d66298181d0db Mon Sep 17 00:00:00 2001 From: Mykhailo Alipa Date: Fri, 10 May 2024 16:18:43 +0200 Subject: [PATCH 02/13] WIP --- composer_local_dev/cli.py | 25 +++++++++- composer_local_dev/docker_files/entrypoint.sh | 12 +++++ composer_local_dev/environment.py | 49 +++++++++++++++++-- tests/unit/test_environment.py | 14 ++++++ 4 files changed, 94 insertions(+), 6 deletions(-) diff --git a/composer_local_dev/cli.py b/composer_local_dev/cli.py index bc8cbda9..393404ff 100644 --- a/composer_local_dev/cli.py +++ b/composer_local_dev/cli.py @@ -48,7 +48,7 @@ }, { "name": "Environment options", - "options": ["--web-server-port", "--dags-path", "--plugins-path"], + "options": ["--web-server-port", "--dags-path", "--plugins-path", "--enable-ssh", "--ssh-port"], }, { "name": "Container Memory and CPUs limit", @@ -178,6 +178,21 @@ def cli(): metavar="PORT", ) +option_enable_ssh = click.option( + "--enable-ssh", + is_flag=True, + default=False, + help="Enable SSH daemon in the environment.", + metavar="ENABLE_SSH", +) + +option_ssh_port = click.option( + "--ssh-port", + type=click.IntRange(min=0, max=65535), + help="Port used by SSH daemon", + show_default="read from the configuration file", + metavar="SSHD_PORT", +) def _complete_environment(ctx, param, incomplete): env_dirs = files.get_environment_directories() @@ -246,6 +261,8 @@ def _complete_environment(ctx, param, incomplete): ) @option_location @option_port +@option_enable_ssh +@option_ssh_port @click.option( "--dags-path", help="Path to DAGs folder. If it does not exist, it will be created.", @@ -279,6 +296,8 @@ def create( project: Optional[str], location: str, web_server_port: Optional[int], + enable_ssh: Optional[bool], + ssh_port: Optional[int], environment: str, verbose: bool, debug: bool, @@ -339,6 +358,8 @@ def create( location=location, env_dir_path=env_dir, web_server_port=web_server_port, + enable_ssh=enable_ssh, + ssh_port=ssh_port, dags_path=dags_path, plugins_path=plugins_path, database_engine=database_engine, @@ -352,6 +373,8 @@ def create( location=location, env_dir_path=env_dir, port=web_server_port, + enable_ssh=enable_ssh, + ssh_port=ssh_port, dags_path=dags_path, plugins_path=plugins_path, database_engine=database_engine, diff --git a/composer_local_dev/docker_files/entrypoint.sh b/composer_local_dev/docker_files/entrypoint.sh index 66172407..d5cc5653 100755 --- a/composer_local_dev/docker_files/entrypoint.sh +++ b/composer_local_dev/docker_files/entrypoint.sh @@ -102,6 +102,14 @@ create_user() { fi } +install_and_run_sshd() { + echo "Installing and running sshd" + sudo apt-get -qq update && sudo DEBIAN_FRONTEND=noninteractive apt-get -qqy install openssh-server > /dev/null 2>&1 + sudo mkdir /run/sshd + echo "airflow:${COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD}" | sudo chpasswd + sudo /usr/sbin/sshd +} + main() { sudo chown airflow:airflow airflow sudo chmod +x $run_as_user @@ -116,6 +124,10 @@ main() { sudo chown -R airflow:airflow "$FAST_API_DIR" fi + if [ "${COMPOSER_CONTAINER_ENABLE_SSHD}" = "True" ]; then + install_and_run_sshd + fi + if [ "${COMPOSER_CONTAINER_RUN_AS_HOST_USER}" = "True" ]; then create_user "${COMPOSER_HOST_USER_NAME}" "${COMPOSER_HOST_USER_ID}" || true echo "Running Airflow as user ${COMPOSER_HOST_USER_NAME}(${COMPOSER_HOST_USER_ID})" diff --git a/composer_local_dev/environment.py b/composer_local_dev/environment.py index 6326d29a..a9316484 100644 --- a/composer_local_dev/environment.py +++ b/composer_local_dev/environment.py @@ -358,7 +358,12 @@ def get_environments_status( class EnvironmentConfig: - def __init__(self, env_dir_path: pathlib.Path, port: Optional[int]): + def __init__( + self, + env_dir_path: pathlib.Path, + port: Optional[int], + ssh_port: Optional[int] = None, + ): self.env_dir_path = env_dir_path self.config = self.load_configuration_from_file() self.project_id = self.get_str_param("composer_project_id") @@ -382,6 +387,12 @@ def __init__(self, env_dir_path: pathlib.Path, port: Optional[int]): else self.parse_int_param("port", allowed_range=(0, 65536)) ) self.database_engine = self.get_str_param("database_engine") + self.enable_ssh = self.get_str_param("enable_ssh") + self.ssh_port = ( + ssh_port + if ssh_port is not None + else self.parse_int_param("ssh_port", allowed_range=(0, 65536)) + ) def load_configuration_from_file(self) -> Dict: """ @@ -462,6 +473,8 @@ def __init__( memory_limit: Optional[str] = None, cpu_count: Optional[int] = None, port: Optional[int] = None, + enable_ssh: Optional[bool] = False, + ssh_port: Optional[int] = None, pypi_packages: Optional[Dict] = None, environment_vars: Optional[Dict] = None, ): @@ -494,6 +507,8 @@ def __init__( self.database_engine == constants.DatabaseEngine.sqlite3 ) self.port: int = port if port is not None else 8080 + self.enable_ssh: bool = enable_ssh + self.ssh_port: int = ssh_port if ssh_port is not None else 10022 self.pypi_packages = ( pypi_packages if pypi_packages is not None else dict() ) @@ -548,9 +563,14 @@ def assert_valid_environment_configuration( ) @classmethod - def load_from_config(cls, env_dir_path: pathlib.Path, port: Optional[int]): + def load_from_config( + cls, + env_dir_path: pathlib.Path, + port: Optional[int], + ssh_port: Optional[int] = None, + ): """Create local environment using 'config.json' configuration file.""" - config = EnvironmentConfig(env_dir_path, port) + config = EnvironmentConfig(env_dir_path, port, ssh_port) environment_vars = load_environment_variables(env_dir_path) Environment.assert_valid_environment_configuration( config, environment_vars @@ -568,6 +588,8 @@ def load_from_config(cls, env_dir_path: pathlib.Path, port: Optional[int]): database_engine=config.database_engine, memory_limit=config.memory_limit, cpu_count=config.cpu_count, + enable_ssh=config.enable_ssh, + ssh_port=config.ssh_port, environment_vars=environment_vars, ) @@ -584,6 +606,8 @@ def from_source_environment( database_engine: str, memory_limit: Optional[str] = None, cpu_count: Optional[int] = None, + enable_ssh: Optional[bool] = False, + ssh_port: Optional[int] = None, ): """ Create Environment using configuration retrieved from Composer @@ -608,6 +632,8 @@ def from_source_environment( plugins_path=plugins_path, dag_dir_list_interval=10, port=web_server_port, + enable_ssh=enable_ssh, + ssh_port=ssh_port, pypi_packages=pypi_packages, environment_vars=env_variables, database_engine=database_engine, @@ -666,7 +692,7 @@ def get_environment_variables_for_image_version(self) -> Dict: return env_vars def get_default_environment_variables( - self, default_db_variables: Dict[str, str] + self, default_db_variables: Dict[str, str], enable_ssh: bool = False ) -> Dict: """Return environment variables that will be set inside container.""" return { @@ -680,6 +706,8 @@ def get_default_environment_variables( # By default, the container runs as the user `airflow` with UID 999. Set # this env variable to "True" to make it run as the current host user. "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", + "COMPOSER_CONTAINER_ENABLE_SSHD": str(enable_ssh), + "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", "COMPOSER_ENVIRONMENT": self.name, @@ -725,6 +753,8 @@ def write_environment_config_to_config_file(self): "database_engine": self.database_engine, "memory_limit": self.container_memory_limit, "cpu_count": self.container_cpu_count, + "enable_ssh": bool(self.enable_ssh), + "ssh_port": int(self.ssh_port), } with open(self.env_dir_path / "config.json", "w") as fp: json.dump(config, fp, indent=4) @@ -803,9 +833,13 @@ def create_docker_container(self): self.requirements_file, db_mounts, ) + db_vars = db_extras["env_vars"] - default_vars = self.get_default_environment_variables(db_vars) + default_vars = self.get_default_environment_variables(db_vars, self.enable_ssh) + env_vars = {**default_vars, **self.environment_vars} + + if ( platform.system() == "Windows" and env_vars["COMPOSER_CONTAINER_RUN_AS_HOST_USER"] == "True" @@ -817,6 +851,7 @@ def create_docker_container(self): ports = { f"8080/tcp": self.port, } + entrypoint = f"bash {constants.ENTRYPOINT_PATH}" memory_limit = ( self.container_memory_limit @@ -825,6 +860,10 @@ def create_docker_container(self): cpu_count = ( self.container_cpu_count or constants.DOCKER_CONTAINER_CPU_COUNT ) + + if env_vars["COMPOSER_CONTAINER_ENABLE_SSHD"] == "True": + ports[f"22/tcp"] = self.ssh_port + try: container = self.create_container( image=self.image_tag, diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py index ea771d8b..19179eb6 100644 --- a/tests/unit/test_environment.py +++ b/tests/unit/test_environment.py @@ -483,6 +483,8 @@ def test_from_image( @pytest.mark.parametrize( "database_engine", constants.DatabaseEngine.choices() ) + @pytest.mark.parametrize("ssh_port", [None, 2222]) + @pytest.mark.parametrize("enable_ssh", [False, True]) @mock.patch("composer_local_dev.environment.docker.from_env") @mock.patch("composer_local_dev.environment.assert_image_exists") def test_create_and_load_from_config( @@ -492,6 +494,8 @@ def test_create_and_load_from_config( pypi_packages, database_engine, port, + enable_ssh, + ssh_port, tmp_path, ): env_dir_path = tmp_path / ".compose" / "my_env" @@ -504,6 +508,8 @@ def test_create_and_load_from_config( dags_path=str(pathlib.Path(tmp_path)), dag_dir_list_interval=10, port=port, + enable_ssh=enable_ssh, + ssh_port=ssh_port, pypi_packages=pypi_packages, database_engine=database_engine, ) @@ -702,7 +708,9 @@ def test_create_docker_container( "DAGS_FOLDER": "/home/airflow/gcs/dags", "COMPOSER_LOCATION": default_env.location, "COMPOSER_ENVIRONMENT": "my_env", + "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", + "COMPOSER_CONTAINER_ENABLE_SSHD": "False", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", "AIRFLOW_HOME": "/home/airflow/airflow", @@ -768,6 +776,8 @@ def test_create_docker_container_with_custom_limits( "COMPOSER_LOCATION": default_env.location, "COMPOSER_ENVIRONMENT": "my_env", "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", + "COMPOSER_CONTAINER_ENABLE_SSHD": "False", + "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", "AIRFLOW_HOME": "/home/airflow/airflow", @@ -1140,6 +1150,8 @@ def test_get_environment_variables(self, mocked_docker): "COMPOSER_LOCATION": "eu-west", "COMPOSER_ENVIRONMENT": "env_name", "AIRFLOW_HOME": "/home/airflow/airflow", + "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", + "COMPOSER_CONTAINER_ENABLE_SSHD": "False", "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", @@ -1199,6 +1211,8 @@ def test_get_environment_variables_airflow_3(self, mocked_docker): "COMPOSER_LOCATION": "eu-west", "COMPOSER_ENVIRONMENT": "env_name", "AIRFLOW_HOME": "/home/airflow/airflow", + "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", + "COMPOSER_CONTAINER_ENABLE_SSHD": "False", "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", From f6845789225b7261fbaebaf13e0c9d88c86903cb Mon Sep 17 00:00:00 2001 From: Mykhailo Alipa Date: Fri, 10 May 2024 21:00:30 +0200 Subject: [PATCH 03/13] added Remote debugging section to README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 799f5e8d..541f0fbd 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,13 @@ composer-dev create example-local-environment \ --plugins-path example_directory/plugins ``` +## Remote debugging +To facilitate remote debugging in the local Airflow environment, use the `--enable-ssh` flag. +This flag allows SSH access into the container, enabling you to debug your DAGs directly. +For SSH connection, use the `airflow` user. The default password for this user is `airflow`. +If you need to change this default password, set the `COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD` environment variable +in the variables.env file. + ## Enable the container user to access mounted files and directories from the host By default, the Composer container runs as the user `airflow` with UID 999. The user needs to have access the files and From 803c5b85c38c921d86212eb8bb4e2d337a462acd Mon Sep 17 00:00:00 2001 From: Mykhailo Alipa Date: Fri, 17 May 2024 10:38:19 +0200 Subject: [PATCH 04/13] fix: check if sshd is already installed in container --- composer_local_dev/docker_files/entrypoint.sh | 11 +++++++---- composer_local_dev/environment.py | 4 ---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/composer_local_dev/docker_files/entrypoint.sh b/composer_local_dev/docker_files/entrypoint.sh index d5cc5653..e17f8429 100755 --- a/composer_local_dev/docker_files/entrypoint.sh +++ b/composer_local_dev/docker_files/entrypoint.sh @@ -103,10 +103,13 @@ create_user() { } install_and_run_sshd() { - echo "Installing and running sshd" - sudo apt-get -qq update && sudo DEBIAN_FRONTEND=noninteractive apt-get -qqy install openssh-server > /dev/null 2>&1 - sudo mkdir /run/sshd - echo "airflow:${COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD}" | sudo chpasswd + echo "Installing sshd" + if ! command -v /usr/sbin/sshd &> /dev/null + then + sudo apt-get -qq update && sudo DEBIAN_FRONTEND=noninteractive apt-get -qqy install openssh-server > /dev/null 2>&1 + sudo mkdir /run/sshd + echo "airflow:${COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD}" | sudo chpasswd + fi sudo /usr/sbin/sshd } diff --git a/composer_local_dev/environment.py b/composer_local_dev/environment.py index a9316484..cf50d003 100644 --- a/composer_local_dev/environment.py +++ b/composer_local_dev/environment.py @@ -833,13 +833,10 @@ def create_docker_container(self): self.requirements_file, db_mounts, ) - db_vars = db_extras["env_vars"] default_vars = self.get_default_environment_variables(db_vars, self.enable_ssh) env_vars = {**default_vars, **self.environment_vars} - - if ( platform.system() == "Windows" and env_vars["COMPOSER_CONTAINER_RUN_AS_HOST_USER"] == "True" @@ -851,7 +848,6 @@ def create_docker_container(self): ports = { f"8080/tcp": self.port, } - entrypoint = f"bash {constants.ENTRYPOINT_PATH}" memory_limit = ( self.container_memory_limit From cef5220cf85139b4ec0c03281ac77fea95892257 Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Thu, 28 May 2026 17:36:28 +1000 Subject: [PATCH 05/13] fix: ensure openssh-server is installed and starts `install_and_run_sshd` previously failed as the base Airflow images do not include the default ubuntu package repositories. As such, the openssh-server package could not be found. Additionally, the ssh_host_keys generated by sshd are owned by root. This causes permission related failures since the container runs as user airflow. --- composer_local_dev/docker_files/entrypoint.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/composer_local_dev/docker_files/entrypoint.sh b/composer_local_dev/docker_files/entrypoint.sh index e17f8429..0f8e5b4f 100755 --- a/composer_local_dev/docker_files/entrypoint.sh +++ b/composer_local_dev/docker_files/entrypoint.sh @@ -106,7 +106,13 @@ install_and_run_sshd() { echo "Installing sshd" if ! command -v /usr/sbin/sshd &> /dev/null then - sudo apt-get -qq update && sudo DEBIAN_FRONTEND=noninteractive apt-get -qqy install openssh-server > /dev/null 2>&1 + sudo bash -c 'cat << EOF > /etc/apt/sources.list.d/default.list +deb http://archive.ubuntu.com/ubuntu/ noble main restricted +deb http://archive.ubuntu.com/ubuntu/ noble-updates main restricted +EOF' + sudo apt-get -qq update > /dev/null 2>&1 + sudo apt-get -qqy install openssh-server > /dev/null 2>&1 + sudo chown airflow:airflow /etc/ssh/ssh_host_*_key sudo mkdir /run/sshd echo "airflow:${COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD}" | sudo chpasswd fi From 4a132eed3a544f3bac8e76a8ac0069506dd1c416 Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Thu, 4 Jun 2026 08:50:03 +1000 Subject: [PATCH 06/13] Remove redundant f-string --- composer_local_dev/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer_local_dev/environment.py b/composer_local_dev/environment.py index cf50d003..a9347897 100644 --- a/composer_local_dev/environment.py +++ b/composer_local_dev/environment.py @@ -858,7 +858,7 @@ def create_docker_container(self): ) if env_vars["COMPOSER_CONTAINER_ENABLE_SSHD"] == "True": - ports[f"22/tcp"] = self.ssh_port + ports["22/tcp"] = self.ssh_port try: container = self.create_container( From 9f2c450aebce8a68654d5166c78d2b1180a21116 Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Thu, 4 Jun 2026 08:53:08 +1000 Subject: [PATCH 07/13] Improve logging in docker entrypoint --- composer_local_dev/docker_files/entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/composer_local_dev/docker_files/entrypoint.sh b/composer_local_dev/docker_files/entrypoint.sh index 0f8e5b4f..f710a1e7 100755 --- a/composer_local_dev/docker_files/entrypoint.sh +++ b/composer_local_dev/docker_files/entrypoint.sh @@ -103,9 +103,9 @@ create_user() { } install_and_run_sshd() { - echo "Installing sshd" if ! command -v /usr/sbin/sshd &> /dev/null then + echo "Installing sshd" sudo bash -c 'cat << EOF > /etc/apt/sources.list.d/default.list deb http://archive.ubuntu.com/ubuntu/ noble main restricted deb http://archive.ubuntu.com/ubuntu/ noble-updates main restricted @@ -116,6 +116,7 @@ EOF' sudo mkdir /run/sshd echo "airflow:${COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD}" | sudo chpasswd fi + echo "Starting sshd" sudo /usr/sbin/sshd } From 5cbda167a0546d5ff0ffcc9e2a3750b0d63b0b87 Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Thu, 4 Jun 2026 09:08:17 +1000 Subject: [PATCH 08/13] Add constant for default ssh port --- composer_local_dev/constants.py | 2 ++ composer_local_dev/environment.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/composer_local_dev/constants.py b/composer_local_dev/constants.py index e470b078..4fb3e89a 100644 --- a/composer_local_dev/constants.py +++ b/composer_local_dev/constants.py @@ -28,6 +28,8 @@ CONFLICT_ERROR_CODE = 409 SERVER_ERROR_CODE = 500 +DEFAULT_SSH_PORT = 10022 + class ContainerStatus(str, enum.Enum): RUNNING = "running" diff --git a/composer_local_dev/environment.py b/composer_local_dev/environment.py index a9347897..284ea5c8 100644 --- a/composer_local_dev/environment.py +++ b/composer_local_dev/environment.py @@ -508,7 +508,7 @@ def __init__( ) self.port: int = port if port is not None else 8080 self.enable_ssh: bool = enable_ssh - self.ssh_port: int = ssh_port if ssh_port is not None else 10022 + self.ssh_port: int = ssh_port if ssh_port is not None else constants.DEFAULT_SSH_PORT self.pypi_packages = ( pypi_packages if pypi_packages is not None else dict() ) From 320d77bee221e9f12e6d5df422cebba7f178269a Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Thu, 4 Jun 2026 09:59:44 +1000 Subject: [PATCH 09/13] docs: add description for `--ssh-port` option --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 541f0fbd..bd4d3c76 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,9 @@ For SSH connection, use the `airflow` user. The default password for this user i If you need to change this default password, set the `COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD` environment variable in the variables.env file. +Use the `--ssh-port` option to change which host port is mapped to the container's `ssh` port. +The default host port used is 10022. + ## Enable the container user to access mounted files and directories from the host By default, the Composer container runs as the user `airflow` with UID 999. The user needs to have access the files and From 6d33fc5ed303d22d312870b2307fe26bfe9daae7 Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Thu, 4 Jun 2026 10:17:22 +1000 Subject: [PATCH 10/13] fix: prevent failures when ssh opts not in config.json --- composer_local_dev/environment.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/composer_local_dev/environment.py b/composer_local_dev/environment.py index 284ea5c8..af07b2e7 100644 --- a/composer_local_dev/environment.py +++ b/composer_local_dev/environment.py @@ -387,12 +387,19 @@ def __init__( else self.parse_int_param("port", allowed_range=(0, 65536)) ) self.database_engine = self.get_str_param("database_engine") - self.enable_ssh = self.get_str_param("enable_ssh") - self.ssh_port = ( - ssh_port - if ssh_port is not None - else self.parse_int_param("ssh_port", allowed_range=(0, 65536)) - ) + + # Backwards compatibility: don't fail on missing enable_ssh + if "enable_ssh" in self.config: + self.enable_ssh = self.get_str_param("enable_ssh") + self.ssh_port = ( + ssh_port + if ssh_port is not None + else self.parse_int_param("ssh_port", allowed_range=(0, 65536)) + ) + else: + self.enable_ssh = False + self.ssh_port = None + def load_configuration_from_file(self) -> Dict: """ From a2ea5482c06511ef98010fae9691894e14b97f2f Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Thu, 4 Jun 2026 10:50:37 +1000 Subject: [PATCH 11/13] misc: pass `enable_ssh` to EnvironmentConfig Also add these arguments to `start` and `restart` commands --- composer_local_dev/cli.py | 12 ++++++++++-- composer_local_dev/environment.py | 10 ++++++++-- tests/unit/test_cli.py | 8 ++++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/composer_local_dev/cli.py b/composer_local_dev/cli.py index 393404ff..246e3b99 100644 --- a/composer_local_dev/cli.py +++ b/composer_local_dev/cli.py @@ -387,12 +387,16 @@ def create( @cli.command() @optional_environment @option_port +@option_enable_ssh +@option_ssh_port @verbose_mode @debug_mode @errors.catch_exceptions() def start( environment: Optional[str], web_server_port: Optional[int], + enable_ssh: Optional[bool], + ssh_port: Optional[int], verbose: bool, debug: bool, ): @@ -400,7 +404,7 @@ def start( utils.setup_logging(verbose, debug) env_path = files.resolve_environment_path(environment) env = composer_environment.Environment.load_from_config( - env_path, web_server_port + env_path, web_server_port,enable_ssh,ssh_port ) console.get_console().print(f"Starting {env.name} composer environment...") env.start() @@ -427,12 +431,16 @@ def stop(environment: Optional[str], verbose: bool, debug: bool): @cli.command() @optional_environment @option_port +@option_enable_ssh +@option_ssh_port @verbose_mode @debug_mode @errors.catch_exceptions() def restart( environment: Optional[str], web_server_port: Optional[int], + enable_ssh: Optional[bool], + ssh_port: Optional[int], verbose: bool, debug: bool, ): @@ -445,7 +453,7 @@ def restart( utils.setup_logging(verbose, debug) env_path = files.resolve_environment_path(environment) env = composer_environment.Environment.load_from_config( - env_path, web_server_port + env_path, web_server_port,enable_ssh,ssh_port ) env.restart() diff --git a/composer_local_dev/environment.py b/composer_local_dev/environment.py index af07b2e7..f23bba7b 100644 --- a/composer_local_dev/environment.py +++ b/composer_local_dev/environment.py @@ -362,6 +362,7 @@ def __init__( self, env_dir_path: pathlib.Path, port: Optional[int], + enable_ssh: Optional[bool] = None, ssh_port: Optional[int] = None, ): self.env_dir_path = env_dir_path @@ -390,7 +391,11 @@ def __init__( # Backwards compatibility: don't fail on missing enable_ssh if "enable_ssh" in self.config: - self.enable_ssh = self.get_str_param("enable_ssh") + self.enable_ssh = ( + enable_ssh + if enable_ssh is not None + else self.get_str_param("enable_ssh") + ) self.ssh_port = ( ssh_port if ssh_port is not None @@ -574,10 +579,11 @@ def load_from_config( cls, env_dir_path: pathlib.Path, port: Optional[int], + enable_ssh: Optional[bool] = None, ssh_port: Optional[int] = None, ): """Create local environment using 'config.json' configuration file.""" - config = EnvironmentConfig(env_dir_path, port, ssh_port) + config = EnvironmentConfig(env_dir_path, port, enable_ssh, ssh_port) environment_vars = load_environment_variables(env_dir_path) Environment.assert_valid_environment_configuration( config, environment_vars diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index c40fee68..8f3fadb6 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -182,15 +182,15 @@ def mocked_resolve_env(self, env_path): ) as mock_check: yield mock_check - def assert_env_loaded(self, mocked_env, env_path, port=None): - mocked_env.load_from_config.assert_called_with(env_path, port) + def assert_env_loaded(self, mocked_env, env_path, port=None, enable_ssh=False, ssh_port=None): + mocked_env.load_from_config.assert_called_with(env_path, port, enable_ssh, ssh_port) - def assert_run_command(self, command, mocked_env, env_path, port=None): + def assert_run_command(self, command, mocked_env, env_path, port=None, enable_ssh=False, ssh_port=None): run_composer_and_assert_exit_code( command, exit_code=0, ) - self.assert_env_loaded(mocked_env, env_path, port) + self.assert_env_loaded(mocked_env, env_path, port, enable_ssh, ssh_port) @pytest.mark.parametrize("command", ["start", "restart"]) def test_start_command( From 2cdd76d10e25905e6419cb203eecaa6fd74c49a5 Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Tue, 16 Jun 2026 15:15:49 +1000 Subject: [PATCH 12/13] fixup! Add constant for default ssh port --- composer_local_dev/constants.py | 1 + composer_local_dev/environment.py | 29 +++++++++++------------------ tests/unit/test_environment.py | 8 ++++---- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/composer_local_dev/constants.py b/composer_local_dev/constants.py index 4fb3e89a..3ebab7a9 100644 --- a/composer_local_dev/constants.py +++ b/composer_local_dev/constants.py @@ -28,6 +28,7 @@ CONFLICT_ERROR_CODE = 409 SERVER_ERROR_CODE = 500 +DEFAULT_ENABLE_SSH = False DEFAULT_SSH_PORT = 10022 diff --git a/composer_local_dev/environment.py b/composer_local_dev/environment.py index f23bba7b..9c495f3c 100644 --- a/composer_local_dev/environment.py +++ b/composer_local_dev/environment.py @@ -389,21 +389,14 @@ def __init__( ) self.database_engine = self.get_str_param("database_engine") - # Backwards compatibility: don't fail on missing enable_ssh - if "enable_ssh" in self.config: - self.enable_ssh = ( - enable_ssh - if enable_ssh is not None - else self.get_str_param("enable_ssh") - ) - self.ssh_port = ( - ssh_port - if ssh_port is not None - else self.parse_int_param("ssh_port", allowed_range=(0, 65536)) - ) - else: - self.enable_ssh = False - self.ssh_port = None + self.enable_ssh = self.config.get("enable_ssh", constants.DEFAULT_ENABLE_SSH) + self.ssh_port = self.config.get("ssh_port", constants.DEFAULT_SSH_PORT) + + if enable_ssh is not None: + self.enable_ssh = enable_ssh + + if ssh_port is not None: + self.ssh_port = self.parse_int_param("ssh_port", allowed_range=(0, 65535)) def load_configuration_from_file(self) -> Dict: @@ -519,7 +512,7 @@ def __init__( self.database_engine == constants.DatabaseEngine.sqlite3 ) self.port: int = port if port is not None else 8080 - self.enable_ssh: bool = enable_ssh + self.enable_ssh: bool = enable_ssh if enable_ssh is not None else constants.DEFAULT_ENABLE_SSH self.ssh_port: int = ssh_port if ssh_port is not None else constants.DEFAULT_SSH_PORT self.pypi_packages = ( pypi_packages if pypi_packages is not None else dict() @@ -719,7 +712,7 @@ def get_default_environment_variables( # By default, the container runs as the user `airflow` with UID 999. Set # this env variable to "True" to make it run as the current host user. "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", - "COMPOSER_CONTAINER_ENABLE_SSHD": str(enable_ssh), + "COMPOSER_CONTAINER_ENABLE_SSH": str(enable_ssh), "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", @@ -870,7 +863,7 @@ def create_docker_container(self): self.container_cpu_count or constants.DOCKER_CONTAINER_CPU_COUNT ) - if env_vars["COMPOSER_CONTAINER_ENABLE_SSHD"] == "True": + if env_vars["COMPOSER_CONTAINER_ENABLE_SSH"] == "True": ports["22/tcp"] = self.ssh_port try: diff --git a/tests/unit/test_environment.py b/tests/unit/test_environment.py index 19179eb6..b2fc6f57 100644 --- a/tests/unit/test_environment.py +++ b/tests/unit/test_environment.py @@ -710,7 +710,7 @@ def test_create_docker_container( "COMPOSER_ENVIRONMENT": "my_env", "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", - "COMPOSER_CONTAINER_ENABLE_SSHD": "False", + "COMPOSER_CONTAINER_ENABLE_SSH": "False", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", "AIRFLOW_HOME": "/home/airflow/airflow", @@ -776,7 +776,7 @@ def test_create_docker_container_with_custom_limits( "COMPOSER_LOCATION": default_env.location, "COMPOSER_ENVIRONMENT": "my_env", "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", - "COMPOSER_CONTAINER_ENABLE_SSHD": "False", + "COMPOSER_CONTAINER_ENABLE_SSH": "False", "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", @@ -1151,7 +1151,7 @@ def test_get_environment_variables(self, mocked_docker): "COMPOSER_ENVIRONMENT": "env_name", "AIRFLOW_HOME": "/home/airflow/airflow", "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", - "COMPOSER_CONTAINER_ENABLE_SSHD": "False", + "COMPOSER_CONTAINER_ENABLE_SSH": "False", "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", @@ -1212,7 +1212,7 @@ def test_get_environment_variables_airflow_3(self, mocked_docker): "COMPOSER_ENVIRONMENT": "env_name", "AIRFLOW_HOME": "/home/airflow/airflow", "COMPOSER_CONTAINER_AIRFLOW_USER_PASSWORD": "airflow", - "COMPOSER_CONTAINER_ENABLE_SSHD": "False", + "COMPOSER_CONTAINER_ENABLE_SSH": "False", "COMPOSER_CONTAINER_RUN_AS_HOST_USER": "False", "COMPOSER_HOST_USER_NAME": f"{getpass.getuser()}", "COMPOSER_HOST_USER_ID": f"{os.getuid() if platform.system() != 'Windows' else ''}", From 0fa9fc49e05f57b57f573ee3b594893db498fd5b Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Wed, 17 Jun 2026 09:53:09 +1000 Subject: [PATCH 13/13] Add e2e test --- tests/e2e/test_enable_ssh.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tests/e2e/test_enable_ssh.py diff --git a/tests/e2e/test_enable_ssh.py b/tests/e2e/test_enable_ssh.py new file mode 100644 index 00000000..a3535de9 --- /dev/null +++ b/tests/e2e/test_enable_ssh.py @@ -0,0 +1,40 @@ +import pathlib + +import pytest + +from tests.e2e import ( + assert_example_dag_listed, + assert_example_dag_succeeded, + run_app, +) + + +@pytest.mark.e2e +def test_enable_ssh_airflow(composer_image_version, valid_project_id, env_name): + dags_dir = pathlib.Path(__file__).parent / "example_dag" + run_app( + f"create --from-image-version {composer_image_version} " + f"-p {valid_project_id} --dags-path {dags_dir} " + f"--enable-ssh {env_name}" + ) + run_app(f"start {env_name}") + assert_example_dag_listed() + assert_example_dag_succeeded(env_name, airflow_major_version=2) + run_app(f"stop {env_name}") + + +@pytest.mark.e2e +def test_enable_ssh_airflow_3( + composer_image_version_airflow_3, valid_project_id, env_name +): + dags_dir = pathlib.Path(__file__).parent / "example_dag" + run_app( + f"create --from-image-version {composer_image_version_airflow_3} " + f"-p {valid_project_id} --dags-path {dags_dir} " + f"--enable-ssh {env_name}" + ) + # Copy requirements.txt with already satisfied deps to our environment + run_app(f"start {env_name}") + assert_example_dag_listed() + assert_example_dag_succeeded(env_name, airflow_major_version=3) + run_app(f"stop {env_name}")