Skip to content

Commit dc03b1c

Browse files
roger-zhanggseshubaws
roger-zhangg
and
seshubaws
authored
integ-test: skip LMI test and improve test quality (#8554)
* fix integ tests * fix * nit --------- Co-authored-by: seshubaws <116689586+seshubaws@users.noreply.github.com>
1 parent 21e6780 commit dc03b1c

13 files changed

Lines changed: 151 additions & 56 deletions

File tree

‎.github/workflows/integration-tests.yml‎

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ env:
2424
CARGO_LAMBDA_VERSION: "v0.17.1"
2525
NOSE_PARAMETERIZED_NO_WARN: 1
2626
BY_CANARY: true
27-
UV_PYTHON: python3.9
27+
UV_PYTHON: python3.11
2828
CREDENTIAL_DISTRIBUTION_LAMBDA_ARN: ${{ secrets.CREDENTIAL_DISTRIBUTION_LAMBDA_ARN }}
2929
ACCOUNT_RESET_LAMBDA_ARN: ${{ secrets.ACCOUNT_RESET_LAMBDA_ARN }}
3030

@@ -297,33 +297,52 @@ jobs:
297297
- name: Get testing resources and credentials
298298
run: |
299299
# Try with skip_role_deletion parameter first
300-
test_env_var=$(python3.9 tests/get_testing_resources.py skip_role_deletion)
300+
test_env_var=$(python3.11 tests/get_testing_resources.py skip_role_deletion)
301301
302302
if [ $? -ne 0 ]; then
303303
echo "First attempt with skip_role_deletion failed, trying without parameter..."
304-
test_env_var=$(python3.9 tests/get_testing_resources.py)
304+
test_env_var=$(python3.11 tests/get_testing_resources.py)
305305
306306
if [ $? -ne 0 ]; then
307307
echo "get_testing_resources failed. Failed to acquire credentials or test resources."
308308
exit 1
309309
fi
310310
fi
311311
312-
# Save current credentials for account reset later
312+
# Save current credentials for account reset later (mask them first)
313+
echo "::add-mask::$AWS_ACCESS_KEY_ID"
314+
echo "::add-mask::$AWS_SECRET_ACCESS_KEY"
315+
echo "::add-mask::$AWS_SESSION_TOKEN"
313316
echo "CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV
314317
echo "CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV
315318
echo "CI_ACCESS_ROLE_AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> $GITHUB_ENV
316319
317-
# Set test credentials
318-
echo "AWS_ACCESS_KEY_ID=$(echo "$test_env_var" | jq -j ".accessKeyID")" >> $GITHUB_ENV
319-
echo "AWS_SECRET_ACCESS_KEY=$(echo "$test_env_var" | jq -j ".secretAccessKey")" >> $GITHUB_ENV
320-
echo "AWS_SESSION_TOKEN=$(echo "$test_env_var" | jq -j ".sessionToken")" >> $GITHUB_ENV
321-
echo "TASK_TOKEN=$(echo "$test_env_var" | jq -j ".taskToken")" >> $GITHUB_ENV
320+
# Extract test credentials and mask them before setting as env vars
321+
TEST_ACCESS_KEY_ID=$(echo "$test_env_var" | jq -j ".accessKeyID")
322+
TEST_SECRET_ACCESS_KEY=$(echo "$test_env_var" | jq -j ".secretAccessKey")
323+
TEST_SESSION_TOKEN=$(echo "$test_env_var" | jq -j ".sessionToken")
324+
TEST_TASK_TOKEN=$(echo "$test_env_var" | jq -j ".taskToken")
325+
326+
# Mask sensitive credentials so they don't appear in logs
327+
echo "::add-mask::$TEST_ACCESS_KEY_ID"
328+
echo "::add-mask::$TEST_SECRET_ACCESS_KEY"
329+
echo "::add-mask::$TEST_SESSION_TOKEN"
330+
echo "::add-mask::$TEST_TASK_TOKEN"
331+
332+
# Set test credentials as environment variables
333+
echo "AWS_ACCESS_KEY_ID=$TEST_ACCESS_KEY_ID" >> $GITHUB_ENV
334+
echo "AWS_SECRET_ACCESS_KEY=$TEST_SECRET_ACCESS_KEY" >> $GITHUB_ENV
335+
echo "AWS_SESSION_TOKEN=$TEST_SESSION_TOKEN" >> $GITHUB_ENV
336+
echo "TASK_TOKEN=$TEST_TASK_TOKEN" >> $GITHUB_ENV
337+
338+
# Set other test resources (non-sensitive)
322339
echo "AWS_S3_TESTING=$(echo "$test_env_var" | jq -j ".TestBucketName")" >> $GITHUB_ENV
323340
echo "AWS_ECR_TESTING=$(echo "$test_env_var" | jq -j ".TestECRURI")" >> $GITHUB_ENV
324341
echo "AWS_KMS_KEY=$(echo "$test_env_var" | jq -j ".TestKMSKeyArn")" >> $GITHUB_ENV
325342
echo "AWS_SIGNING_PROFILE_NAME=$(echo "$test_env_var" | jq -j ".TestSigningProfileName")" >> $GITHUB_ENV
326343
echo "AWS_SIGNING_PROFILE_VERSION_ARN=$(echo "$test_env_var" | jq -j ".TestSigningProfileARN")" >> $GITHUB_ENV
344+
echo "LMI_SUBNET_ID=$(echo "$test_env_var" | jq -j ".LMISubnetId")" >> $GITHUB_ENV
345+
echo "LMI_SECURITY_GROUP_ID=$(echo "$test_env_var" | jq -j ".LMISecurityGroupId")" >> $GITHUB_ENV
327346
328347
- name: Login to Public ECR
329348
if: matrix.container_runtime != 'no-container' && env.BY_CANARY == 'true'
@@ -368,13 +387,13 @@ jobs:
368387
pytest -vv -n 2 --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py -m 'java' -k "${CONTAINER_FILTER}" --json-report --json-report-file=TEST_REPORT-integration-buildcmd-arm64-java-${{ matrix.container_runtime }}.json
369388
;;
370389
"terraform-build")
371-
pytest -vv -n 4 --reruns 4 tests/integration/buildcmd/test_build_terraform_applications.py tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-terraform-${{ matrix.container_runtime }}.json
390+
pytest -vv -n 4 --reruns 3 tests/integration/buildcmd/test_build_terraform_applications.py tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-terraform-${{ matrix.container_runtime }}.json
372391
;;
373392
"package-delete-deploy")
374393
pytest -vv tests/integration/package tests/integration/delete tests/integration/deploy --dist=loadgroup -n 6 --reruns 4 --json-report --json-report-file=TEST_REPORT-integration-package-delete-deploy-${{ matrix.container_runtime }}.json
375394
;;
376395
"sync")
377-
pytest -vv tests/integration/sync -n 6 --reruns 3 --dist loadscope --json-report --json-report-file=TEST_REPORT-integration-sync-${{ matrix.container_runtime }}.json
396+
pytest -vv tests/integration/sync -n 4 --reruns 3 --dist loadscope --json-report --json-report-file=TEST_REPORT-integration-sync-${{ matrix.container_runtime }}.json
378397
;;
379398
"local-invoke")
380399
pytest -vv --reruns 3 tests/integration/local/invoke tests/integration/local/generate_event --ignore tests/integration/local/invoke/test_invoke_durable.py --json-report --json-report-file=TEST_REPORT-integration-local-invoke-${{ matrix.container_runtime }}.json
@@ -389,8 +408,7 @@ jobs:
389408
pytest -vv --reruns 3 tests/integration/local/invoke/test_invoke_durable.py tests/integration/local/start_api/test_start_api_durable.py tests/integration/local/start_lambda/test_start_lambda_durable.py tests/integration/local/callback/test_callback.py tests/integration/local/execution/test_execution.py --json-report --json-report-file=TEST_REPORT-integration-durable-functions-${{ matrix.container_runtime }}.json
390409
;;
391410
"other-and-e2e")
392-
pytest -vv -n 4 --reruns 4 --dist loadgroup tests/integration tests/end_to_end --ignore=tests/integration/buildcmd --ignore=tests/integration/delete --ignore=tests/integration/deploy --ignore=tests/integration/package --ignore=tests/integration/sync --ignore=tests/integration/local --json-report --json-report-file=TEST_REPORT-integration-others-${{ matrix.container_runtime }}.json
393-
pytest -vv --reruns 3 tests/regression --json-report --json-report-file=TEST_REPORT-regression-${{ matrix.container_runtime }}.json
411+
pytest -vv -n 4 --reruns 3 --dist loadgroup tests/integration tests/end_to_end tests/regression --ignore=tests/integration/buildcmd --ignore=tests/integration/delete --ignore=tests/integration/deploy --ignore=tests/integration/package --ignore=tests/integration/sync --ignore=tests/integration/local --json-report --json-report-file=TEST_REPORT-integration-others-${{ matrix.container_runtime }}.json
394412
;;
395413
esac
396414

‎pytest.ini‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ filterwarnings =
1616
; Ignore dateparser deprecation warnings about parsing dates without years (Python 3.13+)
1717
ignore:.*Parsing dates involving a day of month without a year specified.*:DeprecationWarning
1818
; The following deprecation warnings are treated as failures unless we explicitly tell pytest not to
19-
; Remove once we no longer support python3.7
19+
; Remove once we no longer support python3.9
2020
ignore::boto3.exceptions.PythonDeprecationWarning
2121
markers =
2222
ruby

‎tests/integration/deploy/test_deploy_command.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,13 @@
1212
from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME, SamConfig
1313
from samcli.local.docker.utils import get_validated_container_client
1414
from tests.integration.deploy.deploy_integ_base import DeployIntegBase
15-
from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY, UpdatableSARTemplate
15+
from tests.testing_utils import (
16+
RUNNING_ON_CI,
17+
RUNNING_TEST_FOR_MASTER_ON_CI,
18+
RUN_BY_CANARY,
19+
SKIP_LMI_TESTS,
20+
UpdatableSARTemplate,
21+
)
1622

1723
# Deploy tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
1824
# This is to restrict package tests to run outside of CI/CD, when the branch is not master or tests are not run by Canary
@@ -1750,17 +1756,15 @@ def test_deploy_with_valid_config_capabilities_string(self, template_file, confi
17501756
deploy_process_execute = self.run_command(deploy_command_list)
17511757
self.assertEqual(deploy_process_execute.process.returncode, 0)
17521758

1753-
@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
1759+
@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
17541760
def test_deploy_lmi_function(self):
17551761
"""Test deployment of LMI (Lambda Managed Infrastructure) functions with capacity providers."""
17561762
# Validate LMI environment variables are set
17571763
lmi_subnet_id = os.environ.get("LMI_SUBNET_ID")
17581764
lmi_security_group_id = os.environ.get("LMI_SECURITY_GROUP_ID")
1759-
lmi_operator_role_arn = os.environ.get("LMI_OPERATOR_ROLE_ARN")
17601765

17611766
self.assertTrue(lmi_subnet_id, "LMI_SUBNET_ID environment variable must be set")
17621767
self.assertTrue(lmi_security_group_id, "LMI_SECURITY_GROUP_ID environment variable must be set")
1763-
self.assertTrue(lmi_operator_role_arn, "LMI_OPERATOR_ROLE_ARN environment variable must be set")
17641768

17651769
template_path = self.test_data_path.joinpath("lmi_function", "template.yaml")
17661770
stack_name = self._method_to_stack_name(self.id())

‎tests/integration/local/callback/test_callback.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,14 @@ def test_callback_already_completed_execution(self, action, operation_name, call
4848
result = run_command(succeed_command)
4949
self.assertEqual(result.process.returncode, 0)
5050

51-
# Wait for process to complete
51+
# Wait for process to complete and close file handles
5252
process.wait(timeout=30)
5353
thread.join(timeout=5)
54+
# Close file handles to prevent ResourceWarning about unclosed files
55+
if process.stdin:
56+
process.stdin.close()
57+
if process.stdout:
58+
process.stdout.close()
5459

5560
# Try to send another callback (should fail)
5661
second_command = self.get_callback_command_list(action, callback_id)

‎tests/integration/local/invoke/invoke_integ_base.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,12 @@ def run_command(self, command_list, env=None, cwd=None):
111111
return stdout, stderr, process.returncode
112112
except TimeoutExpired:
113113
process.kill()
114+
# Close file handles to prevent resource warnings
115+
if process.stdout:
116+
process.stdout.close()
117+
if process.stderr:
118+
process.stderr.close()
119+
process.wait()
114120
raise
115121

116122

‎tests/integration/pipeline/test_bootstrap_command.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -356,9 +356,7 @@ def test_interactive_pipeline_user_only_created_once(self):
356356
"y", # Create resources confirmation
357357
]
358358

359-
bootstrap_process_execute = run_command_with_input(
360-
bootstrap_command_list, ("\n".join(inputs) + "\n").encode()
361-
)
359+
bootstrap_process_execute = run_command_with_inputs(bootstrap_command_list, inputs)
362360

363361
self.assertEqual(bootstrap_process_execute.process.returncode, 0)
364362
stdout = bootstrap_process_execute.stdout.decode()

‎tests/integration/remote/invoke/test_lambda_invoke_response_stream.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import json
22
import os
33
import uuid
4-
from unittest import skipIf
4+
from unittest import skipIf, SkipTest
55

66
from tests.integration.remote.invoke.remote_invoke_integ_base import RemoteInvokeIntegBase
7-
from tests.testing_utils import run_command, RUNNING_ON_CI
7+
from tests.testing_utils import SKIP_LMI_TESTS, run_command, RUNNING_ON_CI
88

99
from pathlib import Path
1010
import pytest
@@ -105,18 +105,19 @@ def test_invoke_different_boto_options(self):
105105
self.assertEqual(response_event_stream, expected_output_result)
106106

107107

108-
@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
108+
@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
109109
class TestInvokeResponseStreamingCapacityProvider(RemoteInvokeIntegBase):
110110
template = Path("template-lambda-response-capacity-provider-stream-fn.yaml")
111111

112112
@classmethod
113113
def setUpClass(cls):
114+
if SKIP_LMI_TESTS:
115+
raise SkipTest("Skip LMI tests when running on canary")
114116
super().setUpClass()
115117
cls.stack_name = f"{cls.__name__}-{uuid.uuid4().hex}"
116118
# LMI is Lambda Managed Instance
117119
assert os.environ.get("LMI_SUBNET_ID"), "LMI_SUBNET_ID environment variable must be set"
118120
assert os.environ.get("LMI_SECURITY_GROUP_ID"), "LMI_SECURITY_GROUP_ID environment variable must be set"
119-
assert os.environ.get("LMI_OPERATOR_ROLE_ARN"), "LMI_OPERATOR_ROLE_ARN environment variable must be set"
120121

121122
# Read LMI infrastructure from environment variables
122123
cls.parameter_overrides = {

‎tests/integration/remote/invoke/test_remote_invoke.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,29 +6,30 @@
66
import os
77

88
from parameterized import parameterized
9-
from unittest import skipIf
9+
from unittest import skipIf, SkipTest
1010

1111
from tests.integration.remote.invoke.remote_invoke_integ_base import RemoteInvokeIntegBase
12-
from tests.testing_utils import run_command, RUNNING_ON_CI
12+
from tests.testing_utils import SKIP_LMI_TESTS, run_command
1313

1414
from pathlib import Path
1515
import pytest
1616

1717
SQS_WAIT_TIME_SECONDS = 20
1818

1919

20-
@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
20+
@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
2121
class TestingInvokeWithCapacityProvider(RemoteInvokeIntegBase):
2222
template = Path("template-single-lambda-capacity-provider.yaml")
2323

2424
@classmethod
2525
def setUpClass(cls):
26+
if SKIP_LMI_TESTS:
27+
raise SkipTest("Skip LMI tests when running on canary")
2628
super().setUpClass()
2729
cls.stack_name = f"{cls.__name__}-{uuid.uuid4().hex}"
2830
# LMI is Lambda Managed Instance
2931
assert os.environ.get("LMI_SUBNET_ID"), "LMI_SUBNET_ID environment variable must be set"
3032
assert os.environ.get("LMI_SECURITY_GROUP_ID"), "LMI_SECURITY_GROUP_ID environment variable must be set"
31-
assert os.environ.get("LMI_OPERATOR_ROLE_ARN"), "LMI_OPERATOR_ROLE_ARN environment variable must be set"
3233

3334
# Read LMI infrastructure from environment variables
3435
cls.parameter_overrides = {

‎tests/integration/sync/test_sync_code.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import uuid
1010
from pathlib import Path
1111
from typing import Dict
12-
from unittest import skipIf
12+
from unittest import skipIf, SkipTest
1313

1414

1515
import pytest
@@ -24,7 +24,7 @@
2424
)
2525
from tests.integration.sync.sync_integ_base import SyncIntegBase
2626

27-
from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
27+
from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY, SKIP_LMI_TESTS
2828
from tests.testing_utils import run_command_with_input
2929

3030
# Deploy tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
@@ -796,14 +796,14 @@ def test_skip_build(self):
796796
self.assertEqual(lambda_response.get("message"), "hello mars")
797797

798798

799-
@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
800799
@parameterized_class(
801800
[
802801
{"dependency_layer": True},
803802
{"dependency_layer": False},
804803
]
805804
)
806805
@pytest.mark.timeout(300)
806+
@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
807807
class TestSyncCodeLMI(TestSyncCodeBase):
808808
"""Test sync code operations with Lambda Managed Instance functions"""
809809

@@ -812,11 +812,12 @@ class TestSyncCodeLMI(TestSyncCodeBase):
812812

813813
@classmethod
814814
def setUpClass(cls) -> None:
815+
if SKIP_LMI_TESTS:
816+
raise SkipTest("Skip LMI tests when running on canary")
815817
super().setUpClass()
816818
# Validate LMI environment variables are set
817819
assert os.environ.get("LMI_SUBNET_ID"), "LMI_SUBNET_ID environment variable must be set"
818820
assert os.environ.get("LMI_SECURITY_GROUP_ID"), "LMI_SECURITY_GROUP_ID environment variable must be set"
819-
assert os.environ.get("LMI_OPERATOR_ROLE_ARN"), "LMI_OPERATOR_ROLE_ARN environment variable must be set"
820821

821822
# Read LMI infrastructure from environment variables
822823
cls.parameter_overrides = {

‎tests/integration/sync/test_sync_infra.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import uuid
88
from pathlib import Path
99
from typing import Dict
10-
from unittest import skipIf
10+
from unittest import skipIf, SkipTest
1111

1212
import pytest
1313
from parameterized import parameterized, parameterized_class
@@ -18,7 +18,7 @@
1818
AWS_STEPFUNCTIONS_STATEMACHINE,
1919
)
2020
from tests.integration.sync.sync_integ_base import SyncIntegBase
21-
from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
21+
from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY, SKIP_LMI_TESTS
2222
from tests.testing_utils import run_command_with_input
2323

2424
# Deploy tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
@@ -587,7 +587,7 @@ def test_sync_infra_esbuild(self, template_file):
587587
self.assertEqual(lambda_response.get("message"), "hello world")
588588

589589

590-
@skipIf(RUNNING_ON_CI, "Skip LMI tests when running on canary")
590+
@skipIf(SKIP_LMI_TESTS, "Skip LMI tests when running on canary")
591591
@parameterized_class(
592592
[
593593
{"dependency_layer": True},
@@ -603,11 +603,12 @@ class TestSyncInfraLMI(SyncIntegBase):
603603

604604
@classmethod
605605
def setUpClass(cls) -> None:
606+
if SKIP_LMI_TESTS:
607+
raise SkipTest("Skip LMI tests when running on canary")
606608
super().setUpClass()
607609
# Validate LMI environment variables are set
608610
assert os.environ.get("LMI_SUBNET_ID"), "LMI_SUBNET_ID environment variable must be set"
609611
assert os.environ.get("LMI_SECURITY_GROUP_ID"), "LMI_SECURITY_GROUP_ID environment variable must be set"
610-
assert os.environ.get("LMI_OPERATOR_ROLE_ARN"), "LMI_OPERATOR_ROLE_ARN environment variable must be set"
611612

612613
# Read LMI infrastructure from environment variables
613614
cls.parameter_overrides = {

0 commit comments

Comments
 (0)