Merge pull request #79 from FullStackWithLawrence/next

incremental improvements to CI console logging and unit tests

Author: lpm0073

.github/workflows/tests.yml

@@ -30,6 +30,10 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_REGION }}
 
+      - name: Get IAM user info
+        run: |
+          aws sts get-caller-identity
+
       - name: Run Python tests
         uses: ./.github/actions/tests/python
         with:

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
         args: ["--ignore-words=codespell.txt"]
         exclude: 'codespell.txt|\.svg$'
   - repo: https://github.com/psf/black
-    rev: 23.12.0
+    rev: 23.12.1
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/flake8

requirements.txt

@@ -19,7 +19,8 @@ codespell==2.2.6
 
 # project dependencies
 # ------------
-boto3==1.34.7
+boto3==1.34.2
+botocore==1.34.6
 python-dotenv==1.0.0
 pydantic==2.5.3
 pydantic-settings==2.1.0

terraform/python/rekognition_api/__version__.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 # DO NOT EDIT.
 # Managed via automated CI/CD in .github/workflows/semanticVersionBump.yml.
-__version__ = "0.2.11-next.1"
+__version__ = "0.2.12-next.1"

terraform/python/rekognition_api/conf.py

@@ -123,7 +123,9 @@ def to_dict(cls):
         return {
             key: value
             for key, value in Services.__dict__.items()
-            if not key.startswith("__") and not callable(key) and key != "to_dict"
+            if not key.startswith("__")
+            and not callable(key)
+            and key not in ["enabled", "raise_error_on_disabled", "to_dict", "enabled_services"]
         }
 
     @classmethod
@@ -134,7 +136,7 @@ def enabled_services(cls) -> List[str]:
             for key in dir(cls)
             if not key.startswith("__")
             and not callable(getattr(cls, key))
-            and key != "to_dict"
+            and key not in ["enabled", "raise_error_on_disabled", "to_dict", "enabled_services"]
             and getattr(cls, key)[1] is True
         ]
 
@@ -219,27 +221,46 @@ class Settings(BaseSettings):
     _dump: dict = None
     _initialized: bool = False
 
-    # pylint: disable=too-many-branches
-    def __init__(self, **data: Any):
+    # pylint: disable=too-many-branches,too-many-statements
+    def __init__(self, **data: Any):  # noqa: C901
         super().__init__(**data)
         if not Services.enabled(Services.AWS_CLI):
             self._initialized = True
             return
 
         if bool(os.environ.get("AWS_DEPLOYED", False)):
             # If we're running inside AWS Lambda, then we don't need to set the AWS credentials.
+            logger.info("running inside AWS Lambda")
             self._aws_access_key_id_source: str = "overridden by IAM role-based security"
             self._aws_secret_access_key_source: str = "overridden by IAM role-based security"
             self._aws_session = boto3.Session()
             self._initialized = True
 
         if not self.initialized and bool(os.environ.get("GITHUB_ACTIONS", False)):
+            console_handler = logging.StreamHandler()
+            console_handler.setLevel(logging.DEBUG)
+            logger.addHandler(console_handler)
+            logger.setLevel(logging.DEBUG)
+
+            logger.info("running inside GitHub Actions")
+            aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None)
+            aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None)
+            if not aws_access_key_id or not aws_secret_access_key and not self.aws_profile:
+                raise RekognitionConfigurationError(
+                    "required environment variable(s) AWS_ACCESS_KEY_ID and/or AWS_SECRET_ACCESS_KEY not set"
+                )
+            region_name = os.environ.get("AWS_REGION", None)
+            if not region_name and not self.aws_profile:
+                raise RekognitionConfigurationError("required environment variable AWS_REGION not set")
             try:
                 self._aws_session = boto3.Session(
-                    region_name=os.environ.get("AWS_REGION", "us-east-1"),
-                    aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", None),
-                    aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", None),
+                    region_name=region_name,
+                    aws_access_key_id=aws_access_key_id,
+                    aws_secret_access_key=aws_secret_access_key,
                 )
+                self._initialized = True
+                self._aws_access_key_id_source = "environ"
+                self._aws_secret_access_key_source = "environ"
             except ProfileNotFound:
                 logger.warning("aws_profile %s not found", self.aws_profile)
 
@@ -249,6 +270,7 @@ def __init__(self, **data: Any):
             else:
                 self._aws_access_key_id_source = "environ"
                 self._aws_secret_access_key_source = "environ"
+
             self._initialized = True
 
         if not self.initialized:
@@ -351,6 +373,10 @@ def __init__(self, **data: Any):
         pre=True,
         getter=lambda v: empty_str_to_int_default(v, SettingsDefaults.AWS_REKOGNITION_FACE_DETECT_THRESHOLD),
     )
+    init_info: Optional[str] = Field(
+        None,
+        env="INIT_INFO",
+    )
 
     @property
     def initialized(self):
@@ -384,12 +410,15 @@ def aws_secret_access_key_source(self):
     @property
     def aws_auth(self) -> dict:
         """AWS authentication"""
-        return {
+        retval = {
             "aws_profile": self.aws_profile,
             "aws_access_key_id_source": self.aws_access_key_id_source,
             "aws_secret_access_key_source": self.aws_secret_access_key_source,
             "aws_region": self.aws_region,
         }
+        if self.init_info:
+            retval["init_info"] = self.init_info
+        return retval
 
     @property
     def aws_session(self):

terraform/python/rekognition_api/const.py

@@ -24,15 +24,6 @@
 
 logger = logging.getLogger(__name__)
 
-# def read_file_from_github(repo_owner, repo_name, file_path):
-#     """For prod. Read a file from a GitHub repository."""
-#     base_url = "https://raw.githubusercontent.com"
-#     url = f"{base_url}/{repo_owner}/{repo_name}/main/{file_path}"
-#     response = requests.get(url, timeout=5)
-#     if response.status_code == 200:
-#         return response.text
-#     return None
-
 try:
     with open(TERRAFORM_TFVARS, "r", encoding="utf-8") as f:
         TFVARS = hcl2.load(f)

terraform/python/rekognition_api/tests/test_aws.py

@@ -32,15 +32,21 @@ class TestAWSInfrastructture(unittest.TestCase):
     # Get the working directory of this script
     here = os.path.dirname(os.path.abspath(__file__))
 
-    def env_path(self, filename):
-        """Return the path to the .env file."""
-        return os.path.join(self.here, filename)
-
     def setUp(self):
         """Set up test fixtures."""
+        self.saved_env = dict(os.environ)
         env_path = self.env_path(".env")
         load_dotenv(env_path)
 
+    def tearDown(self):
+        # Restore environment variables
+        os.environ.clear()
+        os.environ.update(self.saved_env)
+
+    def env_path(self, filename):
+        """Return the path to the .env file."""
+        return os.path.join(self.here, filename)
+
     def test_rekognition_collection_exists(self):
         """Test that the Rekognition collection exists."""
         if not Services.enabled(Services.AWS_REKOGNITION):

terraform/python/rekognition_api/tests/test_configuration.py

@@ -31,7 +31,13 @@ class TestConfiguration(unittest.TestCase):
     env_vars = dict(os.environ)
 
     def setUp(self):
-        """Set up test fixtures."""
+        # Save current environment variables
+        self.saved_env = dict(os.environ)
+
+    def tearDown(self):
+        # Restore environment variables
+        os.environ.clear()
+        os.environ.update(self.saved_env)
 
     def env_path(self, filename):
         """Return the path to the .env file."""
@@ -40,7 +46,7 @@ def env_path(self, filename):
     def test_conf_defaults(self):
         """Test that settings == SettingsDefaults when no .env is in use."""
         os.environ.clear()
-        mock_settings = Settings()
+        mock_settings = Settings(init_info="test_conf_defaults()")
         os.environ.update(self.env_vars)
 
         self.assertEqual(mock_settings.aws_region, SettingsDefaults.AWS_REGION)
@@ -70,7 +76,7 @@ def test_env_illegal_nulls(self):
         self.assertTrue(loaded)
 
         with self.assertRaises(PydanticValidationError):
-            Settings()
+            Settings(init_info="test_env_illegal_nulls()")
 
         os.environ.update(self.env_vars)
 
@@ -81,7 +87,7 @@ def test_env_nulls(self):
         loaded = load_dotenv(env_path)
         self.assertTrue(loaded)
 
-        mock_settings = Settings()
+        mock_settings = Settings(init_info="test_env_nulls()")
         os.environ.update(self.env_vars)
 
         self.assertEqual(mock_settings.aws_region, SettingsDefaults.AWS_REGION)
@@ -103,7 +109,7 @@ def test_env_overrides(self):
         loaded = load_dotenv(env_path)
         self.assertTrue(loaded)
 
-        mock_settings = Settings()
+        mock_settings = Settings(init_info="test_env_overrides()")
         os.environ.update(self.env_vars)
 
         self.assertEqual(mock_settings.aws_region, "us-west-1")
@@ -122,7 +128,7 @@ def test_env_overrides(self):
     def test_aws_credentials_with_profile(self):
         """Test that key and secret are unset when using profile."""
 
-        mock_settings = Settings()
+        mock_settings = Settings(init_info="test_aws_credentials_with_profile()")
         self.assertEqual(mock_settings.aws_access_key_id_source, "aws_profile")
         self.assertEqual(mock_settings.aws_secret_access_key_source, "aws_profile")
 
@@ -132,7 +138,7 @@ def test_aws_credentials_with_profile(self):
     def test_aws_credentials_without_profile(self):
         """Test that key and secret are set by environment variable when provided."""
 
-        mock_settings = Settings()
+        mock_settings = Settings(init_info="test_aws_credentials_without_profile()")
         # pylint: disable=no-member
         self.assertEqual(mock_settings.aws_access_key_id.get_secret_value(), "TEST_KEY")
         # pylint: disable=no-member
@@ -148,7 +154,7 @@ def test_aws_credentials_without_profile(self):
     def test_aws_credentials_noinfo(self):
         """Test that key and secret remain unset when no profile nor environment variables are provided."""
         os.environ.clear()
-        mock_settings = Settings()
+        mock_settings = Settings(init_info="test_aws_credentials_noinfo()")
         os.environ.update(self.env_vars)
         aws_profile = TFVARS.get("aws_profile", None)
         self.assertEqual(mock_settings.aws_profile, aws_profile)
@@ -169,21 +175,21 @@ def test_invalid_aws_region_code(self):
         """Test that Pydantic raises a validation error for environment variable with non-existent aws region code."""
 
         with self.assertRaises(RekognitionValueError):
-            Settings()
+            Settings(init_info="test_invalid_aws_region_code()")
 
     @patch.dict(os.environ, {"AWS_REKOGNITION_FACE_DETECT_MAX_FACES_COUNT": "-1"})
     def test_invalid_max_faces_count(self):
         """Test that Pydantic raises a validation error for environment variable w negative integer values."""
 
         with self.assertRaises(PydanticValidationError):
-            Settings()
+            Settings(init_info="test_invalid_max_faces_count()")
 
     @patch.dict(os.environ, {"AWS_REKOGNITION_FACE_DETECT_THRESHOLD": "-1"})
     def test_invalid_threshold(self):
         """Test that Pydantic raises a validation error for environment variable w negative integer values."""
 
         with self.assertRaises(PydanticValidationError):
-            Settings()
+            Settings(init_info="test_invalid_threshold()")
 
     def test_configure_with_class_constructor(self):
         """test that we can set values with the class constructor"""
@@ -197,6 +203,7 @@ def test_configure_with_class_constructor(self):
             aws_rekognition_face_detect_quality_filter="TEST_AUTO",
             aws_rekognition_face_detect_threshold=102,
             debug_mode=True,
+            init_info="test_configure_with_class_constructor()",
         )
 
         self.assertEqual(mock_settings.aws_region, "eu-west-1")
@@ -212,15 +219,20 @@ def test_configure_neg_int_with_class_constructor(self):
         """test that we cannot set negative int values with the class constructor"""
 
         with self.assertRaises(PydanticValidationError):
-            Settings(aws_rekognition_face_detect_max_faces_count=-1)
+            Settings(
+                aws_rekognition_face_detect_max_faces_count=-1,
+                init_info="test_configure_neg_int_with_class_constructor()",
+            )
 
         with self.assertRaises(PydanticValidationError):
-            Settings(aws_rekognition_face_detect_threshold=-1)
+            Settings(
+                aws_rekognition_face_detect_threshold=-1, init_info="test_configure_neg_int_with_class_constructor()"
+            )
 
     def test_readonly_settings(self):
         """test that we can't set readonly values with the class constructor"""
 
-        mock_settings = Settings(aws_region="eu-west-1")
+        mock_settings = Settings(aws_region="eu-west-1", init_info="test_readonly_settings()")
         with self.assertRaises(PydanticValidationError):
             mock_settings.aws_region = "us-west-1"
 

terraform/python/rekognition_api/tests/test_configuration_dump.py

@@ -21,16 +21,25 @@
 class TestConfigurationDump(unittest.TestCase):
     """Test configuration."""
 
+    def setUp(self):
+        # Save current environment variables
+        self.saved_env = dict(os.environ)
+
+    def tearDown(self):
+        # Restore environment variables
+        os.environ.clear()
+        os.environ.update(self.saved_env)
+
     def test_dump(self):
         """Test that dump is a dict."""
 
-        mock_settings = Settings(aws_region="us-east-1")
+        mock_settings = Settings(aws_region="us-east-1", init_info="test_dump()")
         self.assertIsInstance(mock_settings.dump, dict)
 
     def test_dump_keys(self):
         """Test that dump contains the expected keys."""
 
-        mock_settings = Settings(aws_region="us-east-1")
+        mock_settings = Settings(aws_region="us-east-1", init_info="test_dump_keys()")
 
         dump = mock_settings.dump
         self.assertIn("environment", dump)
@@ -43,7 +52,7 @@ def test_dump_keys(self):
     def test_dump_values(self):
         """Test that dump contains the expected values."""
 
-        mock_settings = Settings(aws_region="us-east-1")
+        mock_settings = Settings(aws_region="us-east-1", init_info="test_dump_values()")
         environment = mock_settings.dump["environment"]
 
         self.assertEqual(environment["is_using_tfvars_file"], mock_settings.is_using_tfvars_file)

terraform/python/rekognition_api/tests/test_lambda_info.py

@@ -16,9 +16,6 @@
 
 from rekognition_api.lambda_info import lambda_handler  # noqa: E402
 
-# our stuff
-from rekognition_api.tests.test_setup import get_test_file  # noqa: E402
-
 
 class TestLambdaInfo(unittest.TestCase):
     """Test Index Lambda function."""