Add Python 3.7+ compatibility and migrate tests to nose2

.coveragerc

@@ -0,0 +1,14 @@
+[run]
+source =
+    shared
+    commands.prepare
+
+[report]
+fail_under = 60
+show_missing = True
+include =
+    commands/prepare.py
+    shared/*
+
+[html]
+directory = htmlcov

README.md

@@ -49,16 +49,14 @@ If you want to add your own private commands, you can create a `private_commands
 ## Installation
 
 Requirements:
-- python 3 (3.7.0rc1 is known to work), `pip`, and `virtualenv`
-- You will also need `jq` (https://stedolan.github.io/jq/) and the library `pyjq` (https://github.com/doloopwhile/pyjq), which require some additional tools installed that will be shown.
+- python 3 (3.7+ supported), `pip`, and `virtualenv`
+- The project uses the `jq` library for JSON processing, which provides better Python 3 compatibility than the legacy `pyjq`
 
 On macOS:
 
 ```
 # clone the repo
 git clone https://github.com/duo-labs/cloudmapper.git
-# Install pre-reqs for pyjq
-brew install autoconf automake awscli freetype jq libtool python3
 cd cloudmapper/
 python3 -m venv ./venv && source venv/bin/activate
 pip install --prefer-binary -r requirements.txt
@@ -68,11 +66,9 @@ On Linux:
 ```
 # clone the repo
 git clone https://github.com/duo-labs/cloudmapper.git
-# (AWS Linux, Centos, Fedora, RedHat etc.):
-# sudo yum install autoconf automake libtool python3-devel.x86_64 python3-tkinter python-pip jq awscli
-# (Debian, Ubuntu etc.):
-# You may additionally need "build-essential"
-sudo apt-get install autoconf automake libtool python3.7-dev python3-tk jq awscli
+# Install system dependencies (if needed):
+# (RHEL/Fedora): sudo dnf install python3-devel python3-tkinter python-pip jq awscli
+# (Debian/Ubuntu): sudo apt-get install python3-dev python3-tk jq awscli
 cd cloudmapper/
 python3 -m venv ./venv && source venv/bin/activate
 pip install -r requirements.txt

commands/amis.py

@@ -2,7 +2,7 @@
 import sys
 import json
 import argparse
-import pyjq
+import jq as pyjq
 import os.path
 from shared.nodes import Account, Region
 from shared.common import parse_arguments, query_aws

commands/collect.py

@@ -8,7 +8,7 @@
 import time
 import boto3
 import yaml
-import pyjq
+import jq as pyjq
 import urllib.parse
 from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError
 from shared.common import get_account, custom_serializer

commands/prepare.py

@@ -27,7 +27,7 @@
 import operator
 import itertools
 import argparse
-import pyjq
+import jq as pyjq
 import copy
 import urllib.parse
 from netaddr import IPNetwork, IPAddress

commands/sg_ips.py

@@ -1,7 +1,7 @@
 from collections import OrderedDict
 from os import path
 from netaddr import IPNetwork
-import pyjq
+import jq as pyjq
 
 from shared.common import (
     parse_arguments,

commands/weboftrust.py

@@ -2,7 +2,7 @@
 from os import path, listdir
 import json
 import yaml
-import pyjq
+import jq as pyjq
 import urllib.parse
 
 from shared.common import (

requirements.txt

@@ -1,8 +1,8 @@
 astroid==2.8.4
 autoflake==1.4
 autopep8==1.6.0
-boto3==1.19.10
-botocore==1.22.10
+boto3~=1.28.85
+botocore~=1.31.85
 certifi==2023.7.22
 chardet==4.0.0
 charset-normalizer==2.0.7
@@ -18,32 +18,31 @@ kiwisolver==1.3.2
 kwonly-args==1.0.10
 lazy-object-proxy==1.6.0
 MarkupSafe==2.0.1
-matplotlib==3.4.3
+matplotlib>=3.5.0,<3.9.0
 mccabe==0.6.1
 mock==4.0.3
 netaddr==0.8.0
-nose==1.3.7
-numpy==1.22.0
-pandas==1.3.4
+nose2>=0.13.0,<0.14.0
+pandas>=1.3.4
 parliament==1.5.2
-Pillow==10.0.1
+Pillow>=9.5.0,<10.0.0; python_version<'3.8'
+Pillow>=10.0.1,<11.0.0; python_version>='3.8'
 platformdirs==2.4.0
 policyuniverse==1.4.0.20210819
 pycodestyle==2.8.0
 pyflakes==2.4.0
-pyjq==2.4.0
+jq>=1.6.0
 pylint==2.11.1
 pyparsing==3.0.4
 python-dateutil==2.8.2
 pytz==2021.3
-PyYAML==6.0
+PyYAML>=6.0.1
 requests==2.26.0
-s3transfer==0.5.0
-scipy==1.10.0
+s3transfer>=0.7.0,<0.8.0
 seaborn==0.11.2
 six==1.16.0
 toml==0.10.2
-typed-ast==1.4.3
+
 typing-extensions==3.10.0.2
 urllib3==1.26.18
 wrapt==1.13.3

shared/audit.py

@@ -1,7 +1,7 @@
 import json
 import yaml
 from os.path import exists
-import pyjq
+import jq as pyjq
 import traceback
 import re
 import pkgutil

shared/common.py

@@ -2,7 +2,7 @@
 import argparse
 import json
 import datetime
-import pyjq
+import jq as pyjq
 import yaml
 import sys
 from netaddr import IPNetwork

shared/find_unused.py

@@ -1,4 +1,4 @@
-import pyjq
+import jq as pyjq
 
 from shared.common import query_aws, get_regions, get_parameter_file
 from shared.nodes import Account, Region
@@ -130,7 +130,12 @@ def find_unused_elastic_load_balancers(region):
                 "describe-target-health",
                 target_group["TargetGroupArn"],
             )
-            instances = pyjq.one(".TargetHealthDescriptions? | length", target_healths)
+            try:
+                instances = pyjq.first(
+                    ".TargetHealthDescriptions? | length", target_healths
+                )
+            except StopIteration:
+                instances = 0
             if instances > 0:
                 unused_elastic_load_balancers.pop()
                 break

shared/iam_audit.py

@@ -1,6 +1,6 @@
 import json
 from datetime import datetime
-import pyjq
+import jq as pyjq
 import traceback
 import re
 import os.path

shared/nodes.py

@@ -22,7 +22,7 @@
 USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ---------------------------------------------------------------------------
 """
-import pyjq
+import jq as pyjq
 from abc import ABCMeta
 from netaddr import IPNetwork, IPAddress
 from six import add_metaclass

shared/public.py

@@ -1,7 +1,7 @@
 from __future__ import print_function
 import json
 import os
-import pyjq
+import jq as pyjq
 
 from shared.nodes import Account, Region, is_public_ip
 from commands.prepare import build_data_structure
@@ -97,9 +97,12 @@ def get_public_nodes(account, config, use_cache=False):
 
         # Find the node at the other end of this edge
         target = {"arn": edge["target"], "account": account["name"]}
-        target_node = pyjq.first(
-            '.[].data|select(.id=="{}")'.format(target["arn"]), network, {}
-        )
+        try:
+            target_node = pyjq.first(
+                '.[].data|select(.id=="{}")'.format(target["arn"]), network
+            )
+        except StopIteration:
+            target_node = {}
 
         # Depending on the type of node, identify what the IP or hostname is
         if target_node["type"] == "elb":
@@ -141,9 +144,12 @@ def get_public_nodes(account, config, use_cache=False):
         # Check if any protocol is allowed (indicated by IpProtocol == -1)
         ingress = pyjq.all(".[]", edge.get("node_data", {}))
 
-        sg_group_allowing_all_protocols = pyjq.first(
-            '.[]|select(.IpPermissions[]?|.IpProtocol=="-1")|.GroupId', ingress, None
-        )
+        try:
+            sg_group_allowing_all_protocols = pyjq.first(
+                '.[]|select(.IpPermissions[]?|.IpProtocol=="-1")|.GroupId', ingress
+            )
+        except StopIteration:
+            sg_group_allowing_all_protocols = None
         public_sgs = {}
         if sg_group_allowing_all_protocols is not None:
             warnings.append(

tests/scripts/unit_tests.sh

@@ -3,10 +3,4 @@ if [ -f .coverage ]; then
   rm .coverage
 fi
 
-python -m nose tests/unit \
---with-coverage \
---cover-package=commands \
---cover-package=shared \
---cover-min-percentage=60 \
---cover-html \
---cover-html-dir=htmlcov
+python -m nose2 --config unittest.cfg -v

tests/unit/test_audit.py

@@ -1,6 +1,5 @@
 import unittest
 import json
-from nose.tools import assert_equal, assert_true, assert_false
 
 from shared.common import parse_arguments
 from shared.audit import audit
@@ -18,7 +17,7 @@ def test_audit(self):
             print(finding)
             issue_ids.add(finding.issue_id)
 
-        assert_equal(
+        self.assertEqual(
             issue_ids,
             set(
                 [

tests/unit/test_common.py

@@ -1,6 +1,5 @@
 import unittest
 import argparse
-from nose.tools import assert_equal, assert_true, assert_false
 
 from shared.common import (
     make_list,
@@ -14,8 +13,8 @@
 
 class TestCommon(unittest.TestCase):
     def test_make_list(self):
-        assert_equal(["hello"], make_list("hello"))
-        assert_equal(["hello"], make_list(["hello"]))
+        self.assertEqual(["hello"], make_list("hello"))
+        self.assertEqual(["hello"], make_list(["hello"]))
 
     def test_parse_arguments(self):
         parser = argparse.ArgumentParser()
@@ -29,17 +28,17 @@ def test_parse_arguments(self):
             ["--json", "--accounts", "demo", "--config", "config.json.demo"], parser
         )
 
-        assert_equal(args.json, True)
+        self.assertEqual(args.json, True)
 
     def test_get_account_stats(self):
         account = get_account("demo")
 
         stats = get_account_stats(account, True)
-        assert_equal(stats["EC2 instances"]["us-east-1"], 3)
+        self.assertEqual(stats["EC2 instances"]["us-east-1"], 3)
 
     def test_get_collection_date(self):
         account = get_account("demo")
-        assert_equal("2019-05-07T15:40:22+00:00", get_collection_date(account))
+        self.assertEqual("2019-05-07T15:40:22+00:00", get_collection_date(account))
 
     # def test_get_access_advisor_active_counts(self):
     #     account = get_account("demo")

tests/unit/test_find_unused.py

@@ -4,7 +4,6 @@
 
 from unittest import TestCase, mock
 from unittest.mock import MagicMock
-from nose.tools import assert_equal, assert_true, assert_false
 
 
 class TestFindUnused(TestCase):
@@ -41,7 +40,7 @@ def mocked_query_side_effect(account, query, region):
             mock_query.side_effect = mocked_query_side_effect
             from shared.find_unused import find_unused_elastic_ips
 
-            assert_equal(find_unused_elastic_ips(self.mock_region), [])
+            self.assertEqual(find_unused_elastic_ips(self.mock_region), [])
 
     def test_find_unused_elastic_ips(self):
         def mocked_query_side_effect(account, query, region):
@@ -74,7 +73,7 @@ def mocked_query_side_effect(account, query, region):
             mock_query.side_effect = mocked_query_side_effect
             from shared.find_unused import find_unused_elastic_ips
 
-            assert_equal(
+            self.assertEqual(
                 find_unused_elastic_ips(self.mock_region),
                 [{"id": "eipalloc-2", "ip": "2.3.4.5"}],
             )
@@ -137,7 +136,7 @@ def mocked_query_side_effect(account, query, region):
             mock_query.side_effect = mocked_query_side_effect
             from shared.find_unused import find_unused_volumes
 
-            assert_equal(find_unused_volumes(self.mock_region), [{"id": "vol-2222"}])
+            self.assertEqual(find_unused_volumes(self.mock_region), [{"id": "vol-2222"}])
 
     def test_find_unused_security_groups(self):
         def mocked_query_side_effect(account, query, region):
@@ -198,7 +197,7 @@ def mocked_query_side_effect(account, query, region):
             mock_query.side_effect = mocked_query_side_effect
             from shared.find_unused import find_unused_security_groups
 
-            assert_equal(
+            self.assertEqual(
                 find_unused_security_groups(self.mock_region),
                 [
                     {
@@ -271,7 +270,7 @@ def mocked_query_side_effect(account, query, region):
             mock_query.side_effect = mocked_query_side_effect
             from shared.find_unused import find_unused_network_interfaces
 
-            assert_equal(
+            self.assertEqual(
                 find_unused_network_interfaces(self.mock_region),
                 [{"id": "eni-00000001"}],
             )
@@ -355,7 +354,7 @@ def mocked_query_side_effect(account, query, region):
             mock_query.side_effect = mocked_query_side_effect
             from shared.find_unused import find_unused_elastic_load_balancers
 
-            assert_equal(
+            self.assertEqual(
                 find_unused_elastic_load_balancers(self.mock_region),
                 [{"LoadBalancerName": "some-elb", 'Type': 'classic'}],
             )

tests/unit/test_iam_audit.py

@@ -1,6 +1,5 @@
 import unittest
 import json
-from nose.tools import assert_equal, assert_true, assert_false
 
 from shared.iam_audit import find_admins
 from shared.common import parse_arguments
@@ -15,4 +14,4 @@ def test_find_admins(self):
         findings = set()
         admins = find_admins(accounts, args, findings)
 
-        assert_equal(admins, [{"account": "demo", "name": "bad_role", "type": "role"}])
+        self.assertEqual(admins, [{"account": "demo", "name": "bad_role", "type": "role"}])