Merge pull request #7 from escapecloud/task/json_to_sqlite

Issue #5 - Replace JSON with SQLite

Author: bencehezso

README.md

@@ -11,7 +11,8 @@
 
 **Release Date: November 20th, 2024**
 
-cloudexit is an open-source tool that empowers cloud engineers to conduct comprehensive cloud exit assessments. It helps identify and evaluate the risks associated with their cloud environment while providing actionable insights into the challenges and constraints of transitioning away from their current cloud provider. By leveraging EscapeCloud OSS, organizations can better prepare for a potential cloud exit, ensuring a smoother and more informed decision-making process.
+cloudexit is an open-source tool that empowers cloud engineers to conduct comprehensive cloud exit assessments. It helps identify and evaluate the risks associated with their cloud environment while providing actionable insights into the challenges and constraints of transitioning away from their current cloud provider. By leveraging EscapeCloud Community Edition, organizations can better prepare for a potential cloud exit, ensuring a smoother and more informed decision-making process.
+
 
 ## Required Packages
 

assets/template/index.html

@@ -6,7 +6,7 @@
         <meta name="viewport" content="width=device-width, initial-scale=1" />
         <!-- Favicon & Title -->
         <link rel="icon" type="image/png" sizes="16x16" href="assets/img/logo/favicon.png" />
-        <title>EscapeCloud OSS - Cloud Exit Assessment Report</title>
+        <title>EscapeCloud Community Edition - Cloud Exit Assessment Report</title>
         <!-- CSS only -->
         <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.0/font/bootstrap-icons.css">
         <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
@@ -24,7 +24,7 @@
             <div class="top-head">
                 <h4>
                   <img src="assets/img/logo/logo.png" width="30" alt="EscapeCloud" />
-                  EscapeCloud OSS - Cloud Exit Assessment Report
+                  EscapeCloud Community Edition - Cloud Exit Assessment Report
                 </h4>
             </div>
         </header>

core/engine.py

@@ -3,6 +3,9 @@
 import shutil
 import logging
 from datetime import datetime
+from collections import defaultdict
+
+logger = logging.getLogger("core.engine.utils")
 
 def copy_assets(report_path):
     assets_folders = ["css", "img", "icons"]
@@ -19,6 +22,18 @@ def copy_assets(report_path):
         if not os.path.exists(dest_path):
             shutil.copytree(src_path, dest_path, dirs_exist_ok=True)
 
+    # Copy datasets/data.db to data/assessment.db
+    db_src_path = "datasets/data.db"
+    db_dest_dir = os.path.join(report_path, "data")
+    db_dest_path = os.path.join(db_dest_dir, "assessment.db")
+
+    # Create the 'data' directory if it doesn't exist
+    os.makedirs(db_dest_dir, exist_ok=True)
+
+    # Only copy if the destination doesn't already exist
+    if not os.path.exists(db_dest_path):
+        shutil.copyfile(db_src_path, db_dest_path)
+
 
 def get_cost_summary(cost_data):
     months = []
@@ -32,6 +47,13 @@ def get_cost_summary(cost_data):
         "EUR": "€"
     }
 
+    # Convert list to dictionary if necessary
+    if isinstance(cost_data, list):
+        cost_data = {
+            item["month"]: {"cost": item["cost"], "currency": item["currency"]}
+            for item in cost_data
+        }
+
     # Extract currency from the first entry, assuming all costs use the same currency
     first_entry = next(iter(cost_data.values()), None)
     currency_code = first_entry.get("currency", "USD") if first_entry else "USD"
@@ -47,75 +69,72 @@ def get_cost_summary(cost_data):
     return months, cost_values, total_cost, currency_symbol
 
 def get_risk_summary(risk_data, risk_definitions, resource_inventory):
-    logger = logging.getLogger(__name__)
-
     severity_order = {'high': 1, 'medium': 2, 'low': 3}
     severity_counts = {'high': 0, 'medium': 0, 'low': 0}
     sorted_risks = []
 
     # Map resource IDs to resource names for quick lookup
-    resource_name_map = {str(item['resource_type']): item['resource_name'] for item in resource_inventory.values()}
-
-    # Log the resource_name_map to verify it has been built correctly
-    #logger.info(f"Resource Name Map: {resource_name_map}")
+    resource_name_map = {str(key): value['name'] for key, value in resource_inventory.items()}
 
-    # Log the risk_data to verify its structure
-    #logger.info(f"Risk Data: {risk_data}")
-
-    # Group risks by their risk code and track impacted resources
-    risk_map = {}
+    # Group risks by their risk code and impacted resources
+    risk_map = defaultdict(lambda: {"impacted_resources": set(), "count": 0})
     for risk_entry in risk_data:
         risk_code = risk_entry['risk']
-        resource_type = str(risk_entry['resource_type'])  # Convert to string to match the map keys
-
-        # Initialize risk entry in the map if it doesn't exist
-        if risk_code not in risk_map:
-            risk_map[risk_code] = {
-                "impacted_resources": set(),  # To store unique resources
-                "count": 0
-            }
+        resource_type = str(risk_entry['resource_type']) if risk_entry['resource_type'] != "null" else None
 
-        # If resource_type is not "null", add it to impacted resources
-        if resource_type != "null":
+        if resource_type:
+            # Handle risks with associated resource types
             resource_name = resource_name_map.get(resource_type, "Unknown Resource")
             risk_map[risk_code]["impacted_resources"].add(resource_name)
             risk_map[risk_code]["count"] += 1
         else:
-            # Mark this entry as a general risk without specific resources
+            # Handle overall risks with no specific resource type
             risk_map[risk_code]["impacted_resources"] = []
             risk_map[risk_code]["count"] = None
 
-    # Log the intermediate risk_map to verify resource processing
-    #logger.info(f"Risk Map After Processing: {risk_map}")
-
-    # Process each risk code in the map to populate sorted_risks
+    # Process risk definitions
     for risk_code, risk_info in risk_map.items():
-        # Look up the risk definition from risk_definitions
         risk_definition = next((rd for rd in risk_definitions if rd["id"] == risk_code), None)
         if not risk_definition:
             continue
 
         severity = risk_definition['severity']
         severity_counts[severity] += 1
 
-        # Format the impacted resources and count
-        impacted_resources = list(risk_info["impacted_resources"]) if risk_info["impacted_resources"] else []
-        impacted_resources_count = risk_info["count"]
-
-        # Append detailed risk information
         sorted_risks.append({
             'name': risk_definition['name'],
             'description': risk_definition['description'],
-            'impacted_resources': impacted_resources,
-            'impacted_resources_count': impacted_resources_count,
+            'impacted_resources': list(risk_info["impacted_resources"]),
+            'impacted_resources_count': risk_info["count"],
             'severity': severity
         })
 
-    # Sort risks by severity level
+    # Sort risks by severity
     sorted_risks.sort(key=lambda x: severity_order.get(x['severity'], 4))
 
-    # Log the final sorted risks for verification
-    #logger.info(f"Sorted Risks: {sorted_risks}")
-    #logger.info(f"Severity Counts: {severity_counts}")
-
     return sorted_risks, severity_counts
+
+def prepare_alternative_technologies(resource_inventory, alternatives, alternative_technologies, exit_strategy):
+    alt_tech_data = []
+    for resource in resource_inventory:
+        resource_type = resource.get("resource_type")
+        relevant_alternatives = [
+            alt for alt in alternatives
+            if str(alt["resource_type"]) == str(resource_type) and str(alt["strategy_type"]) == str(exit_strategy)
+        ]
+        for alt in relevant_alternatives:
+            tech = next(
+                (t for t in alternative_technologies if t["id"] == alt["alternative_technology"] and t["status"] == "t"),
+                None
+            )
+            if tech:
+                alt_tech_data.append({
+                    "resource_type_id": resource_type,
+                    "product_name": tech.get("product_name"),
+                    "product_description": tech.get("product_description"),
+                    "product_url": tech.get("product_url"),
+                    "open_source": tech.get("open_source") == "t",
+                    "support_plan": tech.get("support_plan") == "t",
+                    "status": tech.get("status") == "t",
+                })
+    return alt_tech_data

core/utils.py

@@ -6,9 +6,12 @@
 import time
 import logging
 from datetime import date, datetime, timedelta
+from collections import defaultdict
 from dateutil.relativedelta import relativedelta
 from botocore.exceptions import NoCredentialsError, ClientError
 
+from .utils_db import connect, load_data
+
 logger = logging.getLogger("core.engine.aws")
 
 def aws_api_call_with_retry(client, function_name, parameters, max_retries, retry_delay):
@@ -59,19 +62,20 @@ def build_aws_resource_inventory(cloud_service_provider, provider_details, repor
             region_name=region
         )
 
-        # Load the ResourceType mapping to include both `id` and `name`
-        with open("datasets/resourcetype.json", "r", encoding="utf-8") as f:
-            resource_type_mapping = {
-                item["code"]: {"id": item["id"], "name": item["name"]}
-                for item in json.load(f)
-                if item["csp"] == "2" and item["status"] == "t"
-            }
+        db_path = os.path.join(report_path, "data", "assessment.db")
+
+        # Load the ResourceType mapping
+        resource_type_mapping = {
+            item["code"]: {"id": item["id"], "name": item["name"]}
+            for item in load_data("resourcetype")
+            if item["csp"] == 2 and item["status"] == "t"
+        }
 
-        resource_summary = {}
+        # Save raw data for debugging and auditing purposes
         raw_data = []
 
-        # Initialize a custom counter
-        resource_inventory_id_counter = 1
+        # Aggregate resources by type and location
+        aggregated_resources = defaultdict(int)
 
         # Iterate through each resource type in the JSON
         for idx, (resource_type_code, resource_info) in enumerate(resource_type_mapping.items(), start=1):
@@ -108,17 +112,9 @@ def build_aws_resource_inventory(cloud_service_provider, provider_details, repor
                     #logger.warning(f"No valid response found for {service_name} operation {operation_name}. Skipping.")
                     continue
 
-                # Count resources and add to summary if count > 0
-                resource_count = len(resources)
-                if resource_count > 0:
-                    resource_inventory_id = str(resource_inventory_id_counter)
-                    resource_summary[resource_inventory_id] = {
-                        "resource_name": resource_info["name"],
-                        "resource_type": resource_info["id"],
-                        "location": region,
-                        "count": resource_count
-                    }
-                    resource_inventory_id_counter += 1
+                # Aggregate the resources
+                for resource in resources:
+                    aggregated_resources[(resource_type_code, region)] += 1
 
                 # Store raw data
                 raw_data.append({
@@ -127,9 +123,6 @@ def build_aws_resource_inventory(cloud_service_provider, provider_details, repor
                     "resources": resources
                 })
 
-
-                #logger.info(f"Processed {resource_count} resources for service {service_name} with operation {operation_name}")
-
             except (NoCredentialsError, ClientError, Exception) as e:
                 #logger.error(f"Error while processing {service_name}: {str(e)}", exc_info=True)
                 continue
@@ -140,13 +133,35 @@ def build_aws_resource_inventory(cloud_service_provider, provider_details, repor
         raw_file_path = os.path.join(raw_data_path, "resource_inventory_raw_data.json")
         with open(raw_file_path, "w", encoding="utf-8") as raw_file:
             json.dump(raw_data, raw_file, indent=4)
-        #logger.info(f"AWS raw resource inventory saved to {raw_file_path}")
 
-        # Save structured data to a JSON file
-        structured_file_path = os.path.join(report_path, "resource_inventory_standard_data.json")
-        with open(structured_file_path, "w", encoding="utf-8") as structured_file:
-            json.dump(resource_summary, structured_file, indent=4)
-        #logger.info(f"AWS structured resource inventory saved to {structured_file_path}")
+        # Insert aggregated data into SQLite
+        with connect(db_path=db_path) as conn:
+            cursor = conn.cursor()
+
+            for (resource_type_code, resource_location), resource_count in aggregated_resources.items():
+                try:
+                    # Map resource type code to resource_type_id
+                    resource_info = resource_type_mapping.get(resource_type_code)
+                    if not resource_info:
+                        #logger.warning(f"Resource type {resource_type_code} not found in resourcetype mapping. Skipping.")
+                        continue
+
+                    resource_type_id = resource_info["id"]
+
+                    cursor.execute(
+                        """
+                        INSERT INTO resource_inventory (resource_type, location, count)
+                        VALUES (?, ?, ?)
+                        ON CONFLICT(resource_type, location) DO UPDATE SET count = excluded.count
+                        """,
+                        (resource_type_id, resource_location, resource_count)
+                    )
+                except sqlite3.Error as e:
+                    logger.error(f"SQLite error while processing aggregated resource: {e}", exc_info=True)
+                except Exception as e:
+                    logger.error(f"Unexpected error while processing aggregated resource: {e}", exc_info=True)
+
+            conn.commit()
 
     except Exception as e:
         logger.error(f"Error creating AWS resource inventory: {str(e)}", exc_info=True)
@@ -172,6 +187,8 @@ def build_aws_cost_inventory(cloud_service_provider, provider_details, report_pa
         )
         cost_explorer = session.client('ce', region_name='us-east-1')
 
+        db_path = os.path.join(report_path, "data", "assessment.db")
+
         end_time = date.today()
         start_time = end_time.replace(day=1) - timedelta(days=180)
         start_time = start_time.replace(day=1)
@@ -189,24 +206,54 @@ def build_aws_cost_inventory(cloud_service_provider, provider_details, report_pa
             }
         )
 
-        cost_inventory_raw_path = os.path.join(report_path, "cost_inventory_raw_data.json")
+        cost_inventory_raw_path = os.path.join(raw_data_path, "cost_inventory_raw_data.json")
         with open(cost_inventory_raw_path, "w", encoding="utf-8") as raw_file:
             json.dump(cost_and_usage, raw_file, indent=4)
 
-        structured_costs = {}
-        for result in cost_and_usage['ResultsByTime']:
-            month_str = result['TimePeriod']['Start']
-            total_cost = sum(float(group['Metrics']['UnblendedCost']['Amount']) for group in result['Groups'])
-            currency = result['Groups'][0]['Metrics']['UnblendedCost']['Unit'] if result['Groups'] else 'USD'
-            structured_costs[month_str] = {"cost": total_cost, "currency": currency}
-
-        missing_months = get_missing_months_aws(structured_costs.keys(), 6)
-        for missing_month in missing_months:
-            structured_costs[missing_month.isoformat()] = {"cost": 0.00, "currency": currency}
-
-        cost_inventory_standard_path = os.path.join(report_path, "cost_inventory_standard_data.json")
-        with open(cost_inventory_standard_path, "w", encoding="utf-8") as structured_file:
-            json.dump(structured_costs, structured_file, indent=4)
+        # Insert structured data into SQLite
+        with connect(db_path=db_path) as conn:
+            cursor = conn.cursor()
+
+            for result in cost_and_usage['ResultsByTime']:
+                month_str = result['TimePeriod']['Start']
+                total_cost = sum(float(group['Metrics']['UnblendedCost']['Amount']) for group in result['Groups'])
+                currency = result['Groups'][0]['Metrics']['UnblendedCost']['Unit'] if result['Groups'] else 'USD'
+                month_date = datetime.strptime(month_str, '%Y-%m-%d').date().replace(day=1).isoformat()
+
+                # Insert or update the cost data for the month
+                cursor.execute(
+                    """
+                    INSERT INTO cost_inventory (month, cost, currency)
+                    VALUES (?, ?, ?)
+                    ON CONFLICT(month) DO UPDATE SET
+                        cost = excluded.cost,
+                        currency = excluded.currency
+                    """,
+                    (month_date, total_cost, currency)
+                )
+
+            # Handle missing months
+            structured_months = {datetime.strptime(result['TimePeriod']['Start'], '%Y-%m-%d').date() for result in cost_and_usage['ResultsByTime']}
+            missing_months = get_missing_months_aws({month.isoformat() for month in structured_months}, 6)
+
+            for missing_month in missing_months:
+                cursor.execute(
+                    """
+                    INSERT INTO cost_inventory (month, cost, currency)
+                    VALUES (?, 0.00, ?)
+                    ON CONFLICT(month) DO UPDATE SET
+                        currency = excluded.currency
+                    """,
+                    (missing_month.isoformat(), currency)
+                )
+
+            conn.commit()
+
+    except sqlite3.Error as e:
+        logger.error(f"SQLite error: {str(e)}", exc_info=True)
+    except Exception as e:
+        logger.error(f"Error creating AWS cost inventory: {str(e)}", exc_info=True)
+        raise
 
     except Exception as e:
         logger.error(f"Error creating AWS cost inventory: {str(e)}", exc_info=True)