Made changes to the python scripts for consistency and accuracty:

- Changed user, project and resource variable consisent with loki/prometheus
- Make python script params more consistent
- Fixed python files to handle mutate: NUMBOOL correctly when calculating rating
- Add scenario name to the db push so that will be target db data to pull

Authored-by @ayefimov-1
Assisted-by: Claude

Author: ayefimov-1

roles/telemetry_chargeback/files/gen_db_summary.py

@@ -119,7 +119,7 @@ def _apply_mutate(qty: float, mutate: str) -> float:
         return math.floor(qty)
     elif mutate_upper == "NUMBOOL":
         # If qty equals 0, leave it at 0. Else, set it to 1.
-        return 0.0 if qty == 0 else 1.0
+        return 0.0 if abs(qty) < 1e-9 else 1.0
     elif mutate_upper == "NOTNUMBOOL":
         # If qty equals 0, set it to 1. Else, set it to 0.
         return 1.0 if qty == 0 else 0.0
@@ -175,8 +175,9 @@ def _parse_numeric(value: Any, default: float = 0) -> float:
 
 def aggregate_rates_by_type(
     pairs: list[tuple[str, str]],
-) -> tuple[dict, float]:
-    sums: defaultdict[str, float] = defaultdict(float)
+) -> tuple[dict, float, dict]:
+    rate_sums: defaultdict[str, float] = defaultdict(float)
+    qty_sums: defaultdict[str, float] = defaultdict(float)
     for _, log_str in pairs:
         try:
             entry = json.loads(log_str)
@@ -196,17 +197,26 @@ def aggregate_rates_by_type(
         except (TypeError, ValueError):
             continue
 
-        # Apply mutate transformation
+        # Track raw qty sum (before any transformation)
+        qty_sums[mtype] += qty
+
+        # Apply mutate transformation for rating calculation
         qty_mutated = _apply_mutate(qty, mutate)
 
         # Apply factor and offset
         qty_rate = qty_mutated * factor + offset
 
         # Calculate rate
-        sums[mtype] += qty_rate * price
-    by_types = {k: {"Rate": round(v, 4)} for k, v in sorted(sums.items())}
-    total = sum(sums.values())
-    return by_types, total
+        rate_sums[mtype] += qty_rate * price
+
+    by_types = {
+        k: {"Rate": round(v, 4)} for k, v in sorted(rate_sums.items())
+    }
+    qty_by_types = {
+        k: {"qty_sum": round(v, 4)} for k, v in sorted(qty_sums.items())
+    }
+    total = sum(rate_sums.values())
+    return by_types, total, qty_by_types
 
 
 def build_summary(pairs: list[tuple[str, str]]) -> dict[str, Any]:
@@ -237,17 +247,35 @@ def build_summary(pairs: list[tuple[str, str]]) -> dict[str, Any]:
         empty = {"nanosec": None, "begin": None, "end": None}
         time_block = {"begin_step": empty.copy(), "end_step": empty.copy()}
 
-    by_types, total_r = aggregate_rates_by_type(pairs)
+    # Get aggregated data by type
+    by_types, total_r, qty_by_types = aggregate_rates_by_type(pairs)
+
+    # Get overall time range for by_type entries
+    begin_time = first.get("start") if pairs else None
+    end_time = last.get("end") if pairs else None
+
+    # Build flat list of entries
+    rate_list = []
+    for type_name in sorted(by_types.keys()):
+        entry = {
+            "Begin": begin_time,
+            "End": end_time,
+            "Qty": qty_by_types.get(type_name, {}).get("qty_sum", 0.0),
+            "Rate": by_types[type_name]["Rate"],
+            "Type": type_name,
+        }
+        rate_list.append(entry)
+
     return {
         "time": time_block,
-        "data_log": {
+        "data_summary": {
             "total_timesteps": n_ts,
             "metrics_per_step": mps,
             "log_count": log_count,
+            "total_rating": round(total_r, 4),
         },
-        "rate": {
-            "by_types": by_types,
-            "total": {"Rating": round(total_r, 4)},
+        "by_type": {
+            "rate": rate_list,
         },
     }
 
@@ -264,10 +292,36 @@ def write_yaml(path: Path, doc: dict[str, Any]) -> None:
         )
 
 
+def _str_to_bool(value: str) -> bool:
+    """
+    Convert string to boolean.
+
+    Args:
+        value: String representation of boolean.
+
+    Returns:
+        Boolean value.
+
+    Raises:
+        argparse.ArgumentTypeError: If value cannot be converted.
+    """
+    if isinstance(value, bool):
+        return value
+    if value.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif value.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError(
+            f"Boolean value expected. Got: {value}"
+        )
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(
         description=(
-            "Summarize Loki JSON log entries to YAML (time, data_log, rate)."
+            "Summarize Loki JSON log entries to YAML "
+            "(time, data_summary, by_type)."
         ),
     )
     parser.add_argument(
@@ -282,11 +336,21 @@ def main() -> None:
     )
     parser.add_argument(
         "--debug",
+        type=_str_to_bool,
+        default=False,
+        metavar="BOOL",
+        help=(
+            "Enable debug mode: write <stem>_diff.txt with one "
+            "[ts,log] JSON per line (true/false)."
+        ),
+    )
+    parser.add_argument(
+        "--debug_dir",
         type=Path,
         default=None,
         metavar="DIR",
         help=(
-            "If set, write <stem>_diff.txt with one [ts,log] JSON per line."
+            "Directory for debug output. Required when --debug is enabled."
         ),
     )
     args = parser.parse_args()
@@ -299,24 +363,24 @@ def main() -> None:
     out_path = args.output or (args.json.parent / f"{stem}_total.yml")
     pairs = extract_and_sort(args.json)
 
-    dbg = str(args.debug).strip() if args.debug is not None else ""
-    if dbg and dbg != ".":
-        args.debug.mkdir(parents=True, exist_ok=True)
-        dbg_file = args.debug / f"{args.json.stem}_diff.txt"
+    if args.debug:
+        # Require debug directory when debug mode is enabled
+        if not args.debug_dir:
+            print(
+                "Error: --debug_dir is required when --debug is enabled",
+                file=sys.stderr
+            )
+            sys.exit(1)
+        debug_dir = args.debug_dir
+        debug_dir.mkdir(parents=True, exist_ok=True)
+        dbg_file = debug_dir / f"{args.json.stem}_diff.txt"
         with dbg_file.open("w", encoding="utf-8") as f:
             for ts, log_str in pairs:
                 print(json.dumps([ts, log_str], ensure_ascii=False), file=f)
 
     doc = build_summary(pairs)
     write_yaml(out_path, doc)
 
-    if doc["data_log"]["metrics_per_step"] == "ERROR":
-        per_ts = Counter(ts for ts, _ in pairs)
-        exp = next(iter(per_ts.values()), 0)
-        for ts in sorted(per_ts, key=int):
-            if per_ts[ts] != exp:
-                print(ts, per_ts[ts], file=sys.stdout)
-
 
 if __name__ == "__main__":
     main()

roles/telemetry_chargeback/files/gen_synth_loki_data.py

@@ -109,6 +109,7 @@ def generate_loki_data(
     end_time: datetime,
     time_step_seconds: int,
     config: Dict[str, Any],
+    scenario_name: str,
     reverse_timestamps: bool = True,
 ):
     """
@@ -121,6 +122,7 @@ def generate_loki_data(
         end_time (datetime): The end time for data generation.
         time_step_seconds (int): The duration of each log entry in seconds.
         config (Dict[str, Any]): Configuration dictionary loaded from file.
+        scenario_name (str): Name of the test scenario for Loki labeling.
         reverse_timestamps (bool): If True, sort timestamps in descending order
             (newest first, oldest last). If False, sort in ascending order
             (oldest first, newest last). Default is True (descending).
@@ -138,6 +140,7 @@ def generate_loki_data(
     logger.debug(f"Time range in epoch seconds: {start_epoch} to {end_epoch}")
 
     log_data_list = []  # This list will hold all our data points
+    last_end_of_step_epoch = None  # Track last entry's end epoch
 
     # Loop through the time range and generate data points
     for current_epoch in range(
@@ -164,13 +167,13 @@ def generate_loki_data(
             "end_time": end_str
         })
 
+        # Track the last end epoch
+        last_end_of_step_epoch = end_of_step_epoch
+
     # Add final entry that ends at end_epoch (current time)
-    if log_data_list and end_epoch > start_epoch:
+    if log_data_list and end_epoch > start_epoch and last_end_of_step_epoch:
         # Calculate start of final entry based on end of last generated entry
-        last_entry_end = log_data_list[-1]["end_time"]
-        # Parse the last entry's end time to get the epoch
-        last_end_dt = datetime.fromisoformat(last_entry_end)
-        final_start_epoch = int(last_end_dt.timestamp()) + 1
+        final_start_epoch = last_end_of_step_epoch + 1
         final_nanoseconds = int(final_start_epoch * 1_000_000_000)
 
         # Only add if the final entry would have a valid duration
@@ -222,8 +225,11 @@ def generate_loki_data(
 
         # Validate required fields
         # metadata is optional for generation; name is not a log-type field
-        required_for_item = set(required_fields) - {"name", "metadata"}
-        missing = required_for_item - set(log_type_config)
+        required_for_item = [
+            f for f in required_fields
+            if f not in ("name", "metadata")
+        ]
+        missing = [f for f in required_for_item if f not in log_type_config]
         if missing:
             logger.error(
                 f"Missing required fields in {type_key!r} config: {missing}"
@@ -255,7 +261,10 @@ def generate_loki_data(
             "qty": qty_list,
             "price": price_list,
             "groupby": groupby.copy(),
-            "metadata": log_type_config.get("metadata", {})
+            "metadata": log_type_config.get("metadata", {}),
+            "mutate": log_type_config.get("mutate"),
+            "factor": log_type_config.get("factor"),
+            "offset": log_type_config.get("offset")
         }
 
     # --- Step 3: Load template and render ---
@@ -333,12 +342,16 @@ def tojson_preserve_order(obj):
 
         log_types_list.append(log_types_with_dates)
 
-    # Get loki_stream configuration
+    # Get loki_stream configuration and add scenario
     loki_stream = config.get("loki_stream", {})
     if not loki_stream:
         logger.warning("No loki_stream configuration found, using defaults")
         loki_stream = {"service": "cloudkitty"}
 
+    # Add scenario name to loki_stream labels
+    loki_stream["scenario"] = scenario_name
+    logger.info(f"Adding scenario label: {scenario_name}")
+
     # Build template context with generic log type information
     template_context = {
         "log_data": log_data_list,
@@ -373,6 +386,31 @@ def tojson_preserve_order(obj):
         sys.exit(1)
 
 
+def _str_to_bool(value: str) -> bool:
+    """
+    Convert string to boolean.
+
+    Args:
+        value: String representation of boolean.
+
+    Returns:
+        Boolean value.
+
+    Raises:
+        argparse.ArgumentTypeError: If value cannot be converted.
+    """
+    if isinstance(value, bool):
+        return value
+    if value.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif value.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError(
+            f"Boolean value expected. Got: {value}"
+        )
+
+
 def main():
     """Main entry point for the script."""
     parser = argparse.ArgumentParser(
@@ -415,8 +453,10 @@ def main():
     )
     parser.add_argument(
         "--debug",
-        action="store_true",
-        help="Enable debug level logging for verbose output."
+        type=_str_to_bool,
+        default=False,
+        metavar="BOOL",
+        help="Enable debug level logging for verbose output (true/false)."
     )
     args = parser.parse_args()
 
@@ -429,7 +469,11 @@ def main():
         config = load_config(args.test)
     except (FileNotFoundError, ValueError) as e:
         logger.critical(f"Failed to load config: {e}")
-        return
+        sys.exit(1)
+
+    # Derive scenario name from test file path
+    scenario_name = args.test.stem
+    logger.info(f"Derived scenario name from test file: {scenario_name}")
 
     # Get generation parameters from config
     generation_config = config.get("generation", {})
@@ -450,14 +494,17 @@ def main():
             end_time=end_time_utc,
             time_step_seconds=step_seconds,
             config=config,
+            scenario_name=scenario_name,
             reverse_timestamps=args.reverse,
         )
     except FileNotFoundError:
         logger.error(
             "Process aborted because the template file was not found."
         )
+        sys.exit(1)
     except Exception as e:
         logger.critical(f"A critical, unhandled error stopped the script: {e}")
+        sys.exit(1)
 
 
 if __name__ == "__main__":