Rebase from master

Author: ayefimov-1

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
 .idea/
+.ansible/

roles/telemetry_chargeback/defaults/main.yml

@@ -6,7 +6,6 @@ openstack_cmd: "openstack"
 cloudkitty_debug: false
 cloudkitty_debug_dir: "{{ (cloudkitty_debug | bool) | ternary(artifacts_dir_zuul + '/debug_ck_db', '') }}"
 
-# Directory paths
 logs_dir_zuul: "{{ cifmw_basedir }}/logs"
 artifacts_dir_zuul: "{{ cifmw_basedir }}/artifacts"
 cert_dir: "{{ cifmw_basedir }}/ck-certs"
@@ -28,3 +27,6 @@ openstackpod: "openstackclient"
 # Time window settings
 lookback: 6
 limit: 50
+
+# List of test scenario files to run
+cloudkitty_test_scenarios: []

roles/telemetry_chargeback/files/gen_db_summary.py

@@ -118,11 +118,11 @@ def _apply_mutate(qty: float, mutate: str) -> float:
     elif mutate_upper == "FLOOR":
         return math.floor(qty)
     elif mutate_upper == "NUMBOOL":
-        # If qty near 0, set it at 0. Else, set it to 1.
-        return 0.0 if abs(qty) < 1e-9 else 1.0
+        # If qty equals 0, leave it at 0. Else, set it to 1.
+        return 0.0 if qty == 0 else 1.0
     elif mutate_upper == "NOTNUMBOOL":
-        # If qty near 0, set it to 1. Else, set it to 0.
-        return 1.0 if abs(qty) < 1e-9 else 0.0
+        # If qty equals 0, set it to 1. Else, set it to 0.
+        return 1.0 if qty == 0 else 0.0
     else:  # NONE or any unrecognized value
         return qty
 
@@ -175,9 +175,8 @@ def _parse_numeric(value: Any, default: float = 0) -> float:
 
 def aggregate_rates_by_type(
     pairs: list[tuple[str, str]],
-) -> tuple[dict, float, dict]:
-    rate_sums: defaultdict[str, float] = defaultdict(float)
-    qty_sums: defaultdict[str, float] = defaultdict(float)
+) -> tuple[dict, float]:
+    sums: defaultdict[str, float] = defaultdict(float)
     for _, log_str in pairs:
         try:
             entry = json.loads(log_str)
@@ -197,34 +196,20 @@ def aggregate_rates_by_type(
         except (TypeError, ValueError):
             continue
 
-        # Track raw qty sum (before any transformation)
-        qty_sums[mtype] += qty
-
-        # Apply mutate transformation for rating calculation
+        # Apply mutate transformation
         qty_mutated = _apply_mutate(qty, mutate)
 
         # Apply factor and offset
         qty_rate = qty_mutated * factor + offset
 
         # Calculate rate
-        rate_sums[mtype] += qty_rate * price
-
-    by_types = {
-        k: {"Rate": round(v, 4)} for k, v in sorted(rate_sums.items())
-    }
-    qty_by_types = {
-        k: {"qty_sum": round(v, 4)} for k, v in sorted(qty_sums.items())
-    }
-    total = sum(rate_sums.values())
-    return by_types, total, qty_by_types
+        sums[mtype] += qty_rate * price
+    by_types = {k: {"Rate": round(v, 4)} for k, v in sorted(sums.items())}
+    total = sum(sums.values())
+    return by_types, total
 
 
 def build_summary(pairs: list[tuple[str, str]]) -> dict[str, Any]:
-    # Early exit if no pairs
-    if not pairs:
-        print("Error: No log entries to summarize", file=sys.stderr)
-        sys.exit(1)
-
     log_count = len(pairs)
     per_ts = Counter(ts for ts, _ in pairs)
     n_ts = len(per_ts)
@@ -233,52 +218,36 @@ def build_summary(pairs: list[tuple[str, str]]) -> dict[str, Any]:
     if counts and len(set(counts)) > 1:
         mps = "ERROR"
 
-    # Parse first and last entries (guaranteed to exist after early exit check)
-    first = json.loads(pairs[0][1])
-    last = json.loads(pairs[-1][1])
-
-    time_block = {
-        "begin_step": {
-            "nanosec": int(pairs[0][0]),
-            "begin": first.get("start"),
-            "end": first.get("end"),
-        },
-        "end_step": {
-            "nanosec": int(pairs[-1][0]),
-            "begin": last.get("start"),
-            "end": last.get("end"),
-        },
-    }
-
-    # Get aggregated data by type
-    by_types, total_r, qty_by_types = aggregate_rates_by_type(pairs)
-
-    # Get overall time range for by_type entries
-    begin_time = first.get("start")
-    end_time = last.get("end")
-
-    # Build flat list of entries
-    rate_list = []
-    for type_name in sorted(by_types.keys()):
-        entry = {
-            "Begin": begin_time,
-            "End": end_time,
-            "Qty": qty_by_types.get(type_name, {}).get("qty_sum", 0.0),
-            "Rate": by_types[type_name]["Rate"],
-            "Type": type_name,
+    if pairs:
+        first = json.loads(pairs[0][1])
+        last = json.loads(pairs[-1][1])
+        time_block = {
+            "begin_step": {
+                "nanosec": int(pairs[0][0]),
+                "begin": first.get("start"),
+                "end": first.get("end"),
+            },
+            "end_step": {
+                "nanosec": int(pairs[-1][0]),
+                "begin": last.get("start"),
+                "end": last.get("end"),
+            },
         }
-        rate_list.append(entry)
+    else:
+        empty = {"nanosec": None, "begin": None, "end": None}
+        time_block = {"begin_step": empty.copy(), "end_step": empty.copy()}
 
+    by_types, total_r = aggregate_rates_by_type(pairs)
     return {
         "time": time_block,
-        "data_summary": {
+        "data_log": {
             "total_timesteps": n_ts,
             "metrics_per_step": mps,
             "log_count": log_count,
-            "total_rating": round(total_r, 4),
         },
-        "by_type": {
-            "rate": rate_list,
+        "rate": {
+            "by_types": by_types,
+            "total": {"Rating": round(total_r, 4)},
         },
     }
 
@@ -295,36 +264,10 @@ def write_yaml(path: Path, doc: dict[str, Any]) -> None:
         )
 
 
-def _str_to_bool(value: str) -> bool:
-    """
-    Convert string to boolean.
-
-    Args:
-        value: String representation of boolean.
-
-    Returns:
-        Boolean value.
-
-    Raises:
-        argparse.ArgumentTypeError: If value cannot be converted.
-    """
-    if isinstance(value, bool):
-        return value
-    if value.lower() in ('yes', 'true', 't', 'y', '1'):
-        return True
-    elif value.lower() in ('no', 'false', 'f', 'n', '0'):
-        return False
-    else:
-        raise argparse.ArgumentTypeError(
-            f"Boolean value expected. Got: {value}"
-        )
-
-
 def main() -> None:
     parser = argparse.ArgumentParser(
         description=(
-            "Summarize Loki JSON log entries to YAML "
-            "(time, data_summary, by_type)."
+            "Summarize Loki JSON log entries to YAML (time, data_log, rate)."
         ),
     )
     parser.add_argument(
@@ -339,21 +282,11 @@ def main() -> None:
     )
     parser.add_argument(
         "--debug",
-        type=_str_to_bool,
-        default=False,
-        metavar="BOOL",
-        help=(
-            "Enable debug mode: write <stem>_diff.txt with one "
-            "[ts,log] JSON per line (true/false)."
-        ),
-    )
-    parser.add_argument(
-        "--debug_dir",
         type=Path,
         default=None,
         metavar="DIR",
         help=(
-            "Directory for debug output. Required when --debug is enabled."
+            "If set, write <stem>_diff.txt with one [ts,log] JSON per line."
         ),
     )
     args = parser.parse_args()
@@ -366,24 +299,24 @@ def main() -> None:
     out_path = args.output or (args.json.parent / f"{stem}_total.yml")
     pairs = extract_and_sort(args.json)
 
-    if args.debug:
-        # Require debug directory when debug mode is enabled
-        if not args.debug_dir:
-            print(
-                "Error: --debug_dir is required when --debug is enabled",
-                file=sys.stderr
-            )
-            sys.exit(1)
-        debug_dir = args.debug_dir
-        debug_dir.mkdir(parents=True, exist_ok=True)
-        dbg_file = debug_dir / f"{args.json.stem}_diff.txt"
+    dbg = str(args.debug).strip() if args.debug is not None else ""
+    if dbg and dbg != ".":
+        args.debug.mkdir(parents=True, exist_ok=True)
+        dbg_file = args.debug / f"{args.json.stem}_diff.txt"
         with dbg_file.open("w", encoding="utf-8") as f:
             for ts, log_str in pairs:
                 print(json.dumps([ts, log_str], ensure_ascii=False), file=f)
 
     doc = build_summary(pairs)
     write_yaml(out_path, doc)
 
+    if doc["data_log"]["metrics_per_step"] == "ERROR":
+        per_ts = Counter(ts for ts, _ in pairs)
+        exp = next(iter(per_ts.values()), 0)
+        for ts in sorted(per_ts, key=int):
+            if per_ts[ts] != exp:
+                print(ts, per_ts[ts], file=sys.stdout)
+
 
 if __name__ == "__main__":
     main()

roles/telemetry_chargeback/files/gen_synth_loki_data.py

@@ -48,7 +48,11 @@ def _get_value_for_step(
 logging.basicConfig(
     level=logging.INFO,
     format='%(asctime)s - %(levelname)s - %(message)s',
+<<<<<<< HEAD
     datefmt='%Y-%m-%dT%H:%M:%S+00:00'
+=======
+    datefmt='%Y-%m-%dT%H:%M:%SZ'
+>>>>>>> 3f9a12c (add the ability to pass test scenario names via)
 )
 logger = logging.getLogger()
 
@@ -167,13 +171,13 @@ def generate_loki_data(
             "end_time": end_str
         })
 
-        # Track the last end epoch
-        last_end_of_step_epoch = end_of_step_epoch
-
     # Add final entry that ends at end_epoch (current time)
-    if log_data_list and end_epoch > start_epoch and last_end_of_step_epoch:
+    if log_data_list and end_epoch > start_epoch:
         # Calculate start of final entry based on end of last generated entry
-        final_start_epoch = last_end_of_step_epoch + 1
+        last_entry_end = log_data_list[-1]["end_time"]
+        # Parse the last entry's end time to get the epoch
+        last_end_dt = datetime.fromisoformat(last_entry_end)
+        final_start_epoch = int(last_end_dt.timestamp()) + 1
         final_nanoseconds = int(final_start_epoch * 1_000_000_000)
 
         # Only add if the final entry would have a valid duration
@@ -225,11 +229,8 @@ def generate_loki_data(
 
         # Validate required fields
         # metadata is optional for generation; name is not a log-type field
-        required_for_item = [
-            f for f in required_fields
-            if f not in ("name", "metadata")
-        ]
-        missing = [f for f in required_for_item if f not in log_type_config]
+        required_for_item = set(required_fields) - {"name", "metadata"}
+        missing = required_for_item - set(log_type_config)
         if missing:
             logger.error(
                 f"Missing required fields in {type_key!r} config: {missing}"
@@ -410,7 +411,6 @@ def _str_to_bool(value: str) -> bool:
             f"Boolean value expected. Got: {value}"
         )
 
-
 def main():
     """Main entry point for the script."""
     parser = argparse.ArgumentParser(
@@ -469,11 +469,7 @@ def main():
         config = load_config(args.test)
     except (FileNotFoundError, ValueError) as e:
         logger.critical(f"Failed to load config: {e}")
-        sys.exit(1)
-
-    # Derive scenario name from test file path
-    scenario_name = args.test.stem
-    logger.info(f"Derived scenario name from test file: {scenario_name}")
+        return
 
     # Get generation parameters from config
     generation_config = config.get("generation", {})

roles/telemetry_chargeback/tasks/gen_synth_loki_data.yml

@@ -10,28 +10,24 @@
     path: "{{ cloudkitty_data_file }}"
   register: file_preexists
 
-- name: "TEST Generate Synthetic Data for {{ scenario_name }}"
+- name: "Generate Synthetic Data for {{ scenario_name }}"
   ansible.builtin.command:
     cmd: >
       python3 "{{ cloudkitty_synth_script }}"
       --tmpl "{{ cloudkitty_data_template }}"
       -t "{{ cloudkitty_test_file }}"
       -o "{{ cloudkitty_data_file }}"
-      --debug "{{ cloudkitty_debug }}"
   register: cloudkitty_data_file_output
   when: not file_preexists.stat.exists | bool
   changed_when: cloudkitty_data_file_output.rc == 0
-  any_errors_fatal: true
 
-- name: "TEST Generate chargeback rating from synthetic data file {{ scenario_name }}"
+- name: "Generate chargeback rating from synthetic data file {{ scenario_name }}"
   ansible.builtin.command:
     cmd: >
       python3 "{{ cloudkitty_summary_script }}"
       -j "{{ cloudkitty_data_file }}"
       -o "{{ cloudkitty_synth_totals_file }}"
-      --debug "{{ cloudkitty_debug }}"
-      --debug_dir "{{ cloudkitty_debug_dir }}"
+      --debug "{{ cloudkitty_debug_dir }}"
   register: cloudkitty_synth_totals_file_output
   when: not file_preexists.stat.exists | bool
   changed_when: cloudkitty_synth_totals_file_output.rc == 0
-  any_errors_fatal: true