Merge pull request #149 from CyberSecDef/copilot/fix-brace-counting-json-parser

Fix /llm_log endpoint: replace brace-counting parser with one-JSON-per-line format

Author: CyberSecDef

novelforge/__init__.py

@@ -1,5 +1,6 @@
 """NovelForge – Flask application factory."""
 
+import collections
 import json
 import logging
 import sys
@@ -236,38 +237,22 @@ def get_llm_log() -> Response:
             return jsonify({"entries": []})
 
         try:
-            entries = []
+            window: collections.deque = collections.deque(maxlen=10)
             with open(log_path, "r", encoding="utf-8") as f:
-                content = f.read()
-
-            json_objects = []
-            current_obj = ""
-            brace_count = 0
-
-            for line in content.split('\n'):
-                if line.strip().startswith('{') and brace_count == 0:
-                    if current_obj:
-                        json_objects.append(current_obj)
-                    current_obj = line + '\n'
-                    brace_count = line.count('{') - line.count('}')
-                elif brace_count > 0:
-                    current_obj += line + '\n'
-                    brace_count += line.count('{') - line.count('}')
-                    if brace_count == 0:
-                        json_objects.append(current_obj)
-                        current_obj = ""
-
-            if current_obj:
-                json_objects.append(current_obj)
-
-            for obj_str in json_objects[-10:]:
-                try:
-                    entry = json.loads(obj_str)
-                    entries.append(entry)
-                except json.JSONDecodeError:
-                    continue
-
-            return jsonify({"entries": entries})
+                for line in f:
+                    stripped = line.strip()
+                    if not stripped:
+                        continue
+
+                    try:
+                        window.append(json.loads(stripped))
+                    except json.JSONDecodeError:
+                        # llm.log is treated as JSONL: one complete JSON object per
+                        # non-empty line. Skip malformed/incomplete lines rather than
+                        # trying to resynchronise based on "{" prefixes, which can
+                        # corrupt pretty-printed nested JSON content.
+                        continue
+            return jsonify({"entries": list(window)})
         except Exception as e:
             logger.error(f"Error reading LLM log: {e}")
             return jsonify({"entries": [], "error": str(e)})

novelforge/llm/client.py

@@ -281,7 +281,7 @@ def _log_llm_error(
     if exception_chain:
         error_log["exception_chain"] = exception_chain
 
-    llm_logger.info(json.dumps(error_log, indent=2))
+    llm_logger.info(json.dumps(error_log))
     logger.error(
         "LLM error [%s] action=%s attempt=%d/%d status=%s: %s | response_body=%s",
         error_type, action, attempt, MAX_RETRIES, status_code,
@@ -350,12 +350,12 @@ def _build_request(
         "provider": provider.label,
         "url": provider.url,
         "headers": {
-            "Authorization": f"Bearer {provider.api_key[:8]}..." if provider.api_key else "None",
+            "Authorization": "Bearer [REDACTED]" if provider.api_key else "None",
             "Content-Type": "application/json",
         },
         "payload": payload,
     }
-    llm_logger.info(json.dumps(request_log, indent=2))
+    llm_logger.info(json.dumps(request_log))
 
     return headers, payload
 
@@ -383,7 +383,7 @@ def _handle_success(
         "headers": dict(resp.headers),
         "response": data,
     }
-    llm_logger.info(json.dumps(response_log, indent=2))
+    llm_logger.info(json.dumps(response_log))
 
     # Accumulate token usage if available
     usage = data.get("usage")

novelforge/llm/image.py

@@ -48,7 +48,7 @@ def call_image_api(prompt: str, *, filename_prefix: str = "illustration") -> str
         "model": config.IMAGE_MODEL,
         "prompt": prompt[:200] + "..." if len(prompt) > 200 else prompt,
     }
-    llm_logger.info(json.dumps(request_log, indent=2))
+    llm_logger.info(json.dumps(request_log))
 
     for attempt in range(1, MAX_RETRIES + 1):
         try:

tests/test_integration.py

@@ -767,6 +767,87 @@ def test_clear_log_available(self, client):
         r = client.post("/clear_log")
         assert r.status_code == 200
 
+    @pytest.fixture
+    def _isolated_app(self, tmp_path, monkeypatch):
+        """Create an app with all dirs isolated under tmp_path."""
+        import novelforge.config as cfg
+        from novelforge import create_app, limiter
+
+        for attr, subdir in (
+            ("NOVELS_DIR", "novels"),
+            ("LOGS_DIR", "logs"),
+            ("SESSION_FILE_DIR", "sessions"),
+            ("EXPORT_DIR", "exports"),
+        ):
+            d = tmp_path / subdir
+            d.mkdir()
+            monkeypatch.setattr(cfg, attr, str(d))
+
+        flask_app = create_app(testing=True)
+        flask_app.config["SECRET_KEY"] = "test-secret"
+        flask_app.config["WTF_CSRF_ENABLED"] = False
+        limiter.enabled = False
+        return flask_app, tmp_path / "logs"
+
+    def test_llm_log_parses_single_line_entries(self, _isolated_app):
+        """One-JSON-per-line log format is parsed correctly."""
+        flask_app, logs_dir = _isolated_app
+        entry1 = {"type": "request", "action": "test1", "timestamp": "2024-01-01 00:00:00"}
+        entry2 = {"type": "response", "action": "test2", "timestamp": "2024-01-01 00:00:01"}
+        (logs_dir / "llm.log").write_text(
+            json.dumps(entry1) + "\n" + json.dumps(entry2) + "\n",
+            encoding="utf-8",
+        )
+
+        with flask_app.test_client() as c:
+            r = c.get("/llm_log")
+
+        assert r.status_code == 200
+        data = r.get_json()
+        assert len(data["entries"]) == 2
+        assert data["entries"][0]["type"] == "request"
+        assert data["entries"][1]["type"] == "response"
+
+    def test_llm_log_parses_entries_with_braces_in_strings(self, _isolated_app):
+        """Brace characters inside JSON string values do not break parsing."""
+        flask_app, logs_dir = _isolated_app
+        entry = {
+            "type": "request",
+            "action": "test",
+            "timestamp": "2024-01-01 00:00:00",
+            "payload": {"key": "value with { brace } and another {brace}"},
+        }
+        (logs_dir / "llm.log").write_text(json.dumps(entry) + "\n", encoding="utf-8")
+
+        with flask_app.test_client() as c:
+            r = c.get("/llm_log")
+
+        assert r.status_code == 200
+        data = r.get_json()
+        assert len(data["entries"]) == 1
+        assert data["entries"][0]["payload"]["key"] == "value with { brace } and another {brace}"
+
+    def test_llm_log_returns_last_ten_entries(self, _isolated_app):
+        """Only the last 10 entries are returned when the log has more."""
+        flask_app, logs_dir = _isolated_app
+        entries = [
+            {"type": "request", "seq": i, "timestamp": f"2024-01-01 00:00:{i:02d}"}
+            for i in range(15)
+        ]
+        (logs_dir / "llm.log").write_text(
+            "\n".join(json.dumps(e) for e in entries) + "\n",
+            encoding="utf-8",
+        )
+
+        with flask_app.test_client() as c:
+            r = c.get("/llm_log")
+
+        assert r.status_code == 200
+        data = r.get_json()
+        assert len(data["entries"]) == 10
+        assert data["entries"][0]["seq"] == 5
+        assert data["entries"][-1]["seq"] == 14
+
 
 class TestNovelforgeDebugEnvVar:
     """Verify that NOVELFORGE_DEBUG controls debug mode in app.py entrypoint."""