From 8b60501529666d075fc50923469a8f8d0fdb2e45 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Apr 2026 18:31:18 +0000 Subject: [PATCH 1/7] Initial plan From 5a78f87db5fd1277fe526beb8a932e2a50c49139 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Apr 2026 18:38:35 +0000 Subject: [PATCH 2/7] Fix brace-counting JSON parser in /llm_log endpoint (Option A) Agent-Logs-Url: https://github.com/CyberSecDef/NovelForge/sessions/3e021d3b-8857-4593-a659-1eae83bff8f4 Co-authored-by: CyberSecDef <17597068+CyberSecDef@users.noreply.github.com> --- novelforge/__init__.py | 40 ++++------------- novelforge/llm/client.py | 6 +-- tests/test_integration.py | 95 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 34 deletions(-) diff --git a/novelforge/__init__.py b/novelforge/__init__.py index ac5190d..eee7487 100644 --- a/novelforge/__init__.py +++ b/novelforge/__init__.py @@ -236,38 +236,16 @@ def get_llm_log() -> Response: return jsonify({"entries": []}) try: - entries = [] + all_entries = [] with open(log_path, "r", encoding="utf-8") as f: - content = f.read() - - json_objects = [] - current_obj = "" - brace_count = 0 - - for line in content.split('\n'): - if line.strip().startswith('{') and brace_count == 0: - if current_obj: - json_objects.append(current_obj) - current_obj = line + '\n' - brace_count = line.count('{') - line.count('}') - elif brace_count > 0: - current_obj += line + '\n' - brace_count += line.count('{') - line.count('}') - if brace_count == 0: - json_objects.append(current_obj) - current_obj = "" - - if current_obj: - json_objects.append(current_obj) - - for obj_str in json_objects[-10:]: - try: - entry = json.loads(obj_str) - entries.append(entry) - except json.JSONDecodeError: - continue - - return jsonify({"entries": entries}) + for line in f: + if line.strip(): + try: + all_entries.append(json.loads(line)) + except json.JSONDecodeError: + continue + + return jsonify({"entries": all_entries[-10:]}) except Exception as e: logger.error(f"Error reading LLM log: {e}") return jsonify({"entries": [], "error": str(e)}) diff --git a/novelforge/llm/client.py b/novelforge/llm/client.py index b1d3df6..b14a706 100644 --- a/novelforge/llm/client.py +++ b/novelforge/llm/client.py @@ -281,7 +281,7 @@ def _log_llm_error( if exception_chain: error_log["exception_chain"] = exception_chain - llm_logger.info(json.dumps(error_log, indent=2)) + llm_logger.info(json.dumps(error_log)) logger.error( "LLM error [%s] action=%s attempt=%d/%d status=%s: %s | response_body=%s", error_type, action, attempt, MAX_RETRIES, status_code, @@ -355,7 +355,7 @@ def _build_request( }, "payload": payload, } - llm_logger.info(json.dumps(request_log, indent=2)) + llm_logger.info(json.dumps(request_log)) return headers, payload @@ -383,7 +383,7 @@ def _handle_success( "headers": dict(resp.headers), "response": data, } - llm_logger.info(json.dumps(response_log, indent=2)) + llm_logger.info(json.dumps(response_log)) # Accumulate token usage if available usage = data.get("usage") diff --git a/tests/test_integration.py b/tests/test_integration.py index eed7f92..4f7da36 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -767,6 +767,101 @@ def test_clear_log_available(self, client): r = client.post("/clear_log") assert r.status_code == 200 + def test_llm_log_parses_single_line_entries(self, tmp_path, monkeypatch): + """One-JSON-per-line log format is parsed correctly.""" + import json as _json + import novelforge.config as cfg + + logs_dir = tmp_path / "logs" + logs_dir.mkdir() + monkeypatch.setattr(cfg, "LOGS_DIR", str(logs_dir)) + + entry1 = {"type": "request", "action": "test1", "timestamp": "2024-01-01 00:00:00"} + entry2 = {"type": "response", "action": "test2", "timestamp": "2024-01-01 00:00:01"} + log_path = logs_dir / "llm.log" + log_path.write_text( + _json.dumps(entry1) + "\n" + _json.dumps(entry2) + "\n", + encoding="utf-8", + ) + + from novelforge import create_app, limiter + flask_app = create_app(testing=True) + flask_app.config["SECRET_KEY"] = "test-secret" + flask_app.config["WTF_CSRF_ENABLED"] = False + limiter.enabled = False + with flask_app.test_client() as c: + r = c.get("/llm_log") + + assert r.status_code == 200 + data = r.get_json() + assert len(data["entries"]) == 2 + assert data["entries"][0]["type"] == "request" + assert data["entries"][1]["type"] == "response" + + def test_llm_log_parses_entries_with_braces_in_strings(self, tmp_path, monkeypatch): + """Brace characters inside JSON string values do not break parsing.""" + import json as _json + import novelforge.config as cfg + + logs_dir = tmp_path / "logs" + logs_dir.mkdir() + monkeypatch.setattr(cfg, "LOGS_DIR", str(logs_dir)) + + entry = { + "type": "request", + "action": "test", + "timestamp": "2024-01-01 00:00:00", + "payload": {"key": "value with { brace } and another {brace}"}, + } + log_path = logs_dir / "llm.log" + log_path.write_text(_json.dumps(entry) + "\n", encoding="utf-8") + + from novelforge import create_app, limiter + flask_app = create_app(testing=True) + flask_app.config["SECRET_KEY"] = "test-secret" + flask_app.config["WTF_CSRF_ENABLED"] = False + limiter.enabled = False + with flask_app.test_client() as c: + r = c.get("/llm_log") + + assert r.status_code == 200 + data = r.get_json() + assert len(data["entries"]) == 1 + assert data["entries"][0]["payload"]["key"] == "value with { brace } and another {brace}" + + def test_llm_log_returns_last_ten_entries(self, tmp_path, monkeypatch): + """Only the last 10 entries are returned when the log has more.""" + import json as _json + import novelforge.config as cfg + + logs_dir = tmp_path / "logs" + logs_dir.mkdir() + monkeypatch.setattr(cfg, "LOGS_DIR", str(logs_dir)) + + entries = [ + {"type": "request", "seq": i, "timestamp": f"2024-01-01 00:00:{i:02d}"} + for i in range(15) + ] + log_path = logs_dir / "llm.log" + log_path.write_text( + "\n".join(_json.dumps(e) for e in entries) + "\n", + encoding="utf-8", + ) + + from novelforge import create_app, limiter + flask_app = create_app(testing=True) + flask_app.config["SECRET_KEY"] = "test-secret" + flask_app.config["WTF_CSRF_ENABLED"] = False + limiter.enabled = False + with flask_app.test_client() as c: + r = c.get("/llm_log") + + assert r.status_code == 200 + data = r.get_json() + assert len(data["entries"]) == 10 + assert data["entries"][0]["seq"] == 5 + assert data["entries"][-1]["seq"] == 14 + class TestNovelforgeDebugEnvVar: """Verify that NOVELFORGE_DEBUG controls debug mode in app.py entrypoint.""" From 20f73bed9a9cb52f7d5b5b8f64792492183f5db8 Mon Sep 17 00:00:00 2001 From: Robert Weber Date: Wed, 8 Apr 2026 14:40:31 -0400 Subject: [PATCH 3/7] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- novelforge/llm/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/novelforge/llm/client.py b/novelforge/llm/client.py index b14a706..552399a 100644 --- a/novelforge/llm/client.py +++ b/novelforge/llm/client.py @@ -350,7 +350,7 @@ def _build_request( "provider": provider.label, "url": provider.url, "headers": { - "Authorization": f"Bearer {provider.api_key[:8]}..." if provider.api_key else "None", + "Authorization": "Bearer [REDACTED]" if provider.api_key else "None", "Content-Type": "application/json", }, "payload": payload, From c1b820c6424ec10fd4ddd32cb0cacfc0d664925f Mon Sep 17 00:00:00 2001 From: Robert Weber Date: Wed, 8 Apr 2026 15:10:16 -0400 Subject: [PATCH 4/7] Update novelforge/__init__.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- novelforge/__init__.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/novelforge/__init__.py b/novelforge/__init__.py index eee7487..9c0f369 100644 --- a/novelforge/__init__.py +++ b/novelforge/__init__.py @@ -238,12 +238,40 @@ def get_llm_log() -> Response: try: all_entries = [] with open(log_path, "r", encoding="utf-8") as f: + buffered_lines: list[str] = [] + for line in f: - if line.strip(): + stripped = line.strip() + if not stripped: + continue + + if buffered_lines: + # If a new JSON object appears before the buffered content + # becomes valid, drop the incomplete buffer and resync. + if stripped.startswith("{"): + try: + all_entries.append(json.loads("\n".join(buffered_lines))) + buffered_lines = [] + except json.JSONDecodeError: + buffered_lines = [line.rstrip("\n")] + continue + + else: + buffered_lines.append(line.rstrip("\n")) + try: + all_entries.append(json.loads("\n".join(buffered_lines))) + buffered_lines = [] + except json.JSONDecodeError: + continue + + else: try: all_entries.append(json.loads(line)) except json.JSONDecodeError: - continue + if stripped.startswith("{"): + buffered_lines = [line.rstrip("\n")] + else: + continue return jsonify({"entries": all_entries[-10:]}) except Exception as e: From 1ede3c1b9cc629db7665164122fb4d30a7bd1c05 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Apr 2026 19:15:59 +0000 Subject: [PATCH 5/7] Address review feedback: deque memory bound, image.py indent=2, test isolation Agent-Logs-Url: https://github.com/CyberSecDef/NovelForge/sessions/f92c009b-bb39-4305-8e31-324b1a695711 Co-authored-by: CyberSecDef <17597068+CyberSecDef@users.noreply.github.com> --- novelforge/__init__.py | 11 +++--- novelforge/llm/image.py | 2 +- tests/test_integration.py | 76 ++++++++++++++++----------------------- 3 files changed, 38 insertions(+), 51 deletions(-) diff --git a/novelforge/__init__.py b/novelforge/__init__.py index 9c0f369..83e25ff 100644 --- a/novelforge/__init__.py +++ b/novelforge/__init__.py @@ -1,5 +1,6 @@ """NovelForge – Flask application factory.""" +import collections import json import logging import sys @@ -236,7 +237,7 @@ def get_llm_log() -> Response: return jsonify({"entries": []}) try: - all_entries = [] + window: collections.deque = collections.deque(maxlen=10) with open(log_path, "r", encoding="utf-8") as f: buffered_lines: list[str] = [] @@ -250,7 +251,7 @@ def get_llm_log() -> Response: # becomes valid, drop the incomplete buffer and resync. if stripped.startswith("{"): try: - all_entries.append(json.loads("\n".join(buffered_lines))) + window.append(json.loads("\n".join(buffered_lines))) buffered_lines = [] except json.JSONDecodeError: buffered_lines = [line.rstrip("\n")] @@ -259,21 +260,21 @@ def get_llm_log() -> Response: else: buffered_lines.append(line.rstrip("\n")) try: - all_entries.append(json.loads("\n".join(buffered_lines))) + window.append(json.loads("\n".join(buffered_lines))) buffered_lines = [] except json.JSONDecodeError: continue else: try: - all_entries.append(json.loads(line)) + window.append(json.loads(line)) except json.JSONDecodeError: if stripped.startswith("{"): buffered_lines = [line.rstrip("\n")] else: continue - return jsonify({"entries": all_entries[-10:]}) + return jsonify({"entries": list(window)}) except Exception as e: logger.error(f"Error reading LLM log: {e}") return jsonify({"entries": [], "error": str(e)}) diff --git a/novelforge/llm/image.py b/novelforge/llm/image.py index 21ae1ca..c8c17c4 100644 --- a/novelforge/llm/image.py +++ b/novelforge/llm/image.py @@ -48,7 +48,7 @@ def call_image_api(prompt: str, *, filename_prefix: str = "illustration") -> str "model": config.IMAGE_MODEL, "prompt": prompt[:200] + "..." if len(prompt) > 200 else prompt, } - llm_logger.info(json.dumps(request_log, indent=2)) + llm_logger.info(json.dumps(request_log)) for attempt in range(1, MAX_RETRIES + 1): try: diff --git a/tests/test_integration.py b/tests/test_integration.py index 4f7da36..bb6d112 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -767,28 +767,38 @@ def test_clear_log_available(self, client): r = client.post("/clear_log") assert r.status_code == 200 - def test_llm_log_parses_single_line_entries(self, tmp_path, monkeypatch): - """One-JSON-per-line log format is parsed correctly.""" - import json as _json + @pytest.fixture + def _isolated_app(self, tmp_path, monkeypatch): + """Create an app with all dirs isolated under tmp_path.""" import novelforge.config as cfg + from novelforge import create_app, limiter + + for attr, subdir in ( + ("NOVELS_DIR", "novels"), + ("LOGS_DIR", "logs"), + ("SESSION_FILE_DIR", "sessions"), + ("EXPORT_DIR", "exports"), + ): + d = tmp_path / subdir + d.mkdir() + monkeypatch.setattr(cfg, attr, str(d)) - logs_dir = tmp_path / "logs" - logs_dir.mkdir() - monkeypatch.setattr(cfg, "LOGS_DIR", str(logs_dir)) + flask_app = create_app(testing=True) + flask_app.config["SECRET_KEY"] = "test-secret" + flask_app.config["WTF_CSRF_ENABLED"] = False + limiter.enabled = False + return flask_app, tmp_path / "logs" + def test_llm_log_parses_single_line_entries(self, _isolated_app): + """One-JSON-per-line log format is parsed correctly.""" + flask_app, logs_dir = _isolated_app entry1 = {"type": "request", "action": "test1", "timestamp": "2024-01-01 00:00:00"} entry2 = {"type": "response", "action": "test2", "timestamp": "2024-01-01 00:00:01"} - log_path = logs_dir / "llm.log" - log_path.write_text( - _json.dumps(entry1) + "\n" + _json.dumps(entry2) + "\n", + (logs_dir / "llm.log").write_text( + json.dumps(entry1) + "\n" + json.dumps(entry2) + "\n", encoding="utf-8", ) - from novelforge import create_app, limiter - flask_app = create_app(testing=True) - flask_app.config["SECRET_KEY"] = "test-secret" - flask_app.config["WTF_CSRF_ENABLED"] = False - limiter.enabled = False with flask_app.test_client() as c: r = c.get("/llm_log") @@ -798,29 +808,17 @@ def test_llm_log_parses_single_line_entries(self, tmp_path, monkeypatch): assert data["entries"][0]["type"] == "request" assert data["entries"][1]["type"] == "response" - def test_llm_log_parses_entries_with_braces_in_strings(self, tmp_path, monkeypatch): + def test_llm_log_parses_entries_with_braces_in_strings(self, _isolated_app): """Brace characters inside JSON string values do not break parsing.""" - import json as _json - import novelforge.config as cfg - - logs_dir = tmp_path / "logs" - logs_dir.mkdir() - monkeypatch.setattr(cfg, "LOGS_DIR", str(logs_dir)) - + flask_app, logs_dir = _isolated_app entry = { "type": "request", "action": "test", "timestamp": "2024-01-01 00:00:00", "payload": {"key": "value with { brace } and another {brace}"}, } - log_path = logs_dir / "llm.log" - log_path.write_text(_json.dumps(entry) + "\n", encoding="utf-8") + (logs_dir / "llm.log").write_text(json.dumps(entry) + "\n", encoding="utf-8") - from novelforge import create_app, limiter - flask_app = create_app(testing=True) - flask_app.config["SECRET_KEY"] = "test-secret" - flask_app.config["WTF_CSRF_ENABLED"] = False - limiter.enabled = False with flask_app.test_client() as c: r = c.get("/llm_log") @@ -829,30 +827,18 @@ def test_llm_log_parses_entries_with_braces_in_strings(self, tmp_path, monkeypat assert len(data["entries"]) == 1 assert data["entries"][0]["payload"]["key"] == "value with { brace } and another {brace}" - def test_llm_log_returns_last_ten_entries(self, tmp_path, monkeypatch): + def test_llm_log_returns_last_ten_entries(self, _isolated_app): """Only the last 10 entries are returned when the log has more.""" - import json as _json - import novelforge.config as cfg - - logs_dir = tmp_path / "logs" - logs_dir.mkdir() - monkeypatch.setattr(cfg, "LOGS_DIR", str(logs_dir)) - + flask_app, logs_dir = _isolated_app entries = [ {"type": "request", "seq": i, "timestamp": f"2024-01-01 00:00:{i:02d}"} for i in range(15) ] - log_path = logs_dir / "llm.log" - log_path.write_text( - "\n".join(_json.dumps(e) for e in entries) + "\n", + (logs_dir / "llm.log").write_text( + "\n".join(json.dumps(e) for e in entries) + "\n", encoding="utf-8", ) - from novelforge import create_app, limiter - flask_app = create_app(testing=True) - flask_app.config["SECRET_KEY"] = "test-secret" - flask_app.config["WTF_CSRF_ENABLED"] = False - limiter.enabled = False with flask_app.test_client() as c: r = c.get("/llm_log") From 4c93d4a9f6dd6a4dc5d2c255af230702fce9cfa1 Mon Sep 17 00:00:00 2001 From: Robert Weber Date: Wed, 8 Apr 2026 15:30:25 -0400 Subject: [PATCH 6/7] Update novelforge/__init__.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- novelforge/__init__.py | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/novelforge/__init__.py b/novelforge/__init__.py index 83e25ff..fab7efd 100644 --- a/novelforge/__init__.py +++ b/novelforge/__init__.py @@ -239,41 +239,19 @@ def get_llm_log() -> Response: try: window: collections.deque = collections.deque(maxlen=10) with open(log_path, "r", encoding="utf-8") as f: - buffered_lines: list[str] = [] - for line in f: stripped = line.strip() if not stripped: continue - if buffered_lines: - # If a new JSON object appears before the buffered content - # becomes valid, drop the incomplete buffer and resync. - if stripped.startswith("{"): - try: - window.append(json.loads("\n".join(buffered_lines))) - buffered_lines = [] - except json.JSONDecodeError: - buffered_lines = [line.rstrip("\n")] - continue - - else: - buffered_lines.append(line.rstrip("\n")) - try: - window.append(json.loads("\n".join(buffered_lines))) - buffered_lines = [] - except json.JSONDecodeError: - continue - - else: - try: - window.append(json.loads(line)) - except json.JSONDecodeError: - if stripped.startswith("{"): - buffered_lines = [line.rstrip("\n")] - else: - continue - + try: + all_entries.append(json.loads(stripped)) + except json.JSONDecodeError: + # llm.log is treated as JSONL: one complete JSON object per + # non-empty line. Skip malformed/incomplete lines rather than + # trying to resynchronise based on "{" prefixes, which can + # corrupt pretty-printed nested JSON content. + continue return jsonify({"entries": list(window)}) except Exception as e: logger.error(f"Error reading LLM log: {e}") From 2f6c168aec57411bc5d2361b2b8a47464351026b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Apr 2026 19:33:17 +0000 Subject: [PATCH 7/7] Fix NameError: append to window deque, not undefined all_entries Agent-Logs-Url: https://github.com/CyberSecDef/NovelForge/sessions/147f948c-2221-4cd9-860c-498288b76c8d Co-authored-by: CyberSecDef <17597068+CyberSecDef@users.noreply.github.com> --- novelforge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/novelforge/__init__.py b/novelforge/__init__.py index fab7efd..9110c21 100644 --- a/novelforge/__init__.py +++ b/novelforge/__init__.py @@ -245,7 +245,7 @@ def get_llm_log() -> Response: continue try: - all_entries.append(json.loads(stripped)) + window.append(json.loads(stripped)) except json.JSONDecodeError: # llm.log is treated as JSONL: one complete JSON object per # non-empty line. Skip malformed/incomplete lines rather than