Merge pull request #19 from codejunkie99/fix/episodic-write-locking

harden: lock episodic writes + fix pi install.sh silent orphan

Author: codejunkie99

.agent/harness/hooks/_episodic_io.py

@@ -0,0 +1,45 @@
+"""Cross-platform locked append for episodic JSONL writes.
+
+POSIX `write(2)` in O_APPEND mode is atomic for payloads up to PIPE_BUF
+(4 KB on Linux, 512 B minimum per POSIX). Most episodic entries fit,
+but failure entries with reflection + context + detail can exceed that,
+and two harness hooks writing from the same process (or from two Pi
+sessions on the same repo) can interleave bytes mid-line. Silent
+corruption is worse than a visible error because every downstream
+reader (`auto_dream.py`, `cluster.py`, `context_budget.py`,
+`show.py`) skips `JSONDecodeError` lines without surfacing the loss.
+
+This module serializes appends with `fcntl.flock(LOCK_EX)` on POSIX.
+On platforms without `fcntl` (native Windows Python) the lock is a
+no-op and behavior matches the pre-lock baseline. WSL, git-bash via
+Cygwin, macOS, and Linux all provide `fcntl`.
+"""
+import json
+import os
+
+try:
+    import fcntl  # POSIX
+    _HAVE_FLOCK = True
+except ImportError:
+    _HAVE_FLOCK = False
+
+
+def append_jsonl(path: str, entry: dict) -> dict:
+    """Serialize `entry` to one JSON line and append to `path`.
+
+    Uses `open(..., "ab")` (append-binary) to bypass Python's text-mode
+    buffering and guarantee a single `write(2)` per call. `fcntl.flock`
+    provides cross-process mutual exclusion on POSIX.
+    """
+    payload = (json.dumps(entry) + "\n").encode("utf-8")
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "ab") as f:
+        if _HAVE_FLOCK:
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+        try:
+            f.write(payload)
+            f.flush()
+        finally:
+            if _HAVE_FLOCK:
+                fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+    return entry

.agent/harness/hooks/on_failure.py

@@ -1,6 +1,7 @@
 """Failures are learning. High pain score + rewrite flag after repeat offenses."""
 import json, datetime, os
 from ._provenance import build_source
+from ._episodic_io import append_jsonl
 
 ROOT = os.path.join(os.path.dirname(__file__), "..", "..")
 EPISODIC = os.path.join(ROOT, "memory/episodic/AGENT_LEARNINGS.jsonl")
@@ -69,7 +70,4 @@ def on_failure(skill_name, action, error, context="", confidence=0.9,
             f"Flag for rewrite."
         )
         entry["pain_score"] = 10
-    os.makedirs(os.path.dirname(EPISODIC), exist_ok=True)
-    with open(EPISODIC, "a") as f:
-        f.write(json.dumps(entry) + "\n")
-    return entry
+    return append_jsonl(EPISODIC, entry)

.agent/harness/hooks/post_execution.py

@@ -1,6 +1,7 @@
 """Runs after every action. Appends a structured entry to episodic memory."""
-import json, datetime, os
+import datetime, os
 from ._provenance import build_source
+from ._episodic_io import append_jsonl
 
 ROOT = os.path.join(os.path.dirname(__file__), "..", "..")
 EPISODIC = os.path.join(ROOT, "memory/episodic/AGENT_LEARNINGS.jsonl")
@@ -15,7 +16,6 @@ def log_execution(skill_name, action, result, success, reflection="",
     a higher value (e.g. 5) for high-importance successful operations so
     recurring patterns cross the dream-cycle promotion threshold (7.0).
     """
-    os.makedirs(os.path.dirname(EPISODIC), exist_ok=True)
     if pain_score is None:
         pain_score = 2 if success else 7
     entry = {
@@ -31,6 +31,4 @@ def log_execution(skill_name, action, result, success, reflection="",
         "source": build_source(skill_name),
         "evidence_ids": list(evidence_ids) if evidence_ids else [],
     }
-    with open(EPISODIC, "a") as f:
-        f.write(json.dumps(entry) + "\n")
-    return entry
+    return append_jsonl(EPISODIC, entry)

install.sh

@@ -133,15 +133,31 @@ case "$ADAPTER" in
       echo "  + AGENTS.md"
     fi
     mkdir -p "$TARGET/.pi"
-    # symlink .pi/skills -> .agent/skills so pi sees the one true skill tree.
-    # ln -sfn atomically replaces an existing symlink; fall back to cp -R
-    # on filesystems that don't support symlinks (e.g. Windows without dev mode).
+    # Keep .pi/skills in sync with .agent/skills (the one true skill tree).
+    # Handle three shapes explicitly: `ln -sfn src dest` against a REAL
+    # directory silently creates `dest/<basename-of-src>` INSIDE the dir
+    # on macOS/Linux and exits 0, which would leave stale orphans forever.
+    # Check -L before -d so existing symlinks take the cheap repoint path,
+    # and reserve the rsync path for real dirs left from a prior copy
+    # fallback install.
     SKILLS_SRC="$(cd "$TARGET/.agent/skills" && pwd)"
-    if ln -sfn "$SKILLS_SRC" "$TARGET/.pi/skills" 2>/dev/null; then
+    SKILLS_DEST="$TARGET/.pi/skills"
+    if [[ -L "$SKILLS_DEST" ]]; then
+      ln -sfn "$SKILLS_SRC" "$SKILLS_DEST"
+      echo "  + .pi/skills -> $SKILLS_SRC"
+    elif [[ -d "$SKILLS_DEST" ]]; then
+      if command -v rsync >/dev/null 2>&1; then
+        rsync -a --delete "$SKILLS_SRC/" "$SKILLS_DEST/"
+        echo "  ~ synced .agent/skills → .pi/skills (rsync --delete)"
+      else
+        rm -rf "$SKILLS_DEST"
+        cp -R "$SKILLS_SRC" "$SKILLS_DEST"
+        echo "  ~ replaced .pi/skills with current .agent/skills (no rsync)"
+      fi
+    elif ln -sfn "$SKILLS_SRC" "$SKILLS_DEST" 2>/dev/null; then
       echo "  + .pi/skills -> $SKILLS_SRC"
     else
-      rm -rf "$TARGET/.pi/skills"
-      cp -R "$SKILLS_SRC" "$TARGET/.pi/skills"
+      cp -R "$SKILLS_SRC" "$SKILLS_DEST"
       echo "  + .pi/skills (copy; symlink not supported here)"
     fi
     ;;