Successfully replaced 4 block(s) in /home/sebas/agents-database/src/shared_agent_memory/service.py.

    1 from __future__ import annotations
    2
    3 import json
    4 import math
+   5 import re
    5 import subprocess
    6 import uuid
    7 from dataclasses import dataclass
    8 from datetime import UTC, datetime, timedelta
      ...
   24 def normalize_text(value: str) -> str:
   25     return " ".join(value.lower().split())
   26
   27
+  29 def normalize_for_dedupe(value: str) -> str:
+  30     text = value.lower()
+  31     text = re.sub(r"\[telegram\]", " ", text)
+  32     text = re.sub(r"https?://\S+", " URL ", text)
+  33     text = re.sub(r"\b(task|run|job|session)_[a-z0-9_-]+\b", " ID ", text)
+  34     text = re.sub(r"\b\d{4}-\d{2}-\d{2}[t ][^\s]+\b", " TS ", text)
+  35     text = re.sub(r"payload:\s*\{.*", " payload_json ", text, flags=re.IGNORECASE | re.DOTALL)
+  36     text = re.sub(r'"route"\s*:\s*\{.*', " route_json ", text, flags=re.IGNORECASE | re.DOTALL)
+  37     text = re.sub(r"[^\w\s]", " ", text)
+  38     return " ".join(text.split())
+  39
+  40
   28 def _line_value(content: str, label: str) -> str:
   29     prefix = f"{label}:"
   30     for line in content.splitlines():
   31         if line.startswith(prefix):
      ...
  367                     (candidate["id"], candidate["type"], candidate["scope"]),
  368                 ).fetchall()
  369                 related_memories = [self._row_to_memory(row) for row in related]
  370
+ 384                 candidate_norm = normalize_for_dedupe(candidate["content"])
  371                 best_match = None
  372                 best_score = 0.0
  373                 for other in related_memories:
- 374                     lexical = self._lexical_score(
- 375                         normalize_text(candidate["content"]).split(),
- 376                         normalize_text(other["content"]),
- 377                     )
+ 388                     other_norm = normalize_for_dedupe(other["content"])
+ 389                     lexical = self._lexical_score(candidate_norm.split(), other_norm)
  378                     semantic = self._semantic_score(candidate.get("embedding") or [], other.get("embedding"))
  379                     score = lexical if semantic == 0.0 else (lexical * 0.6 + semantic * 0.4)
  380                     if score > best_score:
  381                         best_score = score
  382                         best_match = other
  383
- 384                 if best_match and candidate["content"] == best_match["content"]:
+ 396                 if best_match and candidate_norm == normalize_for_dedupe(best_match["content"]):
  385                     self._link_memories(conn, candidate["id"], best_match["id"], "related_to", "exact_duplicate")
  386                     conn.execute(
  387                         "UPDATE memories SET status = ?, updated_at = ? WHERE id = ?",
  388                         ("archived", utcnow(), candidate["id"]),
      ...
  440                         "UPDATE memories SET status = ?, updated_at = ? WHERE id = ?",
  441                         ("active", utcnow(), candidate["id"]),
  442                     )
  443                     stats["promoted"] += 1
- 444                 elif candidate["status"] == "active" and candidate["type"] in {"episode", "artifact", "decision", "project", "procedural", "semantic"} and candidate.get("confidence", 0.0) >= promote_threshold:
- 445                     # Normalize qualifying active records as already-promoted durable memories.
- 446                     stats["promoted"] += 1
  447
  448         return stats
  449
  450     def upsert_profile_fact(
      ...

--- /home/sebas/agents-database/src/shared_agent_memory/service.py
+++ /home/sebas/agents-database/src/shared_agent_memory/service.py
@@ -1,8 +1,9 @@
 from __future__ import annotations

 import json
 import math
+import re
 import subprocess
 import uuid
 from dataclasses import dataclass
 from datetime import UTC, datetime, timedelta
@@ -24,8 +25,20 @@
 def normalize_text(value: str) -> str:
     return " ".join(value.lower().split())

+def normalize_for_dedupe(value: str) -> str:
+    text = value.lower()
+    text = re.sub(r"\[telegram\]", " ", text)
+    text = re.sub(r"https?://\S+", " URL ", text)
+    text = re.sub(r"\b(task|run|job|session)_[a-z0-9_-]+\b", " ID ", text)
+    text = re.sub(r"\b\d{4}-\d{2}-\d{2}[t ][^\s]+\b", " TS ", text)
+    text = re.sub(r"payload:\s*\{.*", " payload_json ", text, flags=re.IGNORECASE | re.DOTALL)
+    text = re.sub(r'"route"\s*:\s*\{.*', " route_json ", text, flags=re.IGNORECASE | re.DOTALL)
+    text = re.sub(r"[^\w\s]", " ", text)
+    return " ".join(text.split())
+
+
 def _line_value(content: str, label: str) -> str:
     prefix = f"{label}:"
     for line in content.splitlines():
         if line.startswith(prefix):
@@ -367,22 +380,21 @@
                     (candidate["id"], candidate["type"], candidate["scope"]),
                 ).fetchall()
                 related_memories = [self._row_to_memory(row) for row in related]

+                candidate_norm = normalize_for_dedupe(candidate["content"])
                 best_match = None
                 best_score = 0.0
                 for other in related_memories:
-                    lexical = self._lexical_score(
-                        normalize_text(candidate["content"]).split(),
-                        normalize_text(other["content"]),
-                    )
+                    other_norm = normalize_for_dedupe(other["content"])
+                    lexical = self._lexical_score(candidate_norm.split(), other_norm)
                     semantic = self._semantic_score(candidate.get("embedding") or [], other.get("embedding"))
                     score = lexical if semantic == 0.0 else (lexical * 0.6 + semantic * 0.4)
                     if score > best_score:
                         best_score = score
                         best_match = other

-                if best_match and candidate["content"] == best_match["content"]:
+                if best_match and candidate_norm == normalize_for_dedupe(best_match["content"]):
                     self._link_memories(conn, candidate["id"], best_match["id"], "related_to", "exact_duplicate")
                     conn.execute(
                         "UPDATE memories SET status = ?, updated_at = ? WHERE id = ?",
                         ("archived", utcnow(), candidate["id"]),
@@ -440,11 +452,8 @@
                         "UPDATE memories SET status = ?, updated_at = ? WHERE id = ?",
                         ("active", utcnow(), candidate["id"]),
                     )
                     stats["promoted"] += 1
-                elif candidate["status"] == "active" and candidate["type"] in {"episode", "artifact", "decision", "project", "procedural", "semantic"} and candidate.get("confidence", 0.0) >= promote_threshold:
-                    # Normalize qualifying active records as already-promoted durable memories.
-                    stats["promoted"] += 1

         return stats

     def upsert_profile_fact(


**Considering Python compilation**

I’m thinking about needing `py_compile`, which is a module for compiling Python source files to bytecode. Maybe I should also consider a dry run, which helps test the code without actually executing it, just to check for errors. I want to ensure I'm prepared for whatever the user needs. It seems like having this toolset could streamline my process and make it easier to troubleshoot any issues that come up.

bash

python3 -m py_compile /home/sebas/pi-config/scripts/session-memory-analyzer.py /home/sebas/agents-database/src/shared_agent_memory/service.py

python3 - <<'PY'
import runpy
ns=runpy.run_path('/home/sebas/pi-config/scripts/session-memory-analyzer.py')
print('loaded analyzer ok')
PY


(no output)


loaded analyzer ok


**Testing dry run logic**

I think I need to test the updated logic live, but without applying it just yet. Maybe I can run a dry run with a script to import the service and call against a copy. That feels safer! I'll mimic the new normalize_for_dedupe process and avoid using any active-promoted inflation in this dry run. Custom code from the service might also be a good option for this setup. It’s all about ensuring accuracy!

python3 - <<'PY'
import json, sqlite3, sys
from datetime import UTC, datetime, timedelta
sys.path.insert(0, '/home/sebas/agents-database/src')
from shared_agent_memory.service import normalize_for_dedupe, MemoryService

p='/home/sebas/agents-database/data/shared-agent-memory.sqlite3'
svc=MemoryService(p)
con=sqlite3.connect(p)
con.row_factory=sqlite3.Row
cur=con.cursor()
config={
    'candidate_age_days':0,
    'max_candidates':200,
    'dedupe_high_threshold':0.9,
    'dedupe_mid_threshold':0.75,
    'promote_confidence':0.7,
    'conflict_threshold':0.55,
    'llm_enabled':False,
}
cutoff=(datetime.now(UTC)-timedelta(days=int(config['candidate_age_days']))).isoformat()
rows=cur.execute("""
SELECT * FROM memories
WHERE (status='inbox' OR (type='episode' AND status='active'))
  AND created_at <= ?
ORDER BY created_at ASC
LIMIT ?
""", (cutoff, int(config['max_candidates']))).fetchall()
candidates=[svc._row_to_memory(r) for r in rows]
stats={'candidates':len(candidates),'promoted':0,'archived':0,'linked':0,'conflicts':0}
examples={'archived':[],'linked':[]}
for candidate in candidates:
    related=cur.execute("""
    SELECT * FROM memories WHERE status='active' AND id != ? AND type = ? AND scope = ?
    ORDER BY updated_at DESC LIMIT 50
    """, (candidate['id'], candidate['type'], candidate['scope'])).fetchall()
    related=[svc._row_to_memory(r) for r in related]
    candidate_norm=normalize_for_dedupe(candidate['content'])
    best_match=None; best_score=0.0
    for other in related:
        other_norm=normalize_for_dedupe(other['content'])
        lexical=svc._lexical_score(candidate_norm.split(), other_norm)
        semantic=svc._semantic_score(candidate.get('embedding') or [], other.get('embedding'))
        score=lexical if semantic==0.0 else (lexical*0.6+semantic*0.4)
        if score>best_score:
            best_score=score; best_match=other
    if best_match and candidate_norm == normalize_for_dedupe(best_match['content']):
        stats['archived'] += 1; stats['linked'] += 1
        if len(examples['archived'])<5:
            examples['archived'].append((candidate['id'], best_match['id'], candidate['title'][:80], best_match['title'][:80]))
        continue
    if best_match and best_score >= float(config['dedupe_high_threshold']):
        stats['archived'] += 1; stats['linked'] += 1
        if len(examples['archived'])<5:
            examples['archived'].append((candidate['id'], best_match['id'], candidate['title'][:80], best_match['title'][:80]))
        continue
    if best_match and best_score >= float(config['dedupe_mid_threshold']):
        stats['linked'] += 1
        if len(examples['linked'])<5:
            examples['linked'].append((candidate['id'], best_match['id'], round(best_score,3), candidate['title'][:80], best_match['title'][:80]))
    if best_match and best_score >= float(config['conflict_threshold']) and svc._conflict_signal(candidate, best_match):
        stats['conflicts'] += 1
    if candidate['status']=='inbox' and candidate.get('confidence',0.0) >= float(config['promote_confidence']):
        stats['promoted'] += 1
print(json.dumps(stats, indent=2))
print('archived_examples', json.dumps(examples['archived'], ensure_ascii=False, indent=2))
print('linked_examples', json.dumps(examples['linked'], ensure_ascii=False, indent=2))
PY


{
  "candidates": 67,
  "promoted": 0,
  "archived": 4,
  "linked": 9,
  "conflicts": 0
}
archived_examples [
  [
    "legacy_run_bfe3ad5d-ed7c-4727-bd72-7b9f5b7b0cc7",
    "legacy_run_2c54c32b-3cf1-426b-98a6-9c43971573ff",
    "Research run: Source 50+ inch China displays for Ballbox advertising in padel ve",
    "Research run: Find China-sourced 50+ inch displays for Ballbox advertising in co"
  ],
  [
    "mem_04dc02a90cad40cc826ed1e8f93aff36",
    "mem_17eea5c0658f4cd3beece90a93d35e36",
    "Subagent intake task_ed6a38c1b49f4e829e067d49d206148b",
    "Subagent intake task_2e1c07a93b9d40598bb4bfa504a5d726"
  ],
  [
    "mem_17eea5c0658f4cd3beece90a93d35e36",
    "mem_04dc02a90cad40cc826ed1e8f93aff36",
    "Subagent intake task_2e1c07a93b9d40598bb4bfa504a5d726",
    "Subagent intake task_ed6a38c1b49f4e829e067d49d206148b"
  ],
  [
    "mem_89d7f9db9cb04d0da7539c33e3ccc024",
    "mem_e80d2be0aecc4eed84d5aa179f83bfdd",
    "pi-config: You are running one deterministic phase-machine pass. Task folder: /h",
    "pi-config: You are running one deterministic phase-machine pass. Task folder: /h"
  ]
]
linked_examples [
  [
    "legacy_run_55b6eaa0-9ff5-438c-a375-50f0452079cb",
    "legacy_run_b07bc538-5cab-4243-a1d3-c7a7fa55a0a7",
    0.888,
    "Research run: Drive Ballbox payment integration toward Volvox-style demo on vend",
    "Research run: Drive ballbox-payment-integration to a demoable vending-machine QR"
  ],
  [
    "legacy_run_b07bc538-5cab-4243-a1d3-c7a7fa55a0a7",
    "legacy_run_a7fe5fbd-4892-4398-ab1a-71337f923ed7",
    0.768,
    "Research run: Drive ballbox-payment-integration to a demoable vending-machine QR",
    "Research run: Analyze the best next steps to make ballbox-payment-integration pr"
  ],
  [
    "legacy_run_35bd54a1-0135-414e-9c6b-245fca772f54",
    "legacy_run_1f1a3fd4-eb28-4e99-891f-07c97243a06f",
    0.852,
    "Research run: Production-readiness plan for ballbox-payment-integration",
    "Research run: Produce a production-readiness plan for ballbox-payment-integratio"
  ],
  [
    "mem_a986376af1554bec8e3909c656585bb6",
    "mem_e80d2be0aecc4eed84d5aa179f83bfdd",
    0.882,
    "pi-config: You are running one deterministic phase-machine pass. Task folder: /h",
    "pi-config: You are running one deterministic phase-machine pass. Task folder: /h"
  ],
  [
    "mem_e80d2be0aecc4eed84d5aa179f83bfdd",
    "mem_a986376af1554bec8e3909c656585bb6",
    0.878,
    "pi-config: You are running one deterministic phase-machine pass. Task folder: /h",
    "pi-config: You are running one deterministic phase-machine pass. Task folder: /h"
  ]
]


systemctl --user list-units --type=service --all | grep 'agents-database\|session-memory'

cd /home/sebas/agents-database && git diff -- src/shared_agent_memory/service.py && echo '---' && cd /home/sebas/pi-config && git diff -- scripts/session-memory-analyzer.py


  agents-database-http.service                 loaded    active   running Agents Database HTTP API
  agents-database-maintenance.service          loaded    active   running Agents Database Maintenance Daemon
  install-session-memory-dispatcher.service    loaded    active   running Pi session memory dispatcher
  session-memory-sweeper.service               loaded    inactive dead    Pi session memory sweeper


diff --git a/src/shared_agent_memory/service.py b/src/shared_agent_memory/service.py
index 1d4c1c4..d3d4961 100644
--- a/src/shared_agent_memory/service.py
+++ b/src/shared_agent_memory/service.py
@@ -2,6 +2,7 @@ from __future__ import annotations

 import json
 import math
+import re
 import subprocess
 import uuid
 from dataclasses import dataclass
@@ -25,6 +26,18 @@ def normalize_text(value: str) -> str:
     return " ".join(value.lower().split())

+def normalize_for_dedupe(value: str) -> str:
+    text = value.lower()
+    text = re.sub(r"\[telegram\]", " ", text)
+    text = re.sub(r"https?://\S+", " URL ", text)
+    text = re.sub(r"\b(task|run|job|session)_[a-z0-9_-]+\b", " ID ", text)
+    text = re.sub(r"\b\d{4}-\d{2}-\d{2}[t ][^\s]+\b", " TS ", text)
+    text = re.sub(r"payload:\s*\{.*", " payload_json ", text, flags=re.IGNORECASE | re.DOTALL)
+    text = re.sub(r'"route"\s*:\s*\{.*', " route_json ", text, flags=re.IGNORECASE | re.DOTALL)
+    text = re.sub(r"[^\w\s]", " ", text)
+    return " ".join(text.split())
+
+
 def _line_value(content: str, label: str) -> str:
     prefix = f"{label}:"
     for line in content.splitlines():
@@ -368,20 +381,19 @@ class MemoryService:
                 ).fetchall()
                 related_memories = [self._row_to_memory(row) for row in related]

+                candidate_norm = normalize_for_dedupe(candidate["content"])
                 best_match = None
                 best_score = 0.0
                 for other in related_memories:
-                    lexical = self._lexical_score(
-                        normalize_text(candidate["content"]).split(),
-                        normalize_text(other["content"]),
-                    )
+                    other_norm = normalize_for_dedupe(other["content"])
+                    lexical = self._lexical_score(candidate_norm.split(), other_norm)
                     semantic = self._semantic_score(candidate.get("embedding") or [], other.get("embedding"))
                     score = lexical if semantic == 0.0 else (lexical * 0.6 + semantic * 0.4)
                     if score > best_score:
                         best_score = score
                         best_match = other

-                if best_match and candidate["content"] == best_match["content"]:
+                if best_match and candidate_norm == normalize_for_dedupe(best_match["content"]):
                     self._link_memories(conn, candidate["id"], best_match["id"], "related_to", "exact_duplicate")
                     conn.execute(
                         "UPDATE memories SET status = ?, updated_at = ? WHERE id = ?",
@@ -441,9 +453,6 @@ class MemoryService:
                         ("active", utcnow(), candidate["id"]),
                     )
                     stats["promoted"] += 1
-                elif candidate["status"] == "active" and candidate["type"] in {"episode", "artifact", "decision", "project", "procedural", "semantic"} and candidate.get("confidence", 0.0) >= promote_threshold:
-                    # Normalize qualifying active records as already-promoted durable memories.
-                    stats["promoted"] += 1

         return stats

---
diff --git a/scripts/session-memory-analyzer.py b/scripts/session-memory-analyzer.py
index 2a195ea..5afad10 100644
--- a/scripts/session-memory-analyzer.py
+++ b/scripts/session-memory-analyzer.py
@@ -4,6 +4,7 @@ from __future__ import annotations
 import argparse
 import json
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -21,6 +22,19 @@ MAX_CANDIDATES = int(os.environ.get("PI_SESSION_MEMORY_MAX_CANDIDATES", "6"))
 ACTIVE_IMPORTANCE_THRESHOLD = float(os.environ.get("PI_SESSION_MEMORY_ACTIVE_IMPORTANCE_THRESHOLD", "0.92"))
 ACTIVE_CONFIDENCE_THRESHOLD = float(os.environ.get("PI_SESSION_MEMORY_ACTIVE_CONFIDENCE_THRESHOLD", "0.9"))
 DURABLE_TYPES = {"profile", "decision", "project", "artifact", "procedural", "semantic"}
+NOISE_PATTERNS = [
+    re.compile(pattern, re.IGNORECASE)
+    for pattern in (
+        r"you are running one deterministic",
+        r"phase-machine pass",
+        r"subagent intake task_",
+        r"^task id:\s*task_",
+        r"^payload:\s*\{",
+        r'"route"\s*:\s*\{',
+        r"you are iteration ",
+        r"continue the bounded ralph-loop",
+    )
+]

 def resolve_pi_bin() -> str:
@@ -206,6 +220,27 @@ def choose_status(memory_type: str, confidence: float, importance: float) -> str
     return "inbox"

+def is_noise_memory(candidate: dict[str, Any]) -> bool:
+    title = str(candidate.get("title") or "")
+    content = str(candidate.get("content") or "")
+    blob = f"{title}\n{content}".strip()
+    if not blob:
+        return True
+    lowered = blob.lower()
+    if any(pattern.search(blob) for pattern in NOISE_PATTERNS):
+        return True
+    if lowered.count("task_") >= 2:
+        return True
+    if lowered.count("payload:") >= 1 and lowered.count("route") >= 1:
+        return True
+    lines = [line.strip() for line in content.splitlines() if line.strip()]
+    if lines:
+        jsonish = sum(1 for line in lines if line.startswith(("Payload:", "{", "}", '"')) or '"' in line)
+        if jsonish / len(lines) >= 0.45:
+            return True
+    return False
+
+
 def build_memory_payload(candidate: dict[str, Any], meta: dict[str, Any], job_file: Path, analysis_reason: str | None, index: int) -> dict[str, Any]:
     memory_type = candidate.get("type") or "episode"
     confidence = float(candidate.get("confidence") or 0.6)
@@ -274,6 +309,8 @@ def main() -> int:
         saved_items: list[dict[str, Any]] = []

         for index, candidate in enumerate(candidates, start=1):
+            if is_noise_memory(candidate):
+                continue
             memory = build_memory_payload(candidate, meta, job_file, analysis.get("reason"), index)
             if not memory["content"].strip() or not memory["title"].strip():
                 continue


systemctl --user restart agents-database-http.service agents-database-maintenance.service && sleep 2 && systemctl --user --no-pager --full status agents-database-http.service agents-database-maintenance.service | sed -n '1,80p'


● agents-database-http.service - Agents Database HTTP API
     Loaded: loaded (/home/sebas/.config/systemd/user/agents-database-http.service; enabled; preset: enabled)
     Active: active (running) since Thu 2026-06-04 09:16:30 -03; 2s ago
 Invocation: 19e87166e4d94e6cad31282239602fa1
   Main PID: 599703 (python3)
      Tasks: 1 (limit: 9575)
        CPU: 136ms
     CGroup: /user.slice/user-1000.slice/user@1000.service/app.slice/agents-database-http.service
             └─599703 python3 -m shared_agent_memory.cli --db /home/sebas/agents-database/data/shared-agent-memory.sqlite3 serve-http --host 0.0.0.0 --port 8091

Jun 04 09:16:30 ballbox-first systemd[1289]: Started agents-database-http.service - Agents Database HTTP API.
Jun 04 09:16:30 ballbox-first run-http-api.sh[599703]: agents-database listening on http://0.0.0.0:8091

● agents-database-maintenance.service - Agents Database Maintenance Daemon
     Loaded: loaded (/home/sebas/.config/systemd/user/agents-database-maintenance.service; enabled; preset: enabled)
     Active: active (running) since Thu 2026-06-04 09:16:30 -03; 2s ago
 Invocation: 3682570e768d43b580b7450a037ebe83
   Main PID: 599704 (python)
      Tasks: 1 (limit: 9575)
        CPU: 118ms
     CGroup: /user.slice/user-1000.slice/user@1000.service/app.slice/agents-database-maintenance.service
             └─599704 /home/sebas/agents-database/.venv/bin/python -m shared_agent_memory.cli --db /home/sebas/agents-database/data/shared-agent-memory.sqlite3 maintenance-daemon --interval-seconds 900 --jitter-seconds 60

Jun 04 09:16:30 ballbox-first systemd[1289]: Started agents-database-maintenance.service - Agents Database Maintenance Daemon.


Updated #2 (in_progress → completed)

Dry run stricter memory consolidation/conflict detection

Inspect current consolidation logic, run a stricter dry-run against recent memories, report what conflicts/consolidations would surface without changing live data.

Implement memory noise filtering and normalized consolidation

Apply minimum safe changes to reduce system-noise memories, normalize dedupe, and make promotion metrics honest; validate with dry run.