bugfixes

dashboard.html

@@ -629,6 +629,7 @@ async function deleteBook(src, groupEl) {
 }
 
 // ── MEMORIES ───────────────────────────────────────────────────
+
 async function loadMemories() {
   document.getElementById('memoriesLoading').style.display = 'block';
   document.getElementById('memoriesContent').innerHTML = '';
@@ -638,15 +639,16 @@ async function loadMemories() {
 
   try {
     await Promise.all(userIds.map(async uid => {
-      const r = await fetch(`${BASE}/memories/recent`, {
+      const r = await fetch(`${BASE}/memories/all`, {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ user_id: uid, limit: 100 })
+        body: JSON.stringify({ user_id: uid })
       });
       const data = await r.json();
       const items = data.results || [];
       if (items.length) allResults[uid] = items;
     }));
+
     stopLoading('memoriesLoading');
     renderMemories(allResults);
   } catch {
@@ -656,6 +658,7 @@ async function loadMemories() {
   }
 }
 
+
 function renderMemories(byUser) {
   const container = document.getElementById('memoriesContent');
   const allEntries = Object.values(byUser).flat();

mem0server.py

@@ -4,7 +4,7 @@ import json
 import sqlite3
 import httpx
 from fastapi import FastAPI, Request
-from fastapi.responses import JSONResponse, FileResponse, HTMLResponse
+from fastapi.responses import JSONResponse, HTMLResponse
 from mem0 import Memory
 
 # =============================================================================
@@ -19,7 +19,9 @@ RERANKER_URL = os.environ.get("RERANKER_URL", "http://192.168.0.200:5200/rerank"
 SQLITE_PATH  = os.path.expanduser("~/.mem0/history.db")
 
 # =============================================================================
-# SAFE JSON RESPONSE  (handles Infinity / NaN from Chroma / reranker scores)
+# SAFE JSON RESPONSE
+# Chroma and the reranker can emit Infinity/NaN which is invalid JSON.
+# Sanitize them to None before serializing.
 # =============================================================================
 
 def _sanitize(obj):
@@ -38,34 +40,97 @@ class SafeJSONResponse(JSONResponse):
         return json.dumps(_sanitize(content), ensure_ascii=False).encode("utf-8")
 
 
+# =============================================================================
+# METADATA SANITIZER
+# Chroma MetadataValue only accepts str, int, float, bool.
+# Drop None values; coerce anything else (lists, dicts) to str.
+# =============================================================================
+
+def sanitize_metadata(meta: dict) -> dict:
+    clean = {}
+    for k, v in meta.items():
+        if v is None:
+            continue
+        if isinstance(v, (str, int, float, bool)):
+            clean[k] = v
+        else:
+            clean[k] = str(v)
+    return clean
+
+
 # =============================================================================
 # PROMPTS
-# Edit these to change how each collection extracts and stores facts.
+# Mapped to MemoryConfig.custom_fact_extraction_prompt /
+#            MemoryConfig.custom_update_memory_prompt  (top-level fields).
+#
+# conversational — active, used by /memories on every add
+# knowledge      — defined for future use; currently bypassed because
+#                  /knowledge always stores verbatim (infer=False)
 # =============================================================================
 
+
 PROMPTS = {
-    # Used by /memories — conversational, user-centric recall for OpenClaw.
     "conversational": {
         "fact_extraction": """
-You are a personal memory assistant. Extract concise, standalone facts about
-the user from the conversation below. Write each fact as a single sentence
-starting with "User" — for example:
-  - "User is interested in generative music."
-  - "User is familiar with Python async patterns."
-  - "User prefers dark mode interfaces."
-Only extract facts that are clearly stated or strongly implied. Ignore filler,
-greetings, and opinions the user is uncertain about.
+You are an intelligent system that extracts useful long-term memory
+from a conversation.
+Your goal is to identify information that could help future interactions.
+Extract facts that describe:
+1. User preferences
+2. Important decisions
+3. Ongoing projects
+4. Tools or technologies being used
+5. Goals or plans
+6. Constraints or requirements
+7. Discoveries or conclusions
+8. Important context about tasks
+Ignore:
+- greetings
+- casual conversation
+- general world knowledge
+- temporary statements
+Return JSON:
+{
+ "facts": [
+   "fact 1",
+   "fact 2"
+ ]
+}
+Only include information that may be useful later.
+If nothing important is present return:
+{"facts": []}
 """.strip(),
+ 
         "update_memory": """
-You manage a long-term memory database for a personal AI assistant.
-You receive existing memories and new information. Update, merge, or add
-memories as needed. Keep each memory as a single concise sentence starting
-with "User". Remove duplicates and outdated facts.
+You manage a long-term memory database.
+You receive:
+1. existing stored memories
+2. new extracted facts
+For each fact decide whether to:
+ADD
+Create a new memory if it contains useful new information.
+UPDATE
+Modify an existing memory if the new fact refines or corrects it.
+DELETE
+Remove a memory if it is clearly outdated or incorrect.
+NONE
+Ignore the fact if it is redundant or trivial.
+Guidelines:
+- Prefer updating over adding duplicates
+- Keep memories concise
+- Avoid storing repeated information
+- Preserve important context
+Return JSON list:
+[
+ { "event": "ADD", "text": "..." },
+ { "event": "UPDATE", "id": "...", "text": "..." }
+]
 """.strip(),
     },
-
-    # Used by /knowledge — objective, source-neutral facts for book/doc ingest.
+ 
     "knowledge": {
+        # Not active during ingest (infer=False bypasses extraction).
+        # Kept here so it can be enabled if infer=True is ever needed.
         "fact_extraction": """
 You are a knowledge extraction system that reads source material and produces
 a list of objective, encyclopedic facts. Write each fact as a precise,
@@ -75,21 +140,26 @@ Examples:
   - "Silvio Gesell proposed demurrage as a mechanism to discourage hoarding of currency."
   - "The MIDI standard uses a 7-bit checksum for SysEx message validation."
 Only extract verifiable facts. Ignore meta-commentary and transitional prose.
+Return JSON: {"facts": ["fact 1", "fact 2"]}
 """.strip(),
+ 
         "update_memory": """
 You manage a knowledge base that stores objective facts extracted from books,
 documents, and reference material. You receive existing facts and new
 information. Update, merge, or add facts as needed. Keep each fact as a
 precise, self-contained sentence. Remove duplicates and outdated entries.
+Return JSON list: [{ "event": "ADD"|"UPDATE"|"DELETE"|"NONE", "text": "..." }]
 """.strip(),
     },
 }
 
 # =============================================================================
 # MEM0 CONFIG FACTORY
+# Prompts are top-level MemoryConfig fields — not nested inside llm.config.
 # =============================================================================
 
 def make_config(collection_name: str, prompt_key: str) -> dict:
+    prompts = PROMPTS[prompt_key]
     return {
         "llm": {
             "provider": "groq",
@@ -114,7 +184,9 @@ def make_config(collection_name: str, prompt_key: str) -> dict:
                 "ollama_base_url": "http://192.168.0.200:11434",
             },
         },
-        "custom_prompts": PROMPTS[prompt_key],
+        # Top-level MemoryConfig fields — confirmed from MemoryConfig source
+        "custom_fact_extraction_prompt": prompts["fact_extraction"],
+        "custom_update_memory_prompt":   prompts["update_memory"],
     }
 
 
@@ -126,7 +198,9 @@ memory_conv = Memory.from_config(make_config("openclaw_mem", "conversational"))
 memory_know = Memory.from_config(make_config("knowledge_mem", "knowledge"))
 
 # =============================================================================
-# CHROMA EMPTY-FILTER PATCH  (applied to both instances)
+# CHROMA EMPTY-FILTER PATCH
+# mem0 sometimes passes an empty filter dict to Chroma which raises an error.
+# Replace with a harmless always-true filter as fallback.
 # =============================================================================
 
 NOOP_WHERE = {"$and": [
@@ -172,10 +246,11 @@ memory_know.vector_store.search = make_safe_search(memory_know)
 
 # =============================================================================
 # RERANKER
+# Calls local reranker to re-order search results by relevance.
+# Falls back to raw mem0 order if unreachable.
 # =============================================================================
 
 def rerank_results(query: str, items: list, top_k: int) -> list:
-    """Re-order results via local reranker. Falls back gracefully."""
     if not items:
         return items
 
@@ -189,9 +264,10 @@ def rerank_results(query: str, items: list, top_k: int) -> list:
         resp.raise_for_status()
         reranked = resp.json()["results"]
     except Exception as exc:
-        print(f"[reranker] unavailable, skipping rerank: {exc}")
+        print(f"[reranker] unavailable, skipping: {exc}")
         return items[:top_k]
 
+    # Re-attach original mem0 metadata by matching text
     text_to_meta = {r.get("memory", ""): r for r in items}
     merged = []
     for r in reranked:
@@ -200,12 +276,15 @@ def rerank_results(query: str, items: list, top_k: int) -> list:
             merged.append({**meta, "rerank_score": r["score"]})
     return merged
 
+
 # =============================================================================
 # SQLITE HELPER
+# mem0 maintains a local SQLite history alongside Chroma.
+# Both must be cleaned together or deleted entries reappear after restart.
 # =============================================================================
 
 def sqlite_delete_ids(memory_ids: list[str]) -> int:
-    """Delete rows from mem0 SQLite by memory_id. Returns count deleted."""
+    """Delete rows by memory_id. Returns count deleted."""
     if not memory_ids:
         return 0
     try:
@@ -224,16 +303,14 @@ def sqlite_delete_ids(memory_ids: list[str]) -> int:
         print(f"[sqlite] warning: {e}")
         return 0
 
+
 # =============================================================================
 # CHROMA PAGINATION HELPER
+# mem0's get_all() is capped at 100 entries. This pages Chroma directly
+# in batches of 500 to retrieve the full collection without limits.
 # =============================================================================
 
 def chroma_get_all(collection, user_id: str, include: list = None) -> list[dict]:
-    """
-    Page through a Chroma collection in batches, filtering by user_id.
-    Returns list of dicts with 'id' and any included fields.
-    Bypasses mem0's 100-entry cap entirely.
-    """
     if include is None:
         include = ["metadatas"]
 
@@ -265,6 +342,7 @@ def chroma_get_all(collection, user_id: str, include: list = None) -> list[dict]
 
     return results
 
+
 # =============================================================================
 # SHARED HANDLERS
 # =============================================================================
@@ -277,21 +355,20 @@ async def handle_add(req: Request, mem: Memory, verbatim_allowed: bool = False):
     """
     Shared add handler for /memories and /knowledge.
 
-    /knowledge  (verbatim_allowed=True)  — always stores verbatim (infer=False).
-                                           The ingestor already summarised; skip
-                                           the second LLM pass.
-    /memories   (verbatim_allowed=False) — always uses LLM extraction for
-                                           conversational recall.
-
-    Supports:
-      - text      — raw string (legacy)
-      - messages  — list of {role, content} dicts (standard mem0)
-      - metadata  — dict, passed through to mem0
-      - user_id / userId
+    /knowledge  (verbatim_allowed=True)  — always infer=False. The ingestor
+                                           already summarised; skip the second
+                                           LLM extraction pass.
+    /memories   (verbatim_allowed=False) — always LLM extraction using the
+                                           conversational prompts above.
+
+    Accepts: text | messages, user_id, metadata.
+    Metadata is sanitized — Chroma rejects None and complex types.
     """
     data     = await req.json()
     user_id  = extract_user_id(data)
-    metadata = data.get("metadata") or {}
+#    metadata = sanitize_metadata(data.get("metadata") or {})
+    raw_meta = data.get("metadata")
+    metadata = sanitize_metadata(raw_meta) if raw_meta else None
     messages = data.get("messages")
     text     = data.get("text")
 
@@ -301,7 +378,7 @@ async def handle_add(req: Request, mem: Memory, verbatim_allowed: bool = False):
         )
 
     if verbatim_allowed:
-        # /knowledge — always verbatim, ingestor already summarised
+        # /knowledge — store verbatim, ingestor already did the summarisation
         content = text or " ".join(
             m["content"] for m in messages if m.get("role") == "user"
         )
@@ -309,17 +386,24 @@ async def handle_add(req: Request, mem: Memory, verbatim_allowed: bool = False):
         print(f"[add verbatim] user={user_id} chars={len(content)} meta={metadata}")
         return SafeJSONResponse(content=result)
 
-    # /memories — always LLM extraction
-    if messages:
-        result = mem.add(messages, user_id=user_id, metadata=metadata)
-    else:
-        result = mem.add(text, user_id=user_id, metadata=metadata)
+    # in the /memories path
+    kwargs = {"user_id": user_id}
+    if metadata:
+        kwargs["metadata"] = metadata
+    result = mem.add(messages or text, **kwargs)
+
+#    # /memories — LLM extracts and deduplicates facts from conversation
+#    if messages:
+#        result = mem.add(messages, user_id=user_id)
+#    else:
+#        result = mem.add(text, user_id=user_id)
 
     print(f"[add conversational] user={user_id} meta={metadata}")
     return SafeJSONResponse(content=result)
 
 
 async def handle_search(req: Request, mem: Memory):
+    """Semantic search with reranking. Fetches limit×3 candidates then reranks."""
     data    = await req.json()
     query   = (data.get("query") or "").strip()
     user_id = extract_user_id(data)
@@ -332,14 +416,15 @@ async def handle_search(req: Request, mem: Memory):
     try:
         result = mem.search(query, user_id=user_id, limit=fetch_k)
     except Exception:
+        # Fallback: get_all + simple text filter
         all_res = mem.get_all(user_id=user_id)
         items   = (
             all_res.get("results", [])
             if isinstance(all_res, dict)
             else (all_res if isinstance(all_res, list) else [])
         )
-        q     = query.lower()
-        items = [r for r in items if q in r.get("memory", "").lower()]
+        q      = query.lower()
+        items  = [r for r in items if q in r.get("memory", "").lower()]
         result = {"results": items}
 
     items = result.get("results", [])
@@ -349,6 +434,7 @@ async def handle_search(req: Request, mem: Memory):
 
 
 async def handle_recent(req: Request, mem: Memory):
+    """Return most recently created memories, sorted by created_at desc."""
     data    = await req.json()
     user_id = extract_user_id(data)
     if not user_id:
@@ -364,6 +450,7 @@ async def handle_recent(req: Request, mem: Memory):
     items = sorted(items, key=lambda r: r.get("created_at", ""), reverse=True)
     return SafeJSONResponse(content={"results": items[:limit]})
 
+
 # =============================================================================
 # APP
 # =============================================================================
@@ -372,7 +459,7 @@ app = FastAPI(title="mem0 server")
 
 
 # ---------------------------------------------------------------------------
-# DASHBOARD
+# DASHBOARD — served from file mounted via docker-compose volume
 # ---------------------------------------------------------------------------
 
 DASHBOARD_HTML = open("dashboard.html").read()
@@ -395,6 +482,7 @@ async def health():
             "conversational": "openclaw_mem",
             "knowledge":      "knowledge_mem",
         },
+        # Show first 80 chars of each prompt for quick verification
         "prompts": {
             k: {pk: pv[:80] + "…" for pk, pv in pv_dict.items()}
             for k, pv_dict in PROMPTS.items()
@@ -403,7 +491,7 @@ async def health():
 
 
 # ---------------------------------------------------------------------------
-# /memories  — conversational, OpenClaw
+# /memories — conversational collection (OpenClaw)
 # ---------------------------------------------------------------------------
 
 @app.post("/memories")
@@ -428,7 +516,7 @@ async def delete_memory(req: Request):
 
 
 # ---------------------------------------------------------------------------
-# /knowledge  — objective facts, book-ingestor
+# /knowledge — objective facts collection (book-ingestor)
 # ---------------------------------------------------------------------------
 
 @app.post("/knowledge")
@@ -456,7 +544,7 @@ async def delete_knowledge(req: Request):
 async def knowledge_sources(req: Request):
     """
     Return distinct source_file values with entry counts.
-    Pages through Chroma directly — no mem0 100-entry cap.
+    Pages Chroma directly — bypasses mem0's 100-entry get_all cap.
     """
     data    = await req.json()
     user_id = extract_user_id(data) or "knowledge_base"
@@ -478,8 +566,8 @@ async def knowledge_sources(req: Request):
 @app.delete("/knowledge/by-source")
 async def delete_knowledge_by_source(req: Request):
     """
-    Delete all knowledge entries for a given source_file.
-    Pages through Chroma directly, then cleans SQLite.
+    Delete all entries for a given source_file from both Chroma and SQLite.
+    Pages Chroma directly to avoid the 100-entry cap on get_all.
     """
     data        = await req.json()
     source_file = data.get("source_file")
@@ -490,6 +578,7 @@ async def delete_knowledge_by_source(req: Request):
             content={"error": "Missing source_file"}, status_code=400
         )
 
+    # Collect all IDs matching source_file across all pages
     rows = chroma_get_all(memory_know.vector_store.collection, user_id)
     to_delete = [
         row["id"] for row in rows
@@ -501,7 +590,7 @@ async def delete_knowledge_by_source(req: Request):
             content={"deleted": 0, "message": "no entries found for that source"}
         )
 
-    # 1. Chroma bulk delete
+    # Delete from Chroma in one bulk call
     try:
         memory_know.vector_store.collection.delete(ids=to_delete)
     except Exception as e:
@@ -509,7 +598,7 @@ async def delete_knowledge_by_source(req: Request):
             content={"error": f"chroma delete failed: {e}"}, status_code=500
         )
 
-    # 2. SQLite cleanup
+    # Clean SQLite so entries don't reappear after server restart
     sqlite_deleted = sqlite_delete_ids(to_delete)
 
     print(f"[delete by-source] source={source_file} "
@@ -523,13 +612,13 @@ async def delete_knowledge_by_source(req: Request):
 
 
 # ---------------------------------------------------------------------------
-# /memory/{id}  — single entry delete (knowledge or conversational)
+# /memory/{id} — single entry delete for dashboard per-row buttons
 # ---------------------------------------------------------------------------
 
 @app.delete("/memory/{memory_id}")
 async def delete_single_memory(memory_id: str, req: Request):
     """
-    Delete a single memory by ID from either collection.
+    Delete one memory by ID from either collection.
     Body: { "collection": "knowledge" | "conversational" }
     Cleans both Chroma and SQLite.
     """
@@ -537,7 +626,6 @@ async def delete_single_memory(memory_id: str, req: Request):
     collection = data.get("collection", "knowledge")
     mem        = memory_know if collection == "knowledge" else memory_conv
 
-    # 1. Chroma delete
     try:
         mem.vector_store.collection.delete(ids=[memory_id])
     except Exception as e:
@@ -545,7 +633,6 @@ async def delete_single_memory(memory_id: str, req: Request):
             content={"error": f"chroma delete failed: {e}"}, status_code=500
         )
 
-    # 2. SQLite cleanup
     sqlite_delete_ids([memory_id])
 
     print(f"[delete single] id={memory_id} collection={collection}")
@@ -553,15 +640,14 @@ async def delete_single_memory(memory_id: str, req: Request):
 
 
 # ---------------------------------------------------------------------------
-# /search  — merged results from both collections (OpenClaw autorecall)
+# /search — merged results from both collections (OpenClaw autorecall)
 # ---------------------------------------------------------------------------
 
 @app.post("/search")
 async def search_all(req: Request):
     """
-    Query both collections and merge results.
-    Results tagged with _source: conversational | knowledge.
-    Accepts same payload as /memories/search.
+    Query both collections simultaneously, tag results with _source,
+    then run a single rerank pass over the merged pool.
     """
     data    = await req.json()
     query   = (data.get("query") or "").strip()
@@ -591,4 +677,35 @@ async def search_all(req: Request):
         f"[search/all] user={user_id} query={query!r} "
         f"conv={len(conv_items)} know={len(know_items)} merged={len(merged)}"
     )
-    return SafeJSONResponse(content={"results": merged})
+    return SafeJSONResponse(content={"results": merged})
+
+
+@app.post("/memories/all")
+async def memories_all(req: Request):
+    """
+    Return all memories for a user, paging Chroma directly.
+    Bypasses mem0's 100-entry get_all cap.
+    """
+    data    = await req.json()
+    user_id = extract_user_id(data) or "main"
+
+    rows = chroma_get_all(
+        memory_conv.vector_store.collection,
+        user_id,
+        include=["metadatas", "documents"]
+    )
+
+    items = []
+    for row in rows:
+        meta = row.get("metadata") or {}
+        items.append({
+            "id":         row["id"],
+            "memory":     row.get("document") or meta.get("data", ""),
+            "created_at": meta.get("created_at"),
+            "metadata":   meta,
+            "user_id":    user_id,
+        })
+
+    items.sort(key=lambda r: r.get("created_at") or "", reverse=True)
+    print(f"[memories/all] user={user_id} total={len(items)}")
+    return SafeJSONResponse(content={"results": items})