diff --git a/mcp/bigmind/bigmind/auto_close.py b/mcp/bigmind/bigmind/auto_close.py index 6a71886..92664cb 100644 --- a/mcp/bigmind/bigmind/auto_close.py +++ b/mcp/bigmind/bigmind/auto_close.py @@ -3,6 +3,7 @@ import logging from datetime import datetime, timezone, timedelta from bigmind.db import db + logger = logging.getLogger("BigMindAutoClose") STALE_THRESHOLD_HOURS = 24 @@ -17,6 +18,7 @@ def auto_close_stale_sessions(user_id: str) -> int: datetime.now(timezone.utc) - timedelta(hours=STALE_THRESHOLD_HOURS) ).isoformat() + now = datetime.now(timezone.utc).isoformat() with db() as conn: stale = conn.execute( """SELECT id, started_at FROM sessions @@ -27,11 +29,11 @@ def auto_close_stale_sessions(user_id: str) -> int: for session in stale: conn.execute( """UPDATE sessions - SET ended_at=CURRENT_TIMESTAMP, + SET ended_at=?, one_liner='[auto-closed — session exceeded 24h]', outcome='Session automatically closed after exceeding 24h without a proper close call.' WHERE id=?""", - (session["id"],), + (now, session["id"]), ) logger.info( "Auto-closed stale session %s (started %s)", @@ -57,15 +59,16 @@ def close_orphaned_sessions(user_id: str, keep_session_id: str) -> list[str]: (user_id, keep_session_id), ).fetchall() + now = datetime.now(timezone.utc).isoformat() closed_ids = [] for session in orphans: conn.execute( """UPDATE sessions - SET ended_at=CURRENT_TIMESTAMP, + SET ended_at=?, one_liner='[orphaned — closed by memory_close_stale_sessions]', outcome='Session was open but never properly closed (IDE crash or forgotten). Cleaned up manually.' WHERE id=?""", - (session["id"],), + (now, session["id"]), ) closed_ids.append(session["id"]) logger.info( diff --git a/mcp/bigmind/bigmind/context_builder.py b/mcp/bigmind/bigmind/context_builder.py index 1a7be02..55dee53 100644 --- a/mcp/bigmind/bigmind/context_builder.py +++ b/mcp/bigmind/bigmind/context_builder.py @@ -21,6 +21,12 @@ def _format_date(iso_str: Optional[str]) -> str: return iso_str[:10] +# Maximum facts loaded into context on session start. +# Keeps context lean as the facts table grows — older/excess facts remain +# searchable via memory_search_facts() but won't bloat every session startup. +_MAX_CONTEXT_FACTS = 50 + + def build_context(user_id: str, n_sessions: int = 10) -> str: """ Assemble the full bootstrapped context markdown for injection at session start. @@ -55,13 +61,19 @@ def build_context(user_id: str, n_sessions: int = 10) -> str: lines.append("") # ── FACTS: Atomic personal facts ───────────────────────────────────────── - facts = memory_store.get_facts(user_id) + all_facts = memory_store.get_facts(user_id) + facts = all_facts[:_MAX_CONTEXT_FACTS] + overflow = len(all_facts) - len(facts) if facts: lines.append("### 🗂️ Stored facts") for f in facts: cat = f.get("category", "") fact = f.get("fact", "") lines.append(f"- **[{cat}]** {fact}") + if overflow > 0: + lines.append( + f"- *… {overflow} more facts available via `memory_search_facts()`*" + ) lines.append("") # ── TIER 1: Recent Sessions ─────────────────────────────────────────────── diff --git a/mcp/bigmind/bigmind/memory_store.py b/mcp/bigmind/bigmind/memory_store.py index 4464f39..b8e2929 100644 --- a/mcp/bigmind/bigmind/memory_store.py +++ b/mcp/bigmind/bigmind/memory_store.py @@ -721,9 +721,14 @@ def health_check(user_id: str, stale_days: int = 30) -> dict: # ── EXPORT ─────────────────────────────────────────────────────────────────────── def export_memory(user_id: str, output_path: str = None) -> dict: - """Export all memory for a user to a portable JSON file.""" + """Export all memory for a user to a portable JSON file. + + Exports ALL tables: facts, sessions (with Tier-2), conversation chunks, + people/contacts, hypotheses, token saves, and upgrade requests. + """ import json from pathlib import Path + from bigmind.db import SCHEMA_VERSION if not output_path: date_str = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") @@ -768,18 +773,57 @@ def export_memory(user_id: str, output_path: str = None) -> dict: ).fetchall() ] + # ── v3+ tables ─────────────────────────────────────────────────────── + hypotheses = [ + dict(r) for r in conn.execute( + "SELECT * FROM hypotheses WHERE user_id=? ORDER BY created_at", + (user_id,), + ).fetchall() + ] + + upgrade_requests = [ + dict(r) for r in conn.execute( + "SELECT * FROM upgrade_requests WHERE user_id=? ORDER BY created_at", + (user_id,), + ).fetchall() + ] + + # ── v6+ tables ─────────────────────────────────────────────────────── + token_saves = [ + dict(r) for r in conn.execute( + "SELECT * FROM token_saves WHERE user_id=? ORDER BY created_at", + (user_id,), + ).fetchall() + ] + + # ── v7+ tables ─────────────────────────────────────────────────────── + people = [ + dict(r) for r in conn.execute( + "SELECT * FROM people WHERE user_id=? ORDER BY created_at", + (user_id,), + ).fetchall() + ] + export_data = { "export_date": datetime.now(timezone.utc).isoformat(), - "bigmind_version": "1.0", + "bigmind_schema_version": SCHEMA_VERSION, "user": user_info, "identity_profile": profile, "facts": facts, "sessions": sessions, "conversation_chunks": chunks, + "hypotheses": hypotheses, + "upgrade_requests": upgrade_requests, + "token_saves": token_saves, + "people": people, "stats": { "facts_count": len(facts), "sessions_count": len(sessions), "chunks_count": len(chunks), + "hypotheses_count": len(hypotheses), + "upgrade_requests_count": len(upgrade_requests), + "token_saves_count": len(token_saves), + "people_count": len(people), }, } @@ -791,6 +835,8 @@ def export_memory(user_id: str, output_path: str = None) -> dict: "facts_count": len(facts), "sessions_count": len(sessions), "chunks_count": len(chunks), + "hypotheses_count": len(hypotheses), + "people_count": len(people), "file_size_kb": round(output.stat().st_size / 1024, 1), } diff --git a/mcp/bigmind/src/server.py b/mcp/bigmind/src/server.py index 618a54f..2a1999b 100644 --- a/mcp/bigmind/src/server.py +++ b/mcp/bigmind/src/server.py @@ -771,6 +771,8 @@ def memory_export(output_path: str = None) -> str: f"| **Facts** | {result['facts_count']} |\n" f"| **Sessions** | {result['sessions_count']} |\n" f"| **Chunks (Tier 3)** | {result['chunks_count']} |\n" + f"| **Hypotheses** | {result['hypotheses_count']} |\n" + f"| **People** | {result['people_count']} |\n" f"| **File size** | {result['file_size_kb']} KB |" ) diff --git a/mcp/bigmind/tests/test_memory_store.py b/mcp/bigmind/tests/test_memory_store.py index f7da1ff..e8f36bf 100644 --- a/mcp/bigmind/tests/test_memory_store.py +++ b/mcp/bigmind/tests/test_memory_store.py @@ -571,8 +571,9 @@ class TestExportMemory: memory_store.export_memory(user["id"], out) data = json.loads(Path(out).read_text()) for key in ( - "export_date", "bigmind_version", "user", "identity_profile", - "facts", "sessions", "conversation_chunks", "stats", + "export_date", "bigmind_schema_version", "user", "identity_profile", + "facts", "sessions", "conversation_chunks", "hypotheses", + "upgrade_requests", "token_saves", "people", "stats", ): assert key in data diff --git a/mcp/bigmind/tests/test_server_tools.py b/mcp/bigmind/tests/test_server_tools.py index 3b74fd5..0b00051 100644 --- a/mcp/bigmind/tests/test_server_tools.py +++ b/mcp/bigmind/tests/test_server_tools.py @@ -541,7 +541,7 @@ class TestMemoryGetInstructions: def test_contains_mandatory_language(self, temp_db): result = memory_get_instructions() - assert "ALWAYS" in result + assert "Always" in result # ── memory_health_check ──────────────────────────────────────────────────────── @@ -674,7 +674,8 @@ class TestMemoryExport: memory_export(output_path=out) data = json.loads(Path(out).read_text()) assert "export_date" in data - assert data["bigmind_version"] == "1.0" + assert "bigmind_schema_version" in data + assert isinstance(data["bigmind_schema_version"], int) # ── memory_deprecate_fact ────────────────────────────────────────────────────── diff --git a/plans/BIGMIND_HEALTH_REPORT_2026-04-04.md b/plans/BIGMIND_HEALTH_REPORT_2026-04-04.md new file mode 100644 index 0000000..120483a --- /dev/null +++ b/plans/BIGMIND_HEALTH_REPORT_2026-04-04.md @@ -0,0 +1,189 @@ +# 🧠 BigMind Health Report +**Date:** 2026-04-04 | **Machine:** Fedora Homelab (AMD Ryzen 5900X) | **Schema:** v7 | **Analyst:** Lumen + +--- + +## 🟢 Executive Summary + +BigMind is **healthy and operational** on the homelab machine. The brain is alive, the DB is clean, and the test suite is 99.7% green. Three bugs and one scalability risk were found during the code scan — all logged as upgrade requests. No data corruption, no orphaned sessions, FTS index is in sync. + +**Verdict: 🟢 HEALTHY** — 1 real bug to fix, 3 improvements to queue. + +--- + +## 📊 Memory Statistics + +| Metric | Value | Notes | +|--------|-------|-------| +| Sessions | **98** | ~5 per day average since March 30 | +| Facts | **97** | Growing fast — see Perf-1 | +| Tier-3 Chunks | **41** | Lean and targeted | +| Global Knowledge | **0** | Phase 3 (Company Brain) not started | +| Database size | **744 KB** | Very healthy | +| DB location | `/home/pplate/.mcp/bigmind/memory.db` | WAL mode active | +| Schema version | **v7** | People/contacts directory live | +| Python version | **3.12.13** | ✅ correct | + +--- + +## 🩺 Health Check Results + +| Check | Status | Detail | +|-------|--------|--------| +| FTS index integrity | ✅ IN SYNC | 41 chunks / 41 index rows — no drift | +| Open sessions | 🟡 1 | Current health check session only — clean | +| Orphaned sessions | ✅ NONE | `memory_close_stale_sessions` returned clean | +| Sessions without Tier-2 | ⚠️ 22 | 22 closed sessions have no narrative summary | +| Stale facts (>30d) | ✅ NONE | All 97 facts updated recently | +| Low-confidence facts (<0.8) | ✅ NONE | All facts at full confidence | + +> **On the 22 sessions without Tier-2:** These are mostly the orphaned sessions that were force-closed by `memory_close_stale_sessions` (IDE crashes). They never had a chance to store a summary. Not a bug — expected behaviour. Worth vacuuming if they clutter the index. + +--- + +## 🧪 Test Suite Results + +**297 tests collected | 296 ✅ PASSED | 1 ❌ FAILED** + +``` +platform linux -- Python 3.12.13, pytest-9.0.2 +rootdir: /home/pplate/pi_mcps/mcp/bigmind +Duration: 2.36s +``` + +### ❌ Failing Test + +``` +tests/test_server_tools.py::TestMemoryGetInstructions::test_contains_mandatory_language +AssertionError: assert 'ALWAYS' in '...Rule 1: Session Start Ritual (Always First Action)...' +``` + +**Root cause:** The `BIGMIND_INSTRUCTIONS` constant in [`src/server.py`](mcp/bigmind/src/server.py:38) was rewritten to use title-case language ("Always First Action") but the test in [`tests/test_server_tools.py`](mcp/bigmind/tests/test_server_tools.py:544) still asserts that the uppercase word `"ALWAYS"` is present. One of them needs to change — the test expectation is the right one to update since "Always" is clearer prose anyway. **Logged as Upgrade Request #6 (HIGH).** + +--- + +## 🐛 Bugs & Issues Found + +### BUG-1 — Test/Code Mismatch (HIGH) `upgrade #6` +- **File:** [`src/server.py:38`](mcp/bigmind/src/server.py:38) + [`tests/test_server_tools.py:544`](mcp/bigmind/tests/test_server_tools.py:544) +- **Issue:** `BIGMIND_INSTRUCTIONS` string was rewritten without updating the test. Test asserts `"ALWAYS"` (uppercase), instructions say `"Always"` (title-case). +- **Impact:** 1 test failing in CI. Easy fix: update the test to assert `"Always"` or add `"ALWAYS"` to the instructions. +- **Fix:** `assert "Always" in result` — one-line change. + +--- + +### BUG-2 — Data Loss in Export (HIGH) `upgrade #7` +- **File:** [`bigmind/memory_store.py:723`](mcp/bigmind/bigmind/memory_store.py:723) — `export_memory()` +- **Issue:** `memory_export` was written before the `people` (v7) and `hypotheses` (v3) tables existed. It exports **facts, sessions, and chunks only**. A full machine migration using the export would **silently lose:** + - The entire contacts directory (Elias, Klaus, etc.) + - The entire thought journal (all 3 open hypotheses + resolved ones) + - Token saves history + - Upgrade requests +- **Impact:** 🔴 Data loss on machine migration. The export is marketed as a full backup but it's not. +- **Fix:** Add `hypotheses`, `people`, `token_saves`, and `upgrade_requests` queries to the export dict. Also fix `bigmind_version: "1.0"` hardcode — should use `SCHEMA_VERSION`. + +--- + +### BUG-3 — Timestamp Format Inconsistency (MEDIUM) `upgrade #8` +- **File:** [`bigmind/auto_close.py:30`](mcp/bigmind/bigmind/auto_close.py:30) and [`auto_close.py:68`](mcp/bigmind/bigmind/auto_close.py:68) +- **Issue:** `auto_close_stale_sessions()` and `close_orphaned_sessions()` write `ended_at` using SQLite's `CURRENT_TIMESTAMP` keyword: + ```sql + SET ended_at=CURRENT_TIMESTAMP + ``` + This produces: `"2026-04-04 09:36:00"` (no T, no timezone) + + The **entire rest of the codebase** uses: + ```python + datetime.now(timezone.utc).isoformat() + ``` + Which produces: `"2026-04-04T09:36:00+00:00"` (ISO 8601 with T and timezone) + +- **Impact:** `datetime.fromisoformat()` in `get_active_sessions()` handles the mixed formats via `.replace("Z", "+00:00")` but does NOT handle the no-T format from SQLite `CURRENT_TIMESTAMP`. Sessions auto-closed by `auto_close_stale_sessions` may have `idle_minutes = None` in the profile page Live Sessions panel. +- **Fix:** Replace `CURRENT_TIMESTAMP` in `auto_close.py` with a Python-level timestamp passed as a parameter: + ```python + now = datetime.now(timezone.utc).isoformat() + conn.execute("UPDATE sessions SET ended_at=? ... WHERE id=?", (now, session["id"])) + ``` + +--- + +## ⚡ Performance / Scalability Concern + +### PERF-1 — Unbounded Fact Loading in Context Builder (MEDIUM) `upgrade #9` +- **File:** [`bigmind/context_builder.py:58`](mcp/bigmind/bigmind/context_builder.py:58) +- **Issue:** `build_context()` calls `get_facts(user_id)` with **no limit**. Currently 97 facts are loaded on every single `memory_start_session()` call. +- **Projection:** + | Facts count | Estimated chars | Estimated tokens | + |-------------|-----------------|------------------| + | 97 (now) | ~12,000 | ~3,000 | + | 200 (3 months) | ~25,000 | ~6,250 | + | 500 (1 year) | ~62,000 | ~15,500 | +- **Impact:** Context window bloat, increased API cost per session, slower startup. The AI also can't meaningfully use 500 facts in a single context — search is more effective for recall. +- **Fix:** Cap at 50 most recently updated facts with a `"... N more facts available via memory_search_facts()"` footer. Optionally sort by `updated_at DESC` to surface the freshest knowledge first. + +--- + +## 📐 Code Quality Observations (Non-Bug) + +| Location | Issue | Severity | +|----------|-------|----------| +| [`bigmind/db.py:357`](mcp/bigmind/bigmind/db.py:357) | Migration functions defined out of numeric order: `v5→v6`, then `v3→v4`, then `_migrate_v6_to_v7`. Works fine in Python (runtime resolution) but confusing for maintainers. | Style | +| [`bigmind/web.py:66`](mcp/bigmind/bigmind/web.py:66) | Bare `except Exception: pass` in `api_search()` for facts, chunks, and sessions searches. Silently swallows errors — makes it impossible to diagnose search bugs through the profile page. | Low | +| [`src/server.py:911`](mcp/bigmind/src/server.py:911) | Double blank line before `memory_open_profile` tool — minor style inconsistency. | Cosmetic | +| [`bigmind/memory_store.py:239`](mcp/bigmind/bigmind/memory_store.py:239) | `update_fields: list = [description, files_json, now]` in `announce_focus()` is built but never actually used (the code branches into two separate `conn.execute` calls). Dead code. | Minor | + +--- + +## 💭 Open Hypotheses Review + +3 hypotheses remain open from previous sessions: + +| # | Age | Confidence | Summary | Action | +|---|-----|------------|---------|--------| +| #38 | ~2d | 92% | NPE at `Center.java:425` from empty `errors` variable | This is an ADP/Paisy hypothesis — still valid, likely confirmed but never closed. Should be resolved. | +| #14 | ~4d | 65% | Elias will hit Windows setup issues with BigMind | Still open/unknown — no update since Elias received the repo. | +| #5 | ~4d | 75% | Elias's AI will have a different name/personality | Still open — awaiting first session report from Elias. | + +> Hypothesis #38 is stale — it's about a Paisy bug investigated days ago. If the fix was merged, it should be resolved as `confirmed`. + +--- + +## 🔧 Upgrade Requests Summary + +| ID | Priority | Status | Title | +|----|----------|--------|-------| +| #2 | MEDIUM | ⏳ OPEN | Session Health Panel on Profile Page | +| #6 | HIGH | ⏳ OPEN | Fix test failure: "ALWAYS" vs "Always" in instructions | +| #7 | HIGH | ⏳ OPEN | `export_memory()` missing `people` + `hypotheses` tables | +| #8 | MEDIUM | ⏳ OPEN | `auto_close.py` timestamp format inconsistency | +| #9 | MEDIUM | ⏳ OPEN | Unbounded fact loading in `context_builder.py` | + +(#1, #3, #4, #5 are already resolved ✅) + +--- + +## 🗓️ Recommended Fix Order + +1. **[30 min] BUG-1** — Fix the failing test. One line. Do it now before it becomes noise. +2. **[2h] BUG-2** — Fix `export_memory()` to include all tables. Critical for machine migration safety. +3. **[1h] BUG-3** — Fix `auto_close.py` timestamp format. Simple Python replace, no schema change. +4. **[1d] PERF-1** — Design and implement smart fact pagination in `context_builder.py`. Needs thought about ranking strategy. +5. **[1d] Upgrade #2** — Session Health Panel on profile page. Quality of life. + +--- + +## ✅ What's Working Well + +- **WAL mode** is active and correctly configured — concurrent IDE access is safe +- **FTS5 index** is perfectly in sync with the chunks table (41/41) +- **Schema migrations** run correctly through v1→v7 on fresh DB +- **Conflict detection** in `announce_focus()` is atomic (uses `BEGIN IMMEDIATE`) — race-condition safe +- **Port conflict handling** in `web.py` is well thought out — second IDE gracefully defers to first +- **People/contacts directory** (v7) is fully functional with FTS search +- **Test coverage** is excellent: 297 tests across 8 test files covering all layers +- **Token efficiency tracker** is working and recording saves +- **`_fts_safe_query()`** helper correctly handles FTS5 reserved words and AND-match semantics + +--- + +*Report generated by Lumen | BigMind session `e04fc9c9` | 2026-04-04 09:40 UTC+2*