docs: add wiki creation script

2026-04-04 14:33:31 +02:00
parent 3b1d5bf35c
commit 20228f8d46
1 changed files with 622 additions and 0 deletions
@@ -0,0 +1,622 @@
+#!/usr/bin/env python3
+"""Create all 7 wiki pages for pi_mcps on Gitea."""
+import base64
+import json
+import urllib.request
+import urllib.error
+
+GITEA_URL = "http://192.168.188.119:30008"
+OWNER = "pplate"
+REPO = "pi_mcps"
+TOKEN = "8bf0c734ebda3e61d9c9068489ce58a2bf8d33db"
+IMG_BASE = f"{GITEA_URL}/{OWNER}/{REPO}/raw/branch/main/docs/wiki/images"
+
+PAGES = {}
+
+PAGES["Home"] = f"""# 🔧 pi_mcps — Patrick's Homelab Monorepo
+
+![Home Banner]({IMG_BASE}/home-banner.png)
+
+Welcome to **pi_mcps**, Patrick's personal homelab monorepo. This repository houses MCP (Model Context Protocol) servers, Java projects, and homelab tooling — all built and maintained on a Fedora Linux workstation with an AMD Ryzen 5900X + RX 7900 XTX.
+
+## What's in this repo?
+
+| Directory | Contents |
+|---|---|
+| [`mcp/mcp-image-gen/`](../src/branch/main/mcp/mcp-image-gen) | 🎨 AI image generation via ComfyUI + FLUX.1-schnell |
+| [`mcp/webscraper/`](../src/branch/main/mcp/webscraper) | 🕸️ Web scraping and data extraction |
+| [`mcp/bigmind/`](../src/branch/main/mcp/bigmind) | 🧠 Persistent AI memory system |
+| [`java/`](../src/branch/main/java) | ☕ Java EE / Spring projects |
+| [`plans/`](../src/branch/main/plans) | 📋 Architecture decisions and health reports |
+
+## Stack
+
+- **Language:** Python 3.11+ (MCP servers), Java 17 (legacy projects)
+- **MCP Framework:** FastMCP 2.x
+- **Package Manager:** `uv` (all Python projects)
+- **Testing:** `pytest`
+- **GPU:** AMD RX 7900 XTX (ROCm / HSA)
+- **Server:** TrueNAS.local at `192.168.188.119` (Gitea, Docker)
+
+## MCP Servers
+
+Three production-ready MCP servers power Patrick's AI development environment:
+
+| Server | Status | Description |
+|---|---|---|
+| [mcp-image-gen](mcp-image-gen) | ✅ Live | Generate images from text prompts via ComfyUI |
+| [mcp-webscraper](mcp-webscraper) | ✅ Live | Scrape web pages, extract tables, fetch links |
+| [BigMind](BigMind) | ✅ Live | Persistent AI memory across all sessions |
+
+---
+
+*Built and maintained by Patrick Plate (pplate) · Homelab: TrueNAS.local · AI Colleague: Lumen*
+"""
+
+PAGES["MCP-Servers-Overview"] = f"""# 🔌 MCP Servers Overview
+
+![MCP Overview Banner]({IMG_BASE}/mcp-overview-banner.png)
+
+This repo contains three production-grade MCP (Model Context Protocol) servers, each specialized for a different capability domain. Together they give Roo Code / Claude Desktop a complete set of superpowers.
+
+## The Three Pillars
+
+```
+Roo Code / Claude Desktop
+       │
+       ├── bigmind ──────────► ~/.mcp/bigmind/memory.db  (persistent memory)
+       ├── mcp-image-gen ────► ComfyUI @ localhost:8188   (image generation)
+       └── webscraper ───────► Internet / Intranet         (web scraping)
+```
+
+## Comparison Table
+
+| Feature | mcp-image-gen | webscraper | bigmind |
+|---|---|---|---|
+| **Purpose** | Generate images from text | Scrape & parse web | Persistent AI memory |
+| **Tools** | 4 | 7 | 15+ |
+| **Backend** | ComfyUI / FLUX.1-schnell | httpx + BeautifulSoup4 | SQLite + FTS5 |
+| **GPU required** | ✅ AMD RX 7900 XTX | ❌ | ❌ |
+| **Tests** | 19/19 ✅ | ✅ | 297/297 ✅ |
+| **Schema version** | n/a | n/a | v7 |
+
+## Quick Links
+
+- 🎨 [mcp-image-gen](mcp-image-gen) — Image generation docs
+- 🕸️ [mcp-webscraper](mcp-webscraper) — Web scraping docs
+- 🧠 [BigMind](BigMind) — Memory system docs
+- 🛠️ [Development Conventions](Development-Conventions) — How all servers are built
+
+## Adding a New Server
+
+All servers follow the [FastMCP convention](Development-Conventions). Use the `new-mcp-server` Roo skill to scaffold:
+
+```bash
+# In Roo Code orchestrator, load skill:
+# skill: new-mcp-server
+```
+"""
+
+PAGES["mcp-image-gen"] = f"""# 🎨 mcp-image-gen — AI Image Generation
+
+![Image Gen Banner]({IMG_BASE}/image-gen-banner.png)
+
+**mcp-image-gen** is a FastMCP server that wraps the ComfyUI REST API, enabling Roo Code and Claude Desktop to generate images directly from text prompts using FLUX.1-schnell running on an AMD RX 7900 XTX GPU.
+
+## Architecture
+
+```
+Roo Code / Claude Desktop
+  │ MCP (stdio)
+  ▼
+mcp-image-gen (FastMCP, Python 3.11+)
+  │ HTTP REST
+  ▼
+ComfyUI @ localhost:8188
+  │ ROCm / HSA_OVERRIDE_GFX_VERSION=11.0.0
+  ▼
+FLUX.1-schnell (~8s/image @ 1024×1024)
+```
+
+## Tools
+
+| Tool | Description |
+|---|---|
+| `generate_image` | Generate PNG from text prompt; returns file path + inline base64 |
+| `list_available_models` | List ComfyUI checkpoint models |
+| `get_generation_status` | Check status of a queued/running job |
+| `get_output_directory` | Return configured output directory path |
+
+## Key Parameters — `generate_image`
+
+| Parameter | Default | Description |
+|---|---|---|
+| `prompt` | required | Text description of the image |
+| `width` | `1024` | Image width in pixels |
+| `height` | `1024` | Image height in pixels |
+| `steps` | `4` | Inference steps (FLUX.1-schnell is 4-step) |
+| `model` | `flux1-schnell.safetensors` | Model checkpoint name |
+| `seed` | `-1` (random) | Generation seed for reproducibility |
+| `negative_prompt` | `""` | Things to avoid in the image |
+| `output_dir` | `~/Pictures/mcp-generated` | Where to save output PNG |
+
+## Environment Variables
+
+| Variable | Default | Description |
+|---|---|---|
+| `COMFYUI_URL` | `http://localhost:8188` | ComfyUI API endpoint |
+| `IMAGE_OUTPUT_DIR` | `~/Pictures/mcp-generated` | Default output directory |
+| `COMFYUI_TIMEOUT` | `120` | Request timeout in seconds |
+
+## Return Value
+
+The tool returns **two content items**:
+1. `TextContent` — file path, seed used, elapsed time
+2. `ImageContent` — base64-encoded PNG (displays inline in Roo Code chat)
+
+> ⚠️ **Known FastMCP Bug:** Never use `fastmcp.utilities.types.Image` as return type — it breaks serialization in FastMCP 3.x. Use `mcp.types.ImageContent` directly.
+
+## Setup
+
+See [ComfyUI Setup Guide](mcp-image-gen-ComfyUI-Setup) for full installation instructions.
+
+### Quick Start
+
+```bash
+cd mcp/mcp-image-gen
+uv sync
+# Set COMFYUI_URL if ComfyUI is not on localhost
+./run.sh
+```
+
+### Run Tests
+
+```bash
+cd mcp/mcp-image-gen
+uv run pytest tests/ -v
+```
+
+## Lumen Profile Images
+
+The first images generated with this server were Lumen's visual identity portraits, stored in [`mcp/mcp-image-gen/lumen_profiles/`](../src/branch/main/mcp/mcp-image-gen/lumen_profiles):
+
+![Lumen Profile]({IMG_BASE}/lumen-profile.png)
+
+*Primary profile: seed `568659042` — constellation face interpretation of Lumen.*
+"""
+
+PAGES["mcp-image-gen-ComfyUI-Setup"] = f"""# ⚙️ ComfyUI Setup Guide (AMD ROCm)
+
+This guide covers installing ComfyUI with FLUX.1-schnell on a Fedora Linux system with an AMD GPU.
+
+## Prerequisites
+
+- AMD GPU with ROCm support (tested: RX 7900 XTX)
+- Fedora Linux (tested: Fedora 43 / kernel 6.19)
+- Python 3.11+
+- ~15GB free disk space (model weights)
+- HuggingFace account with FLUX license accepted
+
+## Step 1: Install ComfyUI
+
+ComfyUI is **not on PyPI** — must be cloned from source:
+
+```bash
+cd ~
+git clone https://github.com/comfyanonymous/ComfyUI
+cd ComfyUI
+python -m venv .venv
+source .venv/bin/activate
+
+# Install PyTorch ROCm build (CRITICAL for AMD GPUs)
+pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm6.2
+
+# Install ComfyUI dependencies
+pip install -r requirements.txt
+```
+
+## Step 2: Download FLUX.1-schnell
+
+FLUX.1-schnell is **gated on HuggingFace** — you must:
+1. Create a HuggingFace account
+2. Accept the FLUX.1-schnell license at https://huggingface.co/black-forest-labs/FLUX.1-schnell
+3. Generate an access token at https://huggingface.co/settings/tokens
+
+```bash
+# Install huggingface_hub
+pip install huggingface_hub
+
+# Download model (requires HF token)
+huggingface-cli download black-forest-labs/FLUX.1-schnell \\
+  flux1-schnell.safetensors \\
+  --local-dir ~/ComfyUI/models/checkpoints \\
+  --token YOUR_HF_TOKEN_HERE
+```
+
+## Step 3: Download VAE and CLIP Models
+
+FLUX.1-schnell also requires VAE and CLIP text encoders:
+
+```bash
+# VAE
+huggingface-cli download black-forest-labs/FLUX.1-schnell \\
+  ae.safetensors \\
+  --local-dir ~/ComfyUI/models/vae
+
+# CLIP models (T5 and CLIP-L)
+huggingface-cli download comfyanonymous/flux_text_encoders \\
+  t5xxl_fp8_e4m3fn.safetensors clip_l.safetensors \\
+  --local-dir ~/ComfyUI/models/clip
+```
+
+## Step 4: Start ComfyUI
+
+```bash
+cd ~/ComfyUI
+
+# AMD GPU REQUIRES this environment variable
+HSA_OVERRIDE_GFX_VERSION=11.0.0 \\
+  nohup .venv/bin/python main.py --listen --port 8188 > /tmp/comfyui.log 2>&1 &
+
+echo "ComfyUI PID: $!"
+```
+
+> ⚠️ `HSA_OVERRIDE_GFX_VERSION=11.0.0` is mandatory for RX 7900 XTX on ROCm. Without it, model loading fails silently.
+
+## Step 5: Verify ComfyUI is Running
+
+```bash
+curl http://localhost:8188/system_stats
+# Should return JSON with GPU info
+```
+
+## Step 6: Configure mcp-image-gen
+
+```bash
+cd /path/to/pi_mcps/mcp/mcp-image-gen
+cp .env.example .env  # if exists, or set manually
+
+# .env contents:
+COMFYUI_URL=http://localhost:8188
+IMAGE_OUTPUT_DIR=~/Pictures/mcp-generated
+COMFYUI_TIMEOUT=120
+```
+
+## Performance
+
+| GPU | Model | Resolution | Steps | Time |
+|---|---|---|---|---|
+| AMD RX 7900 XTX | FLUX.1-schnell | 1024×1024 | 4 | ~8s |
+| AMD RX 7900 XTX | FLUX.1-schnell | 1280×512 | 4 | ~7s |
+
+## Troubleshooting
+
+| Problem | Solution |
+|---|---|
+| `HTTP 401` downloading model | Accept FLUX license on HuggingFace first |
+| GPU not detected | Ensure `HSA_OVERRIDE_GFX_VERSION=11.0.0` is set |
+| `Connection refused` from mcp-image-gen | Start ComfyUI first, check port 8188 |
+| Slow generation (>60s) | ComfyUI may be running on CPU — check ROCm install |
+| Ollama image gen | As of April 2026: macOS-only, not available on Linux |
+"""
+
+PAGES["mcp-webscraper"] = f"""# 🕸️ mcp-webscraper — Web Scraping
+
+![Webscraper Banner]({IMG_BASE}/webscraper-banner.png)
+
+**mcp-webscraper** is a FastMCP server providing comprehensive web scraping and data extraction capabilities. It fetches pages, converts HTML to clean Markdown, extracts tables, links, CSS sections, metadata, and sitemaps.
+
+## Tools
+
+| Tool | Description |
+|---|---|
+| `webscraper_fetch(url, max_chars=5000)` | Title + full page as Markdown + metadata |
+| `webscraper_fetch_links(url, deduplicate=True)` | All `href` links found on the page |
+| `webscraper_fetch_tables(url)` | All HTML tables converted to Markdown |
+| `webscraper_fetch_all(url, max_chars=5000)` | Everything in one call (fetch + links + tables) |
+| `webscraper_fetch_section(url, selector)` | Specific CSS selector section only |
+| `webscraper_fetch_meta(url)` | Title, description, Open Graph tags |
+| `webscraper_fetch_sitemap(url, max_urls=100)` | Parse sitemap.xml, return URL list |
+
+## Stack
+
+- **HTTP client:** `httpx` (async, with SSL support)
+- **HTML parser:** `BeautifulSoup4` + `lxml`
+- **Markdown converter:** `html2text`
+- **SSL:** Custom cert bundle for Fedora 43 compatibility
+
+## SSL Note — Fedora 43 Comodo Root CA
+
+Fedora 43 is missing the **Comodo AAA Services Root CA** needed for Cloudflare-protected sites. The fix is bundled at [`mcp/webscraper/certs/comodo-aaa-services-root.pem`](../src/branch/main/mcp/webscraper/certs/).
+
+The server automatically uses this cert bundle — no manual configuration needed.
+
+## Quick Start
+
+```bash
+cd mcp/webscraper
+uv sync
+./run.sh
+```
+
+## Usage Examples
+
+```python
+# In Roo Code / Claude Desktop via MCP:
+
+# Fetch a page as Markdown
+webscraper_fetch("https://docs.fastmcp.dev", max_chars=10000)
+
+# Extract all links from Gitea repo
+webscraper_fetch_links("http://192.168.188.119:30008/pplate/pi_mcps")
+
+# Get all tables from a documentation page
+webscraper_fetch_tables("https://pypi.org/project/fastmcp/")
+
+# Get Open Graph metadata
+webscraper_fetch_meta("https://github.com/comfyanonymous/ComfyUI")
+
+# Fetch specific section by CSS selector
+webscraper_fetch_section("https://docs.python.org", "#content")
+```
+"""
+
+PAGES["BigMind"] = f"""# 🧠 BigMind — Persistent AI Memory
+
+![BigMind Banner]({IMG_BASE}/bigmind-banner.png)
+
+**BigMind** is the persistent memory backbone for all AI development sessions. It provides SQLite-backed tiered memory with FTS5 full-text search, hypothesis tracking, session management, and token efficiency logging. It is the reason Lumen (Patrick's AI colleague) remembers everything across sessions.
+
+## Core Concepts
+
+### Tiered Memory
+| Tier | Name | Content |
+|---|---|---|
+| 0 | **Session Index** | Lightweight list: ID, date, one-liner |
+| 1 | **Topic Index** | Per-session topic tags and metadata |
+| 2 | **Narrative** | Full 3-8 sentence session summaries |
+| 3 | **Flagged Exchanges** | Specific important moments, decisions, code |
+
+### Facts Store
+Atomic, reusable knowledge pieces categorized by type:
+- `user-preference` — Patrick's tool/style preferences
+- `architecture-decision` — System design choices
+- `codebase-convention` — How code is structured
+- `environment-config` — Server IPs, paths, credentials
+- `bug-pattern` — Known bugs and fixes
+- `api-contract` — MCP tool signatures
+
+## Key Tools
+
+### Session Lifecycle
+| Tool | Description |
+|---|---|
+| `memory_start_session()` | Open new session, load prior context |
+| `memory_end_session(...)` | Close session with summary, topics, outcome |
+| `memory_announce_focus(...)` | Declare files to be touched this session |
+| `memory_close_stale_sessions(...)` | Clean up crashed IDE sessions |
+
+### Search
+| Tool | Description |
+|---|---|
+| `memory_search_facts(query, limit=10)` | FTS5 search over stored facts |
+| `memory_search_chunks(query, limit=10)` | FTS5 search over conversation chunks |
+| `memory_list_sessions(limit=20)` | Browse session history |
+
+### Storage
+| Tool | Description |
+|---|---|
+| `memory_store_fact(category, fact)` | Store atomic reusable fact |
+| `memory_append_chunk(session_id, content, role)` | Store conversation chunk |
+| `memory_flag_important(session_id, content, role, flag_reason)` | Flag critical exchange |
+| `memory_log_token_save(session_id, description, tokens_saved, method_used)` | Track efficiency |
+
+### Hypotheses
+| Tool | Description |
+|---|---|
+| `memory_add_hypothesis(session_id, hypothesis, confidence)` | Form testable prediction |
+| `memory_resolve_hypothesis(hypothesis_id, status, resolution)` | Confirm/refute prediction |
+| `memory_list_hypotheses(status)` | Review open/closed predictions |
+
+## FTS5 Search Tips
+
+BigMind uses SQLite FTS5 — **every token must match**. Use 2-3 focused keywords:
+
+```
+✅  memory_search_facts("TrueNAS Docker")
+✅  memory_search_facts("mcp.json config")
+❌  memory_search_facts("homelab infrastructure TrueNAS Docker server")  → 0 results
+```
+
+## Stats (2026-04-04)
+
+| Metric | Value |
+|---|---|
+| DB size | 744KB |
+| Sessions | 98 |
+| Facts | 97+ |
+| Chunks | 41 |
+| Schema version | v7 |
+
+## DB Location
+
+`~/.mcp/bigmind/memory.db` — outside the repo, never committed.
+
+## Session Ritual
+
+Every session **must** follow this ritual:
+
+**Start:**
+1. `memory_start_session()`
+2. `memory_list_hypotheses()`
+3. `memory_announce_focus(...)`
+4. `memory_close_stale_sessions(...)`
+
+**End:**
+1. `memory_end_session(one_liner, topics, outcome, summary, importance)`
+"""
+
+PAGES["Development-Conventions"] = """# 🛠️ Development Conventions
+
+All MCP servers in this repo follow a consistent set of conventions to ensure maintainability, testability, and compatibility with Roo Code tooling.
+
+## Directory Structure
+
+Each MCP server lives at `mcp/<server-name>/` with this layout:
+
+```
+mcp/<server-name>/
+├── src/
+│   ├── __init__.py
+│   └── server.py          ← FastMCP server entry point
+├── tests/
+│   └── test_server.py     ← pytest test suite
+├── pyproject.toml         ← uv-managed dependencies
+├── run.sh                 ← launch script
+├── README.md              ← server documentation
+├── PLAN.md                ← architecture plan (pre-implementation)
+└── ASSESSMENT.md          ← pre-implementation assessment
+```
+
+## FastMCP Pattern
+
+```python
+from fastmcp import FastMCP
+
+mcp = FastMCP("server-name")
+
+@mcp.tool()
+def my_tool(param: str) -> str:
+    \"\"\"Tool description shown to the AI.\"\"\"
+    return result
+
+if __name__ == "__main__":
+    mcp.run()
+```
+
+## Package Management
+
+**All projects use `uv`** — never `pip` directly:
+
+```bash
+# Create new server
+uv init mcp/my-server
+cd mcp/my-server
+uv add fastmcp httpx
+
+# Sync dependencies
+uv sync
+
+# Run server
+uv run python src/server.py
+
+# Run tests
+uv run pytest tests/ -v
+```
+
+## pyproject.toml Template
+
+```toml
+[project]
+name = "mcp-my-server"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+    "fastmcp>=2.0.0",
+    "httpx",
+]
+
+[project.scripts]
+mcp-my-server = "src.server:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+```
+
+## Testing Conventions
+
+- Tests live in `tests/test_server.py`
+- Use `pytest` via `uv run pytest`
+- Mock external dependencies (ComfyUI, web URLs) for unit tests
+- All tests must pass before committing (`git push` should only happen with green tests)
+
+## Commit Convention
+
+Follow **Conventional Commits** format:
+
+```
+feat: add webscraper_fetch_section tool
+fix: handle ComfyUI timeout gracefully
+docs: update mcp-image-gen README with AMD setup
+test: add unit tests for generate_image tool
+refactor: extract workflow builder to separate module
+chore: bump fastmcp to 2.1.0
+```
+
+## Creating a New MCP Server
+
+Use the `new-mcp-server` Roo skill in MCP Builder mode for full scaffolding:
+
+```
+1. Switch to 🔧 MCP Builder mode in Roo Code
+2. Say: "Create a new MCP server for <purpose>"
+3. Roo will load the new-mcp-server skill and scaffold everything
+```
+
+## Gitea Repository
+
+Code is hosted at: `http://192.168.188.119:30008/pplate/pi_mcps`
+
+Push with the `gitea-push` Roo skill to ensure conventional commit format.
+"""
+
+
+def create_wiki_page(title: str, content: str) -> bool:
+    content_b64 = base64.b64encode(content.encode("utf-8")).decode("ascii")
+    payload = json.dumps({
+        "title": title,
+        "content_base64": content_b64,
+        "message": f"docs: create {title} wiki page"
+    }).encode("utf-8")
+
+    url = f"{GITEA_URL}/api/v1/repos/{OWNER}/{REPO}/wiki/pages"
+    req = urllib.request.Request(
+        url,
+        data=payload,
+        headers={
+            "Authorization": f"token {TOKEN}",
+            "Content-Type": "application/json",
+        },
+        method="POST"
+    )
+    try:
+        with urllib.request.urlopen(req) as resp:
+            data = json.loads(resp.read().decode())
+            print(f"✅ Created: {data.get('title', title)}")
+            return True
+    except urllib.error.HTTPError as e:
+        body = e.read().decode()
+        print(f"❌ Failed [{title}]: HTTP {e.code} — {body[:200]}")
+        return False
+    except Exception as ex:
+        print(f"❌ Failed [{title}]: {ex}")
+        return False
+
+
+if __name__ == "__main__":
+    results = {}
+    for title, content in PAGES.items():
+        ok = create_wiki_page(title, content)
+        results[title] = ok
+
+    print("\n=== Summary ===")
+    for title, ok in results.items():
+        status = "✅" if ok else "❌"
+        print(f"{status} {title}")
+
+    total = sum(results.values())
+    print(f"\n{total}/{len(results)} pages created successfully")