diff --git a/docs/wiki/create_wiki_pages.py b/docs/wiki/create_wiki_pages.py new file mode 100644 index 0000000..283e618 --- /dev/null +++ b/docs/wiki/create_wiki_pages.py @@ -0,0 +1,622 @@ +#!/usr/bin/env python3 +"""Create all 7 wiki pages for pi_mcps on Gitea.""" +import base64 +import json +import urllib.request +import urllib.error + +GITEA_URL = "http://192.168.188.119:30008" +OWNER = "pplate" +REPO = "pi_mcps" +TOKEN = "8bf0c734ebda3e61d9c9068489ce58a2bf8d33db" +IMG_BASE = f"{GITEA_URL}/{OWNER}/{REPO}/raw/branch/main/docs/wiki/images" + +PAGES = {} + +PAGES["Home"] = f"""# πŸ”§ pi_mcps β€” Patrick's Homelab Monorepo + +![Home Banner]({IMG_BASE}/home-banner.png) + +Welcome to **pi_mcps**, Patrick's personal homelab monorepo. This repository houses MCP (Model Context Protocol) servers, Java projects, and homelab tooling β€” all built and maintained on a Fedora Linux workstation with an AMD Ryzen 5900X + RX 7900 XTX. + +## What's in this repo? + +| Directory | Contents | +|---|---| +| [`mcp/mcp-image-gen/`](../src/branch/main/mcp/mcp-image-gen) | 🎨 AI image generation via ComfyUI + FLUX.1-schnell | +| [`mcp/webscraper/`](../src/branch/main/mcp/webscraper) | πŸ•ΈοΈ Web scraping and data extraction | +| [`mcp/bigmind/`](../src/branch/main/mcp/bigmind) | 🧠 Persistent AI memory system | +| [`java/`](../src/branch/main/java) | β˜• Java EE / Spring projects | +| [`plans/`](../src/branch/main/plans) | πŸ“‹ Architecture decisions and health reports | + +## Stack + +- **Language:** Python 3.11+ (MCP servers), Java 17 (legacy projects) +- **MCP Framework:** FastMCP 2.x +- **Package Manager:** `uv` (all Python projects) +- **Testing:** `pytest` +- **GPU:** AMD RX 7900 XTX (ROCm / HSA) +- **Server:** TrueNAS.local at `192.168.188.119` (Gitea, Docker) + +## MCP Servers + +Three production-ready MCP servers power Patrick's AI development environment: + +| Server | Status | Description | +|---|---|---| +| [mcp-image-gen](mcp-image-gen) | βœ… Live | Generate images from text prompts via ComfyUI | +| [mcp-webscraper](mcp-webscraper) | βœ… Live | Scrape web pages, extract tables, fetch links | +| [BigMind](BigMind) | βœ… Live | Persistent AI memory across all sessions | + +--- + +*Built and maintained by Patrick Plate (pplate) Β· Homelab: TrueNAS.local Β· AI Colleague: Lumen* +""" + +PAGES["MCP-Servers-Overview"] = f"""# πŸ”Œ MCP Servers Overview + +![MCP Overview Banner]({IMG_BASE}/mcp-overview-banner.png) + +This repo contains three production-grade MCP (Model Context Protocol) servers, each specialized for a different capability domain. Together they give Roo Code / Claude Desktop a complete set of superpowers. + +## The Three Pillars + +``` +Roo Code / Claude Desktop + β”‚ + β”œβ”€β”€ bigmind ──────────► ~/.mcp/bigmind/memory.db (persistent memory) + β”œβ”€β”€ mcp-image-gen ────► ComfyUI @ localhost:8188 (image generation) + └── webscraper ───────► Internet / Intranet (web scraping) +``` + +## Comparison Table + +| Feature | mcp-image-gen | webscraper | bigmind | +|---|---|---|---| +| **Purpose** | Generate images from text | Scrape & parse web | Persistent AI memory | +| **Tools** | 4 | 7 | 15+ | +| **Backend** | ComfyUI / FLUX.1-schnell | httpx + BeautifulSoup4 | SQLite + FTS5 | +| **GPU required** | βœ… AMD RX 7900 XTX | ❌ | ❌ | +| **Tests** | 19/19 βœ… | βœ… | 297/297 βœ… | +| **Schema version** | n/a | n/a | v7 | + +## Quick Links + +- 🎨 [mcp-image-gen](mcp-image-gen) β€” Image generation docs +- πŸ•ΈοΈ [mcp-webscraper](mcp-webscraper) β€” Web scraping docs +- 🧠 [BigMind](BigMind) β€” Memory system docs +- πŸ› οΈ [Development Conventions](Development-Conventions) β€” How all servers are built + +## Adding a New Server + +All servers follow the [FastMCP convention](Development-Conventions). Use the `new-mcp-server` Roo skill to scaffold: + +```bash +# In Roo Code orchestrator, load skill: +# skill: new-mcp-server +``` +""" + +PAGES["mcp-image-gen"] = f"""# 🎨 mcp-image-gen β€” AI Image Generation + +![Image Gen Banner]({IMG_BASE}/image-gen-banner.png) + +**mcp-image-gen** is a FastMCP server that wraps the ComfyUI REST API, enabling Roo Code and Claude Desktop to generate images directly from text prompts using FLUX.1-schnell running on an AMD RX 7900 XTX GPU. + +## Architecture + +``` +Roo Code / Claude Desktop + β”‚ MCP (stdio) + β–Ό +mcp-image-gen (FastMCP, Python 3.11+) + β”‚ HTTP REST + β–Ό +ComfyUI @ localhost:8188 + β”‚ ROCm / HSA_OVERRIDE_GFX_VERSION=11.0.0 + β–Ό +FLUX.1-schnell (~8s/image @ 1024Γ—1024) +``` + +## Tools + +| Tool | Description | +|---|---| +| `generate_image` | Generate PNG from text prompt; returns file path + inline base64 | +| `list_available_models` | List ComfyUI checkpoint models | +| `get_generation_status` | Check status of a queued/running job | +| `get_output_directory` | Return configured output directory path | + +## Key Parameters β€” `generate_image` + +| Parameter | Default | Description | +|---|---|---| +| `prompt` | required | Text description of the image | +| `width` | `1024` | Image width in pixels | +| `height` | `1024` | Image height in pixels | +| `steps` | `4` | Inference steps (FLUX.1-schnell is 4-step) | +| `model` | `flux1-schnell.safetensors` | Model checkpoint name | +| `seed` | `-1` (random) | Generation seed for reproducibility | +| `negative_prompt` | `""` | Things to avoid in the image | +| `output_dir` | `~/Pictures/mcp-generated` | Where to save output PNG | + +## Environment Variables + +| Variable | Default | Description | +|---|---|---| +| `COMFYUI_URL` | `http://localhost:8188` | ComfyUI API endpoint | +| `IMAGE_OUTPUT_DIR` | `~/Pictures/mcp-generated` | Default output directory | +| `COMFYUI_TIMEOUT` | `120` | Request timeout in seconds | + +## Return Value + +The tool returns **two content items**: +1. `TextContent` β€” file path, seed used, elapsed time +2. `ImageContent` β€” base64-encoded PNG (displays inline in Roo Code chat) + +> ⚠️ **Known FastMCP Bug:** Never use `fastmcp.utilities.types.Image` as return type β€” it breaks serialization in FastMCP 3.x. Use `mcp.types.ImageContent` directly. + +## Setup + +See [ComfyUI Setup Guide](mcp-image-gen-ComfyUI-Setup) for full installation instructions. + +### Quick Start + +```bash +cd mcp/mcp-image-gen +uv sync +# Set COMFYUI_URL if ComfyUI is not on localhost +./run.sh +``` + +### Run Tests + +```bash +cd mcp/mcp-image-gen +uv run pytest tests/ -v +``` + +## Lumen Profile Images + +The first images generated with this server were Lumen's visual identity portraits, stored in [`mcp/mcp-image-gen/lumen_profiles/`](../src/branch/main/mcp/mcp-image-gen/lumen_profiles): + +![Lumen Profile]({IMG_BASE}/lumen-profile.png) + +*Primary profile: seed `568659042` β€” constellation face interpretation of Lumen.* +""" + +PAGES["mcp-image-gen-ComfyUI-Setup"] = f"""# βš™οΈ ComfyUI Setup Guide (AMD ROCm) + +This guide covers installing ComfyUI with FLUX.1-schnell on a Fedora Linux system with an AMD GPU. + +## Prerequisites + +- AMD GPU with ROCm support (tested: RX 7900 XTX) +- Fedora Linux (tested: Fedora 43 / kernel 6.19) +- Python 3.11+ +- ~15GB free disk space (model weights) +- HuggingFace account with FLUX license accepted + +## Step 1: Install ComfyUI + +ComfyUI is **not on PyPI** β€” must be cloned from source: + +```bash +cd ~ +git clone https://github.com/comfyanonymous/ComfyUI +cd ComfyUI +python -m venv .venv +source .venv/bin/activate + +# Install PyTorch ROCm build (CRITICAL for AMD GPUs) +pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm6.2 + +# Install ComfyUI dependencies +pip install -r requirements.txt +``` + +## Step 2: Download FLUX.1-schnell + +FLUX.1-schnell is **gated on HuggingFace** β€” you must: +1. Create a HuggingFace account +2. Accept the FLUX.1-schnell license at https://huggingface.co/black-forest-labs/FLUX.1-schnell +3. Generate an access token at https://huggingface.co/settings/tokens + +```bash +# Install huggingface_hub +pip install huggingface_hub + +# Download model (requires HF token) +huggingface-cli download black-forest-labs/FLUX.1-schnell \\ + flux1-schnell.safetensors \\ + --local-dir ~/ComfyUI/models/checkpoints \\ + --token YOUR_HF_TOKEN_HERE +``` + +## Step 3: Download VAE and CLIP Models + +FLUX.1-schnell also requires VAE and CLIP text encoders: + +```bash +# VAE +huggingface-cli download black-forest-labs/FLUX.1-schnell \\ + ae.safetensors \\ + --local-dir ~/ComfyUI/models/vae + +# CLIP models (T5 and CLIP-L) +huggingface-cli download comfyanonymous/flux_text_encoders \\ + t5xxl_fp8_e4m3fn.safetensors clip_l.safetensors \\ + --local-dir ~/ComfyUI/models/clip +``` + +## Step 4: Start ComfyUI + +```bash +cd ~/ComfyUI + +# AMD GPU REQUIRES this environment variable +HSA_OVERRIDE_GFX_VERSION=11.0.0 \\ + nohup .venv/bin/python main.py --listen --port 8188 > /tmp/comfyui.log 2>&1 & + +echo "ComfyUI PID: $!" +``` + +> ⚠️ `HSA_OVERRIDE_GFX_VERSION=11.0.0` is mandatory for RX 7900 XTX on ROCm. Without it, model loading fails silently. + +## Step 5: Verify ComfyUI is Running + +```bash +curl http://localhost:8188/system_stats +# Should return JSON with GPU info +``` + +## Step 6: Configure mcp-image-gen + +```bash +cd /path/to/pi_mcps/mcp/mcp-image-gen +cp .env.example .env # if exists, or set manually + +# .env contents: +COMFYUI_URL=http://localhost:8188 +IMAGE_OUTPUT_DIR=~/Pictures/mcp-generated +COMFYUI_TIMEOUT=120 +``` + +## Performance + +| GPU | Model | Resolution | Steps | Time | +|---|---|---|---|---| +| AMD RX 7900 XTX | FLUX.1-schnell | 1024Γ—1024 | 4 | ~8s | +| AMD RX 7900 XTX | FLUX.1-schnell | 1280Γ—512 | 4 | ~7s | + +## Troubleshooting + +| Problem | Solution | +|---|---| +| `HTTP 401` downloading model | Accept FLUX license on HuggingFace first | +| GPU not detected | Ensure `HSA_OVERRIDE_GFX_VERSION=11.0.0` is set | +| `Connection refused` from mcp-image-gen | Start ComfyUI first, check port 8188 | +| Slow generation (>60s) | ComfyUI may be running on CPU β€” check ROCm install | +| Ollama image gen | As of April 2026: macOS-only, not available on Linux | +""" + +PAGES["mcp-webscraper"] = f"""# πŸ•ΈοΈ mcp-webscraper β€” Web Scraping + +![Webscraper Banner]({IMG_BASE}/webscraper-banner.png) + +**mcp-webscraper** is a FastMCP server providing comprehensive web scraping and data extraction capabilities. It fetches pages, converts HTML to clean Markdown, extracts tables, links, CSS sections, metadata, and sitemaps. + +## Tools + +| Tool | Description | +|---|---| +| `webscraper_fetch(url, max_chars=5000)` | Title + full page as Markdown + metadata | +| `webscraper_fetch_links(url, deduplicate=True)` | All `href` links found on the page | +| `webscraper_fetch_tables(url)` | All HTML tables converted to Markdown | +| `webscraper_fetch_all(url, max_chars=5000)` | Everything in one call (fetch + links + tables) | +| `webscraper_fetch_section(url, selector)` | Specific CSS selector section only | +| `webscraper_fetch_meta(url)` | Title, description, Open Graph tags | +| `webscraper_fetch_sitemap(url, max_urls=100)` | Parse sitemap.xml, return URL list | + +## Stack + +- **HTTP client:** `httpx` (async, with SSL support) +- **HTML parser:** `BeautifulSoup4` + `lxml` +- **Markdown converter:** `html2text` +- **SSL:** Custom cert bundle for Fedora 43 compatibility + +## SSL Note β€” Fedora 43 Comodo Root CA + +Fedora 43 is missing the **Comodo AAA Services Root CA** needed for Cloudflare-protected sites. The fix is bundled at [`mcp/webscraper/certs/comodo-aaa-services-root.pem`](../src/branch/main/mcp/webscraper/certs/). + +The server automatically uses this cert bundle β€” no manual configuration needed. + +## Quick Start + +```bash +cd mcp/webscraper +uv sync +./run.sh +``` + +## Usage Examples + +```python +# In Roo Code / Claude Desktop via MCP: + +# Fetch a page as Markdown +webscraper_fetch("https://docs.fastmcp.dev", max_chars=10000) + +# Extract all links from Gitea repo +webscraper_fetch_links("http://192.168.188.119:30008/pplate/pi_mcps") + +# Get all tables from a documentation page +webscraper_fetch_tables("https://pypi.org/project/fastmcp/") + +# Get Open Graph metadata +webscraper_fetch_meta("https://github.com/comfyanonymous/ComfyUI") + +# Fetch specific section by CSS selector +webscraper_fetch_section("https://docs.python.org", "#content") +``` +""" + +PAGES["BigMind"] = f"""# 🧠 BigMind β€” Persistent AI Memory + +![BigMind Banner]({IMG_BASE}/bigmind-banner.png) + +**BigMind** is the persistent memory backbone for all AI development sessions. It provides SQLite-backed tiered memory with FTS5 full-text search, hypothesis tracking, session management, and token efficiency logging. It is the reason Lumen (Patrick's AI colleague) remembers everything across sessions. + +## Core Concepts + +### Tiered Memory +| Tier | Name | Content | +|---|---|---| +| 0 | **Session Index** | Lightweight list: ID, date, one-liner | +| 1 | **Topic Index** | Per-session topic tags and metadata | +| 2 | **Narrative** | Full 3-8 sentence session summaries | +| 3 | **Flagged Exchanges** | Specific important moments, decisions, code | + +### Facts Store +Atomic, reusable knowledge pieces categorized by type: +- `user-preference` β€” Patrick's tool/style preferences +- `architecture-decision` β€” System design choices +- `codebase-convention` β€” How code is structured +- `environment-config` β€” Server IPs, paths, credentials +- `bug-pattern` β€” Known bugs and fixes +- `api-contract` β€” MCP tool signatures + +## Key Tools + +### Session Lifecycle +| Tool | Description | +|---|---| +| `memory_start_session()` | Open new session, load prior context | +| `memory_end_session(...)` | Close session with summary, topics, outcome | +| `memory_announce_focus(...)` | Declare files to be touched this session | +| `memory_close_stale_sessions(...)` | Clean up crashed IDE sessions | + +### Search +| Tool | Description | +|---|---| +| `memory_search_facts(query, limit=10)` | FTS5 search over stored facts | +| `memory_search_chunks(query, limit=10)` | FTS5 search over conversation chunks | +| `memory_list_sessions(limit=20)` | Browse session history | + +### Storage +| Tool | Description | +|---|---| +| `memory_store_fact(category, fact)` | Store atomic reusable fact | +| `memory_append_chunk(session_id, content, role)` | Store conversation chunk | +| `memory_flag_important(session_id, content, role, flag_reason)` | Flag critical exchange | +| `memory_log_token_save(session_id, description, tokens_saved, method_used)` | Track efficiency | + +### Hypotheses +| Tool | Description | +|---|---| +| `memory_add_hypothesis(session_id, hypothesis, confidence)` | Form testable prediction | +| `memory_resolve_hypothesis(hypothesis_id, status, resolution)` | Confirm/refute prediction | +| `memory_list_hypotheses(status)` | Review open/closed predictions | + +## FTS5 Search Tips + +BigMind uses SQLite FTS5 β€” **every token must match**. Use 2-3 focused keywords: + +``` +βœ… memory_search_facts("TrueNAS Docker") +βœ… memory_search_facts("mcp.json config") +❌ memory_search_facts("homelab infrastructure TrueNAS Docker server") β†’ 0 results +``` + +## Stats (2026-04-04) + +| Metric | Value | +|---|---| +| DB size | 744KB | +| Sessions | 98 | +| Facts | 97+ | +| Chunks | 41 | +| Schema version | v7 | + +## DB Location + +`~/.mcp/bigmind/memory.db` β€” outside the repo, never committed. + +## Session Ritual + +Every session **must** follow this ritual: + +**Start:** +1. `memory_start_session()` +2. `memory_list_hypotheses()` +3. `memory_announce_focus(...)` +4. `memory_close_stale_sessions(...)` + +**End:** +1. `memory_end_session(one_liner, topics, outcome, summary, importance)` +""" + +PAGES["Development-Conventions"] = """# πŸ› οΈ Development Conventions + +All MCP servers in this repo follow a consistent set of conventions to ensure maintainability, testability, and compatibility with Roo Code tooling. + +## Directory Structure + +Each MCP server lives at `mcp//` with this layout: + +``` +mcp// +β”œβ”€β”€ src/ +β”‚ β”œβ”€β”€ __init__.py +β”‚ └── server.py ← FastMCP server entry point +β”œβ”€β”€ tests/ +β”‚ └── test_server.py ← pytest test suite +β”œβ”€β”€ pyproject.toml ← uv-managed dependencies +β”œβ”€β”€ run.sh ← launch script +β”œβ”€β”€ README.md ← server documentation +β”œβ”€β”€ PLAN.md ← architecture plan (pre-implementation) +└── ASSESSMENT.md ← pre-implementation assessment +``` + +## FastMCP Pattern + +```python +from fastmcp import FastMCP + +mcp = FastMCP("server-name") + +@mcp.tool() +def my_tool(param: str) -> str: + \"\"\"Tool description shown to the AI.\"\"\" + return result + +if __name__ == "__main__": + mcp.run() +``` + +## Package Management + +**All projects use `uv`** β€” never `pip` directly: + +```bash +# Create new server +uv init mcp/my-server +cd mcp/my-server +uv add fastmcp httpx + +# Sync dependencies +uv sync + +# Run server +uv run python src/server.py + +# Run tests +uv run pytest tests/ -v +``` + +## pyproject.toml Template + +```toml +[project] +name = "mcp-my-server" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "fastmcp>=2.0.0", + "httpx", +] + +[project.scripts] +mcp-my-server = "src.server:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.pytest.ini_options] +testpaths = ["tests"] +``` + +## Testing Conventions + +- Tests live in `tests/test_server.py` +- Use `pytest` via `uv run pytest` +- Mock external dependencies (ComfyUI, web URLs) for unit tests +- All tests must pass before committing (`git push` should only happen with green tests) + +## Commit Convention + +Follow **Conventional Commits** format: + +``` +feat: add webscraper_fetch_section tool +fix: handle ComfyUI timeout gracefully +docs: update mcp-image-gen README with AMD setup +test: add unit tests for generate_image tool +refactor: extract workflow builder to separate module +chore: bump fastmcp to 2.1.0 +``` + +## Creating a New MCP Server + +Use the `new-mcp-server` Roo skill in MCP Builder mode for full scaffolding: + +``` +1. Switch to πŸ”§ MCP Builder mode in Roo Code +2. Say: "Create a new MCP server for " +3. Roo will load the new-mcp-server skill and scaffold everything +``` + +## Gitea Repository + +Code is hosted at: `http://192.168.188.119:30008/pplate/pi_mcps` + +Push with the `gitea-push` Roo skill to ensure conventional commit format. +""" + + +def create_wiki_page(title: str, content: str) -> bool: + content_b64 = base64.b64encode(content.encode("utf-8")).decode("ascii") + payload = json.dumps({ + "title": title, + "content_base64": content_b64, + "message": f"docs: create {title} wiki page" + }).encode("utf-8") + + url = f"{GITEA_URL}/api/v1/repos/{OWNER}/{REPO}/wiki/pages" + req = urllib.request.Request( + url, + data=payload, + headers={ + "Authorization": f"token {TOKEN}", + "Content-Type": "application/json", + }, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read().decode()) + print(f"βœ… Created: {data.get('title', title)}") + return True + except urllib.error.HTTPError as e: + body = e.read().decode() + print(f"❌ Failed [{title}]: HTTP {e.code} β€” {body[:200]}") + return False + except Exception as ex: + print(f"❌ Failed [{title}]: {ex}") + return False + + +if __name__ == "__main__": + results = {} + for title, content in PAGES.items(): + ok = create_wiki_page(title, content) + results[title] = ok + + print("\n=== Summary ===") + for title, ok in results.items(): + status = "βœ…" if ok else "❌" + print(f"{status} {title}") + + total = sum(results.values()) + print(f"\n{total}/{len(results)} pages created successfully")