feat(webscraper): add Brave Search hint tool and User-Agent header

- Add webscraper_search_hint() tool using Brave Search as backend
  (no CAPTCHA/GDPR consent wall, works with plain httpx)
- Add User-Agent header to _fetch_page() — fixes 403 on Wikipedia,
  Feynman Lectures, and other sites that block headless requests
- Add 5 new tests for search hint (23 total, 90% coverage)

Brave Search URL: https://search.brave.com/search?q={query}&source=web
Use sparingly — once per research task as orientation, not in loops

mcp/webscraper/src/server.py

@@ -28,9 +28,16 @@ def _build_ssl_context() -> ssl.SSLContext:
 
 _SSL_CTX = _build_ssl_context()
 
+_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+    )
+}
+
 def _fetch_page(url: str) -> Tuple[httpx.Response, BeautifulSoup]:
     """Shared fetch helper — returns response and parsed soup."""
-    response = httpx.get(url, timeout=10.0, verify=_SSL_CTX)
+    response = httpx.get(url, timeout=10.0, verify=_SSL_CTX, headers=_HEADERS)
     response.raise_for_status()
     soup = BeautifulSoup(response.text, 'lxml')
     return response, soup
@@ -255,5 +262,51 @@ def webscraper_fetch_sitemap(url: str, max_urls: int = 100) -> List[str]:
     except (httpx.RequestError, httpx.HTTPStatusError) as e:
         return [f"Error: {str(e)}"]
 
+@mcp.tool()
+def webscraper_search_hint(query: str, max_results: int = 5) -> Dict:
+    """Search Brave Search and return top results as a scraping hint.
+
+    Use this sparingly — once per research task — to get oriented before
+    scraping individual pages. Returns top result URLs + snippets so you
+    can decide which pages are worth scraping deeply.
+
+    Args:
+        query: Search query (e.g. "MacBook Pro M4 price Germany")
+        max_results: Maximum number of results to return (default: 5)
+
+    Returns:
+        Dict with 'query', 'results' (list of {title, url, snippet}), 'hint'
+    """
+    try:
+        search_url = f"https://search.brave.com/search?q={query.replace(' ', '+')}&source=web"
+        _, soup = _fetch_page(search_url)
+
+        results = []
+        # Brave Search result cards: each <a> with class snippet contains title + description
+        for card in soup.select('.snippet')[:max_results]:
+            title_el = card.select_one('.snippet-title')
+            url_el = card.select_one('a')
+            desc_el = card.select_one('.snippet-description')
+
+            title = title_el.get_text(strip=True) if title_el else ""
+            url = url_el['href'] if url_el and url_el.get('href') else ""
+            snippet = desc_el.get_text(strip=True) if desc_el else ""
+
+            if url and url.startswith('http'):
+                results.append({"title": title, "url": url, "snippet": snippet})
+
+        hint = "; ".join(
+            f"{r['title']}: {r['url']}" for r in results
+        ) if results else "No results found"
+
+        return {
+            "query": query,
+            "results": results,
+            "hint": hint,
+        }
+    except (httpx.RequestError, httpx.HTTPStatusError) as e:
+        return {"query": query, "results": [], "hint": f"Error: {str(e)}"}
+
+
 if __name__ == "__main__":
     mcp.run(transport="stdio")