diff options
Diffstat (limited to 'ai-gen.md')
| -rw-r--r-- | ai-gen.md | 567 |
1 files changed, 567 insertions, 0 deletions
diff --git a/ai-gen.md b/ai-gen.md new file mode 100644 index 0000000..ca93714 --- /dev/null +++ b/ai-gen.md @@ -0,0 +1,567 @@ +# 🏛️ AI Genealogy — Local AI for Genealogical Research + +**Complete installation and usage guide** +Gentoo Linux + OpenRC + AMD Radeon 7900 XTX + Docker +*All components are 100% open-source, fully local, private, and can be turned off when not needed.* + +--- + +## Table of Contents + +1. [System Overview](#-system-overview) +2. [Installation](#-installation) +3. [Usage Cheatsheet](#-usage-cheatsheet) +4. [Daily Workflow](#-daily-workflow) + +--- + +## 📦 System Overview + +| Component | Purpose | Port | License | +|-----------|---------|------|---------| +| **Open WebUI** | Web interface, chat, document upload, RAG | 3000 | MIT | +| **Ollama** | LLM server (Qwen2.5, Mistral, BGE-M3) | 11434 | MIT | +| **Qdrant** | Vector database for semantic search | 6333 | Apache 2.0 | +| **SearXNG** | Private meta-search engine | 8080 | AGPLv3 | +| **FastAPI** | OCR, web scraping, indexing pipeline | 8000 | MIT | +| **EasyOCR** | GPU-accelerated OCR (Russian, Polish, Belarusian) | - | Apache 2.0 | +| **Tesseract** | CPU OCR fallback (best for Polish) | - | Apache 2.0 | +| **Crawl4AI** | Website scraping with authentication | - | MIT | +| **BGE-M3** | Multilingual embedding model | - | Apache 2.0 | + +--- + +## 🔧 Installation + +Copy and paste this entire block into your terminal. It will install all dependencies, create all project files, build the Docker images, and launch the system. + +```bash +# ============================================================================= +# STEP 1: System Preparation +# ============================================================================= +echo "=== Step 1: System Preparation ===" +ls /dev/kfd /dev/dri/render* +groups +doas groupadd render 2>/dev/null +doas usermod -a -G video,render $USER +echo "✅ If 'render' group was just created, log out and log back in, then re-run this script." + +# ============================================================================= +# STEP 2: Install Docker +# ============================================================================= +echo "=== Step 2: Install Docker ===" +doas emerge -a app-containers/docker app-containers/docker-cli +doas rc-update add docker default +doas rc-service docker start +doas usermod -aG docker $USER +echo "✅ Log out and log back in for docker group to take effect, then re-run this script." + +# ============================================================================= +# STEP 3: Install Docker Compose Plugin +# ============================================================================= +echo "=== Step 3: Install Docker Compose Plugin ===" +doas mkdir -p /usr/libexec/docker/cli-plugins +doas curl -SL "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-x86_64" \ + -o /usr/libexec/docker/cli-plugins/docker-compose +doas chmod +x /usr/libexec/docker/cli-plugins/docker-compose +docker compose version +echo "✅ Docker Compose installed." + +# ============================================================================= +# STEP 4: Install Tesseract OCR +# ============================================================================= +echo "=== Step 4: Install Tesseract OCR ===" +doas emerge -a app-text/tesseract app-text/tessdata_fast +doas wget -O /usr/share/tessdata/pol.traineddata \ + "https://github.com/tesseract-ocr/tessdata/raw/main/pol.traineddata" +doas wget -O /usr/share/tessdata/rus.traineddata \ + "https://github.com/tesseract-ocr/tessdata/raw/main/rus.traineddata" +doas wget -O /usr/share/tessdata/bel.traineddata \ + "https://github.com/tesseract-ocr/tessdata/raw/main/bel.traineddata" +tesseract --list-langs +echo "✅ Tesseract installed." + +# ============================================================================= +# STEP 5: Create Project Directory and All Files +# ============================================================================= +echo "=== Step 5: Create Project Files ===" +mkdir -p ~/ai-genealogy/{fastapi/app,data} +mkdir -p ~/bin +cd ~/ai-genealogy + +# --- docker-compose.yaml --- +cat > docker-compose.yaml << 'EOF' +services: + ollama: + image: ollama/ollama:rocm + container_name: gen-ollama + devices: + - /dev/kfd + - /dev/dri + volumes: + - ollama_data:/root/.ollama + environment: + - HSA_OVERRIDE_GFX_VERSION=11.0.0 + command: serve + ports: + - "11434:11434" + restart: "no" + + open-webui: + image: ghcr.io/open-webui/open-webui:main + container_name: gen-webui + ports: + - "3000:8080" + volumes: + - open-webui_data:/app/backend/data + - ./fastapi/app/tools.py:/app/tools.py:ro + environment: + - OLLAMA_BASE_URL=http://ollama:11434 + - ENABLE_RAG_WEB_SEARCH=true + - RAG_EMBEDDING_ENGINE=ollama + - RAG_EMBEDDING_MODEL=bge-m3:latest + - WEB_SEARCH_ENGINE=searxng + - SEARXNG_QUERY_URL=http://searxng:8080/search?q=<query> + - TOOLS_ENABLED=true + depends_on: + - ollama + - searxng + - genea-api + restart: "no" + + searxng: + image: searxng/searxng:latest + container_name: gen-searxng + ports: + - "8080:8080" + volumes: + - searxng_data:/etc/searxng + environment: + - SEARXNG_BASE_URL=http://localhost:8080/ + restart: "no" + + qdrant: + image: qdrant/qdrant:latest + container_name: gen-qdrant + ports: + - "6333:6333" + volumes: + - qdrant_storage:/qdrant/storage + restart: "no" + + genea-api: + build: ./fastapi + image: genea-api:latest + container_name: gen-api + ports: + - "8000:8000" + volumes: + - ./data:/data + environment: + - QDRANT_URL=http://qdrant:6333 + - OLLAMA_URL=http://ollama:11434 + - EMBED_MODEL=bge-m3 + - SEARXNG_URL=http://searxng:8080 + devices: + - /dev/kfd + - /dev/dri + depends_on: + - qdrant + - ollama + restart: "no" + +volumes: + ollama_data: + open-webui_data: + searxng_data: + qdrant_storage: +EOF + +# --- fastapi/Dockerfile --- +cat > fastapi/Dockerfile << 'EOF' +FROM rocm/pytorch:latest + +RUN apt update && apt install -y \ + libgl1 \ + libglib2.0-0 \ + wget \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +RUN apt update && apt install -y \ + tesseract-ocr \ + tesseract-ocr-rus \ + tesseract-ocr-pol \ + tesseract-ocr-bel \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY ./app/ /app/ + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +EOF + +# --- fastapi/requirements.txt --- +cat > fastapi/requirements.txt << 'EOF' +fastapi +uvicorn[standard] +httpx +python-multipart +qdrant-client +easyocr +crawl4ai +playwright-stealth +Pillow +opencv-python-headless +EOF + +# --- fastapi/app/__init__.py --- +touch fastapi/app/__init__.py + +# --- fastapi/app/ocr_utils.py --- +cat > fastapi/app/ocr_utils.py << 'EOF' +import easyocr +import logging +import subprocess + +logger = logging.getLogger(__name__) +STANDARD_READERS = {} + +def get_reader(lang: str): + global STANDARD_READERS + if lang not in STANDARD_READERS: + STANDARD_READERS[lang] = easyocr.Reader([lang], gpu=True) + return STANDARD_READERS[lang] + +def recognize_text(image_path: str, lang: str = "ru") -> str: + reader = get_reader(lang) + result = reader.readtext(image_path, detail=0, paragraph=True) + text = ' '.join(result) + if len(text.strip()) < 10 and lang in ['ru', 'pol', 'bel']: + tess_text = tesseract_ocr(image_path, lang=lang) + if len(tess_text) > len(text): + text = tess_text + return text + +def tesseract_ocr(image_path: str, lang: str = 'ru') -> str: + lang_map = {'ru': 'rus', 'pol': 'pol', 'bel': 'bel', 'en': 'eng'} + tess_lang = lang_map.get(lang, 'rus') + try: + result = subprocess.run( + ['tesseract', image_path, 'stdout', '-l', tess_lang, '--psm', '6'], + capture_output=True, text=True, timeout=60 + ) + return result.stdout.strip() + except Exception as e: + logger.error(f"Tesseract error: {e}") + return "" +EOF + +# --- fastapi/app/crawl_utils.py --- +cat > fastapi/app/crawl_utils.py << 'EOF' +import asyncio +import logging +from crawl4ai import AsyncWebCrawler, CacheMode +from typing import Optional, Callable + +logger = logging.getLogger(__name__) + +async def crawl_and_index(url, login=None, password=None, login_url=None, + username_field="username", password_field="password", + submit_button="button[type='submit']", index_func=None, chunk_size=500): + async with AsyncWebCrawler(verbose=False, headless=True) as crawler: + if login and password: + auth_url = login_url or url.rstrip('/') + '/login' + js_code = f""" + (function() {{ + var u = document.querySelector('{username_field}'); + var p = document.querySelector('{password_field}'); + var b = document.querySelector('{submit_button}'); + if (u && p && b) {{ u.value = '{login}'; p.value = '{password}'; b.click(); }} + }})(); + """ + try: + await crawler.arun(url=auth_url, js_code=js_code, cache_mode=CacheMode.BYPASS) + await asyncio.sleep(3) + except Exception as e: + logger.warning(f"Login error: {e}") + result = await crawler.arun(url=url, cache_mode=CacheMode.BYPASS) + if not result.markdown: return 0 + markdown = '\n'.join(line for line in result.markdown.split('\n') if line.strip()) + if index_func is None: return len(markdown) + chunks = [markdown[i:i+chunk_size] for i in range(0, len(markdown), chunk_size)] + indexed = 0 + for chunk in chunks: + if chunk.strip(): + try: + await index_func(text=chunk, source=url) + indexed += 1 + except Exception as e: + logger.error(f"Index error: {e}") + return indexed + +SITE_CONFIGS = { + "forum.vgd.ru": {"username_field": "input[name='login']", "password_field": "input[name='password']", "submit_button": "input[type='submit']"}, + "genealodzy.pl": {"username_field": "input#username", "password_field": "input#password", "submit_button": "button#login-button"}, + "familysearch.org": {"username_field": "input#userName", "password_field": "input#password", "submit_button": "button[type='submit']"}, + "default": {"username_field": "input[name='username']", "password_field": "input[name='password']", "submit_button": "button[type='submit']"} +} + +def get_site_config(url): + for domain, config in SITE_CONFIGS.items(): + if domain in url: return config + return SITE_CONFIGS["default"] +EOF + +# --- fastapi/app/main.py --- +cat > fastapi/app/main.py << 'EOF' +from fastapi import FastAPI, UploadFile, File, HTTPException +import httpx, os, logging +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams, PointStruct +from ocr_utils import recognize_text +from crawl_utils import crawl_and_index, get_site_config + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) +app = FastAPI(title="AI Genealogy API", version="1.0.0") + +client = QdrantClient(url=os.getenv("QDRANT_URL")) +ollama_url = os.getenv("OLLAMA_URL") +embed_model = os.getenv("EMBED_MODEL", "bge-m3") +searxng_url = os.getenv("SEARXNG_URL") +COLLECTION_NAME = "genealogy_docs" + +try: + client.create_collection(collection_name=COLLECTION_NAME, vectors_config=VectorParams(size=1024, distance=Distance.COSINE)) +except Exception: + pass + +async def get_embedding(text: str): + text = text[:512] + async with httpx.AsyncClient(timeout=30) as http: + resp = await http.post(f"{ollama_url}/api/embeddings", json={"model": embed_model, "prompt": text}) + return resp.json()["embedding"] + +@app.get("/") +async def root(): + return {"service": "AI Genealogy API", "status": "running"} + +@app.post("/ocr") +async def ocr_endpoint(file: UploadFile = File(...), lang: str = "ru"): + temp_path = f"/tmp/{file.filename}" + with open(temp_path, "wb") as f: + f.write(await file.read()) + try: + text = recognize_text(temp_path, lang=lang) + return {"text": text, "lang": lang, "filename": file.filename} + finally: + if os.path.exists(temp_path): os.remove(temp_path) + +@app.post("/index") +async def index_text(text: str, source: str = "", lang: str = "ru"): + if len(text.strip()) < 10: + raise HTTPException(400, "Text too short") + embedding = await get_embedding(text) + point_id = abs(hash(f"{text[:100]}{source}")) % (10**9) + client.upsert(collection_name=COLLECTION_NAME, points=[PointStruct(id=point_id, vector=embedding, payload={"text": text, "source": source, "lang": lang})]) + return {"status": "ok", "id": point_id} + +@app.post("/search") +async def search(query: str, top_k: int = 5): + embedding = await get_embedding(query) + hits = client.search(collection_name=COLLECTION_NAME, query_vector=embedding, limit=top_k) + return [{"text": h.payload["text"], "source": h.payload.get("source",""), "score": h.score} for h in hits] + +@app.post("/scrape") +async def scrape(url: str, login: str = None, password: str = None, login_url: str = None): + config = get_site_config(url) + indexed = await crawl_and_index(url=url, login=login, password=password, login_url=login_url, **config, index_func=index_text) + return {"status": "done", "url": url, "indexed_chunks": indexed} + +@app.post("/fullsearch") +async def full_search(query: str, use_web: bool = False, top_k: int = 5): + local = await search(query, top_k=top_k) + web_results = [] + if use_web and searxng_url: + async with httpx.AsyncClient(timeout=15) as http: + resp = await http.get(f"{searxng_url}/search", params={"q": query, "format": "json"}) + for r in resp.json().get("results", [])[:3]: + web_results.append({"text": f"{r.get('title','')}: {r.get('content','')}", "source": r.get("url","web"), "score": 1.0}) + return {"query": query, "results": local + web_results} +EOF + +# --- fastapi/app/tools.py --- +cat > fastapi/app/tools.py << 'EOF' +import httpx + +async def genealogy_search(query: str, search_web: bool = False) -> str: + """Search through personal genealogy archive.""" + async with httpx.AsyncClient(timeout=30) as client: + resp = await client.post("http://genea-api:8000/fullsearch", json={"query": query, "use_web": search_web}) + results = resp.json().get("results", []) + if not results: return "Nothing found." + return "\n\n".join([f"{i}. {r['source']}\n{r['text'][:400]}" for i, r in enumerate(results, 1)]) + +async def index_document(text: str, source: str = "manual input") -> str: + """Add text to the search database.""" + async with httpx.AsyncClient(timeout=30) as client: + resp = await client.post("http://genea-api:8000/index", json={"text": text, "source": source}) + return f"Saved (id: {resp.json().get('id')})" + +async def scrape_website(url: str) -> str: + """Scrape and index website content.""" + async with httpx.AsyncClient(timeout=120) as client: + resp = await client.post("http://genea-api:8000/scrape", json={"url": url}) + return f"Indexed {resp.json().get('indexed_chunks', 0)} chunks" +EOF + +echo "✅ All project files created." + +# ============================================================================= +# STEP 6: Management Script +# ============================================================================= +echo "=== Step 6: Management Script ===" +cat > ~/bin/ai-genealogy << 'EOF' +#!/bin/bash +cd ~/ai-genealogy +case "$1" in + start) docker compose up -d && echo "✅ Started! http://localhost:3000" ;; + stop) docker compose stop && echo "✅ Stopped. GPU freed." ;; + status) docker compose ps ;; + logs) docker compose logs -f --tail=50 ;; + pull-models) + docker exec -it gen-ollama ollama pull qwen2.5:14b + docker exec -it gen-ollama ollama pull bge-m3 + docker exec -it gen-ollama ollama pull mistral-small:22b + ;; + *) echo "Commands: start | stop | status | logs | pull-models" ;; +esac +EOF +chmod +x ~/bin/ai-genealogy +echo 'export PATH="$HOME/bin:$PATH"' >> ~/.zshrc +source ~/.zshrc +echo "✅ Management script installed." + +# ============================================================================= +# STEP 7: Build and Launch +# ============================================================================= +echo "=== Step 7: Build and Launch (15-30 min) ===" +cd ~/ai-genealogy +docker compose up -d +echo "⏳ Building... Wait for completion." +docker compose logs -f --tail=20 +echo "✅ Build complete. Check status: ai-genealogy status" + +# ============================================================================= +# STEP 8: Download AI Models +# ============================================================================= +echo "=== Step 8: Download AI Models ===" +docker exec -it gen-ollama ollama pull qwen2.5:14b +docker exec -it gen-ollama ollama pull bge-m3 +echo "✅ Models downloaded." + +# ============================================================================= +# STEP 9: Configure Web Interface +# ============================================================================= +echo "=== Step 9: Configure Web Interface ===" +echo "" +echo "1. Open http://localhost:3000 in browser" +echo "2. Create admin account" +echo "3. Admin Panel → Settings → Documents:" +echo " - Embedding Model Engine = Ollama" +echo " - Embedding Model = bge-m3:latest" +echo "4. Admin Panel → Settings → Web Search:" +echo " - Web Search Engine = searxng" +echo " - URL = http://searxng:8080/search?q=<query>" +echo "" +echo "🎉 INSTALLATION COMPLETE!" + + +-------------------------------------------------------------------------------- + +📋 Usage Cheatsheet +Power On / Off +bash + +newgrp docker # Fix permissions if needed +ai-genealogy start # Start everything +ai-genealogy stop # Stop (frees GPU) +ai-genealogy status # Check container status + +Web Interfaces +URL Purpose +http://localhost:3000 Chat, document upload, RAG +http://localhost:8000/docs API documentation (Swagger) +OCR — Document Recognition + +Russian: +bash + +cd ~/ai-genealogy && docker compose stop ollama +curl -s -X POST http://localhost:8000/ocr -F "file=@scan.jpg" -F "lang=ru" | python3 -c "import sys,json; print(json.load(sys.stdin)['text'])" +docker compose start ollama + +Polish (Tesseract — better accuracy): +bash + +tesseract scan.jpg stdout -l pol + +Polish (API): +bash + +cd ~/ai-genealogy && docker compose stop ollama +curl -s -X POST http://localhost:8000/ocr -F "file=@scan.jpg" -F "lang=pol" | python3 -c "import sys,json; print(json.load(sys.stdin)['text'])" +docker compose start ollama + +Belarusian: +bash + +cd ~/ai-genealogy && docker compose stop ollama +curl -s -X POST http://localhost:8000/ocr -F "file=@scan.jpg" -F "lang=bel" | python3 -c "import sys,json; print(json.load(sys.stdin)['text'])" +docker compose start ollama + +Indexing Text +bash + +curl -X POST http://localhost:8000/index \ + -H "Content-Type: application/json" \ + -d '{"text": "document text...", "source": "Metric book 1892 p.5"}' + +Search +bash + +# Local only +curl -X POST http://localhost:8000/search \ + -H "Content-Type: application/json" \ + -d '{"query": "Kowalski 1892"}' + +# With web search +curl -X POST http://localhost:8000/fullsearch \ + -H "Content-Type: application/json" \ + -d '{"query": "Kowalski 1892", "use_web": true}' + +Website Scraping +bash + +curl -X POST http://localhost:8000/scrape \ + -H "Content-Type: application/json" \ + -d '{"url": "https://forum.genealodzy.pl/topic/123"}' + +# With login +curl -X POST http://localhost:8000/scrape \ + -H "Content-Type: application/json" \ + -d '{"url": "https://...", "login": "user", "password": "pass"}' + +Model Management +bash + +# List models +curl -s http://localhost:11434/api/tags | python3 -c "import sys,json; [print(m['name']) for m in json.load(sys.stdin)['models']]" + +# Download new model +docker exec -it gen-ollama ollama pull model_name |