chore: improve podcast transcripts

.env.example

@@ -73,19 +73,6 @@ API_URL=http://localhost:5055
 #
 # ESPERANTO_LLM_TIMEOUT=60
 
-# SSL VERIFICATION CONFIGURATION
-# Configure SSL certificate verification for local AI providers (Ollama, LM Studio, etc.)
-# behind reverse proxies with self-signed certificates
-#
-# Option 1: Custom CA Bundle (recommended for self-signed certs)
-# Point to your CA certificate file to verify SSL while using custom certificates
-# ESPERANTO_SSL_CA_BUNDLE=/path/to/your/ca-bundle.pem
-#
-# Option 2: Disable SSL Verification (development only)
-# WARNING: Disabling SSL verification exposes you to man-in-the-middle attacks
-# Only use in trusted development/testing environments
-# ESPERANTO_SSL_VERIFY=false
-
 # SECURITY
 # Set this to protect your Open Notebook instance with a password (for public hosting)
 # OPEN_NOTEBOOK_PASSWORD=
@@ -156,28 +143,10 @@ API_URL=http://localhost:5055
 # OPENAI_COMPATIBLE_API_KEY_TTS=
 
 # AZURE OPENAI
-# Generic configuration (applies to all modalities: language, embedding, STT, TTS)
 # AZURE_OPENAI_API_KEY=
 # AZURE_OPENAI_ENDPOINT=
-# AZURE_OPENAI_API_VERSION=2024-12-01-preview
-
-# Mode-specific configuration (overrides generic if set)
-# Use these when you want different deployments for different AI capabilities
-# AZURE_OPENAI_API_KEY_LLM=
-# AZURE_OPENAI_ENDPOINT_LLM=
-# AZURE_OPENAI_API_VERSION_LLM=
-
-# AZURE_OPENAI_API_KEY_EMBEDDING=
-# AZURE_OPENAI_ENDPOINT_EMBEDDING=
-# AZURE_OPENAI_API_VERSION_EMBEDDING=
-
-# AZURE_OPENAI_API_KEY_STT=
-# AZURE_OPENAI_ENDPOINT_STT=
-# AZURE_OPENAI_API_VERSION_STT=
-
-# AZURE_OPENAI_API_KEY_TTS=
-# AZURE_OPENAI_ENDPOINT_TTS=
-# AZURE_OPENAI_API_VERSION_TTS=
+# AZURE_OPENAI_API_VERSION="2024-12-01-preview"
+# AZURE_OPENAI_DEPLOYMENT_NAME=
 
 # USE THIS IF YOU WANT TO DEBUG THE APP ON LANGSMITH
 # LANGCHAIN_TRACING_V2=true
@@ -191,64 +160,7 @@ SURREAL_URL="ws://surrealdb/rpc:8000"
 SURREAL_USER="root"
 SURREAL_PASSWORD="root"
 SURREAL_NAMESPACE="open_notebook"
-SURREAL_DATABASE="open_notebook"
-
-# RETRY CONFIGURATION (surreal-commands v1.2.0+)
-# Global defaults for all background commands unless explicitly overridden at command level
-# These settings help commands automatically recover from transient failures like:
-# - Database transaction conflicts during concurrent operations
-# - Network timeouts when calling external APIs
-# - Rate limits from LLM/embedding providers
-# - Temporary resource unavailability
-
-# Enable/disable retry globally (default: true)
-# Set to false to disable retries for all commands (useful for debugging)
-SURREAL_COMMANDS_RETRY_ENABLED=true
-
-# Maximum retry attempts before giving up (default: 3)
-# Database operations use 5 attempts (defined per-command)
-# API calls use 3 attempts (defined per-command)
-SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS=3
-
-# Wait strategy between retry attempts (default: exponential_jitter)
-# Options: exponential_jitter, exponential, fixed, random
-# - exponential_jitter: Recommended - prevents thundering herd during DB conflicts
-# - exponential: Good for API rate limits (predictable backoff)
-# - fixed: Use for quick recovery scenarios
-# - random: Use when you want unpredictable retry timing
-SURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter
-
-# Minimum wait time between retries in seconds (default: 1)
-# Database conflicts: 1 second (fast retry for transient issues)
-# API rate limits: 5 seconds (wait for quota reset)
-SURREAL_COMMANDS_RETRY_WAIT_MIN=1
-
-# Maximum wait time between retries in seconds (default: 30)
-# Database conflicts: 30 seconds maximum
-# API rate limits: 120 seconds maximum (defined per-command)
-# Total retry time won't exceed max_attempts * wait_max
-SURREAL_COMMANDS_RETRY_WAIT_MAX=30
-
-# WORKER CONCURRENCY
-# Maximum number of concurrent tasks in the worker pool (default: 5)
-# This affects the likelihood of database transaction conflicts during batch operations
-#
-# Tuning guidelines based on deployment size:
-# - Resource-constrained (low CPU/memory): 1-2 workers
-#   Reduces conflicts and resource usage, but slower processing
-#
-# - Normal deployment (balanced): 5 workers (RECOMMENDED)
-#   Good balance between throughput and conflict rate
-#   Retry logic handles occasional conflicts gracefully
-#
-# - Large instances (high CPU/memory): 10-20 workers
-#   Higher throughput but more frequent DB conflicts
-#   Relies heavily on retry logic with jittered backoff
-#
-# Note: Higher concurrency increases vectorization speed but also increases
-# SurrealDB transaction conflicts. The retry logic with exponential-jitter
-# backoff ensures operations complete successfully even at high concurrency.
-SURREAL_COMMANDS_MAX_TASKS=5
+SURREAL_DATABASE="staging"
 
 # OPEN_NOTEBOOK_PASSWORD=
 

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -83,22 +83,3 @@ body:
       placeholder: "This started happening after I upgraded to v1.5.0..."
     validations:
       required: false
-
-  - type: checkboxes
-    id: willing-to-contribute
-    attributes:
-      label: Contribution
-      description: Would you like to work on fixing this bug?
-      options:
-        - label: I am a developer and would like to work on fixing this issue (pending maintainer approval)
-          required: false
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-        **Next Steps:**
-        1. A maintainer will review your bug report
-        2. If you checked the box above and want to fix it, please propose your solution approach
-        3. Wait for assignment before starting development
-        4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -26,15 +26,6 @@ body:
     validations:
       required: true
 
-  - type: textarea
-    id: proposed-solution
-    attributes:
-      label: Proposed Solution (Optional)
-      description: If you have ideas on how to implement this feature, please share them
-      placeholder: "This could be implemented by..."
-    validations:
-      required: false
-
   - type: textarea
     id: additional-context
     attributes:
@@ -44,22 +35,3 @@ body:
     validations:
       required: false
 
-  - type: checkboxes
-    id: willing-to-contribute
-    attributes:
-      label: Contribution
-      description: Would you like to work on implementing this feature?
-      options:
-        - label: I am a developer and would like to work on implementing this feature (pending maintainer approval)
-          required: false
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-        **Next Steps:**
-        1. A maintainer will review your feature request
-        2. If approved and you checked the box above, the issue will be assigned to you
-        3. Please wait for assignment before starting development
-        4. See our [Contributing Guide](https://github.com/lfnovo/open-notebook/blob/main/CONTRIBUTING.md) for more details
-

.github/pull_request_template.md

@@ -1,107 +0,0 @@
-## Description
-
-<!-- Provide a clear and concise description of what this PR does -->
-
-## Related Issue
-
-<!-- This PR should be linked to an approved issue. If not, please create an issue first. -->
-
-Fixes #<!-- issue number -->
-
-## Type of Change
-
-<!-- Mark the relevant option with an "x" -->
-
-- [ ] Bug fix (non-breaking change that fixes an issue)
-- [ ] New feature (non-breaking change that adds functionality)
-- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
-- [ ] Documentation update
-- [ ] Code refactoring (no functional changes)
-- [ ] Performance improvement
-- [ ] Test coverage improvement
-
-## How Has This Been Tested?
-
-<!-- Describe the tests you ran and/or how you verified your changes work -->
-
-- [ ] Tested locally with Docker
-- [ ] Tested locally with development setup
-- [ ] Added new unit tests
-- [ ] Existing tests pass (`uv run pytest`)
-- [ ] Manual testing performed (describe below)
-
-**Test Details:**
-<!-- Describe your testing approach -->
-
-## Design Alignment
-
-<!-- This section helps ensure your PR aligns with our project vision -->
-
-**Which design principles does this PR support?** (See [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md))
-
-- [ ] Privacy First
-- [ ] Simplicity Over Features
-- [ ] API-First Architecture
-- [ ] Multi-Provider Flexibility
-- [ ] Extensibility Through Standards
-- [ ] Async-First for Performance
-
-**Explanation:**
-<!-- Brief explanation of how your changes align with these principles -->
-
-## Checklist
-
-<!-- Mark completed items with an "x" -->
-
-### Code Quality
-- [ ] My code follows PEP 8 style guidelines (Python)
-- [ ] My code follows TypeScript best practices (Frontend)
-- [ ] I have added type hints to my code (Python)
-- [ ] I have added JSDoc comments where appropriate (TypeScript)
-- [ ] I have performed a self-review of my code
-- [ ] I have commented my code, particularly in hard-to-understand areas
-- [ ] My changes generate no new warnings or errors
-
-### Testing
-- [ ] I have added tests that prove my fix is effective or that my feature works
-- [ ] New and existing unit tests pass locally with my changes
-- [ ] I ran linting: `make ruff` or `ruff check . --fix`
-- [ ] I ran type checking: `make lint` or `uv run python -m mypy .`
-
-### Documentation
-- [ ] I have updated the relevant documentation in `/docs` (if applicable)
-- [ ] I have added/updated docstrings for new/modified functions
-- [ ] I have updated the API documentation (if API changes were made)
-- [ ] I have added comments to complex logic
-
-### Database Changes
-- [ ] I have created migration scripts for any database schema changes (in `/migrations`)
-- [ ] Migration includes both up and down scripts
-- [ ] Migration has been tested locally
-
-### Breaking Changes
-- [ ] This PR includes breaking changes
-- [ ] I have documented the migration path for users
-- [ ] I have updated MIGRATION.md (if applicable)
-
-## Screenshots (if applicable)
-
-<!-- Add screenshots for UI changes -->
-
-## Additional Context
-
-<!-- Add any other context about the PR here -->
-
-## Pre-Submission Verification
-
-Before submitting, please verify:
-
-- [ ] I have read [CONTRIBUTING.md](../CONTRIBUTING.md)
-- [ ] I have read [DESIGN_PRINCIPLES.md](../DESIGN_PRINCIPLES.md)
-- [ ] This PR addresses an approved issue that was assigned to me
-- [ ] I have not included unrelated changes in this PR
-- [ ] My PR title follows conventional commits format (e.g., "feat: add user authentication")
-
----
-
-**Thank you for contributing to Open Notebook!** 🎉

.github/workflows/build-dev.yml

@@ -12,6 +12,15 @@ on:
       - '.github/workflows/claude*.yml'
   workflow_dispatch:
     inputs:
+      dockerfile:
+        description: 'Dockerfile to test'
+        required: true
+        default: 'both'
+        type: choice
+        options:
+          - both
+          - regular
+          - single
       platform:
         description: 'Platform to build'
         required: true
@@ -34,7 +43,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-
+      
       - name: Extract version from pyproject.toml
         id: version
         run: |
@@ -42,9 +51,10 @@ jobs:
           echo "version=$VERSION" >> $GITHUB_OUTPUT
           echo "Extracted version: $VERSION"
 
-  test-build:
+  test-build-regular:
     needs: extract-version
     runs-on: ubuntu-latest
+    if: github.event.inputs.dockerfile == 'regular' || github.event.inputs.dockerfile == 'both' || github.event_name != 'workflow_dispatch'
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -56,18 +66,18 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache-dev
-          key: ${{ runner.os }}-buildx-dev-${{ github.sha }}
+          key: ${{ runner.os }}-buildx-dev-regular-${{ github.sha }}
           restore-keys: |
-            ${{ runner.os }}-buildx-dev-
+            ${{ runner.os }}-buildx-dev-regular-
 
-      - name: Build image (test only)
+      - name: Build regular image (test only)
         uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile
           platforms: ${{ github.event.inputs.platform || 'linux/amd64' }}
           push: false
-          tags: ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-dev
+          tags: ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-dev-regular
           cache-from: type=local,src=/tmp/.buildx-cache-dev
           cache-to: type=local,dest=/tmp/.buildx-cache-dev-new,mode=max
 
@@ -76,8 +86,43 @@ jobs:
           rm -rf /tmp/.buildx-cache-dev
           mv /tmp/.buildx-cache-dev-new /tmp/.buildx-cache-dev
 
+  test-build-single:
+    needs: extract-version
+    runs-on: ubuntu-latest
+    if: github.event.inputs.dockerfile == 'single' || github.event.inputs.dockerfile == 'both' || github.event_name != 'workflow_dispatch'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache-dev-single
+          key: ${{ runner.os }}-buildx-dev-single-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-dev-single-
+
+      - name: Build single-container image (test only)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile.single
+          platforms: ${{ github.event.inputs.platform || 'linux/amd64' }}
+          push: false
+          tags: ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-dev-single
+          cache-from: type=local,src=/tmp/.buildx-cache-dev-single
+          cache-to: type=local,dest=/tmp/.buildx-cache-dev-single-new,mode=max
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache-dev-single
+          mv /tmp/.buildx-cache-dev-single-new /tmp/.buildx-cache-dev-single
+
   summary:
-    needs: [extract-version, test-build]
+    needs: [extract-version, test-build-regular, test-build-single]
     runs-on: ubuntu-latest
     if: always()
     steps:
@@ -86,18 +131,27 @@ jobs:
           echo "## Development Build Summary" >> $GITHUB_STEP_SUMMARY
           echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY
           echo "**Platform:** ${{ github.event.inputs.platform || 'linux/amd64' }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Dockerfile:** ${{ github.event.inputs.dockerfile || 'both' }}" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "### Results:" >> $GITHUB_STEP_SUMMARY
 
-          if [[ "${{ needs.test-build.result }}" == "success" ]]; then
-            echo "✅ **Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY
-          elif [[ "${{ needs.test-build.result }}" == "skipped" ]]; then
-            echo "⏭️ **Dockerfile:** Skipped" >> $GITHUB_STEP_SUMMARY
+          if [[ "${{ needs.test-build-regular.result }}" == "success" ]]; then
+            echo "✅ **Regular Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY
+          elif [[ "${{ needs.test-build-regular.result }}" == "skipped" ]]; then
+            echo "⏭️ **Regular Dockerfile:** Skipped" >> $GITHUB_STEP_SUMMARY
           else
-            echo "❌ **Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY
+            echo "❌ **Regular Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY
           fi
-
+          
+          if [[ "${{ needs.test-build-single.result }}" == "success" ]]; then
+            echo "✅ **Single Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY
+          elif [[ "${{ needs.test-build-single.result }}" == "skipped" ]]; then
+            echo "⏭️ **Single Dockerfile:** Skipped" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Single Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY
+          fi
+          
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "### Notes:" >> $GITHUB_STEP_SUMMARY
           echo "- This is a development build (no images pushed to registry)" >> $GITHUB_STEP_SUMMARY
-          echo "- For production releases, use the 'Build and Release' workflow" >> $GITHUB_STEP_SUMMARY
+          echo "- For production releases, use the 'Build and Release' workflow" >> $GITHUB_STEP_SUMMARY

.github/workflows/claude-code-review.yml

@@ -2,7 +2,7 @@ name: Claude Code Review
 
 on:
   pull_request:
-    types: [opened, synchronize, ready_for_review, reopened]
+    types: [opened, synchronize]
     # Optional: Only run on specific file changes
     # paths:
     #   - "src/**/*.ts"
@@ -17,14 +17,14 @@ jobs:
     #   github.event.pull_request.user.login == 'external-contributor' ||
     #   github.event.pull_request.user.login == 'new-developer' ||
     #   github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
-
+    
     runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: read
       issues: read
       id-token: write
-
+    
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -33,12 +33,43 @@ jobs:
 
       - name: Run Claude Code Review
         id: claude-review
-        uses: anthropics/claude-code-action@v1
+        uses: anthropics/claude-code-action@beta
         with:
-          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
-          plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
-          plugins: 'code-review@claude-code-plugins'
-          prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
-          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
-          # or https://code.claude.com/docs/en/cli-reference for available options
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+          
+          # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4)
+          # model: "claude-opus-4-20250514"
+          
+          # Direct prompt for automated review (no @claude mention needed)
+          direct_prompt: |
+            Please review this pull request and provide feedback on:
+            - Code quality and best practices
+            - Potential bugs or issues
+            - Performance considerations
+            - Security concerns
+            - Test coverage
+            
+            Be constructive and helpful in your feedback.
+          
+          # Optional: Customize review based on file types
+          # direct_prompt: |
+          #   Review this PR focusing on:
+          #   - For TypeScript files: Type safety and proper interface usage
+          #   - For API endpoints: Security, input validation, and error handling
+          #   - For React components: Performance, accessibility, and best practices
+          #   - For tests: Coverage, edge cases, and test quality
+          
+          # Optional: Different prompts for different authors
+          # direct_prompt: |
+          #   ${{ github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' && 
+          #   'Welcome! Please review this PR from a first-time contributor. Be encouraging and provide detailed explanations for any suggestions.' ||
+          #   'Please provide a thorough code review focusing on our coding standards and best practices.' }}
+          
+          # Optional: Add specific tools for running tests or linting
+          # allowed_tools: "Bash(npm run test),Bash(npm run lint),Bash(npm run typecheck)"
+          
+          # Optional: Skip review for certain conditions
+          # if: |
+          #   !contains(github.event.pull_request.title, '[skip-review]') &&
+          #   !contains(github.event.pull_request.title, '[WIP]')
 

.github/workflows/claude.yml

@@ -23,7 +23,6 @@ jobs:
       pull-requests: read
       issues: read
       id-token: write
-      actions: read # Required for Claude to read CI results on PRs
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -32,19 +31,29 @@ jobs:
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@v1
+        uses: anthropics/claude-code-action@beta
         with:
-          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
-
-          # This is an optional setting that allows Claude to read CI results on PRs
-          additional_permissions: |
-            actions: read
-
-          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
-          # prompt: 'Update the pull request description to include a summary of changes.'
-
-          # Optional: Add claude_args to customize behavior and configuration
-          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
-          # or https://code.claude.com/docs/en/cli-reference for available options
-          # claude_args: '--allowed-tools Bash(gh pr:*)'
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+          
+          # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4)
+          # model: "claude-opus-4-20250514"
+          
+          # Optional: Customize the trigger phrase (default: @claude)
+          # trigger_phrase: "/claude"
+          
+          # Optional: Trigger when specific user is assigned to an issue
+          # assignee_trigger: "claude-bot"
+          
+          # Optional: Allow Claude to run specific commands
+          # allowed_tools: "Bash(npm install),Bash(npm run build),Bash(npm run test:*),Bash(npm run lint:*)"
+          
+          # Optional: Add custom instructions for Claude to customize its behavior for your project
+          # custom_instructions: |
+          #   Follow our coding standards
+          #   Ensure all new code has tests
+          #   Use TypeScript for new files
+          
+          # Optional: Custom environment variables for Claude
+          # claude_env: |
+          #   NODE_ENV: test
 

.gitignore

@@ -133,8 +133,4 @@ doc_exports/
 specs/
 .claude
 
-.playwright-mcp/
-
-
-
-**/*.local.md
+.playwright-mcp/

.worktreeinclude

@@ -1,5 +0,0 @@
-.env
-.env.local
-.env.*
-**/.claude/settings.local.json
-CLAUDE.local.md

CHANGELOG.md

@@ -1,26 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [1.2.4] - 2025-12-14
-
-### Added
-- Infinite scroll for notebook sources - no more 50 source limit (#325)
-- Markdown table rendering in chat responses, search results, and insights (#325)
-
-### Fixed
-- Timeout errors with Ollama and local LLMs - increased to 10 minutes (#325)
-- "Unable to Connect to API Server" on Docker startup - frontend now waits for API health check (#325, #315)
-- SSL issues with langchain (#274)
-- Query key consistency for source mutations to properly refresh infinite scroll (#325)
-- Docker compose start-all flow (#323)
-
-### Changed
-- Timeout configuration now uses granular httpx.Timeout (short connect, long read) (#325)
-
-### Dependencies
-- Updated next.js to 15.4.10
-- Updated httpx to >=0.27.0 for SSL fix

CLAUDE.md

@@ -1,219 +1,3 @@
-# Open Notebook - Root CLAUDE.md
 
-This file provides architectural guidance for contributors working on Open Notebook at the project level.
+We have a good amount of documentation on this project on the ./docs folder. Please read through them when necessary, and always review the docs/index.md file before starting a new feature so you know at least which docs are available. 
 
-## Project Overview
-
-**Open Notebook** is an open-source, privacy-focused alternative to Google's Notebook LM. It's an AI-powered research assistant enabling users to upload multi-modal content (PDFs, audio, video, web pages), generate intelligent notes, search semantically, chat with AI models, and produce professional podcasts—all with complete control over data and choice of AI providers.
-
-**Key Values**: Privacy-first, multi-provider AI support, fully self-hosted option, open-source transparency.
-
----
-
-## Three-Tier Architecture
-
-```
-┌─────────────────────────────────────────────────────────┐
-│              Frontend (React/Next.js)                    │
-│              frontend/ @ port 3000                       │
-├─────────────────────────────────────────────────────────┤
-│ - Notebooks, sources, notes, chat, podcasts, search UI  │
-│ - Zustand state management, TanStack Query (React Query)│
-│ - Shadcn/ui component library with Tailwind CSS         │
-└────────────────────────┬────────────────────────────────┘
-                         │ HTTP REST
-┌────────────────────────▼────────────────────────────────┐
-│              API (FastAPI)                              │
-│              api/ @ port 5055                           │
-├─────────────────────────────────────────────────────────┤
-│ - REST endpoints for notebooks, sources, notes, chat    │
-│ - LangGraph workflow orchestration                      │
-│ - Job queue for async operations (podcasts)             │
-│ - Multi-provider AI provisioning via Esperanto          │
-└────────────────────────┬────────────────────────────────┘
-                         │ SurrealQL
-┌────────────────────────▼────────────────────────────────┐
-│         Database (SurrealDB)                            │
-│         Graph database @ port 8000                      │
-├─────────────────────────────────────────────────────────┤
-│ - Records: Notebook, Source, Note, ChatSession, etc.    │
-│ - Relationships: source-to-notebook, note-to-source     │
-│ - Vector embeddings for semantic search                 │
-└─────────────────────────────────────────────────────────┘
-```
-
----
-
-## Useful sources
-
-User documentation is at @docs/
-
-## Tech Stack
-
-### Frontend (`frontend/`)
-- **Framework**: Next.js 15 (React 19)
-- **Language**: TypeScript
-- **State Management**: Zustand
-- **Data Fetching**: TanStack Query (React Query)
-- **Styling**: Tailwind CSS + Shadcn/ui
-- **Build Tool**: Webpack (via Next.js)
-
-### API Backend (`api/` + `open_notebook/`)
-- **Framework**: FastAPI 0.104+
-- **Language**: Python 3.11+
-- **Workflows**: LangGraph state machines
-- **Database**: SurrealDB async driver
-- **AI Providers**: Esperanto library (8+ providers: OpenAI, Anthropic, Google, Groq, Ollama, Mistral, DeepSeek, xAI)
-- **Job Queue**: Surreal-Commands for async jobs (podcasts)
-- **Logging**: Loguru
-- **Validation**: Pydantic v2
-- **Testing**: Pytest
-
-### Database
-- **SurrealDB**: Graph database with built-in embedding storage and vector search
-- **Schema Migrations**: Automatic on API startup via AsyncMigrationManager
-
-### Additional Services
-- **Content Processing**: content-core library (file/URL extraction)
-- **Prompts**: AI-Prompter with Jinja2 templating
-- **Podcast Generation**: podcast-creator library
-- **Embeddings**: Multi-provider via Esperanto
-
----
-
-## Architecture Highlights
-
-### 1. Async-First Design
-- All database queries, graph invocations, and API calls are async (await)
-- SurrealDB async driver with connection pooling
-- FastAPI handles concurrent requests efficiently
-
-### 2. LangGraph Workflows
-- **source.py**: Content ingestion (extract → embed → save)
-- **chat.py**: Conversational agent with message history
-- **ask.py**: Search + synthesis (retrieve relevant sources → LLM)
-- **transformation.py**: Custom transformations on sources
-- All use `provision_langchain_model()` for smart model selection
-
-### 3. Multi-Provider AI
-- **Esperanto library**: Unified interface to 8+ AI providers
-- **ModelManager**: Factory pattern with fallback logic
-- **Smart selection**: Detects large contexts, prefers long-context models
-- **Override support**: Per-request model configuration
-
-### 4. Database Schema
-- **Automatic migrations**: AsyncMigrationManager runs on API startup
-- **SurrealDB graph model**: Records with relationships and embeddings
-- **Vector search**: Built-in semantic search across all content
-- **Transactions**: Repo functions handle ACID operations
-
-### 5. Authentication
-- **Current**: Simple password middleware (insecure, dev-only)
-- **Production**: Replace with OAuth/JWT (see CONFIGURATION.md)
-
----
-
-## Important Quirks & Gotchas
-
-### API Startup
-- **Migrations run automatically** on startup; check logs for errors
-- **Must start API before UI**: UI depends on API for all data
-- **SurrealDB must be running**: API fails without database connection
-
-### Frontend-Backend Communication
-- **Base API URL**: Configured in `.env.local` (default: http://localhost:5055)
-- **CORS enabled**: Configured in `api/main.py` (allow all origins in dev)
-- **Rate limiting**: Not built-in; add at proxy layer for production
-
-### LangGraph Workflows
-- **Blocking operations**: Chat/podcast workflows may take minutes; no timeout
-- **State persistence**: Uses SQLite checkpoint storage in `/data/sqlite-db/`
-- **Model fallback**: If primary model fails, falls back to cheaper/smaller model
-
-### Podcast Generation
-- **Async job queue**: `podcast_service.py` submits jobs but doesn't wait
-- **Track status**: Use `/commands/{command_id}` endpoint to poll status
-- **TTS failures**: Fall back to silent audio if speech synthesis fails
-
-### Content Processing
-- **File extraction**: Uses content-core library; supports 50+ file types
-- **URL handling**: Extracts text + metadata from web pages
-- **Large files**: Content processing is sync; may block API briefly
-
----
-
-## Component References
-
-See dedicated CLAUDE.md files for detailed guidance:
-
-- **[frontend/CLAUDE.md](frontend/CLAUDE.md)**: React/Next.js architecture, state management, API integration
-- **[api/CLAUDE.md](api/CLAUDE.md)**: FastAPI structure, service pattern, endpoint development
-- **[open_notebook/CLAUDE.md](open_notebook/CLAUDE.md)**: Backend core, domain models, LangGraph workflows, AI provisioning
-- **[open_notebook/domain/CLAUDE.md](open_notebook/domain/CLAUDE.md)**: Data models, repository pattern, search functions
-- **[open_notebook/ai/CLAUDE.md](open_notebook/ai/CLAUDE.md)**: ModelManager, AI provider integration, Esperanto usage
-- **[open_notebook/graphs/CLAUDE.md](open_notebook/graphs/CLAUDE.md)**: LangGraph workflow design, state machines
-- **[open_notebook/database/CLAUDE.md](open_notebook/database/CLAUDE.md)**: SurrealDB operations, migrations, async patterns
-
----
-
-## Documentation Map
-
-- **[README.md](README.md)**: Project overview, features, quick start
-- **[docs/index.md](docs/index.md)**: Complete user & deployment documentation
-- **[CONFIGURATION.md](CONFIGURATION.md)**: Environment variables, model configuration
-- **[CONTRIBUTING.md](CONTRIBUTING.md)**: Contribution guidelines
-- **[MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md)**: Release & maintenance procedures
-
----
-
-## Testing Strategy
-
-- **Unit tests**: `tests/test_domain.py`, `test_models_api.py`
-- **Graph tests**: `tests/test_graphs.py` (workflow integration)
-- **Utils tests**: `tests/test_utils.py`
-- **Run all**: `uv run pytest tests/`
-- **Coverage**: Check with `pytest --cov`
-
----
-
-## Common Tasks
-
-### Add a New API Endpoint
-1. Create router in `api/routers/feature.py`
-2. Create service in `api/feature_service.py`
-3. Define schemas in `api/models.py`
-4. Register router in `api/main.py`
-5. Test via http://localhost:5055/docs
-
-### Add a New LangGraph Workflow
-1. Create `open_notebook/graphs/workflow_name.py`
-2. Define StateDict and node functions
-3. Build graph with `.add_node()` / `.add_edge()`
-4. Invoke in service: `graph.ainvoke({"input": ...}, config={"..."})`
-5. Test with sample data in `tests/`
-
-### Add Database Migration
-1. Create `migrations/XXX_description.surql`
-2. Write SurrealQL schema changes
-3. Create `migrations/XXX_description_down.surql` (optional rollback)
-4. API auto-detects on startup; migration runs if newer than recorded version
-
-### Deploy to Production
-1. Review [CONFIGURATION.md](CONFIGURATION.md) for security settings
-2. Use `make docker-release` for multi-platform image
-3. Push to Docker Hub / GitHub Container Registry
-4. Deploy `docker compose --profile multi up`
-5. Verify migrations via API logs
-
----
-
-## Support & Community
-
-- **Documentation**: https://open-notebook.ai
-- **Discord**: https://discord.gg/37XJPXfz2w
-- **Issues**: https://github.com/lfnovo/open-notebook/issues
-- **License**: MIT (see LICENSE)
-
----
-
-**Last Updated**: January 2026 | **Project Version**: 1.2.4+

CONFIGURATION.md

@@ -1,36 +1,108 @@
 # Configuration Guide
 
-**📍 This file has moved!**
+## API Connection Configuration
 
-All configuration documentation has been consolidated into the new documentation structure.
+Starting from version 1.0.0-alpha, Open Notebook uses a simplified API connection system that automatically configures itself based on your deployment environment.
 
-👉 **[Read the Configuration Guide](docs/5-CONFIGURATION/index.md)**
+### How It Works
 
----
+The frontend automatically discovers the API location at runtime by analyzing the incoming HTTP request. This eliminates the need for complex network configurations and works for both Docker deployment modes:
+- Multi-container (docker-compose with separate SurrealDB)
+- Single-container (all services in one container)
 
-## Quick Links
+**Auto-detection logic:**
+1. If `API_URL` environment variable is set → use it (explicit override)
+2. Otherwise, detect from the HTTP request:
+   - Uses the same hostname you're accessing the frontend from
+   - Automatically changes port to 5055 (API port)
+   - Respects `X-Forwarded-Proto` header for reverse proxy setups
+3. Falls back to `http://localhost:5055` if detection fails
 
-- **AI Provider Setup** → [AI Providers](docs/5-CONFIGURATION/ai-providers.md)
-- **Environment Variables Reference** → [Environment Reference](docs/5-CONFIGURATION/environment-reference.md)
-- **Database Configuration** → [Database Setup](docs/5-CONFIGURATION/database.md)
-- **Server Configuration** → [Server Settings](docs/5-CONFIGURATION/server.md)
-- **Security Setup** → [Security Configuration](docs/5-CONFIGURATION/security.md)
-- **Reverse Proxy** → [Reverse Proxy Setup](docs/5-CONFIGURATION/reverse-proxy.md)
-- **Advanced Tuning** → [Advanced Configuration](docs/5-CONFIGURATION/advanced.md)
+**Examples:**
+- Access frontend at `http://localhost:8502` → API at `http://localhost:5055` ✅
+- Access frontend at `http://10.20.30.20:8502` → API at `http://10.20.30.20:5055` ✅
+- Access frontend at `http://my-server:8502` → API at `http://my-server:5055` ✅
 
----
+**No configuration needed** for most deployments!
 
-## What You'll Find
+### Custom Configuration
 
-The new configuration documentation includes:
+If you need to change the API URL (e.g., running on a different host, port, or domain), you can configure it using the `API_URL` environment variable.
 
-- **Complete environment variable reference** with examples
-- **Provider-specific setup guides** for OpenAI, Anthropic, Google, Groq, Ollama, and more
-- **Production deployment configurations** with security best practices
-- **Reverse proxy examples** for Nginx, Caddy, Traefik
-- **Database tuning** for performance optimization
-- **Troubleshooting guides** for common configuration issues
+#### Option 1: Using docker-compose (Recommended)
 
----
+Edit your `docker.env` file:
 
-For all configuration details, see **[docs/5-CONFIGURATION/](docs/5-CONFIGURATION/index.md)**.
+```env
+API_URL=http://your-server-ip:5055
+```
+
+Or add it to your `docker-compose.yml`:
+
+```yaml
+services:
+  open_notebook:
+    image: lfnovo/open_notebook:v1-latest
+    ports:
+      - "8502:8502"
+      - "5055:5055"  # API port must be exposed
+    environment:
+      - API_URL=http://your-server-ip:5055
+```
+
+#### Option 2: Using docker run
+
+```bash
+docker run -e API_URL=http://your-server-ip:5055 \
+  -p 8502:8502 \
+  -p 5055:5055 \
+  lfnovo/open_notebook:v1-latest-single
+```
+
+### Important Notes
+
+1. **Port 5055 must be exposed**: The browser needs direct access to the API, so port 5055 must be mapped in your Docker configuration.
+
+2. **Use the externally accessible URL**: The `API_URL` should be the URL that a browser can reach, not internal Docker networking addresses.
+
+3. **Protocol matters**: Use `http://` for local deployments, `https://` if you've set up SSL.
+
+### Examples
+
+#### Running on a different host
+```env
+API_URL=http://192.168.1.100:5055
+```
+
+#### Running on a custom domain with SSL
+```env
+API_URL=https://notebook.example.com/api
+```
+
+#### Running on a custom port
+```env
+API_URL=http://localhost:3055
+```
+(Remember to update the port mapping in docker-compose accordingly)
+
+### Troubleshooting
+
+**"Unable to connect to server" error on login:**
+1. Verify port 5055 is exposed in your Docker configuration
+2. Check that `API_URL` matches the URL your browser can access
+3. Try accessing `http://localhost:5055/health` directly in your browser
+4. If that fails, the API isn't running or port isn't exposed
+
+**API works but frontend doesn't connect:**
+1. Check browser console for CORS errors
+2. Verify `API_URL` is set correctly
+3. Make sure you're using the same protocol (http/https) throughout
+
+### Migration from Previous Versions
+
+If you were previously exposing port 5055 manually or had custom configurations, you may need to:
+1. Update your `docker.env` or environment variables to include `API_URL`
+2. Ensure port 5055 is exposed in your docker-compose.yml (it's now required)
+3. Remove any custom Next.js configuration or environment variables you may have added
+
+The default configuration will work for most users without any changes.

CONTRIBUTING.md

@@ -1,29 +1,52 @@
 # Contributing to Open Notebook
 
-**📍 This file has moved!**
+First off, thank you for considering contributing to Open Notebook! What makes open source great is the fact that we can work together and accomplish things we would never do on our own. All suggestions are welcome. 
 
-All contribution guidelines have been consolidated into the new development documentation structure.
+## Code of Conduct
 
-👉 **[Read the Contributing Guide](docs/7-DEVELOPMENT/contributing.md)**
+By participating in this project, you are expected to uphold our Code of Conduct (to be created separately).
 
----
+## How Can I Contribute?
 
-## Quick Links
+### Reporting Bugs
 
-- **Want to contribute code?** → [Contributing Guide](docs/7-DEVELOPMENT/contributing.md)
-- **Want to understand the architecture?** → [Architecture Overview](docs/7-DEVELOPMENT/architecture.md)
-- **Want to understand our design philosophy?** → [Design Principles](docs/7-DEVELOPMENT/design-principles.md)
-- **Are you a maintainer?** → [Maintainer Guide](docs/7-DEVELOPMENT/maintainer-guide.md)
-- **New developer?** → [Quick Start](docs/7-DEVELOPMENT/quick-start.md)
+- Ensure the bug was not already reported by searching on GitHub under [Issues](https://github.com/yourusername/open-notebook/issues).
+- If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/yourusername/open-notebook/issues/new). Be sure to include a title and clear description, as much relevant information as possible, and a code sample or an executable test case demonstrating the expected behavior that is not occurring.
 
----
+### Suggesting Enhancements
 
-## The Issue-First Workflow
+- Open a new issue with a clear title and detailed description of the suggested enhancement.
+- Provide any relevant examples or mockups if applicable.
 
-**TL;DR**: Create an issue first, get it assigned, THEN code.
+### Pull Requests
 
-This prevents wasted effort and ensures your work aligns with the project. [See details →](docs/7-DEVELOPMENT/contributing.md)
+1. Fork the repo and create your branch from `main`.
+2. If you've added code that should be tested, add tests.
+3. Ensure the test suite passes.
+4. Make sure your code lints.
+5. Issue that pull request!
 
----
+## Styleguides
 
-For all contribution details, see **[docs/7-DEVELOPMENT/contributing.md](docs/7-DEVELOPMENT/contributing.md)**.
+### Git Commit Messages
+
+- Use the present tense ("Add feature" not "Added feature")
+- Use the imperative mood ("Move cursor to..." not "Moves cursor to...")
+- Limit the first line to 72 characters or less
+- Reference issues and pull requests liberally after the first line
+
+### Python Styleguide
+
+- Follow PEP 8 guidelines
+- Use type hints where possible
+- Write docstrings for all functions, classes, and modules
+
+### Documentation Styleguide
+
+- Use Markdown for documentation files
+- Reference functions and classes appropriately
+
+## Additional Notes
+
+
+Thank you for contributing to Open Notebook!

Dockerfile

@@ -76,10 +76,6 @@ EXPOSE 8502 5055
 
 RUN mkdir -p /app/data
 
-# Copy and make executable the wait-for-api script
-COPY scripts/wait-for-api.sh /app/scripts/wait-for-api.sh
-RUN chmod +x /app/scripts/wait-for-api.sh
-
 # Copy supervisord configuration
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 

Dockerfile.single

@@ -77,10 +77,6 @@ COPY --from=builder /app/frontend/public /app/frontend/public
 # Create directories for data persistence
 RUN mkdir -p /app/data /mydata
 
-# Copy and make executable the wait-for-api script
-COPY scripts/wait-for-api.sh /app/scripts/wait-for-api.sh
-RUN chmod +x /app/scripts/wait-for-api.sh
-
 # Expose ports for Frontend and API
 EXPOSE 8502 5055
 

MAINTAINER_GUIDE.md

@@ -1,19 +0,0 @@
-# Maintainer Guide
-
-**📍 This file has moved!**
-
-All maintainer guidelines have been consolidated into the new development documentation structure.
-
-👉 **[Read the Maintainer Guide](docs/7-DEVELOPMENT/maintainer-guide.md)**
-
----
-
-## Quick Links
-
-- **Maintainer Guide** → [docs/7-DEVELOPMENT/maintainer-guide.md](docs/7-DEVELOPMENT/maintainer-guide.md)
-- **Contributing Guide** → [docs/7-DEVELOPMENT/contributing.md](docs/7-DEVELOPMENT/contributing.md)
-- **Design Principles** → [docs/7-DEVELOPMENT/design-principles.md](docs/7-DEVELOPMENT/design-principles.md)
-
----
-
-For all maintainer details, see **[docs/7-DEVELOPMENT/maintainer-guide.md](docs/7-DEVELOPMENT/maintainer-guide.md)**.

MIGRATION.md

@@ -0,0 +1,397 @@
+# Migration Guide: Streamlit to React/Next.js Frontend
+
+**Version**: 1.0.0
+**Last Updated**: October 2025
+
+This guide helps existing Open Notebook users migrate from the legacy Streamlit frontend to the new React/Next.js frontend.
+
+---
+
+## ⚠️ Breaking Changes in v1.0
+
+Open Notebook v1.0 introduces breaking changes that require manual migration. Please read this section carefully before upgrading.
+
+### Docker Tag Changes
+
+**The "latest" tag is now frozen** at the last Streamlit version. Starting with v1.0, we use versioned tags to prevent unexpected breaking changes:
+
+- **`latest`** and **`latest-single`** → FROZEN at Streamlit version (will not update)
+- **`v1-latest`** and **`v1-latest-single`** → NEW tags for v1.x releases (recommended)
+- **`X.Y.Z`** and **`X.Y.Z-single`** → Specific version tags (unchanged)
+
+**Why this change?**
+The v1.0 release brings significant architectural changes (Streamlit → React/Next.js frontend). Freezing the "latest" tag prevents existing deployments from breaking unexpectedly, while the new "v1-latest" tag allows users to explicitly opt into the v1 architecture.
+
+### Quick Migration for Docker Users
+
+If you're currently using `latest` or `latest-single`, you need to:
+
+1. **Update your docker-compose.yml or docker run command**:
+   ```yaml
+   # Before:
+   image: lfnovo/open_notebook:latest-single
+
+   # After (recommended):
+   image: lfnovo/open_notebook:v1-latest-single
+   ```
+
+2. **Expose port 5055** for the API (required in v1):
+   ```yaml
+   ports:
+     - "8502:8502"  # Frontend
+     - "5055:5055"  # API (NEW - required)
+   ```
+
+3. **Verify API connectivity** after upgrade:
+   ```bash
+   curl http://localhost:5055/api/config
+   ```
+
+### API Connectivity (Port 5055)
+
+**Important:** v1.0 requires port 5055 to be exposed to your host machine so the frontend can communicate with the API.
+
+**Auto-Detection:** The Next.js frontend automatically detects the API URL:
+- If you access the frontend at `http://localhost:8502`, it uses `http://localhost:5055`
+- If you access the frontend at `http://192.168.1.100:8502`, it uses `http://192.168.1.100:5055`
+- If you access the frontend at `http://my-server:8502`, it uses `http://my-server:5055`
+
+**Manual Override:** If auto-detection doesn't work (e.g., reverse proxy, complex networking), set the `API_URL` environment variable:
+
+```bash
+# Docker run example
+docker run -d \
+  --name open-notebook \
+  -p 8502:8502 -p 5055:5055 \
+  -e API_URL=http://my-custom-api:5055 \
+  -v ./notebook_data:/app/data \
+  -v ./surreal_data:/mydata \
+  lfnovo/open_notebook:v1-latest-single
+```
+
+```yaml
+# docker-compose.yml example
+services:
+  open_notebook:
+    image: lfnovo/open_notebook:v1-latest-single
+    ports:
+      - "8502:8502"
+      - "5055:5055"
+    environment:
+      - API_URL=http://my-custom-api:5055
+    volumes:
+      - ./notebook_data:/app/data
+      - ./surreal_data:/mydata
+```
+
+### Health Check
+
+Verify your API is accessible with:
+
+```bash
+# Local deployment
+curl http://localhost:5055/api/config
+
+# Remote deployment
+curl http://your-server-ip:5055/api/config
+```
+
+Expected response:
+```json
+{
+  "version": "1.0.0",
+  "latestVersion": "1.0.0",
+  "hasUpdate": false,
+  "dbStatus": "online"
+}
+```
+
+Note: The API URL is now auto-detected by the frontend from the hostname you're accessing, so `/api/config` no longer returns `apiUrl`.
+
+### Troubleshooting
+
+**Problem:** Frontend shows "Cannot connect to API" error
+- **Check:** Is port 5055 exposed? Run `docker ps` and verify port mapping
+- **Check:** Can you reach the API? Run `curl http://localhost:5055/api/config`
+- **Solution:** If using custom networking, set `API_URL` environment variable
+
+**Problem:** Auto-detection uses wrong hostname
+- **Example:** Frontend at `http://internal-hostname:8502` but API should use `http://public-hostname:5055`
+- **Solution:** Set `API_URL=http://public-hostname:5055` environment variable
+
+**Problem:** Still running the old Streamlit version after `docker pull`
+- **Check:** Are you using the "latest" tag? It's frozen at Streamlit version
+- **Solution:** Update to `v1-latest` or `v1-latest-single` tag
+
+---
+
+## What Changed
+
+Open Notebook has migrated from a Streamlit-based frontend to a modern React/Next.js application. This brings significant improvements in performance, user experience, and maintainability.
+
+### Key Changes
+
+| Aspect | Before (Streamlit) | After (React/Next.js) |
+|--------|-------------------|----------------------|
+| **Frontend Framework** | Streamlit | Next.js 15 + React 18 |
+| **UI Components** | Streamlit widgets | shadcn/ui + Radix UI |
+| **Frontend Port** | 8502 | 8502 (unchanged) |
+| **API Port** | 5055 | 5055 (unchanged) |
+| **Navigation** | Sidebar with emoji icons | Clean sidebar navigation |
+| **Performance** | Server-side rendering | Client-side React with API calls |
+| **Customization** | Limited | Highly customizable |
+
+### What Stayed the Same
+
+- **Core functionality**: All features remain available
+- **API backend**: FastAPI backend unchanged
+- **Database**: SurrealDB unchanged
+- **Data format**: No data migration needed
+- **Configuration**: Same environment variables
+- **Docker deployment**: Same ports and setup
+
+## Migration Paths
+
+### Path 1: Docker Users (Recommended)
+
+If you're running Open Notebook via Docker, migration is automatic:
+
+1. **Stop the current version**:
+   ```bash
+   docker-compose down
+   ```
+
+2. **Update to the latest image**:
+   ```bash
+   # Update docker-compose.yml to use v1-latest
+   # Change from:
+   image: lfnovo/open_notebook:latest-single
+   # To:
+   image: lfnovo/open_notebook:v1-latest-single
+   ```
+
+3. **Start the new version**:
+   ```bash
+   docker-compose pull
+   docker-compose up -d
+   ```
+
+4. **Access the new frontend**:
+   - Frontend: http://localhost:8502 (new React UI)
+   - API Docs: http://localhost:5055/docs
+
+**Your data is automatically preserved!** All notebooks, sources, and notes carry over seamlessly.
+
+### Path 2: Source Code Users
+
+If you're running from source code:
+
+1. **Pull the latest code**:
+   ```bash
+   git pull origin main
+   ```
+
+2. **Install frontend dependencies**:
+   ```bash
+   cd frontend
+   npm install
+   cd ..
+   ```
+
+3. **Update Python dependencies**:
+   ```bash
+   uv sync
+   ```
+
+4. **Start services** (3 terminals):
+   ```bash
+   # Terminal 1: Database
+   make database
+
+   # Terminal 2: API
+   uv run python api/main.py
+
+   # Terminal 3: Frontend (NEW)
+   cd frontend && npm run dev
+   ```
+
+5. **Access the application**:
+   - Frontend: http://localhost:8502
+   - API: http://localhost:5055
+
+## Breaking Changes
+
+### Removed Features
+
+The following Streamlit-specific features are no longer available:
+
+- **Streamlit cache**: Replaced with React Query caching
+- **Streamlit session state**: Replaced with React state management
+- **Direct file access via Streamlit**: Use API endpoints instead
+
+### Changed Navigation
+
+Navigation paths have been simplified:
+
+| Old Path | New Path |
+|----------|----------|
+| Settings → Models | Models |
+| Settings → Advanced | Advanced |
+| Other paths | (Same but cleaner navigation) |
+
+### API Changes
+
+**No breaking API changes!** The REST API remains fully backward compatible.
+
+## New Features in React Version
+
+The React frontend brings several improvements:
+
+### Performance
+- **Faster page loads**: Client-side rendering with React
+- **Better caching**: React Query for intelligent data caching
+- **Optimized builds**: Next.js automatic code splitting
+
+### User Experience
+- **Modern UI**: Clean, professional interface with shadcn/ui
+- **Responsive design**: Better mobile and tablet support
+- **Keyboard shortcuts**: Improved keyboard navigation
+- **Real-time updates**: Better WebSocket support
+
+### Developer Experience
+- **TypeScript**: Full type safety
+- **Component library**: Reusable UI components
+- **Hot reload**: Instant updates during development
+- **Testing**: Better test infrastructure
+
+## Troubleshooting
+
+### Issue: Can't access the frontend
+
+**Solution**:
+```bash
+# Check if services are running
+docker-compose ps
+
+# Check logs
+docker-compose logs open_notebook
+
+# Restart services
+docker-compose restart
+```
+
+### Issue: API errors in new frontend
+
+**Solution**:
+The new frontend requires the API to be running. Ensure:
+```bash
+# API should be accessible at
+curl http://localhost:5055/health
+
+# If not, check API logs
+docker-compose logs open_notebook | grep api
+```
+
+### Issue: Missing data after migration
+
+**Solution**:
+Data is preserved automatically. If you don't see your data:
+
+1. Check database volume is mounted correctly:
+   ```bash
+   docker-compose down
+   # Verify volumes in docker-compose.yml:
+   # - ./surreal_data:/mydata  (for multi-container)
+   # - ./surreal_single_data:/mydata (for single-container)
+   docker-compose up -d
+   ```
+
+2. Check SurrealDB is running:
+   ```bash
+   docker-compose logs surrealdb
+   ```
+
+### Issue: Port conflicts
+
+**Solution**:
+If ports 8502 or 5055 are already in use:
+
+```bash
+# Find what's using the port
+lsof -i :8502
+lsof -i :5055
+
+# Stop conflicting service or change Open Notebook ports
+# Edit docker-compose.yml:
+ports:
+  - "8503:8502"  # Change external port
+  - "5056:5055"  # Change external port
+```
+
+## Rollback Instructions
+
+If you need to roll back to the Streamlit version:
+
+### Docker Users
+
+```bash
+# Stop current version
+docker-compose down
+
+# Edit docker-compose.yml to use old image
+# Change to: lfnovo/open_notebook:streamlit-latest
+
+# Start old version
+docker-compose up -d
+```
+
+### Source Code Users
+
+```bash
+# Checkout the last Streamlit version tag
+git checkout tags/streamlit-final
+
+# Install dependencies
+uv sync
+
+# Start Streamlit
+uv run streamlit run app_home.py
+```
+
+## Getting Help
+
+If you encounter issues during migration:
+
+- **Discord**: Join our [Discord community](https://discord.gg/37XJPXfz2w) for real-time help
+- **GitHub Issues**: Report bugs at [github.com/lfnovo/open-notebook/issues](https://github.com/lfnovo/open-notebook/issues)
+- **Documentation**: Check [full documentation](https://github.com/lfnovo/open-notebook/tree/main/docs)
+
+## FAQs
+
+### Will my notebooks and data be lost?
+No! All data is preserved. The database and API backend are unchanged.
+
+### Do I need to update my API integrations?
+No! The REST API remains fully backward compatible.
+
+### Can I use both frontends simultaneously?
+Technically yes, but not recommended. Choose one for consistency.
+
+### What about my custom Streamlit pages?
+Custom Streamlit pages won't work with the React frontend. Consider:
+- Using the REST API to build custom integrations
+- Contributing React components to the project
+- Requesting features in GitHub issues
+
+### Is the Streamlit version still supported?
+The Streamlit version is no longer actively developed. We recommend migrating to the React version for the best experience and latest features.
+
+## Timeline
+
+- **Legacy (Pre-v1.0)**: Streamlit frontend
+- **Current (v1.0+)**: React/Next.js frontend
+- **Future**: Continued React development with new features
+
+---
+
+**Ready to migrate?** Follow the migration path for your deployment method above. The process is straightforward and your data is safe!

Makefile

@@ -1,6 +1,6 @@
 .PHONY: run frontend check ruff database lint api start-all stop-all status clean-cache worker worker-start worker-stop worker-restart
 .PHONY: docker-buildx-prepare docker-buildx-clean docker-buildx-reset
-.PHONY: docker-push docker-push-latest docker-release docker-build-local tag export-docs
+.PHONY: docker-push docker-push-latest docker-release tag export-docs
 
 # Get version from pyproject.toml
 VERSION := $(shell grep -m1 version pyproject.toml | cut -d'"' -f2)
@@ -45,16 +45,6 @@ docker-buildx-reset: docker-buildx-clean docker-buildx-prepare
 
 # === Docker Build Targets ===
 
-# Build production image for local platform only (no push)
-docker-build-local:
-	@echo "🔨 Building production image locally ($(shell uname -m))..."
-	docker build \
-		-t $(DOCKERHUB_IMAGE):$(VERSION) \
-		-t $(DOCKERHUB_IMAGE):local \
-		.
-	@echo "✅ Built $(DOCKERHUB_IMAGE):$(VERSION) and $(DOCKERHUB_IMAGE):local"
-	@echo "Run with: docker run -p 5055:5055 -p 3000:3000 $(DOCKERHUB_IMAGE):local"
-
 # Build and push version tags ONLY (no latest) for both regular and single images
 docker-push: docker-buildx-prepare
 	@echo "📤 Building and pushing version $(VERSION) to both registries..."
@@ -157,7 +147,7 @@ worker-restart: worker-stop
 start-all:
 	@echo "🚀 Starting Open Notebook (Database + API + Worker + Frontend)..."
 	@echo "📊 Starting SurrealDB..."
-	@docker compose -f docker-compose.dev.yml up -d surrealdb
+	@docker compose up -d surrealdb
 	@sleep 3
 	@echo "🔧 Starting API backend..."
 	@uv run run_api.py &

README.dev.md

@@ -1,408 +0,0 @@
-# Developer Guide
-
-This guide is for developers working on Open Notebook. For end-user documentation, see [README.md](README.md) and [docs/](docs/).
-
-## Quick Start for Development
-
-```bash
-# 1. Clone and setup
-git clone https://github.com/lfnovo/open-notebook.git
-cd open-notebook
-
-# 2. Copy environment files
-cp .env.example .env
-cp .env.example docker.env
-
-# 3. Install dependencies
-uv sync
-
-# 4. Start all services (recommended for development)
-make start-all
-```
-
-## Development Workflows
-
-### When to Use What?
-
-| Workflow | Use Case | Speed | Production Parity |
-|----------|----------|-------|-------------------|
-| **Local Services** (`make start-all`) | Day-to-day development, fastest iteration | ⚡⚡⚡ Fast | Medium |
-| **Docker Compose** (`make dev`) | Testing containerized setup | ⚡⚡ Medium | High |
-| **Local Docker Build** (`make docker-build-local`) | Testing Dockerfile changes | ⚡ Slow | Very High |
-| **Multi-platform Build** (`make docker-push`) | Publishing releases | 🐌 Very Slow | Exact |
-
----
-
-## 1. Local Development (Recommended)
-
-**Best for:** Daily development, hot reload, debugging
-
-### Setup
-
-```bash
-# Start database
-make database
-
-# Start all services (DB + API + Worker + Frontend)
-make start-all
-```
-
-### What This Does
-
-1. Starts SurrealDB in Docker (port 8000)
-2. Starts FastAPI backend (port 5055)
-3. Starts background worker (surreal-commands)
-4. Starts Next.js frontend (port 3000)
-
-### Individual Services
-
-```bash
-# Just the database
-make database
-
-# Just the API
-make api
-
-# Just the frontend
-make frontend
-
-# Just the worker
-make worker
-```
-
-### Checking Status
-
-```bash
-# See what's running
-make status
-
-# Stop everything
-make stop-all
-```
-
-### Advantages
-- ✅ Fastest iteration (hot reload)
-- ✅ Easy debugging (direct process access)
-- ✅ Low resource usage
-- ✅ Direct log access
-
-### Disadvantages
-- ❌ Doesn't test Docker build
-- ❌ Environment may differ from production
-- ❌ Requires local Python/Node setup
-
----
-
-## 2. Docker Compose Development
-
-**Best for:** Testing containerized setup, CI/CD verification
-
-```bash
-# Start with dev profile
-make dev
-
-# Or full stack
-make full
-```
-
-### Configuration Files
-
-- `docker-compose.dev.yml` - Development setup
-- `docker-compose.full.yml` - Full stack setup
-- `docker-compose.yml` - Base configuration
-
-### Advantages
-- ✅ Closer to production environment
-- ✅ Isolated dependencies
-- ✅ Easy to share exact environment
-
-### Disadvantages
-- ❌ Slower rebuilds
-- ❌ More complex debugging
-- ❌ Higher resource usage
-
----
-
-## 3. Testing Production Docker Images
-
-**Best for:** Verifying Dockerfile changes before publishing
-
-### Build Locally
-
-```bash
-# Build production image for your platform only
-make docker-build-local
-```
-
-This creates two tags:
-- `lfnovo/open_notebook:<version>` (from pyproject.toml)
-- `lfnovo/open_notebook:local`
-
-### Run Locally
-
-```bash
-docker run -p 5055:5055 -p 3000:3000 lfnovo/open_notebook:local
-```
-
-### When to Use
-- ✅ Before pushing to registry
-- ✅ Testing Dockerfile changes
-- ✅ Debugging production-specific issues
-- ✅ Verifying build process
-
----
-
-## 4. Publishing Docker Images
-
-### Workflow
-
-```bash
-# 1. Test locally first
-make docker-build-local
-
-# 2. If successful, push version tag (no latest update)
-make docker-push
-
-# 3. Test the pushed version in staging/production
-
-# 4. When ready, promote to latest
-make docker-push-latest
-```
-
-### Available Commands
-
-| Command | What It Does | Updates Latest? |
-|---------|--------------|-----------------|
-| `make docker-build-local` | Build for current platform only | No registry push |
-| `make docker-push` | Push version tags to registries | ❌ No |
-| `make docker-push-latest` | Push version + update v1-latest | ✅ Yes |
-| `make docker-release` | Full release (same as docker-push-latest) | ✅ Yes |
-
-### Publishing Details
-
-- **Platforms:** `linux/amd64`, `linux/arm64`
-- **Registries:** Docker Hub + GitHub Container Registry
-- **Image Variants:** Regular + Single-container (`-single`)
-- **Version Source:** `pyproject.toml`
-
-### Creating Git Tags
-
-```bash
-# Create and push git tag matching pyproject.toml version
-make tag
-```
-
----
-
-## Code Quality
-
-```bash
-# Run linter with auto-fix
-make ruff
-
-# Run type checking
-make lint
-
-# Run tests
-uv run pytest tests/
-
-# Clean cache directories
-make clean-cache
-```
-
----
-
-## Common Development Tasks
-
-### Adding a New Feature
-
-1. Create feature branch
-2. Develop using `make start-all`
-3. Write tests
-4. Run `make ruff` and `make lint`
-5. Test with `make docker-build-local`
-6. Create PR
-
-### Fixing a Bug
-
-1. Reproduce locally with `make start-all`
-2. Add test case demonstrating bug
-3. Fix the bug
-4. Verify test passes
-5. Check with `make docker-build-local`
-
-### Updating Dependencies
-
-```bash
-# Add Python dependency
-uv add package-name
-
-# Update dependencies
-uv sync
-
-# Frontend dependencies
-cd frontend && npm install package-name
-```
-
-### Database Migrations
-
-Database migrations run **automatically** when the API starts.
-
-1. Create migration file: `migrations/XXX_description.surql`
-2. Write SurrealQL schema changes
-3. (Optional) Create rollback: `migrations/XXX_description_down.surql`
-4. Restart API - migration runs on startup
-
----
-
-## Troubleshooting
-
-### Services Won't Start
-
-```bash
-# Check status
-make status
-
-# Check database
-docker compose ps surrealdb
-
-# View logs
-docker compose logs surrealdb
-
-# Restart everything
-make stop-all
-make start-all
-```
-
-### Port Already in Use
-
-```bash
-# Find process using port
-lsof -i :5055
-lsof -i :3000
-lsof -i :8000
-
-# Kill stuck processes
-make stop-all
-```
-
-### Database Connection Issues
-
-```bash
-# Verify SurrealDB is running
-docker compose ps surrealdb
-
-# Check connection settings in .env
-cat .env | grep SURREAL
-```
-
-### Docker Build Fails
-
-```bash
-# Clean Docker cache
-docker builder prune
-
-# Reset buildx
-make docker-buildx-reset
-
-# Try local build first
-make docker-build-local
-```
-
----
-
-## Project Structure
-
-```
-open-notebook/
-├── api/                    # FastAPI backend
-├── frontend/               # Next.js React frontend
-├── open_notebook/          # Python core library
-│   ├── domain/            # Domain models
-│   ├── graphs/            # LangGraph workflows
-│   ├── ai/                # AI provider integration
-│   └── database/          # SurrealDB operations
-├── migrations/             # Database migrations
-├── tests/                  # Test suite
-├── docs/                   # User documentation
-└── Makefile               # Development commands
-```
-
-See component-specific CLAUDE.md files for detailed architecture:
-- [frontend/CLAUDE.md](frontend/CLAUDE.md)
-- [api/CLAUDE.md](api/CLAUDE.md)
-- [open_notebook/CLAUDE.md](open_notebook/CLAUDE.md)
-
----
-
-## Environment Variables
-
-### Required for Local Development
-
-```bash
-# .env file
-SURREAL_URL=ws://localhost:8000
-SURREAL_USER=root
-SURREAL_PASS=root
-SURREAL_DB=open_notebook
-SURREAL_NS=production
-
-# AI Provider (at least one required)
-OPENAI_API_KEY=sk-...
-# OR
-ANTHROPIC_API_KEY=sk-ant-...
-# OR configure other providers (see docs/5-CONFIGURATION/)
-```
-
-See [docs/5-CONFIGURATION/](docs/5-CONFIGURATION/) for complete configuration guide.
-
----
-
-## Performance Tips
-
-### Speed Up Local Development
-
-1. **Use `make start-all`** instead of Docker for daily work
-2. **Keep SurrealDB running** between sessions (`make database`)
-3. **Use `make docker-build-local`** only when testing Dockerfile changes
-4. **Skip multi-platform builds** until ready to publish
-
-### Reduce Resource Usage
-
-```bash
-# Stop unused services
-make stop-all
-
-# Clean up Docker
-docker system prune -a
-
-# Clean Python cache
-make clean-cache
-```
-
----
-
-## TODO: Sections to Add
-
-- [ ] Frontend development guide (hot reload, component structure)
-- [ ] API development guide (adding endpoints, services)
-- [ ] LangGraph workflow development
-- [ ] Testing strategy and coverage
-- [ ] Debugging tips (VSCode/PyCharm setup)
-- [ ] CI/CD pipeline overview
-- [ ] Release process checklist
-- [ ] Common error messages and solutions
-
----
-
-## Resources
-
-- **Documentation:** https://open-notebook.ai
-- **Discord:** https://discord.gg/37XJPXfz2w
-- **Issues:** https://github.com/lfnovo/open-notebook/issues
-- **Contributing:** [CONTRIBUTING.md](CONTRIBUTING.md)
-- **Maintainer Guide:** [MAINTAINER_GUIDE.md](MAINTAINER_GUIDE.md)
-
----
-
-**Last Updated:** January 2025

README.md

@@ -24,20 +24,16 @@
     <a href="https://www.open-notebook.ai"><strong>Checkout our website »</strong></a>
     <br />
     <br />
-    <a href="docs/0-START-HERE/index.md">📚 Get Started</a>
+    <a href="docs/getting-started/index.md">📚 Get Started</a>
     ·
-    <a href="docs/3-USER-GUIDE/index.md">📖 User Guide</a>
+    <a href="docs/user-guide/index.md">📖 User Guide</a>
     ·
-    <a href="docs/2-CORE-CONCEPTS/index.md">✨ Features</a>
+    <a href="docs/features/index.md">✨ Features</a>
     ·
-    <a href="docs/1-INSTALLATION/index.md">🚀 Deploy</a>
+    <a href="docs/deployment/index.md">🚀 Deploy</a>
   </p>
 </div>
 
-<p align="center">
-<a href="https://trendshift.io/repositories/14536" target="_blank"><img src="https://trendshift.io/api/badge/repositories/14536" alt="lfnovo%2Fopen-notebook | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
-</p>
-
 <div align="center">
   <!-- Keep these links. Translations will automatically update with the README. -->
   <a href="https://zdoc.app/de/lfnovo/open-notebook">Deutsch</a> | 
@@ -68,6 +64,19 @@ Learn more about our project at [https://www.open-notebook.ai](https://www.open-
 
 ---
 
+## ⚠️ IMPORTANT: v1.0 Breaking Changes
+
+**If you're upgrading from a previous version**, please note:
+
+- 🏷️ **Docker tags have changed**: The `latest` tag is now **frozen** at the last Streamlit version
+- 🆕 **Use `v1-latest` tag** for the new React/Next.js version (recommended)
+- 🔌 **Port 5055 required**: You must expose port 5055 for the API to work
+- 📖 **Read the migration guide**: See [MIGRATION.md](MIGRATION.md) for detailed upgrade instructions
+
+**New users**: You can ignore this notice and proceed with the Quick Start below using the `v1-latest-single` tag.
+
+---
+
 ## 🆚 Open Notebook vs Google Notebook LM
 
 | Feature | Open Notebook | Google Notebook LM | Advantage |
@@ -75,12 +84,13 @@ Learn more about our project at [https://www.open-notebook.ai](https://www.open-
 | **Privacy & Control** | Self-hosted, your data | Google cloud only | Complete data sovereignty |
 | **AI Provider Choice** | 16+ providers (OpenAI, Anthropic, Ollama, LM Studio, etc.) | Google models only | Flexibility and cost optimization |
 | **Podcast Speakers** | 1-4 speakers with custom profiles | 2 speakers only | Extreme flexibility |
+| **Context Control** | 3 granular levels | All-or-nothing | Privacy and performance tuning |
 | **Content Transformations** | Custom and built-in | Limited options | Unlimited processing power |
 | **API Access** | Full REST API | No API | Complete automation |
 | **Deployment** | Docker, cloud, or local | Google hosted only | Deploy anywhere |
-| **Citations** | Basic references (will improve) | Comprehensive with sources | Research integrity |
+| **Citations** | Comprehensive with sources | Basic references | Research integrity |
 | **Customization** | Open source, fully customizable | Closed system | Unlimited extensibility |
-| **Cost** | Pay only for AI usage | Free tier + Monthly subscription | Transparent and controllable |
+| **Cost** | Pay only for AI usage | Monthly subscription + usage | Transparent and controllable |
 
 **Why Choose Open Notebook?**
 - 🔒 **Privacy First**: Your sensitive research stays completely private
@@ -95,58 +105,181 @@ Learn more about our project at [https://www.open-notebook.ai](https://www.open-
 
 ## 🚀 Quick Start
 
-Choose your installation method:
+**Docker Images Available:**
+- **Docker Hub**: `lfnovo/open_notebook:v1-latest-single`
+- **GitHub Container Registry**: `ghcr.io/lfnovo/open-notebook:v1-latest-single`
 
-### 🐳 **Docker (Recommended)**
+Both registries contain identical images - choose whichever you prefer!
 
-**Best for most users** - Fast setup with Docker Compose:
+### Choose Your Setup:
 
-→ **[Docker Compose Installation Guide](docs/1-INSTALLATION/docker-compose.md)**
-- Multi-container setup (recommended)
-- 5-10 minutes setup time
-- Requires Docker Desktop
+<table>
+<tr>
+<td width="50%">
 
-**Quick Start:**
-- Get an API key (OpenAI, Anthropic, Google, etc.) or setup Ollama
-- Create docker-compose.yml (example in guide)
-- Run: docker compose up -d
-- Access: http://localhost:8502
+#### 🏠 **Local Machine Setup**
+Perfect if Docker runs on the **same computer** where you'll access Open Notebook.
 
----
-
-### 💻 **From Source (Developers)**
-
-**For development and contributors:**
-
-→ **[From Source Installation Guide](docs/1-INSTALLATION/from-source.md)**
-- Clone and run locally
-- 10-15 minutes setup time
-- Requires: Python 3.11+, Node.js 18+, Docker, uv
-
-**Quick Start:**
 ```bash
-git clone https://github.com/lfnovo/open-notebook.git
-uv sync
-make start-all
+mkdir open-notebook && cd open-notebook
+
+docker run -d \
+  --name open-notebook \
+  -p 8502:8502 -p 5055:5055 \
+  -v ./notebook_data:/app/data \
+  -v ./surreal_data:/mydata \
+  -e OPENAI_API_KEY=your_key_here \
+  -e SURREAL_URL="ws://localhost:8000/rpc" \
+  -e SURREAL_USER="root" \
+  -e SURREAL_PASSWORD="root" \
+  -e SURREAL_NAMESPACE="open_notebook" \
+  -e SURREAL_DATABASE="production" \
+  lfnovo/open_notebook:v1-latest-single
 ```
 
-Access: http://localhost:3000 (dev) or http://localhost:8502 (production)
+**Access at:** http://localhost:8502
 
----
+</td>
+<td width="50%">
 
-### 📖 Need Help?
+#### 🌐 **Remote Server Setup**
+Use this for servers, Raspberry Pi, NAS, Proxmox, or any remote machine.
 
-- **🤖 AI Installation Assistant**: [CustomGPT to help you install](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant)
-- **🆘 Troubleshooting**: [5-minute troubleshooting guide](docs/6-TROUBLESHOOTING/quick-fixes.md)
-- **💬 Community Support**: [Discord Server](https://discord.gg/37XJPXfz2w)
-- **🐛 Report Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)
+```bash
+mkdir open-notebook && cd open-notebook
 
----
+docker run -d \
+  --name open-notebook \
+  -p 8502:8502 -p 5055:5055 \
+  -v ./notebook_data:/app/data \
+  -v ./surreal_data:/mydata \
+  -e OPENAI_API_KEY=your_key_here \
+  -e API_URL=http://YOUR_SERVER_IP:5055 \
+  -e SURREAL_URL="ws://localhost:8000/rpc" \
+  -e SURREAL_USER="root" \
+  -e SURREAL_PASSWORD="root" \
+  -e SURREAL_NAMESPACE="open_notebook" \
+  -e SURREAL_DATABASE="production" \
+  lfnovo/open_notebook:v1-latest-single
+```
+
+**Replace `YOUR_SERVER_IP`** with your server's IP (e.g., `192.168.1.100`) or domain
+
+**Access at:** http://YOUR_SERVER_IP:8502
+
+</td>
+</tr>
+</table>
+
+> **⚠️ Critical Setup Notes:**
+>
+> **Both ports are required:**
+> - **Port 8502**: Web interface (what you see in your browser)
+> - **Port 5055**: API backend (required for the app to function)
+>
+> **API_URL must match how YOU access the server:**
+> - ✅ Access via `http://192.168.1.100:8502` → set `API_URL=http://192.168.1.100:5055`
+> - ✅ Access via `http://myserver.local:8502` → set `API_URL=http://myserver.local:5055`
+> - ❌ Don't use `localhost` for remote servers - it won't work from other devices!
+
+### Using Docker Compose (Recommended for Easy Management)
+
+Create a `docker-compose.yml` file:
+
+```yaml
+services:
+  open_notebook:
+    image: lfnovo/open_notebook:v1-latest-single
+    # Or use: ghcr.io/lfnovo/open-notebook:v1-latest-single
+    ports:
+      - "8502:8502"  # Web UI
+      - "5055:5055"  # API (required!)
+    environment:
+      - OPENAI_API_KEY=your_key_here
+      # For remote access, uncomment and set your server IP/domain:
+      # - API_URL=http://192.168.1.100:5055
+      # Database connection (required for single-container)
+      - SURREAL_URL=ws://localhost:8000/rpc
+      - SURREAL_USER=root
+      - SURREAL_PASSWORD=root
+      - SURREAL_NAMESPACE=open_notebook
+      - SURREAL_DATABASE=production
+    volumes:
+      - ./notebook_data:/app/data
+      - ./surreal_data:/mydata
+    restart: always
+```
+
+Start with: `docker compose up -d`
+
+**What gets created:**
+```
+open-notebook/
+├── docker-compose.yml # Your configuration
+├── notebook_data/     # Your notebooks and research content
+└── surreal_data/      # Database files
+```
+
+### 🆘 Quick Troubleshooting
+
+| Problem | Solution |
+|---------|----------|
+| **"Unable to connect to server"** | Set `API_URL` environment variable to match how you access the server (see remote setup above) |
+| **Blank page or errors** | Ensure BOTH ports (8502 and 5055) are exposed in your docker command |
+| **Works on server but not from other computers** | Don't use `localhost` in `API_URL` - use your server's actual IP address |
+| **"404" or "config endpoint" errors** | Don't add `/api` to `API_URL` - use just `http://your-ip:5055` |
+| **Still having issues?** | Check our [5-minute troubleshooting guide](docs/troubleshooting/quick-fixes.md) or [join Discord](https://discord.gg/37XJPXfz2w) |
+
+### How Open Notebook Works
+
+```
+┌─────────────────────────────────────────────────────────┐
+│  Your Browser                                           │
+│  Access: http://your-server-ip:8502                     │
+└────────────────┬────────────────────────────────────────┘
+                 │
+                 ▼
+         ┌───────────────┐
+         │   Port 8502   │  ← Next.js Frontend (what you see)
+         │   Frontend    │    Also proxies API requests internally!
+         └───────┬───────┘
+                 │ proxies /api/* requests ↓
+                 ▼
+         ┌───────────────┐
+         │   Port 5055   │  ← FastAPI Backend (handles requests)
+         │     API       │
+         └───────┬───────┘
+                 │
+                 ▼
+         ┌───────────────┐
+         │   SurrealDB   │  ← Database (internal, auto-configured)
+         │   (Port 8000) │
+         └───────────────┘
+```
+
+**Key Points:**
+- **v1.1+**: Next.js automatically proxies `/api/*` requests to the backend, simplifying reverse proxy setup
+- Your browser loads the frontend from port 8502
+- The frontend needs to know where to find the API - when accessing remotely, set: `API_URL=http://your-server-ip:5055`
+- **Behind reverse proxy?** You only need to proxy to port 8502 now! See [Reverse Proxy Guide](docs/deployment/reverse-proxy.md)
 
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=lfnovo/open-notebook&type=date&legend=top-left)](https://www.star-history.com/#lfnovo/open-notebook&type=date&legend=top-left)
 
+### 🛠️ Full Installation
+For development or customization:
+```bash
+git clone https://github.com/lfnovo/open-notebook
+cd open-notebook
+make start-all
+```
+
+### 📖 Need Help?
+- **🤖 AI Installation Assistant**: We have a [CustomGPT built to help you install Open Notebook](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant) - it will guide you through each step!
+- **New to Open Notebook?** Start with our [Getting Started Guide](docs/getting-started/index.md)
+- **Need installation help?** Check our [Installation Guide](docs/getting-started/installation.md)
+- **Want to see it in action?** Try our [Quick Start Tutorial](docs/getting-started/quick-start.md)
 
 ## Provider Support Matrix
 
@@ -192,35 +325,36 @@ Thanks to the [Esperanto](https://github.com/lfnovo/esperanto) library, we suppo
 - **📊 Fine-Grained Context Control**: Choose exactly what to share with AI models
 - **📎 Citations**: Get answers with proper source citations
 
-
-## Podcast Feature
+### Three-Column Interface
+1. **Sources**: Manage all your research materials
+2. **Notes**: Create manual or AI-generated notes
+3. **Chat**: Converse with AI using your content as context
 
 [![Check out our podcast sample](https://img.youtube.com/vi/D-760MlGwaI/0.jpg)](https://www.youtube.com/watch?v=D-760MlGwaI)
 
 ## 📚 Documentation
 
 ### Getting Started
-- **[📖 Introduction](docs/0-START-HERE/index.md)** - Learn what Open Notebook offers
-- **[⚡ Quick Start](docs/0-START-HERE/quick-start.md)** - Get up and running in 5 minutes
-- **[🔧 Installation](docs/1-INSTALLATION/index.md)** - Comprehensive setup guide
-- **[🎯 Your First Notebook](docs/0-START-HERE/first-notebook.md)** - Step-by-step tutorial
+- **[📖 Introduction](docs/getting-started/introduction.md)** - Learn what Open Notebook offers
+- **[⚡ Quick Start](docs/getting-started/quick-start.md)** - Get up and running in 5 minutes
+- **[🔧 Installation](docs/getting-started/installation.md)** - Comprehensive setup guide
+- **[🎯 Your First Notebook](docs/getting-started/first-notebook.md)** - Step-by-step tutorial
 
 ### User Guide
-- **[📱 Interface Overview](docs/3-USER-GUIDE/interface-overview.md)** - Understanding the layout
-- **[📚 Notebooks](docs/3-USER-GUIDE/notebooks.md)** - Organizing your research
-- **[📄 Sources](docs/3-USER-GUIDE/sources.md)** - Managing content types
-- **[📝 Notes](docs/3-USER-GUIDE/notes.md)** - Creating and managing notes
-- **[💬 Chat](docs/3-USER-GUIDE/chat.md)** - AI conversations
-- **[🔍 Search](docs/3-USER-GUIDE/search.md)** - Finding information
+- **[📱 Interface Overview](docs/user-guide/interface-overview.md)** - Understanding the layout
+- **[📚 Notebooks](docs/user-guide/notebooks.md)** - Organizing your research
+- **[📄 Sources](docs/user-guide/sources.md)** - Managing content types
+- **[📝 Notes](docs/user-guide/notes.md)** - Creating and managing notes
+- **[💬 Chat](docs/user-guide/chat.md)** - AI conversations
+- **[🔍 Search](docs/user-guide/search.md)** - Finding information
 
 ### Advanced Topics
-- **[🎙️ Podcast Generation](docs/2-CORE-CONCEPTS/podcasts.md)** - Create professional podcasts
-- **[🔧 Content Transformations](docs/2-CORE-CONCEPTS/transformations.md)** - Customize content processing
-- **[🤖 AI Models](docs/4-AI-PROVIDERS/index.md)** - AI model configuration
-- **[🔌 MCP Integration](docs/5-CONFIGURATION/mcp-integration.md)** - Connect with Claude Desktop, VS Code and other MCP clients
-- **[🔧 REST API Reference](docs/7-DEVELOPMENT/api-reference.md)** - Complete API documentation
-- **[🔐 Security](docs/5-CONFIGURATION/security.md)** - Password protection and privacy
-- **[🚀 Deployment](docs/1-INSTALLATION/index.md)** - Complete deployment guides for all scenarios
+- **[🎙️ Podcast Generation](docs/features/podcasts.md)** - Create professional podcasts
+- **[🔧 Content Transformations](docs/features/transformations.md)** - Customize content processing
+- **[🤖 AI Models](docs/features/ai-models.md)** - AI model configuration
+- **[🔧 REST API Reference](docs/development/api-reference.md)** - Complete API documentation
+- **[🔐 Security](docs/deployment/security.md)** - Password protection and privacy
+- **[🚀 Deployment](docs/deployment/index.md)** - Complete deployment guides for all scenarios
 
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
@@ -246,12 +380,6 @@ See the [open issues](https://github.com/lfnovo/open-notebook/issues) for a full
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
 
-## 📖 Need Help?
-- **🤖 AI Installation Assistant**: We have a [CustomGPT built to help you install Open Notebook](https://chatgpt.com/g/g-68776e2765b48191bd1bae3f30212631-open-notebook-installation-assistant) - it will guide you through each step!
-- **New to Open Notebook?** Start with our [Getting Started Guide](docs/0-START-HERE/index.md)
-- **Need installation help?** Check our [Installation Guide](docs/1-INSTALLATION/index.md)
-- **Want to see it in action?** Try our [Quick Start Tutorial](docs/0-START-HERE/quick-start.md)
-
 ## 🤝 Community & Contributing
 
 ### Join the Community
@@ -278,12 +406,25 @@ See our [Contributing Guide](CONTRIBUTING.md) for detailed information on how to
 
 Open Notebook is MIT licensed. See the [LICENSE](LICENSE) file for details.
 
+## 📞 Contact
+
+**Luis Novo** - [@lfnovo](https://twitter.com/lfnovo)
 
 **Community Support**:
 - 💬 [Discord Server](https://discord.gg/37XJPXfz2w) - Get help, share ideas, and connect with users
 - 🐛 [GitHub Issues](https://github.com/lfnovo/open-notebook/issues) - Report bugs and request features
 - 🌐 [Website](https://www.open-notebook.ai) - Learn more about the project
 
+## 🙏 Acknowledgments
+
+Open Notebook is built on the shoulders of amazing open-source projects:
+
+* **[Podcast Creator](https://github.com/lfnovo/podcast-creator)** - Advanced podcast generation capabilities
+* **[Surreal Commands](https://github.com/lfnovo/surreal-commands)** - Background job processing
+* **[Content Core](https://github.com/lfnovo/content-core)** - Content processing and management
+* **[Esperanto](https://github.com/lfnovo/esperanto)** - Multi-provider AI model abstraction
+* **[Docling](https://github.com/docling-project/docling)** - Document processing and parsing
+
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
 

api/CLAUDE.md

@@ -1,117 +0,0 @@
-# API Module
-
-FastAPI-based REST backend exposing services for notebooks, sources, notes, chat, podcasts, and AI model management.
-
-## Purpose
-
-FastAPI application serving three architectural layers: routes (HTTP endpoints), services (business logic), and models (request/response schemas). Integrates LangGraph workflows (chat, ask, source_chat), SurrealDB persistence, and AI providers via Esperanto.
-
-## Architecture Overview
-
-**Three layers**:
-1. **Routes** (`routers/*`): HTTP endpoints mapping to services
-2. **Services** (`*_service.py`): Business logic orchestrating domain models, database, graphs, AI providers
-3. **Models** (`models.py`): Pydantic request/response schemas with validation
-
-**Startup flow**:
-- Load .env environment variables
-- Initialize CORS middleware + password auth middleware
-- Run database migrations via AsyncMigrationManager on lifespan startup
-- Register all routers
-
-**Key services**:
-- `chat_service.py`: Invokes chat graph with messages, context
-- `podcast_service.py`: Orchestrates outline + transcript generation
-- `sources_service.py`: Content ingestion, vectorization, metadata
-- `notes_service.py`: Note creation, linking to sources/insights
-- `transformations_service.py`: Applies transformations to content
-- `models_service.py`: Manages AI provider/model configuration
-- `episode_profiles_service.py`: Manages podcast speaker/episode profiles
-
-## Component Catalog
-
-### Main Application
-- **main.py**: FastAPI app initialization, CORS setup, auth middleware, lifespan event, router registration
-- **Lifespan handler**: Runs AsyncMigrationManager on startup (database schema migration)
-- **Auth middleware**: PasswordAuthMiddleware protects endpoints (password-based access control)
-
-### Services (Business Logic)
-- **chat_service.py**: Invokes chat.py graph; handles message history via SqliteSaver
-- **podcast_service.py**: Generates outline (outline.jinja), then transcript (transcript.jinja) for episodes
-- **sources_service.py**: Ingests files/URLs (content_core), extracts text, vectorizes, saves to SurrealDB
-- **transformations_service.py**: Applies transformations via transformation.py graph
-- **models_service.py**: Manages ModelManager config (AI provider overrides)
-- **episode_profiles_service.py**: CRUD for EpisodeProfile and SpeakerProfile models
-- **insights_service.py**: Generates and retrieves source insights
-- **notes_service.py**: Creates notes linked to sources/insights
-
-### Models (Schemas)
-- **models.py**: Pydantic schemas for request/response validation
-- Request bodies: ChatRequest, CreateNoteRequest, PodcastGenerationRequest, etc.
-- Response bodies: ChatResponse, NoteResponse, PodcastResponse, etc.
-- Custom validators for enum fields, file paths, model references
-
-### Routers
-- **routers/chat.py**: POST /chat
-- **routers/source_chat.py**: POST /source/{source_id}/chat
-- **routers/podcasts.py**: POST /podcasts, GET /podcasts/{id}, etc.
-- **routers/notes.py**: POST /notes, GET /notes/{id}
-- **routers/sources.py**: POST /sources, GET /sources/{id}, DELETE /sources/{id}
-- **routers/models.py**: GET /models, POST /models/config
-- **routers/transformations.py**: POST /transformations
-- **routers/insights.py**: GET /sources/{source_id}/insights
-- **routers/auth.py**: POST /auth/password (password-based auth)
-- **routers/commands.py**: GET /commands/{command_id} (job status tracking)
-
-## Common Patterns
-
-- **Service injection via FastAPI**: Routers import services directly; no DI framework
-- **Async/await throughout**: All DB queries, graph invocations, AI calls are async
-- **SurrealDB transactions**: Services use repo_query, repo_create, repo_upsert from database layer
-- **Config override pattern**: Models/config override via models_service passed to graph.ainvoke(config=...)
-- **Error handling**: Services catch exceptions and return HTTP status codes (400 Bad Request, 404 Not Found, 500 Internal Server Error)
-- **Logging**: loguru logger in main.py; services expected to log key operations
-- **Response normalization**: All responses follow standard schema (data + metadata structure)
-
-## Key Dependencies
-
-- `fastapi`: FastAPI app, routers, HTTPException
-- `pydantic`: Validation models with Field, field_validator
-- `open_notebook.graphs`: chat, ask, source_chat, source, transformation graphs
-- `open_notebook.database`: SurrealDB repository functions (repo_query, repo_create, repo_upsert)
-- `open_notebook.domain`: Notebook, Source, Note, SourceInsight models
-- `open_notebook.ai.provision`: provision_langchain_model() factory
-- `ai_prompter`: Prompter for template rendering
-- `content_core`: extract_content() for file/URL processing
-- `esperanto`: AI provider client library (LLM, embeddings, TTS)
-- `surreal_commands`: Job queue for async operations (podcast generation)
-- `loguru`: Structured logging
-
-## Important Quirks & Gotchas
-
-- **Migration auto-run**: Database schema migrations run on every API startup (via lifespan); no manual migration steps
-- **PasswordAuthMiddleware is basic**: Uses simple password check; production deployments should replace with OAuth/JWT
-- **No request rate limiting**: No built-in rate limiting; deployment must add via proxy/middleware
-- **Service state is stateless**: Services don't cache results; each request re-queries database/AI models
-- **Graph invocation is blocking**: chat/podcast workflows may take minutes; no timeout handling in services
-- **Command job fire-and-forget**: podcast_service.py submits jobs but doesn't wait (async job queue pattern)
-- **Model override scoping**: Model config override via RunnableConfig is per-request only (not persistent)
-- **CORS open by default**: main.py CORS settings allow all origins (restrict before production)
-- **No OpenAPI security scheme**: API docs available without auth (disable before production)
-- **Services don't validate user permission**: All endpoints trust authentication layer; no per-notebook permission checks
-
-## How to Add New Endpoint
-
-1. Create router file in `routers/` (e.g., `routers/new_feature.py`)
-2. Import router into `main.py` and register: `app.include_router(new_feature.router, tags=["new_feature"])`
-3. Create service in `new_feature_service.py` with business logic
-4. Define request/response schemas in `models.py` (or create `new_feature_models.py`)
-5. Implement router functions calling service methods
-6. Test with `uv run uvicorn api.main:app --host 0.0.0.0 --port 5055`
-
-## Testing Patterns
-
-- **Interactive docs**: http://localhost:5055/docs (Swagger UI)
-- **Direct service tests**: Import service, call methods directly with test data
-- **Mock graphs**: Replace graph.ainvoke() with mock for testing service logic
-- **Database: Use test database** (separate SurrealDB instance or mock repo_query)

api/chat_service.py

@@ -135,9 +135,7 @@ class ChatService:
             if model_override is not None:
                 data["model_override"] = model_override
             
-            # Short connect timeout (10s), long read timeout (10 min) for Ollama/local LLMs
-            timeout = httpx.Timeout(connect=10.0, read=600.0, write=30.0, pool=10.0)
-            async with httpx.AsyncClient(timeout=timeout) as client:
+            async with httpx.AsyncClient(timeout=120.0) as client:  # Longer timeout for chat
                 response = await client.post(
                     f"{self.base_url}/api/chat/execute",
                     json=data,

api/episode_profiles_service.py

@@ -7,7 +7,7 @@ from typing import List
 from loguru import logger
 
 from api.client import api_client
-from open_notebook.podcasts.models import EpisodeProfile
+from open_notebook.domain.podcast import EpisodeProfile
 
 
 class EpisodeProfilesService:

api/main.py

@@ -1,15 +1,8 @@
-# Load environment variables
-from dotenv import load_dotenv
-
-load_dotenv()
-
 from contextlib import asynccontextmanager
 
-from fastapi import FastAPI, Request
+from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse
 from loguru import logger
-from starlette.exceptions import HTTPException as StarletteHTTPException
 
 from api.auth import PasswordAuthMiddleware
 from api.routers import (
@@ -99,34 +92,6 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-
-# Custom exception handler to ensure CORS headers are included in error responses
-# This helps when errors occur before the CORS middleware can process them
-@app.exception_handler(StarletteHTTPException)
-async def custom_http_exception_handler(request: Request, exc: StarletteHTTPException):
-    """
-    Custom exception handler that ensures CORS headers are included in error responses.
-    This is particularly important for 413 (Payload Too Large) errors during file uploads.
-
-    Note: If a reverse proxy (nginx, traefik) returns 413 before the request reaches
-    FastAPI, this handler won't be called. In that case, configure your reverse proxy
-    to add CORS headers to error responses.
-    """
-    # Get the origin from the request
-    origin = request.headers.get("origin", "*")
-
-    return JSONResponse(
-        status_code=exc.status_code,
-        content={"detail": exc.detail},
-        headers={
-            "Access-Control-Allow-Origin": origin,
-            "Access-Control-Allow-Credentials": "true",
-            "Access-Control-Allow-Methods": "*",
-            "Access-Control-Allow-Headers": "*",
-        },
-    )
-
-
 # Include routers
 app.include_router(auth.router, prefix="/api", tags=["auth"])
 app.include_router(config.router, prefix="/api", tags=["config"])

api/models_service.py

@@ -7,7 +7,7 @@ from typing import List, Optional
 from loguru import logger
 
 from api.client import api_client
-from open_notebook.ai.models import DefaultModels, Model
+from open_notebook.domain.models import DefaultModels, Model
 
 
 class ModelsService:

api/podcast_service.py

@@ -6,7 +6,7 @@ from pydantic import BaseModel
 from surreal_commands import get_command_status, submit_command
 
 from open_notebook.domain.notebook import Notebook
-from open_notebook.podcasts.models import EpisodeProfile, PodcastEpisode, SpeakerProfile
+from open_notebook.domain.podcast import EpisodeProfile, PodcastEpisode, SpeakerProfile
 
 
 class PodcastGenerationRequest(BaseModel):

api/routers/embedding.py

@@ -3,7 +3,7 @@ from loguru import logger
 
 from api.command_service import CommandService
 from api.models import EmbedRequest, EmbedResponse
-from open_notebook.ai.models import model_manager
+from open_notebook.domain.models import model_manager
 from open_notebook.domain.notebook import Note, Source
 
 router = APIRouter()
@@ -62,22 +62,18 @@ async def embed_content(embed_request: EmbedRequest):
                 )
 
         else:
-            # SYNC PATH: Submit job (returns immediately with command_id)
-            # NOTE: "sync" here means "submit and return command_id" - actual processing
-            # still happens asynchronously in the worker pool
+            # SYNC PATH: Execute synchronously (existing behavior)
             logger.info(f"Using sync processing for {item_type} {item_id}")
 
-            command_id = None
-
             # Get the item and embed it
             if item_type == "source":
                 source_item = await Source.get(item_id)
                 if not source_item:
                     raise HTTPException(status_code=404, detail="Source not found")
 
-                # Submit vectorization job (returns command_id for tracking)
-                command_id = await source_item.vectorize()
-                message = "Source vectorization job submitted"
+                # Perform embedding (vectorize is now idempotent - safe to call multiple times)
+                await source_item.vectorize()
+                message = "Source embedded successfully"
 
             elif item_type == "note":
                 note_item = await Note.get(item_id)
@@ -88,7 +84,7 @@ async def embed_content(embed_request: EmbedRequest):
                 message = "Note embedded successfully"
 
             return EmbedResponse(
-                success=True, message=message, item_id=item_id, item_type=item_type, command_id=command_id
+                success=True, message=message, item_id=item_id, item_type=item_type, command_id=None
             )
 
     except HTTPException:

api/routers/episode_profiles.py

@@ -4,7 +4,7 @@ from fastapi import APIRouter, HTTPException
 from loguru import logger
 from pydantic import BaseModel, Field
 
-from open_notebook.podcasts.models import EpisodeProfile
+from open_notebook.domain.podcast import EpisodeProfile
 
 router = APIRouter()
 

api/routers/models.py

@@ -11,7 +11,7 @@ from api.models import (
     ModelResponse,
     ProviderAvailabilityResponse,
 )
-from open_notebook.ai.models import DefaultModels, Model
+from open_notebook.domain.models import DefaultModels, Model
 from open_notebook.exceptions import InvalidInputError
 
 router = APIRouter()
@@ -32,33 +32,6 @@ def _check_openai_compatible_support(mode: str) -> bool:
     return generic or specific
 
 
-def _check_azure_support(mode: str) -> bool:
-    """
-    Check if Azure OpenAI provider is available for a specific mode.
-
-    Args:
-        mode: One of 'LLM', 'EMBEDDING', 'STT', 'TTS'
-
-    Returns:
-        bool: True if either generic or mode-specific env vars are set
-    """
-    # Check generic configuration (applies to all modes)
-    generic = (
-        os.environ.get("AZURE_OPENAI_API_KEY") is not None
-        and os.environ.get("AZURE_OPENAI_ENDPOINT") is not None
-        and os.environ.get("AZURE_OPENAI_API_VERSION") is not None
-    )
-
-    # Check mode-specific configuration (takes precedence)
-    specific = (
-        os.environ.get(f"AZURE_OPENAI_API_KEY_{mode}") is not None
-        and os.environ.get(f"AZURE_OPENAI_ENDPOINT_{mode}") is not None
-        and os.environ.get(f"AZURE_OPENAI_API_VERSION_{mode}") is not None
-    )
-
-    return generic or specific
-
-
 @router.get("/models", response_model=List[ModelResponse])
 async def get_models(
     type: Optional[str] = Query(None, description="Filter by model type")
@@ -98,16 +71,16 @@ async def create_model(model_data: ModelCreate):
                 detail=f"Invalid model type. Must be one of: {valid_types}"
             )
 
-        # Check for duplicate model name under the same provider and type (case-insensitive)
+        # Check for duplicate model name under the same provider (case-insensitive)
         from open_notebook.database.repository import repo_query
         existing = await repo_query(
-            "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND string::lowercase(name) = $name AND string::lowercase(type) = $type LIMIT 1",
-            {"provider": model_data.provider.lower(), "name": model_data.name.lower(), "type": model_data.type.lower()}
+            "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND string::lowercase(name) = $name LIMIT 1",
+            {"provider": model_data.provider.lower(), "name": model_data.name.lower()}
         )
         if existing:
             raise HTTPException(
                 status_code=400,
-                detail=f"Model '{model_data.name}' already exists for provider '{model_data.provider}' with type '{model_data.type}'"
+                detail=f"Model '{model_data.name}' already exists for provider '{model_data.provider}'"
             )
 
         new_model = Model(
@@ -195,9 +168,11 @@ async def update_default_models(defaults_data: DefaultModelsResponse):
             defaults.default_tools_model = defaults_data.default_tools_model  # type: ignore[attr-defined]
         
         await defaults.update()
-
-        # No cache refresh needed - next access will fetch fresh data from DB
-
+        
+        # Refresh the model manager cache
+        from open_notebook.domain.models import model_manager
+        await model_manager.refresh_defaults()
+        
         return DefaultModelsResponse(
             default_chat_model=defaults.default_chat_model,  # type: ignore[attr-defined]
             default_transformation_model=defaults.default_transformation_model,  # type: ignore[attr-defined]
@@ -238,10 +213,10 @@ async def get_provider_availability():
             "elevenlabs": os.environ.get("ELEVENLABS_API_KEY") is not None,
             "voyage": os.environ.get("VOYAGE_API_KEY") is not None,
             "azure": (
-                _check_azure_support("LLM")
-                or _check_azure_support("EMBEDDING")
-                or _check_azure_support("STT")
-                or _check_azure_support("TTS")
+                os.environ.get("AZURE_OPENAI_API_KEY") is not None
+                and os.environ.get("AZURE_OPENAI_ENDPOINT") is not None
+                and os.environ.get("AZURE_OPENAI_DEPLOYMENT_NAME") is not None
+                and os.environ.get("AZURE_OPENAI_API_VERSION") is not None
             ),
             "mistral": os.environ.get("MISTRAL_API_KEY") is not None,
             "deepseek": os.environ.get("DEEPSEEK_API_KEY") is not None,
@@ -264,26 +239,19 @@ async def get_provider_availability():
         for provider in available_providers:
             supported_types[provider] = []
 
-            # Map Esperanto model types to our environment variable modes
-            mode_mapping = {
-                "language": "LLM",
-                "embedding": "EMBEDDING",
-                "speech_to_text": "STT",
-                "text_to_speech": "TTS",
-            }
-
             # Special handling for openai-compatible to check mode-specific availability
             if provider == "openai-compatible":
+                # Map Esperanto model types to our environment variable modes
+                mode_mapping = {
+                    "language": "LLM",
+                    "embedding": "EMBEDDING",
+                    "speech_to_text": "STT",
+                    "text_to_speech": "TTS",
+                }
                 for model_type, mode in mode_mapping.items():
                     if model_type in esperanto_available and provider in esperanto_available[model_type]:
                         if _check_openai_compatible_support(mode):
                             supported_types[provider].append(model_type)
-            # Special handling for azure to check mode-specific availability
-            elif provider == "azure":
-                for model_type, mode in mode_mapping.items():
-                    if model_type in esperanto_available and provider in esperanto_available[model_type]:
-                        if _check_azure_support(mode):
-                            supported_types[provider].append(model_type)
             else:
                 # Standard provider detection
                 for model_type, providers in esperanto_available.items():

api/routers/search.py

@@ -6,7 +6,7 @@ from fastapi.responses import StreamingResponse
 from loguru import logger
 
 from api.models import AskRequest, AskResponse, SearchRequest, SearchResponse
-from open_notebook.ai.models import Model, model_manager
+from open_notebook.domain.models import Model, model_manager
 from open_notebook.domain.notebook import text_search, vector_search
 from open_notebook.exceptions import DatabaseOperationError, InvalidInputError
 from open_notebook.graphs.ask import graph as ask_graph

api/routers/sources.py

@@ -113,7 +113,7 @@ def parse_source_form_data(
         try:
             notebooks_list = json.loads(notebooks)
         except json.JSONDecodeError:
-            logger.error(f"Invalid JSON in notebooks field: {notebooks}")
+            logger.error(f"DEBUG - Invalid JSON in notebooks field: {notebooks}")
             raise ValueError("Invalid JSON in notebooks field")
 
     transformations_list = []
@@ -122,7 +122,7 @@ def parse_source_form_data(
             transformations_list = json.loads(transformations)
         except json.JSONDecodeError:
             logger.error(
-                f"Invalid JSON in transformations field: {transformations}"
+                f"DEBUG - Invalid JSON in transformations field: {transformations}"
             )
             raise ValueError("Invalid JSON in transformations field")
 

api/routers/speaker_profiles.py

@@ -4,7 +4,7 @@ from fastapi import APIRouter, HTTPException
 from loguru import logger
 from pydantic import BaseModel, Field
 
-from open_notebook.podcasts.models import SpeakerProfile
+from open_notebook.domain.podcast import SpeakerProfile
 
 router = APIRouter()
 

api/routers/transformations.py

@@ -12,7 +12,7 @@ from api.models import (
     TransformationResponse,
     TransformationUpdate,
 )
-from open_notebook.ai.models import Model
+from open_notebook.domain.models import Model
 from open_notebook.domain.transformation import DefaultPrompts, Transformation
 from open_notebook.exceptions import InvalidInputError
 from open_notebook.graphs.transformation import graph as transformation_graph
@@ -78,79 +78,6 @@ async def create_transformation(transformation_data: TransformationCreate):
         )
 
 
-@router.post("/transformations/execute", response_model=TransformationExecuteResponse)
-async def execute_transformation(execute_request: TransformationExecuteRequest):
-    """Execute a transformation on input text."""
-    try:
-        # Validate transformation exists
-        transformation = await Transformation.get(execute_request.transformation_id)
-        if not transformation:
-            raise HTTPException(status_code=404, detail="Transformation not found")
-
-        # Validate model exists
-        model = await Model.get(execute_request.model_id)
-        if not model:
-            raise HTTPException(status_code=404, detail="Model not found")
-
-        # Execute the transformation
-        result = await transformation_graph.ainvoke(
-            dict(  # type: ignore[arg-type]
-                input_text=execute_request.input_text,
-                transformation=transformation,
-            ),
-            config=dict(configurable={"model_id": execute_request.model_id}),
-        )
-
-        return TransformationExecuteResponse(
-            output=result["output"],
-            transformation_id=execute_request.transformation_id,
-            model_id=execute_request.model_id,
-        )
-
-    except HTTPException:
-        raise
-    except Exception as e:
-        logger.error(f"Error executing transformation: {str(e)}")
-        raise HTTPException(
-            status_code=500, detail=f"Error executing transformation: {str(e)}"
-        )
-
-
-@router.get("/transformations/default-prompt", response_model=DefaultPromptResponse)
-async def get_default_prompt():
-    """Get the default transformation prompt."""
-    try:
-        default_prompts: DefaultPrompts = await DefaultPrompts.get_instance()  # type: ignore[assignment]
-
-        return DefaultPromptResponse(
-            transformation_instructions=default_prompts.transformation_instructions or ""
-        )
-    except Exception as e:
-        logger.error(f"Error fetching default prompt: {str(e)}")
-        raise HTTPException(
-            status_code=500, detail=f"Error fetching default prompt: {str(e)}"
-        )
-
-
-@router.put("/transformations/default-prompt", response_model=DefaultPromptResponse)
-async def update_default_prompt(prompt_update: DefaultPromptUpdate):
-    """Update the default transformation prompt."""
-    try:
-        default_prompts: DefaultPrompts = await DefaultPrompts.get_instance()  # type: ignore[assignment]
-
-        default_prompts.transformation_instructions = prompt_update.transformation_instructions
-        await default_prompts.update()
-
-        return DefaultPromptResponse(
-            transformation_instructions=default_prompts.transformation_instructions
-        )
-    except Exception as e:
-        logger.error(f"Error updating default prompt: {str(e)}")
-        raise HTTPException(
-            status_code=500, detail=f"Error updating default prompt: {str(e)}"
-        )
-
-
 @router.get(
     "/transformations/{transformation_id}", response_model=TransformationResponse
 )
@@ -245,3 +172,76 @@ async def delete_transformation(transformation_id: str):
         raise HTTPException(
             status_code=500, detail=f"Error deleting transformation: {str(e)}"
         )
+
+
+@router.post("/transformations/execute", response_model=TransformationExecuteResponse)
+async def execute_transformation(execute_request: TransformationExecuteRequest):
+    """Execute a transformation on input text."""
+    try:
+        # Validate transformation exists
+        transformation = await Transformation.get(execute_request.transformation_id)
+        if not transformation:
+            raise HTTPException(status_code=404, detail="Transformation not found")
+
+        # Validate model exists
+        model = await Model.get(execute_request.model_id)
+        if not model:
+            raise HTTPException(status_code=404, detail="Model not found")
+
+        # Execute the transformation
+        result = await transformation_graph.ainvoke(
+            dict(  # type: ignore[arg-type]
+                input_text=execute_request.input_text,
+                transformation=transformation,
+            ),
+            config=dict(configurable={"model_id": execute_request.model_id}),
+        )
+
+        return TransformationExecuteResponse(
+            output=result["output"],
+            transformation_id=execute_request.transformation_id,
+            model_id=execute_request.model_id,
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error executing transformation: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error executing transformation: {str(e)}"
+        )
+
+
+@router.get("/transformations/default-prompt", response_model=DefaultPromptResponse)
+async def get_default_prompt():
+    """Get the default transformation prompt."""
+    try:
+        default_prompts: DefaultPrompts = await DefaultPrompts.get_instance()  # type: ignore[assignment]
+
+        return DefaultPromptResponse(
+            transformation_instructions=default_prompts.transformation_instructions or ""
+        )
+    except Exception as e:
+        logger.error(f"Error fetching default prompt: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching default prompt: {str(e)}"
+        )
+
+
+@router.put("/transformations/default-prompt", response_model=DefaultPromptResponse)
+async def update_default_prompt(prompt_update: DefaultPromptUpdate):
+    """Update the default transformation prompt."""
+    try:
+        default_prompts: DefaultPrompts = await DefaultPrompts.get_instance()  # type: ignore[assignment]
+
+        default_prompts.transformation_instructions = prompt_update.transformation_instructions
+        await default_prompts.update()
+
+        return DefaultPromptResponse(
+            transformation_instructions=default_prompts.transformation_instructions
+        )
+    except Exception as e:
+        logger.error(f"Error updating default prompt: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Error updating default prompt: {str(e)}"
+        )

batch_fix_services.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Batch fix service files for mypy errors."""
+import re
+from pathlib import Path
+
+SERVICE_FILES = [
+    'api/notes_service.py',
+    'api/insights_service.py',
+    'api/episode_profiles_service.py',
+    'api/settings_service.py',
+    'api/sources_service.py',
+    'api/podcast_service.py',
+    'api/command_service.py',
+]
+
+BASE_DIR = Path('/Users/luisnovo/dev/projetos/open-notebook/open-notebook')
+
+for service_file in SERVICE_FILES:
+    file_path = BASE_DIR / service_file
+    if not file_path.exists():
+        print(f"Skipping {service_file} - file not found")
+        continue
+
+    content = file_path.read_text()
+    original_content = content
+
+    # Pattern to find: var_name = api_client.method(args)
+    # Followed by: var_name["key"] or var_name.get("key")
+    lines = content.split('\n')
+    new_lines = []
+    i = 0
+
+    while i < len(lines):
+        line = lines[i]
+
+        # Check if this line has an api_client call assignment
+        match = re.match(r'(\s*)(\w+)\s*=\s*api_client\.(\w+)\((.*)\)\s*$', line)
+        if match and 'response = api_client' not in line:
+            indent = match.group(1)
+            var_name = match.group(2)
+            method_name = match.group(3)
+            args = match.group(4)
+
+            # Look ahead to see if this variable is used with dict access
+            has_dict_access = False
+            for j in range(i+1, min(i+15, len(lines))):
+                next_line = lines[j]
+                if f'{var_name}["' in next_line or f"{var_name}['" in next_line or f'{var_name}.get(' in next_line:
+                    has_dict_access = True
+                    break
+                # Stop looking if we hit a blank line, new function, or new assignment
+                if (not next_line.strip() or
+                    next_line.strip().startswith('def ') or
+                    next_line.strip().startswith('class ') or
+                    (re.match(r'\s*\w+\s*=', next_line) and var_name not in next_line)):
+                    break
+
+            if has_dict_access:
+                # Replace with response and isinstance check
+                new_lines.append(f'{indent}response = api_client.{method_name}({args})')
+                new_lines.append(f'{indent}{var_name} = response if isinstance(response, dict) else response[0]')
+                i += 1
+                continue
+
+        new_lines.append(line)
+        i += 1
+
+    new_content = '\n'.join(new_lines)
+
+    # Check if content changed
+    if new_content != original_content:
+        file_path.write_text(new_content)
+        print(f"✓ Fixed {service_file}")
+    else:
+        print(f"- No changes needed for {service_file}")
+
+print("\nDone!")

commands/CLAUDE.md

@@ -1,51 +0,0 @@
-# Commands Module
-
-**Purpose**: Defines async command handlers for long-running operations via `surreal-commands` job queue system.
-
-## Key Components
-
-- **`process_source_command`**: Ingests content through `source_graph`, creates embeddings (optional), and generates insights. Retries on transaction conflicts (exp. jitter, max 5×).
-- **`embed_single_item_command`**: Embeds individual sources/notes/insights; splits content into chunks for vector storage.
-- **`rebuild_embeddings_command`**: Bulk re-embed all/existing items with selective type filtering.
-- **`generate_podcast_command`**: Creates podcasts via `podcast-creator` library using stored episode/speaker profiles.
-- **`process_text_command`** (example): Test fixture for text operations (uppercase, lowercase, reverse, word_count).
-- **`analyze_data_command`** (example): Test fixture for numeric aggregations.
-
-## Important Patterns
-
-- **Pydantic I/O**: All commands use `CommandInput`/`CommandOutput` subclasses for type safety and serialization.
-- **Error handling**: Permanent errors return failure output; `RuntimeError` exceptions auto-retry via surreal-commands.
-- **Retry configuration**: Aggressive retry settings (15 attempts, 1-120s backoff, DEBUG log level) are a temporary workaround for SurrealDB v2.x transaction conflicts with SEARCH indexes. These can be reduced after migrating to SurrealDB v3.
-- **Model dumping**: Recursive `full_model_dump()` utility converts Pydantic models → dicts for DB/API responses.
-- **Logging**: Uses `loguru.logger` throughout; logs execution start/end and key metrics (processing time, counts). Retry attempts use `retry_log_level: "debug"` to prevent log noise during concurrent chunk processing.
-- **Time tracking**: All commands measure `start_time` → `processing_time` for monitoring.
-
-## Dependencies
-
-**External**: `surreal_commands` (command decorator, job queue), `loguru`, `pydantic`, `podcast_creator`
-**Internal**: `open_notebook.domain.*` (Source, Note, Transformation), `open_notebook.graphs.source`, `open_notebook.ai.models`
-
-## Quirks & Edge Cases
-
-- **source_commands**: `ensure_record_id()` wraps command IDs for DB storage; transaction conflicts trigger exponential backoff retry (1-120s, up to 15 attempts). Non-`RuntimeError` exceptions are permanent. Retry logs at DEBUG level via `retry_log_level` config.
-- **embedding_commands**: Queries DB directly for item state; chunk index must match source's chunk list. Model availability checked at command start. Aggressive retry settings (15 attempts, 120s max wait, DEBUG logging) handle deep queues from large documents without log spam.
-- **podcast_commands**: Profiles loaded from SurrealDB by name (must exist); briefing can be extended with suffix. Episode records created mid-execution.
-- **Example commands**: Accept optional `delay_seconds` for testing async behavior; not for production.
-- **Retry logging**: Uses `retry_log_level: "debug"` in decorator config + manual `logger.debug()` in exception handlers for double protection against retry log noise.
-
-## Code Example
-
-```python
-@command("process_source", app="open_notebook", retry={...})
-async def process_source_command(input_data: SourceProcessingInput) -> SourceProcessingOutput:
-    start_time = time.time()
-    try:
-        transformations = [await Transformation.get(id) for id in input_data.transformations]
-        source = await Source.get(input_data.source_id)
-        result = await source_graph.ainvoke({...})
-        return SourceProcessingOutput(success=True, ...)
-    except RuntimeError as e:
-        raise  # Retry this
-    except Exception as e:
-        return SourceProcessingOutput(success=False, error_message=str(e))
-```

commands/embedding_commands.py

@@ -3,12 +3,11 @@ from typing import Dict, List, Literal, Optional
 
 from loguru import logger
 from pydantic import BaseModel
-from surreal_commands import CommandInput, CommandOutput, command, submit_command
+from surreal_commands import CommandInput, CommandOutput, command
 
-from open_notebook.ai.models import model_manager
 from open_notebook.database.repository import ensure_record_id, repo_query
+from open_notebook.domain.models import model_manager
 from open_notebook.domain.notebook import Note, Source, SourceInsight
-from open_notebook.utils.text_utils import split_text
 
 
 def full_model_dump(model):
@@ -36,32 +35,6 @@ class EmbedSingleItemOutput(CommandOutput):
     error_message: Optional[str] = None
 
 
-class EmbedChunkInput(CommandInput):
-    source_id: str
-    chunk_index: int
-    chunk_text: str
-
-
-class EmbedChunkOutput(CommandOutput):
-    success: bool
-    source_id: str
-    chunk_index: int
-    error_message: Optional[str] = None
-
-
-class VectorizeSourceInput(CommandInput):
-    source_id: str
-
-
-class VectorizeSourceOutput(CommandOutput):
-    success: bool
-    source_id: str
-    total_chunks: int
-    jobs_submitted: int
-    processing_time: float
-    error_message: Optional[str] = None
-
-
 class RebuildEmbeddingsInput(CommandInput):
     mode: Literal["existing", "all"]
     include_sources: bool = True
@@ -186,220 +159,6 @@ async def embed_single_item_command(
         )
 
 
-@command(
-    "embed_chunk",
-    app="open_notebook",
-    retry={
-        "max_attempts": 15,  # Increased from 5 to handle deep queues (workaround for SurrealDB v2 transaction conflicts)
-        "wait_strategy": "exponential_jitter",
-        "wait_min": 1,
-        "wait_max": 120,  # Increased from 30s to 120s to allow queue to drain
-        "retry_on": [RuntimeError, ConnectionError, TimeoutError],
-        "retry_log_level": "debug",  # Use debug level to avoid log noise with hundreds of chunks
-    },
-)
-async def embed_chunk_command(
-    input_data: EmbedChunkInput,
-) -> EmbedChunkOutput:
-    """
-    Process a single text chunk for embedding as part of source vectorization.
-
-    This command is designed to be submitted as a background job for each chunk
-    of a source document, allowing natural concurrency control through the worker pool.
-
-    Retry Strategy (SurrealDB v2 workaround):
-    - Retries up to 15 times for transient failures (increased from 5):
-      * RuntimeError: SurrealDB transaction conflicts ("read or write conflict")
-      * ConnectionError: Network failures when calling embedding provider
-      * TimeoutError: Request timeouts to embedding provider
-    - Uses exponential-jitter backoff (1-120s, increased from 30s max)
-    - Higher retry limits allow deep queues (200+ chunks) to drain during concurrent processing
-    - Does NOT retry permanent failures (ValueError, authentication errors, invalid input)
-
-    Note: These aggressive retry settings are a temporary workaround for SurrealDB v2.x
-    transaction conflict issues. Can be reduced once migrated to SurrealDB v3.
-
-    Exception Handling:
-    - RuntimeError, ConnectionError, TimeoutError: Re-raised to trigger retry mechanism
-    - ValueError and other exceptions: Caught and returned as permanent failures (no retry)
-    """
-    try:
-        logger.debug(
-            f"Processing chunk {input_data.chunk_index} for source {input_data.source_id}"
-        )
-
-        # Get embedding model
-        EMBEDDING_MODEL = await model_manager.get_embedding_model()
-        if not EMBEDDING_MODEL:
-            raise ValueError(
-                "No embedding model configured. Please configure one in the Models section."
-            )
-
-        # Generate embedding for the chunk
-        embedding = (await EMBEDDING_MODEL.aembed([input_data.chunk_text]))[0]
-
-        # Insert chunk embedding into database
-        await repo_query(
-            """
-            CREATE source_embedding CONTENT {
-                "source": $source_id,
-                "order": $order,
-                "content": $content,
-                "embedding": $embedding,
-            };
-            """,
-            {
-                "source_id": ensure_record_id(input_data.source_id),
-                "order": input_data.chunk_index,
-                "content": input_data.chunk_text,
-                "embedding": embedding,
-            },
-        )
-
-        logger.debug(
-            f"Successfully embedded chunk {input_data.chunk_index} for source {input_data.source_id}"
-        )
-
-        return EmbedChunkOutput(
-            success=True,
-            source_id=input_data.source_id,
-            chunk_index=input_data.chunk_index,
-        )
-
-    except RuntimeError:
-        # Re-raise RuntimeError to allow retry mechanism to handle DB transaction conflicts
-        logger.debug(
-            f"Transaction conflict for chunk {input_data.chunk_index} - will be retried by retry mechanism"
-        )
-        raise
-    except (ConnectionError, TimeoutError) as e:
-        # Re-raise network/timeout errors to allow retry mechanism to handle transient provider failures
-        logger.debug(
-            f"Network/timeout error for chunk {input_data.chunk_index} ({type(e).__name__}: {e}) - will be retried by retry mechanism"
-        )
-        raise
-    except Exception as e:
-        # Catch other exceptions (ValueError, etc.) as permanent failures
-        logger.error(
-            f"Failed to embed chunk {input_data.chunk_index} for source {input_data.source_id}: {e}"
-        )
-        logger.exception(e)
-
-        return EmbedChunkOutput(
-            success=False,
-            source_id=input_data.source_id,
-            chunk_index=input_data.chunk_index,
-            error_message=str(e),
-        )
-
-
-@command("vectorize_source", app="open_notebook", retry=None)
-async def vectorize_source_command(
-    input_data: VectorizeSourceInput,
-) -> VectorizeSourceOutput:
-    """
-    Orchestrate source vectorization by splitting text into chunks and submitting
-    individual embed_chunk jobs to the worker queue.
-
-    This command:
-    1. Deletes existing embeddings (idempotency)
-    2. Splits source text into chunks
-    3. Submits each chunk as a separate embed_chunk job
-    4. Returns immediately (jobs run in background)
-
-    Natural concurrency control is provided by the worker pool size.
-
-    Retry Strategy:
-    - Retries disabled (retry=None) - fails fast on job submission errors
-    - This ensures immediate visibility when orchestration fails
-    - Individual embed_chunk jobs have their own retry logic for DB conflicts
-    """
-    start_time = time.time()
-
-    try:
-        logger.info(f"Starting vectorization orchestration for source {input_data.source_id}")
-
-        # 1. Load source
-        source = await Source.get(input_data.source_id)
-        if not source:
-            raise ValueError(f"Source '{input_data.source_id}' not found")
-
-        if not source.full_text:
-            raise ValueError(f"Source {input_data.source_id} has no text to vectorize")
-
-        # 2. Delete existing embeddings (idempotency)
-        logger.info(f"Deleting existing embeddings for source {input_data.source_id}")
-        delete_result = await repo_query(
-            "DELETE source_embedding WHERE source = $source_id",
-            {"source_id": ensure_record_id(input_data.source_id)}
-        )
-        deleted_count = len(delete_result) if delete_result else 0
-        if deleted_count > 0:
-            logger.info(f"Deleted {deleted_count} existing embeddings")
-
-        # 3. Split text into chunks
-        logger.info(f"Splitting text into chunks for source {input_data.source_id}")
-        chunks = split_text(source.full_text)
-        total_chunks = len(chunks)
-        logger.info(f"Split into {total_chunks} chunks")
-
-        if total_chunks == 0:
-            raise ValueError("No chunks created after splitting text")
-
-        # 4. Submit each chunk as a separate job
-        logger.info(f"Submitting {total_chunks} chunk jobs to worker queue")
-        jobs_submitted = 0
-
-        for idx, chunk_text in enumerate(chunks):
-            try:
-                job_id = submit_command(
-                    "open_notebook",  # app name
-                    "embed_chunk",    # command name
-                    {
-                        "source_id": input_data.source_id,
-                        "chunk_index": idx,
-                        "chunk_text": chunk_text,
-                    }
-                )
-                jobs_submitted += 1
-
-                if (idx + 1) % 100 == 0:
-                    logger.info(f"  Submitted {idx + 1}/{total_chunks} chunk jobs")
-
-            except Exception as e:
-                logger.error(f"Failed to submit chunk job {idx}: {e}")
-                # Continue submitting other chunks even if one fails
-
-        processing_time = time.time() - start_time
-
-        logger.info(
-            f"Vectorization orchestration complete for source {input_data.source_id}: "
-            f"{jobs_submitted}/{total_chunks} jobs submitted in {processing_time:.2f}s"
-        )
-
-        return VectorizeSourceOutput(
-            success=True,
-            source_id=input_data.source_id,
-            total_chunks=total_chunks,
-            jobs_submitted=jobs_submitted,
-            processing_time=processing_time,
-        )
-
-    except Exception as e:
-        processing_time = time.time() - start_time
-        logger.error(f"Vectorization orchestration failed for source {input_data.source_id}: {e}")
-        logger.exception(e)
-
-        return VectorizeSourceOutput(
-            success=False,
-            source_id=input_data.source_id,
-            total_chunks=0,
-            jobs_submitted=0,
-            processing_time=processing_time,
-            error_message=str(e),
-        )
-
-
 async def collect_items_for_rebuild(
     mode: str,
     include_sources: bool,
@@ -467,17 +226,12 @@ async def collect_items_for_rebuild(
     return items
 
 
-@command("rebuild_embeddings", app="open_notebook", retry=None)
+@command("rebuild_embeddings", app="open_notebook")
 async def rebuild_embeddings_command(
     input_data: RebuildEmbeddingsInput,
 ) -> RebuildEmbeddingsOutput:
     """
     Rebuild embeddings for sources, notes, and/or insights
-
-    Retry Strategy:
-    - Retries disabled (retry=None) - batch failures are immediately reported
-    - This ensures immediate visibility when batch operations fail
-    - Allows operators to quickly identify and resolve issues
     """
     start_time = time.time()
 

commands/podcast_commands.py

@@ -8,7 +8,7 @@ from surreal_commands import CommandInput, CommandOutput, command
 
 from open_notebook.config import DATA_FOLDER
 from open_notebook.database.repository import ensure_record_id, repo_query
-from open_notebook.podcasts.models import EpisodeProfile, PodcastEpisode, SpeakerProfile
+from open_notebook.domain.podcast import EpisodeProfile, PodcastEpisode, SpeakerProfile
 
 try:
     from podcast_creator import configure, create_podcast

commands/source_commands.py

@@ -44,18 +44,7 @@ class SourceProcessingOutput(CommandOutput):
     error_message: Optional[str] = None
 
 
-@command(
-    "process_source",
-    app="open_notebook",
-    retry={
-        "max_attempts": 15,  # Increased from 5 to handle deep queues (workaround for SurrealDB v2 transaction conflicts)
-        "wait_strategy": "exponential_jitter",
-        "wait_min": 1,
-        "wait_max": 120,  # Increased from 30s to 120s to allow queue to drain
-        "retry_on": [RuntimeError],
-        "retry_log_level": "debug",  # Use debug level to avoid log noise during transaction conflicts
-    },
-)
+@command("process_source", app="open_notebook")
 async def process_source_command(
     input_data: SourceProcessingInput,
 ) -> SourceProcessingOutput:
@@ -135,15 +124,10 @@ async def process_source_command(
             processing_time=processing_time,
         )
 
-    except RuntimeError as e:
-        # Transaction conflicts should be retried by surreal-commands
-        logger.debug(f"Transaction conflict, will retry: {e}")
-        raise
-
     except Exception as e:
-        # Other errors are permanent failures
         processing_time = time.time() - start_time
         logger.error(f"Source processing failed: {e}")
+        logger.exception(e)
 
         return SourceProcessingOutput(
             success=False,

docker-compose.dev.yml

@@ -5,8 +5,6 @@ services:
       - ./surreal_data:/mydata
     environment:
       - SURREAL_EXPERIMENTAL_GRAPHQL=true
-    ports:
-      - "8000:8000"
     command: start --log info --user root --pass root rocksdb:/mydata/mydatabase.db
     pull_policy: always
     user: root

docs/0-START-HERE/index.md

@@ -1,63 +0,0 @@
-# Open Notebook - Start Here
-
-**Open Notebook** is a privacy-focused AI research assistant. Upload documents, chat with AI, generate notes, and create podcasts—all with complete control over your data.
-
-## Choose Your Path
-
-### 🚀 I want to use OpenAI (Fastest)
-**5 minutes to running.** GPT, simple setup, powerful results.
-
-→ [OpenAI Quick Start](quick-start-openai.md)
-
----
-
-### ☁️ I want to use other cloud AI (Anthropic, Google, OpenRouter, etc.)
-**5 minutes to running.** Choose from 15+ AI providers.
-
-→ [Cloud Providers Quick Start](quick-start-cloud.md)
-
----
-
-### 🏠 I want to run locally (Ollama or LMStudio, completely private)
-**5 minutes to running.** Keep everything private, on your machine. No costs.
-
-→ [Local Quick Start](quick-start-local.md)
-
----
-
-## What Can You Do?
-
-- 📄 **Upload Content**: PDFs, web links, audio, video, text
-- 🤖 **Chat with AI**: Ask questions about your documents with citations
-- 📝 **Generate Notes**: AI creates summaries and insights
-- 🎙️ **Create Podcasts**: Turn research into professional audio content
-- 🔍 **Search**: Full-text and semantic search across all content
-- ⚙️ **Transform**: Extract insights, analyze themes, create summaries
-
-## Why Open Notebook?
-
-| Feature | Open Notebook | Notebook LM |
-|---------|---|---|
-| **Privacy** | Self-hosted, your control | Cloud, Google's servers |
-| **AI Choice** | 15+ providers | Google's models only |
-| **Podcast Speakers** | 1-4 customizable | 2 only |
-| **Cost** | Completely free | Free (but your data) |
-| **Offline** | Yes  | No |
-
-## Prerequisites
-
-- **Docker**: All paths use Docker (free)
-- **AI Provider**: Either a cloud API key OR use free local models (Ollama)
-
----
-
-## Next Steps
-
-1. Pick your path above ⬆️
-2. Follow the 5-minute quick start
-3. Create your first notebook
-4. Start uploading documents!
-
----
-
-**Need Help?** Join our [Discord community](https://discord.gg/37XJPXfz2w) or see [Full Documentation](../index.md).

docs/0-START-HERE/quick-start-cloud.md

@@ -1,226 +0,0 @@
-# Quick Start - Cloud AI Providers (5 minutes)
-
-Get Open Notebook running with **Anthropic, Google, Groq, or other cloud providers**. Same simplicity as OpenAI, with more choices.
-
-## Prerequisites
-
-1. **Docker Desktop** installed
-   - [Download here](https://www.docker.com/products/docker-desktop/)
-   - Already have it? Skip to step 2
-
-2. **API Key** from your chosen provider:
-   - **OpenRouter** (100+ models, one key): https://openrouter.ai/keys
-   - **Anthropic (Claude)**: https://console.anthropic.com/
-   - **Google (Gemini)**: https://aistudio.google.com/
-   - **Groq** (fast, free tier): https://console.groq.com/
-   - **Mistral**: https://console.mistral.ai/
-   - **DeepSeek**: https://platform.deepseek.com/
-   - **xAI (Grok)**: https://console.x.ai/
-
-## Step 1: Create Configuration (1 min)
-
-Create a new folder `open-notebook` and add this file:
-
-**docker-compose.yml**:
-```yaml
-services:
-  surrealdb:
-    image: surrealdb/surrealdb:v2
-    command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db
-    ports:
-      - "8000:8000"
-    volumes:
-      - ./surreal_data:/mydata
-
-  open_notebook:
-    image: lfnovo/open_notebook:v1-latest
-    pull_policy: always
-    ports:
-      - "8502:8502"  # Web UI
-      - "5055:5055"  # API
-    environment:
-      # Choose ONE provider (uncomment your choice):
-
-      # OpenRouter - 100+ models with one API key
-      - OPENROUTER_API_KEY=sk-or-...
-
-      # Anthropic (Claude) - Excellent reasoning
-      # - ANTHROPIC_API_KEY=sk-ant-...
-
-      # Google (Gemini) - Large context, cost-effective
-      # - GOOGLE_API_KEY=...
-
-      # Groq - Ultra-fast inference, free tier available
-      # - GROQ_API_KEY=gsk_...
-
-      # Mistral - European provider, good quality
-      # - MISTRAL_API_KEY=...
-
-      # Database (required)
-      - SURREAL_URL=ws://surrealdb:8000/rpc
-      - SURREAL_USER=root
-      - SURREAL_PASSWORD=password
-      - SURREAL_NAMESPACE=open_notebook
-      - SURREAL_DATABASE=open_notebook
-    volumes:
-      - ./notebook_data:/app/data
-    depends_on:
-      - surrealdb
-    restart: always
-
-```
-
-**Edit the file:**
-- Uncomment ONE provider and add your API key
-- Comment out or remove the others
-
----
-
-## Step 2: Start Services (1 min)
-
-Open terminal in your `open-notebook` folder:
-
-```bash
-docker compose up -d
-```
-
-Wait 15-20 seconds for services to start.
-
----
-
-## Step 3: Access Open Notebook (instant)
-
-Open your browser:
-```
-http://localhost:8502
-```
-
-You should see the Open Notebook interface!
-
----
-
-## Step 4: Configure Your Model (1 min)
-
-1. Go to **Settings** (gear icon)
-2. Navigate to **Models**
-3. Select your provider's model:
-
-| Provider | Recommended Model | Notes |
-|----------|-------------------|-------|
-| **OpenRouter** | `anthropic/claude-3.5-sonnet` | Access 100+ models |
-| **Anthropic** | `claude-3-5-sonnet-latest` | Best reasoning |
-| **Google** | `gemini-2.0-flash` | Large context, fast |
-| **Groq** | `llama-3.3-70b-versatile` | Ultra-fast |
-| **Mistral** | `mistral-large-latest` | Strong European option |
-
-4. Click **Save**
-
----
-
-## Step 5: Create Your First Notebook (1 min)
-
-1. Click **New Notebook**
-2. Name: "My Research"
-3. Click **Create**
-
----
-
-## Step 6: Add Content & Chat (2 min)
-
-1. Click **Add Source**
-2. Choose **Web Link**
-3. Paste any article URL
-4. Wait for processing
-5. Go to **Chat** and ask questions!
-
----
-
-## Verification Checklist
-
-- [ ] Docker is running
-- [ ] You can access `http://localhost:8502`
-- [ ] Models are configured for your provider
-- [ ] You created a notebook
-- [ ] Chat works
-
-**All checked?** You're ready to research!
-
----
-
-## Provider Comparison
-
-| Provider | Speed | Quality | Context | Cost |
-|----------|-------|---------|---------|------|
-| **OpenRouter** | Varies | Varies | Varies | Varies (100+ models) |
-| **Anthropic** | Medium | Excellent | 200K | $$$ |
-| **Google** | Fast | Very Good | 1M+ | $$ |
-| **Groq** | Ultra-fast | Good | 128K | $ (free tier) |
-| **Mistral** | Fast | Good | 128K | $$ |
-| **DeepSeek** | Medium | Very Good | 64K | $ |
-
----
-
-## Using Multiple Providers
-
-You can enable multiple providers simultaneously:
-
-```yaml
-environment:
-  - OPENROUTER_API_KEY=sk-or-...
-  - ANTHROPIC_API_KEY=sk-ant-...
-  - GOOGLE_API_KEY=...
-  - GROQ_API_KEY=gsk_...
-```
-
-Then switch between them in **Settings** > **Models** as needed.
-
----
-
-## Troubleshooting
-
-### "Model not found" Error
-
-1. Verify your API key is correct (no extra spaces)
-2. Check you have credits/access for the model
-3. Restart: `docker compose restart api`
-
-### "Cannot connect to server"
-
-```bash
-docker ps  # Check all services running
-docker compose logs  # View logs
-docker compose restart  # Restart everything
-```
-
-### Provider-Specific Issues
-
-**Anthropic**: Ensure key starts with `sk-ant-`
-**Google**: Use AI Studio key, not Cloud Console
-**Groq**: Free tier has rate limits; upgrade if needed
-
----
-
-## Cost Estimates
-
-Approximate costs per 1K tokens:
-
-| Provider | Input | Output |
-|----------|-------|--------|
-| Anthropic (Sonnet) | $0.003 | $0.015 |
-| Google (Flash) | $0.0001 | $0.0004 |
-| Groq (Llama 70B) | Free tier available | - |
-| Mistral (Large) | $0.002 | $0.006 |
-
-Check provider websites for current pricing.
-
----
-
-## Next Steps
-
-1. **Add Your Content**: PDFs, web links, documents
-2. **Explore Features**: Podcasts, transformations, search
-3. **Full Documentation**: [See all features](../3-USER-GUIDE/index.md)
-
----
-
-**Need help?** Join our [Discord community](https://discord.gg/37XJPXfz2w)!

docs/0-START-HERE/quick-start-local.md

@@ -1,285 +0,0 @@
-# Quick Start - Local & Private (5 minutes)
-
-Get Open Notebook running with **100% local AI** using Ollama. No cloud API keys needed, completely private.
-
-## Prerequisites
-
-1. **Docker Desktop** installed
-   - [Download here](https://www.docker.com/products/docker-desktop/)
-   - Already have it? Skip to step 2
-
-2. **Local LLM** - Choose one:
-   - **Ollama** (recommended): [Download here](https://ollama.ai/)
-   - **LM Studio** (GUI alternative): [Download here](https://lmstudio.ai)
-
-## Step 1: Choose Your Setup (1 min)
-
-### 🏠 Local Machine (Same Computer)
-Everything runs on your machine. Recommended for testing/learning.
-
-### 🌐 Remote Server (Raspberry Pi, NAS, Cloud VM)
-Run on a different computer, access from another. Needs network configuration.
-
----
-
-## Step 2: Create Configuration (1 min)
-
-Create a new folder `open-notebook-local` and add this file:
-
-**docker-compose.yml**:
-```yaml
-services:
-  surrealdb:
-    image: surrealdb/surrealdb:v2
-    command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db
-    ports:
-      - "8000:8000"
-    volumes:
-      - ./surreal_data:/mydata
-
-  open_notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    ports:
-      - "8502:8502"  # Web UI (React frontend)
-      - "5055:5055"  # API (required!)
-    environment:
-      # NO API KEYS NEEDED - Using Ollama (free, local)
-      - OLLAMA_API_BASE=http://ollama:11434
-
-      # Database (required)
-      - SURREAL_URL=ws://surrealdb:8000/rpc
-      - SURREAL_USER=root
-      - SURREAL_PASSWORD=password
-      - SURREAL_NAMESPACE=open_notebook
-      - SURREAL_DATABASE=open_notebook
-    volumes:
-      - ./notebook_data:/app/data
-      - ./surreal_data:/mydata
-    depends_on:
-      - surrealdb
-    restart: always
-
-  ollama:
-    image: ollama/ollama:latest
-    ports:
-      - "11434:11434"
-    volumes:
-      - ./ollama_models:/root/.ollama
-    environment:
-      # Optional: set GPU support if available
-      - OLLAMA_NUM_GPU=0
-    restart: always
-
-```
-
-**That's it!** No API keys, no secrets, completely private.
-
----
-
-## Step 3: Start Services (1 min)
-
-Open terminal in your `open-notebook-local` folder:
-
-```bash
-docker compose up -d
-```
-
-Wait 10-15 seconds for all services to start.
-
----
-
-## Step 4: Download a Model (2-3 min)
-
-Ollama needs at least one language model. Pick one:
-
-```bash
-# Fastest & smallest (recommended for testing)
-docker exec open_notebook-ollama-1 ollama pull mistral
-
-# OR: Better quality but slower
-docker exec open_notebook-ollama-1 ollama pull neural-chat
-
-# OR: Even better quality, more VRAM needed
-docker exec open_notebook-ollama-1 ollama pull llama2
-```
-
-This downloads the model (will take 1-5 minutes depending on your internet).
-
----
-
-## Step 5: Access Open Notebook (instant)
-
-Open your browser:
-```
-http://localhost:8502
-```
-
-You should see the Open Notebook interface.
-
----
-
-## Step 6: Configure Local Model (1 min)
-
-1. Click **Settings** (top right) → **Models**
-2. Set:
-   - **Language Model**: `ollama/mistral` (or whichever model you downloaded)
-   - **Embedding Model**: `ollama/nomic-embed-text` (auto-downloads if missing)
-3. Click **Save**
-
----
-
-## Step 7: Create Your First Notebook (1 min)
-
-1. Click **New Notebook**
-2. Name: "My Private Research"
-3. Click **Create**
-
----
-
-## Step 8: Add Local Content (1 min)
-
-1. Click **Add Source**
-2. Choose **Text**
-3. Paste some text or a local document
-4. Click **Add**
-
----
-
-## Step 9: Chat With Your Content (1 min)
-
-1. Go to **Chat**
-2. Type: "What did you learn from this?"
-3. Click **Send**
-4. Watch as the local Ollama model responds!
-
----
-
-## Verification Checklist
-
-- [ ] Docker is running
-- [ ] You can access `http://localhost:8502`
-- [ ] Models are configured
-- [ ] You created a notebook
-- [ ] Chat works with local model
-
-**All checked?** 🎉 You have a completely **private, offline** research assistant!
-
----
-
-## Advantages of Local Setup
-
-✅ **No API costs** - Free forever
-✅ **No internet required** - True offline capability
-✅ **Privacy first** - Your data never leaves your machine
-✅ **No subscriptions** - No monthly bills
-
-**Trade-off:** Slower than cloud models (depends on your CPU/GPU)
-
----
-
-## Troubleshooting
-
-### "ollama: command not found"
-
-Docker image name might be different:
-```bash
-docker ps  # Find the Ollama container name
-docker exec <container_name> ollama pull mistral
-```
-
-### Model Download Stuck
-
-Check internet connection and restart:
-```bash
-docker compose restart ollama
-```
-
-Then retry the model pull command.
-
-### "Address already in use" Error
-
-```bash
-docker compose down
-docker compose up -d
-```
-
-### Low Performance
-
-Check if GPU is available:
-```bash
-# Show available GPUs
-docker exec open_notebook-ollama-1 ollama ps
-
-# Enable GPU in docker-compose.yml:
-# - OLLAMA_NUM_GPU=1
-```
-
-Then restart: `docker compose restart ollama`
-
-### Adding More Models
-
-```bash
-# List available models
-docker exec open_notebook-ollama-1 ollama list
-
-# Pull additional model
-docker exec open_notebook-ollama-1 ollama pull neural-chat
-```
-
----
-
-## Next Steps
-
-**Now that it's running:**
-
-1. **Add Your Own Content**: PDFs, documents, articles (see 3-USER-GUIDE)
-2. **Explore Features**: Podcasts, transformations, search
-3. **Full Documentation**: [See all features](../3-USER-GUIDE/index.md)
-4. **Scale Up**: Deploy to a server with better hardware for faster responses
-5. **Benchmark Models**: Try different models to find the speed/quality tradeoff you prefer
-
----
-
-## Alternative: Using LM Studio Instead of Ollama
-
-**Prefer a GUI?** LM Studio is easier for non-technical users:
-
-1. Download LM Studio: https://lmstudio.ai
-2. Open the app, download a model from the library
-3. Go to "Local Server" tab, start server (port 1234)
-4. Update your docker-compose.yml:
-   ```yaml
-   environment:
-     - OPENAI_COMPATIBLE_BASE_URL=http://host.docker.internal:1234/v1
-     - OPENAI_COMPATIBLE_API_KEY=not-needed
-   ```
-5. Configure in Settings → Models → Select your LM Studio model
-
-**Note**: LM Studio runs outside Docker, use `host.docker.internal` to connect.
-
----
-
-## Going Further
-
-- **Switch models**: Change in Settings → Models anytime
-- **Add more models**:
-  - Ollama: Run `ollama pull <model>`
-  - LM Studio: Download from the app library
-- **Deploy to server**: Same docker-compose.yml works anywhere
-- **Use cloud hybrid**: Keep some local models, add OpenAI/Anthropic for complex tasks
-
----
-
-## Common Model Choices
-
-| Model | Speed | Quality | VRAM | Best For |
-|-------|-------|---------|------|----------|
-| **mistral** | Fast | Good | 4GB | Testing, general use |
-| **neural-chat** | Medium | Better | 6GB | Balanced, recommended |
-| **llama2** | Slow | Best | 8GB+ | Complex reasoning |
-| **phi** | Very Fast | Fair | 2GB | Minimal hardware |
-
----
-
-**Need Help?** Join our [Discord community](https://discord.gg/37XJPXfz2w) - many users run local setups!

docs/0-START-HERE/quick-start-openai.md

@@ -1,178 +0,0 @@
-# Quick Start - OpenAI (5 minutes)
-
-Get Open Notebook running with OpenAI's GPT models. Fast, powerful, and simple.
-
-## Prerequisites
-
-1. **Docker Desktop** installed
-   - [Download here](https://www.docker.com/products/docker-desktop/)
-   - Already have it? Skip to step 2
-
-2. **OpenAI API Key** (required)
-   - Go to https://platform.openai.com/api-keys
-   - Create account → Create new secret key
-   - Add at least $5 in credits to your account
-   - Copy the key (starts with `sk-`)
-
-## Step 1: Create Configuration (1 min)
-
-Create a new folder `open-notebook` and add this file:
-
-**docker-compose.yml**:
-```yaml
-services:
-  surrealdb:
-    image: surrealdb/surrealdb:v2
-    command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db
-    ports:
-      - "8000:8000"
-    volumes:
-      - ./surreal_data:/mydata
-
-  open_notebook:
-    image: lfnovo/open_notebook:v1-latest
-    pull_policy: always
-    ports:
-      - "8502:8502"  # Web UI
-      - "5055:5055"  # API
-    environment:
-      # Your OpenAI key
-      - OPENAI_API_KEY=sk-...
-
-      # Database (required)
-      - SURREAL_URL=ws://surrealdb:8000/rpc
-      - SURREAL_USER=root
-      - SURREAL_PASSWORD=password
-      - SURREAL_NAMESPACE=open_notebook
-      - SURREAL_DATABASE=open_notebook
-    volumes:
-      - ./notebook_data:/app/data
-    depends_on:
-      - surrealdb
-    restart: always
-
-```
-
-**Edit the file:**
-- Replace `sk-...` with your actual OpenAI API key
-
----
-
-## Step 2: Start Services (1 min)
-
-Open terminal in your `open-notebook` folder:
-
-```bash
-docker compose up -d
-```
-
-Wait 15-20 seconds for services to start.
-
----
-
-## Step 3: Access Open Notebook (instant)
-
-Open your browser:
-```
-http://localhost:8502
-```
-
-You should see the Open Notebook interface!
-
----
-
-## Step 4: Create Your First Notebook (1 min)
-
-1. Click **New Notebook**
-2. Name: "My Research"
-3. Click **Create**
-
----
-
-## Step 5: Add a Source (1 min)
-
-1. Click **Add Source**
-2. Choose **Web Link**
-3. Paste: `https://en.wikipedia.org/wiki/Artificial_intelligence`
-4. Click **Add**
-5. Wait for processing (30-60 seconds)
-
----
-
-## Step 6: Chat With Your Content (1 min)
-
-1. Go to **Chat**
-2. Type: "What is artificial intelligence?"
-3. Click **Send**
-4. Watch as GPT responds with information from your source!
-
----
-
-## Verification Checklist
-
-- [ ] Docker is running
-- [ ] You can access `http://localhost:8502`
-- [ ] You created a notebook
-- [ ] You added a source
-- [ ] Chat works
-
-**All checked?** 🎉 You have a fully working AI research assistant!
-
----
-
-## Using Different Models
-
-In your notebook, go to **Settings** → **Models** to choose:
-- `gpt-4o` - Best quality (recommended)
-- `gpt-4o-mini` - Fast and cheap (good for testing)
-
----
-
-## Troubleshooting
-
-### "Port 8502 already in use"
-
-Change the port in docker-compose.yml:
-```yaml
-ports:
-  - "8503:8502"  # Use 8503 instead
-```
-
-Then access at `http://localhost:8503`
-
-### "API key not working"
-
-1. Double-check your API key (no extra spaces)
-2. Verify you added credits at https://platform.openai.com
-3. Restart: `docker compose restart api`
-
-### "Cannot connect to server"
-
-```bash
-docker ps  # Check all services running
-docker compose logs  # View logs
-docker compose restart  # Restart everything
-```
-
----
-
-## Next Steps
-
-1. **Add Your Own Content**: PDFs, web links, documents
-2. **Explore Features**: Podcasts, transformations, search
-3. **Full Documentation**: [See all features](../3-USER-GUIDE/index.md)
-
----
-
-## Cost Estimate
-
-OpenAI pricing (approximate):
-- **Conversation**: $0.01-0.10 per 1K tokens
-- **Embeddings**: $0.02 per 1M tokens
-- **Typical usage**: $1-5/month for light use, $20-50/month for heavy use
-
-Check https://openai.com/pricing for current rates.
-
----
-
-**Need help?** Join our [Discord community](https://discord.gg/37XJPXfz2w)!

docs/1-INSTALLATION/docker-compose.md

@@ -1,313 +0,0 @@
-# Docker Compose Installation (Recommended)
-
-Multi-container setup with separate services. **Best for most users.**
-
-> **Alternative Registry:** All images are available on both Docker Hub (`lfnovo/open_notebook`) and GitHub Container Registry (`ghcr.io/lfnovo/open-notebook`). Use GHCR if Docker Hub is blocked or you prefer GitHub-native workflows.
-
-## Prerequisites
-
-- **Docker Desktop** installed ([Download](https://www.docker.com/products/docker-desktop/))
-- **5-10 minutes** of your time
-- **API key** for at least one AI provider (OpenAI recommended for beginners)
-
-## Step 1: Get an API Key (2 min)
-
-Choose at least one AI provider. **OpenAI recommended if you're unsure:**
-
-```
-OpenAI:     https://platform.openai.com/api-keys
-Anthropic:  https://console.anthropic.com/
-Google:     https://aistudio.google.com/
-Groq:       https://console.groq.com/
-```
-
-Add at least $5 in credits to your account.
-
-(Skip this if using Ollama for free local models)
-
----
-
-## Step 2: Create Configuration (2 min)
-
-Create a folder `open-notebook` and add this file:
-
-**docker-compose.yml**:
-```yaml
-services:
-  surrealdb:
-    image: surrealdb/surrealdb:v2
-    command: start --user root --pass password --bind 0.0.0.0:8000 rocksdb:/mydata/mydatabase.db
-    ports:
-      - "8000:8000"
-    volumes:
-      - ./surreal_data:/mydata
-
-  open_notebook:
-    image: lfnovo/open_notebook:v1-latest
-    pull_policy: always
-    ports:
-      - "8502:8502"  # Web UI
-      - "5055:5055"  # API
-    environment:
-      # AI Provider (choose ONE)
-      - OPENAI_API_KEY=sk-...  # Your OpenAI key
-      # - ANTHROPIC_API_KEY=sk-ant-...  # Or Anthropic
-      # - GOOGLE_API_KEY=...  # Or Google
-
-      # Database
-      - SURREAL_URL=ws://surrealdb:8000/rpc
-      - SURREAL_USER=root
-      - SURREAL_PASSWORD=password
-      - SURREAL_NAMESPACE=open_notebook
-      - SURREAL_DATABASE=open_notebook
-    volumes:
-      - ./notebook_data:/app/data
-    depends_on:
-      - surrealdb
-    restart: always
-
-```
-
-**Edit the file:**
-- Replace `sk-...` with your actual OpenAI API key
-- (Or use Anthropic, Google, Groq keys instead)
-- If you have multiple keys, uncomment the ones you want
-
----
-
-## Step 3: Start Services (2 min)
-
-Open terminal in the `open-notebook` folder:
-
-```bash
-docker compose up -d
-```
-
-Wait 15-20 seconds for all services to start:
-```
-✅ surrealdb running on :8000
-✅ open_notebook running on :8502 (UI) and :5055 (API)
-```
-
-Check status:
-```bash
-docker compose ps
-```
-
----
-
-## Step 4: Verify Installation (1 min)
-
-**API Health:**
-```bash
-curl http://localhost:5055/health
-# Should return: {"status": "healthy"}
-```
-
-**Frontend Access:**
-Open browser to:
-```
-http://localhost:8502
-```
-
-You should see the Open Notebook interface!
-
----
-
-## Step 5: First Notebook (2 min)
-
-1. Click **New Notebook**
-2. Name: "My Research"
-3. Description: "Getting started"
-4. Click **Create**
-
-Done! You now have a fully working Open Notebook instance. 🎉
-
----
-
-## Configuration
-
-### Using Different AI Providers
-
-Change `environment` section in `docker-compose.yml`:
-
-```yaml
-# For Anthropic (Claude)
-- ANTHROPIC_API_KEY=sk-ant-...
-
-# For Google Gemini
-- GOOGLE_API_KEY=...
-
-# For Groq (fast, free tier available)
-- GROQ_API_KEY=...
-
-# For local Ollama docker container (free, offline) --> Virtual machine
-- OLLAMA_API_BASE=http://ollama:11434
-# For localhost Ollama (free, offline) --> Real machine
-# - OLLAMA_API_BASE=http://host.docker.internal:11434
-```
-
-### Adding Ollama container (Free Local Models)
-
-Add to `docker-compose.yml`:
-
-```yaml
-  ollama:
-    image: ollama/ollama:latest
-    ports:
-      - "11434:11434"
-    volumes:
-      - ollama_models:/root/.ollama
-    restart: always
-
-volumes:
-  surreal_data:
-  ollama_models:
-```
-
-Then update API service:
-```yaml
-environment:
-  - OLLAMA_API_BASE=http://ollama:11434
-```
-
-Restart and pull a model:
-```bash
-docker compose restart
-docker exec open_notebook-ollama-1 ollama pull mistral
-```
-
----
-
-## Environment Variables Reference
-
-| Variable | Purpose | Example |
-|----------|---------|---------|
-| `OPENAI_API_KEY` | OpenAI API key | `sk-proj-...` |
-| `ANTHROPIC_API_KEY` | Anthropic/Claude key | `sk-ant-...` |
-| `SURREAL_URL` | Database connection | `ws://surrealdb:8000/rpc` |
-| `SURREAL_USER` | Database user | `root` |
-| `SURREAL_PASSWORD` | Database password | `password` |
-| `API_URL` | API external URL | `http://localhost:5055` |
-| `NEXT_PUBLIC_API_URL` | Frontend API URL | `http://localhost:5055` |
-
----
-
-## Common Tasks
-
-### Stop Services
-```bash
-docker compose down
-```
-
-### View Logs
-```bash
-# All services
-docker compose logs -f
-
-# Specific service
-docker compose logs -f api
-```
-
-### Restart Services
-```bash
-docker compose restart
-```
-
-### Update to Latest Version
-```bash
-docker compose down
-docker compose pull
-docker compose up -d
-```
-
-### Remove All Data
-```bash
-docker compose down -v
-```
-
----
-
-## Troubleshooting
-
-### "Cannot connect to API" Error
-
-1. Check if Docker is running:
-```bash
-docker ps
-```
-
-2. Check if services are running:
-```bash
-docker compose ps
-```
-
-3. Check API logs:
-```bash
-docker compose logs api
-```
-
-4. Wait longer - services can take 20-30 seconds to start on first run
-
----
-
-### Port Already in Use
-
-If you get "Port 8502 already in use", change the port:
-
-```yaml
-ports:
-  - "8503:8502"  # Use 8503 instead
-  - "5055:5055"  # Keep API port same
-```
-
-Then access at `http://localhost:8503`
-
----
-
-### API Key Not Working
-
-1. Double-check your API key in the file (no extra spaces)
-2. Verify key is valid at provider's website
-3. Check you added credits to your account
-4. Restart: `docker compose restart api`
-
----
-
-### Database Connection Issues
-
-Check SurrealDB is running:
-```bash
-docker compose logs surrealdb
-```
-
-Reset database:
-```bash
-docker compose down -v
-docker compose up -d
-```
-
----
-
-## Next Steps
-
-1. **Add Content**: Sources, notebooks, documents
-2. **Configure Models**: Settings → Models (choose your preferences)
-3. **Explore Features**: Chat, search, transformations
-4. **Read Guide**: [User Guide](../3-USER-GUIDE/index.md)
-
----
-
-## Production Deployment
-
-For production use, see:
-- [Security Hardening](https://github.com/lfnovo/open-notebook/blob/main/docs/deployment/security.md)
-- [Reverse Proxy](https://github.com/lfnovo/open-notebook/blob/main/docs/deployment/reverse-proxy.md)
-
----
-
-## Getting Help
-
-- **Discord**: [Community support](https://discord.gg/37XJPXfz2w)
-- **Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)
-- **Docs**: [Full documentation](../index.md)

docs/1-INSTALLATION/from-source.md

@@ -1,153 +0,0 @@
-# From Source Installation
-
-Clone the repository and run locally. **For developers and contributors.**
-
-## Prerequisites
-
-- **Python 3.11+** - [Download](https://www.python.org/)
-- **Node.js 18+** - [Download](https://nodejs.org/)
-- **Git** - [Download](https://git-scm.com/)
-- **Docker** (for SurrealDB) - [Download](https://docker.com/)
-- **uv** (Python package manager) - `curl -LsSf https://astral.sh/uv/install.sh | sh`
-- API key from OpenAI or similar (or use Ollama for free)
-
-## Quick Setup (10 minutes)
-
-### 1. Clone Repository
-
-```bash
-git clone https://github.com/lfnovo/open-notebook.git
-cd open-notebook
-
-# If you forked it:
-git clone https://github.com/YOUR_USERNAME/open-notebook.git
-cd open-notebook
-git remote add upstream https://github.com/lfnovo/open-notebook.git
-```
-
-### 2. Install Python Dependencies
-
-```bash
-uv sync
-uv pip install python-magic
-```
-
-### 3. Start SurrealDB
-
-```bash
-# Terminal 1
-make database
-# or: docker compose up surrealdb
-```
-
-### 4. Set Environment Variables
-
-```bash
-cp .env.example .env
-# Edit .env and add your API key:
-# OPENAI_API_KEY=sk-...
-# (or ANTHROPIC_API_KEY, GROQ_API_KEY, etc.)
-```
-
-### 5. Start API
-
-```bash
-# Terminal 2
-make api
-# or: uv run --env-file .env uvicorn api.main:app --host 0.0.0.0 --port 5055
-```
-
-### 6. Start Frontend
-
-```bash
-# Terminal 3
-cd frontend && npm install && npm run dev
-```
-
-### 7. Access
-
-- **Frontend**: http://localhost:3000
-- **API Docs**: http://localhost:5055/docs
-- **Database**: http://localhost:8000
-
----
-
-## Development Workflow
-
-### Code Quality
-
-```bash
-# Format and lint Python
-make ruff
-# or: ruff check . --fix
-
-# Type checking
-make lint
-# or: uv run python -m mypy .
-```
-
-### Run Tests
-
-```bash
-uv run pytest tests/
-```
-
-### Common Commands
-
-```bash
-# Start everything
-make start-all
-
-# View API docs
-open http://localhost:5055/docs
-
-# Check database migrations
-# (Auto-run on API startup)
-
-# Clean up
-make clean
-```
-
----
-
-## Troubleshooting
-
-### Python version too old
-
-```bash
-python --version  # Check version
-uv sync --python 3.11  # Use specific version
-```
-
-### npm: command not found
-
-Install Node.js from https://nodejs.org/
-
-### Database connection errors
-
-```bash
-docker ps  # Check SurrealDB running
-docker logs surrealdb  # View logs
-```
-
-### Port 5055 already in use
-
-```bash
-# Use different port
-uv run uvicorn api.main:app --port 5056
-```
-
----
-
-## Next Steps
-
-1. Read [Development Guide](../7-DEVELOPMENT/quick-start.md)
-2. See [Architecture Overview](../7-DEVELOPMENT/architecture.md)
-3. Check [Contributing Guide](../7-DEVELOPMENT/contributing.md)
-
----
-
-## Getting Help
-
-- **Discord**: [Community](https://discord.gg/37XJPXfz2w)
-- **Issues**: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)

docs/1-INSTALLATION/index.md

@@ -1,153 +0,0 @@
-# Installation Guide
-
-Choose your installation route based on your setup and use case.
-
-## Quick Decision: Which Route?
-
-### 🚀 I want the easiest setup (Recommended for most)
-**→ [Docker Compose](docker-compose.md)** - Multi-container setup, production-ready
-- ✅ All features working
-- ✅ Clear separation of services
-- ✅ Easy to scale
-- ✅ Works on Mac, Windows, Linux
-- ⏱️ 5 minutes to running
-
----
-
-### 🏠 I want everything in one container (Simplified)
-**→ [Single Container](single-container.md)** - All-in-one for simple deployments
-- ✅ Minimal configuration
-- ✅ Lower resource usage
-- ✅ Good for shared hosting
-- ✅ Works on PikaPods, Railway, etc.
-- ⏱️ 3 minutes to running
-
----
-
-### 👨‍💻 I want to develop/contribute (Developers only)
-**→ [From Source](from-source.md)** - Clone repo, set up locally
-- ✅ Full control over code
-- ✅ Easy to debug
-- ✅ Can modify and test
-- ⚠️ Requires Python 3.11+, Node.js
-- ⏱️ 10 minutes to running
-
----
-
-
-## System Requirements
-
-### Minimum
-- **RAM**: 4GB
-- **Storage**: 2GB for app + space for documents
-- **CPU**: Any modern processor
-- **Network**: Internet (optional for offline setup)
-
-### Recommended
-- **RAM**: 8GB+
-- **Storage**: 10GB+ for documents and models
-- **CPU**: Multi-core processor
-- **GPU**: Optional (speeds up local AI models)
-
----
-
-## AI Provider Options
-
-### Cloud-Based (Pay-as-you-go)
-- **OpenAI** - GPT-4, GPT-4o, fast and capable
-- **Anthropic (Claude)** - Claude 3.5 Sonnet, excellent reasoning
-- **Google Gemini** - Multimodal, cost-effective
-- **Groq** - Ultra-fast inference
-- **Others**: Mistral, DeepSeek, xAI, OpenRouter
-
-**Cost**: Usually $0.01-$0.10 per 1K tokens
-**Speed**: Fast (sub-second)
-**Privacy**: Your data sent to cloud
-
-### Local (Free, Private)
-- **Ollama** - Run open-source models locally
-- **LM Studio** - Desktop app for local models
-- **Hugging Face models** - Download and run
-
-**Cost**: $0 (just electricity)
-**Speed**: Depends on your hardware (slow to medium)
-**Privacy**: 100% offline
-
----
-
-## Choose a Route
-
-**Already know which way to go?** Pick your installation path:
-
-- [Docker Compose](docker-compose.md) - **Most users**
-- [Single Container](single-container.md) - **Shared hosting**
-- [From Source](from-source.md) - **Developers**
-
-> **Privacy-first?** Any installation method works with Ollama for 100% local AI. See [Local Quick Start](../0-START-HERE/quick-start-local.md).
-
----
-
-## Pre-Installation Checklist
-
-Before installing, you'll need:
-
-- [ ] **Docker** (for Docker routes) or **Node.js 18+** (for source)
-- [ ] **AI Provider API key** (OpenAI, Anthropic, etc.) OR willingness to use free local models
-- [ ] **At least 4GB RAM** available
-- [ ] **Stable internet** (or offline setup with Ollama)
-
----
-
-## Detailed Installation Instructions
-
-### For Docker Users
-1. Install [Docker Desktop](https://docker.com/products/docker-desktop)
-2. Choose: [Docker Compose](docker-compose.md) or [Single Container](single-container.md)
-3. Follow the step-by-step guide
-4. Access at `http://localhost:8502`
-
-### For Source Installation (Developers)
-1. Have Python 3.11+, Node.js 18+, Git installed
-2. Follow [From Source](from-source.md)
-3. Run `make start-all`
-4. Access at `http://localhost:8502` (frontend) or `http://localhost:5055` (API)
-
----
-
-## After Installation
-
-Once you're up and running:
-
-1. **Configure Models** - Choose your AI provider in Settings
-2. **Create First Notebook** - Start organizing research
-3. **Add Sources** - PDFs, web links, documents
-4. **Explore Features** - Chat, search, transformations
-5. **Read Full Guide** - [User Guide](../3-USER-GUIDE/index.md)
-
----
-
-## Troubleshooting During Installation
-
-**Having issues?** Check the troubleshooting section in your chosen installation guide, or see [Quick Fixes](../6-TROUBLESHOOTING/quick-fixes.md).
-
----
-
-## Need Help?
-
-- **Discord**: [Join community](https://discord.gg/37XJPXfz2w)
-- **GitHub Issues**: [Report problems](https://github.com/lfnovo/open-notebook/issues)
-- **Docs**: See [Full Documentation](../index.md)
-
----
-
-## Production Deployment
-
-Installing for production use? See additional resources:
-
-- [Security Hardening](https://github.com/lfnovo/open-notebook/blob/main/docs/deployment/security.md)
-- [Reverse Proxy Setup](https://github.com/lfnovo/open-notebook/blob/main/docs/deployment/reverse-proxy.md)
-- [Performance Tuning](https://github.com/lfnovo/open-notebook/blob/main/docs/deployment/retry-configuration.md)
-
----
-
-**Ready to install?** Pick a route above! ⬆️

docs/1-INSTALLATION/single-container.md

@@ -1,122 +0,0 @@
-# Single Container Installation
-
-All-in-one container setup. **Simpler than Docker Compose, but less flexible.**
-
-**Best for:** PikaPods, Railway, shared hosting, minimal setups
-
-> **Alternative Registry:** Images available on both Docker Hub (`lfnovo/open_notebook:v1-latest-single`) and GitHub Container Registry (`ghcr.io/lfnovo/open-notebook:v1-latest-single`).
-
-> ⚠️ **Note**: While this is a simple way to get started, we recommend [Docker Compose](docker-compose.md) for most users. Docker Compose is more flexible and will make it easier if we add more services to the setup in the future. This single-container option is best for platforms that specifically require it (PikaPods, Railway, etc.).
-
-## Prerequisites
-
-- Docker installed (for local testing)
-- API key from OpenAI, Anthropic, or another provider
-- 5 minutes
-
-## Quick Setup
-
-### For Local Testing (Docker)
-
-```yaml
-# docker-compose.yml
-services:
-  open_notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    ports:
-      - "8502:8502"  # Web UI (React frontend)
-      - "5055:5055"  # API
-    environment:
-      - OPENAI_API_KEY=sk-...
-      - SURREAL_URL=ws://localhost:8000/rpc
-      - SURREAL_USER=root
-      - SURREAL_PASSWORD=password
-      - SURREAL_NAMESPACE=open_notebook
-      - SURREAL_DATABASE=open_notebook
-    volumes:
-      - ./data:/app/data
-    restart: always
-```
-
-Run:
-```bash
-docker compose up -d
-```
-
-Access: `http://localhost:8502`
-
-### For Cloud Platforms
-
-**PikaPods:**
-1. Click "New App"
-2. Search "Open Notebook"
-3. Set environment variables
-4. Click "Deploy"
-
-**Railway:**
-1. Create new project
-2. Add `lfnovo/open_notebook:v1-latest-single`
-3. Set environment variables
-4. Deploy
-
-**Render:**
-1. Create new Web Service
-2. Use Docker image: `lfnovo/open_notebook:v1-latest-single`
-3. Set environment variables in dashboard
-4. Configure persistent disk for `/app/data` and `/mydata`
-
-**DigitalOcean App Platform:**
-1. Create new app from Docker Hub
-2. Use image: `lfnovo/open_notebook:v1-latest-single`
-3. Set port to 8502
-4. Add environment variables
-5. Configure persistent storage
-
-**Heroku:**
-```bash
-# Using heroku.yml
-heroku container:push web
-heroku container:release web
-heroku config:set OPENAI_API_KEY=sk-...
-```
-
-**Coolify:**
-1. Add new service → Docker Image
-2. Image: `lfnovo/open_notebook:v1-latest-single`
-3. Port: 8502
-4. Add environment variables
-5. Enable persistent volumes
-6. Coolify handles HTTPS automatically
-
----
-
-## Environment Variables
-
-| Variable | Purpose | Example |
-|----------|---------|---------|
-| `OPENAI_API_KEY` | API key | `sk-...` |
-| `SURREAL_URL` | Database | `ws://localhost:8000/rpc` |
-| `SURREAL_USER` | DB user | `root` |
-| `SURREAL_PASSWORD` | DB password | `password` |
-| `API_URL` | External URL (for remote access) | `https://myapp.example.com` |
-
----
-
-## Limitations vs Docker Compose
-
-| Feature | Single Container | Docker Compose |
-|---------|------------------|-----------------|
-| Setup time | 2 minutes | 5 minutes |
-| Complexity | Minimal | Moderate |
-| Services | All bundled | Separated |
-| Scalability | Limited | Excellent |
-| Memory usage | ~800MB | ~1.2GB |
-
----
-
-## Next Steps
-
-Same as Docker Compose setup - just access via `http://localhost:8502` (local) or your platform's URL (cloud).
-
-See [Docker Compose](docker-compose.md) for full post-install guide.

docs/2-CORE-CONCEPTS/ai-context-rag.md

@@ -1,450 +0,0 @@
-# AI Context & RAG - How Open Notebook Uses Your Research
-
-Open Notebook uses different approaches to make AI models aware of your research depending on the feature. This section explains **RAG** (used in Ask) and **full-content context** (used in Chat).
-
----
-
-## The Problem: Making AI Aware of Your Data
-
-### Traditional Approaches (and their problems)
-
-**Option 1: Fine-Tuning**
-- Train the model on your data
-- Pro: Model becomes specialized
-- Con: Expensive, slow, permanent (can't unlearn)
-
-**Option 2: Send Everything to Cloud**
-- Upload all your data to ChatGPT/Claude API
-- Pro: Works well, fast
-- Con: Privacy nightmare, data leaves your control, expensive
-
-**Option 3: Ignore Your Data**
-- Just use the base model without your research
-- Pro: Private, free
-- Con: AI doesn't know anything about your specific topic
-
-### Open Notebook's Dual Approach
-
-**For Chat**: Sends the entire selected content to the LLM
-- Simple and transparent: You select sources, they're sent in full
-- Maximum context: AI sees everything you choose
-- You control which sources are included
-
-**For Ask (RAG)**: Retrieval-Augmented Generation
-- RAG = Retrieval-Augmented Generation
-- The insight: *Search your content, find relevant pieces, send only those*
-- Automatic: AI decides what's relevant based on your question
-
----
-
-## How RAG Works: Three Stages
-
-### Stage 1: Content Preparation
-
-When you upload a source, Open Notebook prepares it for retrieval:
-
-```
-1. EXTRACT TEXT
-   PDF → text
-   URL → webpage text
-   Audio → transcribed text
-   Video → subtitles + transcription
-
-2. CHUNK INTO PIECES
-   Long documents → break into ~500-word chunks
-   Why? AI context has limits; smaller pieces are more precise
-
-3. CREATE EMBEDDINGS
-   Each chunk → semantic vector (numbers representing meaning)
-   Why? Allows finding chunks by similarity, not just keywords
-
-4. STORE IN DATABASE
-   Chunks + embeddings + metadata → searchable storage
-```
-
-**Example:**
-```
-Source: "AI Safety Research 2026" (50-page PDF)
-↓
-Extracted: 50 pages of text
-↓
-Chunked: 150 chunks (~500 words each)
-↓
-Embedded: Each chunk gets a vector (1536 numbers for OpenAI)
-↓
-Stored: Ready for search
-```
-
----
-
-### Stage 2: Query Time (What You Search For)
-
-When you ask a question, the system finds relevant content:
-
-```
-1. YOU ASK A QUESTION
-   "What does the paper say about alignment?"
-
-2. SYSTEM CONVERTS QUESTION TO EMBEDDING
-   Your question → vector (same way chunks are vectorized)
-
-3. SIMILARITY SEARCH
-   Find chunks most similar to your question
-   (using vector math, not keyword matching)
-
-4. RETURN TOP RESULTS
-   Usually top 5-10 most similar chunks
-
-5. YOU GET BACK
-   ✓ The relevant chunks
-   ✓ Where they came from (sources + page numbers)
-   ✓ Relevance scores
-```
-
-**Example:**
-```
-Q: "What does the paper say about alignment?"
-↓
-Q vector: [0.23, -0.51, 0.88, ..., 0.12]
-↓
-Search: Compare to all chunk vectors
-↓
-Results:
-  - Chunk 47 (alignment section): similarity 0.94
-  - Chunk 63 (safety approaches): similarity 0.88
-  - Chunk 12 (related work): similarity 0.71
-```
-
----
-
-### Stage 3: Augmentation (How AI Uses It)
-
-Now you have the relevant pieces. The AI uses them:
-
-```
-SYSTEM BUILDS A PROMPT:
-  "You are an AI research assistant.
-
-   The user has the following research materials:
-   [CHUNK 47 CONTENT]
-   [CHUNK 63 CONTENT]
-
-   User question: 'What does the paper say about alignment?'
-
-   Answer based on the above materials."
-
-AI RESPONDS:
-  "Based on the research materials, the paper approaches
-   alignment through [pulls from chunks] and emphasizes
-   [pulls from chunks]..."
-
-SYSTEM ADDS CITATIONS:
-  "- See research materials page 15 for approach details
-   - See research materials page 23 for emphasis on X"
-```
-
----
-
-## Two Search Modes: Exact vs. Semantic
-
-Open Notebook provides two different search strategies for different goals.
-
-### 1. Text Search (Keyword Matching)
-
-**How it works:**
-- Uses BM25 ranking (the same algorithm Google uses)
-- Finds chunks containing your keywords
-- Ranks by relevance (how often keywords appear, position, etc.)
-
-**When to use:**
-- "I remember the exact phrase 'X' and want to find it"
-- "I'm looking for a specific name or number"
-- "I need the exact quote"
-
-**Example:**
-```
-Search: "transformer architecture"
-Results:
-  1. Chunk with "transformer architecture" 3 times
-  2. Chunk with "transformer" and "architecture" separately
-  3. Chunk with "transformer-based models"
-```
-
-### 2. Vector Search (Semantic Similarity)
-
-**How it works:**
-- Converts your question to a vector (number embedding)
-- Finds chunks with similar vectors
-- No keywords needed—finds conceptually similar content
-
-**When to use:**
-- "Find content about X (without saying exact words)"
-- "I'm exploring a concept"
-- "Find similar ideas even if worded differently"
-
-**Example:**
-```
-Search: "what's the mechanism for model understanding?"
-Results (no "understanding" in any chunk):
-  1. Chunk about interpretability and mechanistic analysis
-  2. Chunk about feature analysis
-  3. Chunk about attention mechanisms
-
-Why? The vectors are semantically similar to your concept.
-```
-
----
-
-## Context Management: Your Control Panel
-
-Here's where Open Notebook is different: **You decide what the AI sees.**
-
-### The Three Levels
-
-| Level | What's Shared | Example Cost | Privacy | Use Case |
-|-------|---------------|--------------|---------|----------|
-| **Full Content** | Complete source text | 10,000 tokens | Low | Detailed analysis, close reading |
-| **Summary Only** | AI-generated summary | 2,000 tokens | High | Background material, references |
-| **Not in Context** | Nothing | 0 tokens | Max | Confidential, irrelevant, or archived |
-
-### How It Works
-
-**Full Content:**
-```
-You: "What's the methodology in paper A?"
-System:
-  - Searches paper A
-  - Retrieves full paper content (or large chunks)
-  - Sends to AI: "Here's paper A. Answer about methodology."
-  - AI analyzes complete content
-  - Result: Detailed, precise answer
-```
-
-**Summary Only:**
-```
-You: "I want to chat using paper A and B"
-System:
-  - For Paper A: Sends AI-generated summary (not full text)
-  - For Paper B: Sends full content (detailed analysis)
-  - AI sees 2 sources but in different detail levels
-  - Result: Uses summaries for context, details for focused content
-```
-
-**Not in Context:**
-```
-You: "I have 10 sources but only want 5 in context"
-System:
-  - Paper A-E: In context (sent to AI)
-  - Paper F-J: Not in context (AI can't see them, doesn't search them)
-  - AI never knows these 5 sources exist
-  - Result: Tight, focused context
-```
-
-### Why This Matters
-
-**Privacy**: You control what leaves your system
-```
-Scenario: Confidential company docs + public research
-Control: Public research in context → Confidential docs excluded
-Result: AI never sees confidential content
-```
-
-**Cost**: You control token usage
-```
-Scenario: 100 sources for background + 5 for detailed analysis
-Control: Full content for 5 detailed, summaries for 95 background
-Result: 80% lower token cost than sending everything
-```
-
-**Quality**: You control what the AI focuses on
-```
-Scenario: 20 sources, question requires deep analysis
-Control: Full content for relevant source, exclude others
-Result: AI doesn't get distracted; gives better answer
-```
-
----
-
-## The Difference: Chat vs. Ask
-
-**IMPORTANT**: These use completely different approaches!
-
-### Chat: Full-Content Context (NO RAG)
-
-**How it works:**
-```
-YOU:
-  1. Select which sources to include in context
-  2. Set context level (full/summary/excluded)
-  3. Ask question
-
-SYSTEM:
-  - Takes ALL selected sources (respecting context levels)
-  - Sends the ENTIRE content to the LLM at once
-  - NO search, NO retrieval, NO chunking
-  - AI sees everything you selected
-
-AI:
-  - Responds based on the full content you provided
-  - Can reference any part of selected sources
-  - Conversational: context stays for follow-ups
-```
-
-**Use this when**:
-- You know which sources are relevant
-- You want conversational back-and-forth
-- You want AI to see the complete context
-- You're doing close reading or analysis
-
-**Advantages:**
-- Simple and transparent
-- AI sees everything (no missed content)
-- Conversational flow
-
-**Limitations:**
-- Limited by LLM context window
-- You must manually select relevant sources
-- Sends more tokens (higher cost with many sources)
-
----
-
-### Ask: RAG - Automatic Retrieval
-
-**How it works:**
-```
-YOU:
-  Ask one complex question
-
-SYSTEM:
-  1. Analyzes your question
-  2. Searches across ALL your sources automatically
-  3. Finds relevant chunks using vector similarity
-  4. Retrieves only the most relevant pieces
-  5. Sends ONLY those chunks to the LLM
-  6. Synthesizes into comprehensive answer
-
-AI:
-  - Sees ONLY the retrieved chunks (not full sources)
-  - Answers based on what was found to be relevant
-  - One-shot answer (not conversational)
-```
-
-**Use this when**:
-- You have many sources and don't know which are relevant
-- You want the AI to search automatically
-- You need a comprehensive answer to a complex question
-- You want to minimize tokens sent to LLM
-
-**Advantages:**
-- Automatic search (you don't pick sources)
-- Works across many sources at once
-- Cost-effective (sends only relevant chunks)
-
-**Limitations:**
-- Not conversational (single question/answer)
-- AI only sees retrieved chunks (might miss context)
-- Search quality depends on how well question matches content
-
----
-
-## What This Means: Privacy by Design
-
-Open Notebook's RAG approach gives you something you don't get with ChatGPT or Claude directly:
-
-**You control the boundary between:**
-- What stays private (on your system)
-- What goes to AI (explicitly chosen)
-- What the AI can see (context levels)
-
-### The Audit Trail
-
-Because everything is retrieved explicitly, you can ask:
-- "Which sources did the AI use for this answer?" → See citations
-- "What exactly did the AI see?" → See chunks in context level
-- "Is the AI's claim actually in my sources?" → Verify citation
-
-This prevents hallucinations or misrepresentation better than most systems.
-
----
-
-## How Embeddings Work (Simplified)
-
-The magic of semantic search comes from embeddings. Here's the intuition:
-
-### The Idea
-Instead of storing text, store it as a list of numbers (vectors) that represent "meaning."
-
-```
-Chunk: "The transformer uses attention mechanisms"
-Vector: [0.23, -0.51, 0.88, 0.12, ..., 0.34]
-        (1536 numbers for OpenAI)
-
-Another chunk: "Attention allows models to focus on relevant parts"
-Vector: [0.24, -0.48, 0.87, 0.15, ..., 0.35]
-        (similar numbers = similar meaning!)
-```
-
-### Why This Works
-Words that are semantically similar produce similar vectors. So:
-- "alignment" and "interpretability" have similar vectors
-- "transformer" and "attention" have related vectors
-- "cat" and "dog" are more similar than "cat" and "radiator"
-
-### How Search Works
-```
-Your question: "How do models understand their decisions?"
-Question vector: [0.25, -0.50, 0.86, 0.14, ..., 0.33]
-
-Compare to all stored vectors. Find the most similar:
-- Chunk about interpretability: similarity 0.94
-- Chunk about explainability: similarity 0.91
-- Chunk about feature attribution: similarity 0.88
-
-Return the top matches.
-```
-
-This is why semantic search finds conceptually similar content even when words are different.
-
----
-
-## Key Design Decisions
-
-### 1. Search, Don't Train
-**Why?** Fine-tuning is slow and permanent. Search is flexible and reversible.
-
-### 2. Explicit Retrieval, Not Implicit Knowledge
-**Why?** You can verify what the AI saw. You have audit trails. You control what leaves your system.
-
-### 3. Multiple Search Types
-**Why?** Different questions need different search (keyword vs. semantic). Giving you both is more powerful.
-
-### 4. Context as a Permission System
-**Why?** Not everything you save needs to reach AI. You control granularly.
-
----
-
-## Summary
-
-Open Notebook gives you **two ways** to work with AI:
-
-### Chat (Full-Content)
-- Sends entire selected sources to LLM
-- Manual control: you pick sources
-- Conversational: back-and-forth dialog
-- Transparent: you know exactly what AI sees
-- Best for: focused analysis, close reading
-
-### Ask (RAG)
-- Searches and retrieves relevant chunks automatically
-- Automatic: AI finds what's relevant
-- One-shot: single comprehensive answer
-- Efficient: sends only relevant pieces
-- Best for: broad questions across many sources
-
-**Both approaches:**
-1. Keep your data private (doesn't leave your system by default)
-2. Give you control (you choose which features to use)
-3. Create audit trails (citations show what was used)
-4. Support multiple AI providers
-
-**Coming Soon**: The community is working on adding RAG capabilities to Chat as well, giving you the best of both worlds.

docs/2-CORE-CONCEPTS/chat-vs-transformations.md

@@ -1,353 +0,0 @@
-# Chat vs. Ask vs. Transformations - Which Tool for Which Job?
-
-Open Notebook offers different ways to work with your research. Understanding when to use each is key to using the system effectively.
-
----
-
-## The Three Interaction Modes
-
-### 1. CHAT - Conversational Exploration with Manual Context
-
-**What it is:** Have a conversation with AI about selected sources.
-
-**The flow:**
-```
-1. You select which sources to include ("in context")
-2. You ask a question
-3. AI responds using ONLY those sources
-4. You ask follow-up questions (context stays same)
-5. You change sources or context level, then continue
-```
-
-**Context management:** You explicitly choose which sources the AI can see.
-
-**Conversational:** Multiple questions with shared history.
-
-**Example:**
-```
-You: [Select sources: "paper1.pdf", "research_notes.txt"]
-     [Set context: Full content for paper1, Summary for notes]
-
-You: "What's the main argument in these sources?"
-AI:  "Paper 1 argues X [citation]. Your notes emphasize Y [citation]."
-
-You: "How do they differ?"
-AI:  "Paper 1 focuses on X [citation], while your notes highlight Y [citation]..."
-
-You: [Now select different sources]
-
-You: "Compare to this other perspective"
-AI:  "This new source takes a different approach..."
-```
-
-**Best for:**
-- Exploring a focused topic with specific sources
-- Having a dialogue (multiple back-and-forth questions)
-- When you know which sources matter
-- When you want tight control over what goes to AI
-
----
-
-### 2. ASK - Automated Comprehensive Search
-
-**What it is:** Ask one complex question, system automatically finds relevant content.
-
-**The flow:**
-```
-1. You ask a comprehensive question
-2. System analyzes the question
-3. System automatically searches your sources
-4. System retrieves relevant chunks
-5. System synthesizes answer from all results
-6. You get one detailed answer (not conversational)
-```
-
-**Context management:** Automatic. System figures out what's relevant.
-
-**Non-conversational:** One question → one answer. No follow-ups.
-
-**Example:**
-```
-You: "How do these papers compare their approaches to alignment?
-      What does each one recommend?"
-
-System:
-  - Breaks down the question into search strategies
-  - Searches all sources for alignment approaches
-  - Searches all sources for recommendations
-  - Retrieves top 10 relevant chunks
-  - Synthesizes: "Paper A recommends X [citation].
-                  Paper B recommends Y [citation].
-                  They differ in Z."
-
-You: [Get back one comprehensive answer]
-     [If you want to follow up, use Chat instead]
-```
-
-**Best for:**
-- Comprehensive, one-time questions
-- Comparing multiple sources at once
-- When you want the system to decide what's relevant
-- Complex questions that need multiple search angles
-- When you don't need a back-and-forth conversation
-
----
-
-### 3. TRANSFORMATIONS - Template-Based Processing
-
-**What it is:** Apply a reusable template to a source and get structured output.
-
-**The flow:**
-```
-1. You define a transformation (or choose a preset)
-   "Extract: main argument, methodology, limitations"
-
-2. You apply it to ONE source at a time
-   (You can repeat for other sources)
-
-3. For the source:
-   - Source content + transformation prompt → AI
-   - Result stored as new insight/note
-
-4. You get back
-   - Structured output (main argument, methodology, limitations)
-   - Saved as a note in your notebook
-```
-
-**Context management:** Works on one source at a time.
-
-**Reusable:** Apply the same template to different sources (one by one).
-
-**Note**: Currently processes one source at a time. Batch processing (multiple sources at once) is planned for a future release.
-
-**Example:**
-```
-You: Define transformation
-     "For each academic paper, extract:
-      - Main research question
-      - Methodology used
-      - Key findings
-      - Limitations and gaps
-      - Recommended next research"
-
-You: Apply to paper 1
-
-System:
-  - Runs the transformation on paper 1
-  - Result stored as new note
-
-You: Apply same transformation to paper 2, 3, etc.
-
-After 10 papers:
-  - You have 10 structured notes with consistent format
-  - Perfect for writing a literature review or comparison
-```
-
-**Best for:**
-- Extracting the same information from each source (run repeatedly)
-- Creating structured summaries with consistent format
-- Building a knowledge base of categorized insights
-- When you want reusable templates you can apply to each source
-
----
-
-## Decision Tree: Which Tool to Use?
-
-```
-What are you trying to do?
-
-│
-├─→ "I want to have a conversation about this topic"
-│   └─→ Is the conversation exploratory or fixed?
-│       ├─→ Exploratory (I'll ask follow-ups)
-│       │   └─→ USE: CHAT
-│       │
-│       └─→ Fixed (One question → done)
-│           └─→ Go to next question
-│
-├─→ "I need to compare these sources or get a comprehensive answer"
-│   └─→ USE: ASK
-│
-├─→ "I want to extract the same info from each source (one at a time)"
-│   └─→ USE: TRANSFORMATIONS (apply to each source)
-│
-└─→ "I just want to read and search"
-    └─→ USE: Search (text or vector)
-        OR read your notes
-```
-
----
-
-## Side-by-Side Comparison
-
-| Aspect | CHAT | ASK | TRANSFORMATIONS |
-|--------|------|-----|-----------------|
-| **What's it for?** | Conversational exploration | Comprehensive Q&A | Template-based extraction |
-| **# of questions** | Multiple (conversational) | One | One template per source |
-| **Context control** | Manual (you choose) | Automatic (system searches) | One source at a time |
-| **Conversational?** | Yes (follow-ups work) | No (one question only) | No (single operation) |
-| **Output** | Natural conversation | Natural answer | Structured note |
-| **Time** | Quick (back-and-forth) | Longer (comprehensive) | Per source |
-| **Best when** | Exploring & uncertain | Need full picture | Want consistent format |
-| **Model speed** | Any | Fast preferred | Any |
-
----
-
-## Workflow Examples
-
-### Example 1: Academic Research
-
-```
-Goal: Write literature review from 15 papers
-
-Step 1: TRANSFORMATIONS
-  - Define: "Extract abstract, methodology, findings, relevance"
-  - Apply to paper 1 → get structured note
-  - Apply to paper 2 → get structured note
-  - ... repeat for all 15 papers
-  - Result: 15 structured notes with consistent format
-
-Step 2: Read the notes
-  - Now you have consistent summaries
-
-Step 3: CHAT or ASK
-  - Chat: "Help me organize these by theme"
-  - Ask: "What are the common methodologies across these papers?"
-
-Step 4: Write your review
-  - Use the transformations as foundation
-  - Use chat/ask insights for structure
-```
-
-### Example 2: Product Research
-
-```
-Goal: Understand customer feedback from interviews
-
-Step 1: Add sources (interview transcripts)
-
-Step 2: ASK
-  - "What are the top 10 pain points mentioned?"
-  - Get comprehensive answer with citations
-
-Step 3: CHAT
-  - "Can you help me group these by severity?"
-  - Continue conversation to prioritize
-
-Step 4: TRANSFORMATIONS (optional)
-  - Define: "Extract: pain point, frequency, who mentioned it"
-  - Apply to each interview (one by one)
-  - Get structured data for analysis
-```
-
-### Example 3: Policy Analysis
-
-```
-Goal: Compare policy documents
-
-Step 1: Add all policy documents as sources
-
-Step 2: ASK
-  - "How do these policies differ on climate measures?"
-  - System searches all docs, gives comprehensive comparison
-
-Step 3: CHAT (if needed)
-  - "Which policy is most aligned with X goals?"
-  - Have discussion about trade-offs
-
-Step 4: Export notes
-  - Save AI responses as notes for reports
-```
-
----
-
-## Context Management: The Control Panel
-
-All three modes let you control what the AI sees.
-
-### In CHAT and TRANSFORMATIONS
-```
-You choose:
-  - Which sources to include
-  - Context level for each:
-    ✓ Full Content (send complete text)
-    ✓ Summary Only (send AI summary, not full text)
-    ✓ Not in Context (exclude entirely)
-
-Example:
-  Paper A: Full Content (analyzing closely)
-  Paper B: Summary Only (background)
-  Paper C: Not in Context (confidential)
-```
-
-### In ASK
-```
-Context is automatic:
-  - System searches ALL your sources
-  - Retrieves most relevant chunks
-  - Sends those to AI
-
-But you can:
-  - Search in specific notebook
-  - Filter by source type
-  - Use the results to decide context for follow-up Chat
-```
-
----
-
-## Model Selection
-
-Each mode works with different models:
-
-### CHAT
-- **Any model** works fine
-- Fast models (GPT-4o mini, Claude Haiku): Quick responses, good for conversation
-- Powerful models (GPT-4o, Claude Sonnet): Better reasoning, better for complex topics
-
-### ASK
-- **Fast models preferred** (because it processes multiple searches)
-- Can use powerful models if you want deep synthesis
-- Example: GPT-4 for strategy planning, GPT-4o-mini for quick facts
-
-### TRANSFORMATIONS
-- **Any model** works
-- Fast models (cost-effective for batch processing)
-- Powerful models (better quality extractions)
-
----
-
-## Advanced: Chaining Modes Together
-
-You can combine these modes:
-
-```
-TRANSFORMATIONS → CHAT
-  1. Use transformations to extract structured data
-  2. Use chat to discuss the results
-
-ASK → TRANSFORMATIONS
-  1. Use Ask to understand what matters
-  2. Use Transformations to extract it from remaining sources
-
-CHAT → Save as Note → TRANSFORMATIONS
-  1. Have conversation (Chat)
-  2. Save good responses as notes
-  3. Use those notes as context for transformations
-```
-
----
-
-## Summary: When to Use Each
-
-| Situation | Use | Why |
-|-----------|-----|-----|
-| "I want to explore a topic with follow-up questions" | **CHAT** | Conversational, you control context |
-| "I need a comprehensive answer to one complex question" | **ASK** | Automatic search, synthesized answer |
-| "I want consistent summaries from each source" | **TRANSFORMATIONS** | Template reuse, apply to each source |
-| "I'm comparing two specific sources" | **CHAT** | Select just those 2, have discussion |
-| "I need to categorize each source by X criteria" | **TRANSFORMATIONS** | Extract category from each source |
-| "I want to understand the big picture across all sources" | **ASK** | Automatic comprehensive search |
-| "I want to build a knowledge base" | **TRANSFORMATIONS** | Create structured note from each source |
-| "I want to iterate on understanding" | **CHAT** | Multiple questions, refine thinking |
-
-The key insight: **Different questions need different tools.** Open Notebook gives you all three because research rarely fits one mode.

docs/2-CORE-CONCEPTS/index.md

@@ -1,70 +0,0 @@
-# Core Concepts - Understand the Mental Model
-
-Before diving into how to use Open Notebook, it's important to understand **how it thinks**. These core concepts explain the "why" behind the design.
-
-## The Five Mental Models
-
-### 1. [Notebooks, Sources, and Notes](notebooks-sources-notes.md)
-How Open Notebook organizes your research. Understand the three-tier container structure and how information flows from raw materials to finished insights.
-
-**Key idea**: A notebook is a scoped research container. Sources are inputs (PDFs, URLs, etc.). Notes are outputs (your insights, AI-generated summaries, captured responses).
-
----
-
-### 2. [AI Context & RAG](ai-context-rag.md)
-How Open Notebook makes AI aware of your research - two different approaches.
-
-**Key idea**: **Chat** sends entire selected sources to the LLM (full context, conversational). **Ask** uses RAG (retrieval-augmented generation) to automatically search and retrieve only relevant chunks. Different tools for different needs.
-
----
-
-### 3. [Chat vs. Transformations](chat-vs-transformations.md)
-Why Open Notebook has different interaction modes and when to use each one.
-
-**Key idea**: Chat is conversational exploration (you control context). Transformations are insight extractions. They reduced content to smaller bits of concentrated/dense information, which is much more suitable for an AI to use. 
-
----
-
-### 4. [Context Management](chat-vs-transformations.md#context-management-the-control-panel)
-Your control panel for privacy and cost. Decide what data actually reaches AI.
-
-**Key idea**: You choose three levels—not in context (private), summary only (condensed), or full content (complete access). This gives you fine-grained control.
-
----
-
-### 5. [Podcasts Explained](podcasts-explained.md)
-Why Open Notebook can turn research into audio and why this matters.
-
-**Key idea**: Podcasts transform your research into a different consumption format. Instead of reading, someone can listen and absorb your insights passively.
-
----
-
-## Read This Section If:
-
-- **You're new to Open Notebook** — Start here to understand how the system works conceptually before learning the features
-- **You're confused about Chat vs Ask** — Section 2 explains the difference (full-content vs RAG)
-- **You're wondering when to use Chat vs Transformations** — Section 3 clarifies the differences
-- **You want to understand privacy controls** — Section 4 shows you what you can control
-- **You're curious about podcasts** — Section 5 explains the architecture and why it's different from competitors
-
----
-
-## The Big Picture
-
-Open Notebook is built on a simple insight: **Your research deserves to stay yours**.
-
-That means:
-- **Privacy by default** — Your data doesn't leave your infrastructure unless you explicitly choose
-- **AI as a tool, not a gatekeeper** — You decide which sources the AI sees, not the AI deciding for you
-- **Flexible consumption** — Read, listen, search, chat, or transform your research however makes sense
-
-These core concepts explain how that works.
-
----
-
-## Next Steps
-
-1. **Just want to use it?** → Go to [User Guide](../3-USER-GUIDE/index.md)
-2. **Want to understand it first?** → Read the 5 sections above (15 min)
-3. **Setting up for the first time?** → Go to [Installation](../1-INSTALLATION/index.md)
-

docs/2-CORE-CONCEPTS/notebooks-sources-notes.md

@@ -1,284 +0,0 @@
-# Notebooks, Sources, and Notes - The Container Model
-
-Open Notebook organizes research in three connected layers. Understanding this hierarchy is key to using the system effectively.
-
-## The Three-Layer Structure
-
-```
-┌─────────────────────────────────────┐
-│         NOTEBOOK (The Container)    │
-│     "My AI Safety Research 2026"   │
-├─────────────────────────────────────┤
-│                                     │
-│  SOURCES (The Raw Materials)        │
-│  ├─ safety_paper.pdf                │
-│  ├─ alignment_video.mp4             │
-│  └─ prompt_injection_article.html   │
-│                                     │
-│  NOTES (The Processed Insights)     │
-│  ├─ AI Summary (auto-generated)     │
-│  ├─ Key Concepts (transformation)   │
-│  ├─ My Research Notes (manual)      │
-│  └─ Chat Insights (from conversation)
-│                                     │
-└─────────────────────────────────────┘
-```
-
----
-
-## 1. NOTEBOOKS - The Research Container
-
-### What Is a Notebook?
-
-A **notebook** is a *scoped container* for a research project or topic. It's your research workspace.
-
-Think of it like a physical notebook: everything inside is about the same topic, shares the same context, and builds toward the same goals.
-
-### What Goes In?
-
-- **A description** — "This notebook collects research on X topic"
-- **Sources** — The raw materials you add
-- **Notes** — Your insights and outputs
-- **Conversation history** — Your chats and questions
-
-### Why This Matters
-
-**Isolation**: Each notebook is completely separate. Sources in Notebook A never appear in Notebook B. This lets you:
-- Keep different research topics completely isolated
-- Reuse source names across notebooks without conflicts
-- Control which AI context applies to which research
-
-**Shared Context**: All sources and notes in a notebook inherit the notebook's context. If your notebook is titled "AI Safety 2026" with description "Focusing on alignment and interpretability," that context applies to all AI interactions within that notebook.
-
-**Parallel Projects**: You can have 10 notebooks running simultaneously. Each one is its own isolated research environment.
-
-### Example
-
-```
-Notebook: "Customer Research - Product Launch"
-Description: "User interviews and feedback for Q1 2026 launch"
-
-→ All sources added to this notebook are about customer feedback
-→ All notes generated are in that context
-→ When you chat, the AI knows you're analyzing product launch feedback
-→ Different from your "Market Analysis - Competitors" notebook
-```
-
----
-
-## 2. SOURCES - The Raw Materials
-
-### What Is a Source?
-
-A **source** is a *single piece of input material* — the raw content you bring in. Sources never change; they're just processed and indexed.
-
-### What Can Be a Source?
-
-- **PDFs** — Research papers, reports, documents
-- **Web links** — Articles, blog posts, web pages
-- **Audio files** — Podcasts, interviews, lectures
-- **Video files** — Tutorials, presentations, recordings
-- **Plain text** — Notes, transcripts, passages
-- **Uploaded text** — Paste content directly
-
-### What Happens When You Add a Source?
-
-```
-1. EXTRACTION
-   File/URL → Extract text and metadata
-   (OCR for PDFs, web scraping for URLs, speech-to-text for audio)
-
-2. CHUNKING
-   Long text → Break into searchable chunks
-   (Prevents "too much context" in single query)
-
-3. EMBEDDING
-   Each chunk → Generate semantic vector
-   (Allows AI to find conceptually similar content)
-
-4. STORAGE
-   Chunks + vectors → Store in database
-   (Ready for search and retrieval)
-```
-
-### Key Properties
-
-**Immutable**: Once added, the source doesn't change. If you need a new version, add it as a new source.
-
-**Indexed**: Sources are automatically indexed for search (both text and semantic).
-
-**Scoped**: A source belongs to exactly one notebook.
-
-**Referenceable**: Other sources and notes can reference this source by citation.
-
-### Example
-
-```
-Source: "openai_charter.pdf"
-Type: PDF document
-
-What happens:
-→ PDF is uploaded
-→ Text is extracted (including images)
-→ Text is split into 50 chunks (paragraphs, sections)
-→ Each chunk gets an embedding vector
-→ Now searchable by: "OpenAI's approach to safety"
-```
-
----
-
-## 3. NOTES - The Processed Insights
-
-### What Is a Note?
-
-A **note** is a *processed output* — something you created or AI created based on your sources. Notes are the "results" of your research work.
-
-### Types of Notes
-
-#### Manual Notes
-You write them yourself. They're your original thinking, capturing:
-- What you learned from sources
-- Your analysis and interpretations
-- Your next steps and questions
-
-#### AI-Generated Notes
-Created by applying AI processing to sources:
-- **Transformations** — Structured extraction (main points, key concepts, methodology)
-- **Chat Responses** — Answers you saved from conversations
-- **Ask Results** — Comprehensive answers saved to your notebook
-
-#### Captured Insights
-Notes you explicitly saved from interactions:
-- "Save this response as a note"
-- "Save this transformation result"
-- Convert any AI output into a permanent note
-
-### What Can Notes Contain?
-
-- **Text** — Your writing or AI-generated content
-- **Citations** — References to specific sources
-- **Metadata** — When created, how created (manual/AI), which sources influenced it
-- **Tags** — Your categorization (optional but useful)
-
-### Why Notes Matter
-
-**Knowledge Accumulation**: Notes become your actual knowledge base. They're what you take away from the research.
-
-**Searchable**: Notes are searchable along with sources. "Find everything about X" includes your notes, not just sources.
-
-**Citable**: Notes can cite sources, creating an audit trail of where insights came from.
-
-**Shareable**: Notes are your outputs. You can share them, publish them, or build on them in other projects.
-
----
-
-## How They Connect: The Data Flow
-
-```
-YOU
- │
- ├─→ Create Notebook ("AI Research")
- │
- ├─→ Add Sources (papers, articles, videos)
- │    └─→ System: Extract, embed, index
- │
- ├─→ Search Sources (text or semantic)
- │    └─→ System: Find relevant chunks
- │
- ├─→ Apply Transformations (extract insights)
- │    └─→ Creates Notes
- │
- ├─→ Chat with Sources (explore with context control)
- │    ├─→ Can save responses as Notes
- │    └─→ Notes include citations
- │
- ├─→ Ask Questions (automated comprehensive search)
- │    ├─→ Can save results as Notes
- │    └─→ Notes include citations
- │
- └─→ Generate Podcast (transform notebook into audio)
-     └─→ Uses all sources + notes for content
-```
-
----
-
-## Key Design Decisions
-
-### 1. One Notebook Per Source
-
-Each source belongs to exactly one notebook. This creates clear boundaries:
-- No ambiguity about which research project a source is in
-- Easy to isolate or export a complete project
-- Clean permissions model (if someone gets access to notebook, they get access to all its sources)
-
-### 2. Immutable Sources, Mutable Notes
-
-Sources never change (once added, always the same). But notes can be edited or deleted. Why?
-- Sources are evidence → evidence shouldn't be altered
-- Notes are your thinking → thinking evolves as you learn
-
-### 3. Explicit Context Control
-
-Sources don't automatically go to AI. You decide which sources are "in context" for each interaction:
-- Chat: You manually select which sources to include
-- Ask: System automatically figures out which sources to search
-- Transformations: You choose which sources to transform
-
-This is different from systems that always send everything to AI.
-
----
-
-## Mental Models Explained
-
-### Notebook as Boundaries
-Think of a notebook like a Git repository:
-- Everything in it is about the same topic
-- You can clone/fork it (copy to new project)
-- It has clear entry/exit points
-- You know exactly what's included
-
-### Sources as Evidence
-Think of sources like exhibits in a legal case:
-- Once filed, they don't change
-- They can be cited and referenced
-- They're the ground truth for what you're basing claims on
-- Multiple sources can be cross-referenced
-
-### Notes as Synthesis
-Think of notes like your case brief:
-- You write them based on evidence
-- They're your interpretation
-- You can cite which evidence supports each claim
-- They're what you actually share or act on
-
----
-
-## Common Questions
-
-### Can I move a source to a different notebook?
-Not directly. Each source is tied to one notebook. If you want it in multiple notebooks, add it again (uploads are fast if it's already processed).
-
-### Can a note reference sources from a different notebook?
-No. Notes stay within their notebook and reference sources within that notebook. This keeps boundaries clean.
-
-### What if I want to group sources within a notebook?
-Use tags. You can tag sources ("primary research," "background," "methodology") and filter by tags.
-
-### Can I merge two notebooks?
-Not built-in, but you can manually copy sources from one notebook to another by re-uploading them.
-
----
-
-## Summary
-
-| Concept | Purpose | Lifecycle | Scope |
-|---------|---------|-----------|-------|
-| **Notebook** | Container + context | Create once, configure | All its sources + notes |
-| **Source** | Raw material | Add → Process → Store | One notebook |
-| **Note** | Processed output | Create/capture → Edit → Share | One notebook |
-
-This three-layer model gives you:
-- **Clear organization** (everything scoped to projects)
-- **Privacy control** (isolated notebooks)
-- **Audit trails** (notes cite sources)
-- **Flexibility** (notes can be manual or AI-generated)

docs/2-CORE-CONCEPTS/podcasts-explained.md

@@ -1,394 +0,0 @@
-# Podcasts Explained - Research as Audio Dialogue
-
-Podcasts are Open Notebook's highest-level transformation: converting your research into audio dialogue for a different consumption pattern.
-
----
-
-## Why Podcasts Matter
-
-### The Problem
-Research naturally accumulates as text: PDFs, articles, web pages, notes. This creates a friction point:
-
-**To consume research, you must:**
-- Sit down at a desk
-- Focus intently
-- Read actively
-- Take notes
-- Set aside dedicated time
-
-**But much of life is passive time:**
-- Commuting
-- Exercising
-- Doing dishes
-- Driving
-- Walking
-- Idle moments
-
-### The Solution
-Convert your research into audio dialogue so you can consume it passively.
-
-```
-Before (Text-based):
-  Research pile → Must schedule reading time → Requires focus
-
-After (Podcast):
-  Research pile → Podcast → Can listen while commuting
-                         → Absorb while exercising
-                         → Understand while walking
-                         → Engage without screen time
-```
-
----
-
-## What Makes It Special: Open Notebook vs. Competitors
-
-### Google Notebook LM Podcasts
-- **Fixed format**: 2 hosts, always conversational
-- **Limited customization**: You can't choose who the "hosts" are
-- **One TTS voice per speaker**: Can't customize voices
-- **Only uses cloud services**: No local options
-
-### Open Notebook Podcasts
-- **Customizable format**: 1-4 speakers, you design them
-- **Rich speaker profiles**: Create personas with backstories and expertise
-- **Multiple TTS options**:
-  - OpenAI (natural, fast)
-  - Google TTS (high quality)
-  - ElevenLabs (beautiful voices, accents)
-  - Local TTS (privacy-first, no API calls)
-- **Async generation**: Doesn't block your work
-- **Full control**: Choose outline structure, tone, depth
-
----
-
-## How Podcast Generation Works
-
-### Stage 1: Content Selection
-
-You choose what goes into the podcast:
-```
-Notebook content → Which sources? → Which notes?
-                → Which topics to focus on?
-                → Depth of coverage?
-```
-
-### Stage 2: Episode Profile
-
-You define how you want the podcast structured:
-```
-Episode Profile
-├─ Topic: "AI Safety Approaches"
-├─ Length: 20 minutes
-├─ Tone: Academic but accessible
-├─ Format: Debate (2 speakers with opposing views)
-├─ Audience: Researchers new to the field
-└─ Focus areas: Main approaches, pros/cons, open questions
-```
-
-### Stage 3: Speaker Configuration
-
-You create speaker personas (1-4 speakers):
-
-```
-Speaker 1: "Expert Alex"
-├─ Expertise: "Deep knowledge of alignment research"
-├─ Personality: "Rigorous, academic, patient with explanation"
-├─ Accent: (Optional) "British English"
-└─ TTS Voice: "OpenAI Onyx" (or ElevenLabs, Google, etc.)
-
-Speaker 2: "Researcher Sam"
-├─ Expertise: "Field observer, pragmatic perspective"
-├─ Personality: "Curious, asks clarifying questions"
-├─ Accent: "American English"
-└─ TTS Voice: "ElevenLabs - thoughtful"
-```
-
-### Stage 4: Outline Generation
-
-System generates episode outline:
-```
-EPISODE: "AI Safety Approaches"
-
-1. Introduction (2 min)
-   Alex: Introduces topic and speakers
-   Sam: What will we cover today?
-
-2. Main Approaches (8 min)
-   Alex: Explains top 3 approaches
-   Sam: Asks about tradeoffs
-
-3. Debate: Best approach? (6 min)
-   Alex: Advocates for approach A
-   Sam: Argues for approach B
-
-4. Open Questions (3 min)
-   Both: What's unsolved?
-
-5. Conclusion (1 min)
-   Recap and where to learn more
-```
-
-### Stage 5: Dialogue Generation
-
-System generates dialogue based on outline:
-```
-Alex: "Today we're exploring three major approaches to AI alignment..."
-
-Sam: "That's a great start. Can you break down what we mean by alignment?"
-
-Alex: "Good question. Alignment means ensuring AI systems pursue the goals
-       we actually want them to pursue, not just what we literally asked for.
-       There's a classic example of a paperclip maximizer..."
-
-Sam: "Interesting. So it's about solving the intention problem?"
-
-Alex: "Exactly. And that's where the three approaches come in..."
-```
-
-### Stage 6: Text-to-Speech
-
-System converts dialogue to audio:
-```
-Alex's text → OpenAI TTS → Alex's voice (audio file)
-Sam's text → ElevenLabs TTS → Sam's voice (audio file)
-Audio files → Mix together → Final podcast MP3
-```
-
----
-
-## Key Architecture Decisions
-
-### 1. Asynchronous Processing
-Podcasts are generated in the background. You upload → system processes → you download when ready.
-
-**Why?** Podcast generation takes time (10+ minutes for a 30-minute episode). Blocking would lock up your interface.
-
-### 2. Multi-Speaker Support
-Unlike Google Notebook LM (always 2 hosts), you choose 1-4 speakers.
-
-**Why?** Different discussions work better with different formats:
-- Expert monologue (1 speaker)
-- Interview (2 speakers: host + expert)
-- Debate (2 speakers: opposing views)
-- Panel discussion (3-4 speakers: different expertise)
-
-### 3. Speaker Customization
-You create rich speaker profiles, not just "Host A" and "Host B".
-
-**Why?** Makes podcasts more engaging and authentic. Different speakers bring different perspectives.
-
-### 4. Multiple TTS Providers
-You're not locked into one voice provider.
-
-**Why?**
-- Cost optimization (some providers cheaper)
-- Quality preferences (some voices more natural)
-- Privacy options (local TTS for sensitive content)
-- Accessibility (different accents, genders, styles)
-
-### 5. Local TTS Option
-Can generate podcasts entirely offline with local text-to-speech.
-
-**Why?** For sensitive research, never send audio to external APIs.
-
----
-
-## Use Cases Show Why This Matters
-
-### Academic Publishing
-```
-Traditional: Academic paper → PDF
-Problem: Hard to consume, linear reading required
-
-Open Notebook:
-Research materials → Podcast (expert explaining methodology)
-                  → Podcast (debate format: different interpretations)
-                  → Different consumption for different audiences
-```
-
-### Content Creation
-```
-Blog creator: Has research pile on a topic
-Problem: Doesn't have time to write the article
-
-Solution:
-Add research → Create podcast → Transcribe → Becomes article
-OR: Podcast BECOMES the content (upload to podcast platforms)
-```
-
-### Educational Content
-```
-Educator: Has reading materials for a course
-Problem: Students don't read the papers
-
-Solution:
-Create podcast with expert explaining papers
-Students listen → Better engagement → Discussions can reference podcast
-```
-
-### Market Research
-```
-Product manager: Has interviews with customers
-Problem: Too many hours of audio to review
-
-Solution:
-Create podcast with debate format (customer perspective vs. team perspective)
-Much more engaging than raw transcripts
-```
-
-### Knowledge Transfer
-```
-Domain expert: Leaving the organization
-Problem: How to preserve expertise?
-
-Solution:
-Create expert-mode podcast explaining frameworks, decision-making, context
-New team member listens, gets context faster than reading 100 documents
-```
-
----
-
-## The Difference: Active vs. Passive Learning
-
-### Text-Based Research (Active)
-- **Effort**: High (must focus, read, synthesize)
-- **When**: Dedicated study time
-- **Cost**: Time is expensive (can't multitask)
-- **Best for**: Deep dives, precise information
-- **Format**: Whatever you write (notes, articles, books)
-
-### Audio Podcast (Passive)
-- **Effort**: Low (just listen)
-- **When**: Anywhere, anytime
-- **Cost**: Low (can multitask)
-- **Best for**: Overview, context, exploration
-- **Format**: Dialogue (more engaging than narration)
-
-**They complement each other:**
-1. **First encounter**: Listen to podcast (passive, get context)
-2. **Deep dive**: Read source materials (active, precise)
-3. **Mastery**: Both together (understand big picture + details)
-
----
-
-## How Podcasts Fit Into Your Workflow
-
-```
-1. Build notebook (add sources)
-   ↓
-2. Apply transformations (extract insights)
-   ↓
-3. Chat/Ask (explore content)
-   ↓
-4. Decide on podcast
-   ├─→ Create speaker profiles
-   ├─→ Define episode profile
-   ├─→ Choose TTS provider
-   └─→ Generate podcast
-   ↓
-5. Listen while commuting/exercising
-   ↓
-6. Reference sources for deep dive
-   ↓
-7. Repeat for different formats/speakers/focus
-```
-
----
-
-## Advanced: Multiple Podcasts from Same Research
-
-You can create different podcasts from the same sources:
-
-### Example: AI Safety Research
-```
-Podcast 1: "Expert Monologue"
-  Speaker: Researcher explaining field
-  Format: Educational, comprehensive
-  Audience: Students new to field
-
-Podcast 2: "Debate Format"
-  Speakers: Optimist vs. skeptic
-  Format: Discussion of tradeoffs
-  Audience: Advanced researchers
-
-Podcast 3: "Interview Format"
-  Speakers: Journalist + expert
-  Format: Q&A about practical applications
-  Audience: Industry practitioners
-```
-
-Each tells the same story from different angles.
-
----
-
-## Privacy & Data Considerations
-
-### Where Your Data Goes
-
-**Option 1: Cloud TTS (Faster, Higher Quality)**
-```
-Your outline → API call to TTS provider
-            → Audio returned
-            → Stored in your notebook
-
-Provider sees: Your outlined script (not raw sources)
-Privacy level: Medium (outline is shared, sources aren't)
-```
-
-**Option 2: Local TTS (Slower, Maximum Privacy)**
-```
-Your outline → Local TTS engine (runs on your machine)
-            → Audio generated locally
-            → Stored in your notebook
-
-Provider sees: Nothing
-Privacy level: Maximum (everything local)
-```
-
-### Recommendation
-- **Sensitive research**: Use local TTS, no API calls
-- **Less sensitive**: Use ElevenLabs or Google (both handle audio data professionally)
-- **Mixed**: Use local TTS for speakers reading sensitive content
-
----
-
-## Cost Considerations
-
-### Cloud TTS Costs
-| Provider | Cost | Quality | Speed |
-|----------|------|---------|-------|
-| OpenAI | ~$0.015 per minute | Good | Fast |
-| Google | ~$0.004 per minute | Excellent | Fast |
-| ElevenLabs | ~$0.10 per minute | Exceptional | Medium |
-| Local TTS | Free | Basic | Slow |
-
-A 30-minute podcast costs:
-- OpenAI: ~$0.45
-- Google: ~$0.12
-- ElevenLabs: ~$3.00
-- Local: Free (but slow)
-
----
-
-## Summary: Why Podcasts Are Special
-
-**Podcasts transform your research consumption:**
-
-| Aspect | Text | Podcast |
-|--------|------|---------|
-| **How consumed?** | Active reading | Passive listening |
-| **Where consumed?** | Desk | Anywhere |
-| **Multitasking** | Hard | Easy |
-| **Time commitment** | Scheduled | Flexible |
-| **Format** | Whatever | Natural dialogue |
-| **Engagement** | Academic | Conversational |
-| **Accessibility** | Text-based | Audio-based |
-
-**In Open Notebook specifically:**
-- **Full customization** — you create speakers and format
-- **Privacy options** — local TTS for sensitive content
-- **Cost control** — choose TTS provider based on budget
-- **Non-blocking** — generates in background
-- **Multiple versions** — create different podcasts from same research
-
-This is why podcasts matter: they change *when* and *how* you can consume your research.

docs/3-USER-GUIDE/adding-sources.md

@@ -1,429 +0,0 @@
-# Adding Sources - Getting Content Into Your Notebook
-
-Sources are the raw materials of your research. This guide covers how to add different types of content.
-
----
-
-## Quick-Start: Add Your First Source
-
-### Option 1: Upload a File (PDF, Word, etc.)
-
-```
-1. In your notebook, click "Add Source"
-2. Select "Upload File"
-3. Choose a file from your computer
-4. Click "Upload"
-5. Wait 30-60 seconds for processing
-6. Done! Source appears in your notebook
-```
-
-### Option 2: Add a Web Link
-
-```
-1. Click "Add Source"
-2. Select "Web Link"
-3. Paste URL: https://example.com/article
-4. Click "Add"
-5. Wait for processing (usually faster than files)
-6. Done!
-```
-
-### Option 3: Paste Text
-
-```
-1. Click "Add Source"
-2. Select "Text"
-3. Paste or type your content
-4. Click "Save"
-5. Done! Immediately available
-```
-
----
-
-## Supported File Types
-
-### Documents
-- **PDF** (.pdf) — Best support, including scanned PDFs with OCR
-- **Word** (.docx, .doc) — Full support
-- **PowerPoint** (.pptx) — Slides converted to text
-- **Excel** (.xlsx, .xls) — Spreadsheet data
-- **EPUB** (.epub) — eBook files
-- **Markdown** (.md, .txt) — Plain text formats
-- **HTML** (.html, .htm) — Web page files
-
-**File size limits:** Up to ~100MB (varies by system)
-
-**Processing time:** 10 seconds - 2 minutes (depending on length and file type)
-
-### Audio & Video
-- **Audio**: MP3, WAV, M4A, OGG, FLAC (~30 seconds - 3 minutes per hour)
-- **Video**: MP4, AVI, MOV, MKV, WebM (~3-10 minutes per hour)
-- **YouTube**: Direct URL support
-- **Podcasts**: RSS feed URL
-
-**Automatic transcription**: Audio/video is transcribed to text automatically. This requires enabling speech-to-text in settings.
-
-### Web Content
-- **Articles**: Blog posts, news articles, Medium
-- **YouTube**: Full videos or playlists
-- **PDFs online**: Direct PDF links
-- **News**: News site articles
-
-**Just paste the URL** in "Web Link" section.
-
-### What Doesn't Work
-- Paywalled content (WSJ, FT, etc.) — Can't extract
-- Password-protected PDFs — Can't open
-- Pure image files (.jpg, .png) — Except scanned PDFs which have OCR
-- Very large files (>100MB) — Timeout
-
----
-
-## What Happens When You Add a Source
-
-The system automatically does four things:
-
-```
-1. EXTRACT TEXT
-   File/URL → Readable text
-   (PDFs get OCR if scanned)
-   (Videos get transcribed if enabled)
-
-2. BREAK INTO CHUNKS
-   Long text → ~500-word pieces
-   (So search finds specific parts, not whole document)
-
-3. CREATE EMBEDDINGS
-   Each chunk → Vector representation
-   (Enables semantic/concept search)
-
-4. INDEX & STORE
-   Everything → Database
-   (Ready to search and retrieve)
-```
-
-**Time to use:** After the progress bar completes, the source is ready immediately. Embeddings are created in the background.
-
----
-
-## Step-by-Step for Different Types
-
-### PDFs
-
-**Best practices:**
-```
-Clean PDFs:
-  1. Upload → Done
-  2. Processing time: ~30-60 seconds
-
-Scanned/Image PDFs:
-  1. Upload same way
-  2. System auto-detects and uses OCR
-  3. Processing time: ~2-3 minutes
-  4. (Higher, due to OCR overhead)
-
-Large PDFs (50+ pages):
-  1. Consider splitting into smaller files
-  2. Or upload as-is (system handles it)
-  3. Processing time scales with size
-```
-
-**Common issues:**
-- "Can't extract text" → PDF is corrupted or has copy protection
-- Solution: Try opening in Adobe. If it won't, the PDF is likely protected.
-
-### Web Links / Articles
-
-**Best practices:**
-```
-1. Copy full URL from browser: https://example.com/article-title
-2. Paste in "Web Link"
-3. Click Add
-4. Wait for extraction
-
-Processing time: Usually 5-15 seconds
-```
-
-**What works:**
-- Standard web articles
-- Blog posts
-- News articles
-- Wikipedia pages
-- Medium posts
-- Substack articles
-
-**What doesn't work:**
-- Twitter threads (unreliable)
-- Paywalled articles (can't access)
-- JavaScript-heavy sites (content not extracted)
-
-**Pro tip:** If it doesn't work, copy the article text and paste as "Text" instead.
-
-### Audio Files
-
-**Best practices:**
-```
-1. Ensure speech-to-text is enabled in Settings
-2. Upload MP3, WAV, or M4A file
-3. System automatically transcribes to text
-4. Processing time: ~1 minute per 5 minutes of audio
-
-Example:
-  - 1-hour podcast → 12 minutes processing
-  - 10-minute recording → 2 minutes processing
-```
-
-**Quality matters:**
-- Clear audio: Fast transcription
-- Muffled/noisy audio: Slower, less accurate transcription
-- Background noise: Try to minimize before uploading
-
-**Tip:** If audio quality is poor, the AI might misinterpret content. You can manually correct transcription if needed.
-
-### YouTube Videos
-
-**Best practices:**
-```
-Two ways to add:
-
-Method 1: Direct URL
-  1. Copy YouTube URL: https://www.youtube.com/watch?v=...
-  2. Paste in "Web Link"
-  3. Click Add
-  4. System extracts captions (if available) + transcript
-
-Method 2: Playlist
-  1. Paste playlist URL
-  2. System adds all videos as separate sources
-  3. Each video processed separately
-  4. Takes longer (multiple videos)
-```
-
-**What's extracted:**
-- Captions/subtitles (if available)
-- Transcription (if captions aren't available)
-- Basic metadata (title, channel, length)
-
-**Processing:**
-- 10-minute video: ~2-3 minutes
-- 1-hour video: ~10-15 minutes
-
-### Text / Paste Content
-
-**Best practices:**
-```
-1. Select "Text" when adding source
-2. Paste or type content
-3. System processes immediately
-4. No wait time needed
-
-Good for:
-  - Notes you want to reference
-  - Quotes from books
-  - Transcripts you have handy
-  - Quick research snippets
-```
-
----
-
-## Managing Your Sources
-
-### Viewing Source Details
-
-```
-Click on source → See:
-  - Original file name/title
-  - When it was added
-  - Size and format
-  - Processing status
-  - Number of chunks
-```
-
-### Organizing with Metadata
-
-You can add to each source:
-- **Title**: Better name than original filename
-- **Tags**: Category labels ("primary research", "background", "competitor analysis")
-- **Description**: A few notes about what it contains
-
-**Why this matters:**
-- Makes sources easier to find
-- Helps when contextualizing for Chat
-- Useful for organizing large notebooks
-
-### Searching Within Sources
-
-```
-After sources are added, you can:
-
-Text search: "Find exact phrase"
-Vector search: "Find conceptually similar"
-
-Both search across all sources in notebook.
-Results show:
-  - Which source
-  - Which section
-  - Relevance score
-```
-
----
-
-## Context Management: How Sources Get Used
-
-You control how AI accesses sources:
-
-### Three Levels (for Chat)
-
-**Full Content:**
-```
-AI sees: Complete source text
-Cost: 100% of tokens
-Use when: Analyzing in detail, need precise citations
-Example: "Analyze this methodology paper closely"
-```
-
-**Summary Only:**
-```
-AI sees: AI-generated summary (not full text)
-Cost: ~10-20% of tokens
-Use when: Background material, reference context
-Example: "Use this as context but focus on the main source"
-```
-
-**Not in Context:**
-```
-AI sees: Nothing (excluded)
-Cost: 0 tokens
-Use when: Confidential, not relevant, or archived
-Example: "Keep this in notebook but don't use in this conversation"
-```
-
-### How to Set Context (in Chat)
-
-```
-1. Go to Chat
-2. Click "Select Context Sources"
-3. For each source:
-   - Toggle ON/OFF (include/exclude)
-   - Choose level (Full/Summary/Excluded)
-4. Click "Save"
-5. Now chat uses these settings
-```
-
----
-
-## Common Mistakes
-
-| Mistake | What Happens | How to Fix |
-|---------|--------------|-----------|
-| Upload 200 sources at once | System gets slow, processing stalls | Add 10-20 at a time, wait for processing |
-| Use full content for all sources | Token usage skyrockets, expensive | Use "Summary" or "Excluded" for background material |
-| Add huge PDFs without splitting | Processing is slow, search results less precise | Consider splitting large PDFs into chapters |
-| Forget source titles | Can't distinguish between similar sources | Rename sources with descriptive titles right after uploading |
-| Don't tag sources | Hard to find and organize later | Add tags immediately: "primary", "background", etc. |
-| Mix languages in one source | Transcription/embedding quality drops | Keep each language in separate sources |
-| Use same source multiple times | Takes up space, creates confusion | Add once; reuse in multiple chats/notebooks |
-
----
-
-## Processing Status & Troubleshooting
-
-### What the Status Indicators Mean
-
-```
-🟡 Processing
-  → Source is being extracted and embedded
-  → Wait 30 seconds - 3 minutes depending on size
-  → Don't use in Chat yet
-
-🟢 Ready
-  → Source is processed and searchable
-  → Can use immediately in Chat
-  → Can apply transformations
-
-🔴 Error
-  → Something went wrong
-  → Common reasons:
-    - Unsupported file format
-    - File too large or corrupted
-    - Network timeout
-
-⚪ Not in Context
-  → Source added but excluded from Chat
-  → Still searchable, not sent to AI
-```
-
-### Common Errors & Solutions
-
-**"Unsupported file type"**
-- You tried to upload a format not in the list (e.g., `.webp` image)
-- Solution: Convert to supported format (PDF for documents, MP3 for audio)
-
-**"Processing timeout"**
-- Very large file (>100MB) or very long audio
-- Solution: Split into smaller pieces or try uploading again
-
-**"Transcription failed"**
-- Audio quality too poor or language not detected
-- Solution: Re-record with better quality, or paste text transcript manually
-
-**"Web link won't extract"**
-- Website blocks automated access or uses JavaScript for content
-- Solution: Copy the article text and paste as "Text" instead
-
----
-
-## Tips for Best Results
-
-### For PDFs
-- Clean, digital PDFs work best
-- Remove copy protection if present (legally)
-- Scanned PDFs work but take longer
-
-### For Web Articles
-- Use full URL including domain
-- Avoid cookie/popup-laden sites
-- If extraction fails, copy-paste text instead
-
-### For Audio
-- Clear, well-recorded audio transcribes better
-- Remove background noise if possible
-- YouTube videos usually have good transcriptions built-in
-
-### For Large Documents
-- Consider splitting into smaller sources
-- Gives more precise search results
-- Processing is faster for smaller pieces
-
-### For Organization
-- Name sources clearly (not "document_2.pdf")
-- Add tags immediately after uploading
-- Use descriptions for complex documents
-
----
-
-## What Comes After: Using Your Sources
-
-Once you've added sources, you can:
-
-- **Chat** → Ask questions (see [Chat Effectively](chat-effectively.md))
-- **Search** → Find specific content (see [Search Effectively](search.md))
-- **Transformations** → Extract structured insights (see [Working with Notes](working-with-notes.md))
-- **Ask** → Get comprehensive answers (see [Search Effectively](search.md))
-- **Podcasts** → Turn into audio (see [Creating Podcasts](creating-podcasts.md))
-
----
-
-## Summary Checklist
-
-Before adding sources, confirm:
-
-- [ ] File is in supported format
-- [ ] File is under 100MB (or splitting large ones)
-- [ ] Web links are full URLs (not shortened)
-- [ ] Audio files have clear speech (if transcription-dependent)
-- [ ] You've named source clearly
-- [ ] You've added tags for organization
-- [ ] You understand context levels (Full/Summary/Excluded)
-
-Done! Sources are now ready for Chat, Search, Transformations, and more.

docs/3-USER-GUIDE/chat-effectively.md

@@ -1,554 +0,0 @@
-# Chat Effectively - Conversations with Your Research
-
-Chat is your main tool for exploratory questions and back-and-forth dialogue. This guide covers how to use it effectively.
-
----
-
-## Quick-Start: Your First Chat
-
-```
-1. Go to your notebook
-2. Click "Chat"
-3. Select which sources to include (context)
-4. Type your question
-5. Click "Send"
-6. Read the response
-7. Ask a follow-up (context stays same)
-8. Repeat until satisfied
-```
-
-That's it! But doing it *well* requires understanding how context works.
-
----
-
-## Context Management: The Key to Good Chat
-
-Context controls **what the AI is allowed to see**. This is your most important control.
-
-### The Three Levels Explained
-
-**FULL CONTENT**
-- AI sees: Complete source text
-- Cost: 100 tokens per 1K tokens of source
-- Best for: Detailed analysis, precise citations
-- Example: "Analyze this research paper closely"
-
-```
-You set: Paper A → Full Content
-AI sees: Every word of Paper A
-AI can: Cite specific sentences, notice nuances
-Result: Precise, detailed answers (higher cost)
-```
-
-**SUMMARY ONLY**
-- AI sees: AI-generated 200-word summary (not full text)
-- Cost: ~10-20% of full content cost
-- Best for: Background material, reference context
-- Example: "Use this for background, focus on the main paper"
-
-```
-You set: Paper B → Summary Only
-AI sees: Condensed summary, key points
-AI can: Reference main ideas but not details
-Result: Faster, cheaper answers (loses precision)
-```
-
-**NOT IN CONTEXT**
-- AI sees: Nothing
-- Cost: 0 tokens
-- Best for: Confidential, irrelevant, archived content
-- Example: "Keep this in notebook but don't use now"
-
-```
-You set: Paper C → Not in Context
-AI sees: Nothing (completely excluded)
-AI can: Never reference it
-Result: No cost, no privacy risk for that source
-```
-
-### Setting Context (Step by Step)
-
-```
-1. Click "Select Sources"
-   (Shows list of all sources in notebook)
-
-2. For each source:
-   □ Checkbox: Include or exclude
-
-   Level dropdown:
-   ├─ Full Content
-   ├─ Summary Only
-   └─ Excluded
-
-3. Check your selections
-   Example:
-   ✓ Paper A (Full Content) - "Main focus"
-   ✓ Paper B (Summary Only) - "Background"
-   ✓ Paper C (Excluded) - "Keep private"
-   □ Paper D (Not included) - "Not relevant"
-
-4. Click "Save Context"
-
-5. Now chat uses these settings
-```
-
-### Context Strategies
-
-**Strategy 1: Minimalist**
-- Main source: Full Content
-- Everything else: Excluded
-- Result: Focused, cheap, precise
-
-```
-Use when:
-  - Analyzing one source deeply
-  - Budget-conscious
-  - Want focused answers
-```
-
-**Strategy 2: Comprehensive**
-- All sources: Full Content
-- Result: All context considered, expensive
-
-```
-Use when:
-  - Comprehensive analysis
-  - Unlimited budget
-  - Want AI to see everything
-```
-
-**Strategy 3: Tiered**
-- Primary sources: Full Content
-- Secondary sources: Summary Only
-- Background/reference: Excluded
-- Result: Balanced cost/quality
-
-```
-Use when:
-  - Mix of important and reference material
-  - Want thorough but not expensive
-  - Most common strategy
-```
-
-**Strategy 4: Privacy-First**
-- Sensitive docs: Excluded
-- Public research: Full Content
-- Result: Never send confidential data
-
-```
-Use when:
-  - Company confidential materials
-  - Personal sensitive data
-  - Complying with data protection
-```
-
----
-
-## Asking Effective Questions
-
-### Good Questions vs. Poor Questions
-
-**Poor Question**
-```
-"What do you think?"
-
-Problems:
-- Too vague (about what?)
-- No context (what am I analyzing?)
-- Can't verify answer (citing what?)
-
-Result: Generic, shallow answer
-```
-
-**Good Question**
-```
-"Based on the paper's methodology section,
-what are the three main limitations the authors acknowledge?
-Please cite which pages mention each one."
-
-Strengths:
-- Specific about what you want
-- Clear scope (methodology section)
-- Asks for citations
-- Requires deep reading
-
-Result: Precise, verifiable, useful answer
-```
-
-### Question Patterns That Work
-
-**Factual Questions**
-```
-"What does the paper say about X?"
-"Who are the authors?"
-"What year was this published?"
-
-Result: Simple, factual answers with citations
-```
-
-**Analysis Questions**
-```
-"How does this approach differ from the traditional method?"
-"What are the main assumptions underlying this argument?"
-"Why do you think the author chose this methodology?"
-
-Result: Deeper thinking, comparison, critique
-```
-
-**Synthesis Questions**
-```
-"How do these two sources approach the problem differently?"
-"What's the common theme across all three papers?"
-"If we combine these approaches, what would we get?"
-
-Result: Cross-source insights, connections
-```
-
-**Actionable Questions**
-```
-"What are the practical implications of this research?"
-"How could we apply these findings to our situation?"
-"What's the next logical research direction?"
-
-Result: Practical, forward-looking answers
-```
-
-### The SPECIFIC Formula
-
-Good questions have:
-
-1. **SCOPE** - What are you analyzing?
-   "In this research paper..."
-   "Looking at these three articles..."
-   "Based on your experience..."
-
-2. **SPECIFICITY** - Exactly what do you want?
-   "...the methodology..."
-   "...main findings..."
-   "...recommended next steps..."
-
-3. **CONSTRAINT** - Any limits?
-   "...in 3 bullet points..."
-   "...with citations to page numbers..."
-   "...comparing these two approaches..."
-
-4. **VERIFICATION** - How can you check it?
-   "...with specific quotes..."
-   "...cite your sources..."
-   "...link to the relevant section..."
-
-**Example:**
-```
-Poor: "What about transformers?"
-Good: "In this research paper on machine learning,
-      explain the transformer architecture in 2-3 sentences,
-      then cite which page describes the attention mechanism."
-```
-
----
-
-## Follow-Up Questions (The Real Power of Chat)
-
-Chat's strength is dialogue. You ask, get an answer, ask more.
-
-### Building on Responses
-
-```
-First question:
-"What's the main finding?"
-
-AI: "The study shows X [citation]"
-
-Follow-up question:
-"How does that compare to Y research?"
-
-AI: "The key difference is Z [citation]"
-
-Next question:
-"Why do you think that difference matters?"
-
-AI: "Because it affects A, B, C [explained]"
-```
-
-### Iterating Toward Understanding
-
-```
-Round 1: Get overview
-"What's this source about?"
-
-Round 2: Get details
-"What's the most important part?"
-
-Round 3: Compare
-"How does it relate to my notes on X?"
-
-Round 4: Apply
-"What should I do with this information?"
-```
-
-### Changing Direction
-
-```
-Context stays same, but you ask new questions:
-
-Question 1: "What's the methodology?"
-Question 2: "What are the limitations?"
-Question 3: "What about the ethical implications?"
-Question 4: "Who else has done similar work?"
-
-All in one conversation, reusing context.
-```
-
-### Adjusting Context Between Rounds
-
-```
-After question 3, you realize:
-"I need more context from another source"
-
-1. Click "Adjust Context"
-2. Add new source or change context level
-3. Your conversation history stays
-4. Continue asking with new context
-```
-
----
-
-## Citations and Verification
-
-Citations are how you verify that the AI's answer is accurate.
-
-### Understanding Citations
-
-```
-AI Response with Citation:
-"The paper reports a 95% accuracy rate [see page 12]"
-
-What this means:
-✓ The claim "95% accuracy rate" is from page 12
-✓ You can verify by reading page 12
-✓ If page 12 doesn't say that, the AI hallucinated
-```
-
-### Requesting Better Citations
-
-```
-If you get a response without citations:
-
-Ask: "Please cite the page number for that claim"
-or: "Show me where you found that information"
-
-AI will:
-- Find the citation
-- Provide page numbers
-- Show you the source
-```
-
-### Verification Workflow
-
-```
-1. Get answer from Chat
-2. Check citation (which source? which page?)
-3. Click citation link (if available)
-4. See the actual text in source
-5. Does it really say what AI claimed?
-
-If YES: Great, you can use this answer
-If NO: The AI hallucinated, ask for correction
-```
-
----
-
-## Common Chat Patterns
-
-### Pattern 1: Deep Dive into One Source
-
-```
-1. Set context: One source (Full Content)
-2. Question 1: Overview
-3. Question 2: Main argument
-4. Question 3: Evidence for argument
-5. Question 4: Limitations
-6. Question 5: Next steps
-
-Result: Complete understanding of one source
-```
-
-### Pattern 2: Comparative Analysis
-
-```
-1. Set context: 2-3 sources (all Full Content)
-2. Question 1: What does each source say about X?
-3. Question 2: How do they agree?
-4. Question 3: How do they disagree?
-5. Question 4: Which approach is stronger?
-
-Result: Understanding differences and trade-offs
-```
-
-### Pattern 3: Research Exploration
-
-```
-1. Set context: Many sources (mix of Full/Summary)
-2. Question 1: What are the main perspectives?
-3. Question 2: What's missing from these views?
-4. Question 3: What questions does this raise?
-5. Question 4: What should I research next?
-
-Result: Understanding landscape and gaps
-```
-
-### Pattern 4: Problem Solving
-
-```
-1. Set context: Relevant sources (Full Content)
-2. Question 1: What's the problem?
-3. Question 2: What approaches exist?
-4. Question 3: Pros and cons of each?
-5. Question 4: Which would work best for [my situation]?
-
-Result: Decision-making informed by research
-```
-
----
-
-## Optimizing for Cost
-
-Chat uses tokens for every response. Here's how to use efficiently:
-
-### Reduce Token Usage
-
-**Minimize context**
-```
-Option A: All sources, Full Content
-  Cost per response: 5,000 tokens
-
-Option B: Only relevant sources, Summary Only
-  Cost per response: 1,000 tokens
-
-Savings: 80% cheaper, same conversation
-```
-
-**Shorter questions**
-```
-Verbose: "Could you please analyze the methodology
-         section of this paper and explain in detail
-         what the authors did?"
-
-Concise: "Summarize the methodology in 2-3 points."
-
-Savings: 20-30% per response
-```
-
-**Use cheaper models**
-```
-GPT-4o: $0.15 per 1M input tokens
-GPT-4o-mini: $0.03 per 1M input tokens
-Claude Sonnet: $0.90 per 1M input tokens
-
-For chat: Mini/Haiku models are usually fine
-For deep analysis: Sonnet/Opus worth the cost
-```
-
-### Budget Strategies
-
-**Exploration budget**
-- Use cheap model
-- Broad context (understand landscape)
-- Short questions
-- Result: Low cost, good overview
-
-**Analysis budget**
-- Use powerful model
-- Focused context (main source only)
-- Detailed questions
-- Result: Higher cost, deep insights
-
-**Synthesis budget**
-- Use powerful model for final synthesis
-- Multiple sources (Full Content)
-- Complex comparative questions
-- Result: Expensive but valuable output
-
----
-
-## Troubleshooting Chat Issues
-
-### Poor Responses
-
-| Problem | Cause | Solution |
-|---------|-------|----------|
-| Generic answers | Vague question | Be specific (see question patterns) |
-| Missing context | Not enough in context | Add sources or change to Full Content |
-| Incorrect info | Source not in context | Add the relevant source |
-| Hallucinating | Model confused | Ask for citations, verify claims |
-| Shallow analysis | Wrong model | Switch to more powerful model |
-
-### High Costs
-
-| Problem | Cause | Solution |
-|---------|-------|----------|
-| Expensive per response | Too much context | Use Summary Only or exclude sources |
-| Many follow-ups | Exploratory chat | Use Ask instead for single comprehensive answer |
-| Long conversations | Keeping history | Archive old chats, start fresh |
-| Large sources | Full text in context | Use Summary Only for large documents |
-
----
-
-## Best Practices
-
-### Before You Chat
-
-- [ ] Add sources you'll need
-- [ ] Decide context strategy (Tiered is usually best)
-- [ ] Choose model (cheaper for exploration, powerful for analysis)
-- [ ] Have a question in mind
-
-### During Chat
-
-- [ ] Ask specific questions (use SPECIFIC formula)
-- [ ] Check citations for factual claims
-- [ ] Follow up on unclear points
-- [ ] Adjust context if you need different sources
-
-### After Chat
-
-- [ ] Save good responses as notes
-- [ ] Archive conversation if you're done
-- [ ] Organize notes for future reference
-- [ ] Use insights in other features (Ask, Transformations, Podcasts)
-
----
-
-## When to Use Chat vs. Ask
-
-**Use CHAT when:**
-- You want a dialogue
-- You're exploring a topic
-- You'll ask multiple related questions
-- You want to adjust context during conversation
-- You're not sure exactly what you need
-
-**Use ASK when:**
-- You have one specific question
-- You want a comprehensive answer
-- You want the system to auto-search
-- You want one response, not dialogue
-- You want maximum tokens spent on search
-
----
-
-## Summary: Chat as Conversation
-
-Chat is fundamentally different from asking ChatGPT directly:
-
-| Aspect | ChatGPT | Open Notebook Chat |
-|--------|---------|-------------------|
-| **Source control** | None (uses training) | You control which sources are visible |
-| **Cost control** | Per token | Per token, but context is your choice |
-| **Iteration** | Works | Works, with your sources changing dynamically |
-| **Citations** | Made up often | Tied to your sources (verifiable) |
-| **Privacy** | Your data to OpenAI | Your data stays local (unless you choose) |
-
-The key insight: **Chat is retrieval-augmented generation.** AI sees only what you put in context. You control the conversation and the information flow.
-
-That's why Chat is powerful for research. You're not just talking to an AI; you're having a conversation with your research itself.

docs/3-USER-GUIDE/citations.md

@@ -1,299 +0,0 @@
-# Citations - Verify and Trust AI Responses
-
-Citations connect AI responses to your source materials. This guide covers how to use and verify them.
-
----
-
-## Why Citations Matter
-
-Every AI-generated response in Open Notebook includes citations to your sources. This lets you:
-
-- **Verify claims** - Check that AI actually read what it claims
-- **Find original context** - See the full passage around a quote
-- **Catch hallucinations** - Spot when AI makes things up
-- **Build credibility** - Your notes have traceable sources
-
----
-
-## Quick Start: Using Citations
-
-### Reading Citations
-
-```
-AI Response:
-"The study found a 95% accuracy rate [1] using the proposed method."
-
-[1] = Click to see source
-
-What happens when you click:
-→ Opens the source document
-→ Highlights the relevant section
-→ You can verify the claim
-```
-
-### Requesting Better Citations
-
-If a response lacks citations, ask:
-
-```
-"Please cite the specific page or section for that claim."
-"Where in the document does it say that?"
-"Can you quote the exact text?"
-```
-
----
-
-## How Citations Work
-
-### Automatic Generation
-
-When AI references your sources, citations are generated automatically:
-
-```
-1. AI analyzes your question
-2. Retrieves relevant source chunks
-3. Generates response with inline citations
-4. Links citations to original source locations
-```
-
-### Citation Format
-
-```
-Inline format:
-"The researchers concluded X [1] and Y [2]."
-
-Reference list:
-[1] Paper Title - Section 3.2
-[2] Report Name - Page 15
-
-Clickable: Each [number] links to the source
-```
-
----
-
-## Verifying Citations
-
-### The Verification Workflow
-
-```
-Step 1: Read AI response
-        "The model achieved 95% accuracy [1]"
-
-Step 2: Click citation [1]
-        → Opens source document
-        → Shows relevant passage
-
-Step 3: Verify the claim
-        Does source actually say 95%?
-        Is context correct?
-        Any nuance missed?
-
-Step 4: Trust or correct
-        ✓ Accurate → Use the insight
-        ✗ Wrong → Ask AI to correct
-```
-
-### What to Check
-
-| Check | Why |
-|-------|-----|
-| **Exact numbers** | AI sometimes rounds or misremembers |
-| **Context** | Quote might mean something different in context |
-| **Attribution** | Is this the source's claim or someone they cited? |
-| **Completeness** | Did AI miss important caveats? |
-
----
-
-## Citations in Different Features
-
-### Chat Citations
-
-```
-Context: Sources you selected
-Citations: Reference chunks used in response
-Verification: Click to see original text
-Save: Citations preserved when saving as note
-```
-
-### Ask Feature Citations
-
-```
-Context: Auto-searched across all sources
-Citations: Multiple sources synthesized
-Verification: Each source linked separately
-Quality: Often more comprehensive than Chat
-```
-
-### Transformation Citations
-
-```
-Context: Single source being transformed
-Citations: Points back to original document
-Verification: Compare output to source
-Use: When you need structured extraction
-```
-
----
-
-## Saving Citations
-
-### In Notes
-
-When you save an AI response as a note, citations are preserved:
-
-```
-Original response:
-"According to the paper [1], the method works by..."
-
-Saved note includes:
-- The text
-- The citation link
-- Reference to source document
-```
-
-### Exporting
-
-Citations work in exports:
-
-| Format | Citation Behavior |
-|--------|-------------------|
-| **Markdown** | Links preserved as `[text](link)` |
-| **Copy/Paste** | Plain text with reference numbers |
-| **PDF** | Clickable references (if supported) |
-
----
-
-## Citation Quality Tips
-
-### Get Better Citations
-
-**Be specific in questions:**
-```
-Poor: "What does it say about X?"
-Good: "What does page 15 say about X? Please quote directly."
-```
-
-**Request citation format:**
-```
-"Include page numbers for each claim."
-"Cite specific sections, not just document names."
-```
-
-**Use Full Content context:**
-```
-Summary Only → Less precise citations
-Full Content → Exact quotes possible
-```
-
-### When Citations Are Missing
-
-| Situation | Cause | Solution |
-|-----------|-------|----------|
-| No citations | AI used general knowledge | Ask: "Base your answer only on my sources" |
-| Vague citations | Source not in Full Content | Change context level |
-| Wrong citations | AI confused sources | Ask to verify with quotes |
-
----
-
-## Common Issues
-
-### "Citation doesn't match claim"
-
-```
-Problem: AI says X, but source says Y
-
-What happened:
-- AI paraphrased incorrectly
-- AI combined multiple sources confusingly
-- Source was taken out of context
-
-Solution:
-1. Click citation to see original
-2. Note the discrepancy
-3. Ask AI: "The source says Y, not X. Please correct."
-```
-
-### "Can't find cited section"
-
-```
-Problem: Citation link doesn't show relevant text
-
-What happened:
-- Source was chunked differently than expected
-- Information spread across multiple sections
-- Processing missed some content
-
-Solution:
-1. Search within source for key terms
-2. Ask AI for more specific location
-3. Re-process source if needed
-```
-
-### "No citations at all"
-
-```
-Problem: AI response has no source references
-
-What happened:
-- Sources not in context
-- Question asked for opinion/general knowledge
-- Model didn't find relevant content
-
-Solution:
-1. Check context settings
-2. Rephrase: "Based on my sources, what..."
-3. Add more relevant sources
-```
-
----
-
-## Best Practices
-
-### For Research Integrity
-
-1. **Always verify important claims** - Don't trust AI blindly
-2. **Check context** - Quotes can be misleading out of context
-3. **Note limitations** - AI might miss nuance
-4. **Keep source access** - Don't delete sources you cite
-
-### For Academic Work
-
-1. **Use Full Content** for documents you'll cite
-2. **Request specific page numbers**
-3. **Cross-check with original sources**
-4. **Document your verification process**
-
-### For Professional Use
-
-1. **Verify before sharing** - Check claims clients will see
-2. **Keep citation trail** - Save notes with sources linked
-3. **Be transparent** - Note when insights are AI-assisted
-
----
-
-## Summary
-
-```
-Citations = Your verification system
-
-How to use:
-1. Read AI response
-2. Note citation markers [1], [2], etc.
-3. Click to see original source
-4. Verify claim matches source
-5. Trust verified insights
-
-When citations fail:
-- Ask for specific quotes
-- Change to Full Content
-- Request page numbers
-- Verify manually
-
-Why it matters:
-- AI can hallucinate
-- Context can change meaning
-- Trust requires verification
-- Good research needs sources
-```
-
-Citations aren't just references — they're your quality control. Use them to build research you can trust.

docs/3-USER-GUIDE/creating-podcasts.md

@@ -1,676 +0,0 @@
-# Creating Podcasts - Turn Research into Audio
-
-Podcasts let you consume your research passively. This guide covers the complete workflow from setup to download.
-
----
-
-## Quick-Start: Your First Podcast (5 Minutes)
-
-```
-1. Go to your notebook
-2. Click "Generate Podcast"
-3. Select sources to include
-4. Choose a speaker profile (or use default)
-5. Click "Generate"
-6. Wait 3-10 minutes (non-blocking)
-7. Download MP3 when ready
-8. Done!
-```
-
-That's the minimum. Let's make it better.
-
----
-
-## Step-by-Step: The Complete Workflow
-
-### Step 1: Prepare Your Notebook
-
-```
-Before generating, make sure:
-
-✓ You have sources added
-  (At least 1-2 sources)
-
-✓ Sources have been processed
-  (Green "Ready" status)
-
-✓ Notes are organized
-  (If you want notes included)
-
-✓ You know your message
-  (What's the main story?)
-
-Typical preparation: 5-10 minutes
-```
-
-### Step 2: Choose Content
-
-```
-Click "Generate Podcast"
-
-You'll see:
-- List of all sources in notebook
-- List of all notes
-
-Select which to include:
-☑ Paper A (primary source)
-☑ Paper B (supporting source)
-☐ Old note (not relevant)
-✓ Analysis note (important)
-
-What to include:
-- Primary sources: Always include
-- Supporting sources: Usually include
-- Notes: Include your analysis/insights
-- Everything: Can overload podcast
-
-Recommended: 3-5 sources per podcast
-```
-
-### Step 3: Choose Episode Profile
-
-An episode profile defines the structure and tone.
-
-**Option A: Use Preset Profile**
-
-```
-Open Notebook provides templates:
-
-Academic Presentation (Monologue)
-├─ 1 speaker
-├─ Tone: Educational
-└─ Format: Expert explaining topic
-
-Expert Interview (2-speaker)
-├─ 2 speakers: Host + Expert
-├─ Tone: Q&A, conversational
-└─ Format: Interview with expert
-
-Debate Format (2-speaker)
-├─ 2 speakers: Pro vs. Con
-├─ Tone: Discussion, disagreement
-└─ Format: Debate about the topic
-
-Panel Discussion (3-4 speaker)
-├─ 3-4 speakers: Different perspectives
-├─ Tone: Thoughtful discussion
-└─ Format: Each brings different expertise
-
-Solo Explanation (Monologue)
-├─ 1 speaker
-├─ Tone: Conversational, friendly
-└─ Format: Personal explanation
-```
-
-**Pick based on your content:**
-- One main idea → Academic Presentation
-- You want to explain → Solo Explanation
-- Two competing views → Debate Format
-- Multiple perspectives → Panel Discussion
-- Want to explore → Expert Interview
-
-### Step 4: Customize Episode Profile (Optional)
-
-If presets don't fit, customize:
-
-```
-Episode Profile
-├─ Title: "AI Safety in 2026"
-├─ Description: "Exploring current approaches"
-├─ Length target: 20 minutes
-├─ Tone: "Academic but accessible"
-├─ Focus areas:
-│  ├─ Main approaches to alignment
-│  ├─ Pros and cons comparison
-│  └─ Open questions
-├─ Audience: "Researchers new to field"
-└─ Format: "Debate between two perspectives"
-
-How to set:
-1. Click "Customize"
-2. Edit each field
-3. Click "Save Profile"
-4. System uses your profile for outline generation
-```
-
-### Step 5: Create or Select Speakers
-
-Speakers are the "voice" of your podcast.
-
-**Option A: Use Preset Speakers**
-
-```
-Open Notebook provides templates:
-
-"Expert Alex"
-- Expertise: Deep knowledge
-- Personality: Rigorous, patient
-- TTS: OpenAI (clear voice)
-
-"Curious Sam"
-- Expertise: Curious newcomer
-- Personality: Asks questions
-- TTS: Google (natural voice)
-
-"Skeptic Jordan"
-- Expertise: Critical perspective
-- Personality: Challenges assumptions
-- TTS: ElevenLabs (warm voice)
-
-For your first podcast: Use presets
-For custom podcast: Create your own
-```
-
-**Option B: Create Custom Speakers**
-
-```
-Click "Add Speaker"
-
-Fill in:
-
-Name: "Dr. Research Expert"
-
-Expertise:
-"20 years in AI safety research,
- deep knowledge of alignment approaches"
-
-Personality:
-"Rigorous, academic style,
- explains clearly, asks good questions"
-
-Voice Configuration:
-- TTS Provider: OpenAI / Google / ElevenLabs / Local
-- Voice selection: Choose from available voices
-- Accent (optional): British / American / etc.
-
-Example:
-Name: Dr. Research Expert
-Expertise: AI safety alignment research
-Personality: Rigorous, academic but accessible
-Voice: ElevenLabs - professional male voice
-```
-
-### Step 6: Generate Podcast
-
-```
-1. Review your setup:
-   Sources: ✓ Selected
-   Profile: ✓ Episode profile chosen
-   Speakers: ✓ Speakers configured
-
-2. Click "Generate Podcast"
-
-3. System begins:
-   - Analyzing your content
-   - Creating outline
-   - Writing dialogue
-   - Generating audio
-   - Mixing speakers
-
-4. Status shows progress:
-   20% Outline generation
-   40% Dialogue writing
-   60% Audio synthesis
-   80% Mixing
-   100% Complete
-
-Processing time:
-- 5 minutes of content: 3-5 minutes
-- 15 minutes of content: 5-10 minutes
-- 30 minutes of content: 10-20 minutes
-```
-
-### Step 7: Review and Download
-
-```
-When complete:
-
-Preview:
-- Play audio sample
-- Review transcript
-- Check duration
-
-Options:
-✓ Download as MP3 - Save to computer
-✓ Stream directly - Listen in browser
-✓ Share link - Get shareable URL (if public)
-✓ Regenerate - Try different speakers/profile
-
-Download:
-1. Click "Download as MP3"
-2. Choose quality: 128kbps / 192kbps / 320kbps
-3. Save file: podcast_[notebook]_[date].mp3
-4. Listen!
-```
-
----
-
-## Understanding What Happens Behind the Scenes
-
-### The Generation Pipeline
-
-```
-Stage 1: CONTENT ANALYSIS (1 minute)
-  Your sources → What's the main story?
-               → Key themes?
-               → Debate points?
-
-Stage 2: OUTLINE CREATION (2-3 minutes)
-  Themes → Episode structure
-        → Section breakdown
-        → Talking points
-
-Stage 3: DIALOGUE WRITING (2-3 minutes)
-  Outline → Convert to natural dialogue
-         → Add speaker personalities
-         → Create flow and transitions
-
-Stage 4: AUDIO SYNTHESIS (3-5 minutes per speaker)
-  Script + Speaker → Text-to-speech
-                  → Individual audio files
-                  → High quality audio
-
-Stage 5: MIXING & MASTERING (1-2 minutes)
-  Multiple audio → Combine speakers
-               → Level audio
-               → Add polish
-               → Final MP3
-
-Total: 10-20 minutes for typical podcast
-```
-
----
-
-## Text-to-Speech Providers
-
-Different providers, different qualities.
-
-### OpenAI (Recommended)
-
-```
-Voices: 5 options (Alloy, Echo, Fable, Onyx, Shimmer)
-Quality: Good, natural sounding
-Speed: Fast
-Cost: ~$0.015 per minute
-Best for: General purpose, natural speech
-Example: "I have to say, the research shows..."
-```
-
-### Google TTS
-
-```
-Voices: Many options, various accents
-Quality: Excellent, very natural
-Speed: Fast
-Cost: ~$0.004 per minute
-Best for: High quality output, accents
-Example: "The research demonstrates that..."
-```
-
-### ElevenLabs
-
-```
-Voices: 100+ voices, highly customizable
-Quality: Exceptional, very expressive
-Speed: Slower (5-10 seconds per phrase)
-Cost: ~$0.10 per minute
-Best for: Premium quality, emotional range
-Example: [Can convey emotion and tone]
-```
-
-### Local TTS (Free)
-
-```
-Voices: Limited, basic options
-Quality: Basic, robotic
-Speed: Depends on hardware (slow)
-Cost: Free (local processing)
-Best for: Privacy, testing, offline use
-Example: "The research shows..."
-Privacy: Everything stays on your computer
-```
-
-### Which Provider to Choose?
-
-```
-For your first podcast: Google (quality/cost balance)
-For privacy-sensitive: Local TTS (free, private)
-For premium quality: ElevenLabs (best voices)
-For budget: Google (cheapest quality option)
-For speed: OpenAI (fast generation)
-```
-
----
-
-## Tips for Better Podcasts
-
-### Choose Right Profile
-
-```
-Single source analysis → Academic Presentation
-  "Explaining one paper to someone new"
-
-Comparing two approaches → Debate Format
-  "Pros and cons of different methods"
-
-Multiple sources + insights → Panel Discussion
-  "Different experts discussing topic"
-
-Narrative exploration → Expert Interview
-  "Host interviewing research expert"
-
-Personal take → Solo Explanation
-  "You explaining your analysis"
-```
-
-### Create Good Speakers
-
-```
-Good Speaker:
-✓ Clear expertise (know what they're talking about)
-✓ Distinct personality (not generic)
-✓ Good voice choice (matches personality)
-✓ Realistic backstory (feels like real person)
-
-Bad Speaker:
-✗ Generic expertise ("good at research")
-✗ No personality ("just reads")
-✗ Mismatched voice (deep voice for young person)
-✗ Contradicts personality (serious person uses casual voice)
-```
-
-### Focus Content
-
-```
-Better: Podcast on ONE specific topic
-  "How transformers work" (15 minutes, focused)
-
-Worse: Podcast on everything
-  "All of AI 2025" (2 hours, unfocused)
-
-Guideline:
-- 5-10 minutes: One narrow topic
-- 15-20 minutes: One broad topic
-- 30+ minutes: Multiple related subtopics
-
-Shorter is usually better for podcasts.
-```
-
-### Optimize Source Selection
-
-```
-Too much content:
-  "Here are all 20 papers"
-  → Podcast becomes 2+ hours
-  → Unfocused
-  → Low quality
-
-Right amount:
-  "Here are 3 key papers"
-  → Podcast is 15-20 minutes
-  → Focused
-  → High quality
-
-Rule: 3-5 sources per podcast
-     Remove long background papers
-     Keep focused on main topic
-```
-
----
-
-## Quality Troubleshooting
-
-### Audio Sounds Robotic
-
-**Problem**: TTS voice sounds unnatural
-
-**Solutions**:
-```
-1. Switch provider: Try Google or ElevenLabs instead
-2. Choose different voice: Some voices more natural
-3. Shorter sentences: Very long sentences sound robotic
-4. Adjust pacing: Ask for "natural, conversational pacing"
-```
-
-### Audio Sounds Unclear
-
-**Problem**: Hard to understand what's being said
-
-**Solutions**:
-```
-1. Re-generate with different speaker
-2. Try different TTS provider
-3. Use speakers with clear accents
-4. Lower background noise (if any)
-5. Increase speech rate (if too slow)
-```
-
-### Missing Content
-
-**Problem**: Important information isn't in podcast
-
-**Solutions**:
-```
-1. Include that source in content selection
-2. Review generated outline (check before generating)
-3. Regenerate with clearer profile instructions
-4. Try different model (more thorough model)
-```
-
-### Speakers Don't Match
-
-**Problem**: Speakers sound like same person
-
-**Solutions**:
-```
-1. Choose different TTS providers (OpenAI + Google)
-2. Choose very different voice options
-3. Increase personality differences in profile
-4. Try different speaker count (2 vs 3 vs 4)
-```
-
-### Generation Failed
-
-**Problem**: "Podcast generation failed"
-
-**Solutions**:
-```
-1. Check internet connection (especially TTS)
-2. Try again (might be temporary issue)
-3. Use local TTS (doesn't need internet)
-4. Reduce source count (less to process)
-5. Contact support if persistent
-```
-
----
-
-## Advanced: Multiple Podcasts from Same Research
-
-You can generate different podcasts from one notebook:
-
-```
-Podcast 1: Overview
-  Profile: Academic Presentation
-  Sources: Papers A, B, C
-  Speakers: One expert
-  Length: 15 minutes
-
-→ Use for "What's this about?" understanding
-
-Podcast 2: Deep Dive
-  Profile: Expert Interview
-  Sources: Paper A (Full) + B, C (Summary)
-  Speakers: Expert + Interviewer
-  Length: 30 minutes
-
-→ Use for detailed exploration
-
-Podcast 3: Debate
-  Profile: Debate Format
-  Sources: Papers A vs B (different approaches)
-  Speakers: Pro-A speaker + Pro-B speaker
-  Length: 20 minutes
-
-→ Use for comparing approaches
-```
-
-Each tells the same story from different angles.
-
----
-
-## Exporting and Sharing
-
-### Download MP3
-
-```
-1. Generation complete
-2. Click "Download"
-3. Choose quality:
-   - 128 kbps: Smallest file, lower quality
-   - 192 kbps: Balanced (recommended)
-   - 320 kbps: Highest quality, largest file
-4. Save to computer
-5. Use in podcast app, upload to platform, etc.
-```
-
-### Export Transcript
-
-```
-1. Click "Export Transcript"
-2. Get full dialogue as text
-3. Useful for:
-   - Blog post content
-   - Show notes
-   - Searchable text version
-   - Accessibility
-```
-
-### Share Link
-
-```
-If podcast is public:
-1. Click "Share"
-2. Get shareable link
-3. Others can listen/download
-4. Useful for:
-   - Sharing with team
-   - Public distribution
-   - Embedding on website
-```
-
-### Publish to Podcast Platforms
-
-```
-If you want to distribute (future feature):
-1. Download MP3
-2. Upload to platform (Spotify, Apple Podcasts, etc.)
-3. Add metadata (title, description, episode notes)
-4. Your research becomes a published podcast!
-```
-
----
-
-## Best Practices
-
-### Before Generation
-- [ ] Sources are processed and ready
-- [ ] You've chosen content to include
-- [ ] You have a clear episode profile
-- [ ] Speakers are well-defined
-- [ ] Content is focused (3-5 sources max)
-
-### During Generation
-- Don't close the browser (use background processing)
-- Check back in 5-15 minutes
-- Review transcript when complete
-- Listen to sample before downloading
-
-### After Generation
-- [ ] Download MP3 to computer
-- [ ] Save in organized folder
-- [ ] Add metadata (title, description, date)
-- [ ] Test listening in podcast app
-- [ ] Share with colleagues for feedback
-
----
-
-## Use Cases
-
-### Academic Researcher
-```
-Podcast: Explaining your dissertation
-Speakers: You + colleague
-Content: Your papers + supporting research
-Use: Share with advisors, test explanations
-```
-
-### Content Creator
-```
-Podcast: Research-to-podcast article
-Speakers: Narrator + expert
-Content: Articles you've researched
-Use: Transform article into podcast version
-```
-
-### Team Research
-```
-Podcast: Weekly research updates
-Speakers: Multiple team members
-Content: This week's papers
-Use: Team updates, knowledge sharing
-```
-
-### Learning/Teaching
-```
-Podcast: Teaching material
-Speakers: Teacher + inquisitive student
-Content: Textbook + examples
-Use: Students learn while commuting
-```
-
----
-
-## Cost Breakdown Example
-
-### Generate 15-minute podcast with ElevenLabs
-
-```
-Generation (outline + dialogue):
-  No charge (included in service)
-
-Text-to-speech:
-  2 speakers × 15 minutes = 30 minutes TTS
-  ElevenLabs: $0.10 per minute
-  Cost: 30 × $0.10 = $3.00
-
-Processing:
-  Included (no additional cost)
-
-Total: $3.00 per podcast
-
-Cheaper options:
-  With Google TTS: ~$0.12
-  With OpenAI: ~$0.45
-  With Local TTS: ~$0.00
-```
-
----
-
-## Summary: Podcasts as Research Tool
-
-Podcasts transform how you consume research:
-
-```
-Before: Reading papers takes time, focus
-After: Listen while commuting, exercising, doing chores
-
-Before: Can't share complex research easily
-After: Share audio of your analysis
-
-Before: Different consumption styles isolated
-After: Same research, multiple formats (read/listen)
-```
-
-Podcasts aren't just for entertainment—they're a tool for making research more accessible, shareable, and consumable.
-
-That's why they're important for Open Notebook.

docs/3-USER-GUIDE/index.md

@@ -1,193 +0,0 @@
-# User Guide - How to Use Open Notebook
-
-This guide covers practical, step-by-step usage of Open Notebook features. You already understand the concepts; now learn how to actually use them.
-
-> **Prerequisite**: Review [2-CORE-CONCEPTS](../2-CORE-CONCEPTS/index.md) first to understand the mental models (notebooks, sources, notes, chat, transformations, podcasts).
-
----
-
-## Start Here
-
-### [Interface Overview](interface-overview.md)
-Learn the layout before diving in. Understand the three-panel design and where everything is.
-
----
-
-## Eight Core Features
-
-### 1. [Adding Sources](adding-sources.md)
-How to bring content into your notebook. Supports PDFs, web links, audio, video, text, and more.
-
-**Quick links:**
-- Upload a PDF or document
-- Add a web link or article
-- Transcribe audio or video
-- Paste text directly
-- Common mistakes + fixes
-
----
-
-### 2. [Working with Notes](working-with-notes.md)
-Creating, organizing, and using notes (both manual and AI-generated).
-
-**Quick links:**
-- Create a manual note
-- Save AI responses as notes
-- Apply transformations to generate insights
-- Organize with tags and naming
-- Use notes across your notebook
-
----
-
-### 3. [Chat Effectively](chat-effectively.md)
-Have conversations with AI about your sources. Manage context to control what AI sees.
-
-**Quick links:**
-- Start your first chat
-- Select which sources go in context
-- Ask effective questions
-- Use follow-ups productively
-- Understand citations and verify claims
-
----
-
-### 4. [Creating Podcasts](creating-podcasts.md)
-Convert your research into audio dialogue for passive consumption.
-
-**Quick links:**
-- Create your first podcast
-- Choose or customize speakers
-- Select TTS provider
-- Generate and download
-- Common audio quality fixes
-
----
-
-### 5. [Search Effectively](search.md)
-Two search modes: text-based (keyword) and vector-based (semantic). Know when to use each.
-
-**Quick links:**
-- Text search vs vector search (when to use)
-- Running effective searches
-- Using the Ask feature for comprehensive answers
-- Saving search results as notes
-- Troubleshooting poor results
-
----
-
-### 6. [Transformations](transformations.md)
-Batch-process sources with predefined templates. Extract the same insights from multiple documents.
-
-**Quick links:**
-- Built-in transformation templates
-- Creating custom transformations
-- Applying to single or multiple sources
-- Managing transformation output
-
----
-
-### 7. [Citations](citations.md)
-Verify AI claims by tracing them back to source material. Understand the citation system.
-
-**Quick links:**
-- Reading and clicking citations
-- Verifying claims against sources
-- Requesting better citations
-- Saving cited content as notes
-
----
-
-## Which Feature for Which Task?
-
-```
-Task: "I want to explore a topic with follow-ups"
-→ Use: Chat (add sources, select context, have conversation)
-
-Task: "I want one comprehensive answer"
-→ Use: Search / Ask (system finds relevant content)
-
-Task: "I want to extract the same info from many sources"
-→ Use: Transformations (define template, apply to all)
-
-Task: "I want summaries of all my sources"
-→ Use: Transformations (with built-in summary template)
-
-Task: "I want to share my research in audio form"
-→ Use: Podcasts (create speakers, generate episode)
-
-Task: "I want to find that quote I remember"
-→ Use: Search / Text Search (keyword matching)
-
-Task: "I'm exploring a concept without knowing exact words"
-→ Use: Search / Vector Search (semantic similarity)
-```
-
----
-
-## Quick-Start Checklist: First 15 Minutes
-
-**Step 1: Create a Notebook (1 min)**
-- Name: Something descriptive ("Q1 Market Research", "AI Safety Papers", etc.)
-- Description: 1-2 sentences about what you're researching
-- This is your research container
-
-**Step 2: Add Your First Source (3 min)**
-- Pick one: PDF, web link, or text
-- Follow [Adding Sources](adding-sources.md)
-- Wait for processing (usually 30-60 seconds)
-
-**Step 3: Chat About It (3 min)**
-- Go to Chat
-- Select your source (set context to "Full Content")
-- Ask a simple question: "What are the main points?"
-- See AI respond with citations
-
-**Step 4: Save Insight as Note (2 min)**
-- Good response? Click "Save as Note"
-- Name it something useful ("Main points from source X")
-- Now you have a captured insight
-
-**Step 5: Explore More (6 min)**
-- Add another source
-- Chat about both together
-- Ask a question that compares them
-- Follow up with clarifying questions
-
-**Done!** You've used the core workflow: notebook → sources → chat → notes
-
----
-
-## Common Mistakes to Avoid
-
-| Mistake | Problem | Fix |
-|---------|---------|-----|
-| Adding everything to one notebook | No isolation between projects | Create separate notebooks for different topics |
-| Expecting AI to know your context | Questions get generic answers | Describe your research focus in chat context |
-| Forgetting to cite sources | You can't verify claims | Click citations to check source chunks |
-| Using Chat for one-time questions | Slower than Ask | Use Ask for comprehensive Q&A, Chat for exploration |
-| Adding huge PDFs without chunking | Slow processing, poor search | Break into multiple smaller sources if possible |
-| Using same context for all chats | Expensive, unfocused | Adjust context level for each chat |
-| Ignoring vector search | Only finding exact keywords | Use vector search to explore conceptually |
-
----
-
-## Next Steps
-
-1. **Follow each guide** in order (sources → notes → chat → podcasts → search)
-2. **Create your first notebook** with real content
-3. **Practice each feature** with your own research
-4. **Return to CORE-CONCEPTS** if you need to understand the "why"
-
----
-
-## Getting Help
-
-- **Feature not working?** → Check the feature's guide (look for "Troubleshooting" section)
-- **Error message?** → Check [6-TROUBLESHOOTING](../6-TROUBLESHOOTING/index.md)
-- **Understanding how something works?** → Check [2-CORE-CONCEPTS](../2-CORE-CONCEPTS/index.md)
-- **Setting up for the first time?** → Go back to [1-INSTALLATION](../1-INSTALLATION/index.md)
-- **For developers** → See [7-DEVELOPMENT](../7-DEVELOPMENT/index.md)
-
----
-
-**Ready to start?** Pick the guide for what you want to do first!

docs/3-USER-GUIDE/interface-overview.md

@@ -1,377 +0,0 @@
-# Interface Overview - Finding Your Way Around
-
-Open Notebook uses a clean three-panel layout. This guide shows you where everything is.
-
----
-
-## The Main Layout
-
-```
-┌─────────────────────────────────────────────────────────────┐
-│  [Logo]  Notebooks  Search  Podcasts  Models  Settings      │
-├──────────────┬──────────────┬───────────────────────────────┤
-│              │              │                               │
-│   SOURCES    │    NOTES     │           CHAT                │
-│              │              │                               │
-│  Your docs   │  Your        │   Talk to AI about            │
-│  PDFs, URLs  │  insights    │   your sources                │
-│  Videos      │  summaries   │                               │
-│              │              │                               │
-│  [+Add]      │  [+Write]    │   [Type here...]              │
-│              │              │                               │
-└──────────────┴──────────────┴───────────────────────────────┘
-```
-
----
-
-## Navigation Bar
-
-The top navigation takes you to main sections:
-
-| Icon | Page | What It Does |
-|------|------|--------------|
-| **Notebooks** | Main workspace | Your research projects |
-| **Search** | Ask & Search | Query across all notebooks |
-| **Podcasts** | Audio generation | Manage podcast profiles |
-| **Models** | AI configuration | Set up providers and models |
-| **Settings** | Preferences | App configuration |
-
----
-
-## Left Panel: Sources
-
-Your research materials live here.
-
-### What You'll See
-
-```
-┌─────────────────────────┐
-│  Sources (5)            │
-│  [+ Add Source]         │
-├─────────────────────────┤
-│  ┌─────────────────┐    │
-│  │ 📄 Paper.pdf    │    │
-│  │ 🟢 Full Content │    │
-│  │ [⋮ Menu]        │    │
-│  └─────────────────┘    │
-│                         │
-│  ┌─────────────────┐    │
-│  │ 🔗 Article URL  │    │
-│  │ 🟡 Summary Only │    │
-│  │ [⋮ Menu]        │    │
-│  └─────────────────┘    │
-└─────────────────────────┘
-```
-
-### Source Card Elements
-
-- **Icon** - File type (PDF, URL, video, etc.)
-- **Title** - Document name
-- **Context indicator** - What AI can see:
-  - 🟢 Full Content
-  - 🟡 Summary Only
-  - ⛔ Not in Context
-- **Menu (⋮)** - Edit, transform, delete
-
-### Add Source Button
-
-Click to add:
-- File upload (PDF, DOCX, etc.)
-- Web URL
-- YouTube video
-- Plain text
-
----
-
-## Middle Panel: Notes
-
-Your insights and AI-generated content.
-
-### What You'll See
-
-```
-┌─────────────────────────┐
-│  Notes (3)              │
-│  [+ Write Note]         │
-├─────────────────────────┤
-│  ┌─────────────────┐    │
-│  │ 📝 My Analysis  │    │
-│  │ Manual note     │    │
-│  │ Jan 3, 2026     │    │
-│  └─────────────────┘    │
-│                         │
-│  ┌─────────────────┐    │
-│  │ 🤖 Summary      │    │
-│  │ From transform  │    │
-│  │ Jan 2, 2026     │    │
-│  └─────────────────┘    │
-└─────────────────────────┘
-```
-
-### Note Card Elements
-
-- **Icon** - Note type (manual 📝 or AI 🤖)
-- **Title** - Note name
-- **Origin** - How it was created
-- **Date** - When created
-
-### Write Note Button
-
-Click to:
-- Create manual note
-- Add your own insights
-- Markdown supported
-
----
-
-## Right Panel: Chat
-
-Your AI conversation space.
-
-### What You'll See
-
-```
-┌───────────────────────────────┐
-│  Chat                         │
-│  Session: Research Discussion │
-│  [+ New Session] [Sessions ▼] │
-├───────────────────────────────┤
-│                               │
-│  You: What's the main         │
-│       finding?                │
-│                               │
-│  AI: Based on the paper [1],  │
-│      the main finding is...   │
-│      [Save as Note]           │
-│                               │
-│  You: Tell me more about      │
-│       the methodology.        │
-│                               │
-├───────────────────────────────┤
-│  Context: 3 sources (12K tok) │
-├───────────────────────────────┤
-│  [Type your message...]  [↑]  │
-└───────────────────────────────┘
-```
-
-### Chat Elements
-
-- **Session selector** - Switch between conversations
-- **Message history** - Your conversation
-- **Save as Note** - Keep good responses
-- **Context indicator** - What AI can see
-- **Input field** - Type your questions
-
----
-
-## Context Indicators
-
-These show what AI can access:
-
-### Token Counter
-
-```
-Context: 3 sources (12,450 tokens)
-         ↑          ↑
-         Sources    Approximate cost indicator
-         included
-```
-
-### Per-Source Indicators
-
-| Indicator | Meaning | AI Access |
-|-----------|---------|-----------|
-| 🟢 Full Content | Complete text | Everything |
-| 🟡 Summary Only | AI summary | Key points only |
-| ⛔ Not in Context | Excluded | Nothing |
-
-Click any source to change its context level.
-
----
-
-## Podcasts Tab
-
-Inside a notebook, switch to Podcasts:
-
-```
-┌───────────────────────────────┐
-│  [Chat]  [Podcasts]           │
-├───────────────────────────────┤
-│  Episode Profile: [Select ▼]  │
-│                               │
-│  Speakers:                    │
-│  ├─ Host: Alex (OpenAI)       │
-│  └─ Guest: Sam (Google)       │
-│                               │
-│  Include:                     │
-│  ☑ Paper.pdf                  │
-│  ☑ My Analysis (note)         │
-│  ☐ Background article         │
-│                               │
-│  [Generate Podcast]           │
-└───────────────────────────────┘
-```
-
----
-
-## Settings Page
-
-Access via navigation bar → Settings:
-
-### Key Sections
-
-| Section | What It Controls |
-|---------|------------------|
-| **Processing** | Document and URL extraction engines |
-| **Embedding** | Auto-embed settings |
-| **Files** | Auto-delete uploads after processing |
-| **YouTube** | Preferred transcript languages |
-
----
-
-## Models Page
-
-Configure AI providers:
-
-```
-┌───────────────────────────────────────┐
-│  Models                               │
-├───────────────────────────────────────┤
-│  Language Models                      │
-│  ┌─────────────────────────────────┐  │
-│  │ GPT-4o (OpenAI)         [Edit]  │  │
-│  │ Claude Sonnet (Anthropic)       │  │
-│  │ Llama 3.3 (Ollama)      [⭐]    │  │
-│  └─────────────────────────────────┘  │
-│  [+ Add Model]                        │
-│                                       │
-│  Embedding Models                     │
-│  ┌─────────────────────────────────┐  │
-│  │ text-embedding-3-small  [⭐]    │  │
-│  └─────────────────────────────────┘  │
-│                                       │
-│  Text-to-Speech                       │
-│  ┌─────────────────────────────────┐  │
-│  │ OpenAI TTS             [⭐]     │  │
-│  │ Google TTS                      │  │
-│  └─────────────────────────────────┘  │
-└───────────────────────────────────────┘
-```
-
-- **⭐** = Default model for that category
-- **[Edit]** = Modify configuration
-- **[+ Add]** = Add new model
-
----
-
-## Search Page
-
-Query across all notebooks:
-
-```
-┌───────────────────────────────────────┐
-│  Search                               │
-├───────────────────────────────────────┤
-│  [What are you looking for?    ] [🔍] │
-│                                       │
-│  Search type: [Text ▼] [Vector ▼]     │
-│  Search in:   [Sources] [Notes]       │
-├───────────────────────────────────────┤
-│  Results (15)                         │
-│                                       │
-│  📄 Paper.pdf - Notebook: Research    │
-│     "...the transformer model..."     │
-│                                       │
-│  📝 My Analysis - Notebook: Research  │
-│     "...key findings include..."      │
-└───────────────────────────────────────┘
-```
-
----
-
-## Common Actions
-
-### Create a Notebook
-
-```
-Notebooks page → [+ New Notebook] → Enter name → Create
-```
-
-### Add a Source
-
-```
-Inside notebook → [+ Add Source] → Choose type → Upload/paste → Wait for processing
-```
-
-### Ask a Question
-
-```
-Inside notebook → Chat panel → Type question → Enter → Read response
-```
-
-### Save AI Response
-
-```
-Get good response → Click [Save as Note] → Edit title → Save
-```
-
-### Change Context Level
-
-```
-Click source → Context dropdown → Select level → Changes apply immediately
-```
-
-### Generate Podcast
-
-```
-Podcasts tab → Select profile → Choose sources → [Generate] → Wait → Download
-```
-
----
-
-## Keyboard Shortcuts
-
-| Key | Action |
-|-----|--------|
-| `Enter` | Send chat message |
-| `Shift + Enter` | New line in chat |
-| `Escape` | Close dialogs |
-| `Ctrl/Cmd + F` | Browser find |
-
----
-
-## Mobile View
-
-On smaller screens, the three-panel layout stacks vertically:
-
-```
-┌─────────────────┐
-│    SOURCES      │
-│    (tap to expand)
-├─────────────────┤
-│    NOTES        │
-│    (tap to expand)
-├─────────────────┤
-│    CHAT         │
-│    (always visible)
-└─────────────────┘
-```
-
-- Panels collapse to save space
-- Tap headers to expand/collapse
-- Chat remains accessible
-- Full functionality preserved
-
----
-
-## Tips for Efficient Navigation
-
-1. **Use keyboard** - Enter sends messages, Escape closes dialogs
-2. **Context first** - Set source context before chatting
-3. **Sessions** - Create new sessions for different topics
-4. **Search globally** - Use Search page to find across all notebooks
-5. **Models page** - Bookmark your preferred models
-
----
-
-Now you know where everything is. Start with [Adding Sources](adding-sources.md) to begin your research!

docs/3-USER-GUIDE/search.md

@@ -1,475 +0,0 @@
-# Search Effectively - Finding What You Need
-
-Search is your gateway into your research. This guide covers two search modes and when to use each.
-
----
-
-## Quick-Start: Find Something
-
-### Simple Search
-
-```
-1. Go to your notebook
-2. Type in search box
-3. See results (both sources and notes)
-4. Click result to view source/note
-5. Done!
-
-That works for basic searches.
-But you can do much better...
-```
-
----
-
-## Two Search Modes Explained
-
-Open Notebook has two fundamentally different search approaches.
-
-### Search Type 1: TEXT SEARCH (Keyword Matching)
-
-**How it works:**
-- You search for words: "transformer"
-- System finds chunks containing "transformer"
-- Ranked by relevance: frequency, position, context
-
-**Speed:** Very fast (instant)
-
-**When to use:**
-- You remember exact words or phrases
-- You're looking for specific terms
-- You want precise keyword matches
-- You need exact quotes
-
-**Example:**
-```
-Search: "attention mechanism"
-Results:
-  1. "The attention mechanism allows..." (perfect match)
-  2. "Attention and other mechanisms..." (partial match)
-  3. "How mechanisms work in attention..." (includes words separately)
-
-All contain "attention" AND "mechanism"
-Ranked by how close together they are
-```
-
-**What it finds:**
-- Exact phrases: "transformer model"
-- Individual words: transformer OR model (too broad)
-- Names: "Vaswani et al."
-- Numbers: "1994", "GPT-4"
-- Technical terms: "LSTM", "convolution"
-
-**What it doesn't find:**
-- Similar words: searching "attention" won't find "focus"
-- Synonyms: searching "large" won't find "big"
-- Concepts: searching "similarity" won't find "likeness"
-
----
-
-### Search Type 2: VECTOR SEARCH (Semantic/Concept Matching)
-
-**How it works:**
-- Your search converted to embedding (vector)
-- All chunks converted to embeddings
-- System finds most similar embeddings
-- Ranked by semantic similarity
-
-**Speed:** A bit slower (1-2 seconds)
-
-**When to use:**
-- You're exploring a concept
-- You don't know exact words
-- You want semantically similar content
-- You're discovering, not searching
-
-**Example:**
-```
-Search: "What's the mechanism for understanding in models?"
-(Notice: No chunk likely says exactly that)
-
-Results:
-  1. "Mechanistic interpretability allows understanding..." (semantic match)
-  2. "Feature attribution reveals how models work..." (conceptually similar)
-  3. "Attention visualization shows model decisions..." (same topic)
-
-None contain your exact words
-But all are semantically related
-```
-
-**What it finds:**
-- Similar concepts: "understanding" + "interpretation" + "explainability" (all related)
-- Paraphrases: "big" and "large" (same meaning)
-- Related ideas: "safety" relates to "alignment" (connected concepts)
-- Analogies: content about biological learning when searching "learning"
-
-**What it doesn't find:**
-- Exact keywords: if you search a rare word, vector search might miss it
-- Specific numbers: "1994" vs "1993" are semantically different
-- Technical jargon: "LSTM" and "RNN" are different even if related
-
----
-
-## Decision: Text Search vs. Vector Search?
-
-```
-Question: "Do I remember the exact words?"
-
-→ YES: Use TEXT SEARCH
-   Example: "I remember the paper said 'attention is all you need'"
-
-→ NO: Use VECTOR SEARCH
-   Example: "I'm looking for content about how models process information"
-
-→ UNSURE: Try TEXT SEARCH first (faster)
-         If no results, try VECTOR SEARCH
-
-Text search: "I know what I'm looking for"
-Vector search: "I'm exploring an idea"
-```
-
----
-
-## Step-by-Step: Using Each Search
-
-### Text Search
-
-```
-1. Go to search box
-2. Type your keywords: "transformer", "attention", "2017"
-3. Press Enter
-4. Results appear (usually instant)
-5. Click result to see context
-
-Results show:
-  - Which source contains it
-  - How many times it appears
-  - Relevance score
-  - Preview of surrounding text
-```
-
-### Vector Search
-
-```
-1. Go to search box
-2. Type your concept: "How do models understand language?"
-3. Choose "Vector Search" from dropdown
-4. Press Enter
-5. Results appear (1-2 seconds)
-6. Click result to see context
-
-Results show:
-  - Semantically related chunks
-  - Similarity score (higher = more related)
-  - Preview of surrounding text
-  - Different sources mixed together
-```
-
----
-
-## The Ask Feature (Automated Search)
-
-Ask is different from simple search. It automatically searches, synthesizes, and answers.
-
-### How Ask Works
-
-```
-Stage 1: QUESTION UNDERSTANDING
-  "Compare the approaches in my papers"
-  → System: "This asks for comparison"
-
-Stage 2: SEARCH STRATEGY
-  → System: "I should search for each approach separately"
-
-Stage 3: PARALLEL SEARCHES
-  → Search 1: "Approach in paper A"
-  → Search 2: "Approach in paper B"
-  (Multiple searches happen at once)
-
-Stage 4: ANALYSIS & SYNTHESIS
-  → Per-result analysis: "Based on paper A, the approach is..."
-  → Per-result analysis: "Based on paper B, the approach is..."
-  → Final synthesis: "Comparing A and B: A differs from B in..."
-
-Result: Comprehensive answer, not just search results
-```
-
-### When to Use Ask vs. Simple Search
-
-| Task | Use | Why |
-|------|-----|-----|
-| "Find the quote about X" | **TEXT SEARCH** | Need exact words |
-| "What does source A say about X?" | **TEXT SEARCH** | Direct, fast answer |
-| "Find content about X" | **VECTOR SEARCH** | Semantic discovery |
-| "Compare A and B" | **ASK** | Comprehensive synthesis |
-| "What's the big picture?" | **ASK** | Full analysis needed |
-| "How do these sources relate?" | **ASK** | Cross-source synthesis |
-| "I remember something about X" | **TEXT SEARCH** | Recall memory |
-| "I'm exploring the topic of X" | **VECTOR SEARCH** | Discovery mode |
-
----
-
-## Advanced Search Strategies
-
-### Strategy 1: Simple Search with Follow-Up
-
-```
-1. Text search: "attention mechanism"
-   Results: 50 matches
-
-2. Too many. Follow up with vector search:
-   "Why is attention useful?" (concept search)
-   Results: Most relevant papers/notes
-
-3. Better results with less noise
-```
-
-### Strategy 2: Ask for Comprehensive, Then Search for Details
-
-```
-1. Ask: "What are the main approaches to X?"
-   Result: Comprehensive answer about A, B, C
-
-2. Use that to identify specific sources
-
-3. Text search in those specific sources:
-   "Why did they choose method X?"
-   Result: Detailed information
-```
-
-### Strategy 3: Vector Search for Discovery, Text for Verification
-
-```
-1. Vector search: "How do transformers generalize?"
-   Results: Related conceptual papers
-
-2. Skim to understand landscape
-
-3. Text search in promising sources:
-   "generalization", "extrapolation", "transfer"
-   Results: Specific passages to read carefully
-```
-
-### Strategy 4: Combine Search with Chat
-
-```
-1. Vector search: "What's new in AI 2026?"
-   Results: Latest papers
-
-2. Go to Chat
-3. Add those papers to context
-4. Ask detailed follow-up questions
-5. Get deep analysis of results
-```
-
----
-
-## Search Quality Issues & Fixes
-
-### Getting No Results
-
-| Problem | Cause | Solution |
-|---------|-------|----------|
-| Text search: no results | Word doesn't appear | Try vector search instead |
-| Vector search: no results | Concept not in content | Try broader search term |
-| Both empty | Content not in notebook | Add sources to notebook |
-| | Sources not processed | Wait for processing to complete |
-
-### Getting Too Many Results
-
-| Problem | Cause | Solution |
-|---------|-------|----------|
-| 1000+ results | Search too broad | Be more specific |
-| | All sources | Filter by source |
-| | Keyword matches rare words | Use vector search instead |
-
-### Getting Wrong Results
-
-| Problem | Cause | Solution |
-|---------|-------|----------|
-| Results irrelevant | Search term has multiple meanings | Provide more context |
-| | Using text search for concepts | Try vector search |
-| Different meaning | Homonym (word means multiple things) | Add context (e.g., "attention mechanism") |
-
-### Getting Low Quality Results
-
-| Problem | Cause | Solution |
-|---------|-------|----------|
-| Results don't match intent | Vague search term | Be specific ("Who invented X?" vs "X") |
-| | Concept not well-represented | Add more sources on that topic |
-| | Vector embedding not trained on domain | Use text search as fallback |
-
----
-
-## Tips for Better Searches
-
-### For Text Search
-1. **Be specific** — "attention mechanism" not just "attention"
-2. **Use exact phrases** — Put quotes around: "attention is all you need"
-3. **Include context** — "LSTM vs attention" not just "attention"
-4. **Use technical terms** — These are usually more precise
-5. **Try synonyms** — If first search fails, try related terms
-
-### For Vector Search
-1. **Ask a question** — "What's the best way to X?" is better than "best way"
-2. **Use natural language** — Explain what you're looking for
-3. **Be specific about intent** — "Compare X and Y" not "X and Y"
-4. **Include context** — "In machine learning, how..." vs just "how..."
-5. **Think conceptually** — What idea are you exploring?
-
-### General Tips
-1. **Start broad, then narrow** — "AI papers" → "transformers" → "attention mechanism"
-2. **Try both search types** — Each finds different things
-3. **Use Ask for complex questions** — Don't just search
-4. **Save good results as notes** — Create knowledge base
-5. **Filter by source if needed** — "Search in Paper A only"
-
----
-
-## Search Examples
-
-### Example 1: Finding a Specific Fact
-
-**Goal:** "Find the date the transformer was introduced"
-
-```
-Step 1: Text search
-  "transformer 2017" (or year you remember)
-
-If that works: Done!
-
-If no results: Try
-  "attention is all you need" (famous paper title)
-
-Check result for exact date
-```
-
-### Example 2: Exploring a Concept
-
-**Goal:** "Find content about alignment interpretability"
-
-```
-Step 1: Vector search
-  "How do we make AI interpretable?"
-
-Results: Papers on interpretability, transparency, alignment
-
-Step 2: Review results
-  See which papers are most relevant
-
-Step 3: Deep dive
-  Go to Chat, add top 2-3 papers
-  Ask detailed questions about alignment
-```
-
-### Example 3: Comprehensive Answer
-
-**Goal:** "How do different approaches to AI safety compare?"
-
-```
-Step 1: Ask
-  "Compare the main approaches to AI safety in my sources"
-
-Result: Comprehensive analysis comparing approaches
-
-Step 2: Identify sources
-  From answer, see which papers were most relevant
-
-Step 3: Deep dive
-  Text search in those papers:
-  "limitations", "critiques", "open problems"
-
-Step 4: Save as notes
-  Create comparison note from Ask result
-```
-
-### Example 4: Finding Pattern
-
-**Goal:** "Find all papers mentioning transformers"
-
-```
-Step 1: Text search
-  "transformer"
-
-Results: All papers mentioning "transformer"
-
-Step 2: Vector search
-  "neural network architecture for sequence processing"
-
-Results: Papers that don't say "transformer" but discuss similar concept
-
-Step 3: Combine
-  Union of text + vector results shows full landscape
-
-Step 4: Analyze
-  Go to Chat with all results
-  Ask: "What's common across all these?"
-```
-
----
-
-## Search in the Workflow
-
-How search fits with other features:
-
-```
-SOURCES
-  ↓
-SEARCH (find what matters)
-  ├─ Text search (precise)
-  ├─ Vector search (exploration)
-  └─ Ask (comprehensive)
-  ↓
-CHAT (explore with follow-ups)
-  ↓
-TRANSFORMATIONS (batch extract)
-  ↓
-NOTES (save insights)
-```
-
-### Workflow Example
-
-```
-1. Add 10 papers to notebook
-
-2. Search: "What's the state of the art?"
-   (Vector search explores landscape)
-
-3. Ask: "Compare these 3 approaches"
-   (Comprehensive synthesis)
-
-4. Chat: Deep questions about winner
-   (Follow-up exploration)
-
-5. Save best insights as notes
-   (Knowledge capture)
-
-6. Transform remaining papers
-   (Batch extraction for later)
-
-7. Create podcast from notes + sources
-   (Share findings)
-```
-
----
-
-## Summary: Know Your Search
-
-**TEXT SEARCH** — "I know what I'm looking for"
-- Fast, precise, keyword-based
-- Use when you remember exact words/phrases
-- Best for: Finding specific facts, quotes, technical terms
-- Speed: Instant
-
-**VECTOR SEARCH** — "I'm exploring an idea"
-- Slow-ish, concept-based, semantic
-- Use when you're discovering connections
-- Best for: Concept exploration, related ideas, synonyms
-- Speed: 1-2 seconds
-
-**ASK** — "I want a comprehensive answer"
-- Auto-searches, auto-analyzes, synthesizes
-- Use for complex questions needing multiple sources
-- Best for: Comparisons, big-picture questions, synthesis
-- Speed: 10-30 seconds
-
-Pick the right tool for your search goal, and you'll find what you need faster.

docs/3-USER-GUIDE/transformations.md

@@ -1,402 +0,0 @@
-# Transformations - Batch Processing Your Sources
-
-Transformations apply the same analysis to multiple sources at once. Instead of asking the same question repeatedly, define a template and run it across your content.
-
----
-
-## When to Use Transformations
-
-| Use Transformations When | Use Chat Instead When |
-|-------------------------|----------------------|
-| Same analysis on many sources | One-off questions |
-| Consistent output format needed | Exploratory conversation |
-| Batch processing | Follow-up questions needed |
-| Creating structured notes | Context changes between questions |
-
-**Example**: You have 10 papers and want a summary of each. Transformation does it in one operation.
-
----
-
-## Quick Start: Your First Transformation
-
-```
-1. Go to your notebook
-2. Click "Transformations" in navigation
-3. Select a built-in template (e.g., "Summary")
-4. Select sources to transform
-5. Click "Apply"
-6. Wait for processing
-7. New notes appear automatically
-```
-
----
-
-## Built-in Transformations
-
-Open Notebook includes ready-to-use templates:
-
-### Summary
-
-```
-What it does: Creates a 200-300 word overview
-Output: Key points, main arguments, conclusions
-Best for: Quick reference, getting the gist
-```
-
-### Key Concepts
-
-```
-What it does: Extracts main ideas and terminology
-Output: List of concepts with explanations
-Best for: Learning new topics, building vocabulary
-```
-
-### Methodology
-
-```
-What it does: Extracts research approach
-Output: How the study was conducted
-Best for: Academic papers, research review
-```
-
-### Takeaways
-
-```
-What it does: Extracts actionable insights
-Output: What you should do with this information
-Best for: Business documents, practical guides
-```
-
-### Questions
-
-```
-What it does: Generates questions the source raises
-Output: Open questions, gaps, follow-up research
-Best for: Literature review, research planning
-```
-
----
-
-## Creating Custom Transformations
-
-### Step-by-Step
-
-```
-1. Go to "Transformations" page
-2. Click "Create New"
-3. Enter a name: "Academic Paper Analysis"
-4. Write your prompt template:
-
-   "Analyze this academic paper and extract:
-
-   1. **Research Question**: What problem does this address?
-   2. **Hypothesis**: What did they predict?
-   3. **Methodology**: How did they test it?
-   4. **Key Findings**: What did they discover? (numbered list)
-   5. **Limitations**: What caveats do the authors mention?
-   6. **Future Work**: What do they suggest next?
-
-   Be specific and cite page numbers where possible."
-
-5. Click "Save"
-6. Your transformation appears in the list
-```
-
-### Prompt Template Tips
-
-**Be specific about format:**
-```
-Good: "List 5 key points as bullet points"
-Bad: "What are the key points?"
-```
-
-**Request structure:**
-```
-Good: "Create sections for: Summary, Methods, Results"
-Bad: "Tell me about this paper"
-```
-
-**Ask for citations:**
-```
-Good: "Cite page numbers for each claim"
-Bad: (no citation request)
-```
-
-**Set length expectations:**
-```
-Good: "In 200-300 words, summarize..."
-Bad: "Summarize this"
-```
-
----
-
-## Applying Transformations
-
-### To a Single Source
-
-```
-1. In Sources panel, click source menu (⋮)
-2. Select "Transform"
-3. Choose transformation template
-4. Click "Apply"
-5. Note appears when done
-```
-
-### To Multiple Sources (Batch)
-
-```
-1. Go to Transformations page
-2. Select your template
-3. Check multiple sources
-4. Click "Apply to Selected"
-5. Processing runs in parallel
-6. One note per source created
-```
-
-### Processing Time
-
-| Sources | Typical Time |
-|---------|--------------|
-| 1 source | 30 seconds - 1 minute |
-| 5 sources | 2-3 minutes |
-| 10 sources | 4-5 minutes |
-| 20+ sources | 8-10 minutes |
-
-Processing runs in background. You can continue working.
-
----
-
-## Transformation Examples
-
-### Literature Review Template
-
-```
-Name: Literature Review Entry
-
-Prompt:
-"For this research paper, create a literature review entry:
-
-**Citation**: [Author(s), Year, Title, Journal]
-**Research Question**: What problem is addressed?
-**Methodology**: What approach was used?
-**Sample**: What population/data was studied?
-**Key Findings**:
-1. [Finding with page citation]
-2. [Finding with page citation]
-3. [Finding with page citation]
-**Strengths**: What did this study do well?
-**Limitations**: What are the gaps?
-**Relevance**: How does this connect to my research?
-
-Keep each section to 2-3 sentences."
-```
-
-### Meeting Notes Template
-
-```
-Name: Meeting Summary
-
-Prompt:
-"From this meeting transcript, extract:
-
-**Attendees**: Who was present
-**Date/Time**: When it occurred
-**Key Decisions**: What was decided (numbered)
-**Action Items**:
-- [ ] Task (Owner, Due Date)
-**Open Questions**: Unresolved issues
-**Next Steps**: What happens next
-
-Format as clear, scannable notes."
-```
-
-### Competitor Analysis Template
-
-```
-Name: Competitor Analysis
-
-Prompt:
-"Analyze this company/product document:
-
-**Company**: Name and overview
-**Products/Services**: What they offer
-**Target Market**: Who they serve
-**Pricing**: If available
-**Strengths**: Competitive advantages
-**Weaknesses**: Gaps or limitations
-**Opportunities**: How we compare
-**Threats**: What they do better
-
-Be objective and cite specific details."
-```
-
-### Technical Documentation Template
-
-```
-Name: API Documentation Summary
-
-Prompt:
-"Extract from this technical document:
-
-**Overview**: What does this do? (1-2 sentences)
-**Authentication**: How to authenticate
-**Key Endpoints**:
-- Endpoint 1: [method] [path] - [purpose]
-- Endpoint 2: ...
-**Common Parameters**: Frequently used params
-**Rate Limits**: If mentioned
-**Error Codes**: Key error responses
-**Example Usage**: Simple code example if possible
-
-Keep technical but concise."
-```
-
----
-
-## Managing Transformations
-
-### Edit a Transformation
-
-```
-1. Go to Transformations page
-2. Find your template
-3. Click "Edit"
-4. Modify the prompt
-5. Click "Save"
-```
-
-### Delete a Transformation
-
-```
-1. Go to Transformations page
-2. Find the template
-3. Click "Delete"
-4. Confirm
-```
-
-### Reorder/Organize
-
-Built-in transformations appear first, then custom ones alphabetically.
-
----
-
-## Transformation Output
-
-### Where Results Go
-
-- Each source produces one note
-- Notes appear in your notebook's Notes panel
-- Notes are tagged with transformation name
-- Original source is linked
-
-### Note Naming
-
-```
-Default: "[Transformation Name] - [Source Title]"
-Example: "Summary - Research Paper 2025.pdf"
-```
-
-### Editing Output
-
-```
-1. Click the generated note
-2. Click "Edit"
-3. Refine the content
-4. Save
-```
-
----
-
-## Best Practices
-
-### Template Design
-
-1. **Start specific** - Vague prompts give vague results
-2. **Use formatting** - Headings, bullets, numbered lists
-3. **Request citations** - Make results verifiable
-4. **Set length** - Prevent overly long or short output
-5. **Test first** - Run on one source before batch
-
-### Source Selection
-
-1. **Similar content** - Same transformation on similar sources
-2. **Reasonable size** - Very long sources may need splitting
-3. **Processed status** - Ensure sources are fully processed
-
-### Quality Control
-
-1. **Review samples** - Check first few outputs before trusting batch
-2. **Edit as needed** - Transformations are starting points
-3. **Iterate prompts** - Refine based on results
-
----
-
-## Common Issues
-
-### Generic Output
-
-**Problem**: Results are too vague
-**Solution**: Make prompt more specific, add format requirements
-
-### Missing Information
-
-**Problem**: Key details not extracted
-**Solution**: Explicitly ask for what you need in prompt
-
-### Inconsistent Format
-
-**Problem**: Each note looks different
-**Solution**: Add clear formatting instructions to prompt
-
-### Too Long/Short
-
-**Problem**: Output doesn't match expectations
-**Solution**: Specify word count or section lengths
-
-### Processing Fails
-
-**Problem**: Transformation doesn't complete
-**Solution**:
-- Check source is processed
-- Try shorter/simpler prompt
-- Process sources individually
-
----
-
-## Transformations vs. Chat vs. Ask
-
-| Feature | Transformations | Chat | Ask |
-|---------|----------------|------|-----|
-| **Input** | Predefined template | Your questions | Your question |
-| **Scope** | One source at a time | Selected sources | Auto-searched |
-| **Output** | Structured note | Conversation | Comprehensive answer |
-| **Best for** | Batch processing | Exploration | One-shot answers |
-| **Follow-up** | Run again | Ask more | New query |
-
----
-
-## Summary
-
-```
-Transformations = Batch AI Processing
-
-How to use:
-1. Define template (or use built-in)
-2. Select sources
-3. Apply transformation
-4. Get structured notes
-
-When to use:
-- Same analysis on many sources
-- Consistent output needed
-- Building structured knowledge base
-- Saving time on repetitive tasks
-
-Tips:
-- Be specific in prompts
-- Request formatting
-- Test before batch
-- Edit output as needed
-```
-
-Transformations turn repetitive analysis into one-click operations. Define once, apply many times.

docs/3-USER-GUIDE/working-with-notes.md

@@ -1,581 +0,0 @@
-# Working with Notes - Capturing and Organizing Insights
-
-Notes are your processed knowledge. This guide covers how to create, organize, and use them effectively.
-
----
-
-## What Are Notes?
-
-Notes are your **research output** — the insights you capture from analyzing sources. They can be:
-
-- **Manual** — You write them yourself
-- **AI-Generated** — From Chat responses, Ask results, or Transformations
-- **Hybrid** — AI insight + your edits and additions
-
-Unlike sources (which never change), notes are mutable — you edit, refine, and organize them.
-
----
-
-## Quick-Start: Create Your First Note
-
-### Method 1: Manual Note (Write Yourself)
-
-```
-1. In your notebook, go to "Notes" section
-2. Click "Create New Note"
-3. Give it a title: "Key insights from source X"
-4. Write your content (markdown supported)
-5. Click "Save"
-6. Done! Note appears in your notebook
-```
-
-### Method 2: Save from Chat
-
-```
-1. Have a Chat conversation
-2. Get a good response from AI
-3. Click "Save as Note" button under response
-4. Give the note a title
-5. Add any additional context
-6. Click "Save"
-7. Done! Note appears in your notebook
-```
-
-### Method 3: Apply Transformation
-
-```
-1. Go to "Transformations"
-2. Select a template (or create custom)
-3. Click "Apply to sources"
-4. Select which sources to transform
-5. Wait for processing
-6. New notes automatically appear
-7. Done! Each source produces one note
-```
-
----
-
-## Creating Manual Notes
-
-### Basic Structure
-
-```
-Title: "What you're capturing"
-       (Make it descriptive)
-
-Content:
-  - Main points
-  - Your analysis
-  - Questions raised
-  - Next steps
-
-Metadata:
-  - Tags: How to categorize
-  - Related sources: Which documents influenced this
-  - Date: Auto-added when created
-```
-
-### Markdown Support
-
-You can format notes with markdown:
-
-```markdown
-# Heading
-## Subheading
-### Sub-subheading
-
-**Bold text** for emphasis
-*Italic text* for secondary emphasis
-
-- Bullet lists
-- Like this
-
-1. Numbered lists
-2. Like this
-
-> Quotes and important callouts
-
-[Links work](https://example.com)
-```
-
-### Example Note Structure
-
-```markdown
-# Key Findings from "AI Safety Paper 2025"
-
-## Main Argument
-The paper argues that X approach is better than Y because...
-
-## Methodology
-The authors use [methodology] to test this hypothesis.
-
-## Key Results
-- Result 1: [specific finding with citation]
-- Result 2: [specific finding with citation]
-- Result 3: [specific finding with citation]
-
-## Gaps & Limitations
-1. The paper assumes X, which might not hold in Y scenario
-2. Limited to Z population/domain
-3. Future work needed on A, B, C
-
-## My Thoughts
-- This connects to previous research on...
-- Potential application in...
-
-## Next Steps
-- [ ] Read the referenced paper on X
-- [ ] Find similar studies on Y
-- [ ] Discuss implications with team
-```
-
----
-
-## AI-Generated Notes: Three Sources
-
-### 1. Save from Chat
-
-```
-Workflow:
-  Chat → Good response → "Save as Note"
-         → Edit if needed → Save
-
-When to use:
-  - AI response answers your question well
-  - You want to keep the answer for reference
-  - You're building a knowledge base from conversations
-
-Quality:
-  - Quality = quality of your Chat question
-  - Better context = better responses = better notes
-  - Ask specific questions for useful notes
-```
-
-### 2. Save from Ask
-
-```
-Workflow:
-  Ask → Comprehensive answer → "Save as Note"
-      → Edit if needed → Save
-
-When to use:
-  - You need a one-time comprehensive answer
-  - You want to save the synthesized result
-  - Building a knowledge base of comprehensive answers
-
-Quality:
-  - System automatically found relevant sources
-  - Results already have citations
-  - Often higher quality than Chat (more thorough)
-```
-
-### 3. Transformations (Batch Processing)
-
-```
-Workflow:
-  Define transformation → Apply to sources → Notes auto-created
-                      → Review & edit → Organize
-
-Example Transformation:
-  Template: "Extract: main argument, methodology, key findings"
-  Apply to: 5 sources
-  Result: 5 new notes with consistent structure
-
-When to use:
-  - Same extraction from many sources
-  - Building structured knowledge base
-  - Creating consistent summaries
-```
-
----
-
-## Using Transformations for Batch Insights
-
-### Built-in Transformations
-
-Open Notebook comes with presets:
-
-**Summary**
-```
-Extracts: Main points, key arguments, conclusions
-Output: 200-300 word summary of source
-Best for: Quick reference summaries
-```
-
-**Key Concepts**
-```
-Extracts: Main ideas, concepts, terminology
-Output: List of concepts with explanations
-Best for: Learning and terminology
-```
-
-**Methodology**
-```
-Extracts: Research approach, methods, data
-Output: How the research was conducted
-Best for: Academic sources, methodology review
-```
-
-**Takeaways**
-```
-Extracts: Actionable insights, recommendations
-Output: What you should do with this information
-Best for: Practical/business sources
-```
-
-### How to Apply Transformation
-
-```
-1. Go to "Transformations"
-2. Select a template
-3. Click "Apply"
-4. Select which sources (one or many)
-5. Wait for processing (usually 30 seconds - 2 minutes)
-6. New notes appear in your notebook
-7. Edit if needed
-```
-
-### Create Custom Transformation
-
-```
-1. Click "Create Custom Transformation"
-2. Write your extraction template:
-
-   Example:
-   "For this academic paper, extract:
-    - Central research question
-    - Hypothesis tested
-    - Methodology used
-    - Key findings (numbered)
-    - Limitations acknowledged
-    - Recommendations for future work"
-
-3. Click "Save Template"
-4. Apply to one or many sources
-5. System generates notes with consistent structure
-```
-
----
-
-## Organizing Notes
-
-### Naming Conventions
-
-**Option 1: Date-based**
-```
-2026-01-03 - Key points from X source
-2026-01-04 - Comparison between A and B
-Benefit: Easy to see what you did when
-```
-
-**Option 2: Topic-based**
-```
-AI Safety - Alignment approaches
-AI Safety - Interpretability research
-Benefit: Groups by subject matter
-```
-
-**Option 3: Type-based**
-```
-SUMMARY: Paper on X
-QUESTION: What about Y?
-INSIGHT: Connection between Z and W
-Benefit: Easy to filter by type
-```
-
-**Option 4: Source-based**
-```
-From: Paper A - Main insights
-From: Video B - Interesting implications
-Benefit: Easy to trace back to sources
-```
-
-**Best practice:** Combine approaches
-```
-[Date] [Source] - [Topic] - [Type]
-2026-01-03 - Paper A - AI Safety - Takeaways
-```
-
-### Using Tags
-
-Tags are labels for categorization. Add them when creating notes:
-
-```
-Example tags:
-  - "primary-research" (direct source analysis)
-  - "background" (supporting material)
-  - "methodology" (about research methods)
-  - "insights" (your original thinking)
-  - "questions" (open questions raised)
-  - "follow-up" (needs more work)
-  - "published" (ready to share/use)
-```
-
-**Benefits of tags:**
-- Filter notes by tag
-- Find all notes of a type
-- Organize workflow (e.g., find all "follow-up" notes)
-
-### Note Linking & References
-
-You can reference sources within notes:
-
-```markdown
-# Analysis of Paper A
-
-As shown in Paper A (see "main argument" section),
-the authors argue that...
-
-## Related Sources
-- Paper B discusses similar approach
-- Video C shows practical application
-- My note on "Comparative analysis" has more
-```
-
----
-
-## Editing and Refining Notes
-
-### Improving AI-Generated Notes
-
-```
-AI Note:
-  "The paper discusses machine learning"
-
-What you might change:
-  "The paper proposes a supervised learning approach
-   to classification problems, using neural networks
-   with attention mechanisms (see pp. 15-18)."
-
-How to edit:
-  1. Click note
-  2. Click "Edit"
-  3. Refine the content
-  4. Click "Save"
-```
-
-### Adding Citations
-
-```
-When saving from Chat/Ask:
-  - Citations auto-added
-  - Shows which sources informed answer
-  - You can verify by clicking
-
-When manual notes:
-  - Add manually: "From Paper A, page 15: ..."
-  - Or reference: "As discussed in [source]"
-```
-
----
-
-## Searching Your Notes
-
-Notes are fully searchable:
-
-### Text Search
-```
-Find exact phrase: "attention mechanism"
-Results: All notes containing that phrase
-Use when: Looking for specific terms or quotes
-```
-
-### Vector/Semantic Search
-```
-Find concept: "How do models understand?"
-Results: Notes about interpretability, mechanistic understanding, etc.
-Use when: Exploring conceptually (words not exact)
-```
-
-### Combined Search
-```
-Text search notes → Find keyword matches
-Vector search notes → Find conceptual matches
-Both work across sources + notes together
-```
-
----
-
-## Exporting and Sharing Notes
-
-### Options
-
-**Copy to clipboard**
-```
-Click "Share" → "Copy" → Paste anywhere
-Good for: Sharing one note via email/chat
-```
-
-**Export as Markdown**
-```
-Click "Share" → "Export as MD" → Saves as .md file
-Good for: Sharing with others, version control
-```
-
-**Create note collection**
-```
-Select multiple notes → "Export collection"
-→ Creates organized markdown document
-Good for: Sharing a topic overview
-```
-
-**Publish to web**
-```
-Click "Publish" → Get shareable link
-Good for: Publishing publicly (if desired)
-```
-
----
-
-## Organizing Your Notebook's Notes
-
-### By Research Phase
-
-**Phase 1: Discovery**
-- Initial summaries
-- Questions raised
-- Interesting findings
-
-**Phase 2: Deep Dive**
-- Detailed analysis
-- Comparative insights
-- Methodology reviews
-
-**Phase 3: Synthesis**
-- Connections across sources
-- Original thinking
-- Conclusions
-
-### By Content Type
-
-**Summaries**
-- High-level overviews
-- Generated by transformations
-- Quick reference
-
-**Questions**
-- Open questions
-- Things to research more
-- Gaps to fill
-
-**Insights**
-- Your original analysis
-- Connections made
-- Conclusions reached
-
-**Tasks**
-- Follow-up research
-- Sources to add
-- People to contact
-
----
-
-## Using Notes in Other Features
-
-### In Chat
-
-```
-You can reference notes:
-"Based on my note 'Key findings from A',
-how does this compare to B?"
-
-Notes become part of context.
-Treated like sources but smaller/more focused.
-```
-
-### In Transformations
-
-```
-Notes can be transformed:
-1. Select notes as input
-2. Apply transformation
-3. Get new derived notes
-
-Example: Transform 5 analysis notes → Create synthesis
-```
-
-### In Podcasts
-
-```
-Notes are used to create podcast content:
-1. Generate podcast for notebook
-2. System includes notes in content selection
-3. Notes become part of episode outline
-```
-
----
-
-## Best Practices
-
-### For Manual Notes
-1. **Write clearly** — Future you will appreciate it
-2. **Add context** — Why this matters, not just what it says
-3. **Link to sources** — You can verify later
-4. **Date them** — Track your thinking over time
-5. **Tag immediately** — Don't defer organization
-
-### For AI-Generated Notes
-1. **Review before saving** — Verify quality
-2. **Edit for clarity** — AI might miss nuance
-3. **Add your thoughts** — Make it your own
-4. **Include citations** — Understand sources
-5. **Organize right away** — While context is fresh
-
-### For Organization
-1. **Consistent naming** — Your future self will thank you
-2. **Tag everything** — Makes filtering later much easier
-3. **Link related notes** — Create knowledge network
-4. **Review periodically** — Refactor as understanding evolves
-5. **Archive old notes** — Keep working space clean
-
----
-
-## Common Mistakes
-
-| Mistake | Problem | Solution |
-|---------|---------|----------|
-| Save every Chat response | Notebook becomes cluttered with low-quality notes | Only save good responses that answer your questions |
-| Don't add tags | Can't find notes later | Tag immediately when creating |
-| Poor note titles | Can't remember what's in them | Use descriptive titles, include key concept |
-| Never link notes together | Miss connections between ideas | Add references to related notes |
-| Forget the source | Can't verify claims later | Always link back to source |
-| Never edit AI notes | Keep generic AI responses | Refine for clarity and context |
-| Create one giant note | Too long to be useful | Split into focused notes by subtopic |
-
----
-
-## Summary: Note Lifecycle
-
-```
-1. CREATE
-   ├─ Manual: Write from scratch
-   ├─ From Chat: Save good response
-   ├─ From Ask: Save synthesis
-   └─ From Transform: Batch process
-
-2. EDIT & REFINE
-   ├─ Improve clarity
-   ├─ Add context
-   ├─ Fix AI mistakes
-   └─ Add citations
-
-3. ORGANIZE
-   ├─ Name clearly
-   ├─ Add tags
-   ├─ Link related
-   └─ Categorize
-
-4. USE
-   ├─ Reference in Chat
-   ├─ Transform for synthesis
-   ├─ Export for sharing
-   └─ Build on with new questions
-
-5. MAINTAIN
-   ├─ Periodically review
-   ├─ Update as understanding grows
-   ├─ Archive when done
-   └─ Learn from organized knowledge
-```
-
-Your notes become your actual knowledge base. The more you invest in organizing them, the more valuable they become.

docs/4-AI-PROVIDERS/index.md

@@ -1,199 +0,0 @@
-# AI Providers - Comparison & Selection Guide
-
-Open Notebook supports 15+ AI providers. This guide helps you **choose the right provider** for your needs.
-
-> 💡 **Just want to set up a provider?** Skip to the [Configuration Guide](../5-CONFIGURATION/ai-providers.md) for detailed setup instructions.
-
----
-
-## Quick Decision: Which Provider?
-
-### Cloud Providers (Easiest)
-
-**OpenAI (Recommended)**
-- Cost: ~$0.03-0.15 per 1K tokens
-- Speed: Very fast
-- Quality: Excellent
-- Best for: Most users (best quality/price balance)
-
-→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#openai)
-
-**Anthropic (Claude)**
-- Cost: ~$0.80-3.00 per 1M tokens
-- Speed: Fast
-- Quality: Excellent
-- Best for: Long context (200K tokens), reasoning, latest AI
-- Advantage: Superior long-context handling
-
-→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#anthropic-claude)
-
-**Google Gemini**
-- Cost: ~$0.075-0.30 per 1K tokens
-- Speed: Very fast
-- Quality: Good to excellent
-- Best for: Multimodal (images, audio, video)
-- Advantage: Longest context (up to 2M tokens)
-
-→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#google-gemini)
-
-**Groq (Ultra-Fast)**
-- Cost: ~$0.05 per 1M tokens (cheapest)
-- Speed: Ultra-fast (fastest available)
-- Quality: Good
-- Best for: Budget-conscious, transformations, speed-critical tasks
-- Disadvantage: Limited model selection
-
-→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#groq)
-
-**OpenRouter (100+ Models)**
-- Cost: Pay-per-model (varies widely)
-- Speed: Varies by model
-- Quality: Varies by model
-- Best for: Model comparison, testing, unified billing
-- Advantage: One API key for 100+ models from different providers
-
-→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#openrouter)
-
-### Local / Self-Hosted (Free)
-
-**Ollama (Recommended for Local)**
-- Cost: Free (electricity only)
-- Speed: Depends on hardware (slow on CPU, fast on GPU)
-- Quality: Good (open-source models)
-- Setup: 10 minutes
-- Best for: Privacy-first, offline use
-- Privacy: 100% local, nothing leaves your machine
-
-→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#ollama-recommended-for-local)
-
-**LM Studio (Alternative)**
-- Cost: Free (electricity only)
-- Speed: Depends on hardware
-- Quality: Good (same models as Ollama)
-- Setup: 15 minutes (GUI interface)
-- Best for: Non-technical users who prefer GUI over CLI
-- Privacy: 100% local
-
-→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#lm-studio-local-alternative)
-
-### Enterprise
-
-**Azure OpenAI**
-- Cost: Same as OpenAI (usage-based)
-- Speed: Very fast
-- Quality: Excellent (same models as OpenAI)
-- Setup: 10 minutes (more complex)
-- Best for: Enterprise, compliance (HIPAA, SOC2), VPC integration
-
-→ [Setup Guide](../5-CONFIGURATION/ai-providers.md#azure-openai)
-
----
-
-## Comparison Table
-
-| Provider | Speed | Cost | Quality | Privacy | Setup | Context |
-|----------|-------|------|---------|---------|-------|---------|
-| **OpenAI** | Very Fast | $$ | Excellent | Low | 5 min | 128K |
-| **Anthropic** | Fast | $$ | Excellent | Low | 5 min | 200K |
-| **Google** | Very Fast | $$ | Good-Excellent | Low | 5 min | 2M |
-| **Groq** | Ultra Fast | $ | Good | Low | 5 min | 32K |
-| **OpenRouter** | Varies | Varies | Varies | Low | 5 min | Varies |
-| **Ollama** | Slow-Medium | Free | Good | Max | 10 min | Varies |
-| **LM Studio** | Slow-Medium | Free | Good | Max | 15 min | Varies |
-| **Azure** | Very Fast | $$ | Excellent | High | 10 min | 128K |
-
----
-
-## Choosing Your Provider
-
-### I want the easiest setup
-→ **OpenAI** — Most popular, best community support
-
-### I have unlimited budget
-→ **OpenAI** — Best quality
-
-### I want to save money
-→ **Groq** — Cheapest cloud ($0.05 per 1M tokens)
-
-### I want privacy/offline
-→ **Ollama** — Free, local, private
-
-### I want a GUI (not CLI)
-→ **LM Studio** — Desktop app
-
-### I'm in an enterprise
-→ **Azure OpenAI** — Compliance, support
-
-### I need long context (200K+ tokens)
-→ **Anthropic** — Best long-context model
-
-### I need multimodal (images, audio, video)
-→ **Google Gemini** — Best multimodal support
-
-### I want access to many models with one API key
-→ **OpenRouter** — 100+ models, unified billing
-
----
-
-## Ready to Set Up Your Provider?
-
-Now that you've chosen a provider, follow the detailed setup instructions:
-
-→ **[AI Providers Configuration Guide](../5-CONFIGURATION/ai-providers.md)**
-
-This guide includes:
-- Step-by-step setup instructions for each provider
-- Environment variable configuration
-- Model selection and recommendations
-- Provider-specific troubleshooting
-- Hardware requirements (for local providers)
-- Cost optimization tips
-
----
-
-## Cost Estimator
-
-### OpenAI
-```
-Light use (10 chats/day): $1-5/month
-Medium use (50 chats/day): $10-30/month
-Heavy use (all-day use): $50-100+/month
-```
-
-### Anthropic
-```
-Light use: $1-3/month
-Medium use: $5-20/month
-Heavy use: $20-50+/month
-```
-
-### Groq
-```
-Light use: $0-1/month
-Medium use: $2-5/month
-Heavy use: $5-20/month
-```
-
-### Ollama
-```
-Any use: Free (electricity only)
-8GB GPU running 24/7: ~$10/month electricity
-```
-
----
-
-## Next Steps
-
-1. ✅ **You've chosen a provider** (from this comparison guide)
-2. 📖 **Follow the setup guide**: [AI Providers Configuration](../5-CONFIGURATION/ai-providers.md)
-3. ⚙️ **Configure your environment** (detailed in the setup guide)
-4. 🧪 **Test your setup** in Settings → Models
-5. 🚀 **Start using Open Notebook!**
-
----
-
-## Need Help?
-
-- **Setup issues?** See [AI Providers Configuration](../5-CONFIGURATION/ai-providers.md) for detailed troubleshooting per provider
-- **General problems?** Check [Troubleshooting Guide](../6-TROUBLESHOOTING/index.md)
-- **Questions?** Join [Discord community](https://discord.gg/37XJPXfz2w)

docs/5-CONFIGURATION/advanced.md

@@ -1,567 +0,0 @@
-# Advanced Configuration
-
-Performance tuning, debugging, and advanced features.
-
----
-
-## Performance Tuning
-
-### Concurrency Control
-
-```env
-# Max concurrent database operations (default: 5)
-# Increase: Faster processing, more conflicts
-# Decrease: Slower, fewer conflicts
-SURREAL_COMMANDS_MAX_TASKS=5
-```
-
-**Guidelines:**
-- CPU: 2 cores → 2-3 tasks
-- CPU: 4 cores → 5 tasks (default)
-- CPU: 8+ cores → 10-20 tasks
-
-Higher concurrency = more throughput but more database conflicts (retries handle this).
-
-### Retry Strategy
-
-```env
-# How to wait between retries
-SURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter
-
-# Options:
-# - exponential_jitter (recommended)
-# - exponential
-# - fixed
-# - random
-```
-
-For high-concurrency deployments, use `exponential_jitter` to prevent thundering herd.
-
-### Timeout Tuning
-
-```env
-# Client timeout (default: 300 seconds)
-API_CLIENT_TIMEOUT=300
-
-# LLM timeout (default: 60 seconds)
-ESPERANTO_LLM_TIMEOUT=60
-```
-
-**Guideline:** Set `API_CLIENT_TIMEOUT` > `ESPERANTO_LLM_TIMEOUT` + buffer
-
-```
-Example:
-  ESPERANTO_LLM_TIMEOUT=120
-  API_CLIENT_TIMEOUT=180  # 120 + 60 second buffer
-```
-
----
-
-## Batching
-
-### TTS Batch Size
-
-For podcast generation, control concurrent TTS requests:
-
-```env
-# Default: 5
-TTS_BATCH_SIZE=2
-```
-
-**Providers and recommendations:**
-- OpenAI: 5 (can handle many concurrent)
-- Google: 4 (good concurrency)
-- ElevenLabs: 2 (limited concurrent requests)
-- Local TTS: 1 (single-threaded)
-
-Lower = slower but more stable. Higher = faster but more load on provider.
-
----
-
-## Logging & Debugging
-
-### Enable Detailed Logging
-
-```bash
-# Start with debug logging
-RUST_LOG=debug  # For Rust components
-LOGLEVEL=DEBUG  # For Python components
-```
-
-### Debug Specific Components
-
-```bash
-# Only surreal operations
-RUST_LOG=surrealdb=debug
-
-# Only langchain
-LOGLEVEL=langchain:debug
-
-# Only specific module
-RUST_LOG=open_notebook::database=debug
-```
-
-### LangSmith Tracing
-
-For debugging LLM workflows:
-
-```env
-LANGCHAIN_TRACING_V2=true
-LANGCHAIN_ENDPOINT="https://api.smith.langchain.com"
-LANGCHAIN_API_KEY=your-key
-LANGCHAIN_PROJECT="Open Notebook"
-```
-
-Then visit https://smith.langchain.com to see traces.
-
----
-
-## Port Configuration
-
-### Default Ports
-
-```
-Frontend: 8502 (Docker deployment)
-Frontend: 3000 (Development from source)
-API: 5055
-SurrealDB: 8000
-```
-
-### Changing Frontend Port
-
-Edit `docker-compose.yml`:
-
-```yaml
-services:
-  open-notebook:
-    ports:
-      - "8001:8502"  # Change from 8502 to 8001
-```
-
-Access at: `http://localhost:8001`
-
-API auto-detects to: `http://localhost:5055` ✓
-
-### Changing API Port
-
-```yaml
-services:
-  open-notebook:
-    ports:
-      - "127.0.0.1:8502:8502"  # Frontend
-      - "5056:5055"            # Change API from 5055 to 5056
-    environment:
-      - API_URL=http://localhost:5056  # Update API_URL
-```
-
-Access API directly: `http://localhost:5056/docs`
-
-**Note:** When changing API port, you must set `API_URL` explicitly since auto-detection assumes port 5055.
-
-### Changing SurrealDB Port
-
-```yaml
-services:
-  surrealdb:
-    ports:
-      - "8001:8000"  # Change from 8000 to 8001
-    environment:
-      - SURREAL_URL=ws://surrealdb:8001/rpc  # Update connection URL
-```
-
-**Important:** Internal Docker network uses container name (`surrealdb`), not `localhost`.
-
----
-
-## SSL/TLS Configuration
-
-### Custom CA Certificate
-
-For self-signed certs on local providers:
-
-```env
-ESPERANTO_SSL_CA_BUNDLE=/path/to/ca-bundle.pem
-```
-
-### Disable Verification (Development Only)
-
-```env
-# WARNING: Only for testing/development
-# Vulnerable to MITM attacks
-ESPERANTO_SSL_VERIFY=false
-```
-
----
-
-## Multi-Provider Setup
-
-### Use Different Providers for Different Tasks
-
-```env
-# Language model (main)
-OPENAI_API_KEY=sk-proj-...
-
-# Embeddings (alternative)
-# (Future: Configure different embedding provider)
-
-# TTS (different provider)
-ELEVENLABS_API_KEY=...
-```
-
-### OpenAI-Compatible with Fallback
-
-```env
-# Primary
-OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1
-OPENAI_COMPATIBLE_API_KEY=key1
-
-# Can also set specific modality endpoints
-OPENAI_COMPATIBLE_BASE_URL_LLM=http://localhost:1234/v1
-OPENAI_COMPATIBLE_BASE_URL_EMBEDDING=http://localhost:8001/v1
-```
-
----
-
-## Security Hardening
-
-### Change Default Credentials
-
-```env
-# Don't use defaults in production
-SURREAL_USER=your_secure_username
-SURREAL_PASSWORD=$(openssl rand -base64 32)  # Generate secure password
-```
-
-### Add Password Protection
-
-```env
-# Protect your Open Notebook instance
-OPEN_NOTEBOOK_PASSWORD=your_secure_password
-```
-
-### Use HTTPS
-
-```env
-# Always use HTTPS in production
-API_URL=https://mynotebook.example.com
-```
-
-### Firewall Rules
-
-Restrict access to your Open Notebook:
-- Port 8502 (frontend): Only from your IP
-- Port 5055 (API): Only from frontend
-- Port 8000 (SurrealDB): Never expose to internet
-
----
-
-## Web Scraping & Content Extraction
-
-Open Notebook uses multiple services for content extraction:
-
-### Firecrawl
-
-For advanced web scraping:
-
-```env
-FIRECRAWL_API_KEY=your-key
-```
-
-Get key from: https://firecrawl.dev/
-
-### Jina AI
-
-Alternative web extraction:
-
-```env
-JINA_API_KEY=your-key
-```
-
-Get key from: https://jina.ai/
-
----
-
-## Environment Variable Groups
-
-### API Keys (Choose at least one)
-```env
-OPENAI_API_KEY
-ANTHROPIC_API_KEY
-GOOGLE_API_KEY
-GROQ_API_KEY
-MISTRAL_API_KEY
-DEEPSEEK_API_KEY
-OPENROUTER_API_KEY
-XAI_API_KEY
-```
-
-### AI Provider Endpoints
-```env
-OLLAMA_API_BASE
-OPENAI_COMPATIBLE_BASE_URL
-AZURE_OPENAI_ENDPOINT
-GEMINI_API_BASE_URL
-```
-
-### Database
-```env
-SURREAL_URL
-SURREAL_USER
-SURREAL_PASSWORD
-SURREAL_NAMESPACE
-SURREAL_DATABASE
-```
-
-### Performance
-```env
-SURREAL_COMMANDS_MAX_TASKS
-SURREAL_COMMANDS_RETRY_ENABLED
-SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS
-SURREAL_COMMANDS_RETRY_WAIT_STRATEGY
-SURREAL_COMMANDS_RETRY_WAIT_MIN
-SURREAL_COMMANDS_RETRY_WAIT_MAX
-```
-
-### API Settings
-```env
-API_URL
-INTERNAL_API_URL
-API_CLIENT_TIMEOUT
-ESPERANTO_LLM_TIMEOUT
-```
-
-### Audio/TTS
-```env
-ELEVENLABS_API_KEY
-TTS_BATCH_SIZE
-```
-
-### Debugging
-```env
-LANGCHAIN_TRACING_V2
-LANGCHAIN_ENDPOINT
-LANGCHAIN_API_KEY
-LANGCHAIN_PROJECT
-```
-
----
-
-## Testing Configuration
-
-### Quick Test
-
-```bash
-# Add test config
-export OPENAI_API_KEY=sk-test-key
-export API_URL=http://localhost:5055
-
-# Test connection
-curl http://localhost:5055/health
-
-# Test with sample
-curl -X POST http://localhost:5055/api/chat \
-  -H "Content-Type: application/json" \
-  -d '{"message":"Hello"}'
-```
-
-### Validate Config
-
-```bash
-# Check environment variables are set
-env | grep OPENAI_API_KEY
-
-# Verify database connection
-python -c "import os; print(os.getenv('SURREAL_URL'))"
-```
-
----
-
-## Troubleshooting Performance
-
-### High Memory Usage
-
-```env
-# Reduce concurrency
-SURREAL_COMMANDS_MAX_TASKS=2
-
-# Reduce TTS batch size
-TTS_BATCH_SIZE=1
-```
-
-### High CPU Usage
-
-```env
-# Check worker count
-SURREAL_COMMANDS_MAX_TASKS
-
-# Reduce if maxed out:
-SURREAL_COMMANDS_MAX_TASKS=5
-```
-
-### Slow Responses
-
-```env
-# Check timeout settings
-API_CLIENT_TIMEOUT=300
-
-# Check retry config
-SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS=3
-```
-
-### Database Conflicts
-
-```env
-# Reduce concurrency
-SURREAL_COMMANDS_MAX_TASKS=3
-
-# Use jitter strategy
-SURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter
-```
-
----
-
-## Backup & Restore
-
-### Data Locations
-
-| Path | Contents |
-|------|----------|
-| `./data` or `/app/data` | Uploads, podcasts, checkpoints |
-| `./surreal_data` or `/mydata` | SurrealDB database files |
-
-### Quick Backup
-
-```bash
-# Stop services (recommended for consistency)
-docker compose down
-
-# Create timestamped backup
-tar -czf backup-$(date +%Y%m%d-%H%M%S).tar.gz \
-  notebook_data/ surreal_data/
-
-# Restart services
-docker compose up -d
-```
-
-### Automated Backup Script
-
-```bash
-#!/bin/bash
-# backup.sh - Run daily via cron
-
-BACKUP_DIR="/path/to/backups"
-DATE=$(date +%Y%m%d-%H%M%S)
-
-# Create backup
-tar -czf "$BACKUP_DIR/open-notebook-$DATE.tar.gz" \
-  /path/to/notebook_data \
-  /path/to/surreal_data
-
-# Keep only last 7 days
-find "$BACKUP_DIR" -name "open-notebook-*.tar.gz" -mtime +7 -delete
-
-echo "Backup complete: open-notebook-$DATE.tar.gz"
-```
-
-Add to cron:
-```bash
-# Daily backup at 2 AM
-0 2 * * * /path/to/backup.sh >> /var/log/open-notebook-backup.log 2>&1
-```
-
-### Restore
-
-```bash
-# Stop services
-docker compose down
-
-# Remove old data (careful!)
-rm -rf notebook_data/ surreal_data/
-
-# Extract backup
-tar -xzf backup-20240115-120000.tar.gz
-
-# Restart services
-docker compose up -d
-```
-
-### Migration Between Servers
-
-```bash
-# On source server
-docker compose down
-tar -czf open-notebook-migration.tar.gz notebook_data/ surreal_data/
-
-# Transfer to new server
-scp open-notebook-migration.tar.gz user@newserver:/path/
-
-# On new server
-tar -xzf open-notebook-migration.tar.gz
-docker compose up -d
-```
-
----
-
-## Container Management
-
-### Common Commands
-
-```bash
-# Start services
-docker compose up -d
-
-# Stop services
-docker compose down
-
-# View logs (all services)
-docker compose logs -f
-
-# View logs (specific service)
-docker compose logs -f api
-
-# Restart specific service
-docker compose restart api
-
-# Update to latest version
-docker compose down
-docker compose pull
-docker compose up -d
-
-# Check resource usage
-docker stats
-
-# Check service health
-docker compose ps
-```
-
-### Clean Up
-
-```bash
-# Remove stopped containers
-docker compose rm
-
-# Remove unused images
-docker image prune
-
-# Full cleanup (careful!)
-docker system prune -a
-```
-
----
-
-## Summary
-
-**Most deployments need:**
-- One AI provider API key
-- Default database settings
-- Default timeouts
-
-**Tune performance only if:**
-- You have specific bottlenecks
-- High-concurrency workload
-- Custom hardware (very fast or very slow)
-
-**Advanced features:**
-- Firecrawl for better web scraping
-- LangSmith for debugging workflows
-- Custom CA bundles for self-signed certs

docs/5-CONFIGURATION/ai-providers.md

@@ -1,494 +0,0 @@
-# AI Providers - Configuration Reference
-
-Complete setup instructions for each AI provider. Pick the one you're using.
-
----
-
-## Cloud Providers (Recommended for Most)
-
-### OpenAI
-
-**Cost:** ~$0.03-0.15 per 1K tokens (varies by model)
-
-**Setup:**
-```bash
-1. Go to https://platform.openai.com/api-keys
-2. Create account (if needed)
-3. Create new API key (starts with "sk-proj-")
-4. Add $5+ credits to account
-5. Add to .env:
-   OPENAI_API_KEY=sk-proj-...
-6. Restart services
-```
-
-**Environment Variable:**
-```
-OPENAI_API_KEY=sk-proj-xxxxx
-```
-
-**Available Models (in Open Notebook):**
-- `gpt-4o` — Best quality, fast (latest version)
-- `gpt-4o-mini` — Fast, cheap, good for testing
-- `o1` — Advanced reasoning model (slower, more expensive)
-- `o1-mini` — Faster reasoning model
-
-**Recommended:**
-- For general use: `gpt-4o` (best balance)
-- For testing/cheap: `gpt-4o-mini` (90% cheaper)
-- For complex reasoning: `o1` (best for hard problems)
-
-**Cost Estimate:**
-```
-Light use: $1-5/month
-Medium use: $10-30/month
-Heavy use: $50-100+/month
-```
-
-**Troubleshooting:**
-- "Invalid API key" → Check key starts with "sk-proj-"
-- "Rate limit exceeded" → Wait or upgrade account
-- "Model not available" → Try gpt-4o-mini instead
-
----
-
-### Anthropic (Claude)
-
-**Cost:** ~$0.80-3.00 per 1M tokens (cheaper than OpenAI for long context)
-
-**Setup:**
-```bash
-1. Go to https://console.anthropic.com/
-2. Create account or login
-3. Go to API keys section
-4. Create new API key (starts with "sk-ant-")
-5. Add to .env:
-   ANTHROPIC_API_KEY=sk-ant-...
-6. Restart services
-```
-
-**Environment Variable:**
-```
-ANTHROPIC_API_KEY=sk-ant-xxxxx
-```
-
-**Available Models:**
-- `claude-sonnet-4-5-20250929` — Latest, best quality (recommended)
-- `claude-3-5-sonnet-20241022` — Previous generation, still excellent
-- `claude-3-5-haiku-20241022` — Fast, cheap
-- `claude-opus-4-5-20251101` — Most powerful, expensive
-
-**Recommended:**
-- For general use: `claude-sonnet-4-5` (best overall, latest)
-- For cheap: `claude-3-5-haiku` (80% cheaper)
-- For complex: `claude-opus-4-5` (most capable)
-
-**Cost Estimate:**
-```
-Sonnet: $3-20/month (typical use)
-Haiku: $0.50-3/month
-Opus: $10-50+/month
-```
-
-**Advantages:**
-- Great long-context support (200K tokens)
-- Excellent reasoning
-- Fast processing
-
-**Troubleshooting:**
-- "Invalid API key" → Check it starts with "sk-ant-"
-- "Overloaded" → Anthropic is busy, retry later
-- "Model unavailable" → Check model name is correct
-
----
-
-### Google Gemini
-
-**Cost:** ~$0.075-0.30 per 1K tokens (competitive with OpenAI)
-
-**Setup:**
-```bash
-1. Go to https://aistudio.google.com/app/apikey
-2. Create account or login
-3. Create new API key
-4. Add to .env:
-   GOOGLE_API_KEY=AIzaSy...
-5. Restart services
-```
-
-**Environment Variable:**
-```
-GOOGLE_API_KEY=AIzaSy...
-# Optional: override default endpoint
-GEMINI_API_BASE_URL=https://generativelanguage.googleapis.com/v1beta/models
-```
-
-**Available Models:**
-- `gemini-2.0-flash-exp` — Latest experimental, fastest (recommended)
-- `gemini-2.0-flash` — Stable version, fast, cheap
-- `gemini-1.5-pro-latest` — More capable, longer context
-- `gemini-1.5-flash` — Previous generation, very cheap
-
-**Recommended:**
-- For general use: `gemini-2.0-flash-exp` (best value, latest)
-- For cheap: `gemini-1.5-flash` (very cheap)
-- For complex/long context: `gemini-1.5-pro-latest` (2M token context)
-
-**Advantages:**
-- Very long context (1M tokens)
-- Multimodal (images, audio, video)
-- Good for podcasts
-
-**Troubleshooting:**
-- "API key invalid" → Get fresh key from aistudio.google.com
-- "Quota exceeded" → Free tier limited, upgrade account
-- "Model not found" → Check model name spelling
-
----
-
-### Groq
-
-**Cost:** ~$0.05 per 1M tokens (cheapest, but limited models)
-
-**Setup:**
-```bash
-1. Go to https://console.groq.com/keys
-2. Create account or login
-3. Create new API key
-4. Add to .env:
-   GROQ_API_KEY=gsk_...
-5. Restart services
-```
-
-**Environment Variable:**
-```
-GROQ_API_KEY=gsk_xxxxx
-```
-
-**Available Models:**
-- `llama-3.3-70b-versatile` — Best on Groq (recommended)
-- `llama-3.1-70b-versatile` — Fast, capable
-- `mixtral-8x7b-32768` — Good alternative
-- `gemma2-9b-it` — Small, very fast
-
-**Recommended:**
-- For quality: `llama-3.3-70b-versatile` (best overall)
-- For speed: `gemma2-9b-it` (ultra-fast)
-- For balance: `llama-3.1-70b-versatile`
-
-**Advantages:**
-- Ultra-fast inference
-- Very cheap
-- Great for transformations/batch work
-
-**Disadvantages:**
-- Limited model selection
-- Smaller models than OpenAI/Anthropic
-
-**Troubleshooting:**
-- "Rate limited" → Free tier has limits, upgrade
-- "Model not available" → Check supported models list
-
----
-
-### OpenRouter
-
-**Cost:** Varies by model ($0.05-15 per 1M tokens)
-
-**Setup:**
-```bash
-1. Go to https://openrouter.ai/keys
-2. Create account or login
-3. Add credits to your account
-4. Create new API key
-5. Add to .env:
-   OPENROUTER_API_KEY=sk-or-...
-6. Restart services
-```
-
-**Environment Variable:**
-```
-OPENROUTER_API_KEY=sk-or-xxxxx
-```
-
-**Available Models (100+ options):**
-- OpenAI: `openai/gpt-4o`, `openai/o1`
-- Anthropic: `anthropic/claude-sonnet-4.5`, `anthropic/claude-3.5-haiku`
-- Google: `google/gemini-2.0-flash-exp`, `google/gemini-1.5-pro`
-- Meta: `meta-llama/llama-3.3-70b-instruct`, `meta-llama/llama-3.1-405b-instruct`
-- Mistral: `mistralai/mistral-large-2411`
-- DeepSeek: `deepseek/deepseek-chat`
-- And many more...
-
-**Recommended:**
-- For quality: `anthropic/claude-sonnet-4.5` (best overall)
-- For speed/cost: `google/gemini-2.0-flash-exp` (very fast, cheap)
-- For open-source: `meta-llama/llama-3.3-70b-instruct`
-- For reasoning: `openai/o1`
-
-**Advantages:**
-- One API key for 100+ models
-- Unified billing
-- Easy model comparison
-- Access to models that may have waitlists elsewhere
-
-**Cost Estimate:**
-```
-Light use: $1-5/month
-Medium use: $10-30/month
-Heavy use: Depends on models chosen
-```
-
-**Troubleshooting:**
-- "Invalid API key" → Check it starts with "sk-or-"
-- "Insufficient credits" → Add credits at openrouter.ai
-- "Model not available" → Check model ID spelling (use full path)
-
----
-
-## Self-Hosted / Local
-
-### Ollama (Recommended for Local)
-
-**Cost:** Free (electricity only)
-
-**Setup:**
-```bash
-1. Install Ollama: https://ollama.ai
-2. Run Ollama in background:
-   ollama serve
-
-3. Download a model:
-   ollama pull mistral
-   # or llama2, neural-chat, phi, etc.
-
-4. Add to .env:
-   OLLAMA_API_BASE=http://localhost:11434
-   # If on different machine:
-   # OLLAMA_API_BASE=http://10.0.0.5:11434
-
-5. Restart services
-```
-
-**Environment Variable:**
-```
-OLLAMA_API_BASE=http://localhost:11434
-```
-
-**Available Models:**
-- `llama3.3:70b` — Best quality (requires 40GB+ RAM)
-- `llama3.1:8b` — Recommended, balanced (8GB RAM)
-- `qwen2.5:7b` — Excellent for code and reasoning
-- `mistral:7b` — Good general purpose
-- `phi3:3.8b` — Small, fast (4GB RAM)
-- `gemma2:9b` — Google's model, balanced
-- Many more: `ollama list` to see available
-
-**Recommended:**
-- For quality (with GPU): `llama3.3:70b` (best)
-- For general use: `llama3.1:8b` (best balance)
-- For speed/low memory: `phi3:3.8b` (very fast)
-- For coding: `qwen2.5:7b` (excellent at code)
-
-**Hardware Requirements:**
-```
-GPU (NVIDIA/AMD):
-  8GB VRAM: Runs most models fine
-  6GB VRAM: Works, slower
-  4GB VRAM: Small models only
-
-CPU-only:
-  16GB+ RAM: Slow but works
-  8GB RAM: Very slow
-  4GB RAM: Not recommended
-```
-
-**Advantages:**
-- Completely private (runs locally)
-- Free (electricity only)
-- No API key needed
-- Works offline
-
-**Disadvantages:**
-- Slower than cloud (unless on GPU)
-- Smaller models than cloud
-- Requires local hardware
-
-**Troubleshooting:**
-- "Connection refused" → Ollama not running or wrong port
-- "Model not found" → Download it: `ollama pull modelname`
-- "Out of memory" → Use smaller model or add more RAM
-
----
-
-### LM Studio (Local Alternative)
-
-**Cost:** Free
-
-**Setup:**
-```bash
-1. Download LM Studio: https://lmstudio.ai
-2. Open app
-3. Download a model from library
-4. Go to "Local Server" tab
-5. Start server (default port: 1234)
-6. Add to .env:
-   OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1
-   OPENAI_COMPATIBLE_API_KEY=not-needed
-7. Restart services
-```
-
-**Environment Variables:**
-```
-OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1
-OPENAI_COMPATIBLE_API_KEY=lm-studio  # Just a placeholder
-```
-
-**Advantages:**
-- GUI interface (easier than Ollama CLI)
-- Good model selection
-- Privacy-focused
-- Works offline
-
-**Disadvantages:**
-- Desktop only (Mac/Windows/Linux)
-- Slower than cloud
-- Requires local GPU
-
----
-
-### Custom OpenAI-Compatible
-
-For Text Generation UI, vLLM, or other OpenAI-compatible endpoints:
-
-```bash
-Add to .env:
-OPENAI_COMPATIBLE_BASE_URL=http://your-endpoint/v1
-OPENAI_COMPATIBLE_API_KEY=your-api-key
-```
-
-If you need different endpoints for different modalities:
-
-```bash
-# Language model
-OPENAI_COMPATIBLE_BASE_URL_LLM=http://localhost:8000/v1
-OPENAI_COMPATIBLE_API_KEY_LLM=sk-...
-
-# Embeddings
-OPENAI_COMPATIBLE_BASE_URL_EMBEDDING=http://localhost:8001/v1
-OPENAI_COMPATIBLE_API_KEY_EMBEDDING=sk-...
-
-# TTS (text-to-speech)
-OPENAI_COMPATIBLE_BASE_URL_TTS=http://localhost:8002/v1
-OPENAI_COMPATIBLE_API_KEY_TTS=sk-...
-```
-
----
-
-## Enterprise
-
-### Azure OpenAI
-
-**Cost:** Same as OpenAI (usage-based)
-
-**Setup:**
-```bash
-1. Create Azure OpenAI service in Azure portal
-2. Deploy GPT-4/3.5-turbo model
-3. Get your endpoint and key
-4. Add to .env:
-   AZURE_OPENAI_API_KEY=your-key
-   AZURE_OPENAI_ENDPOINT=https://your-name.openai.azure.com/
-   AZURE_OPENAI_API_VERSION=2024-12-01-preview
-5. Restart services
-```
-
-**Environment Variables:**
-```
-AZURE_OPENAI_API_KEY=xxxxx
-AZURE_OPENAI_ENDPOINT=https://your-instance.openai.azure.com/
-AZURE_OPENAI_API_VERSION=2024-12-01-preview
-
-# Optional: Different deployments for different modalities
-AZURE_OPENAI_API_KEY_LLM=xxxxx
-AZURE_OPENAI_ENDPOINT_LLM=https://your-instance.openai.azure.com/
-AZURE_OPENAI_API_VERSION_LLM=2024-12-01-preview
-```
-
-**Advantages:**
-- Enterprise support
-- VPC integration
-- Compliance (HIPAA, SOC2, etc.)
-
-**Disadvantages:**
-- More complex setup
-- Higher overhead
-- Requires Azure account
-
----
-
-## Embeddings (For Search/Semantic Features)
-
-By default, Open Notebook uses the LLM provider's embeddings. To use a different provider:
-
-### OpenAI Embeddings (Default)
-```
-# Uses OpenAI's embedding model automatically
-# Requires OPENAI_API_KEY
-# No separate configuration needed
-```
-
-### Custom Embeddings
-```
-# For other embedding providers (future feature)
-EMBEDDING_PROVIDER=openai  # or custom
-```
-
----
-
-## Choosing Your Provider
-
-**1. Don't want to run locally and don't want to mess around with different providers:** 
-
-Use OpenAI
-- Cloud-based
-- Good quality
-- Reasonable cost
-- Simplest setup, supports all modes (text, embedding, tts, stt, etc)
-
-**For budget-conscious:** Groq, OpenRouter or Ollama
-- Groq: Super cheap cloud
-- Ollama: Free, but local
-- OpenRouter: many open source models very accessible
-
-**For privacy-first:** Ollama or LM Studio and [Speaches](local-tts.md)
-- Everything stays local
-- Works offline
-- No API keys sent anywhere
-
-**For enterprise:** Azure OpenAI
-- Compliance
-- VPC integration
-- Support
-
----
-
-## Next Steps
-
-1. **Choose your provider** from above
-2. **Get API key** (if cloud) or install locally (if Ollama)
-3. **Add to .env**
-4. **Restart services**
-5. **Go to Settings → Models** in Open Notebook
-6. **Verify it works** with a test chat
-
-Done!
-
----
-
-## Related
-
-- **[Environment Reference](environment-reference.md)** - Complete list of all environment variables
-- **[Advanced Configuration](advanced.md)** - Timeouts, SSL, performance tuning
-- **[Ollama Setup](ollama.md)** - Detailed Ollama configuration guide
-- **[OpenAI-Compatible](openai-compatible.md)** - LM Studio and other compatible providers
-- **[Troubleshooting](../6-TROUBLESHOOTING/quick-fixes.md)** - Common issues and fixes

docs/5-CONFIGURATION/database.md

@@ -1,50 +0,0 @@
-# Database - SurrealDB Configuration
-
-Open Notebook uses SurrealDB for its database needs. 
-
----
-
-## Default Configuration
-
-Open Notebook should work out of the box with SurrealDB as long as the environment variables are correctly setup. 
-
-
-### DB running in the same docker compose as Open Notebook (recommended)
-
-The example above is for when you are running SurrealDB as a separate docker container, which is the method described [here](../1-INSTALLATION/docker-compose.md) (and our recommended method). 
-
-```env
-SURREAL_URL="ws://surrealdb:8000/rpc"
-SURREAL_USER="root"
-SURREAL_PASSWORD="root"
-SURREAL_NAMESPACE="open_notebook"
-SURREAL_DATABASE="open_notebook"
-```
-
-### DB running in the host machine and Open Notebook running in Docker
-
-If ON is running in docker and SurrealDB is on your host machine, you need to point to it. 
-
-```env
-SURREAL_URL="ws://your-machine-ip:8000/rpc" #or host.docker.internal
-SURREAL_USER="root"
-SURREAL_PASSWORD="root"
-SURREAL_NAMESPACE="open_notebook"
-SURREAL_DATABASE="open_notebook"
-```
-
-### Open Notebook and Surreal are running on the same machine
-
-If you are running both services locally or if you are using the deprecated [single container setup](../1-INSTALLATION/single-container.md)
-
-```env
-SURREAL_URL="ws://localhost:8000/rpc"
-SURREAL_USER="root"
-SURREAL_PASSWORD="root"
-SURREAL_NAMESPACE="open_notebook"
-SURREAL_DATABASE="open_notebook"
-```
-
-## Multiple databases
-
-You can have multiple namespaces in one SurrealDB instance and you can also have multiple databases in one instance. So, if you want to setup multiple open noteobok deployments for different users, you don't need to deploy multiple databases. 

docs/5-CONFIGURATION/environment-reference.md

@@ -1,320 +0,0 @@
-# Complete Environment Reference
-
-Comprehensive list of all environment variables available in Open Notebook.
-
----
-
-## API Configuration
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `API_URL` | No | Auto-detected | URL where frontend reaches API (e.g., http://localhost:5055) |
-| `INTERNAL_API_URL` | No | http://localhost:5055 | Internal API URL for Next.js server-side proxying |
-| `API_CLIENT_TIMEOUT` | No | 300 | Client timeout in seconds (how long to wait for API response) |
-| `OPEN_NOTEBOOK_PASSWORD` | No | None | Password to protect Open Notebook instance |
-
----
-
-## AI Provider: OpenAI
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `OPENAI_API_KEY` | If using OpenAI | None | OpenAI API key (starts with `sk-`) |
-
-**Setup:** Get from https://platform.openai.com/api-keys
-
----
-
-## AI Provider: Anthropic (Claude)
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `ANTHROPIC_API_KEY` | If using Anthropic | None | Claude API key (starts with `sk-ant-`) |
-
-**Setup:** Get from https://console.anthropic.com/
-
----
-
-## AI Provider: Google Gemini
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `GOOGLE_API_KEY` | If using Google | None | Google API key for Gemini |
-| `GEMINI_API_BASE_URL` | No | Default endpoint | Override Gemini API endpoint |
-| `VERTEX_PROJECT` | If using Vertex AI | None | Google Cloud project ID |
-| `VERTEX_LOCATION` | If using Vertex AI | us-east5 | Vertex AI location |
-| `GOOGLE_APPLICATION_CREDENTIALS` | If using Vertex AI | None | Path to service account JSON |
-
-**Setup:** Get from https://aistudio.google.com/app/apikey
-
----
-
-## AI Provider: Groq
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `GROQ_API_KEY` | If using Groq | None | Groq API key (starts with `gsk_`) |
-
-**Setup:** Get from https://console.groq.com/keys
-
----
-
-## AI Provider: Mistral
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `MISTRAL_API_KEY` | If using Mistral | None | Mistral API key |
-
-**Setup:** Get from https://console.mistral.ai/
-
----
-
-## AI Provider: DeepSeek
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `DEEPSEEK_API_KEY` | If using DeepSeek | None | DeepSeek API key |
-
-**Setup:** Get from https://platform.deepseek.com/
-
----
-
-## AI Provider: xAI
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `XAI_API_KEY` | If using xAI | None | xAI API key |
-
-**Setup:** Get from https://console.x.ai/
-
----
-
-## AI Provider: Ollama (Local)
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `OLLAMA_API_BASE` | If using Ollama | None | Ollama endpoint (e.g., http://localhost:11434) |
-
-**Setup:** Install from https://ollama.ai
-
----
-
-## AI Provider: OpenRouter
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `OPENROUTER_API_KEY` | If using OpenRouter | None | OpenRouter API key |
-| `OPENROUTER_BASE_URL` | No | https://openrouter.ai/api/v1 | OpenRouter endpoint |
-
-**Setup:** Get from https://openrouter.ai/
-
----
-
-## AI Provider: OpenAI-Compatible
-
-For self-hosted LLMs, LM Studio, or OpenAI-compatible endpoints:
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `OPENAI_COMPATIBLE_BASE_URL` | If using compatible | None | Base URL for OpenAI-compatible endpoint |
-| `OPENAI_COMPATIBLE_API_KEY` | If using compatible | None | API key for endpoint |
-| `OPENAI_COMPATIBLE_BASE_URL_LLM` | No | Uses generic | Language model endpoint (overrides generic) |
-| `OPENAI_COMPATIBLE_API_KEY_LLM` | No | Uses generic | Language model API key (overrides generic) |
-| `OPENAI_COMPATIBLE_BASE_URL_EMBEDDING` | No | Uses generic | Embedding endpoint (overrides generic) |
-| `OPENAI_COMPATIBLE_API_KEY_EMBEDDING` | No | Uses generic | Embedding API key (overrides generic) |
-| `OPENAI_COMPATIBLE_BASE_URL_STT` | No | Uses generic | Speech-to-text endpoint (overrides generic) |
-| `OPENAI_COMPATIBLE_API_KEY_STT` | No | Uses generic | STT API key (overrides generic) |
-| `OPENAI_COMPATIBLE_BASE_URL_TTS` | No | Uses generic | Text-to-speech endpoint (overrides generic) |
-| `OPENAI_COMPATIBLE_API_KEY_TTS` | No | Uses generic | TTS API key (overrides generic) |
-
-**Setup:** For LM Studio, typically: `OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1`
-
----
-
-## AI Provider: Azure OpenAI
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `AZURE_OPENAI_API_KEY` | If using Azure | None | Azure OpenAI API key |
-| `AZURE_OPENAI_ENDPOINT` | If using Azure | None | Azure OpenAI endpoint URL |
-| `AZURE_OPENAI_API_VERSION` | No | 2024-12-01-preview | Azure OpenAI API version |
-| `AZURE_OPENAI_API_KEY_LLM` | No | Uses generic | LLM-specific API key |
-| `AZURE_OPENAI_ENDPOINT_LLM` | No | Uses generic | LLM-specific endpoint |
-| `AZURE_OPENAI_API_VERSION_LLM` | No | Uses generic | LLM-specific API version |
-| `AZURE_OPENAI_API_KEY_EMBEDDING` | No | Uses generic | Embedding-specific API key |
-| `AZURE_OPENAI_ENDPOINT_EMBEDDING` | No | Uses generic | Embedding-specific endpoint |
-| `AZURE_OPENAI_API_VERSION_EMBEDDING` | No | Uses generic | Embedding-specific API version |
-
----
-
-## AI Provider: VoyageAI (Embeddings)
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `VOYAGE_API_KEY` | If using Voyage | None | Voyage AI API key (for embeddings) |
-
-**Setup:** Get from https://www.voyageai.com/
-
----
-
-## Text-to-Speech (TTS)
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `ELEVENLABS_API_KEY` | If using ElevenLabs TTS | None | ElevenLabs API key for voice generation |
-| `TTS_BATCH_SIZE` | No | 5 | Concurrent TTS requests (1-5, depends on provider) |
-
-**Setup:** Get from https://elevenlabs.io/
-
----
-
-## Database: SurrealDB
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `SURREAL_URL` | Yes | ws://surrealdb:8000/rpc | SurrealDB WebSocket connection URL |
-| `SURREAL_USER` | Yes | root | SurrealDB username |
-| `SURREAL_PASSWORD` | Yes | root | SurrealDB password |
-| `SURREAL_NAMESPACE` | Yes | open_notebook | SurrealDB namespace |
-| `SURREAL_DATABASE` | Yes | open_notebook | SurrealDB database name |
-
----
-
-## Database: Retry Configuration
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `SURREAL_COMMANDS_RETRY_ENABLED` | No | true | Enable retries on failure |
-| `SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS` | No | 3 | Maximum retry attempts |
-| `SURREAL_COMMANDS_RETRY_WAIT_STRATEGY` | No | exponential_jitter | Retry wait strategy (exponential_jitter/exponential/fixed/random) |
-| `SURREAL_COMMANDS_RETRY_WAIT_MIN` | No | 1 | Minimum wait time between retries (seconds) |
-| `SURREAL_COMMANDS_RETRY_WAIT_MAX` | No | 30 | Maximum wait time between retries (seconds) |
-
----
-
-## Database: Concurrency
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `SURREAL_COMMANDS_MAX_TASKS` | No | 5 | Maximum concurrent database tasks |
-
----
-
-## LLM Timeouts
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `ESPERANTO_LLM_TIMEOUT` | No | 60 | LLM inference timeout in seconds |
-| `ESPERANTO_SSL_VERIFY` | No | true | Verify SSL certificates (false = development only) |
-| `ESPERANTO_SSL_CA_BUNDLE` | No | None | Path to custom CA certificate bundle |
-
----
-
-## Content Extraction
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `FIRECRAWL_API_KEY` | No | None | Firecrawl API key for advanced web scraping |
-| `JINA_API_KEY` | No | None | Jina AI API key for web extraction |
-
-**Setup:**
-- Firecrawl: https://firecrawl.dev/
-- Jina: https://jina.ai/
-
----
-
-## Debugging & Monitoring
-
-| Variable | Required? | Default | Description |
-|----------|-----------|---------|-------------|
-| `LANGCHAIN_TRACING_V2` | No | false | Enable LangSmith tracing |
-| `LANGCHAIN_ENDPOINT` | No | https://api.smith.langchain.com | LangSmith endpoint |
-| `LANGCHAIN_API_KEY` | No | None | LangSmith API key |
-| `LANGCHAIN_PROJECT` | No | Open Notebook | LangSmith project name |
-
-**Setup:** https://smith.langchain.com/
-
----
-
-## Environment Variables by Use Case
-
-### Minimal Setup (Cloud Provider)
-```
-OPENAI_API_KEY (or ANTHROPIC_API_KEY, etc.)
-```
-
-### Local Development (Ollama)
-```
-OLLAMA_API_BASE=http://localhost:11434
-```
-
-### Production (OpenAI + Custom Domain)
-```
-OPENAI_API_KEY=sk-proj-...
-API_URL=https://mynotebook.example.com
-SURREAL_USER=production_user
-SURREAL_PASSWORD=secure_password
-```
-
-### Self-Hosted Behind Reverse Proxy
-```
-OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1
-API_URL=https://mynotebook.example.com
-```
-
-### High-Performance Deployment
-```
-OPENAI_API_KEY=sk-proj-...
-SURREAL_COMMANDS_MAX_TASKS=10
-TTS_BATCH_SIZE=5
-API_CLIENT_TIMEOUT=600
-```
-
-### Debugging
-```
-OPENAI_API_KEY=sk-proj-...
-LANGCHAIN_TRACING_V2=true
-LANGCHAIN_API_KEY=your-key
-```
-
----
-
-## Validation
-
-Check if a variable is set:
-
-```bash
-# Check single variable
-echo $OPENAI_API_KEY
-
-# Check multiple
-env | grep -E "OPENAI|API_URL"
-
-# Print all config
-env | grep -E "^[A-Z_]+=" | sort
-```
-
----
-
-## Notes
-
-- **Case-sensitive:** `OPENAI_API_KEY` ≠ `openai_api_key`
-- **No spaces:** `OPENAI_API_KEY=sk-proj-...` not `OPENAI_API_KEY = sk-proj-...`
-- **Quote values:** Use quotes for values with spaces: `API_URL="http://my server:5055"`
-- **Restart required:** Changes take effect after restarting services
-- **Secrets:** Don't commit API keys to git
-
----
-
-## Quick Setup Checklist
-
-- [ ] Choose AI provider (OpenAI, Anthropic, Ollama, etc.)
-- [ ] Get API key if cloud provider
-- [ ] Add to .env or docker.env
-- [ ] Set `API_URL` if behind reverse proxy
-- [ ] Change `SURREAL_PASSWORD` in production
-- [ ] Verify with: `docker compose logs api | grep -i "error"`
-- [ ] Test in browser: Go to Settings → Models
-- [ ] Try a test chat
-
-Done!

docs/5-CONFIGURATION/index.md

@@ -1,334 +0,0 @@
-# Configuration - Essential Settings
-
-Configuration is how you customize Open Notebook for your specific setup. This section covers what you need to know.
-
----
-
-## What Needs Configuration?
-
-Three things:
-
-1. **AI Provider** — Which LLM/embedding service you're using (OpenAI, Anthropic, Ollama, etc.)
-2. **Database** — How to connect to SurrealDB (usually pre-configured)
-3. **Server** — API URL, ports, timeouts (usually auto-detected)
-
----
-
-## Quick Decision: Which Provider?
-
-### Option 1: Cloud Provider (Fastest)
-- **OpenRouter (recommended)** (access to all models with one key)
-- **OpenAI** (GPT)
-- **Anthropic** (Claude)
-- **Google Gemini** (multi-modal, long context)
-- **Groq** (ultra-fast inference)
-
-Setup: Get API key → Set env var → Done
-
-→ Go to **[AI Providers Guide](ai-providers.md)**
-
-### Option 2: Local (Free & Private)
-- **Ollama** (open-source models, on your machine)
-
-→ Go to **[Ollama Setup](ollama.md)**
-
-### Option 3: OpenAI-Compatible
-- **LM Studio** (local)
-- **Custom endpoints**
-
-→ Go to **[OpenAI-Compatible Guide](openai-compatible.md)**
-
----
-
-## Configuration File
-
-Use the right file depending on your setup.
-
-### `.env` (Local Development)
-
-You will only use .env if you are running Open Notebook locally.
-
-```
-Located in: project root
-Use for: Development on your machine
-Format: KEY=value, one per line
-```
-
-### `docker.env` (Docker Deployment)
-
-You will use this file to hold your environment variables if you are using docker-compose and prefer not to put the variables directly in the compose file. 
-```
-Located in: project root (or ./docker)
-Use for: Docker deployments
-Format: Same as .env
-Loaded by: docker-compose.yml
-```
-
----
-
-## Most Important Settings
-
-All of the settings provided below are to be placed inside your environment file (.env or docker.env depending on your setup).
-
-
-###  Surreal Database
-
-This is the database used by the app.
-
-```
-SURREAL_URL=ws://surrealdb:8000/rpc
-SURREAL_USER=root
-SURREAL_PASSWORD=root  # Change in production!
-SURREAL_NAMESPACE=open_notebook
-SURREAL_DATABASE=open_notebook
-```
-
-> The only thing that is critical to not miss is the hostname in the `SURREAL_URL`. Check what URL to use based on your deployment, [here](database.md).
-
-
-### AI Provider (API Key or URL)
-
-We need access to LLMs in order for the app to work. You can use any of the support AI Providers by adding their API Keys. 
-
-```
-OPENAI_API_KEY=sk-...
-ANTHROPIC_API_KEY=sk-ant-...
-GOOGLE_API_KEY=...
-OPENROUTER_API_KEY=...
-```
-
-Or, if you are planning to use only local providers, you can setup Ollama by configuring it's base URL. This will get you set and ready with text and embeddings in one go: 
-
-```
-OLLAMA_BASE_URL=http://localhost:11434
-```
-
-> A lot of people screw up on the Ollama BASE URL by not knowing how to point to their Ollama installation. if you are having trouble connecting to Ollama, see [here](ollama.md).
-
-You can also use LM Studio locally if you prefer by using it as an OpenAI compatible endpoint. 
-
-```
-OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1
-OPENAI_COMPATIBLE_BASE_URL_EMBEDDING=http://localhost:1234/v1
-```
-
-> For more installation on using OpenAI compatible endpoints, see [here](openai-compatible.md).
-
-
-### API URL (If Behind Reverse Proxy)
-You only need to worry about this if you are deploying on a proxy or if you are changing port information. Otherwise, skip this.
-
-```
-API_URL=https://your-domain.com
-# Usually auto-detected. Only set if needed.
-```
-
-Auto-detection works for most setups.
-
----
-
-## Configuration by Scenario
-
-### Scenario 1: Docker on Localhost (Default)
-```env
-# In docker.env:
-OPENAI_API_KEY=sk-...
-# Everything else uses defaults
-# Done!
-```
-
-### Scenario 2: Docker on Remote Server
-```env
-# In docker.env:
-OPENAI_API_KEY=sk-...
-API_URL=http://your-server-ip:5055
-```
-
-### Scenario 3: Behind Reverse Proxy (Nginx/Cloudflare)
-```env
-# In docker.env:
-OPENAI_API_KEY=sk-...
-API_URL=https://your-domain.com
-# The reverse proxy handles HTTPS
-```
-
-### Scenario 4: Using Ollama Locally
-```env
-# In .env:
-OLLAMA_API_BASE=http://localhost:11434
-# No API key needed
-```
-
-### Scenario 5: Using Azure OpenAI
-```env
-# In docker.env:
-AZURE_OPENAI_API_KEY=your-key
-AZURE_OPENAI_ENDPOINT=https://your-instance.openai.azure.com/
-AZURE_OPENAI_API_VERSION=2024-12-01-preview
-```
-
----
-
-## Configuration Sections
-
-### [AI Providers](ai-providers.md)
-- OpenAI configuration
-- Anthropic configuration
-- Google Gemini configuration
-- Groq configuration
-- Ollama configuration
-- Azure OpenAI configuration
-- OpenAI-compatible configuration
-
-### [Database](database.md)
-- SurrealDB setup
-- Connection strings
-- Database vs. namespace
-- Running your own SurrealDB
-
-### [Advanced](advanced.md)
-- Ports and networking
-- Timeouts and concurrency
-- SSL/security
-- Retry configuration
-- Worker concurrency
-- Language models & embeddings
-- Speech-to-text & text-to-speech
-- Debugging and logging
-
-### [Reverse Proxy](reverse-proxy.md)
-- Nginx, Caddy, Traefik configs
-- Custom domain setup
-- SSL/HTTPS configuration
-- Coolify and other platforms
-
-### [Security](security.md)
-- Password protection
-- API authentication
-- Production hardening
-- Firewall configuration
-
-### [Local TTS](local-tts.md)
-- Speaches setup for local text-to-speech
-- GPU acceleration
-- Voice options
-- Docker networking
-
-### [Ollama](ollama.md)
-- Setting up and pointing to an Ollama server
-- Downloading models
-- Using embedding
-
-### [OpenAI-Compatible Providers](openai-compatible.md)
-- LM Studio, vLLM, Text Generation WebUI
-- Connection configuration
-- Docker networking
-- Troubleshooting
-
-### [Complete Reference](environment-reference.md)
-- All environment variables
-- Grouped by category
-- What each one does
-- Default values
-
----
-
-## How to Add Configuration
-
-### Method 1: Edit `.env` File (Development)
-
-```bash
-1. Open .env in your editor
-2. Find the section for your provider
-3. Uncomment and fill in your API key
-4. Save
-5. Restart services
-```
-
-### Method 2: Set Docker Environment (Deployment)
-
-```bash
-# In docker-compose.yml:
-services:
-  api:
-    environment:
-      - OPENAI_API_KEY=sk-...
-      - API_URL=https://your-domain.com
-```
-
-### Method 3: Export Environment Variables
-
-```bash
-# In your terminal:
-export OPENAI_API_KEY=sk-...
-export API_URL=https://your-domain.com
-
-# Then start services
-docker compose up
-```
-
-### Method 4: Use docker.env File
-
-```bash
-1. Create/edit docker.env
-2. Add your configuration
-3. docker-compose automatically loads it
-4. docker compose up
-```
-
----
-
-## Verification
-
-After configuration, verify it works:
-
-```
-1. Open your notebook
-2. Go to Settings → Models
-3. You should see your configured provider
-4. Try a simple Chat question
-5. If it responds, configuration is correct!
-```
-
----
-
-## Common Mistakes
-
-| Mistake | Problem | Fix |
-|---------|---------|-----|
-| Forget API key | Models not available | Add OPENAI_API_KEY (or your provider) |
-| Wrong database URL | Can't start API | Check SURREAL_URL format |
-| Expose port 5055 | "Can't connect to server" | Expose 5055 in docker-compose |
-| Typo in env var | Settings ignored | Check spelling (case-sensitive!) |
-| Quote mismatch | Value cut off | Use quotes: OPENAI_API_KEY="sk-..." |
-| Don't restart | Old config still used | Restart services after env changes |
-
----
-
-## What Comes After Configuration
-
-Once configured:
-
-1. **[Quick Start](../0-START-HERE/index.md)** — Run your first notebook
-2. **[Installation](../1-INSTALLATION/index.md)** — Multi-route deployment guides
-3. **[User Guide](../3-USER-GUIDE/index.md)** — How to use each feature
-
----
-
-## Getting Help
-
-- **Configuration error?** → Check [Troubleshooting](../6-TROUBLESHOOTING/quick-fixes.md)
-- **Provider-specific issue?** → Check [AI Providers](ai-providers.md)
-- **Need complete reference?** → See [Environment Reference](environment-reference.md)
-
----
-
-## Summary
-
-**Minimal configuration to run:**
-1. Choose an AI provider (or use Ollama locally)
-2. Set API key in .env or docker.env
-3. Start services
-4. Done!
-
-Everything else is optional optimization.

docs/5-CONFIGURATION/local-tts.md

@@ -1,339 +0,0 @@
-# Local Text-to-Speech Setup
-
-Run text-to-speech locally for free, private podcast generation using OpenAI-compatible TTS servers.
-
----
-
-## Why Local TTS?
-
-| Benefit | Description |
-|---------|-------------|
-| **Free** | No per-character costs after setup |
-| **Private** | Audio never leaves your machine |
-| **Unlimited** | No rate limits or quotas |
-| **Offline** | Works without internet |
-
----
-
-## Quick Start with Speaches
-
-[Speaches](https://github.com/speaches-ai/speaches) is an open-source, OpenAI-compatible TTS server.
-
-### Step 1: Create Docker Compose File
-
-Create a folder and add `docker-compose.yml`:
-
-```yaml
-services:
-  speaches:
-    image: ghcr.io/speaches-ai/speaches:latest-cpu
-    container_name: speaches
-    ports:
-      - "8969:8000"
-    volumes:
-      - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub
-    restart: unless-stopped
-
-volumes:
-  hf-hub-cache:
-```
-
-### Step 2: Start and Download Model
-
-```bash
-# Start Speaches
-docker compose up -d
-
-# Wait for startup
-sleep 10
-
-# Download voice model (~500MB)
-docker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX
-```
-
-### Step 3: Test
-
-```bash
-curl "http://localhost:8969/v1/audio/speech" -s \
-  -H "Content-Type: application/json" \
-  --output test.mp3 \
-  --data '{
-    "input": "Hello! Local TTS is working.",
-    "model": "speaches-ai/Kokoro-82M-v1.0-ONNX",
-    "voice": "af_bella"
-  }'
-```
-
-Play `test.mp3` to verify.
-
-### Step 4: Configure Open Notebook
-
-**Docker deployment:**
-```yaml
-# In your Open Notebook docker-compose.yml
-environment:
-  - OPENAI_COMPATIBLE_BASE_URL_TTS=http://host.docker.internal:8969/v1
-```
-
-**Local development:**
-```bash
-export OPENAI_COMPATIBLE_BASE_URL_TTS=http://localhost:8969/v1
-```
-
-### Step 5: Add Model in Open Notebook
-
-1. Go to **Settings** → **Models**
-2. Click **Add Model** in Text-to-Speech section
-3. Configure:
-   - **Provider**: `openai_compatible`
-   - **Model Name**: `speaches-ai/Kokoro-82M-v1.0-ONNX`
-   - **Display Name**: `Local TTS`
-4. Click **Save**
-5. Set as default if desired
-
----
-
-## Available Voices
-
-The Kokoro model includes multiple voices:
-
-### Female Voices
-| Voice ID | Description |
-|----------|-------------|
-| `af_bella` | Clear, professional |
-| `af_sarah` | Warm, friendly |
-| `af_nicole` | Energetic, expressive |
-
-### Male Voices
-| Voice ID | Description |
-|----------|-------------|
-| `am_adam` | Deep, authoritative |
-| `am_michael` | Friendly, conversational |
-
-### British Accents
-| Voice ID | Description |
-|----------|-------------|
-| `bf_emma` | British female, professional |
-| `bm_george` | British male, formal |
-
-### Test Different Voices
-
-```bash
-for voice in af_bella af_sarah am_adam am_michael; do
-  curl "http://localhost:8969/v1/audio/speech" -s \
-    -H "Content-Type: application/json" \
-    --output "test_${voice}.mp3" \
-    --data "{
-      \"input\": \"Hello, this is the ${voice} voice.\",
-      \"model\": \"speaches-ai/Kokoro-82M-v1.0-ONNX\",
-      \"voice\": \"${voice}\"
-    }"
-done
-```
-
----
-
-## GPU Acceleration
-
-For faster generation with NVIDIA GPUs:
-
-```yaml
-services:
-  speaches:
-    image: ghcr.io/speaches-ai/speaches:latest-cuda
-    container_name: speaches
-    ports:
-      - "8969:8000"
-    volumes:
-      - hf-hub-cache:/home/ubuntu/.cache/huggingface/hub
-    restart: unless-stopped
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: 1
-              capabilities: [gpu]
-
-volumes:
-  hf-hub-cache:
-```
-
----
-
-## Docker Networking
-
-### Open Notebook in Docker (macOS/Windows)
-
-```bash
-OPENAI_COMPATIBLE_BASE_URL_TTS=http://host.docker.internal:8969/v1
-```
-
-### Open Notebook in Docker (Linux)
-
-```bash
-# Option 1: Docker bridge IP
-OPENAI_COMPATIBLE_BASE_URL_TTS=http://172.17.0.1:8969/v1
-
-# Option 2: Host networking
-docker run --network host ...
-```
-
-### Remote Server
-
-Run Speaches on a different machine:
-
-```bash
-# On server, bind to all interfaces
-# Then in Open Notebook:
-OPENAI_COMPATIBLE_BASE_URL_TTS=http://server-ip:8969/v1
-```
-
----
-
-## Multi-Speaker Podcasts
-
-Configure different voices for each speaker:
-
-```
-Speaker 1 (Host):
-  Model: speaches-ai/Kokoro-82M-v1.0-ONNX
-  Voice: af_bella
-
-Speaker 2 (Guest):
-  Model: speaches-ai/Kokoro-82M-v1.0-ONNX
-  Voice: am_adam
-
-Speaker 3 (Narrator):
-  Model: speaches-ai/Kokoro-82M-v1.0-ONNX
-  Voice: bf_emma
-```
-
----
-
-## Troubleshooting
-
-### Service Won't Start
-
-```bash
-# Check logs
-docker compose logs speaches
-
-# Verify port available
-lsof -i :8969
-
-# Restart
-docker compose down && docker compose up -d
-```
-
-### Connection Refused
-
-```bash
-# Test Speaches is running
-curl http://localhost:8969/v1/models
-
-# From inside Open Notebook container
-docker exec -it open-notebook curl http://host.docker.internal:8969/v1/models
-```
-
-### Model Not Found
-
-```bash
-# List downloaded models
-docker compose exec speaches uv tool run speaches-cli model list
-
-# Download if missing
-docker compose exec speaches uv tool run speaches-cli model download speaches-ai/Kokoro-82M-v1.0-ONNX
-```
-
-### Poor Audio Quality
-
-- Try different voices
-- Adjust speed: `"speed": 0.9` to `1.2`
-- Check model downloaded completely
-- Allocate more memory
-
-### Slow Generation
-
-| Solution | How |
-|----------|-----|
-| Use GPU | Switch to `latest-cuda` image |
-| More CPU | Allocate more cores in Docker |
-| Faster model | Use smaller/quantized models |
-| SSD storage | Move Docker volumes to SSD |
-
----
-
-## Performance Tips
-
-### Recommended Specs
-
-| Component | Minimum | Recommended |
-|-----------|---------|-------------|
-| CPU | 2 cores | 4+ cores |
-| RAM | 2 GB | 4+ GB |
-| Storage | 5 GB | 10 GB (for multiple models) |
-| GPU | None | NVIDIA (optional) |
-
-### Resource Limits
-
-```yaml
-services:
-  speaches:
-    # ... other config
-    mem_limit: 4g
-    cpus: 2
-```
-
-### Monitor Usage
-
-```bash
-docker stats speaches
-```
-
----
-
-## Comparison: Local vs Cloud
-
-| Aspect | Local (Speaches) | Cloud (OpenAI/ElevenLabs) |
-|--------|------------------|---------------------------|
-| **Cost** | Free | $0.015-0.10/min |
-| **Privacy** | Complete | Data sent to provider |
-| **Speed** | Depends on hardware | Usually faster |
-| **Quality** | Good | Excellent |
-| **Setup** | Moderate | Simple API key |
-| **Offline** | Yes | No |
-| **Voices** | Limited | Many options |
-
-### When to Use Local
-
-- Privacy-sensitive content
-- High-volume generation
-- Development/testing
-- Offline environments
-- Cost control
-
-### When to Use Cloud
-
-- Premium quality needs
-- Multiple languages
-- Time-sensitive projects
-- Limited hardware
-
----
-
-## Other Local TTS Options
-
-Any OpenAI-compatible TTS server works. The key is:
-
-1. Server implements `/v1/audio/speech` endpoint
-2. Set `OPENAI_COMPATIBLE_BASE_URL_TTS` to server URL
-3. Add model with provider `openai_compatible`
-
----
-
-## Related
-
-- **[OpenAI-Compatible Providers](openai-compatible.md)** - General compatible provider setup
-- **[AI Providers](ai-providers.md)** - All provider configuration
-- **[Creating Podcasts](../3-USER-GUIDE/creating-podcasts.md)** - Using TTS for podcasts

docs/5-CONFIGURATION/mcp-integration.md

@@ -1,199 +0,0 @@
-# Model Context Protocol (MCP) Integration
-
-Open Notebook can be seamlessly integrated into your AI workflows using the **Model Context Protocol (MCP)**, enabling direct access to your notebooks, sources, and chat functionality from AI assistants like Claude Desktop and VS Code extensions.
-
-## What is MCP?
-
-The [Model Context Protocol](https://modelcontextprotocol.io) is an open standard that allows AI applications to securely connect to external data sources and tools. With the Open Notebook MCP server, you can:
-
-- 📚 **Access your notebooks** directly from Claude Desktop or VS Code
-- 🔍 **Search your research content** without leaving your AI assistant
-- 💬 **Create and manage chat sessions** with your research as context
-- 📝 **Generate notes** and insights on-the-fly
-- 🤖 **Automate workflows** using the full Open Notebook API
-
-## Quick Setup
-
-### For Claude Desktop
-
-1. **Install the MCP server** (automatically from PyPI):
-
-   ```bash
-   # No manual installation needed! Claude Desktop will use uvx to run it automatically
-   ```
-
-2. **Configure Claude Desktop**:
-
-   **macOS/Linux**: Edit `~/Library/Application Support/Claude/claude_desktop_config.json`
-
-   ```json
-   {
-     "mcpServers": {
-       "open-notebook": {
-         "command": "uvx",
-         "args": ["open-notebook-mcp"],
-         "env": {
-           "OPEN_NOTEBOOK_URL": "http://localhost:5055",
-           "OPEN_NOTEBOOK_PASSWORD": "your_password_here"
-         }
-       }
-     }
-   }
-   ```
-
-   **Windows**: Edit `%APPDATA%\Claude\claude_desktop_config.json`
-
-   ```json
-   {
-     "mcpServers": {
-       "open-notebook": {
-         "command": "uvx",
-         "args": ["open-notebook-mcp"],
-         "env": {
-           "OPEN_NOTEBOOK_URL": "http://localhost:5055",
-           "OPEN_NOTEBOOK_PASSWORD": "your_password_here"
-         }
-       }
-     }
-   }
-   ```
-
-3. **Restart Claude Desktop** and start using your notebooks in conversations!
-
-### For VS Code (Cline and other MCP-compatible extensions)
-
-Add to your VS Code settings or `.vscode/mcp.json`:
-
-```json
-{
-  "servers": {
-    "open-notebook": {
-      "command": "uvx",
-      "args": ["open-notebook-mcp"],
-      "env": {
-        "OPEN_NOTEBOOK_URL": "http://localhost:5055",
-        "OPEN_NOTEBOOK_PASSWORD": "your_password_here"
-      }
-    }
-  }
-}
-```
-
-## Configuration
-
-- **OPEN_NOTEBOOK_URL**: URL to your Open Notebook API (default: `http://localhost:5055`)
-- **OPEN_NOTEBOOK_PASSWORD**: Optional - only needed if you've enabled password protection
-
-### For Remote Servers
-
-If your Open Notebook instance is running on a remote server, update the URL accordingly:
-
-```json
-"OPEN_NOTEBOOK_URL": "http://192.168.1.100:5055"
-```
-
-Or with a domain:
-
-```json
-"OPEN_NOTEBOOK_URL": "https://notebook.yourdomain.com/api"
-```
-
-## What You Can Do
-
-Once connected, you can ask Claude or your AI assistant to:
-
-- _"Search my research notebooks for information about [topic]"_
-- _"Create a new note summarizing the key points from our conversation"_
-- _"List all my notebooks"_
-- _"Start a chat session about [specific source or topic]"_
-- _"What sources do I have in my [notebook name] notebook?"_
-- _"Add this PDF to my research notebook"_
-- _"Show me all notes in [notebook name]"_
-
-The MCP server provides full access to Open Notebook's capabilities, allowing you to manage your research seamlessly from within your AI assistant.
-
-## Available Tools
-
-The Open Notebook MCP server exposes these capabilities:
-
-### Notebooks
-
-- List notebooks
-- Get notebook details
-- Create new notebooks
-- Update notebook information
-- Delete notebooks
-
-### Sources
-
-- List sources in a notebook
-- Get source details
-- Add new sources (links, files, text)
-- Update source metadata
-- Delete sources
-
-### Notes
-
-- List notes in a notebook
-- Get note details
-- Create new notes
-- Update notes
-- Delete notes
-
-### Chat
-
-- Create chat sessions
-- Send messages to chat sessions
-- Get chat history
-- List chat sessions
-
-### Search
-
-- Vector search across content
-- Text search across content
-- Filter by notebook
-
-### Models
-
-- List configured AI models
-- Get model details
-- Create model configurations
-- Update model settings
-
-### Settings
-
-- Get application settings
-- Update settings
-
-## MCP Server Repository
-
-The Open Notebook MCP server is developed and maintained by the Epochal team:
-
-**🔗 GitHub**: [Epochal-dev/open-notebook-mcp](https://github.com/Epochal-dev/open-notebook-mcp)
-
-Contributions, issues, and feature requests are welcome!
-
-## Finding the Server
-
-The Open Notebook MCP server is published to the official MCP Registry:
-
-- **Registry**: Search for "open-notebook" at [registry.modelcontextprotocol.io](https://registry.modelcontextprotocol.io)
-- **PyPI**: [pypi.org/project/open-notebook-mcp](https://pypi.org/project/open-notebook-mcp)
-- **GitHub**: [Epochal-dev/open-notebook-mcp](https://github.com/Epochal-dev/open-notebook-mcp)
-
-## Troubleshooting
-
-### Connection Errors
-
-1. Verify the `OPEN_NOTEBOOK_URL` is correct and accessible
-2. If using password protection, ensure `OPEN_NOTEBOOK_PASSWORD` is set correctly
-3. For remote servers, make sure port 5055 is accessible from your machine
-4. Check firewall settings if connecting to a remote server
-
-## Using with Other MCP Clients
-
-The Open Notebook MCP server follows the standard MCP protocol and can be used with any MCP-compatible client. Check your client's documentation for configuration details.
-
-## Learn More
-
-- [Model Context Protocol Documentation](https://modelcontextprotocol.io)

docs/5-CONFIGURATION/openai-compatible.md

@@ -1,396 +0,0 @@
-# OpenAI-Compatible Providers
-
-Use any server that implements the OpenAI API format with Open Notebook. This includes LM Studio, Text Generation WebUI, vLLM, and many others.
-
----
-
-## What is OpenAI-Compatible?
-
-Many AI tools implement the same API format as OpenAI:
-
-```
-POST /v1/chat/completions
-POST /v1/embeddings
-POST /v1/audio/speech
-```
-
-Open Notebook can connect to any server using this format.
-
----
-
-## Common Compatible Servers
-
-| Server | Use Case | URL |
-|--------|----------|-----|
-| **LM Studio** | Desktop GUI for local models | https://lmstudio.ai |
-| **Text Generation WebUI** | Full-featured local inference | https://github.com/oobabooga/text-generation-webui |
-| **vLLM** | High-performance serving | https://github.com/vllm-project/vllm |
-| **Ollama** | Simple local models | (Use native Ollama provider instead) |
-| **LocalAI** | Local AI inference | https://github.com/mudler/LocalAI |
-| **llama.cpp server** | Lightweight inference | https://github.com/ggerganov/llama.cpp |
-
----
-
-## Quick Setup: LM Studio
-
-### Step 1: Install and Start LM Studio
-
-1. Download from https://lmstudio.ai
-2. Install and launch
-3. Download a model (e.g., Llama 3)
-4. Start the local server (default: port 1234)
-
-### Step 2: Configure Environment
-
-```bash
-# For language models
-export OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1
-export OPENAI_COMPATIBLE_API_KEY=not-needed  # LM Studio doesn't require key
-```
-
-### Step 3: Add Model in Open Notebook
-
-1. Go to **Settings** → **Models**
-2. Click **Add Model**
-3. Configure:
-   - **Provider**: `openai_compatible`
-   - **Model Name**: Your model name from LM Studio
-   - **Display Name**: `LM Studio - Llama 3`
-4. Click **Save**
-
----
-
-## Environment Variables
-
-### Language Models (Chat)
-
-```bash
-OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1
-OPENAI_COMPATIBLE_API_KEY=optional-api-key
-```
-
-### Embeddings
-
-```bash
-OPENAI_COMPATIBLE_BASE_URL_EMBEDDING=http://localhost:1234/v1
-OPENAI_COMPATIBLE_BASE_URL_EMBEDDING=optional-api-key
-```
-
-### Text-to-Speech
-
-```bash
-OPENAI_COMPATIBLE_BASE_URL_TTS=http://localhost:8969/v1
-OPENAI_COMPATIBLE_API_KEY_TTS=optional-api-key
-```
-
-### Speech-to-Text
-
-```bash
-OPENAI_COMPATIBLE_BASE_URL_STT=http://localhost:9000/v1
-OPENAI_COMPATIBLE_API_KEY_STT=optional-api-key
-```
-
----
-
-## Docker Networking
-
-When Open Notebook runs in Docker and your compatible server runs on the host:
-
-### macOS / Windows
-
-```bash
-OPENAI_COMPATIBLE_BASE_URL=http://host.docker.internal:1234/v1
-```
-
-### Linux
-
-```bash
-# Option 1: Docker bridge IP
-OPENAI_COMPATIBLE_BASE_URL=http://172.17.0.1:1234/v1
-
-# Option 2: Host networking mode
-docker run --network host ...
-```
-
-### Same Docker Network
-
-```yaml
-# docker-compose.yml
-services:
-  open-notebook:
-    # ...
-    environment:
-      - OPENAI_COMPATIBLE_BASE_URL=http://lm-studio:1234/v1
-
-  lm-studio:
-    # your LM Studio container
-    ports:
-      - "1234:1234"
-```
-
----
-
-## Text Generation WebUI Setup
-
-### Start with API Enabled
-
-```bash
-python server.py --api --listen
-```
-
-### Configure Open Notebook
-
-```bash
-OPENAI_COMPATIBLE_BASE_URL=http://localhost:5000/v1
-```
-
-### Docker Compose Example
-
-```yaml
-services:
-  text-gen:
-    image: atinoda/text-generation-webui:default
-    ports:
-      - "5000:5000"
-      - "7860:7860"
-    volumes:
-      - ./models:/app/models
-    command: --api --listen
-
-  open-notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    environment:
-      - OPENAI_COMPATIBLE_BASE_URL=http://text-gen:5000/v1
-    depends_on:
-      - text-gen
-```
-
----
-
-## vLLM Setup
-
-### Start vLLM Server
-
-```bash
-python -m vllm.entrypoints.openai.api_server \
-  --model meta-llama/Llama-3.1-8B-Instruct \
-  --port 8000
-```
-
-### Configure Open Notebook
-
-```bash
-OPENAI_COMPATIBLE_BASE_URL=http://localhost:8000/v1
-```
-
-### Docker Compose with GPU
-
-```yaml
-services:
-  vllm:
-    image: vllm/vllm-openai:latest
-    command: --model meta-llama/Llama-3.1-8B-Instruct
-    ports:
-      - "8000:8000"
-    volumes:
-      - ~/.cache/huggingface:/root/.cache/huggingface
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: 1
-              capabilities: [gpu]
-
-  open-notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    environment:
-      - OPENAI_COMPATIBLE_BASE_URL=http://vllm:8000/v1
-    depends_on:
-      - vllm
-```
-
----
-
-## Adding Models in Open Notebook
-
-### Via Settings UI
-
-1. Go to **Settings** → **Models**
-2. Click **Add Model** in appropriate section
-3. Select **Provider**: `openai_compatible`
-4. Enter **Model Name**: exactly as the server expects
-5. Enter **Display Name**: your preferred name
-6. Click **Save**
-
-### Model Name Format
-
-The model name must match what your server expects:
-
-| Server | Model Name Format |
-|--------|-------------------|
-| LM Studio | As shown in LM Studio UI |
-| vLLM | HuggingFace model path |
-| Text Gen WebUI | As loaded in UI |
-| llama.cpp | Model file name |
-
----
-
-## Testing Connection
-
-### Test API Endpoint
-
-```bash
-# Test chat completions
-curl http://localhost:1234/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "your-model-name",
-    "messages": [{"role": "user", "content": "Hello"}]
-  }'
-```
-
-### Test from Inside Docker
-
-```bash
-docker exec -it open-notebook curl http://host.docker.internal:1234/v1/models
-```
-
----
-
-## Troubleshooting
-
-### Connection Refused
-
-```
-Problem: Cannot connect to server
-
-Solutions:
-1. Verify server is running
-2. Check port is correct
-3. Test with curl directly
-4. Check Docker networking (use host.docker.internal)
-5. Verify firewall allows connection
-```
-
-### Model Not Found
-
-```
-Problem: Server returns "model not found"
-
-Solutions:
-1. Check model is loaded in server
-2. Verify exact model name spelling
-3. List available models: curl http://localhost:1234/v1/models
-4. Update model name in Open Notebook
-```
-
-### Slow Responses
-
-```
-Problem: Requests take very long
-
-Solutions:
-1. Check server resources (RAM, GPU)
-2. Use smaller/quantized model
-3. Reduce context length
-4. Enable GPU acceleration if available
-```
-
-### Authentication Errors
-
-```
-Problem: 401 or authentication failed
-
-Solutions:
-1. Check if server requires API key
-2. Set OPENAI_COMPATIBLE_API_KEY
-3. Some servers need any non-empty key
-```
-
-### Timeout Errors
-
-```
-Problem: Request times out
-
-Solutions:
-1. Model may be loading (first request slow)
-2. Increase timeout settings
-3. Check server logs for errors
-4. Reduce request size
-```
-
----
-
-## Multiple Compatible Endpoints
-
-You can use different compatible servers for different purposes:
-
-```bash
-# Chat model from LM Studio
-OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1
-
-# Embeddings from different server
-OPENAI_COMPATIBLE_BASE_URL_EMBEDDING=http://localhost:8080/v1
-
-# TTS from Speaches
-OPENAI_COMPATIBLE_BASE_URL_TTS=http://localhost:8969/v1
-```
-
-Add each as a separate model in Open Notebook settings.
-
----
-
-## Performance Tips
-
-### Model Selection
-
-| Model Size | RAM Needed | Speed |
-|------------|------------|-------|
-| 7B | 8GB | Fast |
-| 13B | 16GB | Medium |
-| 70B | 64GB+ | Slow |
-
-### Quantization
-
-Use quantized models (Q4, Q5) for faster inference with less RAM:
-
-```
-llama-3-8b-q4_k_m.gguf  → ~4GB RAM, fast
-llama-3-8b-f16.gguf     → ~16GB RAM, slower
-```
-
-### GPU Acceleration
-
-Enable GPU in your server for much faster inference:
-- LM Studio: Settings → GPU layers
-- vLLM: Automatic with CUDA
-- llama.cpp: `--n-gpu-layers 35`
-
----
-
-## Comparison: Native vs Compatible
-
-| Aspect | Native Provider | OpenAI Compatible |
-|--------|-----------------|-------------------|
-| **Setup** | API key only | Server + configuration |
-| **Models** | Provider's models | Any compatible model |
-| **Cost** | Pay per token | Free (local) |
-| **Speed** | Usually fast | Depends on hardware |
-| **Features** | Full support | Basic features |
-
-Use OpenAI-compatible when:
-- Running local models
-- Using custom/fine-tuned models
-- Privacy requirements
-- Cost control
-
----
-
-## Related
-
-- **[Local TTS Setup](local-tts.md)** - Text-to-speech with Speaches
-- **[AI Providers](ai-providers.md)** - All provider options
-- **[Ollama Setup](ollama.md)** - Native Ollama integration

docs/5-CONFIGURATION/reverse-proxy.md

@@ -1,828 +0,0 @@
-# Reverse Proxy Configuration
-
-Deploy Open Notebook behind nginx, Caddy, Traefik, or other reverse proxies with custom domains and HTTPS.
-
----
-
-## Simplified Setup (v1.1+)
-
-Starting with v1.1, Open Notebook uses Next.js rewrites to simplify configuration. **You only need to proxy to one port** - Next.js handles internal API routing automatically.
-
-### How It Works
-
-```
-Browser → Reverse Proxy → Port 8502 (Next.js)
-                             ↓ (internal proxy)
-                          Port 5055 (FastAPI)
-```
-
-Next.js automatically forwards `/api/*` requests to the FastAPI backend, so your reverse proxy only needs one port!
-
----
-
-## Quick Configuration Examples
-
-### Nginx (Recommended)
-
-```nginx
-server {
-    listen 443 ssl http2;
-    server_name notebook.example.com;
-
-    ssl_certificate /etc/nginx/ssl/fullchain.pem;
-    ssl_certificate_key /etc/nginx/ssl/privkey.pem;
-
-    # Allow file uploads up to 100MB
-    client_max_body_size 100M;
-
-    # Single location block - that's it!
-    location / {
-        proxy_pass http://open-notebook:8502;
-        proxy_http_version 1.1;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection 'upgrade';
-        proxy_cache_bypass $http_upgrade;
-    }
-}
-
-# HTTP to HTTPS redirect
-server {
-    listen 80;
-    server_name notebook.example.com;
-    return 301 https://$server_name$request_uri;
-}
-```
-
-### Caddy
-
-```caddy
-notebook.example.com {
-    reverse_proxy open-notebook:8502
-}
-```
-
-That's it! Caddy handles HTTPS automatically.
-
-### Traefik
-
-```yaml
-services:
-  open-notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    environment:
-      - API_URL=https://notebook.example.com
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.notebook.rule=Host(`notebook.example.com`)"
-      - "traefik.http.routers.notebook.entrypoints=websecure"
-      - "traefik.http.routers.notebook.tls.certresolver=myresolver"
-      - "traefik.http.services.notebook.loadbalancer.server.port=8502"
-    networks:
-      - traefik-network
-```
-
-### Coolify
-
-1. Create new service with `lfnovo/open_notebook:v1-latest-single`
-2. Set port to **8502**
-3. Add environment: `API_URL=https://your-domain.com`
-4. Enable HTTPS in Coolify
-5. Done!
-
----
-
-## Environment Variables
-
-```bash
-# Required for reverse proxy setups
-API_URL=https://your-domain.com
-
-# Optional: For multi-container deployments
-# INTERNAL_API_URL=http://api-service:5055
-```
-
-**Important**: Set `API_URL` to your public URL (with https://).
-
----
-
-## Understanding API_URL
-
-The frontend uses a three-tier priority system to determine the API URL:
-
-1. **Runtime Configuration** (Highest Priority): `API_URL` environment variable set at container runtime
-2. **Build-time Configuration**: `NEXT_PUBLIC_API_URL` baked into the Docker image
-3. **Auto-detection** (Fallback): Infers from the incoming HTTP request headers
-
-### Auto-Detection Details
-
-When `API_URL` is not set, the Next.js frontend:
-- Analyzes the incoming HTTP request
-- Extracts the hostname from the `host` header
-- Respects the `X-Forwarded-Proto` header (for HTTPS behind reverse proxies)
-- Constructs the API URL as `{protocol}://{hostname}:5055`
-- Example: Request to `http://10.20.30.20:8502` → API URL becomes `http://10.20.30.20:5055`
-
-**Why set API_URL explicitly?**
-- **Reliability**: Auto-detection can fail with complex proxy setups
-- **HTTPS**: Ensures frontend uses `https://` when behind SSL-terminating proxy
-- **Custom domains**: Works correctly with domain names instead of IP addresses
-- **Port mapping**: Avoids exposing port 5055 in the URL when using reverse proxy
-
-**Important**: Don't include `/api` at the end - the system adds this automatically!
-
----
-
-## Complete Docker Compose Example
-
-```yaml
-services:
-  open-notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    container_name: open-notebook
-    environment:
-      - API_URL=https://notebook.example.com
-      - OPENAI_API_KEY=${OPENAI_API_KEY}
-      - OPEN_NOTEBOOK_PASSWORD=${OPEN_NOTEBOOK_PASSWORD}
-    volumes:
-      - ./notebook_data:/app/data
-      - ./surreal_data:/mydata
-    # Only expose to localhost (nginx handles public access)
-    ports:
-      - "127.0.0.1:8502:8502"
-    restart: unless-stopped
-
-  nginx:
-    image: nginx:alpine
-    container_name: nginx-proxy
-    ports:
-      - "80:80"
-      - "443:443"
-    volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf:ro
-      - ./ssl:/etc/nginx/ssl:ro
-    depends_on:
-      - open-notebook
-    restart: unless-stopped
-```
-
----
-
-## Full Nginx Configuration
-
-```nginx
-events {
-    worker_connections 1024;
-}
-
-http {
-    upstream notebook {
-        server open-notebook:8502;
-    }
-
-    # HTTP redirect
-    server {
-        listen 80;
-        server_name notebook.example.com;
-        return 301 https://$server_name$request_uri;
-    }
-
-    # HTTPS server
-    server {
-        listen 443 ssl http2;
-        server_name notebook.example.com;
-
-        ssl_certificate /etc/nginx/ssl/fullchain.pem;
-        ssl_certificate_key /etc/nginx/ssl/privkey.pem;
-        ssl_protocols TLSv1.2 TLSv1.3;
-        ssl_ciphers HIGH:!aNULL:!MD5;
-
-        # Allow file uploads up to 100MB
-        client_max_body_size 100M;
-
-        # Security headers
-        add_header X-Frame-Options DENY;
-        add_header X-Content-Type-Options nosniff;
-        add_header X-XSS-Protection "1; mode=block";
-        add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
-
-        # Proxy settings
-        location / {
-            proxy_pass http://notebook;
-            proxy_http_version 1.1;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-            proxy_set_header Upgrade $http_upgrade;
-            proxy_set_header Connection 'upgrade';
-            proxy_cache_bypass $http_upgrade;
-
-            # Timeouts for long-running operations (podcasts, etc.)
-            proxy_read_timeout 300s;
-            proxy_connect_timeout 60s;
-            proxy_send_timeout 300s;
-        }
-    }
-}
-```
-
----
-
-## Direct API Access (Optional)
-
-If external scripts or integrations need direct API access, route `/api/*` directly:
-
-```nginx
-# Direct API access (for external integrations)
-location /api/ {
-    proxy_pass http://open-notebook:5055/api/;
-    proxy_http_version 1.1;
-    proxy_set_header Host $host;
-    proxy_set_header X-Real-IP $remote_addr;
-    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    proxy_set_header X-Forwarded-Proto $scheme;
-}
-
-# Frontend (handles all other traffic)
-location / {
-    proxy_pass http://open-notebook:8502;
-    # ... same headers as above
-}
-```
-
-**Note**: This is only needed for external API integrations. Browser traffic works fine with single-port setup.
-
----
-
-## Advanced Scenarios
-
-### Remote Server Access (LAN/VPS)
-
-Accessing Open Notebook from a different machine on your network:
-
-**Step 1: Get your server IP**
-```bash
-# On the server running Open Notebook:
-hostname -I
-# or
-ifconfig | grep "inet "
-# Note the IP (e.g., 192.168.1.100)
-```
-
-**Step 2: Configure API_URL**
-```bash
-# In docker-compose.yml or .env:
-API_URL=http://192.168.1.100:5055
-```
-
-**Step 3: Expose ports**
-```yaml
-services:
-  open-notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    environment:
-      - API_URL=http://192.168.1.100:5055
-    ports:
-      - "8502:8502"
-      - "5055:5055"
-```
-
-**Step 4: Access from client machine**
-```bash
-# In browser on other machine:
-http://192.168.1.100:8502
-```
-
-**Troubleshooting**:
-- Check firewall: `sudo ufw allow 8502 && sudo ufw allow 5055`
-- Verify connectivity: `ping 192.168.1.100` from client machine
-- Test port: `telnet 192.168.1.100 8502` from client machine
-
----
-
-### API on Separate Subdomain
-
-Host the API and frontend on different subdomains:
-
-**docker-compose.yml:**
-```yaml
-services:
-  open-notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    environment:
-      - API_URL=https://api.notebook.example.com
-      - OPENAI_API_KEY=${OPENAI_API_KEY}
-    # Don't expose ports (nginx handles routing)
-```
-
-**nginx.conf:**
-```nginx
-# Frontend server
-server {
-    listen 443 ssl http2;
-    server_name notebook.example.com;
-
-    ssl_certificate /etc/nginx/ssl/fullchain.pem;
-    ssl_certificate_key /etc/nginx/ssl/privkey.pem;
-
-    location / {
-        proxy_pass http://open-notebook:8502;
-        proxy_http_version 1.1;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection 'upgrade';
-        proxy_cache_bypass $http_upgrade;
-    }
-}
-
-# API server (separate subdomain)
-server {
-    listen 443 ssl http2;
-    server_name api.notebook.example.com;
-
-    ssl_certificate /etc/nginx/ssl/fullchain.pem;
-    ssl_certificate_key /etc/nginx/ssl/privkey.pem;
-
-    location / {
-        proxy_pass http://open-notebook:5055;
-        proxy_http_version 1.1;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-    }
-}
-```
-
-**Use case**: Separate DNS records, different rate limiting, or isolated API access control.
-
----
-
-### Multi-Container Deployment (Advanced)
-
-For complex deployments with separate frontend and API containers:
-
-**docker-compose.yml:**
-```yaml
-services:
-  frontend:
-    image: lfnovo/open_notebook_frontend:v1-latest
-    pull_policy: always
-    environment:
-      - API_URL=https://notebook.example.com
-    ports:
-      - "8502:8502"
-
-  api:
-    image: lfnovo/open_notebook_api:v1-latest
-    pull_policy: always
-    environment:
-      - OPENAI_API_KEY=${OPENAI_API_KEY}
-    ports:
-      - "5055:5055"
-    depends_on:
-      - surrealdb
-
-  surrealdb:
-    image: surrealdb/surrealdb:latest
-    command: start --log trace --user root --pass root file:/mydata/database.db
-    ports:
-      - "8000:8000"
-    volumes:
-      - ./surreal_data:/mydata
-```
-
-**nginx.conf:**
-```nginx
-http {
-    upstream frontend {
-        server frontend:8502;
-    }
-
-    upstream api {
-        server api:5055;
-    }
-
-    server {
-        listen 443 ssl http2;
-        server_name notebook.example.com;
-
-        # API routes
-        location /api/ {
-            proxy_pass http://api/api/;
-            proxy_http_version 1.1;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-        }
-
-        # Frontend (catch-all)
-        location / {
-            proxy_pass http://frontend;
-            proxy_http_version 1.1;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-            proxy_set_header Upgrade $http_upgrade;
-            proxy_set_header Connection 'upgrade';
-            proxy_cache_bypass $http_upgrade;
-        }
-    }
-}
-```
-
-**Note**: Most users should use the single-container approach (`v1-latest-single`). Multi-container is only needed for custom scaling or isolation requirements.
-
----
-
-## SSL Certificates
-
-### Let's Encrypt with Certbot
-
-```bash
-# Install certbot
-sudo apt install certbot python3-certbot-nginx
-
-# Get certificate
-sudo certbot --nginx -d notebook.example.com
-
-# Auto-renewal (usually configured automatically)
-sudo certbot renew --dry-run
-```
-
-### Let's Encrypt with Caddy
-
-Caddy handles SSL automatically - no configuration needed!
-
-### Self-Signed (Development Only)
-
-```bash
-openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
-  -keyout ssl/privkey.pem \
-  -out ssl/fullchain.pem \
-  -subj "/CN=localhost"
-```
-
----
-
-## Troubleshooting
-
-### "Unable to connect to server"
-
-1. **Check API_URL is set**:
-   ```bash
-   docker exec open-notebook env | grep API_URL
-   ```
-
-2. **Verify reverse proxy reaches container**:
-   ```bash
-   curl -I http://localhost:8502
-   ```
-
-3. **Check browser console** (F12):
-   - Look for connection errors
-   - Check what URL it's trying to reach
-
-### Mixed Content Errors
-
-Frontend using HTTPS but trying to reach HTTP API:
-
-```bash
-# Ensure API_URL uses https://
-API_URL=https://notebook.example.com  # Not http://
-```
-
-### WebSocket Issues
-
-Ensure your proxy supports WebSocket upgrades:
-
-```nginx
-proxy_http_version 1.1;
-proxy_set_header Upgrade $http_upgrade;
-proxy_set_header Connection 'upgrade';
-```
-
-### 502 Bad Gateway
-
-1. Check container is running: `docker ps`
-2. Check container logs: `docker logs open-notebook`
-3. Verify nginx can reach container (same network)
-
-### Timeout Errors
-
-Increase timeouts for long operations (podcast generation):
-
-```nginx
-proxy_read_timeout 300s;
-proxy_send_timeout 300s;
-```
-
----
-
-### How to Debug Configuration Issues
-
-**Step 1: Check browser console** (F12 → Console tab)
-```
-Look for messages starting with 🔧 [Config]
-These show the configuration detection process
-You'll see which API URL is being used
-```
-
-**Example good output:**
-```
-✅ [Config] Runtime API URL from server: https://your-domain.com
-```
-
-**Example bad output:**
-```
-❌ [Config] Failed to fetch runtime config
-⚠️  [Config] Using auto-detected URL: http://localhost:5055
-```
-
-**Step 2: Test API directly**
-```bash
-# Should return JSON config
-curl https://your-domain.com/api/config
-
-# Expected output:
-{"openai_api_key_set":true,"anthropic_api_key_set":false,...}
-```
-
-**Step 3: Check Docker logs**
-```bash
-docker logs open-notebook
-
-# Look for:
-# - Frontend startup: "▲ Next.js ready on http://0.0.0.0:8502"
-# - API startup: "INFO:     Uvicorn running on http://0.0.0.0:5055"
-# - Connection errors or CORS issues
-```
-
-**Step 4: Verify environment variable**
-```bash
-docker exec open-notebook env | grep API_URL
-
-# Should show:
-# API_URL=https://your-domain.com
-```
-
----
-
-### Frontend Adds `:5055` to URL (Versions ≤ 1.0.10)
-
-**Symptoms** (only in older versions):
-- You set `API_URL=https://your-domain.com`
-- Browser console shows: "Attempted URL: https://your-domain.com:5055/api/config"
-- CORS errors with "Status code: (null)"
-
-**Root Cause:**
-In versions ≤ 1.0.10, the frontend's config endpoint was at `/api/runtime-config`, which got intercepted by reverse proxies routing all `/api/*` requests to the backend. This prevented the frontend from reading the `API_URL` environment variable.
-
-**Solution:**
-Upgrade to version 1.0.11 or later. The config endpoint has been moved to `/config` which avoids the `/api/*` routing conflict.
-
-**Verification:**
-Check browser console (F12) - should see: `✅ [Config] Runtime API URL from server: https://your-domain.com`
-
-**If you can't upgrade**, explicitly configure the `/config` route:
-```nginx
-# Only needed for versions ≤ 1.0.10
-location = /config {
-    proxy_pass http://open-notebook:8502;
-    proxy_http_version 1.1;
-    proxy_set_header Host $host;
-    proxy_set_header X-Forwarded-Proto $scheme;
-}
-```
-
----
-
-### File Upload Errors (413 Payload Too Large)
-
-**Symptoms:**
-```
-CORS header 'Access-Control-Allow-Origin' missing. Status code: 413.
-Error creating source. Please try again.
-```
-
-**Root Cause:**
-When uploading files, your reverse proxy may reject the request due to body size limits *before* it reaches the application. Since the error happens at the proxy level, CORS headers are not included in the response.
-
-**Solutions:**
-
-1. **Nginx - Increase body size limit**:
-   ```nginx
-   server {
-       # Allow larger file uploads (default is 1MB)
-       client_max_body_size 100M;
-
-       # Add CORS headers to error responses
-       error_page 413 = @cors_error_413;
-
-       location @cors_error_413 {
-           add_header 'Access-Control-Allow-Origin' '*' always;
-           add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
-           add_header 'Access-Control-Allow-Headers' '*' always;
-           return 413 '{"detail": "File too large. Maximum size is 100MB."}';
-       }
-
-       location / {
-           # ... your existing proxy configuration
-       }
-   }
-   ```
-
-2. **Traefik - Increase buffer size**:
-   ```yaml
-   # In your traefik configuration
-   http:
-     middlewares:
-       large-body:
-         buffering:
-           maxRequestBodyBytes: 104857600  # 100MB
-   ```
-
-   Apply middleware to your router:
-   ```yaml
-   labels:
-     - "traefik.http.routers.notebook.middlewares=large-body"
-   ```
-
-3. **Kubernetes Ingress (nginx-ingress)**:
-   ```yaml
-   apiVersion: networking.k8s.io/v1
-   kind: Ingress
-   metadata:
-     name: open-notebook
-     annotations:
-       nginx.ingress.kubernetes.io/proxy-body-size: "100m"
-       # Add CORS headers for error responses
-       nginx.ingress.kubernetes.io/configuration-snippet: |
-         more_set_headers "Access-Control-Allow-Origin: *";
-   ```
-
-4. **Caddy**:
-   ```caddy
-   notebook.example.com {
-       request_body {
-           max_size 100MB
-       }
-       reverse_proxy open-notebook:8502
-   }
-   ```
-
-**Note:** Open Notebook's API includes CORS headers in error responses, but this only works for errors that reach the application. Proxy-level errors (like 413 from nginx) need to be configured at the proxy level.
-
----
-
-### CORS Errors
-
-**Symptoms:**
-```
-Access-Control-Allow-Origin header is missing
-Cross-Origin Request Blocked
-Response to preflight request doesn't pass access control check
-```
-
-**Possible Causes:**
-
-1. **Missing proxy headers**:
-   ```nginx
-   # Make sure these are set:
-   proxy_set_header X-Forwarded-Proto $scheme;
-   proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-   proxy_set_header Host $host;
-   ```
-
-2. **API_URL protocol mismatch**:
-   ```bash
-   # Frontend is HTTPS, but API_URL is HTTP:
-   API_URL=http://notebook.example.com  # ❌ Wrong
-   API_URL=https://notebook.example.com # ✅ Correct
-   ```
-
-3. **Reverse proxy not forwarding `/api/*` correctly**:
-   ```nginx
-   # Make sure this works:
-   location /api/ {
-       proxy_pass http://open-notebook:5055/api/;  # Note the trailing slash!
-   }
-   ```
-
----
-
-### Missing Authorization Header
-
-**Symptoms:**
-```json
-{"detail": "Missing authorization header"}
-```
-
-This happens when:
-- You have set `OPEN_NOTEBOOK_PASSWORD` for authentication
-- You're trying to access `/api/config` directly without logging in first
-
-**Solution:**
-This is **expected behavior**! The frontend handles authentication automatically. Just:
-1. Access the frontend URL (not `/api/` directly)
-2. Log in through the UI
-3. The frontend will handle authorization headers for all API calls
-
-**For API integrations:** Include the password in the Authorization header:
-```bash
-curl -H "Authorization: Bearer your-password-here" \
-  https://your-domain.com/api/config
-```
-
----
-
-### SSL/TLS Certificate Errors
-
-**Symptoms:**
-- Browser shows "Your connection is not private"
-- Certificate warnings
-- Mixed content errors
-
-**Solutions:**
-
-1. **Use Let's Encrypt** (recommended):
-   ```bash
-   sudo certbot --nginx -d notebook.example.com
-   ```
-
-2. **Check certificate paths** in nginx:
-   ```nginx
-   ssl_certificate /etc/nginx/ssl/fullchain.pem;      # Full chain
-   ssl_certificate_key /etc/nginx/ssl/privkey.pem;    # Private key
-   ```
-
-3. **Verify certificate is valid**:
-   ```bash
-   openssl x509 -in /etc/nginx/ssl/fullchain.pem -text -noout
-   ```
-
-4. **For development**, use self-signed (not for production):
-   ```bash
-   openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
-     -keyout ssl/privkey.pem -out ssl/fullchain.pem \
-     -subj "/CN=localhost"
-   ```
-
----
-
-## Best Practices
-
-1. **Always use HTTPS** in production
-2. **Set API_URL explicitly** when using reverse proxies to avoid auto-detection issues
-3. **Bind to localhost** (`127.0.0.1:8502`) and let proxy handle public access for security
-4. **Enable security headers** (HSTS, X-Frame-Options, X-Content-Type-Options, X-XSS-Protection)
-5. **Set up certificate renewal** for Let's Encrypt (usually automatic with certbot)
-6. **Keep ports 5055 and 8502 accessible** from your reverse proxy container (use Docker networks)
-7. **Use environment files** (`.env` or `docker.env`) to manage configuration securely
-8. **Test your configuration** before going live:
-   - Check browser console for config messages
-   - Test API: `curl https://your-domain.com/api/config`
-   - Verify authentication works
-   - Check long-running operations (podcast generation)
-9. **Monitor logs** regularly: `docker logs open-notebook`
-10. **Don't include `/api` in API_URL** - the system adds this automatically
-
----
-
-## Legacy Configurations (Pre-v1.1)
-
-If you're running Open Notebook **version 1.0.x or earlier**, you may need to use the legacy two-port configuration where you explicitly route `/api/*` to port 5055.
-
-**Check your version:**
-```bash
-docker exec open-notebook cat /app/package.json | grep version
-```
-
-**If version < 1.1.0**, you may need:
-- Explicit `/api/*` routing to port 5055 in reverse proxy
-- Explicit `/config` endpoint routing for versions ≤ 1.0.10
-- See the "Frontend Adds `:5055` to URL" troubleshooting section above
-
-**Recommendation:** Upgrade to v1.1+ for simplified configuration and better performance.
-
----
-
-## Related
-
-- **[Security Configuration](security.md)** - Password protection and hardening
-- **[Advanced Configuration](advanced.md)** - Ports, timeouts, and SSL settings
-- **[Troubleshooting](../6-TROUBLESHOOTING/connection-issues.md)** - Connection problems
-- **[Docker Deployment](../1-INSTALLATION/docker-compose.md)** - Complete deployment guide

docs/5-CONFIGURATION/security.md

@@ -1,335 +0,0 @@
-# Security Configuration
-
-Protect your Open Notebook deployment with password authentication and production hardening.
-
----
-
-## When to Use Password Protection
-
-### Use it for:
-- Public cloud deployments (PikaPods, Railway, DigitalOcean)
-- Shared network environments
-- Any deployment accessible beyond localhost
-
-### You can skip it for:
-- Local development on your machine
-- Private, isolated networks
-- Single-user local setups
-
----
-
-## Quick Setup
-
-### Docker Deployment
-
-```yaml
-# docker-compose.yml
-services:
-  open_notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    environment:
-      - OPENAI_API_KEY=sk-...
-      - OPEN_NOTEBOOK_PASSWORD=your_secure_password
-    # ... rest of config
-```
-
-Or using environment file:
-
-```bash
-# docker.env
-OPENAI_API_KEY=sk-...
-OPEN_NOTEBOOK_PASSWORD=your_secure_password
-```
-
-### Development Setup
-
-```bash
-# .env
-OPEN_NOTEBOOK_PASSWORD=your_secure_password
-```
-
----
-
-## Password Requirements
-
-### Good Passwords
-
-```bash
-# Strong: 20+ characters, mixed case, numbers, symbols
-OPEN_NOTEBOOK_PASSWORD=MySecure2024!Research#Tool
-OPEN_NOTEBOOK_PASSWORD=Notebook$Dev$2024$Strong!
-
-# Generated (recommended)
-OPEN_NOTEBOOK_PASSWORD=$(openssl rand -base64 24)
-```
-
-### Bad Passwords
-
-```bash
-# DON'T use these
-OPEN_NOTEBOOK_PASSWORD=password123
-OPEN_NOTEBOOK_PASSWORD=opennotebook
-OPEN_NOTEBOOK_PASSWORD=admin
-```
-
----
-
-## How It Works
-
-### Frontend Protection
-
-1. Login form appears on first visit
-2. Password stored in browser session
-3. Session persists until browser closes
-4. Clear browser data to log out
-
-### API Protection
-
-All API endpoints require authentication:
-
-```bash
-# Authenticated request
-curl -H "Authorization: Bearer your_password" \
-  http://localhost:5055/api/notebooks
-
-# Unauthenticated (will fail)
-curl http://localhost:5055/api/notebooks
-# Returns: {"detail": "Missing authorization header"}
-```
-
-### Unprotected Endpoints
-
-These work without authentication:
-
-- `/health` - System health check
-- `/docs` - API documentation
-- `/openapi.json` - OpenAPI spec
-
----
-
-## API Authentication Examples
-
-### curl
-
-```bash
-# List notebooks
-curl -H "Authorization: Bearer your_password" \
-  http://localhost:5055/api/notebooks
-
-# Create notebook
-curl -X POST \
-  -H "Authorization: Bearer your_password" \
-  -H "Content-Type: application/json" \
-  -d '{"name": "My Notebook", "description": "Research notes"}' \
-  http://localhost:5055/api/notebooks
-
-# Upload file
-curl -X POST \
-  -H "Authorization: Bearer your_password" \
-  -F "file=@document.pdf" \
-  http://localhost:5055/api/sources/upload
-```
-
-### Python
-
-```python
-import requests
-
-class OpenNotebookClient:
-    def __init__(self, base_url: str, password: str):
-        self.base_url = base_url
-        self.headers = {"Authorization": f"Bearer {password}"}
-
-    def get_notebooks(self):
-        response = requests.get(
-            f"{self.base_url}/api/notebooks",
-            headers=self.headers
-        )
-        return response.json()
-
-    def create_notebook(self, name: str, description: str = None):
-        response = requests.post(
-            f"{self.base_url}/api/notebooks",
-            headers=self.headers,
-            json={"name": name, "description": description}
-        )
-        return response.json()
-
-# Usage
-client = OpenNotebookClient("http://localhost:5055", "your_password")
-notebooks = client.get_notebooks()
-```
-
-### JavaScript/TypeScript
-
-```javascript
-const API_URL = 'http://localhost:5055';
-const PASSWORD = 'your_password';
-
-async function getNotebooks() {
-  const response = await fetch(`${API_URL}/api/notebooks`, {
-    headers: {
-      'Authorization': `Bearer ${PASSWORD}`
-    }
-  });
-  return response.json();
-}
-```
-
----
-
-## Production Hardening
-
-### Docker Security
-
-```yaml
-services:
-  open_notebook:
-    image: lfnovo/open_notebook:v1-latest-single
-    pull_policy: always
-    ports:
-      - "127.0.0.1:8502:8502"  # Bind to localhost only
-    environment:
-      - OPEN_NOTEBOOK_PASSWORD=your_secure_password
-    security_opt:
-      - no-new-privileges:true
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-          cpus: "1.0"
-    restart: always
-```
-
-### Firewall Configuration
-
-```bash
-# UFW (Ubuntu)
-sudo ufw allow ssh
-sudo ufw allow 80/tcp
-sudo ufw allow 443/tcp
-sudo ufw deny 8502/tcp   # Block direct access
-sudo ufw deny 5055/tcp   # Block direct API access
-sudo ufw enable
-
-# iptables
-iptables -A INPUT -p tcp --dport 22 -j ACCEPT
-iptables -A INPUT -p tcp --dport 80 -j ACCEPT
-iptables -A INPUT -p tcp --dport 443 -j ACCEPT
-iptables -A INPUT -p tcp --dport 8502 -j DROP
-iptables -A INPUT -p tcp --dport 5055 -j DROP
-```
-
-### Reverse Proxy with SSL
-
-See [Reverse Proxy Configuration](reverse-proxy.md) for complete nginx/Caddy/Traefik setup with HTTPS.
-
----
-
-## Security Limitations
-
-Open Notebook's password protection provides **basic access control**, not enterprise-grade security:
-
-| Feature | Status |
-|---------|--------|
-| Password transmission | Plain text (use HTTPS!) |
-| Password storage | In memory |
-| User management | Single password for all |
-| Session timeout | None (until browser close) |
-| Rate limiting | None |
-| Audit logging | None |
-
-### Risk Mitigation
-
-1. **Always use HTTPS** - Encrypt traffic with TLS
-2. **Strong passwords** - 20+ characters, complex
-3. **Network security** - Firewall, VPN for sensitive deployments
-4. **Regular updates** - Keep containers and dependencies updated
-5. **Monitoring** - Check logs for suspicious activity
-6. **Backups** - Regular backups of data
-
----
-
-## Enterprise Considerations
-
-For deployments requiring advanced security:
-
-| Need | Solution |
-|------|----------|
-| SSO/OAuth | Implement OAuth2/SAML proxy |
-| Role-based access | Custom middleware |
-| Audit logging | Log aggregation service |
-| Rate limiting | API gateway or nginx |
-| Data encryption | Encrypt volumes at rest |
-| Network segmentation | Docker networks, VPC |
-
----
-
-## Troubleshooting
-
-### Password Not Working
-
-```bash
-# Check env var is set
-docker exec open-notebook env | grep OPEN_NOTEBOOK_PASSWORD
-
-# Check logs
-docker logs open-notebook | grep -i auth
-
-# Test API directly
-curl -H "Authorization: Bearer your_password" \
-  http://localhost:5055/health
-```
-
-### 401 Unauthorized Errors
-
-```bash
-# Check header format
-curl -v -H "Authorization: Bearer your_password" \
-  http://localhost:5055/api/notebooks
-
-# Verify password matches
-echo "Password length: $(echo -n $OPEN_NOTEBOOK_PASSWORD | wc -c)"
-```
-
-### Cannot Access After Setting Password
-
-1. Clear browser cache and cookies
-2. Try incognito/private mode
-3. Check browser console for errors
-4. Verify password is correct in environment
-
-### Security Testing
-
-```bash
-# Without password (should fail)
-curl http://localhost:5055/api/notebooks
-# Expected: {"detail": "Missing authorization header"}
-
-# With correct password (should succeed)
-curl -H "Authorization: Bearer your_password" \
-  http://localhost:5055/api/notebooks
-
-# Health check (should work without password)
-curl http://localhost:5055/health
-```
-
----
-
-## Reporting Security Issues
-
-If you discover security vulnerabilities:
-
-1. **Do NOT open public issues**
-2. Contact maintainers directly
-3. Provide detailed information
-4. Allow time for fixes before disclosure
-
----
-
-## Related
-
-- **[Reverse Proxy](reverse-proxy.md)** - HTTPS and SSL setup
-- **[Advanced Configuration](advanced.md)** - Ports, timeouts, and SSL settings
-- **[Environment Reference](environment-reference.md)** - All configuration options

docs/6-TROUBLESHOOTING/ai-chat-issues.md

@@ -1,400 +0,0 @@
-# AI & Chat Issues - Model Configuration & Quality
-
-Problems with AI models, chat, and response quality.
-
----
-
-## "Models not available" or "Models not showing"
-
-**Symptom:** Settings → Models shows empty, or "No models configured"
-
-**Cause:** Missing or invalid API key
-
-**Solutions:**
-
-### Solution 1: Add API Key
-```bash
-# Check .env has your API key:
-cat .env | grep -i "OPENAI\|ANTHROPIC\|GOOGLE"
-
-# Should see something like:
-# OPENAI_API_KEY=sk-proj-...
-
-# If missing, add it:
-OPENAI_API_KEY=sk-proj-your-key-here
-
-# Save and restart:
-docker compose restart api
-
-# Wait 10 seconds, then refresh browser
-```
-
-### Solution 2: Check Key is Valid
-```bash
-# Test API key directly:
-curl https://api.openai.com/v1/models \
-  -H "Authorization: Bearer sk-proj-..."
-
-# Should return list of models
-# If error: key is invalid
-```
-
-### Solution 3: Switch Provider
-```bash
-# Try a different provider:
-# Remove: OPENAI_API_KEY
-# Add: ANTHROPIC_API_KEY=sk-ant-...
-
-# Restart and check Settings → Models
-```
-
----
-
-## "Invalid API key" or "Unauthorized"
-
-**Symptom:** Error when trying to chat: "Invalid API key"
-
-**Cause:** API key wrong, expired, or revoked
-
-**Solutions:**
-
-### Step 1: Verify Key Format
-```bash
-# OpenAI: Should start with sk-proj-
-# Anthropic: Should start with sk-ant-
-# Google: Should be AIzaSy...
-
-# Check in .env:
-cat .env | grep OPENAI_API_KEY
-```
-
-### Step 2: Get Fresh Key
-```bash
-# Go to provider's dashboard:
-# - OpenAI: https://platform.openai.com/api-keys
-# - Anthropic: https://console.anthropic.com/
-# - Google: https://aistudio.google.com/app/apikey
-
-# Generate new key
-# Copy exactly (no extra spaces)
-```
-
-### Step 3: Update .env
-```bash
-# Edit .env:
-OPENAI_API_KEY=sk-proj-new-key-here
-# No quotes needed, no spaces
-
-# Save and restart:
-docker compose restart api
-```
-
-### Step 4: Verify in UI
-```
-1. Open Open Notebook
-2. Go to Settings → Models
-3. Select your provider
-4. Should show available models
-```
-
----
-
-## Chat Returns Generic/Bad Responses
-
-**Symptom:** AI responses are shallow, generic, or wrong
-
-**Cause:** Bad context, vague question, or wrong model
-
-**Solutions:**
-
-### Solution 1: Check Context
-```
-1. In Chat, click "Select Sources"
-2. Verify sources you want are CHECKED
-3. Set them to "Full Content" (not "Summary Only")
-4. Click "Save"
-5. Try chat again
-```
-
-### Solution 2: Ask Better Question
-```
-Bad:     "What do you think?"
-Good:    "Based on the paper's methodology, what are 3 limitations?"
-
-Bad:     "Tell me about X"
-Good:    "Summarize X in 3 bullet points with page citations"
-```
-
-### Solution 3: Use Stronger Model
-```
-OpenAI:
-  Current: gpt-4o-mini → Switch to: gpt-4o
-
-Anthropic:
-  Current: claude-3-5-haiku → Switch to: claude-3-5-sonnet
-
-To change:
-1. Settings → Models
-2. Select model
-3. Try chat again
-```
-
-### Solution 4: Add More Sources
-```
-If:  "Response seems incomplete"
-Try: Add more relevant sources to provide context
-```
-
----
-
-## Chat is Very Slow
-
-**Symptom:** Chat responses take minutes
-
-**Cause:** Large context, slow model, or overloaded API
-
-**Solutions:**
-
-### Solution 1: Use Faster Model
-```bash
-Fastest: Groq (any model)
-Fast: OpenAI gpt-4o-mini
-Medium: Anthropic claude-3-5-haiku
-Slow: Anthropic claude-3-5-sonnet
-
-Switch in: Settings → Models
-```
-
-### Solution 2: Reduce Context
-```
-1. Chat → Select Sources
-2. Uncheck sources you don't need
-3. Or switch to "Summary Only" for background sources
-4. Save and try again
-```
-
-### Solution 3: Increase Timeout
-```bash
-# In .env:
-API_CLIENT_TIMEOUT=600  # 10 minutes
-
-# Restart:
-docker compose restart
-```
-
-### Solution 4: Check System Load
-```bash
-# See if API is overloaded:
-docker stats
-
-# If CPU >80% or memory >90%:
-# Reduce: SURREAL_COMMANDS_MAX_TASKS=2
-# Restart: docker compose restart
-```
-
----
-
-## Chat Doesn't Remember History
-
-**Symptom:** Each message treated as separate, no context between questions
-
-**Cause:** Chat history not saved or new chat started
-
-**Solution:**
-
-```
-1. Make sure you're in same Chat (not new Chat)
-2. Check Chat title at top
-3. If it's blank, start new Chat with a title
-4. Each named Chat keeps its history
-5. If you start new Chat, history is separate
-```
-
----
-
-## "Rate limit exceeded"
-
-**Symptom:** Error: "Rate limit exceeded" or "Too many requests"
-
-**Cause:** Hit provider's API rate limit
-
-**Solutions:**
-
-### For Cloud Providers (OpenAI, Anthropic, etc.)
-
-**Immediate:**
-- Wait 1-2 minutes
-- Try again
-
-**Short term:**
-- Use cheaper/smaller model
-- Reduce concurrent operations
-- Space out requests
-
-**Long term:**
-- Upgrade your account
-- Switch to different provider
-- Use Ollama (local, no limits)
-
-### Check Account Status
-```
-OpenAI: https://platform.openai.com/account/usage/overview
-Anthropic: https://console.anthropic.com/account/billing/overview
-Google: Google Cloud Console
-```
-
-### For Ollama (Local)
-- No rate limits
-- Use `ollama pull mistral` for best model
-- Restart if hitting resource limits
-
----
-
-## "Context length exceeded" or "Token limit"
-
-**Symptom:** Error about too many tokens
-
-**Cause:** Sources too large for model
-
-**Solutions:**
-
-### Solution 1: Use Model with Longer Context
-```
-Current: GPT-4o (128K tokens) → Switch to: Claude (200K tokens)
-Current: Claude Haiku (200K) → Switch to: Gemini (1M tokens)
-
-To change: Settings → Models
-```
-
-### Solution 2: Reduce Context
-```
-1. Select fewer sources
-2. Or use "Summary Only" instead of "Full Content"
-3. Or split large documents into smaller pieces
-```
-
-### Solution 3: For Ollama (Local)
-```bash
-# Use smaller model:
-ollama pull phi  # Very small
-# Instead of: ollama pull neural-chat  # Large
-```
-
----
-
-## "API call failed" or Timeout
-
-**Symptom:** Generic API error, response times out
-
-**Cause:** Provider API down, network issue, or slow service
-
-**Solutions:**
-
-### Check Provider Status
-```
-OpenAI: https://status.openai.com/
-Anthropic: Check website
-Google: Google Cloud Status
-Groq: Check website
-```
-
-### Retry Operation
-```
-1. Wait 30 seconds
-2. Try again
-```
-
-### Use Different Model/Provider
-```
-1. Settings → Models
-2. Try different provider
-3. If OpenAI down, use Anthropic
-```
-
-### Check Network
-```bash
-# Verify internet working:
-ping google.com
-
-# Test API endpoint directly:
-curl https://api.openai.com/v1/models \
-  -H "Authorization: Bearer YOUR_KEY"
-```
-
----
-
-## Responses Include Hallucinations
-
-**Symptom:** AI makes up facts that aren't in sources
-
-**Cause:** Sources not in context, or model guessing
-
-**Solutions:**
-
-### Solution 1: Verify Context
-```
-1. Click citation in response
-2. Check source actually says that
-3. If not, sources weren't in context
-4. Add source to context and try again
-```
-
-### Solution 2: Request Citations
-```
-Ask: "Answer this with citations to specific pages"
-
-The AI will be more careful if asked for citations
-```
-
-### Solution 3: Use Stronger Model
-```
-Weaker models hallucinate more
-Switch to: GPT-4o or Claude Sonnet
-```
-
----
-
-## High API Costs
-
-**Symptom:** API bills are higher than expected
-
-**Cause:** Using expensive model, large context, many requests
-
-**Solutions:**
-
-### Use Cheaper Model
-```
-Expensive: gpt-4o
-Cheaper: gpt-4o-mini (10x cheaper)
-
-Expensive: Claude Sonnet
-Cheaper: Claude Haiku (5x cheaper)
-
-Groq: Ultra cheap but fewer models
-```
-
-### Reduce Context
-```
-In Chat:
-1. Select fewer sources
-2. Use "Summary Only" for background
-3. Ask more specific questions
-```
-
-### Switch to Ollama (Free)
-```bash
-# Install Ollama
-# Run: ollama serve
-# Download: ollama pull mistral
-# Set: OLLAMA_API_BASE=http://localhost:11434
-# Cost: Free!
-```
-
----
-
-## Still Having Chat Issues?
-
-- Try [Quick Fixes](quick-fixes.md)
-- Try [Chat Effectively Guide](../3-USER-GUIDE/chat-effectively.md)
-- Check logs: `docker compose logs api | grep -i "error"`
-- Ask for help: [Troubleshooting Index](index.md#getting-help)

docs/6-TROUBLESHOOTING/connection-issues.md

@@ -1,447 +0,0 @@
-# Connection Issues - Network & API Problems
-
-Frontend can't reach API or services won't communicate.
-
----
-
-## "Cannot connect to server" (Most Common)
-
-**What it looks like:**
-- Browser shows error page
-- "Unable to reach API"
-- "Cannot connect to server"
-- UI loads but can't create notebooks
-
-**Diagnosis:**
-
-```bash
-# Check if API is running
-docker ps | grep api
-# Should see "api" service running
-
-# Check if API is responding
-curl http://localhost:5055/health
-# Should show: {"status":"ok"}
-
-# Check if frontend is running
-docker ps | grep frontend
-# Should see "frontend" or React service running
-```
-
-**Solutions:**
-
-### Solution 1: API Not Running
-```bash
-# Start API
-docker compose up api -d
-
-# Wait 5 seconds
-sleep 5
-
-# Verify it's running
-docker compose logs api | tail -20
-```
-
-### Solution 2: Port Not Exposed
-```bash
-# Check docker-compose.yml has port mapping:
-# api:
-#   ports:
-#     - "5055:5055"
-
-# If missing, add it and restart:
-docker compose down
-docker compose up -d
-```
-
-### Solution 3: API_URL Mismatch
-```bash
-# In .env, check API_URL:
-cat .env | grep API_URL
-
-# Should match your frontend URL:
-# Frontend: http://localhost:8502
-# API_URL: http://localhost:5055
-
-# If wrong, fix it:
-# API_URL=http://localhost:5055
-# Then restart:
-docker compose restart frontend
-```
-
-### Solution 4: Firewall Blocking
-```bash
-# Verify port 5055 is accessible
-netstat -tlnp | grep 5055
-# Should show port listening
-
-# If on different machine, try:
-# Instead of localhost, use your IP:
-API_URL=http://192.168.1.100:5055
-```
-
-### Solution 5: Services Not Started
-```bash
-# Restart everything
-docker compose restart
-
-# Wait 10 seconds
-sleep 10
-
-# Check all services
-docker compose ps
-# All should show "Up"
-```
-
----
-
-## Connection Refused
-
-**What it looks like:**
-```
-Connection refused
-ECONNREFUSED
-Error: socket hang up
-```
-
-**Diagnosis:**
-- API port (5055) not open
-- API crashed
-- Wrong IP/hostname
-
-**Solution:**
-
-```bash
-# Step 1: Check if API is running
-docker ps | grep api
-
-# Step 2: Check if port is listening
-lsof -i :5055
-# or
-netstat -tlnp | grep 5055
-
-# Step 3: Check API logs
-docker compose logs api | tail -30
-# Look for errors
-
-# Step 4: Restart API
-docker compose restart api
-docker compose logs api | grep -i "error"
-```
-
----
-
-## Timeout / Slow Connection
-
-**What it looks like:**
-- Page loads slowly
-- Request times out
-- "Gateway timeout" error
-
-**Causes:**
-- API is overloaded
-- Network is slow
-- Reverse proxy issue
-
-**Solutions:**
-
-### Check API Performance
-```bash
-# See CPU/memory usage
-docker stats
-
-# Check logs for slow operations
-docker compose logs api | grep "slow\|timeout"
-```
-
-### Reduce Load
-```bash
-# In .env:
-SURREAL_COMMANDS_MAX_TASKS=2
-API_CLIENT_TIMEOUT=600
-
-# Restart
-docker compose restart
-```
-
-### Check Network
-```bash
-# Test latency
-ping localhost
-
-# Test API directly
-time curl http://localhost:5055/health
-
-# Should be < 100ms
-```
-
----
-
-## 502 Bad Gateway (Reverse Proxy)
-
-**What it looks like:**
-```
-502 Bad Gateway
-The server is temporarily unable to service the request
-```
-
-**Cause:** Reverse proxy can't reach API
-
-**Solutions:**
-
-### Check Backend is Running
-```bash
-# From the reverse proxy server
-curl http://localhost:5055/health
-
-# Should work
-```
-
-### Check Reverse Proxy Config
-```nginx
-# Nginx example (correct):
-location /api {
-    proxy_pass http://localhost:5055/api;
-    proxy_http_version 1.1;
-}
-
-# Common mistake (wrong):
-location /api {
-    proxy_pass http://localhost:5055;  # Missing /api
-}
-```
-
-### Set API_URL for HTTPS
-```bash
-# In .env:
-API_URL=https://yourdomain.com
-
-# Restart
-docker compose restart
-```
-
----
-
-## Intermittent Disconnects
-
-**What it looks like:**
-- Works sometimes, fails other times
-- Sporadic "cannot connect" errors
-- Works then stops working
-
-**Cause:** Transient network issue or database conflicts
-
-**Solutions:**
-
-### Enable Retry Logic
-```bash
-# In .env:
-SURREAL_COMMANDS_RETRY_ENABLED=true
-SURREAL_COMMANDS_RETRY_MAX_ATTEMPTS=5
-SURREAL_COMMANDS_RETRY_WAIT_STRATEGY=exponential_jitter
-
-# Restart
-docker compose restart
-```
-
-### Reduce Concurrency
-```bash
-# In .env:
-SURREAL_COMMANDS_MAX_TASKS=2
-
-# Restart
-docker compose restart
-```
-
-### Check Network Stability
-```bash
-# Monitor connection
-ping google.com
-
-# Long-running test
-ping -c 100 google.com | grep "packet loss"
-# Should be 0% loss
-```
-
----
-
-## Different Machine / Remote Access
-
-**You want to access Open Notebook from another computer**
-
-**Solution:**
-
-### Step 1: Get Your Machine IP
-```bash
-# On the server running Open Notebook:
-ifconfig | grep "inet "
-# or
-hostname -I
-# Note the IP (e.g., 192.168.1.100)
-```
-
-### Step 2: Update API_URL
-```bash
-# In .env:
-API_URL=http://192.168.1.100:5055
-
-# Restart
-docker compose restart
-```
-
-### Step 3: Access from Other Machine
-```bash
-# In browser on other machine:
-http://192.168.1.100:8502
-# (or your server IP)
-```
-
-### Step 4: Verify Port is Exposed
-```bash
-# On server:
-docker compose ps
-
-# Should show port mapping:
-# 0.0.0.0:8502->8502/tcp
-# 0.0.0.0:5055->5055/tcp
-```
-
-### If Still Doesn't Work
-```bash
-# Check firewall on server
-sudo ufw status
-# May need to open ports:
-sudo ufw allow 8502
-sudo ufw allow 5055
-
-# Check on different machine:
-telnet 192.168.1.100 5055
-# Should connect
-```
-
----
-
-## CORS Error (Browser Console)
-
-**What it looks like:**
-```
-Cross-Origin Request Blocked
-Access-Control-Allow-Origin
-```
-
-**In browser console (F12):**
-```
-CORS policy: Response to preflight request doesn't pass access control check
-```
-
-**Cause:** Frontend and API URLs don't match
-
-**Solution:**
-
-```bash
-# Check browser console error for what URLs are being used
-# The error shows:
-# - Requesting from: http://localhost:8502
-# - Trying to reach: http://localhost:5055
-
-# Make sure API_URL matches:
-API_URL=http://localhost:5055
-
-# And protocol matches (http/https)
-# Restart
-docker compose restart frontend
-```
-
----
-
-## Testing Connection
-
-**Full diagnostic:**
-
-```bash
-# 1. Services running?
-docker compose ps
-# All should show "Up"
-
-# 2. Ports listening?
-netstat -tlnp | grep -E "8502|5055|8000"
-
-# 3. API responding?
-curl http://localhost:5055/health
-
-# 4. Frontend accessible?
-curl http://localhost:8502 | head
-
-# 5. Network OK?
-ping google.com
-
-# 6. No firewall?
-sudo ufw status | grep -E "5055|8502|8000"
-```
-
----
-
-## Checklist for Remote Access
-
-- [ ] Server IP noted (e.g., 192.168.1.100)
-- [ ] Ports 8502, 5055, 8000 exposed in docker-compose
-- [ ] API_URL set to server IP
-- [ ] Firewall allows ports 8502, 5055, 8000
-- [ ] Can reach server from client machine (ping IP)
-- [ ] All services running (docker compose ps)
-- [ ] Can curl API from client (curl http://IP:5055/health)
-
----
-
-## SSL Certificate Errors
-
-**What it looks like:**
-```
-[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed
-Connection error when using HTTPS endpoints
-Works with HTTP but fails with HTTPS
-```
-
-**Cause:** Self-signed certificates not trusted by Python's SSL verification
-
-**Solutions:**
-
-### Solution 1: Use Custom CA Bundle (Recommended)
-```bash
-# In .env:
-ESPERANTO_SSL_CA_BUNDLE=/path/to/your/ca-bundle.pem
-
-# For Docker, mount the certificate:
-# In docker-compose.yml:
-volumes:
-  - /path/to/your/ca-bundle.pem:/certs/ca-bundle.pem:ro
-environment:
-  - ESPERANTO_SSL_CA_BUNDLE=/certs/ca-bundle.pem
-```
-
-### Solution 2: Disable SSL Verification (Development Only)
-```bash
-# WARNING: Only use in trusted development environments
-# In .env:
-ESPERANTO_SSL_VERIFY=false
-```
-
-### Solution 3: Use HTTP Instead
-If services are on a trusted local network, HTTP is acceptable:
-```bash
-# Change endpoint from https:// to http://
-OPENAI_COMPATIBLE_BASE_URL=http://localhost:1234/v1
-```
-
-> **Security Note:** Disabling SSL verification exposes you to man-in-the-middle attacks. Always prefer custom CA bundle or HTTP on trusted networks.
-
----
-
-## Still Having Issues?
-
-- Check [Quick Fixes](quick-fixes.md)
-- Check [FAQ](faq.md)
-- Check logs: `docker compose logs`
-- Try restart: `docker compose restart`
-- Check firewall: `sudo ufw status`
-- Ask for help on [Discord](https://discord.gg/37XJPXfz2w)

docs/6-TROUBLESHOOTING/faq.md

@@ -1,258 +0,0 @@
-# Frequently Asked Questions
-
-Common questions about Open Notebook usage, configuration, and best practices.
-
----
-
-## General Usage
-
-### What is Open Notebook?
-
-Open Notebook is an open-source, privacy-focused alternative to Google's Notebook LM. It allows you to:
-- Create and manage research notebooks
-- Chat with your documents using AI
-- Generate podcasts from your content
-- Search across all your sources with semantic search
-- Transform and analyze your content
-
-### How is it different from Google Notebook LM?
-
-**Privacy**: Your data stays local by default. Only your chosen AI providers receive queries.
-**Flexibility**: Support for 15+ AI providers (OpenAI, Anthropic, Google, local models, etc.)
-**Customization**: Open source, so you can modify and extend functionality
-**Control**: You control your data, models, and processing
-
-### Can I use Open Notebook offline?
-
-**Partially**: The application runs locally, but requires internet for:
-- AI model API calls (unless using local models like Ollama)
-- Web content scraping
-
-**Fully offline**: Possible with local models (Ollama) for basic functionality.
-
-### What file types are supported?
-
-**Documents**: PDF, DOCX, TXT, Markdown
-**Web Content**: URLs, YouTube videos
-**Media**: MP3, WAV, M4A (audio), MP4, AVI, MOV (video)
-**Other**: Direct text input, CSV, code files
-
-### How much does it cost?
-
-**Software**: Free (open source)
-**AI API costs**: Pay-per-use to providers:
-- OpenAI: ~$0.50-5 per 1M tokens
-- Anthropic: ~$3-75 per 1M tokens
-- Google: Often free tier available
-- Local models: Free after initial setup
-
-**Typical monthly costs**: $5-50 for moderate usage.
-
----
-
-## AI Models and Providers
-
-### Which AI provider should I choose?
-
-**For beginners**: OpenAI (reliable, well-documented)
-**For privacy**: Local models (Ollama) or European providers (Mistral)
-**For cost optimization**: Groq, Google (free tier), or OpenRouter
-**For long context**: Anthropic (200K tokens) or Google Gemini (1M tokens)
-
-### Can I use multiple providers?
-
-**Yes**: Configure different providers for different tasks:
-- OpenAI for chat
-- Google for embeddings
-- ElevenLabs for text-to-speech
-- Anthropic for complex reasoning
-
-### What are the best model combinations?
-
-**Budget-friendly**:
-- Language: `gpt-4o-mini` (OpenAI) or `deepseek-chat`
-- Embedding: `text-embedding-3-small` (OpenAI)
-
-**High-quality**:
-- Language: `claude-3-5-sonnet` (Anthropic) or `gpt-4o` (OpenAI)
-- Embedding: `text-embedding-3-large` (OpenAI)
-
-**Privacy-focused**:
-- Language: Local Ollama models (mistral, llama3)
-- Embedding: Local embedding models
-
-### How do I optimize AI costs?
-
-**Model selection**:
-- Use smaller models for simple tasks (gpt-4o-mini, claude-3-5-haiku)
-- Use larger models only for complex reasoning
-- Leverage free tiers when available
-
-**Usage optimization**:
-- Use "Summary Only" context for background sources
-- Ask more specific questions
-- Use local models (Ollama) for frequent tasks
-
----
-
-## Data Management
-
-### Where is my data stored?
-
-**Local storage**: By default, all data is stored locally:
-- Database: SurrealDB files in `surreal_data/`
-- Uploads: Files in `data/uploads/`
-- Podcasts: Generated audio in `data/podcasts/`
-- No external data transmission (except to chosen AI providers)
-
-### How do I backup my data?
-
-```bash
-# Create backup
-tar -czf backup-$(date +%Y%m%d).tar.gz data/ surreal_data/
-
-# Restore backup
-tar -xzf backup-20240101.tar.gz
-```
-
-### Can I sync data between devices?
-
-**Currently**: No built-in sync functionality.
-**Workarounds**:
-- Use shared network storage for data directories
-- Manual backup/restore between devices
-
-### What happens if I delete a notebook?
-
-**Soft deletion**: Notebooks are marked as archived, not permanently deleted.
-**Recovery**: Archived notebooks can be restored from the database.
-
----
-
-## Best Practices
-
-### How should I organize my notebooks?
-
-- **By topic**: Separate notebooks for different research areas
-- **By project**: One notebook per project or course
-- **By time period**: Monthly or quarterly notebooks
-
-**Recommended size**: 20-100 sources per notebook for best performance.
-
-### How do I get the best search results?
-
-- Use descriptive queries ("data analysis methods" not just "data")
-- Combine multiple related terms
-- Use natural language (ask questions as you would to a human)
-- Try both text search (keywords) and vector search (concepts)
-
-### How can I improve chat responses?
-
-- Provide context: Reference specific sources or topics
-- Be specific: Ask detailed questions rather than general ones
-- Request citations: "Answer with page citations"
-- Use follow-up questions: Build on previous responses
-
-### What are the security best practices?
-
-- Never share API keys publicly
-- Use `OPEN_NOTEBOOK_PASSWORD` for public deployments
-- Use HTTPS for production (via reverse proxy)
-- Keep Docker images updated
-- Encrypt backups if they contain sensitive data
-
----
-
-## Technical Questions
-
-### Can I use Open Notebook programmatically?
-
-**Yes**: Open Notebook provides a REST API:
-- Full API documentation at `http://localhost:5055/docs`
-- Support for all UI functionality
-- Authentication via password header
-
-### Can I run Open Notebook in production?
-
-**Yes**: Designed for production use with:
-- Docker deployment
-- Security features (password protection)
-- Monitoring and logging
-- Reverse proxy support (nginx, Caddy, Traefik)
-
-### What are the system requirements?
-
-**Minimum**:
-- 4GB RAM
-- 2 CPU cores
-- 10GB disk space
-
-**Recommended**:
-- 8GB+ RAM
-- 4+ CPU cores
-- SSD storage
-- For local models: 16GB+ RAM, GPU recommended
-
----
-
-## Timeout and Performance
-
-### Why do I get timeout errors?
-
-**Common causes**:
-- Large context (too many sources)
-- Slow AI provider
-- Local models on CPU (slow)
-- First request (model loading)
-
-**Solutions**:
-```bash
-# In .env:
-API_CLIENT_TIMEOUT=600  # 10 minutes for slow setups
-ESPERANTO_LLM_TIMEOUT=180  # 3 minutes for model inference
-```
-
-### Recommended timeouts by setup:
-
-| Setup | API_CLIENT_TIMEOUT |
-|-------|-------------------|
-| Cloud APIs (OpenAI, Anthropic) | 300 (default) |
-| Local Ollama with GPU | 600 |
-| Local Ollama with CPU | 1200 |
-| Remote LM Studio | 900 |
-
----
-
-## Getting Help
-
-### My question isn't answered here
-
-1. Check the troubleshooting guides in this section
-2. Search existing GitHub issues
-3. Ask in the Discord community
-4. Create a GitHub issue with detailed information
-
-### How do I report a bug?
-
-Include:
-- Steps to reproduce
-- Expected vs actual behavior
-- Error messages and logs
-- System information
-- Configuration details (without API keys)
-
-Submit to: [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)
-
-### Where can I get help?
-
-- **Discord**: https://discord.gg/37XJPXfz2w (fastest)
-- **GitHub Issues**: Bug reports and feature requests
-- **Documentation**: This docs site
-
----
-
-## Related
-
-- [Quick Fixes](quick-fixes.md) - Common issues with 1-minute solutions
-- [AI & Chat Issues](ai-chat-issues.md) - Model and chat problems
-- [Connection Issues](connection-issues.md) - Network and API problems

docs/6-TROUBLESHOOTING/index.md

@@ -1,239 +0,0 @@
-# Troubleshooting - Problem Solving Guide
-
-Having issues? Use this guide to diagnose and fix problems.
-
----
-
-## How to Use This Guide
-
-**Step 1: Identify your problem**
-- What's the symptom? (error message, behavior, something not working?)
-- When did it happen? (during install, while using, after update?)
-
-**Step 2: Find the right guide**
-- Look below for your symptom
-- Go to the specific troubleshooting guide
-
-**Step 3: Follow the steps**
-- Guides are organized by symptom, not by root cause
-- Each has diagnostic steps and solutions
-
----
-
-## Quick Problem Map
-
-### During Installation
-
-- **Docker won't start** → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)
-- **Port already in use** → [Quick Fixes](quick-fixes.md#3-port-x-already-in-use)
-- **Permission denied** → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)
-- **Can't connect to database** → [Connection Issues](connection-issues.md)
-
-### When Starting
-
-- **API won't start** → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)
-- **Frontend won't load** → [Connection Issues](connection-issues.md)
-- **"Cannot connect to server" error** → [Connection Issues](connection-issues.md)
-
-### Settings / Configuration
-
-- **Models not showing** → [AI & Chat Issues](ai-chat-issues.md)
-- **"Invalid API key"** → [AI & Chat Issues](ai-chat-issues.md)
-- **Can't find Settings** → [Quick Fixes](quick-fixes.md)
-
-### Using Features
-
-- **Chat not working** → [AI & Chat Issues](ai-chat-issues.md)
-- **Chat responses are slow** → [AI & Chat Issues](ai-chat-issues.md)
-- **Chat gives bad answers** → [AI & Chat Issues](ai-chat-issues.md)
-
-### Adding Content
-
-- **Can't upload PDF** → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)
-- **File won't process** → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)
-- **Web link won't extract** → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)
-
-### Search
-
-- **Search returns no results** → [Quick Fixes](quick-fixes.md#7-search-returns-nothing)
-- **Search returns wrong results** → [Quick Fixes](quick-fixes.md#7-search-returns-nothing)
-
-### Podcasts
-
-- **Can't generate podcast** → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)
-- **Podcast audio is robotic** → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)
-- **Podcast generation times out** → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)
-
----
-
-## Troubleshooting by Error Message
-
-### "Cannot connect to server"
-→ [Connection Issues](connection-issues.md) — Frontend can't reach API
-
-### "Invalid API key"
-→ [AI & Chat Issues](ai-chat-issues.md) — Wrong or missing API key
-
-### "Models not available"
-→ [AI & Chat Issues](ai-chat-issues.md) — Model not configured
-
-### "Connection refused"
-→ [Connection Issues](connection-issues.md) — Service not running or port wrong
-
-### "Port already in use"
-→ [Quick Fixes](quick-fixes.md#3-port-x-already-in-use) — Port conflict
-
-### "Permission denied"
-→ [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error) — File permissions issue
-
-### "Unsupported file type"
-→ [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format) — File format not supported
-
-### "Processing timeout"
-→ [Quick Fixes](quick-fixes.md#5-chat-is-very-slow) — File too large or slow processing
-
----
-
-## Troubleshooting by Component
-
-### Frontend (Browser/UI)
-- Can't access UI → [Connection Issues](connection-issues.md)
-- UI is slow → [Quick Fixes](quick-fixes.md)
-- Button/feature missing → [Quick Fixes](quick-fixes.md)
-
-### API (Backend)
-- API won't start → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)
-- API errors in logs → [Quick Fixes](quick-fixes.md#9-services-wont-start-or-docker-error)
-- API is slow → [Quick Fixes](quick-fixes.md)
-
-### Database
-- Can't connect to database → [Connection Issues](connection-issues.md)
-- Data lost after restart → [FAQ](faq.md#how-do-i-backup-my-data)
-
-### AI / Chat
-- Chat not working → [AI & Chat Issues](ai-chat-issues.md)
-- Bad responses → [AI & Chat Issues](ai-chat-issues.md)
-- Cost too high → [AI & Chat Issues](ai-chat-issues.md#high-api-costs)
-
-### Sources
-- Can't upload file → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)
-- File won't process → [Quick Fixes](quick-fixes.md#4-cannot-process-file-or-unsupported-format)
-
-### Podcasts
-- Won't generate → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)
-- Bad audio quality → [Quick Fixes](quick-fixes.md#8-podcast-generation-failed)
-
----
-
-## Diagnostic Checklist
-
-**When something isn't working:**
-
-- [ ] Check if services are running: `docker ps`
-- [ ] Check logs: `docker compose logs api` (or frontend, surrealdb)
-- [ ] Verify ports are exposed: `netstat -tlnp` or `lsof -i :5055`
-- [ ] Test connectivity: `curl http://localhost:5055/health`
-- [ ] Check environment variables: `docker inspect <container>`
-- [ ] Try restarting: `docker compose restart`
-- [ ] Check firewall/antivirus isn't blocking
-
----
-
-## Getting Help
-
-If you can't find the answer here:
-
-1. **Check the relevant guide** — Read completely, try all steps
-2. **Check the FAQ** — [Frequently Asked Questions](faq.md)
-3. **Search our Discord** — Others may have had same issue
-4. **Check logs** — Most issues show error messages in logs
-5. **Report on GitHub** — Include error message, steps to reproduce
-
-### How to Report an Issue
-
-Include:
-1. Error message (exact)
-2. Steps to reproduce
-3. Logs: `docker compose logs`
-4. Your setup: Docker/local, provider, OS
-5. What you've already tried
-
-→ [Report on GitHub](https://github.com/lfnovo/open-notebook/issues)
-
----
-
-## Guides
-
-### [Quick Fixes](quick-fixes.md)
-Top 10 most common issues with 1-minute solutions.
-
-### [Connection Issues](connection-issues.md)
-Frontend can't reach API, network problems.
-
-### [AI & Chat Issues](ai-chat-issues.md)
-Chat not working, bad responses, slow performance.
-
-### [FAQ](faq.md)
-Frequently asked questions about usage, costs, and best practices.
-
----
-
-## Common Solutions
-
-**Service won't start?**
-```bash
-# Check logs
-docker compose logs
-
-# Restart everything
-docker compose restart
-
-# Nuclear option: rebuild
-docker compose down
-docker compose up --build
-```
-
-**Port conflict?**
-```bash
-# Find what's using port 5055
-lsof -i :5055
-# Kill it or use different port
-```
-
-**Can't connect?**
-```bash
-# Test API directly
-curl http://localhost:5055/health
-# Should return: {"status":"ok"}
-```
-
-**Slow performance?**
-```bash
-# Check resource usage
-docker stats
-
-# Reduce concurrency in .env
-SURREAL_COMMANDS_MAX_TASKS=2
-```
-
-**High costs?**
-```bash
-# Switch to cheaper model
-# In Settings → Models → Choose gpt-4o-mini (OpenAI)
-# Or use Ollama (free)
-```
-
----
-
-## Still Stuck?
-
-**Before asking for help:**
-1. Read the relevant guide completely
-2. Try all the steps
-3. Check the logs
-4. Restart services
-5. Search existing issues on GitHub
-
-**Then:**
-- **Discord**: https://discord.gg/37XJPXfz2w (fastest response)
-- **GitHub Issues**: https://github.com/lfnovo/open-notebook/issues

docs/6-TROUBLESHOOTING/quick-fixes.md

@@ -1,380 +0,0 @@
-# Quick Fixes - Top 11 Issues & Solutions
-
-Common problems with 1-minute solutions.
-
----
-
-## #1: "Cannot connect to server"
-
-**Symptom:** Browser shows error "Cannot connect to server" or "Unable to reach API"
-
-**Cause:** Frontend can't reach API
-
-**Solution (1 minute):**
-
-```bash
-# Step 1: Check if API is running
-docker ps | grep api
-
-# Step 2: Verify port 5055 is accessible
-curl http://localhost:5055/health
-
-# Expected output: {"status":"ok"}
-
-# If that doesn't work:
-# Step 3: Restart services
-docker compose restart
-
-# Step 4: Try again
-# Open http://localhost:8502 in browser
-```
-
-**If still broken:**
-- Check `API_URL` in .env (should match your frontend URL)
-- See [Connection Issues](connection-issues.md)
-
----
-
-## #2: "Invalid API key" or "Models not showing"
-
-**Symptom:** Settings → Models shows "No models available"
-
-**Cause:** API key missing, wrong, or not set
-
-**Solution (1 minute):**
-
-```bash
-# Step 1: Check your .env has API key
-cat .env | grep OPENAI_API_KEY
-
-# Step 2: Verify it's correct (from https://platform.openai.com/api-keys)
-# Should look like: sk-proj-xxx...
-
-# Step 3: Restart services
-docker compose restart api
-
-# Step 4: Wait 10 seconds, then refresh browser
-# Go to Settings → Models
-
-# If still no models:
-# Check logs for error
-docker compose logs api | grep -i "api key\|error"
-```
-
-**If still broken:**
-- Make sure key has no extra spaces
-- Generate a fresh key from provider dashboard
-- See [AI & Chat Issues](ai-chat-issues.md)
-
----
-
-## #3: "Port X already in use"
-
-**Symptom:** Docker error "Port 8502 is already allocated"
-
-**Cause:** Another service using that port
-
-**Solution (1 minute):**
-
-```bash
-# Option 1: Stop the other service
-# Find what's using port 8502
-lsof -i :8502
-# Kill it or close the app
-
-# Option 2: Use different port
-# Edit docker-compose.yml
-# Change: - "8502:8502"
-# To:     - "8503:8502"
-
-# Then restart
-docker compose restart
-# Access at: http://localhost:8503
-```
-
----
-
-## #4: "Cannot process file" or "Unsupported format"
-
-**Symptom:** Upload fails or says "File format not supported"
-
-**Cause:** File type not supported or too large
-
-**Solution (1 minute):**
-
-```bash
-# Check if file format is supported:
-# ✓ PDF, DOCX, PPTX, XLSX (documents)
-# ✓ MP3, WAV, M4A (audio)
-# ✓ MP4, AVI, MOV (video)
-# ✓ URLs/web links
-
-# ✗ Pure images (.jpg without OCR)
-# ✗ Files > 100MB
-
-# Try these:
-# - Convert to PDF if possible
-# - Split large files
-# - Try uploading again
-```
-
----
-
-## #5: "Chat is very slow"
-
-**Symptom:** Chat responses take minutes or timeout
-
-**Cause:** Slow AI provider, large context, or overloaded system
-
-**Solution (1 minute):**
-
-```bash
-# Step 1: Check which model you're using
-# Settings → Models
-# Note the model name
-
-# Step 2: Try a cheaper/faster model
-# OpenAI: Switch to gpt-4o-mini (10x cheaper, slightly faster)
-# Anthropic: Switch to claude-3-5-haiku (fastest)
-# Groq: Use any model (ultra-fast)
-
-# Step 3: Reduce context
-# Chat: Select fewer sources
-# Use "Summary Only" instead of "Full Content"
-
-# Step 4: Check if API is overloaded
-docker stats
-# Look at CPU/memory usage
-```
-
-For deep dive: See [AI & Chat Issues](ai-chat-issues.md)
-
----
-
-## #6: "Chat gives bad responses"
-
-**Symptom:** AI responses are generic, wrong, or irrelevant
-
-**Cause:** Bad context, vague question, or wrong model
-
-**Solution (1 minute):**
-
-```bash
-# Step 1: Make sure sources are in context
-# Click "Select Sources" in Chat
-# Verify relevant sources are checked and set to "Full Content"
-
-# Step 2: Ask a specific question
-# Bad: "What do you think?"
-# Good: "Based on the paper's methodology section, what are the 3 main limitations?"
-
-# Step 3: Try a more powerful model
-# OpenAI: Use gpt-4o (better reasoning)
-# Anthropic: Use claude-3-5-sonnet (best reasoning)
-
-# Step 4: Check citations
-# Click citations to verify AI actually saw those sources
-```
-
-For detailed help: See [Chat Effectively](../3-USER-GUIDE/chat-effectively.md)
-
----
-
-## #7: "Search returns nothing"
-
-**Symptom:** Search shows 0 results even though content exists
-
-**Cause:** Wrong search type or poor query
-
-**Solution (1 minute):**
-
-```bash
-# Try a different search type:
-
-# If you searched with KEYWORDS:
-# Try VECTOR SEARCH instead
-# (Concept-based, not keyword-based)
-
-# If you searched for CONCEPTS:
-# Try TEXT SEARCH instead
-# (Look for specific words in your query)
-
-# Try simpler search:
-# Instead of: "How do transformers work in neural networks?"
-# Try: "transformers" or "neural networks"
-
-# Check sources are processed:
-# Go to notebook
-# All sources should show green "Ready" status
-```
-
-For detailed help: See [Search Effectively](../3-USER-GUIDE/search.md)
-
----
-
-## #8: "Podcast generation failed"
-
-**Symptom:** "Podcast generation failed" error
-
-**Cause:** Insufficient content, API quota, or network issue
-
-**Solution (1 minute):**
-
-```bash
-# Step 1: Make sure you have content
-# Select at least 1-2 sources
-# Avoid single-sentence sources
-
-# Step 2: Try again
-# Sometimes it's a temporary API issue
-# Wait 30 seconds and retry
-
-# Step 3: Check your TTS provider has quota
-# OpenAI: Check account has credits
-# ElevenLabs: Check monthly quota
-# Google: Check API quota
-
-# Step 4: Try different TTS provider
-# In podcast generation, choose "Google" or "Local"
-# instead of "ElevenLabs"
-```
-
-For detailed help: See [FAQ](faq.md)
-
----
-
-## #9: "Services won't start" or Docker error
-
-**Symptom:** Docker error when running `docker compose up`
-
-**Cause:** Corrupt configuration, permission issue, or resource issue
-
-**Solution (1 minute):**
-
-```bash
-# Step 1: Check logs
-docker compose logs
-
-# Step 2: Try restart
-docker compose restart
-
-# Step 3: If that fails, rebuild
-docker compose down
-docker compose up --build
-
-# Step 4: Check disk space
-df -h
-# Need at least 5GB free
-
-# Step 5: Check Docker has enough memory
-# Docker settings → Resources → Memory: 4GB+
-```
-
----
-
-## #10: "Database says 'too many connections'"
-
-**Symptom:** Error about database connections
-
-**Cause:** Too many concurrent operations
-
-**Solution (1 minute):**
-
-```bash
-# In .env, reduce concurrency:
-SURREAL_COMMANDS_MAX_TASKS=2
-
-# Then restart:
-docker compose restart
-
-# This makes it slower but more stable
-```
-
----
-
-## #11: Slow Startup or Download Timeouts (China/Slow Networks)
-
-**Symptom:** Container crashes on startup, worker enters FATAL state, or pip/uv downloads fail
-
-**Cause:** Slow network or restricted access to Python package repositories
-
-**Solution:**
-
-### Increase Download Timeout
-```yaml
-# In docker-compose.yml environment:
-environment:
-  - UV_HTTP_TIMEOUT=600  # 10 minutes (default is 30s)
-```
-
-### Use Chinese Mirrors (if in China)
-```yaml
-environment:
-  - UV_HTTP_TIMEOUT=600
-  - UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
-  - PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
-```
-
-**Alternative Chinese mirrors:**
-- Tsinghua: `https://pypi.tuna.tsinghua.edu.cn/simple`
-- Aliyun: `https://mirrors.aliyun.com/pypi/simple/`
-- Huawei: `https://repo.huaweicloud.com/repository/pypi/simple`
-
-**Note:** First startup may take several minutes while dependencies download. Subsequent starts will be faster.
-
----
-
-## Quick Troubleshooting Checklist
-
-When something breaks:
-
-- [ ] **Restart services:** `docker compose restart`
-- [ ] **Check logs:** `docker compose logs`
-- [ ] **Verify connectivity:** `curl http://localhost:5055/health`
-- [ ] **Check .env:** API keys set? API_URL correct?
-- [ ] **Check resources:** `docker stats` (CPU/memory)
-- [ ] **Clear cache:** `docker system prune` (free space)
-- [ ] **Rebuild if needed:** `docker compose up --build`
-
----
-
-## Nuclear Options (Last Resort)
-
-**Completely reset (will lose all data in Docker):**
-
-```bash
-docker compose down -v
-docker compose up --build
-```
-
-**Reset to defaults:**
-```bash
-# Backup your .env first!
-cp .env .env.backup
-
-# Reset to example
-cp .env.example .env
-
-# Edit with your API keys
-# Restart
-docker compose up
-```
-
----
-
-## Prevention Tips
-
-1. **Keep backups** — Export your notebooks regularly
-2. **Monitor logs** — Check `docker compose logs` periodically
-3. **Update regularly** — Pull latest image: `docker pull lfnovo/open_notebook:latest`
-4. **Document changes** — Keep notes on what you configured
-5. **Test after updates** — Verify everything works
-
----
-
-## Still Stuck?
-
-- **Look up your exact error** in [Troubleshooting Index](index.md)
-- **Check the FAQ** in [FAQ](faq.md)
-- **Check logs:** `docker compose logs | head -50`
-- **Ask for help:** [Discord](https://discord.gg/37XJPXfz2w) or [GitHub Issues](https://github.com/lfnovo/open-notebook/issues)

docs/7-DEVELOPMENT/api-reference.md

@@ -1,213 +0,0 @@
-# API Reference
-
-Complete REST API for Open Notebook. All endpoints are served from the API backend (default: `http://localhost:5055`).
-
-**Base URL**: `http://localhost:5055` (development) or environment-specific production URL
-
-**Interactive Docs**: Use FastAPI's built-in Swagger UI at `http://localhost:5055/docs` for live testing and exploration. This is the primary reference for all endpoints, request/response schemas, and real-time testing.
-
----
-
-## Quick Start
-
-### 1. Authentication
-
-Simple password-based (development only):
-
-```bash
-curl http://localhost:5055/api/notebooks \
-  -H "Authorization: Bearer your_password"
-```
-
-**⚠️ Production**: Replace with OAuth/JWT. See [Security Configuration](../5-CONFIGURATION/security.md) for details.
-
-### 2. Base API Flow
-
-Most operations follow this pattern:
-1. Create a **Notebook** (container for research)
-2. Add **Sources** (PDFs, URLs, text)
-3. Query via **Chat** or **Search**
-4. View results and **Notes**
-
-### 3. Testing Endpoints
-
-Instead of memorizing endpoints, use the interactive API docs:
-- Navigate to `http://localhost:5055/docs`
-- Try requests directly in the browser
-- See request/response schemas in real-time
-- Test with your own data
-
----
-
-## API Endpoints Overview
-
-### Main Resource Types
-
-**Notebooks** - Research projects containing sources and notes
-- `GET/POST /notebooks` - List and create
-- `GET/PUT/DELETE /notebooks/{id}` - Read, update, delete
-
-**Sources** - Content items (PDFs, URLs, text)
-- `GET/POST /sources` - List and add content
-- `GET /sources/{id}` - Fetch source details
-- `POST /sources/{id}/retry` - Retry failed processing
-- `GET /sources/{id}/download` - Download original file
-
-**Notes** - User-created or AI-generated research notes
-- `GET/POST /notes` - List and create
-- `GET/PUT/DELETE /notes/{id}` - Read, update, delete
-
-**Chat** - Conversational AI interface
-- `GET/POST /chat/sessions` - Manage chat sessions
-- `POST /chat/execute` - Send message and get response
-- `POST /chat/context/build` - Prepare context for chat
-
-**Search** - Find content by text or semantic similarity
-- `POST /search` - Full-text or vector search
-- `POST /ask` - Ask a question (search + synthesize)
-
-**Transformations** - Custom prompts for extracting insights
-- `GET/POST /transformations` - Create custom extraction rules
-- `POST /sources/{id}/insights` - Apply transformation to source
-
-**Models** - Configure AI providers
-- `GET /models` - Available models
-- `GET /models/defaults` - Current defaults
-- `POST /models/config` - Set defaults
-
-**Health & Status**
-- `GET /health` - Health check
-- `GET /commands/{id}` - Track async operations
-
----
-
-## Authentication
-
-### Current (Development)
-
-All requests require password header:
-
-```bash
-curl -H "Authorization: Bearer your_password" http://localhost:5055/api/notebooks
-```
-
-Password configured via `OPEN_NOTEBOOK_PASSWORD` environment variable.
-
-> **📖 See [Security Configuration](../5-CONFIGURATION/security.md)** for complete authentication setup, API examples, and production hardening.
-
-### Production
-
-**⚠️ Not secure.** Replace with:
-- OAuth 2.0 (recommended)
-- JWT tokens
-- API keys
-
-See [Security Configuration](../5-CONFIGURATION/security.md) for production setup.
-
----
-
-## Common Patterns
-
-### Pagination
-
-```bash
-# List sources with limit/offset
-curl 'http://localhost:5055/sources?limit=20&offset=10'
-```
-
-### Filtering & Sorting
-
-```bash
-# Filter by notebook, sort by date
-curl 'http://localhost:5055/sources?notebook_id=notebook:abc&sort_by=created&sort_order=asc'
-```
-
-### Async Operations
-
-Some operations (source processing, podcast generation) return immediately with a command ID:
-
-```bash
-# Submit async operation
-curl -X POST http://localhost:5055/sources -F async_processing=true
-# Response: {"id": "source:src001", "command_id": "command:cmd123"}
-
-# Poll status
-curl http://localhost:5055/commands/command:cmd123
-```
-
-### Streaming Responses
-
-The `/ask` endpoint streams responses as Server-Sent Events:
-
-```bash
-curl -N 'http://localhost:5055/ask' \
-  -H "Content-Type: application/json" \
-  -d '{"question": "What is AI?"}'
-
-# Outputs: data: {"type":"strategy",...}
-#          data: {"type":"answer",...}
-#          data: {"type":"final_answer",...}
-```
-
-### Multipart File Upload
-
-```bash
-curl -X POST http://localhost:5055/sources \
-  -F "type=upload" \
-  -F "notebook_id=notebook:abc" \
-  -F "file=@document.pdf"
-```
-
----
-
-## Error Handling
-
-All errors return JSON with status code:
-
-```json
-{"detail": "Notebook not found"}
-```
-
-### Common Status Codes
-
-| Code | Meaning | Example |
-|------|---------|---------|
-| 200 | Success | Operation completed |
-| 400 | Bad Request | Invalid input |
-| 404 | Not Found | Resource doesn't exist |
-| 409 | Conflict | Resource already exists |
-| 500 | Server Error | Database/processing error |
-
----
-
-## Tips for Developers
-
-1. **Start with interactive docs** (`http://localhost:5055/docs`) - this is the definitive reference
-2. **Enable logging** for debugging (check API logs: `docker logs`)
-3. **Streaming endpoints** require special handling (Server-Sent Events, not standard JSON)
-4. **Async operations** return immediately; always poll status before assuming completion
-5. **Vector search** requires embedding model configured (check `/models`)
-6. **Model overrides** are per-request; set in body, not config
-7. **CORS enabled** in development; configure for production
-
----
-
-## Learning Path
-
-1. **Authentication**: Add `X-Password` header to all requests
-2. **Create a notebook**: `POST /notebooks` with name and description
-3. **Add a source**: `POST /sources` with file, URL, or text
-4. **Query your content**: `POST /chat/execute` to ask questions
-5. **Explore advanced features**: Search, transformations, streaming
-
----
-
-## Production Considerations
-
-- Replace password auth with OAuth/JWT (see [Security](../5-CONFIGURATION/security.md))
-- Add rate limiting via reverse proxy (Nginx, CloudFlare, Kong)
-- Enable CORS restrictions (currently allows all origins)
-- Use HTTPS via reverse proxy (see [Reverse Proxy](../5-CONFIGURATION/reverse-proxy.md))
-- Set up API versioning strategy (currently implicit)
-
-See [Security Configuration](../5-CONFIGURATION/security.md) and [Reverse Proxy Setup](../5-CONFIGURATION/reverse-proxy.md) for complete production setup.

docs/7-DEVELOPMENT/architecture.md

@@ -1,891 +0,0 @@
-# Open Notebook Architecture
-
-## High-Level Overview
-
-Open Notebook follows a three-tier architecture with clear separation of concerns:
-
-```
-┌─────────────────────────────────────────────────────────┐
-│  Your Browser                                           │
-│  Access: http://your-server-ip:8502                     │
-└────────────────┬────────────────────────────────────────┘
-                 │
-                 ▼
-         ┌───────────────┐
-         │   Port 8502   │  ← Next.js Frontend (what you see)
-         │   Frontend    │    Also proxies API requests internally!
-         └───────┬───────┘
-                 │ proxies /api/* requests ↓
-                 ▼
-         ┌───────────────┐
-         │   Port 5055   │  ← FastAPI Backend (handles requests)
-         │     API       │
-         └───────┬───────┘
-                 │
-                 ▼
-         ┌───────────────┐
-         │   SurrealDB   │  ← Database (internal, auto-configured)
-         │   (Port 8000) │
-         └───────────────┘
-```
-
-**Key Points:**
-- **v1.1+**: Next.js automatically proxies `/api/*` requests to the backend, simplifying reverse proxy setup
-- Your browser loads the frontend from port 8502
-- The frontend needs to know where to find the API - when accessing remotely, set: `API_URL=http://your-server-ip:5055`
-- **Behind reverse proxy?** You only need to proxy to port 8502 now! See [Reverse Proxy Configuration](../5-CONFIGURATION/reverse-proxy.md)
-
----
-
-## Detailed Architecture
-
-Open Notebook is built on a **three-tier, async-first architecture** designed for scalability, modularity, and multi-provider AI flexibility. The system separates concerns across frontend, API, and database layers, with LangGraph powering intelligent workflows and Esperanto enabling seamless integration with 8+ AI providers.
-
-**Core Philosophy**:
-- Privacy-first: Users control their data and AI provider choice
-- Async/await throughout: Non-blocking operations for responsive UX
-- Domain-Driven Design: Clear separation between domain models, repositories, and orchestrators
-- Multi-provider flexibility: Swap AI providers without changing application code
-- Self-hosted capable: All components deployable in isolated environments
-
----
-
-## Three-Tier Architecture
-
-### Layer 1: Frontend (React/Next.js @ port 3000)
-
-**Purpose**: Responsive, interactive user interface for research, notes, chat, and podcast management.
-
-**Technology Stack**:
-- **Framework**: Next.js 15 with React 19
-- **Language**: TypeScript with strict type checking
-- **State Management**: Zustand (lightweight store) + TanStack Query (server state)
-- **Styling**: Tailwind CSS + Shadcn/ui component library
-- **Build Tool**: Webpack (bundled via Next.js)
-
-**Key Responsibilities**:
-- Render notebooks, sources, notes, chat sessions, and podcasts
-- Handle user interactions (create, read, update, delete operations)
-- Manage complex UI state (modals, file uploads, real-time search)
-- Stream responses from API (chat, podcast generation)
-- Display embeddings, vector search results, and insights
-
-**Communication Pattern**:
-- All data fetched via REST API (async requests to port 5055)
-- Configured base URL: `http://localhost:5055` (dev) or environment-specific (prod)
-- TanStack Query handles caching, refetching, and data synchronization
-- Zustand stores global state (user, notebooks, selected context)
-- CORS enabled on API side for cross-origin requests
-
-**Component Architecture**:
-- `/src/app/`: Next.js App Router (pages, layouts)
-- `/src/components/`: Reusable React components (buttons, forms, cards)
-- `/src/hooks/`: Custom hooks (useNotebook, useChat, useSearch)
-- `/src/lib/`: Utility functions, API clients, validators
-- `/src/styles/`: Global CSS, Tailwind config
-
----
-
-### Layer 2: API (FastAPI @ port 5055)
-
-**Purpose**: RESTful backend exposing operations on notebooks, sources, notes, chat sessions, and AI models.
-
-**Technology Stack**:
-- **Framework**: FastAPI 0.104+ (async Python web framework)
-- **Language**: Python 3.11+
-- **Validation**: Pydantic v2 (request/response schemas)
-- **Logging**: Loguru (structured JSON logging)
-- **Testing**: Pytest (unit and integration tests)
-
-**Architecture**:
-```
-FastAPI App (main.py)
-  ├── Routers (HTTP endpoints)
-  │   ├── routers/notebooks.py (CRUD operations)
-  │   ├── routers/sources.py (content ingestion, upload)
-  │   ├── routers/notes.py (note management)
-  │   ├── routers/chat.py (conversation sessions)
-  │   ├── routers/search.py (full-text + vector search)
-  │   ├── routers/transformations.py (custom transformations)
-  │   ├── routers/models.py (AI model configuration)
-  │   └── routers/*.py (11 additional routers)
-  │
-  ├── Services (business logic)
-  │   ├── *_service.py (orchestration, graph invocation)
-  │   ├── command_service.py (async job submission)
-  │   └── middleware (auth, logging)
-  │
-  ├── Models (Pydantic schemas)
-  │   └── models.py (validation, serialization)
-  │
-  └── Lifespan (startup/shutdown)
-      └── AsyncMigrationManager (database schema migrations)
-```
-
-**Key Responsibilities**:
-1. **HTTP Interface**: Accept REST requests, validate, return JSON responses
-2. **Business Logic**: Orchestrate domain models, repository operations, and workflows
-3. **Async Job Queue**: Submit long-running tasks (podcast generation, source processing)
-4. **Database Migrations**: Run schema updates on startup
-5. **Error Handling**: Catch exceptions, return appropriate HTTP status codes
-6. **Logging**: Track operations for debugging and monitoring
-
-**Startup Flow**:
-1. Load `.env` environment variables
-2. Initialize FastAPI app with CORS + auth middleware
-3. Run AsyncMigrationManager (creates/updates database schema)
-4. Register all routers (20+ endpoints)
-5. Server ready on port 5055
-
-**Request-Response Cycle**:
-```
-HTTP Request → Router → Service → Domain/Repository → SurrealDB
-                                       ↓
-                                  LangGraph (optional)
-                                       ↓
-Response ← Pydantic serialization ← Service ← Result
-```
-
----
-
-### Layer 3: Database (SurrealDB @ port 8000)
-
-**Purpose**: Graph database with built-in vector embeddings, semantic search, and relationship management.
-
-**Technology Stack**:
-- **Database**: SurrealDB (multi-model, ACID transactions)
-- **Query Language**: SurrealQL (SQL-like syntax with graph operations)
-- **Async Driver**: Async Rust client for Python
-- **Migrations**: Manual `.surql` files in `/migrations/` (auto-run on API startup)
-
-**Core Tables**:
-
-| Table | Purpose | Key Fields |
-|-------|---------|-----------|
-| `notebook` | Research project container | id, name, description, archived, created, updated |
-| `source` | Content item (PDF, URL, text) | id, title, full_text, topics, asset, created, updated |
-| `source_embedding` | Vector embeddings for semantic search | id, source, embedding, chunk_text, chunk_index |
-| `note` | User-created research notes | id, title, content, note_type (human/ai), created, updated |
-| `chat_session` | Conversation session | id, notebook_id, title, messages (JSON), created, updated |
-| `transformation` | Custom transformation rules | id, name, description, prompt, created, updated |
-| `source_insight` | Transformation output | id, source_id, insight_type, content, created, updated |
-| `reference` | Relationship: source → notebook | out (source), in (notebook) |
-| `artifact` | Relationship: note → notebook | out (note), in (notebook) |
-
-**Relationship Graph**:
-```
-Notebook
-  ↓ (referenced_by)
-Source
-  ├→ SourceEmbedding (1:many for chunked text)
-  ├→ SourceInsight (1:many for transformation outputs)
-  └→ Note (via artifact relationship)
-    ├→ Embedding (semantic search)
-    └→ Topics (tags)
-
-ChatSession
-  ├→ Notebook
-  └→ Messages (stored as JSON array)
-```
-
-**Vector Search Capability**:
-- Embeddings stored natively in SurrealDB
-- Full-text search on `source.full_text` and `note.content`
-- Cosine similarity search on embedding vectors
-- Semantic search integrates with search endpoint
-
-**Connection Management**:
-- Async connection pooling (configurable size)
-- Transaction support for multi-record operations
-- Schema auto-validation via migrations
-- Query timeout protection (prevent infinite queries)
-
----
-
-## Tech Stack Rationale
-
-### Why Python + FastAPI?
-
-**Python**:
-- Rich AI/ML ecosystem (LangChain, LangGraph, transformers, scikit-learn)
-- Rapid prototyping and deployment
-- Extensive async support (asyncio, async/await)
-- Strong type hints (Pydantic, mypy)
-
-**FastAPI**:
-- Modern, async-first framework
-- Automatic OpenAPI documentation (Swagger UI @ /docs)
-- Built-in request validation (Pydantic)
-- Excellent performance (benchmarked near C/Rust speeds)
-- Easy middleware/dependency injection
-
-### Why Next.js + React + TypeScript?
-
-**Next.js**:
-- Full-stack React framework with SSR/SSG
-- File-based routing (intuitive project structure)
-- Built-in API routes (optional backend co-location)
-- Optimized image/code splitting
-- Easy deployment (Vercel, Docker, self-hosted)
-
-**React 19**:
-- Component-based UI (reusable, testable)
-- Excellent tooling and community
-- Client-side state management (Zustand)
-- Server-side state sync (TanStack Query)
-
-**TypeScript**:
-- Type safety catches errors at compile time
-- Better IDE autocomplete and refactoring
-- Documentation via types (self-documenting code)
-- Easier onboarding for new contributors
-
-### Why SurrealDB?
-
-**SurrealDB**:
-- Native graph database (relationships are first-class)
-- Built-in vector embeddings (no separate vector DB)
-- ACID transactions (data consistency)
-- Multi-model (relational + document + graph)
-- Full-text search + semantic search in one query
-- Self-hosted (unlike managed Pinecone/Weaviate)
-- Flexible SurrealQL (SQL-like syntax)
-
-**Alternative Considered**: PostgreSQL + pgvector (more mature but separate extensions)
-
-### Why Esperanto for AI Providers?
-
-**Esperanto Library**:
-- Unified interface to 8+ LLM providers (OpenAI, Anthropic, Google, Groq, Ollama, Mistral, DeepSeek, xAI)
-- Multi-provider embeddings (OpenAI, Google, Ollama, Mistral, Voyage)
-- TTS/STT integration (OpenAI, Groq, ElevenLabs, Google)
-- Smart provider selection (fallback logic, cost optimization)
-- Per-request model override support
-- Local Ollama support (completely self-hosted option)
-
-**Alternative Considered**: LangChain's provider abstraction (more verbose, less flexible)
-
----
-
-## LangGraph Workflows
-
-LangGraph is a state machine library that orchestrates multi-step AI workflows. Open Notebook uses five core workflows:
-
-### 1. **Source Processing Workflow** (`open_notebook/graphs/source.py`)
-
-**Purpose**: Ingest content (PDF, URL, text) and prepare for search/insights.
-
-**Flow**:
-```
-Input (file/URL/text)
-  ↓
-Extract Content (content-core library)
-  ↓
-Clean & tokenize text
-  ↓
-Generate Embeddings (Esperanto)
-  ↓
-Create SourceEmbedding records (chunked + indexed)
-  ↓
-Extract Topics (LLM summarization)
-  ↓
-Save to SurrealDB
-  ↓
-Output (Source record with embeddings)
-```
-
-**State Dict**:
-```python
-{
-  "content_state": {"file_path" | "url" | "content": str},
-  "source_id": str,
-  "full_text": str,
-  "embeddings": List[Dict],
-  "topics": List[str],
-  "notebook_ids": List[str],
-}
-```
-
-**Invoked By**: Sources API (`POST /sources`)
-
----
-
-### 2. **Chat Workflow** (`open_notebook/graphs/chat.py`)
-
-**Purpose**: Conduct multi-turn conversations with AI model, referencing notebook context.
-
-**Flow**:
-```
-User Message
-  ↓
-Build Context (selected sources/notes)
-  ↓
-Add Message to Session
-  ↓
-Create Chat Prompt (system + history + context)
-  ↓
-Call LLM (via Esperanto)
-  ↓
-Stream Response
-  ↓
-Save AI Message to ChatSession
-  ↓
-Output (complete message)
-```
-
-**State Dict**:
-```python
-{
-  "session_id": str,
-  "messages": List[BaseMessage],
-  "context": Dict[str, Any],  # sources, notes, snippets
-  "response": str,
-  "model_override": Optional[str],
-}
-```
-
-**Key Features**:
-- Message history persisted in SurrealDB (SqliteSaver checkpoint)
-- Context building via `build_context_for_chat()` utility
-- Token counting to prevent overflow
-- Per-message model override support
-
-**Invoked By**: Chat API (`POST /chat/execute`)
-
----
-
-### 3. **Ask Workflow** (`open_notebook/graphs/ask.py`)
-
-**Purpose**: Answer user questions by searching sources and synthesizing responses.
-
-**Flow**:
-```
-User Question
-  ↓
-Plan Search Strategy (LLM generates searches)
-  ↓
-Execute Searches (vector + text search)
-  ↓
-Score & Rank Results
-  ↓
-Provide Answers (LLM synthesizes from results)
-  ↓
-Stream Responses
-  ↓
-Output (final answer)
-```
-
-**State Dict**:
-```python
-{
-  "question": str,
-  "strategy": SearchStrategy,
-  "answers": List[str],
-  "final_answer": str,
-  "sources_used": List[Source],
-}
-```
-
-**Streaming**: Uses `astream()` to emit updates in real-time (strategy → answers → final answer)
-
-**Invoked By**: Search API (`POST /ask` with streaming)
-
----
-
-### 4. **Transformation Workflow** (`open_notebook/graphs/transformation.py`)
-
-**Purpose**: Apply custom transformations to sources (extract summaries, key points, etc).
-
-**Flow**:
-```
-Source + Transformation Rule
-  ↓
-Generate Prompt (Jinja2 template)
-  ↓
-Call LLM
-  ↓
-Parse Output
-  ↓
-Create SourceInsight record
-  ↓
-Output (insight with type + content)
-```
-
-**Example Transformations**:
-- Summary (5-sentence overview)
-- Key Points (bulleted list)
-- Quotes (notable excerpts)
-- Q&A (generated questions and answers)
-
-**Invoked By**: Sources API (`POST /sources/{id}/insights`)
-
----
-
-### 5. **Prompt Workflow** (`open_notebook/graphs/prompt.py`)
-
-**Purpose**: Generic LLM task execution (e.g., auto-generate note titles, analyze content).
-
-**Flow**:
-```
-Input Text + Prompt
-  ↓
-Call LLM (simple request-response)
-  ↓
-Output (completion)
-```
-
-**Used For**: Note title generation, content analysis, etc.
-
----
-
-## AI Provider Integration Pattern
-
-### ModelManager: Centralized Factory
-
-Located in `open_notebook/ai/models.py`, ModelManager handles:
-
-1. **Provider Detection**: Check environment variables for available providers
-2. **Model Selection**: Choose best model based on context size and task
-3. **Fallback Logic**: If primary provider unavailable, try backup
-4. **Cost Optimization**: Prefer cheaper models for simple tasks
-5. **Token Calculation**: Estimate cost before LLM call
-
-**Usage**:
-```python
-from open_notebook.ai.provision import provision_langchain_model
-
-# Get best LLM for context size
-model = await provision_langchain_model(
-    task="chat",  # or "search", "extraction"
-    model_override="anthropic/claude-opus-4",  # optional
-    context_size=8000,  # estimated tokens
-)
-
-# Invoke model
-response = await model.ainvoke({"input": prompt})
-```
-
-### Multi-Provider Support
-
-**LLM Providers**:
-- OpenAI (gpt-4, gpt-4-turbo, gpt-3.5-turbo)
-- Anthropic (claude-opus, claude-sonnet, claude-haiku)
-- Google (gemini-pro, gemini-1.5)
-- Groq (mixtral, llama-2)
-- Ollama (local models)
-- Mistral (mistral-large, mistral-medium)
-- DeepSeek (deepseek-chat)
-- xAI (grok)
-
-**Embedding Providers**:
-- OpenAI (text-embedding-3-large, text-embedding-3-small)
-- Google (embedding-001)
-- Ollama (local embeddings)
-- Mistral (mistral-embed)
-- Voyage (voyage-large-2)
-
-**TTS Providers**:
-- OpenAI (tts-1, tts-1-hd)
-- Groq (no TTS, fallback to OpenAI)
-- ElevenLabs (multilingual voices)
-- Google TTS (text-to-speech)
-
-### Per-Request Override
-
-Every LangGraph invocation accepts a `config` parameter to override models:
-
-```python
-result = await graph.ainvoke(
-    input={...},
-    config={
-        "configurable": {
-            "model_override": "anthropic/claude-opus-4"  # Use Claude instead
-        }
-    }
-)
-```
-
----
-
-## Design Patterns
-
-### 1. **Domain-Driven Design (DDD)**
-
-**Domain Objects** (`open_notebook/domain/`):
-- `Notebook`: Research container with relationships to sources/notes
-- `Source`: Content item (PDF, URL, text) with embeddings
-- `Note`: User-created or AI-generated research note
-- `ChatSession`: Conversation history for a notebook
-- `Transformation`: Custom rule for extracting insights
-
-**Repository Pattern**:
-- Database access layer (`open_notebook/database/repository.py`)
-- `repo_query()`: Execute SurrealQL queries
-- `repo_create()`: Insert records
-- `repo_upsert()`: Merge records
-- `repo_delete()`: Remove records
-
-**Entity Methods**:
-```python
-# Domain methods (business logic)
-notebook = await Notebook.get(id)
-await notebook.save()
-notes = await notebook.get_notes()
-sources = await notebook.get_sources()
-```
-
-### 2. **Async-First Architecture**
-
-**All I/O is async**:
-- Database queries: `await repo_query(...)`
-- LLM calls: `await model.ainvoke(...)`
-- File I/O: `await upload_file.read()`
-- Graph invocations: `await graph.ainvoke(...)`
-
-**Benefits**:
-- Non-blocking request handling (FastAPI serves multiple concurrent requests)
-- Better resource utilization (I/O waiting doesn't block CPU)
-- Natural fit for Python async/await syntax
-
-**Example**:
-```python
-@router.post("/sources")
-async def create_source(source_data: SourceCreate):
-    # All operations are non-blocking
-    source = Source(title=source_data.title)
-    await source.save()  # async database operation
-    await graph.ainvoke({...})  # async LangGraph invocation
-    return SourceResponse(...)
-```
-
-### 3. **Service Pattern**
-
-Services orchestrate domain objects, repositories, and workflows:
-
-```python
-# api/notebook_service.py
-class NotebookService:
-    async def get_notebook_with_stats(notebook_id: str):
-        notebook = await Notebook.get(notebook_id)
-        sources = await notebook.get_sources()
-        notes = await notebook.get_notes()
-        return {
-            "notebook": notebook,
-            "source_count": len(sources),
-            "note_count": len(notes),
-        }
-```
-
-**Responsibilities**:
-- Validate inputs (Pydantic)
-- Orchestrate database operations
-- Invoke workflows (LangGraph graphs)
-- Handle errors and return appropriate status codes
-- Log operations
-
-### 4. **Streaming Pattern**
-
-For long-running operations (ask workflow, podcast generation), stream results as Server-Sent Events:
-
-```python
-@router.post("/ask", response_class=StreamingResponse)
-async def ask(request: AskRequest):
-    async def stream_response():
-        async for chunk in ask_graph.astream(input={...}):
-            yield f"data: {json.dumps(chunk)}\n\n"
-    return StreamingResponse(stream_response(), media_type="text/event-stream")
-```
-
-### 5. **Job Queue Pattern**
-
-For async background tasks (source processing), use Surreal-Commands job queue:
-
-```python
-# Submit job
-command_id = await CommandService.submit_command_job(
-    app="open_notebook",
-    command="process_source",
-    input={...}
-)
-
-# Poll status
-status = await source.get_status()
-```
-
----
-
-## Service Communication Patterns
-
-### Frontend → API
-
-1. **REST requests** (HTTP GET/POST/PUT/DELETE)
-2. **JSON request/response bodies**
-3. **Standard HTTP status codes** (200, 400, 404, 500)
-4. **Optional streaming** (Server-Sent Events for long operations)
-
-**Example**:
-```typescript
-// Frontend
-const response = await fetch("http://localhost:5055/sources", {
-  method: "POST",
-  body: formData,  // multipart/form-data for file upload
-});
-const source = await response.json();
-```
-
-### API → SurrealDB
-
-1. **SurrealQL queries** (similar to SQL)
-2. **Async driver** with connection pooling
-3. **Type-safe record IDs** (record_id syntax)
-4. **Transaction support** for multi-step operations
-
-**Example**:
-```python
-# API
-result = await repo_query(
-    "SELECT * FROM source WHERE notebook = $notebook_id",
-    {"notebook_id": ensure_record_id(notebook_id)}
-)
-```
-
-### API → AI Providers (via Esperanto)
-
-1. **Esperanto unified interface**
-2. **Per-request provider override**
-3. **Automatic fallback on failure**
-4. **Token counting and cost estimation**
-
-**Example**:
-```python
-# API
-model = await provision_langchain_model(task="chat")
-response = await model.ainvoke({"input": prompt})
-```
-
-### API → Job Queue (Surreal-Commands)
-
-1. **Async job submission**
-2. **Fire-and-forget pattern**
-3. **Status polling via `/commands/{id}` endpoint**
-4. **Job completion callbacks (optional)**
-
-**Example**:
-```python
-# Submit async source processing
-command_id = await CommandService.submit_command_job(...)
-
-# Client polls status
-response = await fetch(f"http://localhost:5055/commands/{command_id}")
-status = await response.json()  # returns { status: "running|queued|completed|failed" }
-```
-
----
-
-## Database Schema Overview
-
-### Core Schema Structure
-
-**Tables** (20+):
-- Notebooks (with soft-delete via `archived` flag)
-- Sources (content + metadata)
-- SourceEmbeddings (vector chunks)
-- Notes (user-created + AI-generated)
-- ChatSessions (conversation history)
-- Transformations (custom rules)
-- SourceInsights (transformation outputs)
-- Relationships (notebook→source, notebook→note)
-
-**Migrations**:
-- Automatic on API startup
-- Located in `/migrations/` directory
-- Numbered sequentially (001_*.surql, 002_*.surql, etc)
-- Tracked in `_sbl_migrations` table
-- Rollback via `_down.surql` files (manual)
-
-### Relationship Model
-
-**Graph Relationships**:
-```
-Notebook
-  ← reference ← Source (many:many)
-  ← artifact ← Note (many:many)
-
-Source
-  → source_embedding (one:many)
-  → source_insight (one:many)
-  → embedding (via source_embedding)
-
-ChatSession
-  → messages (JSON array in database)
-  → notebook_id (reference to Notebook)
-
-Transformation
-  → source_insight (one:many)
-```
-
-**Query Example** (get all sources in a notebook with counts):
-```sql
-SELECT id, title,
-  count(<-reference.in) as note_count,
-  count(<-embedding.in) as embedded_chunks
-FROM source
-WHERE notebook = $notebook_id
-ORDER BY updated DESC
-```
-
----
-
-## Key Architectural Decisions
-
-### 1. **Async Throughout**
-
-All I/O operations are non-blocking to maximize concurrency and responsiveness.
-
-**Trade-off**: Slightly more complex code (async/await syntax) vs. high throughput.
-
-### 2. **Multi-Provider from Day 1**
-
-Built-in support for 8+ AI providers prevents vendor lock-in.
-
-**Trade-off**: Added complexity in ModelManager vs. flexibility and cost optimization.
-
-### 3. **Graph-First Workflows**
-
-LangGraph state machines for complex multi-step operations (ask, chat, transformations).
-
-**Trade-off**: Steeper learning curve vs. maintainable, debuggable workflows.
-
-### 4. **Self-Hosted Database**
-
-SurrealDB for graph + vector search in one system (no external dependencies).
-
-**Trade-off**: Operational responsibility vs. simplified architecture and cost savings.
-
-### 5. **Job Queue for Long-Running Tasks**
-
-Async job submission (source processing, podcast generation) prevents request timeouts.
-
-**Trade-off**: Eventual consistency vs. responsive user experience.
-
----
-
-## Important Quirks & Gotchas
-
-### API Startup
-
-- **Migrations run automatically** on every startup; check logs for errors
-- **SurrealDB must be running** before starting API (connection test in lifespan)
-- **Auth middleware is basic** (password-only); upgrade to OAuth/JWT for production
-
-### Database Operations
-
-- **Record IDs use SurrealDB syntax** (table:id format, e.g., "notebook:abc123")
-- **ensure_record_id()** helper prevents malformed IDs
-- **Soft deletes** via `archived` field (data not removed, just marked inactive)
-- **Timestamps in ISO 8601 format** (created, updated fields)
-
-### LangGraph Workflows
-
-- **State persistence** via SqliteSaver in `/data/sqlite-db/`
-- **No built-in timeout**; long workflows may block requests (use streaming for UX)
-- **Model fallback** automatic if primary provider unavailable
-- **Checkpoint IDs** must be unique per session (avoid collisions)
-
-### AI Provider Integration
-
-- **Esperanto library** handles all provider APIs (no direct API calls)
-- **Per-request override** via RunnableConfig (temporary, not persistent)
-- **Cost estimation** via token counting (not 100% accurate, use for guidance)
-- **Fallback logic** tries cheaper models if primary fails
-
-### File Uploads
-
-- **Stored in `/data/uploads/`** directory (not database)
-- **Unique filename generation** prevents overwrites (counter suffix)
-- **Content-core library** extracts text from 50+ file types
-- **Large files** may block API briefly (sync content extraction)
-
----
-
-## Performance Considerations
-
-### Optimization Strategies
-
-1. **Connection Pooling**: SurrealDB async driver with configurable pool size
-2. **Query Caching**: TanStack Query on frontend (client-side caching)
-3. **Embedding Reuse**: Vector search uses pre-computed embeddings
-4. **Chunking**: Sources split into chunks for better search relevance
-5. **Async Operations**: Non-blocking I/O for high concurrency
-6. **Lazy Loading**: Frontend requests only needed data (pagination)
-
-### Bottlenecks
-
-1. **LLM Calls**: Latency depends on provider (typically 1-30 seconds)
-2. **Embedding Generation**: Time proportional to content size and provider
-3. **Vector Search**: Similarity computation over all embeddings
-4. **Content Extraction**: Sync operation in source processing
-
-### Monitoring
-
-- **API Logs**: Check loguru output for errors and slow operations
-- **Database Queries**: SurrealDB metrics available via admin UI
-- **Token Usage**: Estimated via `estimate_tokens()` utility
-- **Job Status**: Poll `/commands/{id}` for async operations
-
----
-
-## Extension Points
-
-### Adding a New Workflow
-
-1. Create `open_notebook/graphs/workflow_name.py`
-2. Define StateDict and node functions
-3. Build graph with `.add_node()` / `.add_edge()`
-4. Create service in `api/workflow_service.py`
-5. Register router in `api/main.py`
-6. Add tests in `tests/test_workflow.py`
-
-### Adding a New Data Model
-
-1. Create model in `open_notebook/domain/model_name.py`
-2. Inherit from BaseModel (domain object)
-3. Implement `save()`, `get()`, `delete()` methods (CRUD)
-4. Add repository functions if complex queries needed
-5. Create database migration in `migrations/`
-6. Add API routes and models in `api/`
-
-### Adding a New AI Provider
-
-1. Configure Esperanto for new provider (see .env.example)
-2. ModelManager automatically detects via environment variables
-3. Override via per-request config (no code changes needed)
-4. Test fallback logic if provider unavailable
-
----
-
-## Deployment Considerations
-
-### Development
-
-- All services on localhost (3000, 5055, 8000)
-- Auto-reload on file changes (Next.js, FastAPI)
-- Hot-reload database migrations
-- Open API docs at http://localhost:5055/docs
-
-### Production
-
-- **Frontend**: Deploy to Vercel, Netlify, or Docker
-- **API**: Docker container (see Dockerfile)
-- **Database**: SurrealDB container or managed service
-- **Environment**: Secure .env file with API keys
-- **SSL/TLS**: Reverse proxy (Nginx, CloudFlare)
-- **Rate Limiting**: Add at proxy layer
-- **Auth**: Replace PasswordAuthMiddleware with OAuth/JWT
-- **Monitoring**: Log aggregation (CloudWatch, DataDog, etc)
-
----
-
-## Summary
-
-Open Notebook's architecture provides a solid foundation for privacy-focused, AI-powered research. The separation of concerns (frontend/API/database), async-first design, and multi-provider flexibility enable rapid development and easy deployment. LangGraph workflows orchestrate complex AI tasks, while Esperanto abstracts provider details. The result is a scalable, maintainable system that puts users in control of their data and AI provider choice.

docs/7-DEVELOPMENT/code-standards.md

@@ -1,375 +0,0 @@
-# Code Standards
-
-This document outlines coding standards and best practices for Open Notebook contributions. All code should follow these guidelines to ensure consistency, readability, and maintainability.
-
-## Python Standards
-
-### Code Formatting
-
-We follow **PEP 8** with some specific guidelines:
-
-- Use **Ruff** for linting and formatting
-- Maximum line length: **88 characters**
-- Use **double quotes** for strings
-- Use **trailing commas** in multi-line structures
-
-### Type Hints
-
-Always use type hints for function parameters and return values:
-
-```python
-from typing import List, Optional, Dict, Any
-from pydantic import BaseModel
-
-async def process_content(
-    content: str,
-    options: Optional[Dict[str, Any]] = None
-) -> ProcessedContent:
-    """Process content with optional configuration."""
-    # Implementation
-```
-
-### Async/Await Patterns
-
-Use async/await consistently throughout the codebase:
-
-```python
-# Good
-async def fetch_data(url: str) -> Dict[str, Any]:
-    async with aiohttp.ClientSession() as session:
-        async with session.get(url) as response:
-            return await response.json()
-
-# Bad - mixing sync and async
-def fetch_data(url: str) -> Dict[str, Any]:
-    loop = asyncio.get_event_loop()
-    return loop.run_until_complete(async_fetch(url))
-```
-
-### Error Handling
-
-Use structured error handling with custom exceptions:
-
-```python
-from open_notebook.exceptions import DatabaseOperationError, InvalidInputError
-
-async def create_notebook(name: str, description: str) -> Notebook:
-    """Create a new notebook with validation."""
-    if not name.strip():
-        raise InvalidInputError("Notebook name cannot be empty")
-
-    try:
-        notebook = Notebook(name=name, description=description)
-        await notebook.save()
-        return notebook
-    except Exception as e:
-        raise DatabaseOperationError(f"Failed to create notebook: {str(e)}")
-```
-
-### Documentation (Google-style Docstrings)
-
-Use Google-style docstrings for all functions, classes, and modules:
-
-```python
-async def vector_search(
-    query: str,
-    limit: int = 10,
-    minimum_score: float = 0.2
-) -> List[SearchResult]:
-    """Perform vector search across embedded content.
-
-    Args:
-        query: Search query string
-        limit: Maximum number of results to return
-        minimum_score: Minimum similarity score for results
-
-    Returns:
-        List of search results sorted by relevance score
-
-    Raises:
-        InvalidInputError: If query is empty or limit is invalid
-        DatabaseOperationError: If search operation fails
-    """
-    # Implementation
-```
-
-#### Module Docstrings
-```python
-"""
-Notebook domain model and operations.
-
-This module contains the core Notebook class and related operations for
-managing research notebooks within the Open Notebook system.
-"""
-```
-
-#### Class Docstrings
-```python
-class Notebook(BaseModel):
-    """A research notebook containing sources, notes, and chat sessions.
-
-    Notebooks are the primary organizational unit in Open Notebook, allowing
-    users to group related research materials and maintain separate contexts
-    for different projects.
-
-    Attributes:
-        name: The notebook's display name
-        description: Optional description of the notebook's purpose
-        archived: Whether the notebook is archived (default: False)
-        created: Timestamp of creation
-        updated: Timestamp of last update
-    """
-```
-
-#### Function Docstrings
-```python
-async def create_notebook(
-    name: str,
-    description: str = "",
-    user_id: Optional[str] = None
-) -> Notebook:
-    """Create a new notebook with validation.
-
-    Args:
-        name: The notebook name (required, non-empty)
-        description: Optional notebook description
-        user_id: Optional user ID for multi-user deployments
-
-    Returns:
-        The created notebook instance
-
-    Raises:
-        InvalidInputError: If name is empty or invalid
-        DatabaseOperationError: If creation fails
-
-    Example:
-        ```python
-        notebook = await create_notebook(
-            name="AI Research",
-            description="Research on AI applications"
-        )
-        ```
-    """
-```
-
-## FastAPI Standards
-
-### Router Organization
-
-Organize endpoints by domain:
-
-```python
-# api/routers/notebooks.py
-from fastapi import APIRouter, HTTPException, Query
-from typing import List, Optional
-
-router = APIRouter()
-
-@router.get("/notebooks", response_model=List[NotebookResponse])
-async def get_notebooks(
-    archived: Optional[bool] = Query(None, description="Filter by archived status"),
-    order_by: str = Query("updated desc", description="Order by field and direction"),
-):
-    """Get all notebooks with optional filtering and ordering."""
-    # Implementation
-```
-
-### Request/Response Models
-
-Use Pydantic models for validation:
-
-```python
-from pydantic import BaseModel, Field
-from typing import Optional
-
-class NotebookCreate(BaseModel):
-    name: str = Field(..., description="Name of the notebook", min_length=1)
-    description: str = Field(default="", description="Description of the notebook")
-
-class NotebookResponse(BaseModel):
-    id: str
-    name: str
-    description: str
-    archived: bool
-    created: str
-    updated: str
-```
-
-### Error Handling
-
-Use consistent error responses:
-
-```python
-from fastapi import HTTPException
-from loguru import logger
-
-try:
-    result = await some_operation()
-    return result
-except InvalidInputError as e:
-    raise HTTPException(status_code=400, detail=str(e))
-except DatabaseOperationError as e:
-    logger.error(f"Database error: {str(e)}")
-    raise HTTPException(status_code=500, detail="Internal server error")
-```
-
-### API Documentation
-
-Use FastAPI's automatic documentation features:
-
-```python
-@router.post(
-    "/notebooks",
-    response_model=NotebookResponse,
-    summary="Create a new notebook",
-    description="Create a new notebook with the specified name and description.",
-    responses={
-        201: {"description": "Notebook created successfully"},
-        400: {"description": "Invalid input data"},
-        500: {"description": "Internal server error"}
-    }
-)
-async def create_notebook(notebook: NotebookCreate):
-    """Create a new notebook."""
-    # Implementation
-```
-
-## Database Standards
-
-### SurrealDB Patterns
-
-Use the repository pattern consistently:
-
-```python
-from open_notebook.database.repository import repo_create, repo_query, repo_update
-
-# Create records
-async def create_notebook(data: Dict[str, Any]) -> Dict[str, Any]:
-    """Create a new notebook record."""
-    return await repo_create("notebook", data)
-
-# Query with parameters
-async def find_notebooks_by_user(user_id: str) -> List[Dict[str, Any]]:
-    """Find notebooks for a specific user."""
-    return await repo_query(
-        "SELECT * FROM notebook WHERE user_id = $user_id",
-        {"user_id": user_id}
-    )
-
-# Update records
-async def update_notebook(notebook_id: str, data: Dict[str, Any]) -> Dict[str, Any]:
-    """Update a notebook record."""
-    return await repo_update("notebook", notebook_id, data)
-```
-
-### Schema Management
-
-Use migrations for schema changes:
-
-```surrealql
--- migrations/8.surrealql
-DEFINE TABLE IF NOT EXISTS new_feature SCHEMAFULL;
-DEFINE FIELD IF NOT EXISTS name ON TABLE new_feature TYPE string;
-DEFINE FIELD IF NOT EXISTS description ON TABLE new_feature TYPE option<string>;
-DEFINE FIELD IF NOT EXISTS created ON TABLE new_feature TYPE datetime DEFAULT time::now();
-DEFINE FIELD IF NOT EXISTS updated ON TABLE new_feature TYPE datetime DEFAULT time::now();
-```
-
-## TypeScript Standards
-
-### Basic Guidelines
-
-Follow TypeScript best practices:
-
-- Use strict mode enabled in `tsconfig.json`
-- Use proper type annotations for all variables and functions
-- Avoid using `any` type unless absolutely necessary
-- Use `interface` for object shapes, `type` for unions and other advanced types
-
-### Component Structure
-
-- Use functional components with hooks
-- Keep components focused and single-responsibility
-- Extract reusable logic into custom hooks
-- Use proper TypeScript types for props
-
-### Error Handling
-
-- Handle errors explicitly
-- Provide meaningful error messages
-- Log errors appropriately
-- Don't suppress errors silently
-
-## Code Quality Tools
-
-We use these tools to maintain code quality:
-
-- **Ruff**: Linting and code formatting
-  - Run with: `uv run ruff check . --fix`
-  - Format with: `uv run ruff format .`
-
-- **MyPy**: Static type checking
-  - Run with: `uv run python -m mypy .`
-
-- **Pytest**: Testing framework
-  - Run with: `uv run pytest`
-
-## Common Patterns
-
-### Async Database Operations
-
-```python
-async def get_notebook_with_sources(notebook_id: str) -> Notebook:
-    """Retrieve notebook with all related sources."""
-    notebook_data = await repo_query(
-        "SELECT * FROM notebook WHERE id = $id",
-        {"id": notebook_id}
-    )
-    if not notebook_data:
-        raise InvalidInputError(f"Notebook {notebook_id} not found")
-
-    sources_data = await repo_query(
-        "SELECT * FROM source WHERE notebook_id = $notebook_id",
-        {"notebook_id": notebook_id}
-    )
-
-    return Notebook(
-        **notebook_data[0],
-        sources=[Source(**s) for s in sources_data]
-    )
-```
-
-### Model Validation
-
-```python
-from pydantic import BaseModel, validator
-
-class NotebookInput(BaseModel):
-    name: str
-    description: str = ""
-
-    @validator('name')
-    def name_not_empty(cls, v):
-        if not v.strip():
-            raise ValueError('Name cannot be empty')
-        return v.strip()
-```
-
-## Code Review Checklist
-
-Before submitting code for review, ensure:
-
-- [ ] Code follows PEP 8 / TypeScript best practices
-- [ ] Type hints are present for all functions
-- [ ] Docstrings are complete and accurate
-- [ ] Error handling is appropriate
-- [ ] Tests are included and passing
-- [ ] No debug code (console.logs, print statements) left behind
-- [ ] Commit messages are clear and follow conventions
-- [ ] Documentation is updated if needed
-
----
-
-**See also:**
-- [Testing Guide](testing.md) - How to write tests
-- [Contributing Guide](contributing.md) - Overall contribution workflow

docs/7-DEVELOPMENT/contributing.md

@@ -1,201 +0,0 @@
-# Contributing to Open Notebook
-
-Thank you for your interest in contributing to Open Notebook! We welcome contributions from developers of all skill levels. This guide will help you understand our contribution workflow and what makes a good contribution.
-
-## 🚨 Issue-First Workflow
-
-**To maintain project coherence and avoid wasted effort, please follow this process:**
-
-1. **Create an issue first** - Before writing any code, create an issue describing the bug or feature
-2. **Propose your solution** - Explain how you plan to implement the fix or feature
-3. **Wait for assignment** - A maintainer will review and assign the issue to you if approved
-4. **Only then start coding** - This ensures your work aligns with the project's vision and architecture
-
-**Why this process?**
-- Prevents duplicate work
-- Ensures solutions align with our architecture and design principles
-- Saves your time by getting feedback before coding
-- Helps maintainers manage the project direction
-
-> ⚠️ **Pull requests without an assigned issue may be closed**, even if the code is good. We want to respect your time by making sure work is aligned before it starts.
-
-## Code of Conduct
-
-By participating in this project, you are expected to uphold our Code of Conduct. Be respectful, constructive, and collaborative.
-
-## How Can I Contribute?
-
-### Reporting Bugs
-
-1. **Search existing issues** - Check if the bug was already reported
-2. **Create a bug report** - Use the [Bug Report template](https://github.com/lfnovo/open-notebook/issues/new?template=bug_report.yml)
-3. **Provide details** - Include:
-   - Steps to reproduce
-   - Expected vs actual behavior
-   - Logs, screenshots, or error messages
-   - Your environment (OS, Docker version, Open Notebook version)
-4. **Indicate if you want to fix it** - Check the "I would like to work on this" box if you're interested
-
-### Suggesting Features
-
-1. **Search existing issues** - Check if the feature was already suggested
-2. **Create a feature request** - Use the [Feature Request template](https://github.com/lfnovo/open-notebook/issues/new?template=feature_request.yml)
-3. **Explain the value** - Describe why this feature would be helpful
-4. **Propose implementation** - If you have ideas on how to implement it, share them
-5. **Indicate if you want to build it** - Check the "I would like to work on this" box if you're interested
-
-### Contributing Code (Pull Requests)
-
-**IMPORTANT: Follow the issue-first workflow above before starting any PR**
-
-Once your issue is assigned:
-
-1. **Fork the repo** and create your branch from `main`
-2. **Understand our vision and principles** - Read [design-principles.md](design-principles.md) to understand what guides our decisions
-3. **Follow our architecture** - Refer to the architecture documentation to understand project structure
-4. **Write quality code** - Follow the standards outlined in [code-standards.md](code-standards.md)
-5. **Test your changes** - See [testing.md](testing.md) for test guidelines
-6. **Update documentation** - If you changed functionality, update the relevant docs
-7. **Create your PR**:
-   - Reference the issue number (e.g., "Fixes #123")
-   - Describe what changed and why
-   - Include screenshots for UI changes
-   - Keep PRs focused - one issue per PR
-
-### What Makes a Good Contribution?
-
-✅ **We love PRs that:**
-- Solve a real problem described in an issue
-- Follow our architecture and coding standards
-- Include tests and documentation
-- Are well-scoped (focused on one thing)
-- Have clear commit messages
-
-❌ **We may close PRs that:**
-- Don't have an associated approved issue
-- Introduce breaking changes without discussion
-- Conflict with our architectural vision
-- Lack tests or documentation
-- Try to solve multiple unrelated problems
-
-## Git Commit Messages
-
-- Use the present tense ("Add feature" not "Added feature")
-- Use the imperative mood ("Move cursor to..." not "Moves cursor to...")
-- Limit the first line to 72 characters or less
-- Reference issues and pull requests liberally after the first line
-
-## Development Workflow
-
-### Branch Strategy
-
-We use a **feature branch workflow**:
-
-1. **Main Branch**: `main` - production-ready code
-2. **Feature Branches**: `feature/description` - new features
-3. **Bug Fixes**: `fix/description` - bug fixes
-4. **Documentation**: `docs/description` - documentation updates
-
-### Making Changes
-
-1. **Create a feature branch**:
-```bash
-git checkout -b feature/amazing-new-feature
-```
-
-2. **Make your changes** following our coding standards
-
-3. **Test your changes**:
-```bash
-# Run tests
-uv run pytest
-
-# Run linting
-uv run ruff check .
-
-# Run formatting
-uv run ruff format .
-```
-
-4. **Commit your changes**:
-```bash
-git add .
-git commit -m "feat: add amazing new feature"
-```
-
-5. **Push and create PR**:
-```bash
-git push origin feature/amazing-new-feature
-# Then create a Pull Request on GitHub
-```
-
-### Keeping Your Fork Updated
-
-```bash
-# Fetch upstream changes
-git fetch upstream
-
-# Switch to main and merge
-git checkout main
-git merge upstream/main
-
-# Push to your fork
-git push origin main
-```
-
-## Pull Request Process
-
-When you create a pull request:
-
-1. **Link your issue** - Reference the issue number in PR description
-2. **Describe your changes** - Explain what changed and why
-3. **Provide test evidence** - Screenshots, test results, or logs
-4. **Check PR template** - Ensure you've completed all required sections
-5. **Wait for review** - A maintainer will review your PR within a week
-
-### PR Review Expectations
-
-- Code review feedback is about the code, not the person
-- Be open to suggestions and alternative approaches
-- Address review comments with clarity and respect
-- Ask questions if feedback is unclear
-
-## Current Priority Areas
-
-We're actively looking for contributions in these areas:
-
-1. **Frontend Enhancement** - Help improve the Next.js/React UI with real-time updates and better UX
-2. **Testing** - Expand test coverage across all components
-3. **Performance** - Async processing improvements and caching
-4. **Documentation** - API examples and user guides
-5. **Integrations** - New content sources and AI providers
-
-## Getting Help
-
-### Community Support
-
-- **Discord**: [Join our Discord server](https://discord.gg/37XJPXfz2w) for real-time help
-- **GitHub Discussions**: For longer-form questions and ideas
-- **GitHub Issues**: For bug reports and feature requests
-
-### Documentation References
-
-- [Design Principles](design-principles.md) - Understanding our project vision
-- [Code Standards](code-standards.md) - Coding guidelines by language
-- [Testing Guide](testing.md) - How to write tests
-- [Development Setup](development-setup.md) - Getting started locally
-
-## Recognition
-
-We recognize contributions through:
-
-- **GitHub credits** on releases
-- **Community recognition** in Discord
-- **Contribution statistics** in project analytics
-- **Maintainer consideration** for active contributors
-
----
-
-Thank you for contributing to Open Notebook! Your contributions help make research more accessible and private for everyone.
-
-For questions about this guide or contributing in general, please reach out on [Discord](https://discord.gg/37XJPXfz2w) or open a GitHub Discussion.

docs/7-DEVELOPMENT/design-principles.md

@@ -1,351 +0,0 @@
-# Design Principles & Project Vision
-
-This document outlines the core principles, vision, and design philosophy that guide Open Notebook's development. All contributors should read and understand these principles before proposing changes or new features.
-
-## 🎯 Project Vision
-
-Open Notebook aims to be a **privacy-focused, self-hosted alternative to Google's Notebook LM** that empowers users to:
-
-1. **Own their research data** - Full control over where data lives and who can access it
-2. **Choose their AI providers** - Freedom to use any AI provider or run models locally
-3. **Customize their workflows** - Flexibility to adapt the tool to different research needs
-4. **Access their work anywhere** - Through web UI, API, or integrations
-
-### What Open Notebook IS
-
-- A **research assistant** for managing and understanding content
-- A **platform** that connects various AI providers
-- A **privacy-first** tool that keeps your data under your control
-- An **extensible system** with APIs and customization options
-
-### What Open Notebook IS NOT
-
-- A document editor (use Google Docs, Notion, etc. for that)
-- A file storage system (use Dropbox, S3, etc. for that)
-- A general-purpose chatbot (use ChatGPT, Claude, etc. for that)
-- A replacement for your entire workflow (it's one tool in your toolkit)
-
-## 🏗️ Core Design Principles
-
-### 1. Privacy First
-
-**Principle**: User data and research should stay under user control by default.
-
-**In Practice**:
-- Self-hosted deployment is the primary use case
-- No telemetry or analytics without explicit opt-in
-- No hard dependency on specific cloud services
-- Clear documentation on what data goes where
-
-**Example Decisions**:
-- ✅ Support for local Ollama models
-- ✅ Configurable AI provider selection
-- ❌ Hard-coded cloud service integrations
-- ❌ Required external service dependencies
-
-### 2. Simplicity Over Features
-
-**Principle**: The tool should be easy to understand and use, even if it means fewer features.
-
-**In Practice**:
-- Clear, focused UI with well-defined sections
-- Sensible defaults that work for most users
-- Advanced features hidden behind optional configuration
-- Documentation written for non-technical users
-
-**Example Decisions**:
-- ✅ Three-column layout (Sources, Notes, Chat)
-- ✅ Default models that work out of the box
-- ❌ Overwhelming users with too many options upfront
-- ❌ Complex multi-step workflows for basic tasks
-
-### 3. API-First Architecture
-
-**Principle**: All functionality should be accessible via API, not just the UI.
-
-**In Practice**:
-- UI calls the same API that external clients use
-- Comprehensive REST API with OpenAPI documentation
-- No "UI-only" features that can't be automated
-- Clear separation between frontend and backend
-
-**Example Decisions**:
-- ✅ FastAPI backend with full API documentation
-- ✅ Consistent API patterns across all endpoints
-- ❌ Business logic in UI components
-- ❌ Features that require direct database access
-
-### 4. Multi-Provider Flexibility
-
-**Principle**: Users should never be locked into a single AI provider.
-
-**In Practice**:
-- Support for multiple AI providers through Esperanto library
-- Easy switching between providers and models
-- Clear documentation on provider limitations
-- Graceful degradation when providers are unavailable
-
-**Example Decisions**:
-- ✅ Support for 16+ AI providers
-- ✅ Per-feature model selection (chat, embeddings, TTS)
-- ❌ Features that only work with OpenAI
-- ❌ Hard-coded API endpoints for specific providers
-
-### 5. Extensibility Through Standards
-
-**Principle**: The system should be extensible through well-defined interfaces, not by forking.
-
-**In Practice**:
-- Plugin systems for transformations and commands
-- Standard data formats (JSON, Markdown)
-- Clear extension points in the architecture
-- Documentation for common customization scenarios
-
-**Example Decisions**:
-- ✅ Custom transformation templates
-- ✅ Background command system
-- ✅ Jinja2 prompt templates
-- ❌ Hard-coded business logic without extension points
-
-### 6. Async-First for Performance
-
-**Principle**: Long-running operations should not block the user interface or API.
-
-**In Practice**:
-- Async/await patterns throughout the backend
-- Background job processing for heavy workloads
-- Status updates and progress tracking
-- Graceful handling of slow AI provider responses
-
-**Example Decisions**:
-- ✅ AsyncIO for database operations
-- ✅ Background commands for podcast generation
-- ✅ Streaming responses for chat
-- ❌ Synchronous blocking operations in API endpoints
-
-## 🎨 UI/UX Principles
-
-### Focus on Content, Not Chrome
-
-- Minimize UI clutter and distractions
-- Content should occupy most of the screen space
-- Controls appear when needed, not always visible
-- Consistent layout across different views
-
-### Progressive Disclosure
-
-- Show simple options first, advanced options on demand
-- Don't overwhelm new users with every possible setting
-- Provide sensible defaults that work for 80% of use cases
-- Make power features discoverable but not intrusive
-
-### Responsive and Fast
-
-- UI should feel instant for common operations
-- Show loading states for operations that take time
-- Cache and optimize where possible
-- Degrade gracefully on slow connections
-
-## 🔧 Technical Principles
-
-### Clean Separation of Concerns
-
-**Layers should not leak**:
-- Frontend should not know about database structure
-- API should not contain business logic (delegate to domain layer)
-- Domain models should not know about HTTP requests
-- Database layer should not know about AI providers
-
-### Type Safety and Validation
-
-**Catch errors early**:
-- Use Pydantic models for all API boundaries
-- Type hints throughout Python codebase
-- TypeScript for frontend code
-- Validate data at system boundaries
-
-### Test What Matters
-
-**Focus on valuable tests**:
-- Test business logic and domain models
-- Test API contracts and error handling
-- Don't test framework code (FastAPI, React, etc.)
-- Integration tests for critical workflows
-
-### Database as Source of Truth
-
-**SurrealDB is our single source of truth**:
-- All state persisted in database
-- No business logic in database layer
-- Use SurrealDB features (record links, queries) appropriately
-- Schema migrations for all schema changes
-
-## 🚫 Anti-Patterns to Avoid
-
-### Feature Creep
-
-**What it looks like**:
-- Adding features because they're "cool" or "easy"
-- Building features for edge cases before common cases work well
-- Trying to be everything to everyone
-
-**Why we avoid it**:
-- Increases complexity and maintenance burden
-- Makes the tool harder to learn and use
-- Dilutes the core value proposition
-
-**Instead**:
-- Focus on core use cases
-- Say no to features that don't align with vision
-- Build extensibility points for edge cases
-
-### Premature Optimization
-
-**What it looks like**:
-- Optimizing code before knowing if it's slow
-- Complex caching strategies without measuring impact
-- Trading code clarity for marginal performance gains
-
-**Why we avoid it**:
-- Makes code harder to understand and maintain
-- Optimizes the wrong things
-- Wastes development time
-
-**Instead**:
-- Measure first, optimize second
-- Focus on algorithmic improvements
-- Profile before making performance changes
-
-### Over-Engineering
-
-**What it looks like**:
-- Building abstraction layers "in case we need them later"
-- Implementing design patterns for 3-line functions
-- Creating frameworks instead of solving problems
-
-**Why we avoid it**:
-- Increases cognitive load for contributors
-- Makes simple changes require touching many files
-- Hides the actual business logic
-
-**Instead**:
-- Start simple, refactor when patterns emerge
-- Optimize for readability and clarity
-- Use abstractions when they simplify, not complicate
-
-### Breaking Changes Without Migration Path
-
-**What it looks like**:
-- Changing database schema without migration scripts
-- Modifying API contracts without versioning
-- Removing features without deprecation warnings
-
-**Why we avoid it**:
-- Breaks existing installations
-- Frustrates users and contributors
-- Creates maintenance nightmares
-
-**Instead**:
-- Always provide migration scripts for schema changes
-- Deprecate before removing
-- Document breaking changes clearly
-
-## 🤝 Decision-Making Framework
-
-When evaluating new features or changes, ask:
-
-### 1. Does it align with our vision?
-- Does it help users own their research data?
-- Does it support privacy and self-hosting?
-- Does it fit our core use cases?
-
-### 2. Does it follow our principles?
-- Is it simple to use and understand?
-- Does it work via API?
-- Does it support multiple providers?
-- Can it be extended by users?
-
-### 3. Is the implementation sound?
-- Does it maintain separation of concerns?
-- Is it properly typed and validated?
-- Does it include tests?
-- Is it documented?
-
-### 4. What is the cost?
-- How much complexity does it add?
-- How much maintenance burden?
-- Does it introduce new dependencies?
-- Will it be used enough to justify the cost?
-
-### 5. Are there alternatives?
-- Can existing features solve this problem?
-- Can this be built as a plugin or extension?
-- Should this be a separate tool instead?
-
-## 📚 Examples of Principle-Driven Decisions
-
-### Why we migrated from Streamlit to Next.js
-
-**Principle**: API-First Architecture
-
-**Reasoning**:
-- Streamlit coupled UI and backend logic
-- Difficult to build external integrations
-- Limited control over API behavior
-- Next.js + FastAPI provides clear separation
-
-### Why we use Esperanto for AI providers
-
-**Principle**: Multi-Provider Flexibility
-
-**Reasoning**:
-- Abstracts provider-specific details
-- Easy to add new providers
-- Consistent interface across providers
-- No vendor lock-in
-
-### Why we have a Background Command System
-
-**Principle**: Async-First for Performance
-
-**Reasoning**:
-- Podcast generation takes minutes
-- Users shouldn't wait for long operations
-- Need status tracking and error handling
-- Supports future batch operations
-
-### Why we support Local Ollama
-
-**Principle**: Privacy First
-
-**Reasoning**:
-- Enables fully offline operation
-- No data sent to external services
-- Free for users after hardware cost
-- Aligns with self-hosted philosophy
-
-## 🔄 Evolution of Principles
-
-These principles are not set in stone. As the project grows and we learn from users, some principles may evolve. However, changes to core principles should be:
-
-1. **Well-justified** - Clear reasoning for why the change is needed
-2. **Discussed openly** - Community input on major changes
-3. **Documented** - Updated in this document with explanation
-4. **Gradual** - Not implemented as breaking changes when possible
-
----
-
-## For Contributors
-
-When proposing a feature or change:
-
-1. **Reference these principles** - Explain how your proposal aligns
-2. **Identify trade-offs** - Be honest about what you're trading for what
-3. **Suggest alternatives** - Show you've considered other approaches
-4. **Be open to feedback** - Maintainers may see concerns you don't
-
-**Remember**: A "no" to a feature isn't a judgment on you or your idea. It means we're staying focused on our core vision. We appreciate all contributions and ideas!
-
----
-
-**Questions about these principles?** Open a discussion on GitHub or join our [Discord](https://discord.gg/37XJPXfz2w).

docs/7-DEVELOPMENT/development-setup.md

@@ -1,409 +0,0 @@
-# Local Development Setup
-
-This guide walks you through setting up Open Notebook for local development. Follow these steps to get the full stack running on your machine.
-
-## Prerequisites
-
-Before you start, ensure you have the following installed:
-
-- **Python 3.11+** - Check with: `python --version`
-- **uv** (recommended) or **pip** - Install from: https://github.com/astral-sh/uv
-- **SurrealDB** - Via Docker or binary (see below)
-- **Docker** (optional) - For containerized database
-- **Node.js 18+** (optional) - For frontend development
-- **Git** - For version control
-
-## Step 1: Clone and Initial Setup
-
-```bash
-# Clone the repository
-git clone https://github.com/lfnovo/open-notebook.git
-cd open-notebook
-
-# Add upstream remote for keeping your fork updated
-git remote add upstream https://github.com/lfnovo/open-notebook.git
-```
-
-## Step 2: Install Python Dependencies
-
-```bash
-# Using uv (recommended)
-uv sync
-
-# Or using pip
-pip install -e .
-```
-
-## Step 3: Environment Variables
-
-Create a `.env` file in the project root with your configuration:
-
-```bash
-# Copy from example
-cp .env.example .env
-```
-
-Edit `.env` with your settings:
-
-```bash
-# Database
-SURREAL_URL=ws://localhost:8000/rpc
-SURREAL_USER=root
-SURREAL_PASSWORD=password
-SURREAL_NAMESPACE=open_notebook
-SURREAL_DATABASE=development
-
-# AI Providers (add your API keys)
-OPENAI_API_KEY=sk-...
-ANTHROPIC_API_KEY=sk-ant-...
-GOOGLE_API_KEY=AI...
-GROQ_API_KEY=gsk-...
-
-# Application
-APP_PASSWORD=  # Optional password protection
-DEBUG=true
-LOG_LEVEL=DEBUG
-```
-
-### AI Provider Keys
-
-You'll need at least one AI provider. Popular options:
-
-- **OpenAI** - https://platform.openai.com/api-keys
-- **Anthropic (Claude)** - https://console.anthropic.com/
-- **Google** - https://ai.google.dev/
-- **Groq** - https://console.groq.com/
-
-For local development, you can also use:
-- **Ollama** - Run locally without API keys (see "Local Ollama" below)
-
-## Step 4: Start SurrealDB
-
-### Option A: Using Docker (Recommended)
-
-```bash
-# Start SurrealDB in memory
-docker run -d --name surrealdb -p 8000:8000 \
-  surrealdb/surrealdb:v2 start \
-  --user root --pass password \
-  --bind 0.0.0.0:8000 memory
-
-# Or with persistent storage
-docker run -d --name surrealdb -p 8000:8000 \
-  -v surrealdb_data:/data \
-  surrealdb/surrealdb:v2 start \
-  --user root --pass password \
-  --bind 0.0.0.0:8000 file:/data/surreal.db
-```
-
-### Option B: Using Make
-
-```bash
-make database
-```
-
-### Option C: Using Docker Compose
-
-```bash
-docker compose up -d surrealdb
-```
-
-### Verify SurrealDB is Running
-
-```bash
-# Should show server information
-curl http://localhost:8000/
-```
-
-## Step 5: Run Database Migrations
-
-Database migrations run automatically when you start the API. The first startup will apply any pending migrations.
-
-To verify migrations manually:
-
-```bash
-# API will run migrations on startup
-uv run python -m api.main
-```
-
-Check the logs - you should see messages like:
-```
-Running migration 001_initial_schema
-Running migration 002_add_vectors
-...
-Migrations completed successfully
-```
-
-## Step 6: Start the API Server
-
-In a new terminal window:
-
-```bash
-# Terminal 2: Start API (port 5055)
-uv run --env-file .env uvicorn api.main:app --host 0.0.0.0 --port 5055
-
-# Or using the shortcut
-make api
-```
-
-You should see:
-```
-INFO:     Application startup complete
-INFO:     Uvicorn running on http://0.0.0.0:5055
-```
-
-### Verify API is Running
-
-```bash
-# Check health endpoint
-curl http://localhost:5055/health
-
-# View API documentation
-open http://localhost:5055/docs
-```
-
-## Step 7: Start the Frontend (Optional)
-
-If you want to work on the frontend, start Next.js in another terminal:
-
-```bash
-# Terminal 3: Start Next.js frontend (port 3000)
-cd frontend
-npm install  # First time only
-npm run dev
-```
-
-You should see:
-```
-> next dev
-  ▲ Next.js 15.x
-  - Local:        http://localhost:3000
-```
-
-### Access the Frontend
-
-Open your browser to: http://localhost:3000
-
-## Verification Checklist
-
-After setup, verify everything is working:
-
-- [ ] **SurrealDB**: `curl http://localhost:8000/` returns content
-- [ ] **API**: `curl http://localhost:5055/health` returns `{"status": "ok"}`
-- [ ] **API Docs**: `open http://localhost:5055/docs` works
-- [ ] **Database**: API logs show migrations completing
-- [ ] **Frontend** (optional): `http://localhost:3000` loads
-
-## Starting Services Together
-
-### Quick Start All Services
-
-```bash
-make start-all
-```
-
-This starts SurrealDB, API, and frontend in one command.
-
-### Individual Terminals (Recommended for Development)
-
-**Terminal 1 - Database:**
-```bash
-make database
-```
-
-**Terminal 2 - API:**
-```bash
-make api
-```
-
-**Terminal 3 - Frontend:**
-```bash
-cd frontend && npm run dev
-```
-
-## Development Tools Setup
-
-### Pre-commit Hooks (Optional but Recommended)
-
-Install git hooks to automatically check code quality:
-
-```bash
-uv run pre-commit install
-```
-
-Now your commits will be checked before they're made.
-
-### Code Quality Commands
-
-```bash
-# Lint Python code (auto-fix)
-make ruff
-# or: ruff check . --fix
-
-# Type check Python code
-make lint
-# or: uv run python -m mypy .
-
-# Run tests
-uv run pytest
-
-# Run tests with coverage
-uv run pytest --cov=open_notebook
-```
-
-## Common Development Tasks
-
-### Running Tests
-
-```bash
-# Run all tests
-uv run pytest
-
-# Run specific test file
-uv run pytest tests/test_notebooks.py
-
-# Run with coverage report
-uv run pytest --cov=open_notebook --cov-report=html
-```
-
-### Creating a Feature Branch
-
-```bash
-# Create and switch to new branch
-git checkout -b feature/my-feature
-
-# Make changes, then commit
-git add .
-git commit -m "feat: add my feature"
-
-# Push to your fork
-git push origin feature/my-feature
-```
-
-### Updating from Upstream
-
-```bash
-# Fetch latest changes
-git fetch upstream
-
-# Rebase your branch
-git rebase upstream/main
-
-# Push updated branch
-git push origin feature/my-feature -f
-```
-
-## Troubleshooting
-
-### "Connection refused" on SurrealDB
-
-**Problem**: API can't connect to SurrealDB
-
-**Solutions**:
-1. Check if SurrealDB is running: `docker ps | grep surrealdb`
-2. Verify URL in `.env`: Should be `ws://localhost:8000/rpc`
-3. Restart SurrealDB: `docker stop surrealdb && docker rm surrealdb`
-4. Then restart with: `docker run -d --name surrealdb -p 8000:8000 surrealdb/surrealdb:v2 start --user root --pass password --bind 0.0.0.0:8000 memory`
-
-### "Address already in use"
-
-**Problem**: Port 5055 or 3000 is already in use
-
-**Solutions**:
-```bash
-# Find process using port
-lsof -i :5055  # Check port 5055
-
-# Kill process (macOS/Linux)
-kill -9 <PID>
-
-# Or use different port
-uvicorn api.main:app --port 5056
-```
-
-### Module not found errors
-
-**Problem**: Import errors when running API
-
-**Solutions**:
-```bash
-# Reinstall dependencies
-uv sync
-
-# Or with pip
-pip install -e .
-```
-
-### Database migration failures
-
-**Problem**: API fails to start with migration errors
-
-**Solutions**:
-1. Check SurrealDB is running: `curl http://localhost:8000/`
-2. Check credentials in `.env` match your SurrealDB setup
-3. Check logs for specific migration error: `make api 2>&1 | grep -i migration`
-4. Verify database exists: Check SurrealDB console at http://localhost:8000/
-
-### Migrations not applying
-
-**Problem**: Database schema seems outdated
-
-**Solutions**:
-1. Restart API - migrations run on startup: `make api`
-2. Check logs show "Migrations completed successfully"
-3. Verify `/migrations/` folder exists and has files
-4. Check SurrealDB is writable and not in read-only mode
-
-## Optional: Local Ollama Setup
-
-For testing with local AI models:
-
-```bash
-# Install Ollama from https://ollama.ai
-
-# Pull a model (e.g., Mistral 7B)
-ollama pull mistral
-
-# Add to .env
-OLLAMA_BASE_URL=http://localhost:11434
-```
-
-Then in your code, you can use Ollama through the Esperanto library.
-
-## Optional: Docker Development Environment
-
-Run entire stack in Docker:
-
-```bash
-# Start all services
-docker compose --profile multi up
-
-# Logs
-docker compose logs -f
-
-# Stop services
-docker compose down
-```
-
-## Next Steps
-
-After setup is complete:
-
-1. **Read the Contributing Guide** - [contributing.md](contributing.md)
-2. **Explore the Architecture** - Check the documentation
-3. **Find an Issue** - Look for "good first issue" on GitHub
-4. **Set Up Pre-commit** - Install git hooks for code quality
-5. **Join Discord** - https://discord.gg/37XJPXfz2w
-
-## Getting Help
-
-If you get stuck:
-
-- **Discord**: [Join our server](https://discord.gg/37XJPXfz2w) for real-time help
-- **GitHub Issues**: Check existing issues for similar problems
-- **GitHub Discussions**: Ask questions in discussions
-- **Documentation**: See [code-standards.md](code-standards.md) and [testing.md](testing.md)
-
----
-
-**Ready to contribute?** Go to [contributing.md](contributing.md) for the contribution workflow.

docs/7-DEVELOPMENT/index.md

@@ -1,96 +0,0 @@
-# Development
-
-Welcome to the Open Notebook development documentation! Whether you're contributing code, understanding our architecture, or maintaining the project, you'll find guidance here.
-
-## 🎯 Pick Your Path
-
-### 👨‍💻 I Want to Contribute Code
-
-Start with **[Contributing Guide](contributing.md)** for the workflow, then check:
-- **[Quick Start](quick-start.md)** - Clone, install, verify in 5 minutes
-- **[Development Setup](development-setup.md)** - Complete local environment guide
-- **[Code Standards](code-standards.md)** - How to write code that fits our style
-- **[Testing](testing.md)** - How to write and run tests
-
-**First time?** Check out our [Contributing Guide](contributing.md) for the issue-first workflow.
-
----
-
-### 🏗️ I Want to Understand the Architecture
-
-**[Architecture Overview](architecture.md)** covers:
-- 3-tier system design
-- Tech stack and rationale
-- Key components and workflows
-- Design patterns we use
-
-For deeper dives, check `/open_notebook/` CLAUDE.md for component-specific guidance.
-
----
-
-### 👨‍🔧 I'm a Maintainer
-
-**[Maintainer Guide](maintainer-guide.md)** covers:
-- Issue triage and management
-- Pull request review process
-- Communication templates
-- Best practices
-
----
-
-## 📚 Quick Links
-
-| Document | For | Purpose |
-|---|---|---|
-| [Quick Start](quick-start.md) | New developers | Clone, install, and verify setup (5 min) |
-| [Development Setup](development-setup.md) | Local development | Complete environment setup guide |
-| [Contributing](contributing.md) | Code contributors | Workflow: issue → code → PR |
-| [Code Standards](code-standards.md) | Writing code | Style guides for Python, FastAPI, DB |
-| [Testing](testing.md) | Testing code | How to write and run tests |
-| [Architecture](architecture.md) | Understanding system | System design, tech stack, workflows |
-| [Design Principles](design-principles.md) | All developers | What guides our decisions |
-| [API Reference](api-reference.md) | Building integrations | Complete REST API documentation |
-| [Maintainer Guide](maintainer-guide.md) | Maintainers | Managing issues, PRs, releases |
-
----
-
-## 🚀 Current Development Priorities
-
-We're actively looking for help with:
-
-1. **Frontend Enhancement** - Improve Next.js/React UI with real-time updates
-2. **Performance** - Async processing and caching optimizations
-3. **Testing** - Expand test coverage across components
-4. **Documentation** - API examples and developer guides
-5. **Integrations** - New content sources and AI providers
-
-See GitHub Issues labeled `good first issue` or `help wanted`.
-
----
-
-## 💬 Getting Help
-
-- **Discord**: [Join our server](https://discord.gg/37XJPXfz2w) for real-time discussions
-- **GitHub Discussions**: For architecture questions
-- **GitHub Issues**: For bugs and features
-
-Don't be shy! We're here to help new contributors succeed.
-
----
-
-## 📖 Additional Resources
-
-### External Documentation
-- [FastAPI Docs](https://fastapi.tiangolo.com/)
-- [SurrealDB Docs](https://surrealdb.com/docs)
-- [LangChain Docs](https://python.langchain.com/)
-- [Next.js Docs](https://nextjs.org/docs)
-
-### Our Libraries
-- [Esperanto](https://github.com/lfnovo/esperanto) - Multi-provider AI abstraction
-- [Content Core](https://github.com/lfnovo/content-core) - Content processing
-- [Podcast Creator](https://github.com/lfnovo/podcast-creator) - Podcast generation
-
----
-
-Ready to get started? Head over to **[Quick Start](quick-start.md)**! 🎉

docs/7-DEVELOPMENT/maintainer-guide.md

@@ -1,408 +0,0 @@
-# Maintainer Guide
-
-This guide is for project maintainers to help manage contributions effectively while maintaining project quality and vision.
-
-## Table of Contents
-
-- [Issue Management](#issue-management)
-- [Pull Request Review](#pull-request-review)
-- [Common Scenarios](#common-scenarios)
-- [Communication Templates](#communication-templates)
-
-## Issue Management
-
-### When a New Issue is Created
-
-**1. Initial Triage** (within 24-48 hours)
-
-- Add appropriate labels:
-  - `bug`, `enhancement`, `documentation`, etc.
-  - `good first issue` for beginner-friendly tasks
-  - `needs-triage` until reviewed
-  - `help wanted` if you'd welcome community contributions
-
-- Quick assessment:
-  - Is it clear and well-described?
-  - Is it aligned with project vision? (See [design-principles.md](design-principles.md))
-  - Does it duplicate an existing issue?
-
-**2. Initial Response**
-
-```markdown
-Thanks for opening this issue! We'll review it and get back to you soon.
-
-[If it's a bug] In the meantime, have you checked our troubleshooting guide?
-
-[If it's a feature] You might find our [design principles](design-principles.md) helpful for understanding what we're building toward.
-```
-
-**3. Decision Making**
-
-Ask yourself:
-- Does this align with our [design principles](design-principles.md)?
-- Is this something we want in the core project, or better as a plugin/extension?
-- Do we have the capacity to support this feature long-term?
-- Will this benefit most users, or just a specific use case?
-
-**4. Issue Assignment**
-
-If the contributor checked "I am a developer and would like to work on this":
-
-**For Accepted Issues:**
-```markdown
-Great idea! This aligns well with our goals, particularly [specific design principle].
-
-I see you'd like to work on this. Before you start:
-
-1. Please share your proposed approach/solution
-2. Review our [Contributing Guide](contributing.md) and [Design Principles](design-principles.md)
-3. Once we agree on the approach, I'll assign this to you
-
-Looking forward to your thoughts!
-```
-
-**For Issues Needing Clarification:**
-```markdown
-Thanks for offering to work on this! Before we proceed, we need to clarify a few things:
-
-1. [Question 1]
-2. [Question 2]
-
-Once we have these details, we can discuss the best approach.
-```
-
-**For Issues Not Aligned with Vision:**
-```markdown
-Thank you for the suggestion and for offering to work on this!
-
-After reviewing against our [design principles](design-principles.md), we've decided not to pursue this in the core project because [specific reason].
-
-However, you might be able to achieve this through [alternative approach, if applicable].
-
-We appreciate your interest in contributing! Feel free to check out our [open issues](link) for other ways to contribute.
-```
-
-### Labels to Use
-
-**Priority:**
-- `priority: critical` - Security issues, data loss bugs
-- `priority: high` - Major functionality broken
-- `priority: medium` - Annoying bugs, useful features
-- `priority: low` - Nice to have, edge cases
-
-**Status:**
-- `needs-triage` - Not yet reviewed by maintainer
-- `needs-info` - Waiting for more information from reporter
-- `needs-discussion` - Requires community/team discussion
-- `ready` - Approved and ready to be worked on
-- `in-progress` - Someone is actively working on this
-- `blocked` - Cannot proceed due to external dependency
-
-**Type:**
-- `bug` - Something is broken
-- `enhancement` - New feature or improvement
-- `documentation` - Documentation improvements
-- `question` - General questions
-- `refactor` - Code cleanup/restructuring
-
-**Difficulty:**
-- `good first issue` - Good for newcomers
-- `help wanted` - Community contributions welcome
-- `advanced` - Requires deep codebase knowledge
-
-## Pull Request Review
-
-### Initial PR Review Checklist
-
-**Before diving into code:**
-
-- [ ] Is there an associated approved issue?
-- [ ] Does the PR reference the issue number?
-- [ ] Is the PR description clear about what changed and why?
-- [ ] Did the contributor check the relevant boxes in the PR template?
-- [ ] Are there tests? Screenshots (for UI changes)?
-
-**Red Flags** (may require closing PR):
-- No associated issue
-- Issue was not assigned to contributor
-- PR tries to solve multiple unrelated problems
-- Breaking changes without discussion
-- Conflicts with project vision
-
-### Code Review Process
-
-**1. High-Level Review**
-
-- Does the approach align with our architecture?
-- Is the solution appropriately scoped?
-- Are there simpler alternatives?
-- Does it follow our design principles?
-
-**2. Code Quality Review**
-
-Python:
-- [ ] Follows PEP 8
-- [ ] Has type hints
-- [ ] Has docstrings
-- [ ] Proper error handling
-- [ ] No security vulnerabilities
-
-TypeScript/Frontend:
-- [ ] Follows TypeScript best practices
-- [ ] Proper component structure
-- [ ] No console.logs left in production code
-- [ ] Accessible UI components
-
-**3. Testing Review**
-
-- [ ] Has appropriate test coverage
-- [ ] Tests are meaningful (not just for coverage percentage)
-- [ ] Tests pass locally and in CI
-- [ ] Edge cases are tested
-
-**4. Documentation Review**
-
-- [ ] Code is well-commented
-- [ ] Complex logic is explained
-- [ ] User-facing documentation updated (if applicable)
-- [ ] API documentation updated (if API changed)
-- [ ] Migration guide provided (if breaking change)
-
-### Providing Feedback
-
-**Positive Feedback** (important!):
-```markdown
-Thanks for this PR! I really like [specific thing they did well].
-
-[Feedback on what needs to change]
-```
-
-**Requesting Changes:**
-```markdown
-This is a great start! A few things to address:
-
-1. **[High-level concern]**: [Explanation and suggested approach]
-2. **[Code quality issue]**: [Specific example and fix]
-3. **[Testing gap]**: [What scenarios need coverage]
-
-Let me know if you have questions about any of this!
-```
-
-**Suggesting Alternative Approach:**
-```markdown
-I appreciate the effort you put into this! However, I'm concerned about [specific issue].
-
-Have you considered [alternative approach]? It might be better because [reasons].
-
-What do you think?
-```
-
-## Common Scenarios
-
-### Scenario 1: Good Code, Wrong Approach
-
-**Situation**: Contributor wrote quality code, but solved the problem in a way that doesn't fit our architecture.
-
-**Response:**
-```markdown
-Thank you for this PR! The code quality is great, and I can see you put thought into this.
-
-However, I'm concerned that this approach [specific architectural concern]. In our architecture, we [explain the pattern we follow].
-
-Would you be open to refactoring this to [suggested approach]? I'm happy to provide guidance on the specifics.
-
-Alternatively, if you don't have time for a refactor, I can take over and finish this up (with credit to you, of course).
-
-Let me know what you prefer!
-```
-
-### Scenario 2: PR Without Assigned Issue
-
-**Situation**: Contributor submitted PR without going through issue approval process.
-
-**Response:**
-```markdown
-Thanks for the PR! I appreciate you taking the time to contribute.
-
-However, to maintain project coherence, we require all PRs to be linked to an approved issue that was assigned to the contributor. This is explained in our [Contributing Guide](contributing.md).
-
-This helps us:
-- Ensure work aligns with project vision
-- Prevent duplicate efforts
-- Discuss approach before implementation
-
-Could you please:
-1. Create an issue describing this change
-2. Wait for it to be reviewed and assigned to you
-3. We can then reopen this PR or you can create a new one
-
-Sorry for the inconvenience - this process helps us manage the project effectively.
-```
-
-### Scenario 3: Feature Request Not Aligned with Vision
-
-**Situation**: Well-intentioned feature that doesn't fit project goals.
-
-**Response:**
-```markdown
-Thank you for this suggestion! I can see how this would be useful for [specific use case].
-
-After reviewing against our [design principles](design-principles.md), we've decided not to include this in the core project because [specific reason - e.g., "it conflicts with our 'Simplicity Over Features' principle" or "it would require dependencies that conflict with our privacy-first approach"].
-
-Some alternatives:
-- [If applicable] This could be built as a plugin/extension
-- [If applicable] This functionality might be achievable through [existing feature]
-- [If applicable] You might be interested in [other tool] which is designed for this use case
-
-We appreciate your contribution and hope you understand. Feel free to check our roadmap or open issues for other ways to contribute!
-```
-
-### Scenario 4: Contributor Ghosts After Feedback
-
-**Situation**: You requested changes, but contributor hasn't responded in 2+ weeks.
-
-**After 2 weeks:**
-```markdown
-Hey there! Just checking in on this PR. Do you have time to address the feedback, or would you like someone else to take over?
-
-No pressure either way - just want to make sure this doesn't fall through the cracks.
-```
-
-**After 1 month with no response:**
-```markdown
-Thanks again for starting this work! Since we haven't heard back, I'm going to close this PR for now.
-
-If you want to pick this up again in the future, feel free to reopen it or create a new PR. Alternatively, I'll mark the issue as available for someone else to work on.
-
-We appreciate your contribution!
-```
-
-Then:
-- Close the PR
-- Unassign the issue
-- Add `help wanted` label to the issue
-
-### Scenario 5: Breaking Changes Without Discussion
-
-**Situation**: PR introduces breaking changes that weren't discussed.
-
-**Response:**
-```markdown
-Thanks for this PR! However, I notice this introduces breaking changes that weren't discussed in the original issue.
-
-Breaking changes require:
-1. Prior discussion and approval
-2. Migration guide for users
-3. Deprecation period (when possible)
-4. Clear documentation of the change
-
-Could we discuss the breaking changes first? Specifically:
-- [What breaks and why]
-- [Who will be affected]
-- [Migration path]
-
-We may need to adjust the approach to minimize impact on existing users.
-```
-
-## Communication Templates
-
-### Closing a PR (Misaligned with Vision)
-
-```markdown
-Thank you for taking the time to contribute! We really appreciate it.
-
-After careful review, we've decided not to merge this PR because [specific reason related to design principles].
-
-This isn't a reflection on your code quality - it's about maintaining focus on our core goals as outlined in [design-principles.md](design-principles.md).
-
-We'd love to have you contribute in other ways! Check out:
-- Good first issues
-- Help wanted issues
-- Our roadmap
-
-Thanks again for your interest in Open Notebook!
-```
-
-### Closing a Stale Issue
-
-```markdown
-We're closing this issue due to inactivity. If this is still relevant, feel free to reopen it with updated information.
-
-Thanks!
-```
-
-### Asking for More Information
-
-```markdown
-Thanks for reporting this! To help us investigate, could you provide:
-
-1. [Specific information needed]
-2. [Logs, screenshots, etc.]
-3. [Steps to reproduce]
-
-This will help us understand the issue better and find a solution.
-```
-
-### Thanking a Contributor
-
-```markdown
-Merged!
-
-Thank you so much for this contribution, @username! [Specific thing they did well].
-
-This will be included in the next release.
-```
-
-## Best Practices
-
-### Be Kind and Respectful
-
-- Thank contributors for their time and effort
-- Assume good intentions
-- Be patient with newcomers
-- Explain *why*, not just *what*
-
-### Be Clear and Direct
-
-- Don't leave ambiguity about next steps
-- Be specific about what needs to change
-- Explain architectural decisions
-- Set clear expectations
-
-### Be Consistent
-
-- Apply the same standards to all contributors
-- Follow the process you've defined
-- Document decisions for future reference
-
-### Be Protective of Project Vision
-
-- It's okay to say "no"
-- Prioritize long-term maintainability
-- Don't accept features you can't support
-- Keep the project focused
-
-### Be Responsive
-
-- Respond to issues within 48 hours (even just to acknowledge)
-- Review PRs within a week when possible
-- Keep contributors updated on status
-- Close stale issues/PRs to keep things tidy
-
-## When in Doubt
-
-Ask yourself:
-1. Does this align with our [design principles](design-principles.md)?
-2. Will we be able to maintain this feature long-term?
-3. Does this benefit most users, or just an edge case?
-4. Is there a simpler alternative?
-5. Would I want to support this in 2 years?
-
-If you're unsure, it's perfectly fine to:
-- Ask for input from other maintainers
-- Start a discussion issue
-- Sleep on it before making a decision
-
----
-
-**Remember**: Good maintainership is about balancing openness to contributions with protection of project vision. You're not being mean by saying "no" to things that don't fit - you're being a responsible steward of the project.

docs/7-DEVELOPMENT/quick-start.md

@@ -1,128 +0,0 @@
-# Quick Start - Development
-
-Get Open Notebook running locally in 5 minutes.
-
-## Prerequisites
-
-- **Python 3.11+**
-- **Git**
-- **uv** (package manager) - install with `curl -LsSf https://astral.sh/uv/install.sh | sh`
-- **Docker** (optional, for SurrealDB)
-
-## 1. Clone the Repository (2 min)
-
-```bash
-# Fork the repository on GitHub first, then clone your fork
-git clone https://github.com/YOUR_USERNAME/open-notebook.git
-cd open-notebook
-
-# Add upstream remote for updates
-git remote add upstream https://github.com/lfnovo/open-notebook.git
-```
-
-## 2. Install Dependencies (2 min)
-
-```bash
-# Install Python dependencies
-uv sync
-
-# Verify uv is working
-uv --version
-```
-
-## 3. Start Services (1 min)
-
-In separate terminal windows:
-
-```bash
-# Terminal 1: Start SurrealDB (database)
-make database
-# or: docker run -d --name surrealdb -p 8000:8000 surrealdb/surrealdb:v2 start --user root --pass password --bind 0.0.0.0:8000 memory
-
-# Terminal 2: Start API (backend on port 5055)
-make api
-# or: uv run --env-file .env uvicorn api.main:app --host 0.0.0.0 --port 5055
-
-# Terminal 3: Start Frontend (UI on port 3000)
-cd frontend && npm run dev
-```
-
-## 4. Verify Everything Works (instant)
-
-- **API Health**: http://localhost:5055/health → should return `{"status": "ok"}`
-- **API Docs**: http://localhost:5055/docs → interactive API documentation
-- **Frontend**: http://localhost:3000 → Open Notebook UI
-
-**All three show up?** ✅ You're ready to develop!
-
----
-
-## Next Steps
-
-- **First Issue?** Pick a [good first issue](https://github.com/lfnovo/open-notebook/issues?q=label%3A%22good+first+issue%22)
-- **Understand the code?** Read [Architecture Overview](architecture.md)
-- **Make changes?** Follow [Contributing Guide](contributing.md)
-- **Setup details?** See [Development Setup](development-setup.md)
-
----
-
-## Troubleshooting
-
-### "Port 5055 already in use"
-```bash
-# Find what's using the port
-lsof -i :5055
-
-# Use a different port
-uv run uvicorn api.main:app --port 5056
-```
-
-### "Can't connect to SurrealDB"
-```bash
-# Check if SurrealDB is running
-docker ps | grep surrealdb
-
-# Restart it
-make database
-```
-
-### "Python version is too old"
-```bash
-# Check your Python version
-python --version  # Should be 3.11+
-
-# Use Python 3.11 specifically
-uv sync --python 3.11
-```
-
-### "npm: command not found"
-```bash
-# Install Node.js from https://nodejs.org/
-# Then install frontend dependencies
-cd frontend && npm install
-```
-
----
-
-## Common Development Commands
-
-```bash
-# Run tests
-uv run pytest
-
-# Format code
-make ruff
-
-# Type checking
-make lint
-
-# Run the full stack
-make start-all
-
-# View API documentation
-open http://localhost:5055/docs
-```
-
----
-
-Need more help? See [Development Setup](development-setup.md) for details or join our [Discord](https://discord.gg/37XJPXfz2w).

docs/7-DEVELOPMENT/testing.md

@@ -1,423 +0,0 @@
-# Testing Guide
-
-This document provides guidelines for writing tests in Open Notebook. Testing is critical to maintaining code quality and preventing regressions.
-
-## Testing Philosophy
-
-### What to Test
-
-Focus on testing the things that matter most:
-
-- **Business Logic** - Core domain models and their operations
-- **API Contracts** - HTTP endpoint behavior and error handling
-- **Critical Workflows** - End-to-end flows that users depend on
-- **Data Persistence** - Database operations and data integrity
-- **Error Conditions** - How the system handles failures gracefully
-
-### What NOT to Test
-
-Don't waste time testing framework code:
-
-- Framework functionality (FastAPI, React, etc.)
-- Third-party library implementation
-- Simple getters/setters without logic
-- View/presentation layer rendering (unless it contains logic)
-
-## Test Structure
-
-We use **pytest** with async support for all Python tests:
-
-```python
-import pytest
-from httpx import AsyncClient
-from open_notebook.domain.notebook import Notebook
-
-@pytest.mark.asyncio
-async def test_create_notebook():
-    """Test notebook creation."""
-    notebook = Notebook(name="Test Notebook", description="Test description")
-    await notebook.save()
-
-    assert notebook.id is not None
-    assert notebook.name == "Test Notebook"
-    assert notebook.created is not None
-
-@pytest.mark.asyncio
-async def test_api_create_notebook():
-    """Test notebook creation via API."""
-    async with AsyncClient(app=app, base_url="http://test") as client:
-        response = await client.post(
-            "/api/notebooks",
-            json={"name": "Test Notebook", "description": "Test description"}
-        )
-        assert response.status_code == 200
-        data = response.json()
-        assert data["name"] == "Test Notebook"
-```
-
-## Test Categories
-
-### 1. Unit Tests
-
-Test individual functions and methods in isolation:
-
-```python
-@pytest.mark.asyncio
-async def test_notebook_validation():
-    """Test that notebook name validation works."""
-    with pytest.raises(InvalidInputError):
-        Notebook(name="", description="test")
-
-@pytest.mark.asyncio
-async def test_notebook_archive():
-    """Test notebook archiving."""
-    notebook = Notebook(name="Test", description="")
-    notebook.archive()
-    assert notebook.archived is True
-```
-
-**Location**: `tests/unit/`
-
-### 2. Integration Tests
-
-Test component interactions and database operations:
-
-```python
-@pytest.mark.asyncio
-async def test_create_notebook_with_sources():
-    """Test creating a notebook and adding sources."""
-    notebook = await create_notebook(name="Research", description="")
-    source = await add_source(notebook_id=notebook.id, url="https://example.com")
-
-    retrieved = await get_notebook_with_sources(notebook.id)
-    assert len(retrieved.sources) == 1
-    assert retrieved.sources[0].id == source.id
-```
-
-**Location**: `tests/integration/`
-
-### 3. API Tests
-
-Test HTTP endpoints and error responses:
-
-```python
-@pytest.mark.asyncio
-async def test_get_notebooks_endpoint():
-    """Test GET /notebooks endpoint."""
-    async with AsyncClient(app=app, base_url="http://test") as client:
-        response = await client.get("/api/notebooks")
-        assert response.status_code == 200
-        data = response.json()
-        assert isinstance(data, list)
-
-@pytest.mark.asyncio
-async def test_create_notebook_validation():
-    """Test that invalid input is rejected."""
-    async with AsyncClient(app=app, base_url="http://test") as client:
-        response = await client.post(
-            "/api/notebooks",
-            json={"name": "", "description": ""}
-        )
-        assert response.status_code == 400
-```
-
-**Location**: `tests/api/`
-
-### 4. Database Tests
-
-Test data persistence and query correctness:
-
-```python
-@pytest.mark.asyncio
-async def test_save_and_retrieve_notebook():
-    """Test saving and retrieving a notebook from database."""
-    notebook = Notebook(name="Test", description="desc")
-    await notebook.save()
-
-    retrieved = await Notebook.get(notebook.id)
-    assert retrieved.name == "Test"
-    assert retrieved.description == "desc"
-
-@pytest.mark.asyncio
-async def test_query_by_criteria():
-    """Test querying notebooks by criteria."""
-    await create_notebook("Active", "")
-    await create_notebook("Archived", "")
-
-    active = await repo_query(
-        "SELECT * FROM notebook WHERE archived = false"
-    )
-    assert len(active) >= 1
-```
-
-**Location**: `tests/database/`
-
-## Running Tests
-
-### Run All Tests
-
-```bash
-uv run pytest
-```
-
-### Run Specific Test File
-
-```bash
-uv run pytest tests/test_notebooks.py
-```
-
-### Run Specific Test Function
-
-```bash
-uv run pytest tests/test_notebooks.py::test_create_notebook
-```
-
-### Run with Coverage Report
-
-```bash
-uv run pytest --cov=open_notebook
-```
-
-### Run Only Unit Tests
-
-```bash
-uv run pytest tests/unit/
-```
-
-### Run Only Integration Tests
-
-```bash
-uv run pytest tests/integration/
-```
-
-### Run Tests in Verbose Mode
-
-```bash
-uv run pytest -v
-```
-
-### Run Tests with Output
-
-```bash
-uv run pytest -s
-```
-
-## Test Fixtures
-
-Use pytest fixtures for common setup and teardown:
-
-```python
-import pytest
-
-@pytest.fixture
-async def test_notebook():
-    """Create a test notebook."""
-    notebook = Notebook(name="Test Notebook", description="Test description")
-    await notebook.save()
-    yield notebook
-    await notebook.delete()
-
-@pytest.fixture
-async def api_client():
-    """Create an API test client."""
-    async with AsyncClient(app=app, base_url="http://test") as client:
-        yield client
-
-@pytest.fixture
-async def test_notebook_with_sources(test_notebook):
-    """Create a test notebook with sample sources."""
-    source1 = Source(notebook_id=test_notebook.id, url="https://example.com")
-    source2 = Source(notebook_id=test_notebook.id, url="https://example.org")
-    await source1.save()
-    await source2.save()
-
-    test_notebook.sources = [source1, source2]
-    yield test_notebook
-
-    # Cleanup
-    await source1.delete()
-    await source2.delete()
-```
-
-## Best Practices
-
-### 1. Write Descriptive Test Names
-
-```python
-# Good - clearly describes what is being tested
-async def test_create_notebook_with_valid_name_succeeds():
-    ...
-
-# Bad - vague about what's being tested
-async def test_notebook():
-    ...
-```
-
-### 2. Use Docstrings
-
-```python
-@pytest.mark.asyncio
-async def test_vector_search_returns_sorted_results():
-    """Test that vector search results are sorted by relevance score."""
-    # Implementation
-```
-
-### 3. Test Edge Cases
-
-```python
-@pytest.mark.asyncio
-async def test_search_with_empty_query():
-    """Test that empty query raises error."""
-    with pytest.raises(InvalidInputError):
-        await vector_search("")
-
-@pytest.mark.asyncio
-async def test_search_with_very_long_query():
-    """Test that very long query is handled."""
-    long_query = "x" * 10000
-    results = await vector_search(long_query)
-    assert isinstance(results, list)
-
-@pytest.mark.asyncio
-async def test_search_with_special_characters():
-    """Test that special characters are handled."""
-    results = await vector_search("@#$%^&*()")
-    assert isinstance(results, list)
-```
-
-### 4. Use Assertions Effectively
-
-```python
-# Good - specific assertions
-assert notebook.name == "Test"
-assert len(notebook.sources) == 3
-assert notebook.created is not None
-
-# Less good - too broad
-assert notebook is not None
-assert notebook  # ambiguous what's being tested
-```
-
-### 5. Test Both Success and Failure Cases
-
-```python
-@pytest.mark.asyncio
-async def test_create_notebook_success():
-    """Test successful notebook creation."""
-    notebook = await create_notebook(name="Research", description="AI")
-    assert notebook.id is not None
-    assert notebook.name == "Research"
-
-@pytest.mark.asyncio
-async def test_create_notebook_empty_name_fails():
-    """Test that empty name raises error."""
-    with pytest.raises(InvalidInputError):
-        await create_notebook(name="", description="")
-
-@pytest.mark.asyncio
-async def test_create_notebook_duplicate_fails():
-    """Test that duplicate names are handled."""
-    await create_notebook(name="Research", description="")
-    with pytest.raises(DuplicateError):
-        await create_notebook(name="Research", description="")
-```
-
-### 6. Keep Tests Independent
-
-```python
-# Good - test is self-contained
-@pytest.mark.asyncio
-async def test_archive_notebook():
-    notebook = Notebook(name="Test", description="")
-    await notebook.save()
-    await notebook.archive()
-    assert notebook.archived is True
-
-# Bad - depends on another test's state
-@pytest.mark.asyncio
-async def test_archive_existing_notebook():
-    # Assumes test_create_notebook ran first
-    await notebook.archive()  # notebook undefined
-```
-
-### 7. Use Fixtures for Reusable Setup
-
-```python
-# Instead of repeating setup:
-@pytest.fixture
-async def client_with_auth(api_client, mock_auth):
-    """Client with authentication set up."""
-    api_client.headers.update({"Authorization": f"Bearer {mock_auth.token}"})
-    yield api_client
-
-@pytest.mark.asyncio
-async def test_protected_endpoint(client_with_auth):
-    """Test protected endpoint."""
-    response = await client_with_auth.get("/api/protected")
-    assert response.status_code == 200
-```
-
-## Coverage Goals
-
-- Aim for 70%+ overall coverage
-- 90%+ coverage for critical business logic
-- Don't obsess over 100% - focus on meaningful tests
-- Use `--cov` flag to check coverage: `uv run pytest --cov=open_notebook`
-
-## Async Test Patterns
-
-### Testing Async Functions
-
-```python
-@pytest.mark.asyncio
-async def test_async_operation():
-    """Test async function."""
-    result = await some_async_function()
-    assert result is not None
-```
-
-### Testing Concurrent Operations
-
-```python
-@pytest.mark.asyncio
-async def test_concurrent_notebook_creation():
-    """Test creating multiple notebooks concurrently."""
-    tasks = [
-        create_notebook(f"Notebook {i}", "")
-        for i in range(10)
-    ]
-    notebooks = await asyncio.gather(*tasks)
-    assert len(notebooks) == 10
-    assert all(n.id for n in notebooks)
-```
-
-## Common Testing Errors
-
-### Error: "event loop is closed"
-
-Solution: Use the async fixture properly:
-```python
-@pytest.fixture
-async def notebook():  # Use async fixture
-    notebook = Notebook(name="Test", description="")
-    await notebook.save()
-    yield notebook
-    await notebook.delete()
-```
-
-### Error: "object is not awaitable"
-
-Solution: Make sure you're using await:
-```python
-# Wrong
-result = create_notebook("Test", "")
-
-# Right
-result = await create_notebook("Test", "")
-```
-
----
-
-**See also:**
-- [Code Standards](code-standards.md) - Code formatting and style
-- [Contributing Guide](contributing.md) - Overall contribution workflow

docs/deployment/development.md

@@ -0,0 +1,512 @@
+# Development Setup
+
+This guide covers setting up Open Notebook for local development, contributing to the project, and running from source code.
+
+## 🎯 Who This Guide Is For
+
+This setup is ideal if you want to:
+- **Contribute to Open Notebook** - Fix bugs, add features, or improve documentation
+- **Customize the application** - Modify the code for your specific needs
+- **Understand the codebase** - Learn how Open Notebook works internally
+- **Develop integrations** - Build custom plugins or extensions
+
+## 🛠️ Prerequisites
+
+### System Requirements
+
+- **Python 3.11+** - Required for the application
+- **Node.js 18+** - For frontend development (if contributing to UI)
+- **Git** - For version control
+- **Docker** - For SurrealDB and optional services
+
+### Development Tools
+
+- **Code editor** - VS Code, PyCharm, or your preferred IDE
+- **Terminal** - Command line access
+- **Web browser** - For testing the application
+
+## 📥 Installation
+
+### Step 1: Clone the Repository
+
+```bash
+git clone https://github.com/lfnovo/open-notebook.git
+cd open-notebook
+```
+
+### Step 2: Python Environment Setup
+
+Open Notebook uses **uv** for dependency management:
+
+```bash
+# Install uv (if not already installed)
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Create and activate virtual environment
+uv venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+
+# Install dependencies
+uv sync
+```
+
+### Step 3: Database Setup
+
+#### Option A: Docker SurrealDB (Recommended)
+
+```bash
+# Start SurrealDB with Docker
+docker run -d \
+  --name surrealdb-dev \
+  -p 8000:8000 \
+  surrealdb/surrealdb:v1-latest \
+  start --log trace --user root --pass root memory
+```
+
+#### Option B: Local SurrealDB Installation
+
+```bash
+# Install SurrealDB locally
+curl -sSf https://install.surrealdb.com | sh
+
+# Start SurrealDB
+surreal start --log trace --user root --pass root memory
+```
+
+### Step 4: Environment Configuration
+
+Create a `.env` file in the project root:
+
+```env
+# Database Configuration
+SURREAL_URL=ws://localhost:8000/rpc
+SURREAL_USER=root
+SURREAL_PASSWORD=root
+SURREAL_NAMESPACE=open_notebook
+SURREAL_DATABASE=development
+
+# Required: At least one AI provider
+OPENAI_API_KEY=sk-your-openai-key
+
+# Optional: Additional providers for testing
+ANTHROPIC_API_KEY=sk-ant-your-anthropic-key
+GOOGLE_API_KEY=your-google-key
+GROQ_API_KEY=gsk_your-groq-key
+
+# Optional: Development settings
+LOG_LEVEL=DEBUG
+ENABLE_ANALYTICS=false
+```
+
+### Step 5: Frontend Setup
+
+Install frontend dependencies:
+
+```bash
+cd frontend
+npm install
+cd ..
+```
+
+> **Note**: Database migrations now run automatically when the API starts. No manual migration step is required.
+
+### Step 6: Start the Application
+
+#### Option A: Full Stack with Make
+
+```bash
+# Start all services (recommended for development)
+make start-all
+```
+
+This starts:
+- **SurrealDB** (if not already running)
+- **FastAPI backend** on port 5055
+- **Background worker** for async tasks
+- **React frontend** on port 8502
+
+#### Option B: Individual Services
+
+Start services separately for debugging:
+
+```bash
+# Terminal 1: Start the API
+uv run python api/main.py
+
+# Terminal 2: Start the background worker
+uv run python -m open_notebook.worker
+
+# Terminal 3: Start the React frontend
+cd frontend && npm run dev
+```
+
+## 🔧 Development Workflow
+
+### Project Structure
+
+```
+open-notebook/
+├── api/                    # FastAPI backend
+│   ├── routers/           # API routes
+│   └── main.py           # API entry point
+├── frontend/              # React frontend (Next.js)
+│   ├── src/              # React components and pages
+│   └── public/           # Static assets
+├── open_notebook/         # Core application
+│   ├── domain/           # Business logic
+│   ├── database/         # Database layer
+│   └── graphs/           # LangGraph workflows
+├── prompts/              # Jinja2 templates
+├── docs/                 # Documentation
+└── tests/                # Test files
+```
+
+### Development Commands
+
+```bash
+# Install new dependencies
+uv add package-name
+
+# Run tests
+uv run pytest
+
+# Run linting
+uv run ruff check
+uv run ruff format
+
+# Type checking
+uv run mypy .
+
+# Start development server
+make start-dev
+```
+
+### Making Changes
+
+1. **Create a branch** for your feature/fix:
+   ```bash
+   git checkout -b feature/your-feature-name
+   ```
+
+2. **Make your changes** in the appropriate files
+
+3. **Test your changes**:
+   ```bash
+   uv run pytest
+   ```
+
+4. **Format code**:
+   ```bash
+   uv run ruff format
+   ```
+
+5. **Commit your changes**:
+   ```bash
+   git add .
+   git commit -m "feat: your descriptive commit message"
+   ```
+
+6. **Push and create a pull request**:
+   ```bash
+   git push origin feature/your-feature-name
+   ```
+
+## 🧪 Testing
+
+### Running Tests
+
+```bash
+# Run all tests
+uv run pytest
+
+# Run specific test file
+uv run pytest tests/test_specific.py
+
+# Run with coverage
+uv run pytest --cov=open_notebook
+
+# Run integration tests
+uv run pytest tests/integration/
+```
+
+### Test Structure
+
+```
+tests/
+├── unit/               # Unit tests
+├── integration/        # Integration tests
+├── fixtures/          # Test fixtures
+└── conftest.py        # Test configuration
+```
+
+### Writing Tests
+
+```python
+# Example test file
+import pytest
+from open_notebook.domain.notebook import Notebook
+
+def test_notebook_creation():
+    notebook = Notebook(name="Test Notebook", description="Test")
+    assert notebook.name == "Test Notebook"
+    assert notebook.description == "Test"
+```
+
+## 🚀 Building and Deployment
+
+### Local Docker Build
+
+```bash
+# Build multi-container version
+make docker-build-dev
+
+# Build single-container version
+make docker-build-single-dev
+
+# Test the built image
+docker run -p 8502:8502 \
+  -v ./notebook_data:/app/data \
+  -v ./surreal_data:/mydata \
+  open_notebook:v1-latest
+```
+
+### Production Build
+
+```bash
+# Build with multi-platform support
+make docker-build
+
+# Build and push to registry
+make docker-push
+```
+
+## 🔍 Debugging
+
+### Common Development Issues
+
+#### Database Connection Errors
+
+```bash
+# Check if SurrealDB is running
+docker ps | grep surrealdb
+
+# Check SurrealDB logs
+docker logs surrealdb-dev
+
+# Test connection
+curl -X POST http://localhost:8000/sql \
+  -H "Content-Type: application/json" \
+  -d '{"sql": "SELECT * FROM VERSION"}'
+```
+
+#### API Not Starting
+
+```bash
+# Check Python environment
+uv run python --version
+
+# Check dependencies
+uv run pip list | grep fastapi
+
+# Start with debug mode
+uv run python api/main.py --debug
+```
+
+#### Frontend Issues
+
+```bash
+# Check Node.js and npm versions
+node --version
+npm --version
+
+# Reinstall frontend dependencies
+cd frontend
+rm -rf node_modules package-lock.json
+npm install
+
+# Start frontend in development mode
+npm run dev
+```
+
+### Debugging Tools
+
+#### VS Code Configuration
+
+Create `.vscode/launch.json`:
+
+```json
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "FastAPI",
+      "type": "python",
+      "request": "launch",
+      "program": "api/main.py",
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}"
+      }
+    },
+    {
+      "name": "React Frontend",
+      "type": "node",
+      "request": "launch",
+      "cwd": "${workspaceFolder}/frontend",
+      "runtimeExecutable": "npm",
+      "runtimeArgs": ["run", "dev"],
+      "console": "integratedTerminal"
+    }
+  ]
+}
+```
+
+#### Python Debugging
+
+```python
+# Add breakpoints in code
+import pdb; pdb.set_trace()
+
+# Or use debugger
+import debugpy
+debugpy.listen(5678)
+debugpy.wait_for_client()
+```
+
+## 📝 Code Style and Standards
+
+### Python Style Guide
+
+- **Formatting**: Use `ruff format` for code formatting
+- **Linting**: Use `ruff check` for linting
+- **Type hints**: Use type hints for all functions
+- **Docstrings**: Document all public functions and classes
+
+### Example Code Style
+
+```python
+from typing import List, Optional
+from pydantic import BaseModel
+
+class Notebook(BaseModel):
+    """A notebook for organizing research sources."""
+    
+    name: str
+    description: Optional[str] = None
+    sources: List[str] = []
+    
+    def add_source(self, source_id: str) -> None:
+        """Add a source to the notebook.
+        
+        Args:
+            source_id: The ID of the source to add
+        """
+        if source_id not in self.sources:
+            self.sources.append(source_id)
+```
+
+### Commit Message Format
+
+Follow conventional commits:
+
+```
+feat: add new podcast generation feature
+fix: resolve database connection issue
+docs: update deployment guide
+refactor: improve source processing logic
+test: add tests for notebook creation
+```
+
+## 🤝 Contributing
+
+### Before Contributing
+
+1. **Read the contribution guidelines** in `CONTRIBUTING.md`
+2. **Join the Discord** for discussion: [discord.gg/37XJPXfz2w](https://discord.gg/37XJPXfz2w)
+3. **Check existing issues** to avoid duplicates
+4. **Discuss major changes** before implementing
+
+### Contribution Process
+
+1. **Fork the repository** on GitHub
+2. **Create a feature branch** from `main`
+3. **Make your changes** following the coding standards
+4. **Add tests** for new functionality
+5. **Update documentation** as needed
+6. **Submit a pull request** with a clear description
+
+### Areas for Contribution
+
+- **Frontend Development** - Modern React/Next.js UI improvements
+- **Backend Features** - API endpoints, new functionality
+- **AI Integrations** - New model providers, better prompts
+- **Documentation** - Guides, tutorials, API docs
+- **Testing** - Unit tests, integration tests
+- **Bug Fixes** - Resolve existing issues
+
+## 📚 Development Resources
+
+### Documentation
+
+- **[API Documentation](../api-reference.md)** - REST API reference
+- **[Architecture Guide](../architecture.md)** - System architecture
+- **[Plugin Development](../plugins.md)** - Creating custom plugins
+
+### External Resources
+
+- **[SurrealDB Documentation](https://surrealdb.com/docs)** - Database queries and schema
+- **[FastAPI Documentation](https://fastapi.tiangolo.com/)** - API framework
+- **[Next.js Documentation](https://nextjs.org/docs)** - React framework
+- **[LangChain Documentation](https://python.langchain.com/)** - AI workflows
+
+### Getting Help
+
+- **[Discord Server](https://discord.gg/37XJPXfz2w)** - Real-time development help
+- **[GitHub Discussions](https://github.com/lfnovo/open-notebook/discussions)** - Design discussions
+- **[GitHub Issues](https://github.com/lfnovo/open-notebook/issues)** - Bug reports and feature requests
+
+## 🔄 Maintenance
+
+### Keeping Your Fork Updated
+
+```bash
+# Add upstream remote
+git remote add upstream https://github.com/lfnovo/open-notebook.git
+
+# Fetch upstream changes
+git fetch upstream
+
+# Merge upstream changes
+git checkout main
+git merge upstream/main
+```
+
+### Dependency Updates
+
+```bash
+# Update dependencies
+uv sync --upgrade
+
+# Check for security issues
+uv audit
+
+# Update pre-commit hooks
+pre-commit autoupdate
+```
+
+### Database Migrations
+
+Database migrations now run automatically when the API starts. When you need to create new migrations:
+
+```bash
+# Create new migration file
+# Add your migration to migrations/ folder with incremental number
+
+# Migrations are automatically applied on API startup
+uv run python api/main.py
+```
+
+---
+
+**Ready to contribute?** Start by forking the repository and following the installation steps above. Join our Discord for real-time help and discussion!

docs/deployment/docker.md

@@ -0,0 +1,528 @@
+# Docker Deployment Guide
+
+**The complete Docker setup guide for Open Notebook - from beginner to advanced configurations.**
+
+This guide covers everything you need to deploy Open Notebook using Docker, from a simple single-provider setup to advanced multi-provider configurations with local models.
+
+## 📋 What You'll Get
+
+Open Notebook is a powerful AI-powered research and note-taking tool that:
+- Modern Next.js/React interface for a smooth user experience
+- Helps you organize research across multiple notebooks
+- Lets you chat with your documents using AI
+- Supports 16+ AI providers (OpenAI, Anthropic, Google, Ollama, and more)
+- Creates AI-generated podcasts from your content
+- Works with PDFs, web links, videos, audio files, and more
+
+## 📦 Docker Image Registries
+
+Open Notebook images are available from two registries:
+
+- **GitHub Container Registry (GHCR)**: `ghcr.io/lfnovo/open-notebook` - Hosted on GitHub, no Docker Hub account needed
+- **Docker Hub**: `lfnovo/open_notebook` - Traditional Docker registry
+
+Both registries contain identical images. Choose based on your preference:
+- Use **GHCR** if you prefer GitHub-native workflows or Docker Hub is blocked
+- Use **Docker Hub** if you're already using it or prefer the traditional registry
+
+All examples in this guide use Docker Hub (`lfnovo/open_notebook`), but you can replace it with `ghcr.io/lfnovo/open-notebook` anywhere.
+
+## 🚀 Quick Start (5 Minutes)
+
+### Step 1: Install Docker
+
+#### Windows
+1. Download Docker Desktop from [docker.com](https://www.docker.com/products/docker-desktop/)
+2. Run the installer and follow the setup wizard
+3. Restart your computer when prompted
+4. Launch Docker Desktop
+
+#### macOS
+1. Download Docker Desktop from [docker.com](https://www.docker.com/products/docker-desktop/)
+2. Choose Intel or Apple Silicon based on your Mac
+3. Drag Docker to Applications folder
+4. Open Docker from Applications
+
+#### Linux (Ubuntu/Debian)
+```bash
+sudo apt update
+sudo apt install docker.io docker-compose
+sudo systemctl start docker
+sudo systemctl enable docker
+sudo usermod -aG docker $USER
+```
+Log out and log back in after installation.
+
+### Step 2: Get Your OpenAI API Key
+
+OpenAI provides everything you need to get started:
+- **Text generation** for chat and notes
+- **Embeddings** for search functionality  
+- **Text-to-speech** for podcast generation
+- **Speech-to-text** for audio transcription
+
+1. Go to [platform.openai.com](https://platform.openai.com/)
+2. Create an account or sign in
+3. Navigate to **API Keys** in the sidebar
+4. Click **"Create new secret key"**
+5. Name your key (e.g., "Open Notebook")
+6. Copy the key (starts with "sk-")
+7. **Save it safely** - you won't see it again!
+
+**Important**: Add at least $5 in credits to your OpenAI account before using the API.
+
+### Step 3: Deploy Open Notebook
+
+1. **Create a project directory**:
+   ```bash
+   mkdir open-notebook
+   cd open-notebook
+   ```
+
+2. **Create `docker-compose.yml`**:
+   ```yaml
+   services:
+     open_notebook:
+       image: lfnovo/open_notebook:v1-latest-single
+       ports:
+         - "8502:8502"  # Frontend
+         - "5055:5055"  # API
+       environment:
+         - OPENAI_API_KEY=your_openai_key_here
+       volumes:
+         - ./notebook_data:/app/data
+         - ./surreal_data:/mydata
+       restart: always
+   ```
+
+3. **Create `docker.env` file** (optional but recommended):
+   ```env
+   # Required: Your OpenAI API key
+   OPENAI_API_KEY=sk-your-actual-key-here
+   
+   # Optional: Security for public deployments
+   OPEN_NOTEBOOK_PASSWORD=your_secure_password
+   
+   # Database settings (auto-configured)
+   SURREAL_URL=ws://localhost:8000/rpc
+   SURREAL_USER=root
+   SURREAL_PASSWORD=root
+   SURREAL_NAMESPACE=open_notebook
+   SURREAL_DATABASE=production
+   ```
+
+4. **Start Open Notebook**:
+   ```bash
+   docker compose up -d
+   ```
+
+5. **Access the application**:
+   - **Next.js UI**: http://localhost:8502 - Modern, responsive interface
+   - **API Documentation**: http://localhost:5055/docs - Full REST API access
+   - You should see the Open Notebook interface!
+
+**Alternative: Using GHCR**
+To use GitHub Container Registry instead, simply replace the image name:
+```yaml
+services:
+  open_notebook:
+    image: ghcr.io/lfnovo/open-notebook:v1-latest-single
+    # ... rest of configuration stays the same
+```
+
+### Step 4: Configure Your Models
+
+Before creating your first notebook, configure your AI models:
+
+1. Click **"⚙️ Settings"** in the sidebar
+2. Click **"🤖 Models"** tab
+3. Configure these recommended models:
+   - **Language Model**: `gpt-5-mini` (cost-effective)
+   - **Embedding Model**: `text-embedding-3-small` (required for search)
+   - **Text-to-Speech**: `gpt-4o-mini-tts` (for podcast generation)
+   - **Speech-to-Text**: `whisper-1` (for audio transcription)
+4. Click **"Save"** after configuring all models
+
+### Step 5: Create Your First Notebook
+
+1. Click **"Create New Notebook"**
+2. Give it a name (e.g., "My Research")
+3. Add a description
+4. Click **"Create"**
+5. Add your first source (web link, PDF, or text)
+6. Start chatting with your content!
+
+## 🔧 Advanced Configuration
+
+### Multi-Container Setup
+
+For production deployments or development, use the multi-container setup:
+
+```yaml
+services:
+  surrealdb:
+    image: surrealdb/surrealdb:v1-latest
+    ports:
+      - "8000:8000"
+    command: start --log trace --user root --pass root memory
+    restart: always
+
+  open_notebook:
+    image: lfnovo/open_notebook:v1-latest
+    # Or use: ghcr.io/lfnovo/open-notebook:v1-latest
+    ports:
+      - "8502:8502"  # Next.js Frontend
+      - "5055:5055"  # REST API
+    env_file:
+      - ./docker.env
+    volumes:
+      - ./notebook_data:/app/data
+    depends_on:
+      - surrealdb
+    restart: always
+```
+
+### Environment Configuration
+
+Create a comprehensive `docker.env` file:
+
+```env
+# Required: Database connection
+SURREAL_URL=ws://surrealdb:8000/rpc
+SURREAL_USER=root
+SURREAL_PASSWORD=root
+SURREAL_NAMESPACE=open_notebook
+SURREAL_DATABASE=production
+
+# Required: At least one AI provider
+OPENAI_API_KEY=sk-your-openai-key
+
+# Optional: Additional AI providers
+ANTHROPIC_API_KEY=sk-ant-your-anthropic-key
+GOOGLE_API_KEY=your-google-key
+GROQ_API_KEY=gsk_your-groq-key
+
+# Optional: Security
+OPEN_NOTEBOOK_PASSWORD=your_secure_password
+
+# Optional: Advanced features
+ELEVENLABS_API_KEY=your-elevenlabs-key
+```
+
+## 🌟 Advanced Provider Setup
+
+### OpenRouter (100+ Models)
+
+OpenRouter gives you access to virtually every AI model through a single API:
+
+1. **Get your API key** at [openrouter.ai](https://openrouter.ai/keys)
+2. **Add to your `docker.env`**:
+   ```env
+   OPENROUTER_API_KEY=sk-or-your-openrouter-key
+   ```
+3. **Restart the container**:
+   ```bash
+   docker compose restart
+   ```
+4. **Configure models** in Models
+
+**Recommended OpenRouter models**:
+- `anthropic/claude-3-haiku` - Fast and cost-effective
+- `google/gemini-pro` - Good reasoning capabilities
+- `meta-llama/llama-3-8b-instruct` - Open source option
+
+### Ollama (Local Models)
+
+Run AI models locally for complete privacy:
+
+1. **Install Ollama** on your host machine from [ollama.ai](https://ollama.ai)
+2. **Start Ollama**:
+   ```bash
+   ollama serve
+   ```
+3. **Download models**:
+   ```bash
+   ollama pull llama2        # 7B model (~4GB)
+   ollama pull mistral       # 7B model (~4GB)
+   ollama pull llama2:13b    # 13B model (~8GB)
+   ```
+4. **Find your IP address**:
+   - Windows: `ipconfig` (look for IPv4 Address)
+   - macOS/Linux: `ifconfig` or `ip addr show`
+5. **Configure Open Notebook**:
+   ```env
+   OLLAMA_API_BASE=http://192.168.1.100:11434
+   ```
+   Replace `192.168.1.100` with your actual IP.
+
+6. **Restart and configure** models in Models
+
+### Other Providers
+
+**Anthropic (Direct)**:
+```env
+ANTHROPIC_API_KEY=sk-ant-your-key
+```
+
+**Google Gemini**:
+```env
+GOOGLE_API_KEY=AIzaSy-your-key
+```
+
+**Groq (Fast Inference)**:
+```env
+GROQ_API_KEY=gsk_your-key
+```
+
+## 🔒 Security & Production
+
+### Password Protection
+
+For public deployments, always set a password:
+
+```env
+OPEN_NOTEBOOK_PASSWORD=your_secure_password
+```
+
+This protects both the web interface and API endpoints.
+
+### Production Best Practices
+
+1. **Use HTTPS**: Deploy behind a reverse proxy with SSL
+2. **Regular Updates**: Keep containers updated
+3. **Monitor Resources**: Set up resource limits
+4. **Backup Data**: Regular backups of volumes
+5. **Network Security**: Configure firewall rules
+
+### Example Production Setup
+
+```yaml
+services:
+  surrealdb:
+    image: surrealdb/surrealdb:v1-latest
+    ports:
+      - "127.0.0.1:8000:8000"  # Bind to localhost only
+    command: start --log warn --user root --pass root file:///mydata/database.db
+    volumes:
+      - ./surreal_data:/mydata
+    restart: always
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+          cpus: "0.5"
+
+  open_notebook:
+    image: lfnovo/open_notebook:v1-latest
+    ports:
+      - "127.0.0.1:8502:8502"
+      - "127.0.0.1:5055:5055"
+    env_file:
+      - ./docker.env
+    volumes:
+      - ./notebook_data:/app/data
+    depends_on:
+      - surrealdb
+    restart: always
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+          cpus: "1.0"
+```
+
+## 🛠️ Management & Maintenance
+
+### Container Management
+
+```bash
+# Start services
+docker compose up -d
+
+# Stop services
+docker compose down
+
+# View logs
+docker compose logs -f
+
+# Restart specific service
+docker compose restart open_notebook
+
+# Update to latest version
+docker compose pull
+docker compose up -d
+```
+
+### Data Management
+
+```bash
+# Backup data
+tar -czf backup-$(date +%Y%m%d).tar.gz notebook_data surreal_data
+
+# Restore data
+tar -xzf backup-20240101.tar.gz
+
+# Clean up old containers
+docker system prune -a
+```
+
+### Monitoring
+
+```bash
+# Check resource usage
+docker stats
+
+# Check service health
+docker compose ps
+
+# View detailed logs
+docker compose logs --tail=100 -f open_notebook
+```
+
+## 📊 Performance Optimization
+
+### Resource Allocation
+
+**Minimum requirements**:
+- 2GB RAM
+- 2 CPU cores
+- 10GB storage
+
+**Recommended for production**:
+- 4GB+ RAM
+- 4+ CPU cores
+- 50GB+ storage
+
+### Model Selection Tips
+
+**For cost optimization**:
+- Use OpenRouter for expensive models
+- Use Ollama for simple tasks
+- Monitor usage at provider dashboards
+
+**For performance**:
+- Use Groq for fast inference
+- Use local models for privacy
+- Use OpenAI for reliability
+
+## 🔍 Troubleshooting
+
+### Common Issues
+
+**Port conflicts**:
+```bash
+# Check what's using port 8502
+lsof -i :8502
+
+# Use different port
+docker compose -p 8503:8502 up -d
+```
+
+**API key errors**:
+1. Verify keys are set correctly in `docker.env`
+2. Check you have credits with your AI provider
+3. Ensure no extra spaces in the key
+
+**Database connection issues**:
+1. Check SurrealDB container is running
+2. Verify database files are writable
+3. Try restarting containers
+
+**Out of memory errors**:
+1. Increase Docker memory allocation
+2. Use smaller models
+3. Monitor resource usage
+
+### Getting Help
+
+1. **Check logs**: `docker compose logs -f`
+2. **Verify environment**: `docker compose config`
+3. **Test connectivity**: `docker compose exec open_notebook ping surrealdb`
+4. **Join Discord**: [discord.gg/37XJPXfz2w](https://discord.gg/37XJPXfz2w)
+5. **GitHub Issues**: [github.com/lfnovo/open-notebook/issues](https://github.com/lfnovo/open-notebook/issues)
+
+## 🎯 Next Steps
+
+After successful deployment:
+
+1. **Create your first notebook** - Start with a simple research project
+2. **Explore features** - Try podcasts, transformations, and search
+3. **Optimize models** - Experiment with different providers
+4. **Join the community** - Share your experience and get help
+
+## 📚 Complete Configuration Reference
+
+### All Environment Variables
+
+```env
+# Database Configuration
+SURREAL_URL=ws://surrealdb:8000/rpc
+SURREAL_USER=root
+SURREAL_PASSWORD=root
+SURREAL_NAMESPACE=open_notebook
+SURREAL_DATABASE=production
+
+# Required: At least one AI provider
+OPENAI_API_KEY=sk-your-openai-key
+
+# Optional: Additional AI providers
+ANTHROPIC_API_KEY=sk-ant-your-anthropic-key
+GOOGLE_API_KEY=your-google-key
+GROQ_API_KEY=gsk_your-groq-key
+OPENROUTER_API_KEY=sk-or-your-openrouter-key
+OLLAMA_API_BASE=http://192.168.1.100:11434
+
+# Optional: Advanced TTS
+ELEVENLABS_API_KEY=your-elevenlabs-key
+
+# Optional: Security
+OPEN_NOTEBOOK_PASSWORD=your_secure_password
+
+# Optional: Advanced settings
+LOG_LEVEL=INFO
+MAX_UPLOAD_SIZE=100MB
+ENABLE_ANALYTICS=false
+```
+
+### Complete Docker Compose
+
+```yaml
+version: '3.8'
+services:
+  surrealdb:
+    image: surrealdb/surrealdb:v1-latest
+    ports:
+      - "8000:8000"
+    command: start --log warn --user root --pass root file:///mydata/database.db
+    volumes:
+      - ./surreal_data:/mydata
+    restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  open_notebook:
+    image: lfnovo/open_notebook:v1-latest
+    ports:
+      - "8502:8502"  # Next.js Frontend
+      - "5055:5055"  # REST API
+    env_file:
+      - ./docker.env
+    volumes:
+      - ./notebook_data:/app/data
+    depends_on:
+      surrealdb:
+        condition: service_healthy
+    restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5055/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+```
+
+---
+
+**Ready to get started?** Follow the Quick Start section above and you'll be up and running in 5 minutes!