diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..271c6bc
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,83 @@
+# Environment Configuration
+NODE_ENV=development
+DEBUG=true
+ENVIRONMENT=development
+
+# API Configuration
+API_HOST=0.0.0.0
+API_PORT=8000
+SECRET_KEY=your-secret-key-change-in-production
+
+# Database Configuration
+DATABASE_URL=postgresql+asyncpg://aisearch:aisearch_password@localhost:5432/aisearch
+DATABASE_POOL_SIZE=10
+DATABASE_MAX_OVERFLOW=20
+
+# Redis Configuration
+REDIS_URL=redis://localhost:6379
+REDIS_MAX_CONNECTIONS=10
+
+# OpenAI Configuration
+OPENAI_API_KEY=your-openai-api-key
+OPENAI_MODEL=gpt-4-turbo-preview
+OPENAI_EMBEDDING_MODEL=text-embedding-ada-002
+
+# Anthropic Configuration
+ANTHROPIC_API_KEY=your-anthropic-api-key
+ANTHROPIC_MODEL=claude-3-sonnet-20240229
+
+# HuggingFace Configuration
+HUGGINGFACE_API_KEY=your-huggingface-api-key
+HUGGINGFACE_MODEL=sentence-transformers/all-MiniLM-L6-v2
+
+# Cohere Configuration
+COHERE_API_KEY=your-cohere-api-key
+COHERE_MODEL=embed-english-v2.0
+
+# Vector Database Configuration (Milvus)
+MILVUS_HOST=localhost
+MILVUS_PORT=19530
+MILVUS_USER=
+MILVUS_PASSWORD=
+
+# Vector Database Configuration (Pinecone)
+PINECONE_API_KEY=your-pinecone-api-key
+PINECONE_ENVIRONMENT=your-pinecone-environment
+PINECONE_INDEX_NAME=ai-search
+
+# Vector Database Configuration (Weaviate)
+WEAVIATE_URL=http://localhost:8080
+WEAVIATE_API_KEY=your-weaviate-api-key
+WEAVIATE_CLASS_NAME=Document
+
+# Vector Database Configuration (Qdrant)
+QDRANT_URL=http://localhost:6333
+QDRANT_API_KEY=your-qdrant-api-key
+QDRANT_COLLECTION_NAME=documents
+
+# Vector Database Selection
+VECTOR_DB_PROVIDER=milvus
+VECTOR_DB_DIMENSIONS=1536
+
+# Content Processing
+MAX_DOCUMENT_SIZE=52428800 # 50MB
+CHUNK_SIZE=1000
+CHUNK_OVERLAP=200
+
+# Rate Limiting
+RATE_LIMIT_REQUESTS=100
+RATE_LIMIT_WINDOW=60
+
+# Celery Configuration
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/0
+
+# Monitoring
+SENTRY_DSN=your-sentry-dsn
+LOG_LEVEL=INFO
+
+# File Storage
+UPLOAD_DIR=uploads
+
+# CORS Configuration
+ALLOWED_ORIGINS=["http://localhost:3000","http://localhost:8080","https://yourdomain.com"]
\ No newline at end of file
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
new file mode 100644
index 0000000..7a09de3
--- /dev/null
+++ b/.github/workflows/ci-cd.yml
@@ -0,0 +1,200 @@
+name: CI/CD Pipeline
+
+on:
+ push:
+ branches: [ main, develop ]
+ pull_request:
+ branches: [ main ]
+
+env:
+ NODE_VERSION: '18'
+ PYTHON_VERSION: '3.11'
+
+jobs:
+ # Frontend Tests
+ frontend-tests:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: ${{ env.NODE_VERSION }}
+ cache: 'npm'
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Run linting
+ run: npm run lint
+
+ - name: Run type checking
+ run: npm run type-check
+
+ - name: Run tests
+ run: npm test
+
+ - name: Build packages
+ run: npm run build
+
+ # Backend Tests
+ backend-tests:
+ runs-on: ubuntu-latest
+
+ services:
+ postgres:
+ image: postgres:15
+ env:
+ POSTGRES_PASSWORD: postgres
+ POSTGRES_DB: test_aisearch
+ options: >-
+ --health-cmd pg_isready
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ ports:
+ - 5432:5432
+
+ redis:
+ image: redis:7
+ options: >-
+ --health-cmd "redis-cli ping"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ ports:
+ - 6379:6379
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Setup Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Install dependencies
+ working-directory: ./packages/api
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r requirements.txt
+
+ - name: Run linting
+ working-directory: ./packages/api
+ run: |
+ flake8 .
+ black --check .
+ mypy .
+
+ - name: Run tests
+ working-directory: ./packages/api
+ env:
+ DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_aisearch
+ REDIS_URL: redis://localhost:6379
+ SECRET_KEY: test-secret-key
+ run: pytest tests/ -v --cov=app --cov-report=xml
+
+ - name: Upload coverage
+ uses: codecov/codecov-action@v3
+ with:
+ file: ./packages/api/coverage.xml
+ flags: backend
+
+ # Security Scanning
+ security-scan:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Run Trivy vulnerability scanner
+ uses: aquasecurity/trivy-action@master
+ with:
+ scan-type: 'fs'
+ scan-ref: '.'
+ format: 'sarif'
+ output: 'trivy-results.sarif'
+
+ - name: Upload Trivy scan results
+ uses: github/codeql-action/upload-sarif@v2
+ with:
+ sarif_file: 'trivy-results.sarif'
+
+ # Build and Deploy
+ build-deploy:
+ needs: [frontend-tests, backend-tests]
+ runs-on: ubuntu-latest
+ if: github.ref == 'refs/heads/main'
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Setup Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Login to Container Registry
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Build and push API image
+ uses: docker/build-push-action@v5
+ with:
+ context: ./packages/api
+ push: true
+ tags: |
+ ghcr.io/${{ github.repository }}/api:latest
+ ghcr.io/${{ github.repository }}/api:${{ github.sha }}
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+
+ - name: Build and push Demo image
+ uses: docker/build-push-action@v5
+ with:
+ context: ./apps/demo
+ push: true
+ tags: |
+ ghcr.io/${{ github.repository }}/demo:latest
+ ghcr.io/${{ github.repository }}/demo:${{ github.sha }}
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+
+ - name: Deploy to staging
+ if: github.ref == 'refs/heads/main'
+ run: |
+ echo "Deploying to staging environment..."
+ # Add your deployment commands here
+ # e.g., kubectl apply, terraform apply, etc.
+
+ - name: Run integration tests
+ run: |
+ echo "Running integration tests..."
+ # Add integration test commands here
+
+ - name: Deploy to production
+ if: github.ref == 'refs/heads/main'
+ run: |
+ echo "Deploying to production environment..."
+ # Add production deployment commands here
+
+ # Performance Tests
+ performance-tests:
+ needs: [build-deploy]
+ runs-on: ubuntu-latest
+ if: github.ref == 'refs/heads/main'
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Run load tests
+ run: |
+ echo "Running performance tests..."
+ # Add load testing commands here (e.g., k6, artillery)
+
+ - name: Upload performance results
+ uses: actions/upload-artifact@v3
+ with:
+ name: performance-results
+ path: performance-results/
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0ddd8ff
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,118 @@
+# Dependencies
+node_modules/
+.pnp
+.pnp.js
+
+# Production builds
+dist/
+build/
+.next/
+out/
+
+# Environment variables
+.env
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+# Logs
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Coverage directory used by tools like istanbul
+coverage/
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Dependency directories
+jspm_packages/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+
+# Nuxt.js build / generate output
+.nuxt
+
+# Storybook build outputs
+.out
+.storybook-out
+
+# Temporary folders
+tmp/
+temp/
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+*.swp
+*.swo
+*~
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Turbo
+.turbo
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+.venv/
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Database
+*.db
+*.sqlite
+*.sqlite3
+
+# Docker
+.dockerignore
+Dockerfile
+docker-compose*.yml
\ No newline at end of file
diff --git a/.prettierrc b/.prettierrc
new file mode 100644
index 0000000..46f2372
--- /dev/null
+++ b/.prettierrc
@@ -0,0 +1,8 @@
+{
+ "semi": true,
+ "trailingComma": "es5",
+ "singleQuote": true,
+ "printWidth": 80,
+ "tabWidth": 2,
+ "useTabs": false
+}
\ No newline at end of file
diff --git a/README.md b/README.md
index 08094e2..b5cdcf3 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,85 @@
-# possibility
-edit in local machine
-This is Shaswat this is my first project
+# AI-Powered Website Search Platform
+
+A production-grade AI-powered website search and content discovery platform that delivers instant, personalized search experiences via RAG, vector search, and LLM integrations.
+
+## Features
+
+- **Semantic (RAG) Search**: Advanced AI-powered search using vector embeddings
+- **Customizable Widget SDK**: Embeddable search components for any website
+- **Content Guardrails**: Validation and content filtering mechanisms
+- **Real-time Analytics**: Engagement tracking and performance insights
+- **Content Optimization**: AI-driven recommendations and improvements
+
+## Architecture
+
+### Frontend Components
+- **Website Widget** (JS/TS): Chat UI, search box, result cards
+- **Analytics Dashboard**: Real-time metrics and insights
+- **Configuration Panel**: Easy setup and customization
+
+### Backend Services
+- **Search API**: Semantic search with RAG capabilities
+- **Content Ingestion API**: Document processing and indexing
+- **Analytics API**: Event tracking and reporting
+- **API Gateway**: Authentication, rate limiting, load balancing
+
+### AI/ML Layer
+- **Embedding Generation**: OpenAI, HuggingFace integrations
+- **Vector Search**: Milvus/Pinecone/Weaviate support
+- **LLM Integration**: GPT-4, Claude, and custom models
+- **Content Guardrails**: Validation and hallucination filtering
+
+## Development Phases
+
+### Phase 1: MVP (Months 1-4)
+- Core search functionality
+- Basic widget SDK
+- Document ingestion
+- Simple analytics
+
+### Phase 2: Enhancement (Months 5-7)
+- Advanced guardrails
+- Multi-LLM support
+- A/B testing
+- Performance optimization
+
+### Phase 3: Enterprise Scale (Months 8-11)
+- Enterprise features
+- Multi-language support
+- White-label solutions
+- Advanced security
+
+## Quick Start
+
+```bash
+# Install dependencies
+npm install
+
+# Start development servers
+npm run dev
+
+# Run tests
+npm test
+
+# Build for production
+npm run build
+```
+
+## Project Structure
+
+```
+├── packages/
+│ ├── widget/ # Embeddable search widget
+│ ├── dashboard/ # Analytics dashboard
+│ ├── api/ # Backend API services
+│ ├── shared/ # Shared types and utilities
+│ └── docs/ # Documentation
+├── apps/
+│ ├── demo/ # Demo application
+│ └── admin/ # Admin interface
+└── infrastructure/ # Deployment configs
+```
+
+## License
+
+MIT License - see LICENSE file for details.
\ No newline at end of file
diff --git a/apps/demo/app/globals.css b/apps/demo/app/globals.css
new file mode 100644
index 0000000..f2bb22a
--- /dev/null
+++ b/apps/demo/app/globals.css
@@ -0,0 +1,45 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+:root {
+ --foreground-rgb: 0, 0, 0;
+ --background-start-rgb: 214, 219, 220;
+ --background-end-rgb: 255, 255, 255;
+}
+
+@media (prefers-color-scheme: dark) {
+ :root {
+ --foreground-rgb: 255, 255, 255;
+ --background-start-rgb: 0, 0, 0;
+ --background-end-rgb: 0, 0, 0;
+ }
+}
+
+body {
+ color: rgb(var(--foreground-rgb));
+ background: linear-gradient(
+ to bottom,
+ transparent,
+ rgb(var(--background-end-rgb))
+ )
+ rgb(var(--background-start-rgb));
+}
+
+/* Custom scrollbar */
+::-webkit-scrollbar {
+ width: 8px;
+}
+
+::-webkit-scrollbar-track {
+ background: #f1f1f1;
+}
+
+::-webkit-scrollbar-thumb {
+ background: #c1c1c1;
+ border-radius: 4px;
+}
+
+::-webkit-scrollbar-thumb:hover {
+ background: #a8a8a8;
+}
\ No newline at end of file
diff --git a/apps/demo/app/layout.tsx b/apps/demo/app/layout.tsx
new file mode 100644
index 0000000..cec8932
--- /dev/null
+++ b/apps/demo/app/layout.tsx
@@ -0,0 +1,22 @@
+import './globals.css'
+import type { Metadata } from 'next'
+import { Inter } from 'next/font/google'
+
+const inter = Inter({ subsets: ['latin'] })
+
+export const metadata: Metadata = {
+ title: 'AI Search Platform Demo',
+ description: 'Experience the power of AI-driven search and content discovery',
+}
+
+export default function RootLayout({
+ children,
+}: {
+ children: React.ReactNode
+}) {
+ return (
+
+
{children}
+
+ )
+}
\ No newline at end of file
diff --git a/apps/demo/app/page.tsx b/apps/demo/app/page.tsx
new file mode 100644
index 0000000..d7d546f
--- /dev/null
+++ b/apps/demo/app/page.tsx
@@ -0,0 +1,330 @@
+'use client'
+
+import { useEffect, useState } from 'react'
+import { Search, Zap, Shield, BarChart3, Sparkles, Code, Globe, Users } from 'lucide-react'
+import { SearchWidget } from '@ai-search/widget'
+import type { WidgetConfig } from '@ai-search/shared'
+
+export default function Home() {
+ const [widgetLoaded, setWidgetLoaded] = useState(false)
+
+ // Demo widget configuration
+ const widgetConfig: WidgetConfig = {
+ id: 'demo-widget-001',
+ name: 'AI Search Demo',
+ apiKey: 'demo-api-key-12345',
+ theme: {
+ primaryColor: '#3b82f6',
+ secondaryColor: '#6b7280',
+ backgroundColor: '#ffffff',
+ textColor: '#1f2937',
+ borderRadius: 12,
+ fontFamily: 'Inter, system-ui, sans-serif',
+ fontSize: 14,
+ },
+ layout: {
+ position: 'bottom-right',
+ width: 420,
+ height: 600,
+ offset: { x: 24, y: 24 },
+ zIndex: 9999,
+ },
+ behavior: {
+ autoOpen: false,
+ showWelcomeMessage: true,
+ enableVoiceInput: false,
+ enableFileUpload: true,
+ enableFeedback: true,
+ enableAnalytics: true,
+ },
+ content: {
+ welcomeMessage: "👋 Hi! I'm your AI search assistant. Ask me anything about our platform!",
+ placeholder: 'Ask me anything...',
+ noResultsMessage: "I couldn't find what you're looking for. Try rephrasing your question!",
+ errorMessage: 'Oops! Something went wrong. Please try again.',
+ loadingMessage: 'Searching...',
+ poweredByText: 'Powered by AI Search Platform',
+ feedbackPrompt: 'Was this helpful?',
+ },
+ features: {
+ enableSemanticSearch: true,
+ enableEnhancedAnswers: true,
+ enableAutoComplete: true,
+ enableSearchSuggestions: true,
+ enableResultHighlighting: true,
+ enableSourceCitations: true,
+ maxResults: 8,
+ enableFilters: false,
+ },
+ security: {
+ enableCSP: true,
+ enableCORS: true,
+ rateLimitPerMinute: 60,
+ },
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ }
+
+ useEffect(() => {
+ // Simulate widget loading
+ const timer = setTimeout(() => setWidgetLoaded(true), 1000)
+ return () => clearTimeout(timer)
+ }, [])
+
+ const features = [
+ {
+ icon: ,
+ title: 'Semantic Search',
+ description: 'AI-powered search that understands context and intent, not just keywords.',
+ },
+ {
+ icon: ,
+ title: 'Enhanced Answers',
+ description: 'Get comprehensive answers with source citations powered by advanced LLMs.',
+ },
+ {
+ icon: ,
+ title: 'Easy Integration',
+ description: 'Drop-in widget that works with any website. No complex setup required.',
+ },
+ {
+ icon: ,
+ title: 'Real-time Analytics',
+ description: 'Track search behavior, user engagement, and content performance.',
+ },
+ {
+ icon: ,
+ title: 'Content Guardrails',
+ description: 'Built-in validation and filtering to ensure high-quality responses.',
+ },
+ {
+ icon: ,
+ title: 'Multi-language',
+ description: 'Support for multiple languages with automatic detection.',
+ },
+ ]
+
+ const stats = [
+ { label: 'Search Accuracy', value: '95%' },
+ { label: 'Response Time', value: '<200ms' },
+ { label: 'Uptime', value: '99.9%' },
+ { label: 'Languages', value: '25+' },
+ ]
+
+ return (
+
+ {/* Header */}
+
+
+ {/* Hero Section */}
+
+
+
+
+ AI-Powered Search
+ Made Simple
+
+
+ Transform your website with intelligent search capabilities.
+ Deliver instant, personalized experiences with semantic search,
+ enhanced answers, and real-time analytics.
+
+
+
+
+
+
+
+
+
+ {/* Stats Section */}
+
+
+
+ {stats.map((stat, index) => (
+
+
+ {stat.value}
+
+
{stat.label}
+
+ ))}
+
+
+
+
+ {/* Features Section */}
+
+
+
+
+ Powerful Features
+
+
+ Everything you need to deliver exceptional search experiences
+
+
+
+
+ {features.map((feature, index) => (
+
+
{feature.icon}
+
+ {feature.title}
+
+
{feature.description}
+
+ ))}
+
+
+
+
+ {/* Demo Section */}
+
+
+
+
+ See It In Action
+
+
+ Try our AI search widget right here! Click the chat button in the bottom-right corner.
+
+
+
+
+
+
+
+ Interactive Demo
+
+
+
+
+
+
Ask Natural Questions
+
Try "How do I integrate the search widget?" or "What are the pricing plans?"
+
+
+
+
+
+
Get Enhanced Answers
+
Receive comprehensive responses with source citations and related content.
+
+
+
+
+
+
Provide Feedback
+
Help improve the system by rating responses and providing feedback.
+
+
+
+
+
+
+
+
+
+
+ Widget Active
+
+
+ The AI search widget is now available in the bottom-right corner.
+ Click it to start exploring!
+
+
+
+
+
+
+
+
+ {/* CTA Section */}
+
+
+
+ Ready to Transform Your Search?
+
+
+ Join thousands of websites already using AI Search Platform to deliver
+ exceptional user experiences.
+
+
+
+
+
+
+
+
+ {/* Footer */}
+
+
+ {/* AI Search Widget */}
+ {widgetLoaded && (
+
console.error('Widget error:', error)}
+ />
+ )}
+
+ )
+}
\ No newline at end of file
diff --git a/apps/demo/next.config.js b/apps/demo/next.config.js
new file mode 100644
index 0000000..8300a26
--- /dev/null
+++ b/apps/demo/next.config.js
@@ -0,0 +1,9 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+ experimental: {
+ appDir: true,
+ },
+ transpilePackages: ['@ai-search/shared', '@ai-search/widget'],
+}
+
+module.exports = nextConfig
\ No newline at end of file
diff --git a/apps/demo/package.json b/apps/demo/package.json
new file mode 100644
index 0000000..739becc
--- /dev/null
+++ b/apps/demo/package.json
@@ -0,0 +1,33 @@
+{
+ "name": "@ai-search/demo",
+ "version": "1.0.0",
+ "description": "Demo application showcasing AI Search Platform",
+ "private": true,
+ "scripts": {
+ "dev": "next dev",
+ "build": "next build",
+ "start": "next start",
+ "lint": "next lint",
+ "type-check": "tsc --noEmit"
+ },
+ "dependencies": {
+ "@ai-search/shared": "workspace:*",
+ "@ai-search/widget": "workspace:*",
+ "next": "14.0.3",
+ "react": "^18.2.0",
+ "react-dom": "^18.2.0",
+ "lucide-react": "^0.292.0",
+ "clsx": "^2.0.0",
+ "tailwindcss": "^3.3.6"
+ },
+ "devDependencies": {
+ "@types/node": "^20.8.0",
+ "@types/react": "^18.2.37",
+ "@types/react-dom": "^18.2.15",
+ "autoprefixer": "^10.4.16",
+ "eslint": "^8.51.0",
+ "eslint-config-next": "14.0.3",
+ "postcss": "^8.4.31",
+ "typescript": "^5.2.2"
+ }
+}
\ No newline at end of file
diff --git a/apps/demo/postcss.config.js b/apps/demo/postcss.config.js
new file mode 100644
index 0000000..96bb01e
--- /dev/null
+++ b/apps/demo/postcss.config.js
@@ -0,0 +1,6 @@
+module.exports = {
+ plugins: {
+ tailwindcss: {},
+ autoprefixer: {},
+ },
+}
\ No newline at end of file
diff --git a/apps/demo/tailwind.config.js b/apps/demo/tailwind.config.js
new file mode 100644
index 0000000..d8f4a05
--- /dev/null
+++ b/apps/demo/tailwind.config.js
@@ -0,0 +1,21 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+ content: [
+ './pages/**/*.{js,ts,jsx,tsx,mdx}',
+ './components/**/*.{js,ts,jsx,tsx,mdx}',
+ './app/**/*.{js,ts,jsx,tsx,mdx}',
+ ],
+ theme: {
+ extend: {
+ colors: {
+ primary: {
+ 50: '#eff6ff',
+ 500: '#3b82f6',
+ 600: '#2563eb',
+ 700: '#1d4ed8',
+ }
+ }
+ },
+ },
+ plugins: [],
+}
\ No newline at end of file
diff --git a/infrastructure/kubernetes/api-deployment.yaml b/infrastructure/kubernetes/api-deployment.yaml
new file mode 100644
index 0000000..60ceb44
--- /dev/null
+++ b/infrastructure/kubernetes/api-deployment.yaml
@@ -0,0 +1,98 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: ai-search-api
+ labels:
+ app: ai-search-api
+spec:
+ replicas: 3
+ selector:
+ matchLabels:
+ app: ai-search-api
+ template:
+ metadata:
+ labels:
+ app: ai-search-api
+ spec:
+ containers:
+ - name: api
+ image: ghcr.io/ai-search-platform/api:latest
+ ports:
+ - containerPort: 8000
+ env:
+ - name: DATABASE_URL
+ valueFrom:
+ secretKeyRef:
+ name: ai-search-secrets
+ key: database-url
+ - name: REDIS_URL
+ valueFrom:
+ secretKeyRef:
+ name: ai-search-secrets
+ key: redis-url
+ - name: SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ name: ai-search-secrets
+ key: secret-key
+ - name: OPENAI_API_KEY
+ valueFrom:
+ secretKeyRef:
+ name: ai-search-secrets
+ key: openai-api-key
+ resources:
+ requests:
+ memory: "512Mi"
+ cpu: "250m"
+ limits:
+ memory: "1Gi"
+ cpu: "500m"
+ livenessProbe:
+ httpGet:
+ path: /health
+ port: 8000
+ initialDelaySeconds: 30
+ periodSeconds: 10
+ readinessProbe:
+ httpGet:
+ path: /health/ready
+ port: 8000
+ initialDelaySeconds: 5
+ periodSeconds: 5
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: ai-search-api-service
+spec:
+ selector:
+ app: ai-search-api
+ ports:
+ - protocol: TCP
+ port: 80
+ targetPort: 8000
+ type: ClusterIP
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: ai-search-api-ingress
+ annotations:
+ nginx.ingress.kubernetes.io/rewrite-target: /
+ cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+ tls:
+ - hosts:
+ - api.aisearch.com
+ secretName: api-tls
+ rules:
+ - host: api.aisearch.com
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: ai-search-api-service
+ port:
+ number: 80
\ No newline at end of file
diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf
new file mode 100644
index 0000000..c92072d
--- /dev/null
+++ b/infrastructure/terraform/main.tf
@@ -0,0 +1,271 @@
+terraform {
+ required_version = ">= 1.0"
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 5.0"
+ }
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.23"
+ }
+ }
+}
+
+provider "aws" {
+ region = var.aws_region
+}
+
+# VPC Configuration
+module "vpc" {
+ source = "terraform-aws-modules/vpc/aws"
+
+ name = "${var.project_name}-vpc"
+ cidr = "10.0.0.0/16"
+
+ azs = ["${var.aws_region}a", "${var.aws_region}b", "${var.aws_region}c"]
+ private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
+ public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"]
+
+ enable_nat_gateway = true
+ enable_vpn_gateway = false
+ enable_dns_hostnames = true
+ enable_dns_support = true
+
+ tags = {
+ Environment = var.environment
+ Project = var.project_name
+ }
+}
+
+# EKS Cluster
+module "eks" {
+ source = "terraform-aws-modules/eks/aws"
+
+ cluster_name = "${var.project_name}-${var.environment}"
+ cluster_version = "1.28"
+
+ vpc_id = module.vpc.vpc_id
+ subnet_ids = module.vpc.private_subnets
+
+ # EKS Managed Node Groups
+ eks_managed_node_groups = {
+ main = {
+ desired_size = 2
+ max_size = 10
+ min_size = 1
+
+ instance_types = ["t3.medium"]
+ capacity_type = "ON_DEMAND"
+
+ k8s_labels = {
+ Environment = var.environment
+ NodeGroup = "main"
+ }
+ }
+ }
+
+ tags = {
+ Environment = var.environment
+ Project = var.project_name
+ }
+}
+
+# RDS PostgreSQL
+resource "aws_db_subnet_group" "main" {
+ name = "${var.project_name}-db-subnet-group"
+ subnet_ids = module.vpc.private_subnets
+
+ tags = {
+ Name = "${var.project_name} DB subnet group"
+ }
+}
+
+resource "aws_db_instance" "postgres" {
+ identifier = "${var.project_name}-postgres-${var.environment}"
+
+ engine = "postgres"
+ engine_version = "15.4"
+ instance_class = var.db_instance_class
+
+ allocated_storage = 20
+ max_allocated_storage = 100
+ storage_type = "gp3"
+ storage_encrypted = true
+
+ db_name = "aisearch"
+ username = "aisearch"
+ password = var.db_password
+
+ vpc_security_group_ids = [aws_security_group.rds.id]
+ db_subnet_group_name = aws_db_subnet_group.main.name
+
+ backup_retention_period = 7
+ backup_window = "03:00-04:00"
+ maintenance_window = "sun:04:00-sun:05:00"
+
+ skip_final_snapshot = var.environment != "production"
+ deletion_protection = var.environment == "production"
+
+ tags = {
+ Environment = var.environment
+ Project = var.project_name
+ }
+}
+
+# ElastiCache Redis
+resource "aws_elasticache_subnet_group" "main" {
+ name = "${var.project_name}-cache-subnet"
+ subnet_ids = module.vpc.private_subnets
+}
+
+resource "aws_elasticache_replication_group" "redis" {
+ replication_group_id = "${var.project_name}-redis-${var.environment}"
+ description = "Redis cluster for ${var.project_name}"
+
+ node_type = var.redis_node_type
+ port = 6379
+ parameter_group_name = "default.redis7"
+
+ num_cache_clusters = 2
+
+ subnet_group_name = aws_elasticache_subnet_group.main.name
+ security_group_ids = [aws_security_group.redis.id]
+
+ at_rest_encryption_enabled = true
+ transit_encryption_enabled = true
+
+ tags = {
+ Environment = var.environment
+ Project = var.project_name
+ }
+}
+
+# Security Groups
+resource "aws_security_group" "rds" {
+ name_prefix = "${var.project_name}-rds-"
+ vpc_id = module.vpc.vpc_id
+
+ ingress {
+ from_port = 5432
+ to_port = 5432
+ protocol = "tcp"
+ cidr_blocks = [module.vpc.vpc_cidr_block]
+ }
+
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ tags = {
+ Name = "${var.project_name}-rds-sg"
+ }
+}
+
+resource "aws_security_group" "redis" {
+ name_prefix = "${var.project_name}-redis-"
+ vpc_id = module.vpc.vpc_id
+
+ ingress {
+ from_port = 6379
+ to_port = 6379
+ protocol = "tcp"
+ cidr_blocks = [module.vpc.vpc_cidr_block]
+ }
+
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ tags = {
+ Name = "${var.project_name}-redis-sg"
+ }
+}
+
+# S3 Bucket for file storage
+resource "aws_s3_bucket" "storage" {
+ bucket = "${var.project_name}-storage-${var.environment}-${random_string.bucket_suffix.result}"
+
+ tags = {
+ Environment = var.environment
+ Project = var.project_name
+ }
+}
+
+resource "random_string" "bucket_suffix" {
+ length = 8
+ special = false
+ upper = false
+}
+
+resource "aws_s3_bucket_versioning" "storage" {
+ bucket = aws_s3_bucket.storage.id
+ versioning_configuration {
+ status = "Enabled"
+ }
+}
+
+resource "aws_s3_bucket_encryption" "storage" {
+ bucket = aws_s3_bucket.storage.id
+
+ server_side_encryption_configuration {
+ rule {
+ apply_server_side_encryption_by_default {
+ sse_algorithm = "AES256"
+ }
+ }
+ }
+}
+
+# CloudWatch Log Groups
+resource "aws_cloudwatch_log_group" "api" {
+ name = "/aws/eks/${var.project_name}-${var.environment}/api"
+ retention_in_days = 30
+
+ tags = {
+ Environment = var.environment
+ Project = var.project_name
+ }
+}
+
+# Outputs
+output "cluster_endpoint" {
+ description = "Endpoint for EKS control plane"
+ value = module.eks.cluster_endpoint
+}
+
+output "cluster_security_group_id" {
+ description = "Security group ids attached to the cluster control plane"
+ value = module.eks.cluster_security_group_id
+}
+
+output "cluster_iam_role_name" {
+ description = "IAM role name associated with EKS cluster"
+ value = module.eks.cluster_iam_role_name
+}
+
+output "cluster_certificate_authority_data" {
+ description = "Base64 encoded certificate data required to communicate with the cluster"
+ value = module.eks.cluster_certificate_authority_data
+}
+
+output "rds_endpoint" {
+ description = "RDS instance endpoint"
+ value = aws_db_instance.postgres.endpoint
+}
+
+output "redis_endpoint" {
+ description = "ElastiCache Redis endpoint"
+ value = aws_elasticache_replication_group.redis.configuration_endpoint_address
+}
+
+output "s3_bucket_name" {
+ description = "Name of the S3 bucket"
+ value = aws_s3_bucket.storage.bucket
+}
\ No newline at end of file
diff --git a/infrastructure/terraform/variables.tf b/infrastructure/terraform/variables.tf
new file mode 100644
index 0000000..d63f6f1
--- /dev/null
+++ b/infrastructure/terraform/variables.tf
@@ -0,0 +1,35 @@
+variable "aws_region" {
+ description = "AWS region"
+ type = string
+ default = "us-west-2"
+}
+
+variable "environment" {
+ description = "Environment name"
+ type = string
+ default = "staging"
+}
+
+variable "project_name" {
+ description = "Name of the project"
+ type = string
+ default = "ai-search-platform"
+}
+
+variable "db_instance_class" {
+ description = "RDS instance class"
+ type = string
+ default = "db.t3.micro"
+}
+
+variable "db_password" {
+ description = "Database password"
+ type = string
+ sensitive = true
+}
+
+variable "redis_node_type" {
+ description = "ElastiCache node type"
+ type = string
+ default = "cache.t3.micro"
+}
\ No newline at end of file
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..43f9fc4
--- /dev/null
+++ b/package.json
@@ -0,0 +1,31 @@
+{
+ "name": "ai-search-platform",
+ "version": "1.0.0",
+ "description": "Production-grade AI-powered website search and content discovery platform",
+ "private": true,
+ "workspaces": [
+ "packages/*",
+ "apps/*"
+ ],
+ "scripts": {
+ "dev": "turbo run dev",
+ "build": "turbo run build",
+ "test": "turbo run test",
+ "lint": "turbo run lint",
+ "clean": "turbo run clean",
+ "type-check": "turbo run type-check",
+ "format": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md}\"",
+ "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md}\""
+ },
+ "devDependencies": {
+ "@turbo/gen": "^1.10.12",
+ "turbo": "^1.10.12",
+ "prettier": "^3.0.3",
+ "typescript": "^5.2.2",
+ "@types/node": "^20.8.0"
+ },
+ "engines": {
+ "node": ">=18.0.0",
+ "npm": ">=8.0.0"
+ }
+}
\ No newline at end of file
diff --git a/packages/api/app/__init__.py b/packages/api/app/__init__.py
new file mode 100644
index 0000000..2b4d0a1
--- /dev/null
+++ b/packages/api/app/__init__.py
@@ -0,0 +1 @@
+# AI Search Platform API
\ No newline at end of file
diff --git a/packages/api/app/api/__init__.py b/packages/api/app/api/__init__.py
new file mode 100644
index 0000000..8aace33
--- /dev/null
+++ b/packages/api/app/api/__init__.py
@@ -0,0 +1 @@
+# API routes package
\ No newline at end of file
diff --git a/packages/api/app/api/endpoints/__init__.py b/packages/api/app/api/endpoints/__init__.py
new file mode 100644
index 0000000..480aee4
--- /dev/null
+++ b/packages/api/app/api/endpoints/__init__.py
@@ -0,0 +1 @@
+# API endpoints package
\ No newline at end of file
diff --git a/packages/api/app/api/endpoints/documents.py b/packages/api/app/api/endpoints/documents.py
new file mode 100644
index 0000000..b4eeb87
--- /dev/null
+++ b/packages/api/app/api/endpoints/documents.py
@@ -0,0 +1,342 @@
+"""Document management API endpoints."""
+
+import asyncio
+from typing import Dict, Any, List, Optional
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form
+from sqlalchemy.ext.asyncio import AsyncSession
+from pydantic import BaseModel, Field, HttpUrl
+
+from app.core.database import get_db
+from app.core.security import require_api_key
+from app.core.exceptions import ValidationError, NotFoundError, ServiceUnavailableError
+from app.services.document_service import DocumentService
+from app.services.ingestion_service import IngestionService
+
+# Pydantic models
+class DocumentIngestionRequest(BaseModel):
+ url: Optional[HttpUrl] = None
+ content: Optional[str] = None
+ title: str = Field(min_length=1, max_length=500)
+ content_type: str = Field(default="text", pattern="^(text|html|markdown|pdf|doc|docx)$")
+ metadata: Optional[Dict[str, Any]] = None
+ tags: List[str] = Field(default_factory=list)
+ source_id: Optional[str] = None
+
+class DocumentResponse(BaseModel):
+ id: str
+ url: Optional[str]
+ title: str
+ content_type: str
+ status: str
+ metadata: Optional[Dict[str, Any]]
+ tags: List[str]
+ source_id: Optional[str]
+ created_at: str
+ updated_at: str
+ indexed_at: Optional[str]
+
+class DocumentListResponse(BaseModel):
+ documents: List[DocumentResponse]
+ total: int
+ page: int
+ limit: int
+ has_next: bool
+ has_prev: bool
+
+class BatchIngestionRequest(BaseModel):
+ documents: List[DocumentIngestionRequest]
+
+class BatchIngestionResponse(BaseModel):
+ job_id: str
+ status: str
+ total_documents: int
+ message: str
+
+router = APIRouter()
+
+
+@router.post("/ingest", response_model=DocumentResponse)
+async def ingest_document(
+ request_data: DocumentIngestionRequest,
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+) -> DocumentResponse:
+ """
+ Ingest a single document for indexing.
+
+ Supports multiple content types and sources:
+ - Direct content input
+ - URL-based content fetching
+ - File upload (via separate endpoint)
+ """
+ try:
+ ingestion_service = IngestionService(db)
+
+ # Validate that either URL or content is provided
+ if not request_data.url and not request_data.content:
+ raise ValidationError("Either 'url' or 'content' must be provided")
+
+ document = await ingestion_service.ingest_document(
+ client_id=client_info["client_id"],
+ url=str(request_data.url) if request_data.url else None,
+ content=request_data.content,
+ title=request_data.title,
+ content_type=request_data.content_type,
+ metadata=request_data.metadata,
+ tags=request_data.tags,
+ source_id=request_data.source_id,
+ )
+
+ return DocumentResponse(
+ id=str(document.id),
+ url=document.url,
+ title=document.title,
+ content_type=document.content_type,
+ status=document.status,
+ metadata=document.metadata,
+ tags=document.tags or [],
+ source_id=document.source_id,
+ created_at=document.created_at.isoformat(),
+ updated_at=document.updated_at.isoformat(),
+ indexed_at=document.indexed_at.isoformat() if document.indexed_at else None,
+ )
+
+ except ValidationError:
+ raise
+ except Exception as e:
+ raise ServiceUnavailableError(f"Document ingestion error: {str(e)}")
+
+
+@router.post("/ingest/batch", response_model=BatchIngestionResponse)
+async def ingest_documents_batch(
+ request_data: BatchIngestionRequest,
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+) -> BatchIngestionResponse:
+ """
+ Ingest multiple documents in batch for efficient processing.
+
+ Creates a background job to process all documents asynchronously.
+ """
+ try:
+ ingestion_service = IngestionService(db)
+
+ job = await ingestion_service.create_batch_ingestion_job(
+ client_id=client_info["client_id"],
+ documents=[doc.dict() for doc in request_data.documents],
+ )
+
+ return BatchIngestionResponse(
+ job_id=str(job.id),
+ status=job.status,
+ total_documents=len(request_data.documents),
+ message="Batch ingestion job created successfully",
+ )
+
+ except Exception as e:
+ raise ServiceUnavailableError(f"Batch ingestion error: {str(e)}")
+
+
+@router.post("/upload", response_model=DocumentResponse)
+async def upload_document(
+ file: UploadFile = File(...),
+ title: str = Form(...),
+ content_type: str = Form(default="auto"),
+ metadata: Optional[str] = Form(default=None),
+ tags: Optional[str] = Form(default=None),
+ source_id: Optional[str] = Form(default=None),
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+) -> DocumentResponse:
+ """
+ Upload and ingest a document file.
+
+ Supports various file formats:
+ - PDF documents
+ - Word documents (DOC, DOCX)
+ - Text files
+ - HTML files
+ - Markdown files
+ """
+ try:
+ ingestion_service = IngestionService(db)
+
+ # Parse optional JSON fields
+ import json
+ parsed_metadata = json.loads(metadata) if metadata else None
+ parsed_tags = json.loads(tags) if tags else []
+
+ document = await ingestion_service.ingest_file(
+ client_id=client_info["client_id"],
+ file=file,
+ title=title,
+ content_type=content_type,
+ metadata=parsed_metadata,
+ tags=parsed_tags,
+ source_id=source_id,
+ )
+
+ return DocumentResponse(
+ id=str(document.id),
+ url=document.url,
+ title=document.title,
+ content_type=document.content_type,
+ status=document.status,
+ metadata=document.metadata,
+ tags=document.tags or [],
+ source_id=document.source_id,
+ created_at=document.created_at.isoformat(),
+ updated_at=document.updated_at.isoformat(),
+ indexed_at=document.indexed_at.isoformat() if document.indexed_at else None,
+ )
+
+ except ValidationError:
+ raise
+ except Exception as e:
+ raise ServiceUnavailableError(f"File upload error: {str(e)}")
+
+
+@router.get("/", response_model=DocumentListResponse)
+async def list_documents(
+ page: int = 1,
+ limit: int = 20,
+ status: Optional[str] = None,
+ content_type: Optional[str] = None,
+ source_id: Optional[str] = None,
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+) -> DocumentListResponse:
+ """
+ List documents for the authenticated client.
+
+ Supports filtering by status, content type, and source.
+ """
+ try:
+ document_service = DocumentService(db)
+
+ result = await document_service.list_documents(
+ client_id=client_info["client_id"],
+ page=page,
+ limit=limit,
+ status=status,
+ content_type=content_type,
+ source_id=source_id,
+ )
+
+ return DocumentListResponse(
+ documents=[
+ DocumentResponse(
+ id=str(doc.id),
+ url=doc.url,
+ title=doc.title,
+ content_type=doc.content_type,
+ status=doc.status,
+ metadata=doc.metadata,
+ tags=doc.tags or [],
+ source_id=doc.source_id,
+ created_at=doc.created_at.isoformat(),
+ updated_at=doc.updated_at.isoformat(),
+ indexed_at=doc.indexed_at.isoformat() if doc.indexed_at else None,
+ ) for doc in result["documents"]
+ ],
+ total=result["total"],
+ page=page,
+ limit=limit,
+ has_next=result["has_next"],
+ has_prev=result["has_prev"],
+ )
+
+ except Exception as e:
+ raise ServiceUnavailableError(f"Document listing error: {str(e)}")
+
+
+@router.get("/{document_id}", response_model=DocumentResponse)
+async def get_document(
+ document_id: UUID,
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+) -> DocumentResponse:
+ """Get a specific document by ID."""
+ try:
+ document_service = DocumentService(db)
+
+ document = await document_service.get_document(
+ document_id=document_id,
+ client_id=client_info["client_id"],
+ )
+
+ if not document:
+ raise NotFoundError("Document not found")
+
+ return DocumentResponse(
+ id=str(document.id),
+ url=document.url,
+ title=document.title,
+ content_type=document.content_type,
+ status=document.status,
+ metadata=document.metadata,
+ tags=document.tags or [],
+ source_id=document.source_id,
+ created_at=document.created_at.isoformat(),
+ updated_at=document.updated_at.isoformat(),
+ indexed_at=document.indexed_at.isoformat() if document.indexed_at else None,
+ )
+
+ except NotFoundError:
+ raise
+ except Exception as e:
+ raise ServiceUnavailableError(f"Document retrieval error: {str(e)}")
+
+
+@router.delete("/{document_id}")
+async def delete_document(
+ document_id: UUID,
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+):
+ """Delete a document and all associated data."""
+ try:
+ document_service = DocumentService(db)
+
+ success = await document_service.delete_document(
+ document_id=document_id,
+ client_id=client_info["client_id"],
+ )
+
+ if not success:
+ raise NotFoundError("Document not found")
+
+ return {"success": True, "message": "Document deleted successfully"}
+
+ except NotFoundError:
+ raise
+ except Exception as e:
+ raise ServiceUnavailableError(f"Document deletion error: {str(e)}")
+
+
+@router.post("/{document_id}/reindex")
+async def reindex_document(
+ document_id: UUID,
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+):
+ """Reindex a document with updated embeddings."""
+ try:
+ ingestion_service = IngestionService(db)
+
+ success = await ingestion_service.reindex_document(
+ document_id=document_id,
+ client_id=client_info["client_id"],
+ )
+
+ if not success:
+ raise NotFoundError("Document not found")
+
+ return {"success": True, "message": "Document reindexing initiated"}
+
+ except NotFoundError:
+ raise
+ except Exception as e:
+ raise ServiceUnavailableError(f"Document reindexing error: {str(e)}")
\ No newline at end of file
diff --git a/packages/api/app/api/endpoints/search.py b/packages/api/app/api/endpoints/search.py
new file mode 100644
index 0000000..e2fad43
--- /dev/null
+++ b/packages/api/app/api/endpoints/search.py
@@ -0,0 +1,227 @@
+"""Search API endpoints."""
+
+import time
+from typing import Dict, Any, List
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.database import get_db
+from app.core.security import require_api_key
+from app.core.exceptions import ValidationError, ServiceUnavailableError
+from app.services.search_service import SearchService
+from app.services.analytics_service import AnalyticsService
+
+# Import shared types (in a real implementation, these would be properly imported)
+# For now, we'll define basic Pydantic models
+
+from pydantic import BaseModel, Field
+from typing import Optional, List as ListType
+
+class SearchFilters(BaseModel):
+ content_type: Optional[ListType[str]] = None
+ tags: Optional[ListType[str]] = None
+ language: Optional[str] = None
+ source_id: Optional[str] = None
+
+class SearchOptions(BaseModel):
+ limit: int = Field(default=10, ge=1, le=100)
+ offset: int = Field(default=0, ge=0)
+ include_content: bool = True
+ include_metadata: bool = False
+ min_score: Optional[float] = Field(default=None, ge=0, le=1)
+ search_type: str = Field(default="semantic", pattern="^(semantic|keyword|hybrid)$")
+ enhance_with_llm: bool = False
+
+class SearchRequest(BaseModel):
+ query: str = Field(min_length=1, max_length=1000)
+ filters: Optional[SearchFilters] = None
+ options: Optional[SearchOptions] = None
+ session_id: Optional[str] = None
+ user_id: Optional[str] = None
+
+class SearchResultItem(BaseModel):
+ id: str
+ document_id: str
+ title: str
+ content: str
+ url: Optional[str] = None
+ score: float
+ metadata: Optional[Dict[str, Any]] = None
+ highlights: Optional[List[Dict[str, Any]]] = None
+ chunk_index: Optional[int] = None
+
+class EnhancedAnswer(BaseModel):
+ content: str
+ sources: List[str]
+ confidence: float
+
+class SearchResponse(BaseModel):
+ results: List[SearchResultItem]
+ total: int
+ query: str
+ processing_time: float
+ enhanced_answer: Optional[EnhancedAnswer] = None
+ suggestions: Optional[List[str]] = None
+
+class AutoCompleteRequest(BaseModel):
+ query: str = Field(min_length=1, max_length=100)
+ limit: int = Field(default=5, ge=1, le=20)
+
+class AutoCompleteSuggestion(BaseModel):
+ text: str
+ score: float
+ type: str
+
+class AutoCompleteResponse(BaseModel):
+ suggestions: List[AutoCompleteSuggestion]
+
+router = APIRouter()
+
+
+@router.post("/", response_model=SearchResponse)
+async def search(
+ request_data: SearchRequest,
+ request: Request,
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+) -> SearchResponse:
+ """
+ Perform semantic search across indexed documents.
+
+ This endpoint provides AI-powered search capabilities including:
+ - Semantic search using vector embeddings
+ - Keyword search with full-text indexing
+ - Hybrid search combining both approaches
+ - Optional LLM-enhanced answers with source citations
+ """
+ start_time = time.time()
+
+ try:
+ # Initialize services
+ search_service = SearchService(db)
+ analytics_service = AnalyticsService(db)
+
+ # Perform search
+ search_result = await search_service.search(
+ query=request_data.query,
+ client_id=client_info["client_id"],
+ filters=request_data.filters.dict() if request_data.filters else None,
+ options=request_data.options.dict() if request_data.options else None,
+ )
+
+ processing_time = (time.time() - start_time) * 1000 # Convert to milliseconds
+
+ # Track analytics
+ await analytics_service.track_search_query(
+ client_id=client_info["client_id"],
+ session_id=request_data.session_id,
+ user_id=request_data.user_id,
+ query=request_data.query,
+ results_count=len(search_result["results"]),
+ processing_time=processing_time,
+ search_type=request_data.options.search_type if request_data.options else "semantic",
+ enhanced_answer=bool(search_result.get("enhanced_answer")),
+ )
+
+ return SearchResponse(
+ results=[
+ SearchResultItem(**result) for result in search_result["results"]
+ ],
+ total=search_result["total"],
+ query=request_data.query,
+ processing_time=processing_time,
+ enhanced_answer=EnhancedAnswer(**search_result["enhanced_answer"])
+ if search_result.get("enhanced_answer") else None,
+ suggestions=search_result.get("suggestions", []),
+ )
+
+ except ValidationError:
+ raise
+ except Exception as e:
+ raise ServiceUnavailableError(f"Search service error: {str(e)}")
+
+
+@router.post("/autocomplete", response_model=AutoCompleteResponse)
+async def autocomplete(
+ request_data: AutoCompleteRequest,
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+) -> AutoCompleteResponse:
+ """
+ Get search query autocompletion suggestions.
+
+ Provides intelligent query suggestions based on:
+ - Popular search queries
+ - Document titles and content
+ - User's search history
+ """
+ try:
+ search_service = SearchService(db)
+
+ suggestions = await search_service.get_autocomplete_suggestions(
+ query=request_data.query,
+ client_id=client_info["client_id"],
+ limit=request_data.limit,
+ )
+
+ return AutoCompleteResponse(
+ suggestions=[
+ AutoCompleteSuggestion(**suggestion) for suggestion in suggestions
+ ]
+ )
+
+ except Exception as e:
+ raise ServiceUnavailableError(f"Autocomplete service error: {str(e)}")
+
+
+@router.post("/feedback")
+async def submit_feedback(
+ feedback_data: Dict[str, Any],
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+):
+ """
+ Submit user feedback on search results or enhanced answers.
+
+ Feedback helps improve search quality and relevance.
+ """
+ try:
+ analytics_service = AnalyticsService(db)
+
+ await analytics_service.track_feedback(
+ client_id=client_info["client_id"],
+ feedback_data=feedback_data,
+ )
+
+ return {"success": True, "message": "Feedback submitted successfully"}
+
+ except Exception as e:
+ raise ServiceUnavailableError(f"Feedback service error: {str(e)}")
+
+
+@router.get("/suggestions")
+async def get_search_suggestions(
+ client_id: str,
+ limit: int = 10,
+ db: AsyncSession = Depends(get_db),
+ client_info: Dict[str, Any] = Depends(require_api_key),
+):
+ """
+ Get popular search suggestions for the client.
+
+ Returns trending and popular queries to help users discover content.
+ """
+ try:
+ search_service = SearchService(db)
+
+ suggestions = await search_service.get_popular_queries(
+ client_id=client_info["client_id"],
+ limit=limit,
+ )
+
+ return {"suggestions": suggestions}
+
+ except Exception as e:
+ raise ServiceUnavailableError(f"Suggestions service error: {str(e)}")
\ No newline at end of file
diff --git a/packages/api/app/api/routes.py b/packages/api/app/api/routes.py
new file mode 100644
index 0000000..7e08c64
--- /dev/null
+++ b/packages/api/app/api/routes.py
@@ -0,0 +1,33 @@
+"""Main API router configuration."""
+
+from fastapi import APIRouter
+
+from app.api.endpoints import search, documents, analytics, health
+
+# Create main API router
+api_router = APIRouter()
+
+# Include endpoint routers
+api_router.include_router(
+ search.router,
+ prefix="/search",
+ tags=["search"],
+)
+
+api_router.include_router(
+ documents.router,
+ prefix="/documents",
+ tags=["documents"],
+)
+
+api_router.include_router(
+ analytics.router,
+ prefix="/analytics",
+ tags=["analytics"],
+)
+
+api_router.include_router(
+ health.router,
+ prefix="/health",
+ tags=["health"],
+)
\ No newline at end of file
diff --git a/packages/api/app/core/config.py b/packages/api/app/core/config.py
new file mode 100644
index 0000000..2ad832a
--- /dev/null
+++ b/packages/api/app/core/config.py
@@ -0,0 +1,146 @@
+"""
+Application configuration management.
+"""
+
+import os
+from functools import lru_cache
+from typing import List, Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+ """Application settings."""
+
+ # Environment
+ DEBUG: bool = Field(default=False, env="DEBUG")
+ ENVIRONMENT: str = Field(default="development", env="ENVIRONMENT")
+
+ # API Configuration
+ API_HOST: str = Field(default="0.0.0.0", env="API_HOST")
+ API_PORT: int = Field(default=8000, env="API_PORT")
+ API_WORKERS: int = Field(default=1, env="API_WORKERS")
+
+ # Security
+ SECRET_KEY: str = Field(env="SECRET_KEY")
+ JWT_ALGORITHM: str = Field(default="HS256", env="JWT_ALGORITHM")
+ JWT_EXPIRE_MINUTES: int = Field(default=1440, env="JWT_EXPIRE_MINUTES") # 24 hours
+
+ # CORS
+ ALLOWED_ORIGINS: List[str] = Field(
+ default=["http://localhost:3000", "http://localhost:8080"],
+ env="ALLOWED_ORIGINS"
+ )
+
+ # Database
+ DATABASE_URL: str = Field(env="DATABASE_URL")
+ DATABASE_POOL_SIZE: int = Field(default=10, env="DATABASE_POOL_SIZE")
+ DATABASE_MAX_OVERFLOW: int = Field(default=20, env="DATABASE_MAX_OVERFLOW")
+
+ # Redis
+ REDIS_URL: str = Field(env="REDIS_URL")
+ REDIS_MAX_CONNECTIONS: int = Field(default=10, env="REDIS_MAX_CONNECTIONS")
+
+ # OpenAI
+ OPENAI_API_KEY: Optional[str] = Field(default=None, env="OPENAI_API_KEY")
+ OPENAI_MODEL: str = Field(default="gpt-4-turbo-preview", env="OPENAI_MODEL")
+ OPENAI_EMBEDDING_MODEL: str = Field(
+ default="text-embedding-ada-002", env="OPENAI_EMBEDDING_MODEL"
+ )
+
+ # Anthropic
+ ANTHROPIC_API_KEY: Optional[str] = Field(default=None, env="ANTHROPIC_API_KEY")
+ ANTHROPIC_MODEL: str = Field(
+ default="claude-3-sonnet-20240229", env="ANTHROPIC_MODEL"
+ )
+
+ # HuggingFace
+ HUGGINGFACE_API_KEY: Optional[str] = Field(default=None, env="HUGGINGFACE_API_KEY")
+ HUGGINGFACE_MODEL: str = Field(
+ default="sentence-transformers/all-MiniLM-L6-v2", env="HUGGINGFACE_MODEL"
+ )
+
+ # Cohere
+ COHERE_API_KEY: Optional[str] = Field(default=None, env="COHERE_API_KEY")
+ COHERE_MODEL: str = Field(default="embed-english-v2.0", env="COHERE_MODEL")
+
+ # Vector Databases
+ # Milvus
+ MILVUS_HOST: str = Field(default="localhost", env="MILVUS_HOST")
+ MILVUS_PORT: int = Field(default=19530, env="MILVUS_PORT")
+ MILVUS_USER: Optional[str] = Field(default=None, env="MILVUS_USER")
+ MILVUS_PASSWORD: Optional[str] = Field(default=None, env="MILVUS_PASSWORD")
+
+ # Pinecone
+ PINECONE_API_KEY: Optional[str] = Field(default=None, env="PINECONE_API_KEY")
+ PINECONE_ENVIRONMENT: Optional[str] = Field(default=None, env="PINECONE_ENVIRONMENT")
+ PINECONE_INDEX_NAME: str = Field(default="ai-search", env="PINECONE_INDEX_NAME")
+
+ # Weaviate
+ WEAVIATE_URL: Optional[str] = Field(default=None, env="WEAVIATE_URL")
+ WEAVIATE_API_KEY: Optional[str] = Field(default=None, env="WEAVIATE_API_KEY")
+ WEAVIATE_CLASS_NAME: str = Field(default="Document", env="WEAVIATE_CLASS_NAME")
+
+ # Qdrant
+ QDRANT_URL: Optional[str] = Field(default=None, env="QDRANT_URL")
+ QDRANT_API_KEY: Optional[str] = Field(default=None, env="QDRANT_API_KEY")
+ QDRANT_COLLECTION_NAME: str = Field(default="documents", env="QDRANT_COLLECTION_NAME")
+
+ # Vector Database Selection
+ VECTOR_DB_PROVIDER: str = Field(default="milvus", env="VECTOR_DB_PROVIDER")
+ VECTOR_DB_DIMENSIONS: int = Field(default=1536, env="VECTOR_DB_DIMENSIONS")
+
+ # Content Processing
+ MAX_DOCUMENT_SIZE: int = Field(default=50 * 1024 * 1024, env="MAX_DOCUMENT_SIZE") # 50MB
+ CHUNK_SIZE: int = Field(default=1000, env="CHUNK_SIZE")
+ CHUNK_OVERLAP: int = Field(default=200, env="CHUNK_OVERLAP")
+
+ # Rate Limiting
+ RATE_LIMIT_REQUESTS: int = Field(default=100, env="RATE_LIMIT_REQUESTS")
+ RATE_LIMIT_WINDOW: int = Field(default=60, env="RATE_LIMIT_WINDOW") # seconds
+
+ # Celery (for background tasks)
+ CELERY_BROKER_URL: str = Field(env="CELERY_BROKER_URL")
+ CELERY_RESULT_BACKEND: str = Field(env="CELERY_RESULT_BACKEND")
+
+ # Monitoring
+ SENTRY_DSN: Optional[str] = Field(default=None, env="SENTRY_DSN")
+ LOG_LEVEL: str = Field(default="INFO", env="LOG_LEVEL")
+
+ # File Storage
+ UPLOAD_DIR: str = Field(default="uploads", env="UPLOAD_DIR")
+
+ class Config:
+ env_file = ".env"
+ env_file_encoding = "utf-8"
+ case_sensitive = True
+
+
+@lru_cache()
+def get_settings() -> Settings:
+ """Get cached application settings."""
+ return Settings()
+
+
+# Environment-specific configurations
+def get_database_url() -> str:
+ """Get database URL with proper configuration."""
+ settings = get_settings()
+ return settings.DATABASE_URL
+
+
+def get_redis_url() -> str:
+ """Get Redis URL with proper configuration."""
+ settings = get_settings()
+ return settings.REDIS_URL
+
+
+def is_development() -> bool:
+ """Check if running in development mode."""
+ return get_settings().ENVIRONMENT == "development"
+
+
+def is_production() -> bool:
+ """Check if running in production mode."""
+ return get_settings().ENVIRONMENT == "production"
\ No newline at end of file
diff --git a/packages/api/app/core/database.py b/packages/api/app/core/database.py
new file mode 100644
index 0000000..7e994a1
--- /dev/null
+++ b/packages/api/app/core/database.py
@@ -0,0 +1,125 @@
+"""
+Database configuration and connection management.
+"""
+
+import asyncio
+from typing import AsyncGenerator
+
+from sqlalchemy import MetaData, create_engine, pool
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+
+from app.core.config import get_settings
+
+# Create base class for models
+Base = declarative_base()
+
+# Naming convention for constraints
+convention = {
+ "ix": "ix_%(column_0_label)s",
+ "uq": "uq_%(table_name)s_%(column_0_name)s",
+ "ck": "ck_%(table_name)s_%(constraint_name)s",
+ "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+ "pk": "pk_%(table_name)s",
+}
+
+Base.metadata = MetaData(naming_convention=convention)
+
+# Database engines and sessions
+engine = None
+async_engine = None
+SessionLocal = None
+AsyncSessionLocal = None
+
+
+def create_database_engine():
+ """Create synchronous database engine."""
+ settings = get_settings()
+
+ # Convert async URL to sync URL for SQLAlchemy
+ sync_url = settings.DATABASE_URL.replace("postgresql+asyncpg://", "postgresql://")
+
+ return create_engine(
+ sync_url,
+ pool_size=settings.DATABASE_POOL_SIZE,
+ max_overflow=settings.DATABASE_MAX_OVERFLOW,
+ pool_pre_ping=True,
+ pool_recycle=300,
+ echo=settings.DEBUG,
+ )
+
+
+def create_async_database_engine():
+ """Create asynchronous database engine."""
+ settings = get_settings()
+
+ return create_async_engine(
+ settings.DATABASE_URL,
+ pool_size=settings.DATABASE_POOL_SIZE,
+ max_overflow=settings.DATABASE_MAX_OVERFLOW,
+ pool_pre_ping=True,
+ pool_recycle=300,
+ echo=settings.DEBUG,
+ )
+
+
+async def init_db():
+ """Initialize database connections."""
+ global engine, async_engine, SessionLocal, AsyncSessionLocal
+
+ # Create engines
+ engine = create_database_engine()
+ async_engine = create_async_database_engine()
+
+ # Create session makers
+ SessionLocal = sessionmaker(
+ bind=engine,
+ autocommit=False,
+ autoflush=False,
+ )
+
+ AsyncSessionLocal = async_sessionmaker(
+ bind=async_engine,
+ class_=AsyncSession,
+ autocommit=False,
+ autoflush=False,
+ expire_on_commit=False,
+ )
+
+
+async def get_db() -> AsyncGenerator[AsyncSession, None]:
+ """Get async database session."""
+ async with AsyncSessionLocal() as session:
+ try:
+ yield session
+ await session.commit()
+ except Exception:
+ await session.rollback()
+ raise
+ finally:
+ await session.close()
+
+
+def get_sync_db():
+ """Get synchronous database session."""
+ db = SessionLocal()
+ try:
+ yield db
+ db.commit()
+ except Exception:
+ db.rollback()
+ raise
+ finally:
+ db.close()
+
+
+async def close_db():
+ """Close database connections."""
+ global engine, async_engine
+
+ if async_engine:
+ await async_engine.dispose()
+
+ if engine:
+ engine.dispose()
\ No newline at end of file
diff --git a/packages/api/app/core/exceptions.py b/packages/api/app/core/exceptions.py
new file mode 100644
index 0000000..8c64656
--- /dev/null
+++ b/packages/api/app/core/exceptions.py
@@ -0,0 +1,133 @@
+"""
+Custom exception classes for the API.
+"""
+
+from typing import Any, Dict, Optional
+
+
+class APIException(Exception):
+ """Base exception class for API errors."""
+
+ def __init__(
+ self,
+ message: str,
+ status_code: int = 500,
+ error_code: str = "INTERNAL_SERVER_ERROR",
+ details: Optional[Dict[str, Any]] = None,
+ ):
+ self.message = message
+ self.status_code = status_code
+ self.error_code = error_code
+ self.details = details
+ super().__init__(message)
+
+
+class ValidationError(APIException):
+ """Validation error exception."""
+
+ def __init__(self, message: str, details: Optional[Dict[str, Any]] = None):
+ super().__init__(
+ message=message,
+ status_code=422,
+ error_code="VALIDATION_ERROR",
+ details=details,
+ )
+
+
+class AuthenticationError(APIException):
+ """Authentication error exception."""
+
+ def __init__(self, message: str = "Authentication failed"):
+ super().__init__(
+ message=message,
+ status_code=401,
+ error_code="AUTHENTICATION_ERROR",
+ )
+
+
+class AuthorizationError(APIException):
+ """Authorization error exception."""
+
+ def __init__(self, message: str = "Insufficient permissions"):
+ super().__init__(
+ message=message,
+ status_code=403,
+ error_code="AUTHORIZATION_ERROR",
+ )
+
+
+class NotFoundError(APIException):
+ """Resource not found exception."""
+
+ def __init__(self, message: str = "Resource not found"):
+ super().__init__(
+ message=message,
+ status_code=404,
+ error_code="NOT_FOUND",
+ )
+
+
+class ConflictError(APIException):
+ """Resource conflict exception."""
+
+ def __init__(self, message: str = "Resource conflict"):
+ super().__init__(
+ message=message,
+ status_code=409,
+ error_code="CONFLICT",
+ )
+
+
+class RateLimitError(APIException):
+ """Rate limit exceeded exception."""
+
+ def __init__(self, message: str = "Rate limit exceeded"):
+ super().__init__(
+ message=message,
+ status_code=429,
+ error_code="RATE_LIMIT_EXCEEDED",
+ )
+
+
+class ServiceUnavailableError(APIException):
+ """Service unavailable exception."""
+
+ def __init__(self, message: str = "Service temporarily unavailable"):
+ super().__init__(
+ message=message,
+ status_code=503,
+ error_code="SERVICE_UNAVAILABLE",
+ )
+
+
+class ExternalServiceError(APIException):
+ """External service error exception."""
+
+ def __init__(self, service: str, message: str):
+ super().__init__(
+ message=f"{service} service error: {message}",
+ status_code=502,
+ error_code="EXTERNAL_SERVICE_ERROR",
+ details={"service": service},
+ )
+
+
+class EmbeddingServiceError(ExternalServiceError):
+ """Embedding service error exception."""
+
+ def __init__(self, message: str):
+ super().__init__("Embedding", message)
+
+
+class VectorDatabaseError(ExternalServiceError):
+ """Vector database error exception."""
+
+ def __init__(self, message: str):
+ super().__init__("Vector Database", message)
+
+
+class LLMServiceError(ExternalServiceError):
+ """LLM service error exception."""
+
+ def __init__(self, message: str):
+ super().__init__("LLM", message)
\ No newline at end of file
diff --git a/packages/api/app/core/security.py b/packages/api/app/core/security.py
new file mode 100644
index 0000000..d423c46
--- /dev/null
+++ b/packages/api/app/core/security.py
@@ -0,0 +1,209 @@
+"""
+Security utilities for authentication and authorization.
+"""
+
+import secrets
+from datetime import datetime, timedelta
+from typing import Any, Dict, Optional, Union
+
+import jwt
+from fastapi import Depends, HTTPException, Request, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from passlib.context import CryptContext
+
+from app.core.config import get_settings
+from app.core.exceptions import AuthenticationError, AuthorizationError
+
+# Password hashing
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+# JWT token scheme
+security = HTTPBearer()
+
+
+def create_access_token(
+ subject: Union[str, Any], expires_delta: Optional[timedelta] = None
+) -> str:
+ """Create JWT access token."""
+ settings = get_settings()
+
+ if expires_delta:
+ expire = datetime.utcnow() + expires_delta
+ else:
+ expire = datetime.utcnow() + timedelta(minutes=settings.JWT_EXPIRE_MINUTES)
+
+ to_encode = {"exp": expire, "sub": str(subject)}
+
+ encoded_jwt = jwt.encode(
+ to_encode, settings.SECRET_KEY, algorithm=settings.JWT_ALGORITHM
+ )
+
+ return encoded_jwt
+
+
+def verify_token(token: str) -> Dict[str, Any]:
+ """Verify and decode JWT token."""
+ settings = get_settings()
+
+ try:
+ payload = jwt.decode(
+ token, settings.SECRET_KEY, algorithms=[settings.JWT_ALGORITHM]
+ )
+ return payload
+ except jwt.ExpiredSignatureError:
+ raise AuthenticationError("Token has expired")
+ except jwt.JWTError:
+ raise AuthenticationError("Invalid token")
+
+
+def get_password_hash(password: str) -> str:
+ """Hash a password."""
+ return pwd_context.hash(password)
+
+
+def verify_password(plain_password: str, hashed_password: str) -> bool:
+ """Verify a password against its hash."""
+ return pwd_context.verify(plain_password, hashed_password)
+
+
+def generate_api_key() -> str:
+ """Generate a secure API key."""
+ return secrets.token_urlsafe(32)
+
+
+async def get_current_user(
+ credentials: HTTPAuthorizationCredentials = Depends(security),
+) -> Dict[str, Any]:
+ """Get current authenticated user from JWT token."""
+ try:
+ payload = verify_token(credentials.credentials)
+ user_id = payload.get("sub")
+
+ if user_id is None:
+ raise AuthenticationError("Invalid token payload")
+
+ # In a real application, you would fetch user data from database
+ return {"id": user_id, "is_active": True}
+
+ except AuthenticationError:
+ raise
+ except Exception:
+ raise AuthenticationError("Could not validate credentials")
+
+
+async def get_api_key(request: Request) -> str:
+ """Extract and validate API key from request."""
+ # Check Authorization header
+ auth_header = request.headers.get("Authorization")
+ if auth_header and auth_header.startswith("Bearer "):
+ api_key = auth_header[7:] # Remove "Bearer " prefix
+ return api_key
+
+ # Check X-API-Key header
+ api_key = request.headers.get("X-API-Key")
+ if api_key:
+ return api_key
+
+ # Check query parameter
+ api_key = request.query_params.get("api_key")
+ if api_key:
+ return api_key
+
+ raise AuthenticationError("API key required")
+
+
+async def validate_api_key(api_key: str) -> Dict[str, Any]:
+ """Validate API key and return associated client info."""
+ # In a real application, you would validate against database
+ # For now, we'll do basic validation
+
+ if not api_key or len(api_key) < 20:
+ raise AuthenticationError("Invalid API key format")
+
+ # Mock validation - replace with actual database lookup
+ return {
+ "api_key": api_key,
+ "client_id": "client_123",
+ "is_active": True,
+ "rate_limit": 1000,
+ "features": ["search", "analytics", "enhanced_answers"],
+ }
+
+
+class RateLimiter:
+ """Rate limiting utility."""
+
+ def __init__(self, redis_client=None):
+ self.redis_client = redis_client
+
+ async def is_allowed(
+ self,
+ key: str,
+ limit: int,
+ window: int
+ ) -> bool:
+ """Check if request is within rate limit."""
+ if not self.redis_client:
+ return True # No rate limiting if Redis not available
+
+ try:
+ current = await self.redis_client.get(key)
+ if current is None:
+ await self.redis_client.setex(key, window, 1)
+ return True
+
+ if int(current) >= limit:
+ return False
+
+ await self.redis_client.incr(key)
+ return True
+
+ except Exception:
+ # If Redis fails, allow the request
+ return True
+
+
+# Dependency to check API key and rate limits
+async def require_api_key(request: Request) -> Dict[str, Any]:
+ """Dependency to require and validate API key."""
+ api_key = await get_api_key(request)
+ client_info = await validate_api_key(api_key)
+
+ # Add rate limiting
+ rate_limiter = RateLimiter() # Initialize with Redis client
+ client_id = client_info["client_id"]
+
+ settings = get_settings()
+ is_allowed = await rate_limiter.is_allowed(
+ f"rate_limit:{client_id}",
+ settings.RATE_LIMIT_REQUESTS,
+ settings.RATE_LIMIT_WINDOW,
+ )
+
+ if not is_allowed:
+ raise HTTPException(
+ status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+ detail="Rate limit exceeded",
+ )
+
+ return client_info
+
+
+# Permission decorators
+def require_permission(permission: str):
+ """Decorator to require specific permission."""
+ def decorator(func):
+ async def wrapper(*args, **kwargs):
+ # Get client info from dependency
+ client_info = kwargs.get("client_info")
+ if not client_info:
+ raise AuthorizationError("Client information not available")
+
+ # Check if client has required permission
+ features = client_info.get("features", [])
+ if permission not in features:
+ raise AuthorizationError(f"Permission '{permission}' required")
+
+ return await func(*args, **kwargs)
+ return wrapper
+ return decorator
\ No newline at end of file
diff --git a/packages/api/app/models/__init__.py b/packages/api/app/models/__init__.py
new file mode 100644
index 0000000..69a5685
--- /dev/null
+++ b/packages/api/app/models/__init__.py
@@ -0,0 +1,14 @@
+"""Database models for the AI Search Platform."""
+
+from .document import Document, DocumentChunk
+from .analytics import AnalyticsEvent, SearchSession
+from .client import Client, APIKey
+
+__all__ = [
+ "Document",
+ "DocumentChunk",
+ "AnalyticsEvent",
+ "SearchSession",
+ "Client",
+ "APIKey",
+]
\ No newline at end of file
diff --git a/packages/api/app/models/analytics.py b/packages/api/app/models/analytics.py
new file mode 100644
index 0000000..be0e637
--- /dev/null
+++ b/packages/api/app/models/analytics.py
@@ -0,0 +1,170 @@
+"""Analytics models for tracking search behavior and performance."""
+
+from datetime import datetime
+from typing import Dict, Optional
+
+from sqlalchemy import (
+ Column,
+ DateTime,
+ Enum,
+ ForeignKey,
+ Integer,
+ JSON,
+ String,
+ Float,
+ Index,
+)
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import relationship
+from sqlalchemy.sql import func
+import uuid
+
+from app.core.database import Base
+
+
+class SearchSession(Base):
+ """Search session model for tracking user interactions."""
+
+ __tablename__ = "search_sessions"
+
+ # Primary key
+ id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+
+ # Session info
+ session_id = Column(String(255), nullable=False, unique=True, index=True)
+ user_id = Column(String(255), nullable=True, index=True)
+
+ # Client association
+ client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True)
+
+ # Session metadata
+ user_agent = Column(Text, nullable=True)
+ ip_address = Column(String(45), nullable=True) # IPv6 compatible
+ referrer = Column(String(2048), nullable=True)
+ country = Column(String(2), nullable=True)
+ device_type = Column(String(50), nullable=True)
+
+ # Session metrics
+ total_queries = Column(Integer, nullable=False, default=0)
+ total_clicks = Column(Integer, nullable=False, default=0)
+ session_duration = Column(Integer, nullable=True) # in seconds
+
+ # Timestamps
+ started_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+ ended_at = Column(DateTime(timezone=True), nullable=True)
+ last_activity = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False)
+
+ # Relationships
+ client = relationship("Client", back_populates="search_sessions")
+ events = relationship("AnalyticsEvent", back_populates="session", cascade="all, delete-orphan")
+
+ # Indexes
+ __table_args__ = (
+ Index("ix_search_sessions_client_started", "client_id", "started_at"),
+ Index("ix_search_sessions_user_started", "user_id", "started_at"),
+ )
+
+ def __repr__(self) -> str:
+ return f""
+
+
+class AnalyticsEvent(Base):
+ """Analytics event model for detailed tracking."""
+
+ __tablename__ = "analytics_events"
+
+ # Primary key
+ id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+
+ # Session reference
+ session_id = Column(UUID(as_uuid=True), ForeignKey("search_sessions.id"), nullable=False, index=True)
+
+ # Event details
+ event_type = Column(
+ Enum(
+ "search_query",
+ "search_result_click",
+ "widget_load",
+ "widget_interaction",
+ "document_view",
+ "feedback_positive",
+ "feedback_negative",
+ "auto_complete",
+ "enhanced_answer_view",
+ "source_click",
+ name="event_type_enum"
+ ),
+ nullable=False,
+ index=True,
+ )
+
+ # Event data
+ data = Column(JSON, nullable=False)
+
+ # Performance metrics
+ processing_time = Column(Float, nullable=True) # in milliseconds
+
+ # Timestamps
+ timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False, index=True)
+
+ # Relationships
+ session = relationship("SearchSession", back_populates="events")
+
+ # Indexes
+ __table_args__ = (
+ Index("ix_analytics_events_session_type", "session_id", "event_type"),
+ Index("ix_analytics_events_type_timestamp", "event_type", "timestamp"),
+ Index("ix_analytics_events_timestamp", "timestamp"),
+ )
+
+ def __repr__(self) -> str:
+ return f""
+
+
+class SearchQuery(Base):
+ """Search query model for tracking and analysis."""
+
+ __tablename__ = "search_queries"
+
+ # Primary key
+ id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+
+ # Query details
+ query_text = Column(Text, nullable=False, index=True)
+ query_hash = Column(String(64), nullable=False, index=True) # MD5 hash for deduplication
+
+ # Search configuration
+ search_type = Column(
+ Enum("semantic", "keyword", "hybrid", name="search_type_enum"),
+ nullable=False,
+ default="semantic",
+ )
+
+ # Results
+ results_count = Column(Integer, nullable=False, default=0)
+ processing_time = Column(Float, nullable=False) # in milliseconds
+
+ # Enhanced answer
+ enhanced_answer_generated = Column(Boolean, nullable=False, default=False)
+ enhanced_answer_id = Column(UUID(as_uuid=True), nullable=True)
+
+ # Session and client
+ session_id = Column(UUID(as_uuid=True), ForeignKey("search_sessions.id"), nullable=False, index=True)
+ client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True)
+
+ # Timestamps
+ created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+
+ # Relationships
+ session = relationship("SearchSession")
+ client = relationship("Client")
+
+ # Indexes
+ __table_args__ = (
+ Index("ix_search_queries_client_created", "client_id", "created_at"),
+ Index("ix_search_queries_hash_client", "query_hash", "client_id"),
+ Index("ix_search_queries_text_trgm", "query_text", postgresql_using="gin"), # Full-text search
+ )
+
+ def __repr__(self) -> str:
+ return f""
\ No newline at end of file
diff --git a/packages/api/app/models/client.py b/packages/api/app/models/client.py
new file mode 100644
index 0000000..347b866
--- /dev/null
+++ b/packages/api/app/models/client.py
@@ -0,0 +1,163 @@
+"""Client and API key models."""
+
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from sqlalchemy import (
+ Column,
+ DateTime,
+ Boolean,
+ ForeignKey,
+ Integer,
+ JSON,
+ String,
+ Text,
+ Index,
+)
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import relationship
+from sqlalchemy.sql import func
+import uuid
+
+from app.core.database import Base
+
+
+class Client(Base):
+ """Client model for API access management."""
+
+ __tablename__ = "clients"
+
+ # Primary key
+ id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+
+ # Client info
+ name = Column(String(255), nullable=False)
+ email = Column(String(255), nullable=False, unique=True, index=True)
+ company = Column(String(255), nullable=True)
+ website = Column(String(255), nullable=True)
+
+ # Status
+ is_active = Column(Boolean, nullable=False, default=True, index=True)
+ is_verified = Column(Boolean, nullable=False, default=False)
+
+ # Subscription and limits
+ plan = Column(String(50), nullable=False, default="free") # free, pro, enterprise
+ monthly_quota = Column(Integer, nullable=False, default=1000)
+ monthly_usage = Column(Integer, nullable=False, default=0)
+
+ # Configuration
+ settings = Column(JSON, nullable=True)
+ allowed_domains = Column(JSON, nullable=True) # Array of allowed domains
+
+ # Timestamps
+ created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+ updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False)
+ last_login = Column(DateTime(timezone=True), nullable=True)
+
+ # Relationships
+ api_keys = relationship("APIKey", back_populates="client", cascade="all, delete-orphan")
+ documents = relationship("Document", back_populates="client", cascade="all, delete-orphan")
+ search_sessions = relationship("SearchSession", back_populates="client", cascade="all, delete-orphan")
+
+ # Indexes
+ __table_args__ = (
+ Index("ix_clients_active_plan", "is_active", "plan"),
+ Index("ix_clients_created_at", "created_at"),
+ )
+
+ def __repr__(self) -> str:
+ return f""
+
+
+class APIKey(Base):
+ """API key model for authentication."""
+
+ __tablename__ = "api_keys"
+
+ # Primary key
+ id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+
+ # Key details
+ key_hash = Column(String(255), nullable=False, unique=True, index=True)
+ key_prefix = Column(String(20), nullable=False) # First few characters for identification
+ name = Column(String(255), nullable=False)
+ description = Column(Text, nullable=True)
+
+ # Client reference
+ client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True)
+
+ # Status and permissions
+ is_active = Column(Boolean, nullable=False, default=True, index=True)
+ permissions = Column(JSON, nullable=True) # Array of permissions
+
+ # Rate limiting
+ rate_limit_per_minute = Column(Integer, nullable=False, default=60)
+ rate_limit_per_hour = Column(Integer, nullable=False, default=1000)
+ rate_limit_per_day = Column(Integer, nullable=False, default=10000)
+
+ # Usage tracking
+ total_requests = Column(Integer, nullable=False, default=0)
+ last_used = Column(DateTime(timezone=True), nullable=True)
+
+ # Timestamps
+ created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+ updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False)
+ expires_at = Column(DateTime(timezone=True), nullable=True)
+
+ # Relationships
+ client = relationship("Client", back_populates="api_keys")
+
+ # Indexes
+ __table_args__ = (
+ Index("ix_api_keys_client_active", "client_id", "is_active"),
+ Index("ix_api_keys_expires_at", "expires_at"),
+ )
+
+ def __repr__(self) -> str:
+ return f""
+
+
+class Usage(Base):
+ """Usage tracking model for billing and analytics."""
+
+ __tablename__ = "usage"
+
+ # Primary key
+ id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+
+ # Client and API key
+ client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True)
+ api_key_id = Column(UUID(as_uuid=True), ForeignKey("api_keys.id"), nullable=False, index=True)
+
+ # Usage metrics
+ endpoint = Column(String(255), nullable=False, index=True)
+ method = Column(String(10), nullable=False)
+ requests_count = Column(Integer, nullable=False, default=0)
+
+ # Billing metrics
+ tokens_used = Column(Integer, nullable=False, default=0)
+ compute_time = Column(Float, nullable=False, default=0.0) # in seconds
+
+ # Time period
+ date = Column(DateTime(timezone=True), nullable=False, index=True)
+ hour = Column(Integer, nullable=False) # 0-23
+
+ # Timestamps
+ created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+ updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False)
+
+ # Relationships
+ client = relationship("Client")
+ api_key = relationship("APIKey")
+
+ # Indexes
+ __table_args__ = (
+ Index("ix_usage_client_date", "client_id", "date"),
+ Index("ix_usage_api_key_date", "api_key_id", "date"),
+ Index("ix_usage_endpoint_date", "endpoint", "date"),
+ # Unique constraint for deduplication
+ Index("uq_usage_client_endpoint_date_hour", "client_id", "endpoint", "date", "hour", unique=True),
+ )
+
+ def __repr__(self) -> str:
+ return f""
\ No newline at end of file
diff --git a/packages/api/app/models/document.py b/packages/api/app/models/document.py
new file mode 100644
index 0000000..cd79be7
--- /dev/null
+++ b/packages/api/app/models/document.py
@@ -0,0 +1,125 @@
+"""Document and document chunk models."""
+
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from sqlalchemy import (
+ Column,
+ DateTime,
+ Enum,
+ ForeignKey,
+ Integer,
+ JSON,
+ String,
+ Text,
+ Index,
+)
+from sqlalchemy.dialects.postgresql import UUID, ARRAY
+from sqlalchemy.orm import relationship
+from sqlalchemy.sql import func
+import uuid
+
+from app.core.database import Base
+
+
+class Document(Base):
+ """Document model for storing indexed content."""
+
+ __tablename__ = "documents"
+
+ # Primary key
+ id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+
+ # Basic document info
+ url = Column(String(2048), nullable=True, index=True)
+ title = Column(String(500), nullable=False, index=True)
+ content = Column(Text, nullable=False)
+ content_type = Column(
+ Enum(
+ "text", "html", "markdown", "pdf", "doc", "docx",
+ name="content_type_enum"
+ ),
+ nullable=False,
+ default="text",
+ )
+
+ # Metadata
+ metadata = Column(JSON, nullable=True)
+ language = Column(String(10), nullable=False, default="en", index=True)
+ tags = Column(ARRAY(String), nullable=True, index=True)
+ source_id = Column(String(255), nullable=True, index=True)
+
+ # Status and processing
+ status = Column(
+ Enum(
+ "pending", "indexed", "failed", "archived",
+ name="document_status_enum"
+ ),
+ nullable=False,
+ default="pending",
+ index=True,
+ )
+
+ # Timestamps
+ created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+ updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False)
+ indexed_at = Column(DateTime(timezone=True), nullable=True)
+
+ # Client association
+ client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True)
+
+ # Relationships
+ chunks = relationship("DocumentChunk", back_populates="document", cascade="all, delete-orphan")
+ client = relationship("Client", back_populates="documents")
+
+ # Indexes
+ __table_args__ = (
+ Index("ix_documents_client_status", "client_id", "status"),
+ Index("ix_documents_client_language", "client_id", "language"),
+ Index("ix_documents_created_at", "created_at"),
+ Index("ix_documents_url_hash", func.md5("url")), # For faster URL lookups
+ )
+
+ def __repr__(self) -> str:
+ return f""
+
+
+class DocumentChunk(Base):
+ """Document chunk model for vector storage."""
+
+ __tablename__ = "document_chunks"
+
+ # Primary key
+ id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+
+ # Document reference
+ document_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=False, index=True)
+
+ # Chunk content
+ content = Column(Text, nullable=False)
+ chunk_index = Column(Integer, nullable=False)
+ start_offset = Column(Integer, nullable=False)
+ end_offset = Column(Integer, nullable=False)
+
+ # Vector embedding (stored as JSON array)
+ embedding = Column(JSON, nullable=True)
+ embedding_model = Column(String(255), nullable=True)
+
+ # Metadata
+ metadata = Column(JSON, nullable=True)
+
+ # Timestamps
+ created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+ updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False)
+
+ # Relationships
+ document = relationship("Document", back_populates="chunks")
+
+ # Indexes
+ __table_args__ = (
+ Index("ix_document_chunks_document_id", "document_id"),
+ Index("ix_document_chunks_document_chunk", "document_id", "chunk_index"),
+ )
+
+ def __repr__(self) -> str:
+ return f""
\ No newline at end of file
diff --git a/packages/api/app/services/__init__.py b/packages/api/app/services/__init__.py
new file mode 100644
index 0000000..c66a0b2
--- /dev/null
+++ b/packages/api/app/services/__init__.py
@@ -0,0 +1 @@
+# Services package
\ No newline at end of file
diff --git a/packages/api/app/services/embedding_service.py b/packages/api/app/services/embedding_service.py
new file mode 100644
index 0000000..e42554d
--- /dev/null
+++ b/packages/api/app/services/embedding_service.py
@@ -0,0 +1,176 @@
+"""
+Embedding service for generating vector embeddings from text.
+"""
+
+import asyncio
+import httpx
+from typing import List, Dict, Any, Optional
+import numpy as np
+
+from app.core.config import get_settings
+from app.core.exceptions import EmbeddingServiceError
+
+
+class EmbeddingService:
+ """Service for generating text embeddings using various providers."""
+
+ def __init__(self):
+ self.settings = get_settings()
+ self.client = httpx.AsyncClient(timeout=30.0)
+
+ async def generate_embeddings(
+ self,
+ texts: List[str],
+ provider: str = "openai",
+ model: Optional[str] = None,
+ ) -> List[List[float]]:
+ """Generate embeddings for a list of texts."""
+ if provider == "openai":
+ return await self._generate_openai_embeddings(texts, model)
+ elif provider == "huggingface":
+ return await self._generate_huggingface_embeddings(texts, model)
+ elif provider == "cohere":
+ return await self._generate_cohere_embeddings(texts, model)
+ else:
+ raise EmbeddingServiceError(f"Unsupported embedding provider: {provider}")
+
+ async def _generate_openai_embeddings(
+ self,
+ texts: List[str],
+ model: Optional[str] = None
+ ) -> List[List[float]]:
+ """Generate embeddings using OpenAI API."""
+ if not self.settings.OPENAI_API_KEY:
+ raise EmbeddingServiceError("OpenAI API key not configured")
+
+ model = model or self.settings.OPENAI_EMBEDDING_MODEL
+
+ try:
+ # Process in batches to avoid rate limits
+ batch_size = 100
+ all_embeddings = []
+
+ for i in range(0, len(texts), batch_size):
+ batch = texts[i:i + batch_size]
+
+ response = await self.client.post(
+ "https://api.openai.com/v1/embeddings",
+ headers={
+ "Authorization": f"Bearer {self.settings.OPENAI_API_KEY}",
+ "Content-Type": "application/json",
+ },
+ json={
+ "input": batch,
+ "model": model,
+ },
+ )
+
+ if response.status_code != 200:
+ error_data = response.json()
+ raise EmbeddingServiceError(
+ f"OpenAI API error: {error_data.get('error', {}).get('message', 'Unknown error')}"
+ )
+
+ data = response.json()
+ batch_embeddings = [item["embedding"] for item in data["data"]]
+ all_embeddings.extend(batch_embeddings)
+
+ return all_embeddings
+
+ except httpx.RequestError as e:
+ raise EmbeddingServiceError(f"OpenAI API request failed: {str(e)}")
+ except Exception as e:
+ raise EmbeddingServiceError(f"OpenAI embedding generation failed: {str(e)}")
+
+ async def _generate_huggingface_embeddings(
+ self,
+ texts: List[str],
+ model: Optional[str] = None
+ ) -> List[List[float]]:
+ """Generate embeddings using HuggingFace API."""
+ model = model or self.settings.HUGGINGFACE_MODEL
+
+ try:
+ # For HuggingFace, we'll use the sentence-transformers library locally
+ # In production, you might want to use their API or host your own model
+ from sentence_transformers import SentenceTransformer
+
+ # Load model (cache it in production)
+ transformer_model = SentenceTransformer(model)
+
+ # Generate embeddings
+ embeddings = transformer_model.encode(texts, convert_to_tensor=False)
+
+ return embeddings.tolist()
+
+ except ImportError:
+ raise EmbeddingServiceError("sentence-transformers library not installed")
+ except Exception as e:
+ raise EmbeddingServiceError(f"HuggingFace embedding generation failed: {str(e)}")
+
+ async def _generate_cohere_embeddings(
+ self,
+ texts: List[str],
+ model: Optional[str] = None
+ ) -> List[List[float]]:
+ """Generate embeddings using Cohere API."""
+ if not self.settings.COHERE_API_KEY:
+ raise EmbeddingServiceError("Cohere API key not configured")
+
+ model = model or self.settings.COHERE_MODEL
+
+ try:
+ response = await self.client.post(
+ "https://api.cohere.ai/v1/embed",
+ headers={
+ "Authorization": f"Bearer {self.settings.COHERE_API_KEY}",
+ "Content-Type": "application/json",
+ },
+ json={
+ "texts": texts,
+ "model": model,
+ },
+ )
+
+ if response.status_code != 200:
+ error_data = response.json()
+ raise EmbeddingServiceError(
+ f"Cohere API error: {error_data.get('message', 'Unknown error')}"
+ )
+
+ data = response.json()
+ return data["embeddings"]
+
+ except httpx.RequestError as e:
+ raise EmbeddingServiceError(f"Cohere API request failed: {str(e)}")
+ except Exception as e:
+ raise EmbeddingServiceError(f"Cohere embedding generation failed: {str(e)}")
+
+ async def get_embedding_dimensions(self, provider: str = "openai", model: Optional[str] = None) -> int:
+ """Get the dimensions of embeddings for a given provider/model."""
+ if provider == "openai":
+ model = model or self.settings.OPENAI_EMBEDDING_MODEL
+ if "ada-002" in model:
+ return 1536
+ elif "3-small" in model:
+ return 1536
+ elif "3-large" in model:
+ return 3072
+ else:
+ return 1536 # Default
+ elif provider == "huggingface":
+ model = model or self.settings.HUGGINGFACE_MODEL
+ if "all-MiniLM-L6-v2" in model:
+ return 384
+ elif "all-mpnet-base-v2" in model:
+ return 768
+ else:
+ return 384 # Default
+ elif provider == "cohere":
+ return 4096 # Cohere embed-english-v2.0
+ else:
+ raise EmbeddingServiceError(f"Unknown provider: {provider}")
+
+ async def close(self):
+ """Close the HTTP client."""
+ await self.client.aclose()
\ No newline at end of file
diff --git a/packages/api/app/services/llm_service.py b/packages/api/app/services/llm_service.py
new file mode 100644
index 0000000..4cdb2a5
--- /dev/null
+++ b/packages/api/app/services/llm_service.py
@@ -0,0 +1,338 @@
+"""
+LLM service for generating enhanced answers and content.
+"""
+
+import asyncio
+import httpx
+from typing import Dict, Any, Optional, List
+import json
+
+from app.core.config import get_settings
+from app.core.exceptions import LLMServiceError
+
+
+class LLMService:
+ """Service for LLM-powered content generation."""
+
+ def __init__(self):
+ self.settings = get_settings()
+ self.client = httpx.AsyncClient(timeout=60.0)
+
+ async def generate_answer(
+ self,
+ query: str,
+ context: str,
+ max_length: int = 500,
+ provider: str = "openai",
+ model: Optional[str] = None,
+ ) -> Optional[Dict[str, Any]]:
+ """Generate an enhanced answer based on query and context."""
+ try:
+ if provider == "openai":
+ return await self._generate_openai_answer(query, context, max_length, model)
+ elif provider == "anthropic":
+ return await self._generate_anthropic_answer(query, context, max_length, model)
+ else:
+ raise LLMServiceError(f"Unsupported LLM provider: {provider}")
+ except Exception as e:
+ raise LLMServiceError(f"Answer generation failed: {str(e)}")
+
+ async def _generate_openai_answer(
+ self,
+ query: str,
+ context: str,
+ max_length: int,
+ model: Optional[str] = None,
+ ) -> Optional[Dict[str, Any]]:
+ """Generate answer using OpenAI API."""
+ if not self.settings.OPENAI_API_KEY:
+ raise LLMServiceError("OpenAI API key not configured")
+
+ model = model or self.settings.OPENAI_MODEL
+
+ # Construct the prompt
+ prompt = self._build_answer_prompt(query, context, max_length)
+
+ try:
+ response = await self.client.post(
+ "https://api.openai.com/v1/chat/completions",
+ headers={
+ "Authorization": f"Bearer {self.settings.OPENAI_API_KEY}",
+ "Content-Type": "application/json",
+ },
+ json={
+ "model": model,
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful AI assistant that provides accurate, concise answers based on the given context. Always cite your sources and be honest about limitations."
+ },
+ {
+ "role": "user",
+ "content": prompt
+ }
+ ],
+ "max_tokens": max_length,
+ "temperature": 0.3,
+ "top_p": 0.9,
+ },
+ )
+
+ if response.status_code != 200:
+ error_data = response.json()
+ raise LLMServiceError(
+ f"OpenAI API error: {error_data.get('error', {}).get('message', 'Unknown error')}"
+ )
+
+ data = response.json()
+
+ if not data.get("choices"):
+ return None
+
+ content = data["choices"][0]["message"]["content"].strip()
+
+ return {
+ "content": content,
+ "confidence": 0.8, # Could be calculated based on various factors
+ "model": model,
+ "provider": "openai",
+ "usage": data.get("usage", {}),
+ }
+
+ except httpx.RequestError as e:
+ raise LLMServiceError(f"OpenAI API request failed: {str(e)}")
+
+ async def _generate_anthropic_answer(
+ self,
+ query: str,
+ context: str,
+ max_length: int,
+ model: Optional[str] = None,
+ ) -> Optional[Dict[str, Any]]:
+ """Generate answer using Anthropic API."""
+ if not self.settings.ANTHROPIC_API_KEY:
+ raise LLMServiceError("Anthropic API key not configured")
+
+ model = model or self.settings.ANTHROPIC_MODEL
+
+ # Construct the prompt
+ prompt = self._build_answer_prompt(query, context, max_length)
+
+ try:
+ response = await self.client.post(
+ "https://api.anthropic.com/v1/messages",
+ headers={
+ "x-api-key": self.settings.ANTHROPIC_API_KEY,
+ "Content-Type": "application/json",
+ "anthropic-version": "2023-06-01",
+ },
+ json={
+ "model": model,
+ "max_tokens": max_length,
+ "temperature": 0.3,
+ "messages": [
+ {
+ "role": "user",
+ "content": prompt
+ }
+ ],
+ },
+ )
+
+ if response.status_code != 200:
+ error_data = response.json()
+ raise LLMServiceError(
+ f"Anthropic API error: {error_data.get('error', {}).get('message', 'Unknown error')}"
+ )
+
+ data = response.json()
+
+ if not data.get("content"):
+ return None
+
+ content = data["content"][0]["text"].strip()
+
+ return {
+ "content": content,
+ "confidence": 0.8,
+ "model": model,
+ "provider": "anthropic",
+ "usage": data.get("usage", {}),
+ }
+
+ except httpx.RequestError as e:
+ raise LLMServiceError(f"Anthropic API request failed: {str(e)}")
+
+ def _build_answer_prompt(self, query: str, context: str, max_length: int) -> str:
+ """Build a prompt for answer generation."""
+ return f"""Based on the following context, please provide a comprehensive and accurate answer to the user's question.
+
+Context:
+{context}
+
+Question: {query}
+
+Instructions:
+1. Provide a clear, concise answer based solely on the information in the context
+2. If the context doesn't contain enough information to fully answer the question, say so
+3. Keep your response under {max_length} characters
+4. Use a professional but friendly tone
+5. Structure your answer with clear paragraphs if needed
+
+Answer:"""
+
+ async def generate_summary(
+ self,
+ content: str,
+ max_length: int = 200,
+ provider: str = "openai",
+ ) -> Optional[str]:
+ """Generate a summary of the given content."""
+ try:
+ prompt = f"""Please provide a concise summary of the following content in no more than {max_length} characters:
+
+{content}
+
+Summary:"""
+
+ if provider == "openai":
+ result = await self._generate_openai_completion(prompt, max_length)
+ elif provider == "anthropic":
+ result = await self._generate_anthropic_completion(prompt, max_length)
+ else:
+ raise LLMServiceError(f"Unsupported provider: {provider}")
+
+ return result.get("content") if result else None
+
+ except Exception as e:
+ raise LLMServiceError(f"Summary generation failed: {str(e)}")
+
+ async def _generate_openai_completion(
+ self,
+ prompt: str,
+ max_tokens: int,
+ ) -> Optional[Dict[str, Any]]:
+ """Generate a completion using OpenAI API."""
+ try:
+ response = await self.client.post(
+ "https://api.openai.com/v1/chat/completions",
+ headers={
+ "Authorization": f"Bearer {self.settings.OPENAI_API_KEY}",
+ "Content-Type": "application/json",
+ },
+ json={
+ "model": self.settings.OPENAI_MODEL,
+ "messages": [{"role": "user", "content": prompt}],
+ "max_tokens": max_tokens,
+ "temperature": 0.3,
+ },
+ )
+
+ if response.status_code == 200:
+ data = response.json()
+ if data.get("choices"):
+ return {
+ "content": data["choices"][0]["message"]["content"].strip(),
+ "usage": data.get("usage", {}),
+ }
+
+ return None
+
+ except Exception:
+ return None
+
+ async def _generate_anthropic_completion(
+ self,
+ prompt: str,
+ max_tokens: int,
+ ) -> Optional[Dict[str, Any]]:
+ """Generate a completion using Anthropic API."""
+ try:
+ response = await self.client.post(
+ "https://api.anthropic.com/v1/messages",
+ headers={
+ "x-api-key": self.settings.ANTHROPIC_API_KEY,
+ "Content-Type": "application/json",
+ "anthropic-version": "2023-06-01",
+ },
+ json={
+ "model": self.settings.ANTHROPIC_MODEL,
+ "max_tokens": max_tokens,
+ "temperature": 0.3,
+ "messages": [{"role": "user", "content": prompt}],
+ },
+ )
+
+ if response.status_code == 200:
+ data = response.json()
+ if data.get("content"):
+ return {
+ "content": data["content"][0]["text"].strip(),
+ "usage": data.get("usage", {}),
+ }
+
+ return None
+
+ except Exception:
+ return None
+
+ async def validate_content(
+ self,
+ content: str,
+ guidelines: Optional[List[str]] = None,
+ ) -> Dict[str, Any]:
+ """Validate content against guidelines and detect potential issues."""
+ try:
+ # Basic content validation
+ issues = []
+
+ # Check for potential inappropriate content (basic keywords)
+ inappropriate_keywords = [
+ "spam", "scam", "fraud", "illegal", "hate", "violence"
+ ]
+
+ content_lower = content.lower()
+ for keyword in inappropriate_keywords:
+ if keyword in content_lower:
+ issues.append({
+ "type": "inappropriate_content",
+ "severity": "medium",
+ "message": f"Potentially inappropriate content detected: {keyword}",
+ })
+
+ # Check content length
+ if len(content) < 10:
+ issues.append({
+ "type": "content_too_short",
+ "severity": "low",
+ "message": "Content appears to be too short to be meaningful",
+ })
+
+ # Check for excessive repetition
+ words = content.split()
+ if len(words) > 10:
+ unique_words = set(words)
+ repetition_ratio = len(unique_words) / len(words)
+ if repetition_ratio < 0.3:
+ issues.append({
+ "type": "excessive_repetition",
+ "severity": "medium",
+ "message": "Content contains excessive repetition",
+ })
+
+ return {
+ "is_valid": len([i for i in issues if i["severity"] == "high"]) == 0,
+ "issues": issues,
+ "confidence": 0.7, # Basic validation confidence
+ }
+
+ except Exception as e:
+ return {
+ "is_valid": True, # Default to valid if validation fails
+ "issues": [{"type": "validation_error", "severity": "low", "message": str(e)}],
+ "confidence": 0.1,
+ }
+
+ async def close(self):
+ """Close the HTTP client."""
+ await self.client.aclose()
\ No newline at end of file
diff --git a/packages/api/app/services/search_service.py b/packages/api/app/services/search_service.py
new file mode 100644
index 0000000..d04ad6a
--- /dev/null
+++ b/packages/api/app/services/search_service.py
@@ -0,0 +1,458 @@
+"""
+Search service for semantic and keyword search functionality.
+"""
+
+import asyncio
+from typing import List, Dict, Any, Optional, Tuple
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select, func, text
+import time
+
+from app.models.document import Document, DocumentChunk
+from app.services.embedding_service import EmbeddingService
+from app.services.vector_service import VectorService
+from app.services.llm_service import LLMService
+from app.core.exceptions import ValidationError, ServiceUnavailableError
+
+
+class SearchService:
+ """Service for performing semantic and keyword searches."""
+
+ def __init__(self, db: AsyncSession):
+ self.db = db
+ self.embedding_service = EmbeddingService()
+ self.vector_service = VectorService()
+ self.llm_service = LLMService()
+
+ async def search(
+ self,
+ query: str,
+ client_id: str,
+ filters: Optional[Dict[str, Any]] = None,
+ options: Optional[Dict[str, Any]] = None,
+ ) -> Dict[str, Any]:
+ """
+ Perform comprehensive search across documents.
+
+ Supports semantic, keyword, and hybrid search modes.
+ """
+ start_time = time.time()
+
+ # Parse options
+ search_options = options or {}
+ search_type = search_options.get("search_type", "semantic")
+ limit = search_options.get("limit", 10)
+ offset = search_options.get("offset", 0)
+ min_score = search_options.get("min_score", 0.1)
+ enhance_with_llm = search_options.get("enhance_with_llm", False)
+
+ # Validate search type
+ if search_type not in ["semantic", "keyword", "hybrid"]:
+ raise ValidationError("Invalid search_type. Must be 'semantic', 'keyword', or 'hybrid'")
+
+ results = []
+ total_results = 0
+
+ if search_type == "semantic":
+ results, total_results = await self._semantic_search(
+ query, client_id, filters, limit, offset, min_score
+ )
+ elif search_type == "keyword":
+ results, total_results = await self._keyword_search(
+ query, client_id, filters, limit, offset
+ )
+ elif search_type == "hybrid":
+ results, total_results = await self._hybrid_search(
+ query, client_id, filters, limit, offset, min_score
+ )
+
+ processing_time = (time.time() - start_time) * 1000
+
+ # Generate enhanced answer if requested
+ enhanced_answer = None
+ if enhance_with_llm and results:
+ try:
+ enhanced_answer = await self._generate_enhanced_answer(query, results[:5])
+ except Exception as e:
+ # Don't fail the entire search if LLM enhancement fails
+ pass
+
+ # Generate suggestions
+ suggestions = await self._generate_suggestions(query, client_id)
+
+ return {
+ "results": results,
+ "total": total_results,
+ "processing_time": processing_time,
+ "enhanced_answer": enhanced_answer,
+ "suggestions": suggestions,
+ }
+
+ async def _semantic_search(
+ self,
+ query: str,
+ client_id: str,
+ filters: Optional[Dict[str, Any]],
+ limit: int,
+ offset: int,
+ min_score: float,
+ ) -> Tuple[List[Dict[str, Any]], int]:
+ """Perform semantic search using vector embeddings."""
+ try:
+ # Generate query embedding
+ query_embeddings = await self.embedding_service.generate_embeddings([query])
+ query_vector = query_embeddings[0]
+
+ # Search vector database
+ vector_results = await self.vector_service.search_vectors(
+ query_vector=query_vector,
+ limit=limit + offset, # Get more to handle offset
+ filters=self._build_vector_filters(client_id, filters),
+ )
+
+ # Filter by minimum score and apply offset
+ filtered_results = [
+ r for r in vector_results
+ if r["score"] >= min_score
+ ][offset:offset + limit]
+
+ # Get document information
+ results = []
+ for vector_result in filtered_results:
+ # Get document chunk info
+ chunk_query = select(DocumentChunk).where(
+ DocumentChunk.id == vector_result["id"]
+ ).join(Document).where(
+ Document.client_id == client_id,
+ Document.status == "indexed"
+ )
+
+ chunk_result = await self.db.execute(chunk_query)
+ chunk = chunk_result.scalar_one_or_none()
+
+ if chunk and chunk.document:
+ results.append({
+ "id": str(chunk.id),
+ "document_id": str(chunk.document_id),
+ "title": chunk.document.title,
+ "content": chunk.content,
+ "url": chunk.document.url,
+ "score": vector_result["score"],
+ "metadata": chunk.metadata,
+ "chunk_index": chunk.chunk_index,
+ })
+
+ return results, len(vector_results)
+
+ except Exception as e:
+ raise ServiceUnavailableError(f"Semantic search failed: {str(e)}")
+
+ async def _keyword_search(
+ self,
+ query: str,
+ client_id: str,
+ filters: Optional[Dict[str, Any]],
+ limit: int,
+ offset: int,
+ ) -> Tuple[List[Dict[str, Any]], int]:
+ """Perform keyword search using full-text search."""
+ try:
+ # Build the base query
+ base_query = select(Document).where(
+ Document.client_id == client_id,
+ Document.status == "indexed"
+ )
+
+ # Add text search
+ search_query = base_query.where(
+ text("to_tsvector('english', title || ' ' || content) @@ plainto_tsquery('english', :query)")
+ ).params(query=query)
+
+ # Apply filters
+ if filters:
+ search_query = self._apply_document_filters(search_query, filters)
+
+ # Add ranking and ordering
+ ranked_query = search_query.add_columns(
+ text("ts_rank(to_tsvector('english', title || ' ' || content), plainto_tsquery('english', :query)) as rank")
+ ).params(query=query).order_by(text("rank DESC"))
+
+ # Get total count
+ count_query = select(func.count()).select_from(
+ ranked_query.subquery()
+ )
+ count_result = await self.db.execute(count_query)
+ total_results = count_result.scalar()
+
+ # Apply pagination
+ paginated_query = ranked_query.offset(offset).limit(limit)
+
+ # Execute search
+ search_result = await self.db.execute(paginated_query)
+ documents = search_result.fetchall()
+
+ # Format results
+ results = []
+ for doc, rank in documents:
+ results.append({
+ "id": str(doc.id),
+ "document_id": str(doc.id),
+ "title": doc.title,
+ "content": doc.content[:1000], # Truncate for display
+ "url": doc.url,
+ "score": float(rank) if rank else 0.0,
+ "metadata": doc.metadata,
+ "chunk_index": 0, # Full document
+ })
+
+ return results, total_results
+
+ except Exception as e:
+ raise ServiceUnavailableError(f"Keyword search failed: {str(e)}")
+
+ async def _hybrid_search(
+ self,
+ query: str,
+ client_id: str,
+ filters: Optional[Dict[str, Any]],
+ limit: int,
+ offset: int,
+ min_score: float,
+ ) -> Tuple[List[Dict[str, Any]], int]:
+ """Perform hybrid search combining semantic and keyword search."""
+ try:
+ # Perform both searches in parallel
+ semantic_task = asyncio.create_task(
+ self._semantic_search(query, client_id, filters, limit * 2, 0, min_score * 0.5)
+ )
+ keyword_task = asyncio.create_task(
+ self._keyword_search(query, client_id, filters, limit * 2, 0)
+ )
+
+ semantic_results, semantic_total = await semantic_task
+ keyword_results, keyword_total = await keyword_task
+
+ # Combine and re-rank results
+ combined_results = self._combine_search_results(
+ semantic_results, keyword_results, query
+ )
+
+ # Apply offset and limit
+ paginated_results = combined_results[offset:offset + limit]
+
+ return paginated_results, len(combined_results)
+
+ except Exception as e:
+ raise ServiceUnavailableError(f"Hybrid search failed: {str(e)}")
+
+ def _combine_search_results(
+ self,
+ semantic_results: List[Dict[str, Any]],
+ keyword_results: List[Dict[str, Any]],
+ query: str,
+ ) -> List[Dict[str, Any]]:
+ """Combine and re-rank semantic and keyword search results."""
+ # Create a mapping of document_id to results
+ result_map = {}
+
+ # Add semantic results with higher weight
+ for result in semantic_results:
+ doc_id = result["document_id"]
+ result["combined_score"] = result["score"] * 0.7 # Semantic weight
+ result_map[doc_id] = result
+
+ # Add keyword results, combining scores if document already exists
+ for result in keyword_results:
+ doc_id = result["document_id"]
+ keyword_score = result["score"] * 0.3 # Keyword weight
+
+ if doc_id in result_map:
+ # Combine scores
+ result_map[doc_id]["combined_score"] += keyword_score
+ # Use the better content (longer usually means more context)
+ if len(result["content"]) > len(result_map[doc_id]["content"]):
+ result_map[doc_id]["content"] = result["content"]
+ else:
+ result["combined_score"] = keyword_score
+ result_map[doc_id] = result
+
+ # Sort by combined score
+ combined_results = list(result_map.values())
+ combined_results.sort(key=lambda x: x["combined_score"], reverse=True)
+
+ # Update the score field to reflect combined score
+ for result in combined_results:
+ result["score"] = result["combined_score"]
+ del result["combined_score"]
+
+ return combined_results
+
+ async def _generate_enhanced_answer(
+ self,
+ query: str,
+ top_results: List[Dict[str, Any]],
+ ) -> Optional[Dict[str, Any]]:
+ """Generate an enhanced answer using LLM."""
+ try:
+ if not top_results:
+ return None
+
+ # Prepare context from search results
+ context_pieces = []
+ sources = []
+
+ for result in top_results:
+ context_pieces.append(f"Title: {result['title']}\nContent: {result['content']}")
+ sources.append({
+ "id": result["document_id"],
+ "title": result["title"],
+ "url": result.get("url"),
+ })
+
+ context = "\n\n---\n\n".join(context_pieces)
+
+ # Generate enhanced answer
+ answer = await self.llm_service.generate_answer(
+ query=query,
+ context=context,
+ max_length=500,
+ )
+
+ if answer:
+ return {
+ "content": answer["content"],
+ "sources": [s["id"] for s in sources],
+ "confidence": answer.get("confidence", 0.8),
+ }
+
+ return None
+
+ except Exception as e:
+ # Log error but don't fail the search
+ return None
+
+ async def _generate_suggestions(
+ self,
+ query: str,
+ client_id: str,
+ limit: int = 5,
+ ) -> List[str]:
+ """Generate search suggestions based on query."""
+ try:
+ # Get popular queries for this client
+ popular_queries = await self.get_popular_queries(client_id, limit * 2)
+
+ # Simple similarity matching (in production, use more sophisticated methods)
+ query_words = set(query.lower().split())
+ suggestions = []
+
+ for popular_query in popular_queries:
+ popular_words = set(popular_query.lower().split())
+ # Calculate Jaccard similarity
+ intersection = len(query_words & popular_words)
+ union = len(query_words | popular_words)
+ similarity = intersection / union if union > 0 else 0
+
+ if similarity > 0.2 and popular_query.lower() != query.lower():
+ suggestions.append(popular_query)
+
+ return suggestions[:limit]
+
+ except Exception:
+ return []
+
+ async def get_popular_queries(
+ self,
+ client_id: str,
+ limit: int = 10,
+ ) -> List[str]:
+ """Get popular search queries for a client."""
+ try:
+ # This would typically query an analytics table
+ # For now, return some mock data
+ return [
+ "getting started guide",
+ "API documentation",
+ "troubleshooting",
+ "best practices",
+ "configuration",
+ ][:limit]
+
+ except Exception:
+ return []
+
+ async def get_autocomplete_suggestions(
+ self,
+ query: str,
+ client_id: str,
+ limit: int = 5,
+ ) -> List[Dict[str, Any]]:
+ """Get autocomplete suggestions for a partial query."""
+ try:
+ suggestions = []
+
+ # Get document titles that match the query
+ title_query = select(Document.title).where(
+ Document.client_id == client_id,
+ Document.status == "indexed",
+ Document.title.ilike(f"%{query}%")
+ ).limit(limit)
+
+ title_result = await self.db.execute(title_query)
+ titles = title_result.scalars().all()
+
+ for title in titles:
+ suggestions.append({
+ "text": title,
+ "score": 0.9,
+ "type": "document_title",
+ })
+
+ # Add popular queries if we need more suggestions
+ if len(suggestions) < limit:
+ popular = await self.get_popular_queries(client_id, limit - len(suggestions))
+ for query_text in popular:
+ if query.lower() in query_text.lower():
+ suggestions.append({
+ "text": query_text,
+ "score": 0.7,
+ "type": "query",
+ })
+
+ return suggestions[:limit]
+
+ except Exception:
+ return []
+
+ def _build_vector_filters(
+ self,
+ client_id: str,
+ filters: Optional[Dict[str, Any]],
+ ) -> Dict[str, Any]:
+ """Build filters for vector database queries."""
+ vector_filters = {"client_id": client_id}
+
+ if filters:
+ if "content_type" in filters:
+ vector_filters["content_type"] = {"$in": filters["content_type"]}
+ if "tags" in filters:
+ vector_filters["tags"] = {"$in": filters["tags"]}
+ if "language" in filters:
+ vector_filters["language"] = filters["language"]
+ if "source_id" in filters:
+ vector_filters["source_id"] = filters["source_id"]
+
+ return vector_filters
+
+ def _apply_document_filters(self, query, filters: Dict[str, Any]):
+ """Apply filters to a document query."""
+ if "content_type" in filters:
+ query = query.where(Document.content_type.in_(filters["content_type"]))
+ if "tags" in filters:
+ query = query.where(Document.tags.overlap(filters["tags"]))
+ if "language" in filters:
+ query = query.where(Document.language == filters["language"])
+ if "source_id" in filters:
+ query = query.where(Document.source_id == filters["source_id"])
+
+ return query
\ No newline at end of file
diff --git a/packages/api/app/services/vector_service.py b/packages/api/app/services/vector_service.py
new file mode 100644
index 0000000..10d1d9d
--- /dev/null
+++ b/packages/api/app/services/vector_service.py
@@ -0,0 +1,357 @@
+"""
+Vector database service for storing and searching embeddings.
+"""
+
+import asyncio
+from typing import List, Dict, Any, Optional, Tuple
+import numpy as np
+
+from app.core.config import get_settings
+from app.core.exceptions import VectorDatabaseError
+
+
+class VectorService:
+ """Service for vector database operations."""
+
+ def __init__(self):
+ self.settings = get_settings()
+ self.client = None
+ self.provider = self.settings.VECTOR_DB_PROVIDER
+
+ async def initialize(self):
+ """Initialize the vector database connection."""
+ if self.provider == "milvus":
+ await self._init_milvus()
+ elif self.provider == "pinecone":
+ await self._init_pinecone()
+ elif self.provider == "weaviate":
+ await self._init_weaviate()
+ elif self.provider == "qdrant":
+ await self._init_qdrant()
+ else:
+ raise VectorDatabaseError(f"Unsupported vector database: {self.provider}")
+
+ async def _init_milvus(self):
+ """Initialize Milvus connection."""
+ try:
+ from pymilvus import connections, Collection, CollectionSchema, FieldSchema, DataType
+
+ # Connect to Milvus
+ connections.connect(
+ alias="default",
+ host=self.settings.MILVUS_HOST,
+ port=self.settings.MILVUS_PORT,
+ user=self.settings.MILVUS_USER,
+ password=self.settings.MILVUS_PASSWORD,
+ )
+
+ # Define collection schema
+ fields = [
+ FieldSchema(name="id", dtype=DataType.VARCHAR, max_length=100, is_primary=True),
+ FieldSchema(name="document_id", dtype=DataType.VARCHAR, max_length=100),
+ FieldSchema(name="chunk_index", dtype=DataType.INT64),
+ FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=self.settings.VECTOR_DB_DIMENSIONS),
+ FieldSchema(name="content", dtype=DataType.VARCHAR, max_length=65535),
+ FieldSchema(name="metadata", dtype=DataType.JSON),
+ ]
+
+ schema = CollectionSchema(fields=fields, description="Document chunks collection")
+
+ # Create or get collection
+ collection_name = "document_chunks"
+ try:
+ self.collection = Collection(name=collection_name, schema=schema)
+ except Exception:
+ # Collection might already exist
+ self.collection = Collection(name=collection_name)
+
+ # Create index
+ index_params = {
+ "metric_type": "COSINE",
+ "index_type": "IVF_FLAT",
+ "params": {"nlist": 1024}
+ }
+
+ try:
+ self.collection.create_index(field_name="embedding", index_params=index_params)
+ except Exception:
+ pass # Index might already exist
+
+ self.collection.load()
+
+ except ImportError:
+ raise VectorDatabaseError("pymilvus library not installed")
+ except Exception as e:
+ raise VectorDatabaseError(f"Milvus initialization failed: {str(e)}")
+
+ async def _init_pinecone(self):
+ """Initialize Pinecone connection."""
+ try:
+ import pinecone
+
+ pinecone.init(
+ api_key=self.settings.PINECONE_API_KEY,
+ environment=self.settings.PINECONE_ENVIRONMENT,
+ )
+
+ # Create index if it doesn't exist
+ index_name = self.settings.PINECONE_INDEX_NAME
+
+ if index_name not in pinecone.list_indexes():
+ pinecone.create_index(
+ name=index_name,
+ dimension=self.settings.VECTOR_DB_DIMENSIONS,
+ metric="cosine",
+ )
+
+ self.index = pinecone.Index(index_name)
+
+ except ImportError:
+ raise VectorDatabaseError("pinecone-client library not installed")
+ except Exception as e:
+ raise VectorDatabaseError(f"Pinecone initialization failed: {str(e)}")
+
+ async def _init_weaviate(self):
+ """Initialize Weaviate connection."""
+ try:
+ import weaviate
+
+ auth_config = None
+ if self.settings.WEAVIATE_API_KEY:
+ auth_config = weaviate.AuthApiKey(api_key=self.settings.WEAVIATE_API_KEY)
+
+ self.client = weaviate.Client(
+ url=self.settings.WEAVIATE_URL,
+ auth_client_secret=auth_config,
+ )
+
+ # Create class schema if it doesn't exist
+ class_name = self.settings.WEAVIATE_CLASS_NAME
+
+ if not self.client.schema.exists(class_name):
+ schema = {
+ "class": class_name,
+ "vectorizer": "none", # We provide our own vectors
+ "properties": [
+ {
+ "name": "document_id",
+ "dataType": ["string"],
+ },
+ {
+ "name": "chunk_index",
+ "dataType": ["int"],
+ },
+ {
+ "name": "content",
+ "dataType": ["text"],
+ },
+ {
+ "name": "metadata",
+ "dataType": ["object"],
+ },
+ ],
+ }
+
+ self.client.schema.create_class(schema)
+
+ except ImportError:
+ raise VectorDatabaseError("weaviate-client library not installed")
+ except Exception as e:
+ raise VectorDatabaseError(f"Weaviate initialization failed: {str(e)}")
+
+ async def _init_qdrant(self):
+ """Initialize Qdrant connection."""
+ try:
+ from qdrant_client import QdrantClient
+ from qdrant_client.http import models
+
+ self.client = QdrantClient(
+ url=self.settings.QDRANT_URL,
+ api_key=self.settings.QDRANT_API_KEY,
+ )
+
+ collection_name = self.settings.QDRANT_COLLECTION_NAME
+
+ # Create collection if it doesn't exist
+ try:
+ self.client.create_collection(
+ collection_name=collection_name,
+ vectors_config=models.VectorParams(
+ size=self.settings.VECTOR_DB_DIMENSIONS,
+ distance=models.Distance.COSINE,
+ ),
+ )
+ except Exception:
+ pass # Collection might already exist
+
+ except ImportError:
+ raise VectorDatabaseError("qdrant-client library not installed")
+ except Exception as e:
+ raise VectorDatabaseError(f"Qdrant initialization failed: {str(e)}")
+
+ async def insert_vectors(
+ self,
+ vectors: List[Dict[str, Any]],
+ ) -> bool:
+ """Insert vectors into the database."""
+ try:
+ if self.provider == "milvus":
+ return await self._insert_milvus(vectors)
+ elif self.provider == "pinecone":
+ return await self._insert_pinecone(vectors)
+ elif self.provider == "weaviate":
+ return await self._insert_weaviate(vectors)
+ elif self.provider == "qdrant":
+ return await self._insert_qdrant(vectors)
+ else:
+ raise VectorDatabaseError(f"Unsupported provider: {self.provider}")
+ except Exception as e:
+ raise VectorDatabaseError(f"Vector insertion failed: {str(e)}")
+
+ async def _insert_milvus(self, vectors: List[Dict[str, Any]]) -> bool:
+ """Insert vectors into Milvus."""
+ data = [
+ [v["id"] for v in vectors],
+ [v["document_id"] for v in vectors],
+ [v["chunk_index"] for v in vectors],
+ [v["embedding"] for v in vectors],
+ [v["content"] for v in vectors],
+ [v["metadata"] for v in vectors],
+ ]
+
+ self.collection.insert(data)
+ self.collection.flush()
+ return True
+
+ async def _insert_pinecone(self, vectors: List[Dict[str, Any]]) -> bool:
+ """Insert vectors into Pinecone."""
+ upsert_data = [
+ (
+ v["id"],
+ v["embedding"],
+ {
+ "document_id": v["document_id"],
+ "chunk_index": v["chunk_index"],
+ "content": v["content"],
+ "metadata": v["metadata"],
+ }
+ )
+ for v in vectors
+ ]
+
+ self.index.upsert(vectors=upsert_data)
+ return True
+
+ async def search_vectors(
+ self,
+ query_vector: List[float],
+ limit: int = 10,
+ filters: Optional[Dict[str, Any]] = None,
+ ) -> List[Dict[str, Any]]:
+ """Search for similar vectors."""
+ try:
+ if self.provider == "milvus":
+ return await self._search_milvus(query_vector, limit, filters)
+ elif self.provider == "pinecone":
+ return await self._search_pinecone(query_vector, limit, filters)
+ elif self.provider == "weaviate":
+ return await self._search_weaviate(query_vector, limit, filters)
+ elif self.provider == "qdrant":
+ return await self._search_qdrant(query_vector, limit, filters)
+ else:
+ raise VectorDatabaseError(f"Unsupported provider: {self.provider}")
+ except Exception as e:
+ raise VectorDatabaseError(f"Vector search failed: {str(e)}")
+
+ async def _search_milvus(
+ self,
+ query_vector: List[float],
+ limit: int,
+ filters: Optional[Dict[str, Any]] = None,
+ ) -> List[Dict[str, Any]]:
+ """Search vectors in Milvus."""
+ search_params = {"metric_type": "COSINE", "params": {"nprobe": 10}}
+
+ results = self.collection.search(
+ data=[query_vector],
+ anns_field="embedding",
+ param=search_params,
+ limit=limit,
+ output_fields=["id", "document_id", "chunk_index", "content", "metadata"],
+ )
+
+ formatted_results = []
+ for hit in results[0]:
+ formatted_results.append({
+ "id": hit.entity.get("id"),
+ "document_id": hit.entity.get("document_id"),
+ "chunk_index": hit.entity.get("chunk_index"),
+ "content": hit.entity.get("content"),
+ "metadata": hit.entity.get("metadata"),
+ "score": hit.score,
+ })
+
+ return formatted_results
+
+ async def _search_pinecone(
+ self,
+ query_vector: List[float],
+ limit: int,
+ filters: Optional[Dict[str, Any]] = None,
+ ) -> List[Dict[str, Any]]:
+ """Search vectors in Pinecone."""
+ results = self.index.query(
+ vector=query_vector,
+ top_k=limit,
+ include_metadata=True,
+ filter=filters,
+ )
+
+ formatted_results = []
+ for match in results["matches"]:
+ formatted_results.append({
+ "id": match["id"],
+ "document_id": match["metadata"]["document_id"],
+ "chunk_index": match["metadata"]["chunk_index"],
+ "content": match["metadata"]["content"],
+ "metadata": match["metadata"]["metadata"],
+ "score": match["score"],
+ })
+
+ return formatted_results
+
+ async def delete_vectors(self, document_id: str) -> bool:
+ """Delete all vectors for a document."""
+ try:
+ if self.provider == "milvus":
+ return await self._delete_milvus(document_id)
+ elif self.provider == "pinecone":
+ return await self._delete_pinecone(document_id)
+ elif self.provider == "weaviate":
+ return await self._delete_weaviate(document_id)
+ elif self.provider == "qdrant":
+ return await self._delete_qdrant(document_id)
+ else:
+ raise VectorDatabaseError(f"Unsupported provider: {self.provider}")
+ except Exception as e:
+ raise VectorDatabaseError(f"Vector deletion failed: {str(e)}")
+
+ async def _delete_milvus(self, document_id: str) -> bool:
+ """Delete vectors from Milvus."""
+ expr = f'document_id == "{document_id}"'
+ self.collection.delete(expr)
+ return True
+
+ async def _delete_pinecone(self, document_id: str) -> bool:
+ """Delete vectors from Pinecone."""
+ self.index.delete(filter={"document_id": document_id})
+ return True
+
+ async def close(self):
+ """Close database connections."""
+ if self.provider == "milvus":
+ from pymilvus import connections
+ connections.disconnect("default")
+ elif hasattr(self, 'client') and self.client:
+ if hasattr(self.client, 'close'):
+ self.client.close()
\ No newline at end of file
diff --git a/packages/api/main.py b/packages/api/main.py
new file mode 100644
index 0000000..384d6dd
--- /dev/null
+++ b/packages/api/main.py
@@ -0,0 +1,250 @@
+"""
+AI Search Platform API
+Production-grade FastAPI application for AI-powered search and content discovery.
+"""
+
+import logging
+import time
+from contextlib import asynccontextmanager
+from typing import AsyncGenerator
+
+import structlog
+from fastapi import FastAPI, Request, Response
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import JSONResponse
+from prometheus_client import Counter, Histogram, generate_latest
+from starlette.middleware.base import BaseHTTPMiddleware
+
+from app.api.routes import api_router
+from app.core.config import get_settings
+from app.core.database import init_db
+from app.core.exceptions import APIException
+from app.services.vector_service import VectorService
+from app.services.embedding_service import EmbeddingService
+from app.services.llm_service import LLMService
+
+# Configure structured logging
+structlog.configure(
+ processors=[
+ structlog.stdlib.filter_by_level,
+ structlog.stdlib.add_logger_name,
+ structlog.stdlib.add_log_level,
+ structlog.stdlib.PositionalArgumentsFormatter(),
+ structlog.processors.TimeStamper(fmt="iso"),
+ structlog.processors.StackInfoRenderer(),
+ structlog.processors.format_exc_info,
+ structlog.processors.UnicodeDecoder(),
+ structlog.processors.JSONRenderer(),
+ ],
+ context_class=dict,
+ logger_factory=structlog.stdlib.LoggerFactory(),
+ wrapper_class=structlog.stdlib.BoundLogger,
+ cache_logger_on_first_use=True,
+)
+
+logger = structlog.get_logger()
+
+# Metrics
+REQUEST_COUNT = Counter(
+ "http_requests_total", "Total HTTP requests", ["method", "endpoint", "status"]
+)
+REQUEST_DURATION = Histogram(
+ "http_request_duration_seconds", "HTTP request duration", ["method", "endpoint"]
+)
+SEARCH_COUNT = Counter("search_requests_total", "Total search requests")
+EMBEDDING_COUNT = Counter("embedding_requests_total", "Total embedding requests")
+
+
+class MetricsMiddleware(BaseHTTPMiddleware):
+ """Middleware to collect Prometheus metrics."""
+
+ async def dispatch(self, request: Request, call_next):
+ start_time = time.time()
+
+ response = await call_next(request)
+
+ duration = time.time() - start_time
+ endpoint = request.url.path
+ method = request.method
+ status = str(response.status_code)
+
+ REQUEST_COUNT.labels(method=method, endpoint=endpoint, status=status).inc()
+ REQUEST_DURATION.labels(method=method, endpoint=endpoint).observe(duration)
+
+ return response
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
+ """Application lifespan events."""
+ settings = get_settings()
+
+ logger.info("Starting AI Search Platform API", version="1.0.0")
+
+ # Initialize database
+ await init_db()
+
+ # Initialize services
+ try:
+ # Initialize vector service
+ vector_service = VectorService()
+ await vector_service.initialize()
+ app.state.vector_service = vector_service
+
+ # Initialize embedding service
+ embedding_service = EmbeddingService()
+ app.state.embedding_service = embedding_service
+
+ # Initialize LLM service
+ llm_service = LLMService()
+ app.state.llm_service = llm_service
+
+ logger.info("All services initialized successfully")
+
+ except Exception as e:
+ logger.error("Failed to initialize services", error=str(e))
+ raise
+
+ yield
+
+ # Cleanup
+ logger.info("Shutting down AI Search Platform API")
+
+ # Close vector service connections
+ if hasattr(app.state, "vector_service"):
+ await app.state.vector_service.close()
+
+
+# Create FastAPI app
+app = FastAPI(
+ title="AI Search Platform API",
+ description="Production-grade AI-powered website search and content discovery platform",
+ version="1.0.0",
+ docs_url="/docs" if get_settings().DEBUG else None,
+ redoc_url="/redoc" if get_settings().DEBUG else None,
+ lifespan=lifespan,
+)
+
+# Add middleware
+app.add_middleware(
+ CORSMiddleware,
+ allow_origins=get_settings().ALLOWED_ORIGINS,
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+)
+
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+app.add_middleware(MetricsMiddleware)
+
+
+# Exception handlers
+@app.exception_handler(APIException)
+async def api_exception_handler(request: Request, exc: APIException) -> JSONResponse:
+ """Handle custom API exceptions."""
+ logger.error(
+ "API exception occurred",
+ error_code=exc.error_code,
+ message=exc.message,
+ details=exc.details,
+ path=request.url.path,
+ )
+
+ return JSONResponse(
+ status_code=exc.status_code,
+ content={
+ "success": False,
+ "error": {
+ "code": exc.error_code,
+ "message": exc.message,
+ "details": exc.details,
+ },
+ "metadata": {
+ "requestId": getattr(request.state, "request_id", None),
+ "timestamp": time.time(),
+ "version": "1.0.0",
+ },
+ },
+ )
+
+
+@app.exception_handler(Exception)
+async def general_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+ """Handle unexpected exceptions."""
+ logger.error(
+ "Unexpected exception occurred",
+ error=str(exc),
+ path=request.url.path,
+ exc_info=True,
+ )
+
+ return JSONResponse(
+ status_code=500,
+ content={
+ "success": False,
+ "error": {
+ "code": "INTERNAL_SERVER_ERROR",
+ "message": "An unexpected error occurred",
+ "details": None,
+ },
+ "metadata": {
+ "requestId": getattr(request.state, "request_id", None),
+ "timestamp": time.time(),
+ "version": "1.0.0",
+ },
+ },
+ )
+
+
+# Health check endpoints
+@app.get("/health")
+async def health_check():
+ """Basic health check endpoint."""
+ return {"status": "healthy", "timestamp": time.time()}
+
+
+@app.get("/health/ready")
+async def readiness_check():
+ """Readiness check endpoint."""
+ # Check if all services are ready
+ checks = {
+ "database": True, # Add actual database check
+ "vector_service": hasattr(app.state, "vector_service"),
+ "embedding_service": hasattr(app.state, "embedding_service"),
+ "llm_service": hasattr(app.state, "llm_service"),
+ }
+
+ all_ready = all(checks.values())
+
+ return {
+ "status": "ready" if all_ready else "not_ready",
+ "checks": checks,
+ "timestamp": time.time(),
+ }
+
+
+@app.get("/metrics")
+async def metrics():
+ """Prometheus metrics endpoint."""
+ return Response(
+ generate_latest(),
+ media_type="text/plain; version=0.0.4; charset=utf-8",
+ )
+
+
+# Include API routes
+app.include_router(api_router, prefix="/api/v1")
+
+
+if __name__ == "__main__":
+ import uvicorn
+
+ settings = get_settings()
+ uvicorn.run(
+ "main:app",
+ host="0.0.0.0",
+ port=8000,
+ reload=settings.DEBUG,
+ log_config=None, # Use structlog instead
+ )
\ No newline at end of file
diff --git a/packages/api/package.json b/packages/api/package.json
new file mode 100644
index 0000000..b6cdfc1
--- /dev/null
+++ b/packages/api/package.json
@@ -0,0 +1,17 @@
+{
+ "name": "@ai-search/api",
+ "version": "1.0.0",
+ "description": "FastAPI backend for AI search platform",
+ "scripts": {
+ "dev": "python -m uvicorn main:app --reload --host 0.0.0.0 --port 8000",
+ "start": "python -m uvicorn main:app --host 0.0.0.0 --port 8000",
+ "test": "python -m pytest tests/ -v",
+ "lint": "python -m flake8 . && python -m black --check .",
+ "format": "python -m black . && python -m isort .",
+ "type-check": "python -m mypy .",
+ "clean": "find . -type d -name __pycache__ -exec rm -rf {} + || true"
+ },
+ "dependencies": {
+ "@ai-search/shared": "workspace:*"
+ }
+}
\ No newline at end of file
diff --git a/packages/api/pyproject.toml b/packages/api/pyproject.toml
new file mode 100644
index 0000000..f2662a2
--- /dev/null
+++ b/packages/api/pyproject.toml
@@ -0,0 +1,47 @@
+[tool.black]
+line-length = 88
+target-version = ['py311']
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+ # directories
+ \.eggs
+ | \.git
+ | \.hg
+ | \.mypy_cache
+ | \.tox
+ | \.venv
+ | build
+ | dist
+)/
+'''
+
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+line_length = 88
+known_first_party = ["app"]
+
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+strict_equality = true
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = "-ra -q --strict-markers"
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+asyncio_mode = "auto"
\ No newline at end of file
diff --git a/packages/api/requirements.txt b/packages/api/requirements.txt
new file mode 100644
index 0000000..edf3665
--- /dev/null
+++ b/packages/api/requirements.txt
@@ -0,0 +1,55 @@
+# FastAPI and web framework
+fastapi==0.104.1
+uvicorn[standard]==0.24.0
+python-multipart==0.0.6
+python-jose[cryptography]==3.3.0
+passlib[bcrypt]==1.7.4
+
+# Database
+sqlalchemy==2.0.23
+alembic==1.12.1
+psycopg2-binary==2.9.9
+redis==5.0.1
+
+# AI/ML
+openai==1.3.5
+anthropic==0.7.7
+transformers==4.35.2
+torch==2.1.1
+sentence-transformers==2.2.2
+numpy==1.24.4
+
+# Vector databases
+pymilvus==2.3.4
+pinecone-client==2.2.4
+weaviate-client==3.25.3
+qdrant-client==1.7.0
+
+# Text processing
+beautifulsoup4==4.12.2
+pypdf==3.17.1
+python-docx==1.1.0
+markdown==3.5.1
+html2text==2020.1.16
+
+# Utilities
+pydantic==2.5.0
+pydantic-settings==2.1.0
+httpx==0.25.2
+aiofiles==23.2.1
+celery==5.3.4
+python-dotenv==1.0.0
+
+# Monitoring and logging
+structlog==23.2.0
+sentry-sdk[fastapi]==1.38.0
+prometheus-client==0.19.0
+
+# Development
+pytest==7.4.3
+pytest-asyncio==0.21.1
+pytest-cov==4.1.0
+black==23.11.0
+flake8==6.1.0
+mypy==1.7.1
+isort==5.12.0
\ No newline at end of file
diff --git a/packages/shared/package.json b/packages/shared/package.json
new file mode 100644
index 0000000..d645428
--- /dev/null
+++ b/packages/shared/package.json
@@ -0,0 +1,24 @@
+{
+ "name": "@ai-search/shared",
+ "version": "1.0.0",
+ "description": "Shared types, utilities, and configurations",
+ "main": "dist/index.js",
+ "types": "dist/index.d.ts",
+ "scripts": {
+ "build": "tsc",
+ "dev": "tsc --watch",
+ "clean": "rm -rf dist",
+ "type-check": "tsc --noEmit",
+ "lint": "eslint src --ext .ts,.tsx"
+ },
+ "dependencies": {
+ "zod": "^3.22.4"
+ },
+ "devDependencies": {
+ "typescript": "^5.2.2",
+ "@types/node": "^20.8.0",
+ "eslint": "^8.51.0",
+ "@typescript-eslint/eslint-plugin": "^6.7.4",
+ "@typescript-eslint/parser": "^6.7.4"
+ }
+}
\ No newline at end of file
diff --git a/packages/shared/src/config/constants.ts b/packages/shared/src/config/constants.ts
new file mode 100644
index 0000000..bb23333
--- /dev/null
+++ b/packages/shared/src/config/constants.ts
@@ -0,0 +1,229 @@
+/**
+ * Application constants and configuration
+ */
+
+// API Configuration
+export const API_CONFIG = {
+ DEFAULT_TIMEOUT: 30000,
+ MAX_RETRIES: 3,
+ RETRY_DELAY: 1000,
+ MAX_REQUEST_SIZE: 10 * 1024 * 1024, // 10MB
+ DEFAULT_PAGE_SIZE: 20,
+ MAX_PAGE_SIZE: 100,
+} as const;
+
+// Search Configuration
+export const SEARCH_CONFIG = {
+ DEFAULT_LIMIT: 10,
+ MAX_LIMIT: 100,
+ MIN_QUERY_LENGTH: 1,
+ MAX_QUERY_LENGTH: 1000,
+ DEFAULT_MIN_SCORE: 0.1,
+ MAX_RESULTS_FOR_ENHANCEMENT: 5,
+ CHUNK_SIZE: 1000,
+ CHUNK_OVERLAP: 200,
+} as const;
+
+// Embedding Configuration
+export const EMBEDDING_CONFIG = {
+ OPENAI: {
+ MODEL: 'text-embedding-ada-002',
+ DIMENSIONS: 1536,
+ MAX_TOKENS: 8191,
+ BATCH_SIZE: 100,
+ },
+ HUGGINGFACE: {
+ MODEL: 'sentence-transformers/all-MiniLM-L6-v2',
+ DIMENSIONS: 384,
+ MAX_TOKENS: 512,
+ BATCH_SIZE: 32,
+ },
+ COHERE: {
+ MODEL: 'embed-english-v2.0',
+ DIMENSIONS: 4096,
+ MAX_TOKENS: 512,
+ BATCH_SIZE: 96,
+ },
+} as const;
+
+// LLM Configuration
+export const LLM_CONFIG = {
+ OPENAI: {
+ MODELS: {
+ GPT_4: 'gpt-4',
+ GPT_4_TURBO: 'gpt-4-turbo-preview',
+ GPT_3_5_TURBO: 'gpt-3.5-turbo',
+ },
+ MAX_TOKENS: 4096,
+ DEFAULT_TEMPERATURE: 0.7,
+ },
+ ANTHROPIC: {
+ MODELS: {
+ CLAUDE_3_OPUS: 'claude-3-opus-20240229',
+ CLAUDE_3_SONNET: 'claude-3-sonnet-20240229',
+ CLAUDE_3_HAIKU: 'claude-3-haiku-20240307',
+ },
+ MAX_TOKENS: 4096,
+ DEFAULT_TEMPERATURE: 0.7,
+ },
+} as const;
+
+// Vector Database Configuration
+export const VECTOR_DB_CONFIG = {
+ MILVUS: {
+ DEFAULT_COLLECTION: 'documents',
+ INDEX_TYPE: 'IVF_FLAT',
+ METRIC_TYPE: 'COSINE',
+ NLIST: 1024,
+ },
+ PINECONE: {
+ DEFAULT_NAMESPACE: 'default',
+ METRIC: 'cosine',
+ PODS: 1,
+ },
+ WEAVIATE: {
+ DEFAULT_CLASS: 'Document',
+ DISTANCE_METRIC: 'cosine',
+ },
+} as const;
+
+// Widget Configuration
+export const WIDGET_CONFIG = {
+ DEFAULT_THEME: {
+ PRIMARY_COLOR: '#007bff',
+ SECONDARY_COLOR: '#6c757d',
+ BACKGROUND_COLOR: '#ffffff',
+ TEXT_COLOR: '#212529',
+ BORDER_RADIUS: 8,
+ FONT_FAMILY: 'system-ui, -apple-system, sans-serif',
+ FONT_SIZE: 14,
+ },
+ DEFAULT_LAYOUT: {
+ POSITION: 'bottom-right',
+ WIDTH: 400,
+ HEIGHT: 500,
+ Z_INDEX: 9999,
+ OFFSET: { X: 20, Y: 20 },
+ },
+ RATE_LIMITS: {
+ SEARCHES_PER_MINUTE: 60,
+ SEARCHES_PER_HOUR: 1000,
+ SEARCHES_PER_DAY: 10000,
+ },
+} as const;
+
+// Analytics Configuration
+export const ANALYTICS_CONFIG = {
+ BATCH_SIZE: 100,
+ FLUSH_INTERVAL: 5000, // 5 seconds
+ MAX_QUEUE_SIZE: 1000,
+ RETENTION_DAYS: 90,
+ AGGREGATION_INTERVALS: ['hour', 'day', 'week', 'month'] as const,
+} as const;
+
+// Content Processing
+export const CONTENT_CONFIG = {
+ MAX_DOCUMENT_SIZE: 50 * 1024 * 1024, // 50MB
+ SUPPORTED_FORMATS: ['text', 'html', 'markdown', 'pdf', 'doc', 'docx'] as const,
+ MAX_CHUNKS_PER_DOCUMENT: 1000,
+ MIN_CHUNK_SIZE: 100,
+ MAX_CHUNK_SIZE: 2000,
+} as const;
+
+// Security Configuration
+export const SECURITY_CONFIG = {
+ JWT_EXPIRY: '24h',
+ API_KEY_LENGTH: 32,
+ MAX_LOGIN_ATTEMPTS: 5,
+ LOCKOUT_DURATION: 15 * 60 * 1000, // 15 minutes
+ BCRYPT_ROUNDS: 12,
+ CORS_MAX_AGE: 86400, // 24 hours
+} as const;
+
+// Error Codes
+export const ERROR_CODES = {
+ // Authentication
+ INVALID_API_KEY: 'INVALID_API_KEY',
+ EXPIRED_TOKEN: 'EXPIRED_TOKEN',
+ INSUFFICIENT_PERMISSIONS: 'INSUFFICIENT_PERMISSIONS',
+
+ // Validation
+ INVALID_INPUT: 'INVALID_INPUT',
+ MISSING_REQUIRED_FIELD: 'MISSING_REQUIRED_FIELD',
+ INVALID_FORMAT: 'INVALID_FORMAT',
+
+ // Search
+ QUERY_TOO_SHORT: 'QUERY_TOO_SHORT',
+ QUERY_TOO_LONG: 'QUERY_TOO_LONG',
+ NO_RESULTS_FOUND: 'NO_RESULTS_FOUND',
+ SEARCH_TIMEOUT: 'SEARCH_TIMEOUT',
+
+ // Content
+ DOCUMENT_NOT_FOUND: 'DOCUMENT_NOT_FOUND',
+ DOCUMENT_TOO_LARGE: 'DOCUMENT_TOO_LARGE',
+ UNSUPPORTED_FORMAT: 'UNSUPPORTED_FORMAT',
+ INDEXING_FAILED: 'INDEXING_FAILED',
+
+ // Rate Limiting
+ RATE_LIMIT_EXCEEDED: 'RATE_LIMIT_EXCEEDED',
+ QUOTA_EXCEEDED: 'QUOTA_EXCEEDED',
+
+ // External Services
+ EMBEDDING_SERVICE_ERROR: 'EMBEDDING_SERVICE_ERROR',
+ LLM_SERVICE_ERROR: 'LLM_SERVICE_ERROR',
+ VECTOR_DB_ERROR: 'VECTOR_DB_ERROR',
+
+ // System
+ INTERNAL_SERVER_ERROR: 'INTERNAL_SERVER_ERROR',
+ SERVICE_UNAVAILABLE: 'SERVICE_UNAVAILABLE',
+ TIMEOUT: 'TIMEOUT',
+} as const;
+
+// HTTP Status Codes
+export const HTTP_STATUS = {
+ OK: 200,
+ CREATED: 201,
+ NO_CONTENT: 204,
+ BAD_REQUEST: 400,
+ UNAUTHORIZED: 401,
+ FORBIDDEN: 403,
+ NOT_FOUND: 404,
+ METHOD_NOT_ALLOWED: 405,
+ CONFLICT: 409,
+ UNPROCESSABLE_ENTITY: 422,
+ TOO_MANY_REQUESTS: 429,
+ INTERNAL_SERVER_ERROR: 500,
+ BAD_GATEWAY: 502,
+ SERVICE_UNAVAILABLE: 503,
+ GATEWAY_TIMEOUT: 504,
+} as const;
+
+// Environment Variables
+export const ENV_VARS = {
+ NODE_ENV: 'NODE_ENV',
+ PORT: 'PORT',
+ DATABASE_URL: 'DATABASE_URL',
+ REDIS_URL: 'REDIS_URL',
+
+ // API Keys
+ OPENAI_API_KEY: 'OPENAI_API_KEY',
+ ANTHROPIC_API_KEY: 'ANTHROPIC_API_KEY',
+ HUGGINGFACE_API_KEY: 'HUGGINGFACE_API_KEY',
+ COHERE_API_KEY: 'COHERE_API_KEY',
+
+ // Vector Databases
+ PINECONE_API_KEY: 'PINECONE_API_KEY',
+ PINECONE_ENVIRONMENT: 'PINECONE_ENVIRONMENT',
+ MILVUS_HOST: 'MILVUS_HOST',
+ MILVUS_PORT: 'MILVUS_PORT',
+ WEAVIATE_URL: 'WEAVIATE_URL',
+ WEAVIATE_API_KEY: 'WEAVIATE_API_KEY',
+
+ // Security
+ JWT_SECRET: 'JWT_SECRET',
+ ENCRYPTION_KEY: 'ENCRYPTION_KEY',
+
+ // Monitoring
+ SENTRY_DSN: 'SENTRY_DSN',
+ LOG_LEVEL: 'LOG_LEVEL',
+} as const;
\ No newline at end of file
diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts
new file mode 100644
index 0000000..f8fe567
--- /dev/null
+++ b/packages/shared/src/index.ts
@@ -0,0 +1,9 @@
+// Export all types
+export * from './types';
+
+// Export utilities
+export * from './utils/validation';
+export * from './utils/text-processing';
+
+// Export configuration
+export * from './config/constants';
\ No newline at end of file
diff --git a/packages/shared/src/types/analytics.ts b/packages/shared/src/types/analytics.ts
new file mode 100644
index 0000000..6fddc7a
--- /dev/null
+++ b/packages/shared/src/types/analytics.ts
@@ -0,0 +1,114 @@
+import { z } from 'zod';
+
+// Analytics event base schema
+export const AnalyticsEventSchema = z.object({
+ id: z.string().uuid(),
+ sessionId: z.string(),
+ userId: z.string().optional(),
+ timestamp: z.date(),
+ eventType: z.enum([
+ 'search_query',
+ 'search_result_click',
+ 'widget_load',
+ 'widget_interaction',
+ 'document_view',
+ 'feedback_positive',
+ 'feedback_negative',
+ 'auto_complete',
+ 'enhanced_answer_view',
+ 'source_click'
+ ]),
+ data: z.record(z.any()),
+ metadata: z.object({
+ userAgent: z.string().optional(),
+ referrer: z.string().optional(),
+ ip: z.string().optional(),
+ country: z.string().optional(),
+ device: z.string().optional(),
+ widgetVersion: z.string().optional(),
+ }).optional(),
+});
+
+// Specific event schemas
+export const SearchQueryEventSchema = AnalyticsEventSchema.extend({
+ eventType: z.literal('search_query'),
+ data: z.object({
+ query: z.string(),
+ searchType: z.enum(['semantic', 'keyword', 'hybrid']),
+ resultsCount: z.number().int().min(0),
+ processingTime: z.number().min(0),
+ filters: z.record(z.any()).optional(),
+ enhancedAnswer: z.boolean().optional(),
+ }),
+});
+
+export const SearchResultClickEventSchema = AnalyticsEventSchema.extend({
+ eventType: z.literal('search_result_click'),
+ data: z.object({
+ query: z.string(),
+ documentId: z.string().uuid(),
+ resultPosition: z.number().int().min(0),
+ score: z.number().min(0).max(1),
+ url: z.string().url().optional(),
+ }),
+});
+
+export const FeedbackEventSchema = AnalyticsEventSchema.extend({
+ eventType: z.enum(['feedback_positive', 'feedback_negative']),
+ data: z.object({
+ query: z.string().optional(),
+ documentId: z.string().uuid().optional(),
+ enhancedAnswerId: z.string().uuid().optional(),
+ comment: z.string().optional(),
+ rating: z.number().int().min(1).max(5).optional(),
+ }),
+});
+
+// Analytics aggregation schemas
+export const SearchAnalyticsSchema = z.object({
+ period: z.enum(['hour', 'day', 'week', 'month']),
+ startDate: z.date(),
+ endDate: z.date(),
+ metrics: z.object({
+ totalSearches: z.number().int().min(0),
+ uniqueUsers: z.number().int().min(0),
+ avgResultsPerQuery: z.number().min(0),
+ avgProcessingTime: z.number().min(0),
+ clickThroughRate: z.number().min(0).max(1),
+ zeroResultsRate: z.number().min(0).max(1),
+ enhancedAnswerUsage: z.number().min(0).max(1),
+ }),
+ topQueries: z.array(z.object({
+ query: z.string(),
+ count: z.number().int().min(0),
+ avgScore: z.number().min(0).max(1),
+ })),
+ topDocuments: z.array(z.object({
+ documentId: z.string().uuid(),
+ title: z.string(),
+ clicks: z.number().int().min(0),
+ avgPosition: z.number().min(0),
+ })),
+});
+
+// User engagement metrics
+export const UserEngagementSchema = z.object({
+ userId: z.string(),
+ sessionId: z.string(),
+ startTime: z.date(),
+ endTime: z.date().optional(),
+ totalQueries: z.number().int().min(0),
+ totalClicks: z.number().int().min(0),
+ avgQueryLength: z.number().min(0),
+ sessionDuration: z.number().min(0), // in seconds
+ bounceRate: z.number().min(0).max(1),
+ conversionEvents: z.array(z.string()),
+});
+
+// Export types
+export type AnalyticsEvent = z.infer;
+export type SearchQueryEvent = z.infer;
+export type SearchResultClickEvent = z.infer;
+export type FeedbackEvent = z.infer;
+export type SearchAnalytics = z.infer;
+export type UserEngagement = z.infer;
\ No newline at end of file
diff --git a/packages/shared/src/types/document.ts b/packages/shared/src/types/document.ts
new file mode 100644
index 0000000..429a2d6
--- /dev/null
+++ b/packages/shared/src/types/document.ts
@@ -0,0 +1,57 @@
+import { z } from 'zod';
+
+// Base document schema
+export const DocumentSchema = z.object({
+ id: z.string().uuid(),
+ url: z.string().url(),
+ title: z.string().min(1).max(500),
+ content: z.string().min(1),
+ metadata: z.record(z.any()).optional(),
+ createdAt: z.date(),
+ updatedAt: z.date(),
+ indexedAt: z.date().optional(),
+ status: z.enum(['pending', 'indexed', 'failed', 'archived']),
+ contentType: z.enum(['text', 'html', 'markdown', 'pdf', 'doc']),
+ language: z.string().default('en'),
+ tags: z.array(z.string()).default([]),
+ sourceId: z.string().optional(), // For tracking content source
+});
+
+// Document chunk for vector storage
+export const DocumentChunkSchema = z.object({
+ id: z.string().uuid(),
+ documentId: z.string().uuid(),
+ content: z.string().min(1),
+ embedding: z.array(z.number()).optional(),
+ chunkIndex: z.number().int().min(0),
+ startOffset: z.number().int().min(0),
+ endOffset: z.number().int().min(0),
+ metadata: z.record(z.any()).optional(),
+});
+
+// Document ingestion request
+export const DocumentIngestionRequestSchema = z.object({
+ url: z.string().url().optional(),
+ content: z.string().min(1).optional(),
+ title: z.string().min(1).max(500),
+ contentType: z.enum(['text', 'html', 'markdown', 'pdf', 'doc']).default('text'),
+ metadata: z.record(z.any()).optional(),
+ tags: z.array(z.string()).default([]),
+ sourceId: z.string().optional(),
+}).refine(data => data.url || data.content, {
+ message: "Either URL or content must be provided"
+});
+
+// Export types
+export type Document = z.infer;
+export type DocumentChunk = z.infer;
+export type DocumentIngestionRequest = z.infer;
+
+// Document status update
+export const DocumentStatusUpdateSchema = z.object({
+ id: z.string().uuid(),
+ status: z.enum(['pending', 'indexed', 'failed', 'archived']),
+ error: z.string().optional(),
+});
+
+export type DocumentStatusUpdate = z.infer;
\ No newline at end of file
diff --git a/packages/shared/src/types/embedding.ts b/packages/shared/src/types/embedding.ts
new file mode 100644
index 0000000..399d759
--- /dev/null
+++ b/packages/shared/src/types/embedding.ts
@@ -0,0 +1,90 @@
+import { z } from 'zod';
+
+// Embedding provider configuration
+export const EmbeddingProviderSchema = z.object({
+ provider: z.enum(['openai', 'huggingface', 'cohere', 'custom']),
+ model: z.string(),
+ apiKey: z.string().optional(),
+ apiUrl: z.string().url().optional(),
+ dimensions: z.number().int().min(1),
+ maxTokens: z.number().int().min(1).optional(),
+ batchSize: z.number().int().min(1).max(100).default(10),
+});
+
+// Embedding request
+export const EmbeddingRequestSchema = z.object({
+ texts: z.array(z.string().min(1)),
+ model: z.string().optional(),
+ provider: z.enum(['openai', 'huggingface', 'cohere', 'custom']).optional(),
+ options: z.object({
+ normalize: z.boolean().default(true),
+ truncate: z.boolean().default(true),
+ }).optional(),
+});
+
+// Embedding response
+export const EmbeddingResponseSchema = z.object({
+ embeddings: z.array(z.array(z.number())),
+ model: z.string(),
+ provider: z.string(),
+ usage: z.object({
+ totalTokens: z.number().int().min(0),
+ promptTokens: z.number().int().min(0),
+ }).optional(),
+});
+
+// Vector search request
+export const VectorSearchRequestSchema = z.object({
+ queryVector: z.array(z.number()),
+ limit: z.number().int().min(1).max(100).default(10),
+ minScore: z.number().min(0).max(1).optional(),
+ filters: z.record(z.any()).optional(),
+ includeMetadata: z.boolean().default(false),
+ includeVectors: z.boolean().default(false),
+});
+
+// Vector search result
+export const VectorSearchResultSchema = z.object({
+ id: z.string(),
+ score: z.number().min(0).max(1),
+ metadata: z.record(z.any()).optional(),
+ vector: z.array(z.number()).optional(),
+});
+
+// Vector database configuration
+export const VectorDatabaseConfigSchema = z.object({
+ provider: z.enum(['milvus', 'pinecone', 'weaviate', 'qdrant', 'chroma']),
+ connectionString: z.string(),
+ collectionName: z.string(),
+ dimensions: z.number().int().min(1),
+ metricType: z.enum(['cosine', 'euclidean', 'dot_product']).default('cosine'),
+ indexType: z.string().optional(),
+ apiKey: z.string().optional(),
+});
+
+// Batch embedding job
+export const BatchEmbeddingJobSchema = z.object({
+ id: z.string().uuid(),
+ status: z.enum(['pending', 'processing', 'completed', 'failed']),
+ documentIds: z.array(z.string().uuid()),
+ provider: z.string(),
+ model: z.string(),
+ createdAt: z.date(),
+ startedAt: z.date().optional(),
+ completedAt: z.date().optional(),
+ progress: z.object({
+ total: z.number().int().min(0),
+ processed: z.number().int().min(0),
+ failed: z.number().int().min(0),
+ }),
+ error: z.string().optional(),
+});
+
+// Export types
+export type EmbeddingProvider = z.infer;
+export type EmbeddingRequest = z.infer;
+export type EmbeddingResponse = z.infer;
+export type VectorSearchRequest = z.infer;
+export type VectorSearchResult = z.infer;
+export type VectorDatabaseConfig = z.infer;
+export type BatchEmbeddingJob = z.infer;
\ No newline at end of file
diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts
new file mode 100644
index 0000000..5f393b8
--- /dev/null
+++ b/packages/shared/src/types/index.ts
@@ -0,0 +1,56 @@
+// Document types
+export * from './document';
+
+// Search types
+export * from './search';
+
+// Analytics types
+export * from './analytics';
+
+// Embedding types
+export * from './embedding';
+
+// LLM types
+export * from './llm';
+
+// Widget types
+export * from './widget';
+
+// Common API response types
+import { z } from 'zod';
+
+export const ApiResponseSchema = z.object({
+ success: z.boolean(),
+ data: z.any().optional(),
+ error: z.object({
+ code: z.string(),
+ message: z.string(),
+ details: z.any().optional(),
+ }).optional(),
+ metadata: z.object({
+ requestId: z.string().uuid(),
+ timestamp: z.date(),
+ version: z.string(),
+ processingTime: z.number().min(0).optional(),
+ }),
+});
+
+export const PaginatedResponseSchema = z.object({
+ items: z.array(z.any()),
+ pagination: z.object({
+ page: z.number().int().min(1),
+ limit: z.number().int().min(1),
+ total: z.number().int().min(0),
+ totalPages: z.number().int().min(0),
+ hasNext: z.boolean(),
+ hasPrev: z.boolean(),
+ }),
+});
+
+export type ApiResponse = z.infer & {
+ data?: T;
+};
+
+export type PaginatedResponse = z.infer & {
+ items: T[];
+};
\ No newline at end of file
diff --git a/packages/shared/src/types/llm.ts b/packages/shared/src/types/llm.ts
new file mode 100644
index 0000000..43134b2
--- /dev/null
+++ b/packages/shared/src/types/llm.ts
@@ -0,0 +1,142 @@
+import { z } from 'zod';
+
+// LLM provider configuration
+export const LLMProviderSchema = z.object({
+ provider: z.enum(['openai', 'anthropic', 'huggingface', 'custom']),
+ model: z.string(),
+ apiKey: z.string().optional(),
+ apiUrl: z.string().url().optional(),
+ maxTokens: z.number().int().min(1).max(32000).default(2000),
+ temperature: z.number().min(0).max(2).default(0.7),
+ topP: z.number().min(0).max(1).default(1),
+ frequencyPenalty: z.number().min(-2).max(2).default(0),
+ presencePenalty: z.number().min(-2).max(2).default(0),
+});
+
+// Enhanced answer request
+export const EnhancedAnswerRequestSchema = z.object({
+ query: z.string().min(1),
+ searchResults: z.array(z.object({
+ id: z.string(),
+ title: z.string(),
+ content: z.string(),
+ url: z.string().url().optional(),
+ score: z.number().min(0).max(1),
+ })),
+ options: z.object({
+ maxLength: z.number().int().min(50).max(2000).default(500),
+ includeSourceCitations: z.boolean().default(true),
+ tone: z.enum(['professional', 'casual', 'technical', 'friendly']).default('professional'),
+ language: z.string().default('en'),
+ customInstructions: z.string().optional(),
+ }).optional(),
+ context: z.object({
+ userId: z.string().optional(),
+ sessionId: z.string().optional(),
+ previousQueries: z.array(z.string()).optional(),
+ }).optional(),
+});
+
+// Enhanced answer response
+export const EnhancedAnswerResponseSchema = z.object({
+ id: z.string().uuid(),
+ content: z.string(),
+ sources: z.array(z.object({
+ id: z.string(),
+ title: z.string(),
+ url: z.string().url().optional(),
+ relevanceScore: z.number().min(0).max(1),
+ })),
+ confidence: z.number().min(0).max(1),
+ model: z.string(),
+ provider: z.string(),
+ usage: z.object({
+ promptTokens: z.number().int().min(0),
+ completionTokens: z.number().int().min(0),
+ totalTokens: z.number().int().min(0),
+ }).optional(),
+ processingTime: z.number().min(0),
+ createdAt: z.date(),
+});
+
+// Content guardrails
+export const ContentGuardrailsSchema = z.object({
+ enableProfanityFilter: z.boolean().default(true),
+ enableToxicityFilter: z.boolean().default(true),
+ enableFactualityCheck: z.boolean().default(false),
+ enableSourceAttribution: z.boolean().default(true),
+ enableHallucinationDetection: z.boolean().default(true),
+ customFilters: z.array(z.object({
+ name: z.string(),
+ pattern: z.string(),
+ action: z.enum(['block', 'warn', 'flag']),
+ })).optional(),
+ maxResponseLength: z.number().int().min(10).max(5000).default(1000),
+ requireSourceCitation: z.boolean().default(false),
+});
+
+// Guardrail violation
+export const GuardrailViolationSchema = z.object({
+ type: z.enum(['profanity', 'toxicity', 'factuality', 'hallucination', 'custom']),
+ severity: z.enum(['low', 'medium', 'high']),
+ message: z.string(),
+ confidence: z.number().min(0).max(1),
+ action: z.enum(['blocked', 'warned', 'flagged']),
+ details: z.record(z.any()).optional(),
+});
+
+// LLM chat message
+export const ChatMessageSchema = z.object({
+ id: z.string().uuid(),
+ role: z.enum(['system', 'user', 'assistant']),
+ content: z.string(),
+ timestamp: z.date(),
+ metadata: z.object({
+ sources: z.array(z.string()).optional(),
+ confidence: z.number().min(0).max(1).optional(),
+ processingTime: z.number().min(0).optional(),
+ model: z.string().optional(),
+ }).optional(),
+});
+
+// Chat session
+export const ChatSessionSchema = z.object({
+ id: z.string().uuid(),
+ userId: z.string().optional(),
+ messages: z.array(ChatMessageSchema),
+ createdAt: z.date(),
+ updatedAt: z.date(),
+ metadata: z.object({
+ title: z.string().optional(),
+ tags: z.array(z.string()).optional(),
+ language: z.string().optional(),
+ }).optional(),
+});
+
+// A/B test configuration for LLM responses
+export const LLMTestConfigSchema = z.object({
+ id: z.string().uuid(),
+ name: z.string(),
+ description: z.string().optional(),
+ variants: z.array(z.object({
+ id: z.string(),
+ name: z.string(),
+ weight: z.number().min(0).max(1),
+ config: LLMProviderSchema,
+ promptTemplate: z.string().optional(),
+ })),
+ enabled: z.boolean().default(false),
+ startDate: z.date().optional(),
+ endDate: z.date().optional(),
+ targetMetrics: z.array(z.enum(['response_quality', 'response_time', 'user_satisfaction'])),
+});
+
+// Export types
+export type LLMProvider = z.infer;
+export type EnhancedAnswerRequest = z.infer;
+export type EnhancedAnswerResponse = z.infer;
+export type ContentGuardrails = z.infer;
+export type GuardrailViolation = z.infer;
+export type ChatMessage = z.infer;
+export type ChatSession = z.infer;
+export type LLMTestConfig = z.infer;
\ No newline at end of file
diff --git a/packages/shared/src/types/search.ts b/packages/shared/src/types/search.ts
new file mode 100644
index 0000000..ae5e5f5
--- /dev/null
+++ b/packages/shared/src/types/search.ts
@@ -0,0 +1,87 @@
+import { z } from 'zod';
+
+// Search query schema
+export const SearchQuerySchema = z.object({
+ query: z.string().min(1).max(1000),
+ filters: z.object({
+ contentType: z.array(z.string()).optional(),
+ tags: z.array(z.string()).optional(),
+ dateRange: z.object({
+ from: z.date().optional(),
+ to: z.date().optional(),
+ }).optional(),
+ language: z.string().optional(),
+ sourceId: z.string().optional(),
+ }).optional(),
+ options: z.object({
+ limit: z.number().int().min(1).max(100).default(10),
+ offset: z.number().int().min(0).default(0),
+ includeContent: z.boolean().default(true),
+ includeMetadata: z.boolean().default(false),
+ minScore: z.number().min(0).max(1).optional(),
+ searchType: z.enum(['semantic', 'keyword', 'hybrid']).default('semantic'),
+ enhanceWithLLM: z.boolean().default(false),
+ }).optional(),
+ sessionId: z.string().optional(),
+ userId: z.string().optional(),
+});
+
+// Search result item
+export const SearchResultItemSchema = z.object({
+ id: z.string().uuid(),
+ documentId: z.string().uuid(),
+ title: z.string(),
+ content: z.string(),
+ url: z.string().url().optional(),
+ score: z.number().min(0).max(1),
+ metadata: z.record(z.any()).optional(),
+ highlights: z.array(z.object({
+ field: z.string(),
+ fragments: z.array(z.string()),
+ })).optional(),
+ chunkIndex: z.number().int().optional(),
+});
+
+// Search response
+export const SearchResponseSchema = z.object({
+ results: z.array(SearchResultItemSchema),
+ total: z.number().int().min(0),
+ query: z.string(),
+ processingTime: z.number().min(0),
+ enhancedAnswer: z.object({
+ content: z.string(),
+ sources: z.array(z.string().uuid()),
+ confidence: z.number().min(0).max(1),
+ }).optional(),
+ suggestions: z.array(z.string()).optional(),
+ facets: z.record(z.array(z.object({
+ value: z.string(),
+ count: z.number().int().min(0),
+ }))).optional(),
+});
+
+// Auto-complete suggestion
+export const AutoCompleteRequestSchema = z.object({
+ query: z.string().min(1).max(100),
+ limit: z.number().int().min(1).max(20).default(5),
+ filters: z.object({
+ contentType: z.array(z.string()).optional(),
+ tags: z.array(z.string()).optional(),
+ language: z.string().optional(),
+ }).optional(),
+});
+
+export const AutoCompleteResponseSchema = z.object({
+ suggestions: z.array(z.object({
+ text: z.string(),
+ score: z.number().min(0).max(1),
+ type: z.enum(['query', 'document_title', 'tag']),
+ })),
+});
+
+// Export types
+export type SearchQuery = z.infer;
+export type SearchResultItem = z.infer;
+export type SearchResponse = z.infer;
+export type AutoCompleteRequest = z.infer;
+export type AutoCompleteResponse = z.infer;
\ No newline at end of file
diff --git a/packages/shared/src/types/widget.ts b/packages/shared/src/types/widget.ts
new file mode 100644
index 0000000..b7d88ad
--- /dev/null
+++ b/packages/shared/src/types/widget.ts
@@ -0,0 +1,149 @@
+import { z } from 'zod';
+
+// Widget configuration
+export const WidgetConfigSchema = z.object({
+ id: z.string().uuid(),
+ name: z.string().min(1).max(100),
+ apiKey: z.string(),
+ theme: z.object({
+ primaryColor: z.string().regex(/^#[0-9A-F]{6}$/i).default('#007bff'),
+ secondaryColor: z.string().regex(/^#[0-9A-F]{6}$/i).default('#6c757d'),
+ backgroundColor: z.string().regex(/^#[0-9A-F]{6}$/i).default('#ffffff'),
+ textColor: z.string().regex(/^#[0-9A-F]{6}$/i).default('#212529'),
+ borderRadius: z.number().min(0).max(50).default(8),
+ fontFamily: z.string().default('system-ui, -apple-system, sans-serif'),
+ fontSize: z.number().min(10).max(24).default(14),
+ }).optional(),
+ layout: z.object({
+ position: z.enum(['bottom-right', 'bottom-left', 'top-right', 'top-left', 'center']).default('bottom-right'),
+ width: z.number().min(200).max(800).default(400),
+ height: z.number().min(300).max(600).default(500),
+ offset: z.object({
+ x: z.number().default(20),
+ y: z.number().default(20),
+ }).optional(),
+ zIndex: z.number().min(1).max(999999).default(9999),
+ }).optional(),
+ behavior: z.object({
+ autoOpen: z.boolean().default(false),
+ showWelcomeMessage: z.boolean().default(true),
+ enableVoiceInput: z.boolean().default(false),
+ enableFileUpload: z.boolean().default(false),
+ maxFileSize: z.number().min(1).max(50).default(10), // MB
+ allowedFileTypes: z.array(z.string()).default(['pdf', 'doc', 'docx', 'txt']),
+ enableFeedback: z.boolean().default(true),
+ enableAnalytics: z.boolean().default(true),
+ }).optional(),
+ content: z.object({
+ welcomeMessage: z.string().default('Hi! How can I help you find what you\'re looking for?'),
+ placeholder: z.string().default('Ask me anything...'),
+ noResultsMessage: z.string().default('I couldn\'t find any relevant results. Try rephrasing your question.'),
+ errorMessage: z.string().default('Something went wrong. Please try again.'),
+ loadingMessage: z.string().default('Searching...'),
+ poweredByText: z.string().default('Powered by AI Search'),
+ feedbackPrompt: z.string().default('Was this helpful?'),
+ }).optional(),
+ features: z.object({
+ enableSemanticSearch: z.boolean().default(true),
+ enableEnhancedAnswers: z.boolean().default(true),
+ enableAutoComplete: z.boolean().default(true),
+ enableSearchSuggestions: z.boolean().default(true),
+ enableResultHighlighting: z.boolean().default(true),
+ enableSourceCitations: z.boolean().default(true),
+ maxResults: z.number().min(1).max(50).default(10),
+ enableFilters: z.boolean().default(false),
+ availableFilters: z.array(z.enum(['contentType', 'tags', 'dateRange', 'language'])).optional(),
+ }).optional(),
+ security: z.object({
+ allowedDomains: z.array(z.string().url()).optional(),
+ enableCSP: z.boolean().default(true),
+ enableCORS: z.boolean().default(true),
+ rateLimitPerMinute: z.number().min(1).max(1000).default(60),
+ }).optional(),
+ createdAt: z.date(),
+ updatedAt: z.date(),
+});
+
+// Widget installation code
+export const WidgetInstallationSchema = z.object({
+ widgetId: z.string().uuid(),
+ scriptTag: z.string(),
+ htmlSnippet: z.string(),
+ customization: z.record(z.any()).optional(),
+ version: z.string().default('1.0.0'),
+});
+
+// Widget analytics event
+export const WidgetAnalyticsEventSchema = z.object({
+ widgetId: z.string().uuid(),
+ eventType: z.enum([
+ 'widget_loaded',
+ 'widget_opened',
+ 'widget_closed',
+ 'search_initiated',
+ 'result_clicked',
+ 'feedback_given',
+ 'error_occurred',
+ 'file_uploaded'
+ ]),
+ timestamp: z.date(),
+ sessionId: z.string(),
+ userId: z.string().optional(),
+ data: z.record(z.any()),
+ metadata: z.object({
+ userAgent: z.string().optional(),
+ referrer: z.string().optional(),
+ viewport: z.object({
+ width: z.number(),
+ height: z.number(),
+ }).optional(),
+ widgetVersion: z.string().optional(),
+ }).optional(),
+});
+
+// Widget performance metrics
+export const WidgetPerformanceSchema = z.object({
+ widgetId: z.string().uuid(),
+ period: z.enum(['hour', 'day', 'week', 'month']),
+ metrics: z.object({
+ totalLoads: z.number().int().min(0),
+ uniqueUsers: z.number().int().min(0),
+ totalSearches: z.number().int().min(0),
+ avgResponseTime: z.number().min(0),
+ errorRate: z.number().min(0).max(1),
+ conversionRate: z.number().min(0).max(1),
+ userSatisfaction: z.number().min(0).max(5).optional(),
+ bounceRate: z.number().min(0).max(1),
+ }),
+ topQueries: z.array(z.object({
+ query: z.string(),
+ count: z.number().int().min(0),
+ })),
+ deviceBreakdown: z.object({
+ desktop: z.number().min(0).max(1),
+ mobile: z.number().min(0).max(1),
+ tablet: z.number().min(0).max(1),
+ }),
+});
+
+// Widget customization template
+export const WidgetTemplateSchema = z.object({
+ id: z.string().uuid(),
+ name: z.string(),
+ description: z.string().optional(),
+ category: z.enum(['ecommerce', 'documentation', 'support', 'blog', 'corporate']),
+ config: WidgetConfigSchema.omit({ id: true, apiKey: true, createdAt: true, updatedAt: true }),
+ preview: z.object({
+ thumbnail: z.string().url(),
+ demo: z.string().url().optional(),
+ }),
+ isPublic: z.boolean().default(false),
+ createdBy: z.string().optional(),
+});
+
+// Export types
+export type WidgetConfig = z.infer;
+export type WidgetInstallation = z.infer;
+export type WidgetAnalyticsEvent = z.infer;
+export type WidgetPerformance = z.infer;
+export type WidgetTemplate = z.infer;
\ No newline at end of file
diff --git a/packages/shared/src/utils/text-processing.ts b/packages/shared/src/utils/text-processing.ts
new file mode 100644
index 0000000..8aca6cb
--- /dev/null
+++ b/packages/shared/src/utils/text-processing.ts
@@ -0,0 +1,201 @@
+/**
+ * Text processing utilities for search and content analysis
+ */
+
+/**
+ * Split text into chunks for embedding
+ */
+export function chunkText(
+ text: string,
+ chunkSize: number = 1000,
+ overlap: number = 200
+): Array<{ content: string; startOffset: number; endOffset: number }> {
+ const chunks: Array<{ content: string; startOffset: number; endOffset: number }> = [];
+
+ if (text.length <= chunkSize) {
+ return [{ content: text, startOffset: 0, endOffset: text.length }];
+ }
+
+ let start = 0;
+ let chunkIndex = 0;
+
+ while (start < text.length) {
+ let end = Math.min(start + chunkSize, text.length);
+
+ // Try to break at sentence boundaries
+ if (end < text.length) {
+ const sentenceEnd = text.lastIndexOf('.', end);
+ const questionEnd = text.lastIndexOf('?', end);
+ const exclamationEnd = text.lastIndexOf('!', end);
+
+ const lastSentenceEnd = Math.max(sentenceEnd, questionEnd, exclamationEnd);
+
+ if (lastSentenceEnd > start + chunkSize * 0.5) {
+ end = lastSentenceEnd + 1;
+ }
+ }
+
+ const content = text.substring(start, end).trim();
+ if (content.length > 0) {
+ chunks.push({
+ content,
+ startOffset: start,
+ endOffset: end,
+ });
+ }
+
+ start = Math.max(start + chunkSize - overlap, end);
+ chunkIndex++;
+ }
+
+ return chunks;
+}
+
+/**
+ * Extract keywords from text
+ */
+export function extractKeywords(text: string, maxKeywords: number = 10): string[] {
+ // Simple keyword extraction - in production, use more sophisticated NLP
+ const words = text
+ .toLowerCase()
+ .replace(/[^\w\s]/g, '')
+ .split(/\s+/)
+ .filter(word => word.length > 3);
+
+ // Remove common stop words
+ const stopWords = new Set([
+ 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with',
+ 'by', 'from', 'up', 'about', 'into', 'through', 'during', 'before',
+ 'after', 'above', 'below', 'between', 'among', 'this', 'that', 'these',
+ 'those', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have',
+ 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should'
+ ]);
+
+ const filteredWords = words.filter(word => !stopWords.has(word));
+
+ // Count word frequency
+ const wordCount = new Map();
+ filteredWords.forEach(word => {
+ wordCount.set(word, (wordCount.get(word) || 0) + 1);
+ });
+
+ // Sort by frequency and return top keywords
+ return Array.from(wordCount.entries())
+ .sort(([, a], [, b]) => b - a)
+ .slice(0, maxKeywords)
+ .map(([word]) => word);
+}
+
+/**
+ * Calculate text similarity using Jaccard index
+ */
+export function calculateTextSimilarity(text1: string, text2: string): number {
+ const words1 = new Set(text1.toLowerCase().split(/\s+/));
+ const words2 = new Set(text2.toLowerCase().split(/\s+/));
+
+ const intersection = new Set([...words1].filter(x => words2.has(x)));
+ const union = new Set([...words1, ...words2]);
+
+ return intersection.size / union.size;
+}
+
+/**
+ * Highlight search terms in text
+ */
+export function highlightSearchTerms(
+ text: string,
+ searchTerms: string[],
+ highlightTag: string = 'mark'
+): string {
+ let highlightedText = text;
+
+ searchTerms.forEach(term => {
+ const regex = new RegExp(`\\b${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'gi');
+ highlightedText = highlightedText.replace(
+ regex,
+ `<${highlightTag}>$&${highlightTag}>`
+ );
+ });
+
+ return highlightedText;
+}
+
+/**
+ * Generate text summary
+ */
+export function generateSummary(text: string, maxLength: number = 200): string {
+ if (text.length <= maxLength) return text;
+
+ // Split into sentences
+ const sentences = text.split(/[.!?]+/).filter(s => s.trim().length > 0);
+
+ if (sentences.length === 0) return text.substring(0, maxLength) + '...';
+
+ let summary = '';
+ for (const sentence of sentences) {
+ const trimmedSentence = sentence.trim();
+ if (summary.length + trimmedSentence.length + 1 <= maxLength) {
+ summary += (summary ? '. ' : '') + trimmedSentence;
+ } else {
+ break;
+ }
+ }
+
+ return summary || text.substring(0, maxLength) + '...';
+}
+
+/**
+ * Clean HTML content for indexing
+ */
+export function cleanHtmlContent(html: string): string {
+ // Remove script and style tags
+ let cleaned = html.replace(/