From a71409731fd589baa8b41e5947089929588c4c22 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 13 Oct 2025 06:42:21 +0000 Subject: [PATCH] feat: Initialize project structure and core components This commit sets up the foundational structure for the AI Search Platform. It includes: - README with project overview and features. - .env.example for environment configuration. - CI/CD pipeline setup (.github/workflows/ci-cd.yml). - .gitignore and .prettierrc for project standards. - Demo application (Next.js) with basic UI and widget integration. - Infrastructure configurations (Kubernetes, Terraform). - Monorepo setup with Turbo.json and package.json. - API backend (FastAPI) with core services (search, documents, analytics), models, and database setup. - Shared library for common types and utilities. - Widget component for frontend integration. Co-authored-by: 251450122889 <251450122889@mujonline.edu.in> --- .env.example | 83 +++ .github/workflows/ci-cd.yml | 200 +++++++ .gitignore | 118 ++++ .prettierrc | 8 + README.md | 88 ++- apps/demo/app/globals.css | 45 ++ apps/demo/app/layout.tsx | 22 + apps/demo/app/page.tsx | 330 ++++++++++++ apps/demo/next.config.js | 9 + apps/demo/package.json | 33 ++ apps/demo/postcss.config.js | 6 + apps/demo/tailwind.config.js | 21 + infrastructure/kubernetes/api-deployment.yaml | 98 ++++ infrastructure/terraform/main.tf | 271 ++++++++++ infrastructure/terraform/variables.tf | 35 ++ package.json | 31 ++ packages/api/app/__init__.py | 1 + packages/api/app/api/__init__.py | 1 + packages/api/app/api/endpoints/__init__.py | 1 + packages/api/app/api/endpoints/documents.py | 342 ++++++++++++ packages/api/app/api/endpoints/search.py | 227 ++++++++ packages/api/app/api/routes.py | 33 ++ packages/api/app/core/config.py | 146 +++++ packages/api/app/core/database.py | 125 +++++ packages/api/app/core/exceptions.py | 133 +++++ packages/api/app/core/security.py | 209 +++++++ packages/api/app/models/__init__.py | 14 + packages/api/app/models/analytics.py | 170 ++++++ packages/api/app/models/client.py | 163 ++++++ packages/api/app/models/document.py | 125 +++++ packages/api/app/services/__init__.py | 1 + .../api/app/services/embedding_service.py | 176 ++++++ packages/api/app/services/llm_service.py | 338 ++++++++++++ packages/api/app/services/search_service.py | 458 ++++++++++++++++ packages/api/app/services/vector_service.py | 357 ++++++++++++ packages/api/main.py | 250 +++++++++ packages/api/package.json | 17 + packages/api/pyproject.toml | 47 ++ packages/api/requirements.txt | 55 ++ packages/shared/package.json | 24 + packages/shared/src/config/constants.ts | 229 ++++++++ packages/shared/src/index.ts | 9 + packages/shared/src/types/analytics.ts | 114 ++++ packages/shared/src/types/document.ts | 57 ++ packages/shared/src/types/embedding.ts | 90 ++++ packages/shared/src/types/index.ts | 56 ++ packages/shared/src/types/llm.ts | 142 +++++ packages/shared/src/types/search.ts | 87 +++ packages/shared/src/types/widget.ts | 149 +++++ packages/shared/src/utils/text-processing.ts | 201 +++++++ packages/shared/src/utils/validation.ts | 108 ++++ packages/shared/tsconfig.json | 23 + packages/widget/package.json | 46 ++ .../widget/src/components/ChatInterface.tsx | 168 ++++++ .../widget/src/components/MessageBubble.tsx | 190 +++++++ .../widget/src/components/ResultsList.tsx | 275 ++++++++++ packages/widget/src/components/SearchBox.tsx | 244 +++++++++ .../widget/src/components/SearchWidget.tsx | 163 ++++++ .../widget/src/components/WidgetButton.tsx | 94 ++++ .../widget/src/components/WidgetContainer.tsx | 182 +++++++ packages/widget/src/hooks/useSearch.ts | 155 ++++++ packages/widget/src/index.ts | 250 +++++++++ packages/widget/src/store/search-store.ts | 113 ++++ packages/widget/src/styles/widget.css | 509 ++++++++++++++++++ packages/widget/src/utils/api.ts | 196 +++++++ packages/widget/tsconfig.json | 29 + packages/widget/vite.config.ts | 33 ++ turbo.json | 27 + 68 files changed, 8747 insertions(+), 3 deletions(-) create mode 100644 .env.example create mode 100644 .github/workflows/ci-cd.yml create mode 100644 .gitignore create mode 100644 .prettierrc create mode 100644 apps/demo/app/globals.css create mode 100644 apps/demo/app/layout.tsx create mode 100644 apps/demo/app/page.tsx create mode 100644 apps/demo/next.config.js create mode 100644 apps/demo/package.json create mode 100644 apps/demo/postcss.config.js create mode 100644 apps/demo/tailwind.config.js create mode 100644 infrastructure/kubernetes/api-deployment.yaml create mode 100644 infrastructure/terraform/main.tf create mode 100644 infrastructure/terraform/variables.tf create mode 100644 package.json create mode 100644 packages/api/app/__init__.py create mode 100644 packages/api/app/api/__init__.py create mode 100644 packages/api/app/api/endpoints/__init__.py create mode 100644 packages/api/app/api/endpoints/documents.py create mode 100644 packages/api/app/api/endpoints/search.py create mode 100644 packages/api/app/api/routes.py create mode 100644 packages/api/app/core/config.py create mode 100644 packages/api/app/core/database.py create mode 100644 packages/api/app/core/exceptions.py create mode 100644 packages/api/app/core/security.py create mode 100644 packages/api/app/models/__init__.py create mode 100644 packages/api/app/models/analytics.py create mode 100644 packages/api/app/models/client.py create mode 100644 packages/api/app/models/document.py create mode 100644 packages/api/app/services/__init__.py create mode 100644 packages/api/app/services/embedding_service.py create mode 100644 packages/api/app/services/llm_service.py create mode 100644 packages/api/app/services/search_service.py create mode 100644 packages/api/app/services/vector_service.py create mode 100644 packages/api/main.py create mode 100644 packages/api/package.json create mode 100644 packages/api/pyproject.toml create mode 100644 packages/api/requirements.txt create mode 100644 packages/shared/package.json create mode 100644 packages/shared/src/config/constants.ts create mode 100644 packages/shared/src/index.ts create mode 100644 packages/shared/src/types/analytics.ts create mode 100644 packages/shared/src/types/document.ts create mode 100644 packages/shared/src/types/embedding.ts create mode 100644 packages/shared/src/types/index.ts create mode 100644 packages/shared/src/types/llm.ts create mode 100644 packages/shared/src/types/search.ts create mode 100644 packages/shared/src/types/widget.ts create mode 100644 packages/shared/src/utils/text-processing.ts create mode 100644 packages/shared/src/utils/validation.ts create mode 100644 packages/shared/tsconfig.json create mode 100644 packages/widget/package.json create mode 100644 packages/widget/src/components/ChatInterface.tsx create mode 100644 packages/widget/src/components/MessageBubble.tsx create mode 100644 packages/widget/src/components/ResultsList.tsx create mode 100644 packages/widget/src/components/SearchBox.tsx create mode 100644 packages/widget/src/components/SearchWidget.tsx create mode 100644 packages/widget/src/components/WidgetButton.tsx create mode 100644 packages/widget/src/components/WidgetContainer.tsx create mode 100644 packages/widget/src/hooks/useSearch.ts create mode 100644 packages/widget/src/index.ts create mode 100644 packages/widget/src/store/search-store.ts create mode 100644 packages/widget/src/styles/widget.css create mode 100644 packages/widget/src/utils/api.ts create mode 100644 packages/widget/tsconfig.json create mode 100644 packages/widget/vite.config.ts create mode 100644 turbo.json diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..271c6bc --- /dev/null +++ b/.env.example @@ -0,0 +1,83 @@ +# Environment Configuration +NODE_ENV=development +DEBUG=true +ENVIRONMENT=development + +# API Configuration +API_HOST=0.0.0.0 +API_PORT=8000 +SECRET_KEY=your-secret-key-change-in-production + +# Database Configuration +DATABASE_URL=postgresql+asyncpg://aisearch:aisearch_password@localhost:5432/aisearch +DATABASE_POOL_SIZE=10 +DATABASE_MAX_OVERFLOW=20 + +# Redis Configuration +REDIS_URL=redis://localhost:6379 +REDIS_MAX_CONNECTIONS=10 + +# OpenAI Configuration +OPENAI_API_KEY=your-openai-api-key +OPENAI_MODEL=gpt-4-turbo-preview +OPENAI_EMBEDDING_MODEL=text-embedding-ada-002 + +# Anthropic Configuration +ANTHROPIC_API_KEY=your-anthropic-api-key +ANTHROPIC_MODEL=claude-3-sonnet-20240229 + +# HuggingFace Configuration +HUGGINGFACE_API_KEY=your-huggingface-api-key +HUGGINGFACE_MODEL=sentence-transformers/all-MiniLM-L6-v2 + +# Cohere Configuration +COHERE_API_KEY=your-cohere-api-key +COHERE_MODEL=embed-english-v2.0 + +# Vector Database Configuration (Milvus) +MILVUS_HOST=localhost +MILVUS_PORT=19530 +MILVUS_USER= +MILVUS_PASSWORD= + +# Vector Database Configuration (Pinecone) +PINECONE_API_KEY=your-pinecone-api-key +PINECONE_ENVIRONMENT=your-pinecone-environment +PINECONE_INDEX_NAME=ai-search + +# Vector Database Configuration (Weaviate) +WEAVIATE_URL=http://localhost:8080 +WEAVIATE_API_KEY=your-weaviate-api-key +WEAVIATE_CLASS_NAME=Document + +# Vector Database Configuration (Qdrant) +QDRANT_URL=http://localhost:6333 +QDRANT_API_KEY=your-qdrant-api-key +QDRANT_COLLECTION_NAME=documents + +# Vector Database Selection +VECTOR_DB_PROVIDER=milvus +VECTOR_DB_DIMENSIONS=1536 + +# Content Processing +MAX_DOCUMENT_SIZE=52428800 # 50MB +CHUNK_SIZE=1000 +CHUNK_OVERLAP=200 + +# Rate Limiting +RATE_LIMIT_REQUESTS=100 +RATE_LIMIT_WINDOW=60 + +# Celery Configuration +CELERY_BROKER_URL=redis://localhost:6379/0 +CELERY_RESULT_BACKEND=redis://localhost:6379/0 + +# Monitoring +SENTRY_DSN=your-sentry-dsn +LOG_LEVEL=INFO + +# File Storage +UPLOAD_DIR=uploads + +# CORS Configuration +ALLOWED_ORIGINS=["http://localhost:3000","http://localhost:8080","https://yourdomain.com"] \ No newline at end of file diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 0000000..7a09de3 --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,200 @@ +name: CI/CD Pipeline + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main ] + +env: + NODE_VERSION: '18' + PYTHON_VERSION: '3.11' + +jobs: + # Frontend Tests + frontend-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run linting + run: npm run lint + + - name: Run type checking + run: npm run type-check + + - name: Run tests + run: npm test + + - name: Build packages + run: npm run build + + # Backend Tests + backend-tests: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: test_aisearch + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + redis: + image: redis:7 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + working-directory: ./packages/api + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run linting + working-directory: ./packages/api + run: | + flake8 . + black --check . + mypy . + + - name: Run tests + working-directory: ./packages/api + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_aisearch + REDIS_URL: redis://localhost:6379 + SECRET_KEY: test-secret-key + run: pytest tests/ -v --cov=app --cov-report=xml + + - name: Upload coverage + uses: codecov/codecov-action@v3 + with: + file: ./packages/api/coverage.xml + flags: backend + + # Security Scanning + security-scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' + + # Build and Deploy + build-deploy: + needs: [frontend-tests, backend-tests] + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + + steps: + - uses: actions/checkout@v4 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push API image + uses: docker/build-push-action@v5 + with: + context: ./packages/api + push: true + tags: | + ghcr.io/${{ github.repository }}/api:latest + ghcr.io/${{ github.repository }}/api:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push Demo image + uses: docker/build-push-action@v5 + with: + context: ./apps/demo + push: true + tags: | + ghcr.io/${{ github.repository }}/demo:latest + ghcr.io/${{ github.repository }}/demo:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Deploy to staging + if: github.ref == 'refs/heads/main' + run: | + echo "Deploying to staging environment..." + # Add your deployment commands here + # e.g., kubectl apply, terraform apply, etc. + + - name: Run integration tests + run: | + echo "Running integration tests..." + # Add integration test commands here + + - name: Deploy to production + if: github.ref == 'refs/heads/main' + run: | + echo "Deploying to production environment..." + # Add production deployment commands here + + # Performance Tests + performance-tests: + needs: [build-deploy] + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + + steps: + - uses: actions/checkout@v4 + + - name: Run load tests + run: | + echo "Running performance tests..." + # Add load testing commands here (e.g., k6, artillery) + + - name: Upload performance results + uses: actions/upload-artifact@v3 + with: + name: performance-results + path: performance-results/ \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0ddd8ff --- /dev/null +++ b/.gitignore @@ -0,0 +1,118 @@ +# Dependencies +node_modules/ +.pnp +.pnp.js + +# Production builds +dist/ +build/ +.next/ +out/ + +# Environment variables +.env +.env.local +.env.development.local +.env.test.local +.env.production.local + +# Logs +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Coverage directory used by tools like istanbul +coverage/ +*.lcov + +# nyc test coverage +.nyc_output + +# Dependency directories +jspm_packages/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next + +# Nuxt.js build / generate output +.nuxt + +# Storybook build outputs +.out +.storybook-out + +# Temporary folders +tmp/ +temp/ + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +*.swp +*.swo +*~ + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Turbo +.turbo + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +venv/ +.venv/ +pip-log.txt +pip-delete-this-directory.txt + +# Database +*.db +*.sqlite +*.sqlite3 + +# Docker +.dockerignore +Dockerfile +docker-compose*.yml \ No newline at end of file diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..46f2372 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,8 @@ +{ + "semi": true, + "trailingComma": "es5", + "singleQuote": true, + "printWidth": 80, + "tabWidth": 2, + "useTabs": false +} \ No newline at end of file diff --git a/README.md b/README.md index 08094e2..b5cdcf3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,85 @@ -# possibility -edit in local machine -This is Shaswat this is my first project +# AI-Powered Website Search Platform + +A production-grade AI-powered website search and content discovery platform that delivers instant, personalized search experiences via RAG, vector search, and LLM integrations. + +## Features + +- **Semantic (RAG) Search**: Advanced AI-powered search using vector embeddings +- **Customizable Widget SDK**: Embeddable search components for any website +- **Content Guardrails**: Validation and content filtering mechanisms +- **Real-time Analytics**: Engagement tracking and performance insights +- **Content Optimization**: AI-driven recommendations and improvements + +## Architecture + +### Frontend Components +- **Website Widget** (JS/TS): Chat UI, search box, result cards +- **Analytics Dashboard**: Real-time metrics and insights +- **Configuration Panel**: Easy setup and customization + +### Backend Services +- **Search API**: Semantic search with RAG capabilities +- **Content Ingestion API**: Document processing and indexing +- **Analytics API**: Event tracking and reporting +- **API Gateway**: Authentication, rate limiting, load balancing + +### AI/ML Layer +- **Embedding Generation**: OpenAI, HuggingFace integrations +- **Vector Search**: Milvus/Pinecone/Weaviate support +- **LLM Integration**: GPT-4, Claude, and custom models +- **Content Guardrails**: Validation and hallucination filtering + +## Development Phases + +### Phase 1: MVP (Months 1-4) +- Core search functionality +- Basic widget SDK +- Document ingestion +- Simple analytics + +### Phase 2: Enhancement (Months 5-7) +- Advanced guardrails +- Multi-LLM support +- A/B testing +- Performance optimization + +### Phase 3: Enterprise Scale (Months 8-11) +- Enterprise features +- Multi-language support +- White-label solutions +- Advanced security + +## Quick Start + +```bash +# Install dependencies +npm install + +# Start development servers +npm run dev + +# Run tests +npm test + +# Build for production +npm run build +``` + +## Project Structure + +``` +├── packages/ +│ ├── widget/ # Embeddable search widget +│ ├── dashboard/ # Analytics dashboard +│ ├── api/ # Backend API services +│ ├── shared/ # Shared types and utilities +│ └── docs/ # Documentation +├── apps/ +│ ├── demo/ # Demo application +│ └── admin/ # Admin interface +└── infrastructure/ # Deployment configs +``` + +## License + +MIT License - see LICENSE file for details. \ No newline at end of file diff --git a/apps/demo/app/globals.css b/apps/demo/app/globals.css new file mode 100644 index 0000000..f2bb22a --- /dev/null +++ b/apps/demo/app/globals.css @@ -0,0 +1,45 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +:root { + --foreground-rgb: 0, 0, 0; + --background-start-rgb: 214, 219, 220; + --background-end-rgb: 255, 255, 255; +} + +@media (prefers-color-scheme: dark) { + :root { + --foreground-rgb: 255, 255, 255; + --background-start-rgb: 0, 0, 0; + --background-end-rgb: 0, 0, 0; + } +} + +body { + color: rgb(var(--foreground-rgb)); + background: linear-gradient( + to bottom, + transparent, + rgb(var(--background-end-rgb)) + ) + rgb(var(--background-start-rgb)); +} + +/* Custom scrollbar */ +::-webkit-scrollbar { + width: 8px; +} + +::-webkit-scrollbar-track { + background: #f1f1f1; +} + +::-webkit-scrollbar-thumb { + background: #c1c1c1; + border-radius: 4px; +} + +::-webkit-scrollbar-thumb:hover { + background: #a8a8a8; +} \ No newline at end of file diff --git a/apps/demo/app/layout.tsx b/apps/demo/app/layout.tsx new file mode 100644 index 0000000..cec8932 --- /dev/null +++ b/apps/demo/app/layout.tsx @@ -0,0 +1,22 @@ +import './globals.css' +import type { Metadata } from 'next' +import { Inter } from 'next/font/google' + +const inter = Inter({ subsets: ['latin'] }) + +export const metadata: Metadata = { + title: 'AI Search Platform Demo', + description: 'Experience the power of AI-driven search and content discovery', +} + +export default function RootLayout({ + children, +}: { + children: React.ReactNode +}) { + return ( + + {children} + + ) +} \ No newline at end of file diff --git a/apps/demo/app/page.tsx b/apps/demo/app/page.tsx new file mode 100644 index 0000000..d7d546f --- /dev/null +++ b/apps/demo/app/page.tsx @@ -0,0 +1,330 @@ +'use client' + +import { useEffect, useState } from 'react' +import { Search, Zap, Shield, BarChart3, Sparkles, Code, Globe, Users } from 'lucide-react' +import { SearchWidget } from '@ai-search/widget' +import type { WidgetConfig } from '@ai-search/shared' + +export default function Home() { + const [widgetLoaded, setWidgetLoaded] = useState(false) + + // Demo widget configuration + const widgetConfig: WidgetConfig = { + id: 'demo-widget-001', + name: 'AI Search Demo', + apiKey: 'demo-api-key-12345', + theme: { + primaryColor: '#3b82f6', + secondaryColor: '#6b7280', + backgroundColor: '#ffffff', + textColor: '#1f2937', + borderRadius: 12, + fontFamily: 'Inter, system-ui, sans-serif', + fontSize: 14, + }, + layout: { + position: 'bottom-right', + width: 420, + height: 600, + offset: { x: 24, y: 24 }, + zIndex: 9999, + }, + behavior: { + autoOpen: false, + showWelcomeMessage: true, + enableVoiceInput: false, + enableFileUpload: true, + enableFeedback: true, + enableAnalytics: true, + }, + content: { + welcomeMessage: "👋 Hi! I'm your AI search assistant. Ask me anything about our platform!", + placeholder: 'Ask me anything...', + noResultsMessage: "I couldn't find what you're looking for. Try rephrasing your question!", + errorMessage: 'Oops! Something went wrong. Please try again.', + loadingMessage: 'Searching...', + poweredByText: 'Powered by AI Search Platform', + feedbackPrompt: 'Was this helpful?', + }, + features: { + enableSemanticSearch: true, + enableEnhancedAnswers: true, + enableAutoComplete: true, + enableSearchSuggestions: true, + enableResultHighlighting: true, + enableSourceCitations: true, + maxResults: 8, + enableFilters: false, + }, + security: { + enableCSP: true, + enableCORS: true, + rateLimitPerMinute: 60, + }, + createdAt: new Date(), + updatedAt: new Date(), + } + + useEffect(() => { + // Simulate widget loading + const timer = setTimeout(() => setWidgetLoaded(true), 1000) + return () => clearTimeout(timer) + }, []) + + const features = [ + { + icon: , + title: 'Semantic Search', + description: 'AI-powered search that understands context and intent, not just keywords.', + }, + { + icon: , + title: 'Enhanced Answers', + description: 'Get comprehensive answers with source citations powered by advanced LLMs.', + }, + { + icon: , + title: 'Easy Integration', + description: 'Drop-in widget that works with any website. No complex setup required.', + }, + { + icon: , + title: 'Real-time Analytics', + description: 'Track search behavior, user engagement, and content performance.', + }, + { + icon: , + title: 'Content Guardrails', + description: 'Built-in validation and filtering to ensure high-quality responses.', + }, + { + icon: , + title: 'Multi-language', + description: 'Support for multiple languages with automatic detection.', + }, + ] + + const stats = [ + { label: 'Search Accuracy', value: '95%' }, + { label: 'Response Time', value: '<200ms' }, + { label: 'Uptime', value: '99.9%' }, + { label: 'Languages', value: '25+' }, + ] + + return ( +
+ {/* Header */} +
+
+
+
+
+ +
+

AI Search Platform

+
+ +
+
+
+ + {/* Hero Section */} +
+
+
+

+ AI-Powered Search + Made Simple +

+

+ Transform your website with intelligent search capabilities. + Deliver instant, personalized experiences with semantic search, + enhanced answers, and real-time analytics. +

+
+ + +
+
+
+
+ + {/* Stats Section */} +
+
+
+ {stats.map((stat, index) => ( +
+
+ {stat.value} +
+
{stat.label}
+
+ ))} +
+
+
+ + {/* Features Section */} +
+
+
+

+ Powerful Features +

+

+ Everything you need to deliver exceptional search experiences +

+
+ +
+ {features.map((feature, index) => ( +
+
{feature.icon}
+

+ {feature.title} +

+

{feature.description}

+
+ ))} +
+
+
+ + {/* Demo Section */} +
+
+
+

+ See It In Action +

+

+ Try our AI search widget right here! Click the chat button in the bottom-right corner. +

+
+ +
+
+
+

+ Interactive Demo +

+
+
+
+
+
+
+

Ask Natural Questions

+

Try "How do I integrate the search widget?" or "What are the pricing plans?"

+
+
+
+
+
+
+
+

Get Enhanced Answers

+

Receive comprehensive responses with source citations and related content.

+
+
+
+
+
+
+
+

Provide Feedback

+

Help improve the system by rating responses and providing feedback.

+
+
+
+
+ +
+
+ +
+
+ Widget Active +
+

+ The AI search widget is now available in the bottom-right corner. + Click it to start exploring! +

+
+
+ Ready to help +
+
+
+
+
+
+ + {/* CTA Section */} +
+
+

+ Ready to Transform Your Search? +

+

+ Join thousands of websites already using AI Search Platform to deliver + exceptional user experiences. +

+
+ + +
+
+
+ + {/* Footer */} +
+
+
+
+
+ +
+ AI Search Platform +
+
+ © 2024 AI Search Platform. All rights reserved. +
+
+
+
+ + {/* AI Search Widget */} + {widgetLoaded && ( + console.error('Widget error:', error)} + /> + )} +
+ ) +} \ No newline at end of file diff --git a/apps/demo/next.config.js b/apps/demo/next.config.js new file mode 100644 index 0000000..8300a26 --- /dev/null +++ b/apps/demo/next.config.js @@ -0,0 +1,9 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + experimental: { + appDir: true, + }, + transpilePackages: ['@ai-search/shared', '@ai-search/widget'], +} + +module.exports = nextConfig \ No newline at end of file diff --git a/apps/demo/package.json b/apps/demo/package.json new file mode 100644 index 0000000..739becc --- /dev/null +++ b/apps/demo/package.json @@ -0,0 +1,33 @@ +{ + "name": "@ai-search/demo", + "version": "1.0.0", + "description": "Demo application showcasing AI Search Platform", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "@ai-search/shared": "workspace:*", + "@ai-search/widget": "workspace:*", + "next": "14.0.3", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "lucide-react": "^0.292.0", + "clsx": "^2.0.0", + "tailwindcss": "^3.3.6" + }, + "devDependencies": { + "@types/node": "^20.8.0", + "@types/react": "^18.2.37", + "@types/react-dom": "^18.2.15", + "autoprefixer": "^10.4.16", + "eslint": "^8.51.0", + "eslint-config-next": "14.0.3", + "postcss": "^8.4.31", + "typescript": "^5.2.2" + } +} \ No newline at end of file diff --git a/apps/demo/postcss.config.js b/apps/demo/postcss.config.js new file mode 100644 index 0000000..96bb01e --- /dev/null +++ b/apps/demo/postcss.config.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} \ No newline at end of file diff --git a/apps/demo/tailwind.config.js b/apps/demo/tailwind.config.js new file mode 100644 index 0000000..d8f4a05 --- /dev/null +++ b/apps/demo/tailwind.config.js @@ -0,0 +1,21 @@ +/** @type {import('tailwindcss').Config} */ +module.exports = { + content: [ + './pages/**/*.{js,ts,jsx,tsx,mdx}', + './components/**/*.{js,ts,jsx,tsx,mdx}', + './app/**/*.{js,ts,jsx,tsx,mdx}', + ], + theme: { + extend: { + colors: { + primary: { + 50: '#eff6ff', + 500: '#3b82f6', + 600: '#2563eb', + 700: '#1d4ed8', + } + } + }, + }, + plugins: [], +} \ No newline at end of file diff --git a/infrastructure/kubernetes/api-deployment.yaml b/infrastructure/kubernetes/api-deployment.yaml new file mode 100644 index 0000000..60ceb44 --- /dev/null +++ b/infrastructure/kubernetes/api-deployment.yaml @@ -0,0 +1,98 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ai-search-api + labels: + app: ai-search-api +spec: + replicas: 3 + selector: + matchLabels: + app: ai-search-api + template: + metadata: + labels: + app: ai-search-api + spec: + containers: + - name: api + image: ghcr.io/ai-search-platform/api:latest + ports: + - containerPort: 8000 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: ai-search-secrets + key: database-url + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: ai-search-secrets + key: redis-url + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: ai-search-secrets + key: secret-key + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: ai-search-secrets + key: openai-api-key + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health/ready + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: ai-search-api-service +spec: + selector: + app: ai-search-api + ports: + - protocol: TCP + port: 80 + targetPort: 8000 + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ai-search-api-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + tls: + - hosts: + - api.aisearch.com + secretName: api-tls + rules: + - host: api.aisearch.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ai-search-api-service + port: + number: 80 \ No newline at end of file diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf new file mode 100644 index 0000000..c92072d --- /dev/null +++ b/infrastructure/terraform/main.tf @@ -0,0 +1,271 @@ +terraform { + required_version = ">= 1.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + } +} + +provider "aws" { + region = var.aws_region +} + +# VPC Configuration +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + + name = "${var.project_name}-vpc" + cidr = "10.0.0.0/16" + + azs = ["${var.aws_region}a", "${var.aws_region}b", "${var.aws_region}c"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] + + enable_nat_gateway = true + enable_vpn_gateway = false + enable_dns_hostnames = true + enable_dns_support = true + + tags = { + Environment = var.environment + Project = var.project_name + } +} + +# EKS Cluster +module "eks" { + source = "terraform-aws-modules/eks/aws" + + cluster_name = "${var.project_name}-${var.environment}" + cluster_version = "1.28" + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + # EKS Managed Node Groups + eks_managed_node_groups = { + main = { + desired_size = 2 + max_size = 10 + min_size = 1 + + instance_types = ["t3.medium"] + capacity_type = "ON_DEMAND" + + k8s_labels = { + Environment = var.environment + NodeGroup = "main" + } + } + } + + tags = { + Environment = var.environment + Project = var.project_name + } +} + +# RDS PostgreSQL +resource "aws_db_subnet_group" "main" { + name = "${var.project_name}-db-subnet-group" + subnet_ids = module.vpc.private_subnets + + tags = { + Name = "${var.project_name} DB subnet group" + } +} + +resource "aws_db_instance" "postgres" { + identifier = "${var.project_name}-postgres-${var.environment}" + + engine = "postgres" + engine_version = "15.4" + instance_class = var.db_instance_class + + allocated_storage = 20 + max_allocated_storage = 100 + storage_type = "gp3" + storage_encrypted = true + + db_name = "aisearch" + username = "aisearch" + password = var.db_password + + vpc_security_group_ids = [aws_security_group.rds.id] + db_subnet_group_name = aws_db_subnet_group.main.name + + backup_retention_period = 7 + backup_window = "03:00-04:00" + maintenance_window = "sun:04:00-sun:05:00" + + skip_final_snapshot = var.environment != "production" + deletion_protection = var.environment == "production" + + tags = { + Environment = var.environment + Project = var.project_name + } +} + +# ElastiCache Redis +resource "aws_elasticache_subnet_group" "main" { + name = "${var.project_name}-cache-subnet" + subnet_ids = module.vpc.private_subnets +} + +resource "aws_elasticache_replication_group" "redis" { + replication_group_id = "${var.project_name}-redis-${var.environment}" + description = "Redis cluster for ${var.project_name}" + + node_type = var.redis_node_type + port = 6379 + parameter_group_name = "default.redis7" + + num_cache_clusters = 2 + + subnet_group_name = aws_elasticache_subnet_group.main.name + security_group_ids = [aws_security_group.redis.id] + + at_rest_encryption_enabled = true + transit_encryption_enabled = true + + tags = { + Environment = var.environment + Project = var.project_name + } +} + +# Security Groups +resource "aws_security_group" "rds" { + name_prefix = "${var.project_name}-rds-" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.project_name}-rds-sg" + } +} + +resource "aws_security_group" "redis" { + name_prefix = "${var.project_name}-redis-" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 6379 + to_port = 6379 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.project_name}-redis-sg" + } +} + +# S3 Bucket for file storage +resource "aws_s3_bucket" "storage" { + bucket = "${var.project_name}-storage-${var.environment}-${random_string.bucket_suffix.result}" + + tags = { + Environment = var.environment + Project = var.project_name + } +} + +resource "random_string" "bucket_suffix" { + length = 8 + special = false + upper = false +} + +resource "aws_s3_bucket_versioning" "storage" { + bucket = aws_s3_bucket.storage.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_encryption" "storage" { + bucket = aws_s3_bucket.storage.id + + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } + } +} + +# CloudWatch Log Groups +resource "aws_cloudwatch_log_group" "api" { + name = "/aws/eks/${var.project_name}-${var.environment}/api" + retention_in_days = 30 + + tags = { + Environment = var.environment + Project = var.project_name + } +} + +# Outputs +output "cluster_endpoint" { + description = "Endpoint for EKS control plane" + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane" + value = module.eks.cluster_security_group_id +} + +output "cluster_iam_role_name" { + description = "IAM role name associated with EKS cluster" + value = module.eks.cluster_iam_role_name +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = module.eks.cluster_certificate_authority_data +} + +output "rds_endpoint" { + description = "RDS instance endpoint" + value = aws_db_instance.postgres.endpoint +} + +output "redis_endpoint" { + description = "ElastiCache Redis endpoint" + value = aws_elasticache_replication_group.redis.configuration_endpoint_address +} + +output "s3_bucket_name" { + description = "Name of the S3 bucket" + value = aws_s3_bucket.storage.bucket +} \ No newline at end of file diff --git a/infrastructure/terraform/variables.tf b/infrastructure/terraform/variables.tf new file mode 100644 index 0000000..d63f6f1 --- /dev/null +++ b/infrastructure/terraform/variables.tf @@ -0,0 +1,35 @@ +variable "aws_region" { + description = "AWS region" + type = string + default = "us-west-2" +} + +variable "environment" { + description = "Environment name" + type = string + default = "staging" +} + +variable "project_name" { + description = "Name of the project" + type = string + default = "ai-search-platform" +} + +variable "db_instance_class" { + description = "RDS instance class" + type = string + default = "db.t3.micro" +} + +variable "db_password" { + description = "Database password" + type = string + sensitive = true +} + +variable "redis_node_type" { + description = "ElastiCache node type" + type = string + default = "cache.t3.micro" +} \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..43f9fc4 --- /dev/null +++ b/package.json @@ -0,0 +1,31 @@ +{ + "name": "ai-search-platform", + "version": "1.0.0", + "description": "Production-grade AI-powered website search and content discovery platform", + "private": true, + "workspaces": [ + "packages/*", + "apps/*" + ], + "scripts": { + "dev": "turbo run dev", + "build": "turbo run build", + "test": "turbo run test", + "lint": "turbo run lint", + "clean": "turbo run clean", + "type-check": "turbo run type-check", + "format": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md}\"", + "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md}\"" + }, + "devDependencies": { + "@turbo/gen": "^1.10.12", + "turbo": "^1.10.12", + "prettier": "^3.0.3", + "typescript": "^5.2.2", + "@types/node": "^20.8.0" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + } +} \ No newline at end of file diff --git a/packages/api/app/__init__.py b/packages/api/app/__init__.py new file mode 100644 index 0000000..2b4d0a1 --- /dev/null +++ b/packages/api/app/__init__.py @@ -0,0 +1 @@ +# AI Search Platform API \ No newline at end of file diff --git a/packages/api/app/api/__init__.py b/packages/api/app/api/__init__.py new file mode 100644 index 0000000..8aace33 --- /dev/null +++ b/packages/api/app/api/__init__.py @@ -0,0 +1 @@ +# API routes package \ No newline at end of file diff --git a/packages/api/app/api/endpoints/__init__.py b/packages/api/app/api/endpoints/__init__.py new file mode 100644 index 0000000..480aee4 --- /dev/null +++ b/packages/api/app/api/endpoints/__init__.py @@ -0,0 +1 @@ +# API endpoints package \ No newline at end of file diff --git a/packages/api/app/api/endpoints/documents.py b/packages/api/app/api/endpoints/documents.py new file mode 100644 index 0000000..b4eeb87 --- /dev/null +++ b/packages/api/app/api/endpoints/documents.py @@ -0,0 +1,342 @@ +"""Document management API endpoints.""" + +import asyncio +from typing import Dict, Any, List, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form +from sqlalchemy.ext.asyncio import AsyncSession +from pydantic import BaseModel, Field, HttpUrl + +from app.core.database import get_db +from app.core.security import require_api_key +from app.core.exceptions import ValidationError, NotFoundError, ServiceUnavailableError +from app.services.document_service import DocumentService +from app.services.ingestion_service import IngestionService + +# Pydantic models +class DocumentIngestionRequest(BaseModel): + url: Optional[HttpUrl] = None + content: Optional[str] = None + title: str = Field(min_length=1, max_length=500) + content_type: str = Field(default="text", pattern="^(text|html|markdown|pdf|doc|docx)$") + metadata: Optional[Dict[str, Any]] = None + tags: List[str] = Field(default_factory=list) + source_id: Optional[str] = None + +class DocumentResponse(BaseModel): + id: str + url: Optional[str] + title: str + content_type: str + status: str + metadata: Optional[Dict[str, Any]] + tags: List[str] + source_id: Optional[str] + created_at: str + updated_at: str + indexed_at: Optional[str] + +class DocumentListResponse(BaseModel): + documents: List[DocumentResponse] + total: int + page: int + limit: int + has_next: bool + has_prev: bool + +class BatchIngestionRequest(BaseModel): + documents: List[DocumentIngestionRequest] + +class BatchIngestionResponse(BaseModel): + job_id: str + status: str + total_documents: int + message: str + +router = APIRouter() + + +@router.post("/ingest", response_model=DocumentResponse) +async def ingest_document( + request_data: DocumentIngestionRequest, + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +) -> DocumentResponse: + """ + Ingest a single document for indexing. + + Supports multiple content types and sources: + - Direct content input + - URL-based content fetching + - File upload (via separate endpoint) + """ + try: + ingestion_service = IngestionService(db) + + # Validate that either URL or content is provided + if not request_data.url and not request_data.content: + raise ValidationError("Either 'url' or 'content' must be provided") + + document = await ingestion_service.ingest_document( + client_id=client_info["client_id"], + url=str(request_data.url) if request_data.url else None, + content=request_data.content, + title=request_data.title, + content_type=request_data.content_type, + metadata=request_data.metadata, + tags=request_data.tags, + source_id=request_data.source_id, + ) + + return DocumentResponse( + id=str(document.id), + url=document.url, + title=document.title, + content_type=document.content_type, + status=document.status, + metadata=document.metadata, + tags=document.tags or [], + source_id=document.source_id, + created_at=document.created_at.isoformat(), + updated_at=document.updated_at.isoformat(), + indexed_at=document.indexed_at.isoformat() if document.indexed_at else None, + ) + + except ValidationError: + raise + except Exception as e: + raise ServiceUnavailableError(f"Document ingestion error: {str(e)}") + + +@router.post("/ingest/batch", response_model=BatchIngestionResponse) +async def ingest_documents_batch( + request_data: BatchIngestionRequest, + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +) -> BatchIngestionResponse: + """ + Ingest multiple documents in batch for efficient processing. + + Creates a background job to process all documents asynchronously. + """ + try: + ingestion_service = IngestionService(db) + + job = await ingestion_service.create_batch_ingestion_job( + client_id=client_info["client_id"], + documents=[doc.dict() for doc in request_data.documents], + ) + + return BatchIngestionResponse( + job_id=str(job.id), + status=job.status, + total_documents=len(request_data.documents), + message="Batch ingestion job created successfully", + ) + + except Exception as e: + raise ServiceUnavailableError(f"Batch ingestion error: {str(e)}") + + +@router.post("/upload", response_model=DocumentResponse) +async def upload_document( + file: UploadFile = File(...), + title: str = Form(...), + content_type: str = Form(default="auto"), + metadata: Optional[str] = Form(default=None), + tags: Optional[str] = Form(default=None), + source_id: Optional[str] = Form(default=None), + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +) -> DocumentResponse: + """ + Upload and ingest a document file. + + Supports various file formats: + - PDF documents + - Word documents (DOC, DOCX) + - Text files + - HTML files + - Markdown files + """ + try: + ingestion_service = IngestionService(db) + + # Parse optional JSON fields + import json + parsed_metadata = json.loads(metadata) if metadata else None + parsed_tags = json.loads(tags) if tags else [] + + document = await ingestion_service.ingest_file( + client_id=client_info["client_id"], + file=file, + title=title, + content_type=content_type, + metadata=parsed_metadata, + tags=parsed_tags, + source_id=source_id, + ) + + return DocumentResponse( + id=str(document.id), + url=document.url, + title=document.title, + content_type=document.content_type, + status=document.status, + metadata=document.metadata, + tags=document.tags or [], + source_id=document.source_id, + created_at=document.created_at.isoformat(), + updated_at=document.updated_at.isoformat(), + indexed_at=document.indexed_at.isoformat() if document.indexed_at else None, + ) + + except ValidationError: + raise + except Exception as e: + raise ServiceUnavailableError(f"File upload error: {str(e)}") + + +@router.get("/", response_model=DocumentListResponse) +async def list_documents( + page: int = 1, + limit: int = 20, + status: Optional[str] = None, + content_type: Optional[str] = None, + source_id: Optional[str] = None, + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +) -> DocumentListResponse: + """ + List documents for the authenticated client. + + Supports filtering by status, content type, and source. + """ + try: + document_service = DocumentService(db) + + result = await document_service.list_documents( + client_id=client_info["client_id"], + page=page, + limit=limit, + status=status, + content_type=content_type, + source_id=source_id, + ) + + return DocumentListResponse( + documents=[ + DocumentResponse( + id=str(doc.id), + url=doc.url, + title=doc.title, + content_type=doc.content_type, + status=doc.status, + metadata=doc.metadata, + tags=doc.tags or [], + source_id=doc.source_id, + created_at=doc.created_at.isoformat(), + updated_at=doc.updated_at.isoformat(), + indexed_at=doc.indexed_at.isoformat() if doc.indexed_at else None, + ) for doc in result["documents"] + ], + total=result["total"], + page=page, + limit=limit, + has_next=result["has_next"], + has_prev=result["has_prev"], + ) + + except Exception as e: + raise ServiceUnavailableError(f"Document listing error: {str(e)}") + + +@router.get("/{document_id}", response_model=DocumentResponse) +async def get_document( + document_id: UUID, + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +) -> DocumentResponse: + """Get a specific document by ID.""" + try: + document_service = DocumentService(db) + + document = await document_service.get_document( + document_id=document_id, + client_id=client_info["client_id"], + ) + + if not document: + raise NotFoundError("Document not found") + + return DocumentResponse( + id=str(document.id), + url=document.url, + title=document.title, + content_type=document.content_type, + status=document.status, + metadata=document.metadata, + tags=document.tags or [], + source_id=document.source_id, + created_at=document.created_at.isoformat(), + updated_at=document.updated_at.isoformat(), + indexed_at=document.indexed_at.isoformat() if document.indexed_at else None, + ) + + except NotFoundError: + raise + except Exception as e: + raise ServiceUnavailableError(f"Document retrieval error: {str(e)}") + + +@router.delete("/{document_id}") +async def delete_document( + document_id: UUID, + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +): + """Delete a document and all associated data.""" + try: + document_service = DocumentService(db) + + success = await document_service.delete_document( + document_id=document_id, + client_id=client_info["client_id"], + ) + + if not success: + raise NotFoundError("Document not found") + + return {"success": True, "message": "Document deleted successfully"} + + except NotFoundError: + raise + except Exception as e: + raise ServiceUnavailableError(f"Document deletion error: {str(e)}") + + +@router.post("/{document_id}/reindex") +async def reindex_document( + document_id: UUID, + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +): + """Reindex a document with updated embeddings.""" + try: + ingestion_service = IngestionService(db) + + success = await ingestion_service.reindex_document( + document_id=document_id, + client_id=client_info["client_id"], + ) + + if not success: + raise NotFoundError("Document not found") + + return {"success": True, "message": "Document reindexing initiated"} + + except NotFoundError: + raise + except Exception as e: + raise ServiceUnavailableError(f"Document reindexing error: {str(e)}") \ No newline at end of file diff --git a/packages/api/app/api/endpoints/search.py b/packages/api/app/api/endpoints/search.py new file mode 100644 index 0000000..e2fad43 --- /dev/null +++ b/packages/api/app/api/endpoints/search.py @@ -0,0 +1,227 @@ +"""Search API endpoints.""" + +import time +from typing import Dict, Any, List +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import get_db +from app.core.security import require_api_key +from app.core.exceptions import ValidationError, ServiceUnavailableError +from app.services.search_service import SearchService +from app.services.analytics_service import AnalyticsService + +# Import shared types (in a real implementation, these would be properly imported) +# For now, we'll define basic Pydantic models + +from pydantic import BaseModel, Field +from typing import Optional, List as ListType + +class SearchFilters(BaseModel): + content_type: Optional[ListType[str]] = None + tags: Optional[ListType[str]] = None + language: Optional[str] = None + source_id: Optional[str] = None + +class SearchOptions(BaseModel): + limit: int = Field(default=10, ge=1, le=100) + offset: int = Field(default=0, ge=0) + include_content: bool = True + include_metadata: bool = False + min_score: Optional[float] = Field(default=None, ge=0, le=1) + search_type: str = Field(default="semantic", pattern="^(semantic|keyword|hybrid)$") + enhance_with_llm: bool = False + +class SearchRequest(BaseModel): + query: str = Field(min_length=1, max_length=1000) + filters: Optional[SearchFilters] = None + options: Optional[SearchOptions] = None + session_id: Optional[str] = None + user_id: Optional[str] = None + +class SearchResultItem(BaseModel): + id: str + document_id: str + title: str + content: str + url: Optional[str] = None + score: float + metadata: Optional[Dict[str, Any]] = None + highlights: Optional[List[Dict[str, Any]]] = None + chunk_index: Optional[int] = None + +class EnhancedAnswer(BaseModel): + content: str + sources: List[str] + confidence: float + +class SearchResponse(BaseModel): + results: List[SearchResultItem] + total: int + query: str + processing_time: float + enhanced_answer: Optional[EnhancedAnswer] = None + suggestions: Optional[List[str]] = None + +class AutoCompleteRequest(BaseModel): + query: str = Field(min_length=1, max_length=100) + limit: int = Field(default=5, ge=1, le=20) + +class AutoCompleteSuggestion(BaseModel): + text: str + score: float + type: str + +class AutoCompleteResponse(BaseModel): + suggestions: List[AutoCompleteSuggestion] + +router = APIRouter() + + +@router.post("/", response_model=SearchResponse) +async def search( + request_data: SearchRequest, + request: Request, + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +) -> SearchResponse: + """ + Perform semantic search across indexed documents. + + This endpoint provides AI-powered search capabilities including: + - Semantic search using vector embeddings + - Keyword search with full-text indexing + - Hybrid search combining both approaches + - Optional LLM-enhanced answers with source citations + """ + start_time = time.time() + + try: + # Initialize services + search_service = SearchService(db) + analytics_service = AnalyticsService(db) + + # Perform search + search_result = await search_service.search( + query=request_data.query, + client_id=client_info["client_id"], + filters=request_data.filters.dict() if request_data.filters else None, + options=request_data.options.dict() if request_data.options else None, + ) + + processing_time = (time.time() - start_time) * 1000 # Convert to milliseconds + + # Track analytics + await analytics_service.track_search_query( + client_id=client_info["client_id"], + session_id=request_data.session_id, + user_id=request_data.user_id, + query=request_data.query, + results_count=len(search_result["results"]), + processing_time=processing_time, + search_type=request_data.options.search_type if request_data.options else "semantic", + enhanced_answer=bool(search_result.get("enhanced_answer")), + ) + + return SearchResponse( + results=[ + SearchResultItem(**result) for result in search_result["results"] + ], + total=search_result["total"], + query=request_data.query, + processing_time=processing_time, + enhanced_answer=EnhancedAnswer(**search_result["enhanced_answer"]) + if search_result.get("enhanced_answer") else None, + suggestions=search_result.get("suggestions", []), + ) + + except ValidationError: + raise + except Exception as e: + raise ServiceUnavailableError(f"Search service error: {str(e)}") + + +@router.post("/autocomplete", response_model=AutoCompleteResponse) +async def autocomplete( + request_data: AutoCompleteRequest, + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +) -> AutoCompleteResponse: + """ + Get search query autocompletion suggestions. + + Provides intelligent query suggestions based on: + - Popular search queries + - Document titles and content + - User's search history + """ + try: + search_service = SearchService(db) + + suggestions = await search_service.get_autocomplete_suggestions( + query=request_data.query, + client_id=client_info["client_id"], + limit=request_data.limit, + ) + + return AutoCompleteResponse( + suggestions=[ + AutoCompleteSuggestion(**suggestion) for suggestion in suggestions + ] + ) + + except Exception as e: + raise ServiceUnavailableError(f"Autocomplete service error: {str(e)}") + + +@router.post("/feedback") +async def submit_feedback( + feedback_data: Dict[str, Any], + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +): + """ + Submit user feedback on search results or enhanced answers. + + Feedback helps improve search quality and relevance. + """ + try: + analytics_service = AnalyticsService(db) + + await analytics_service.track_feedback( + client_id=client_info["client_id"], + feedback_data=feedback_data, + ) + + return {"success": True, "message": "Feedback submitted successfully"} + + except Exception as e: + raise ServiceUnavailableError(f"Feedback service error: {str(e)}") + + +@router.get("/suggestions") +async def get_search_suggestions( + client_id: str, + limit: int = 10, + db: AsyncSession = Depends(get_db), + client_info: Dict[str, Any] = Depends(require_api_key), +): + """ + Get popular search suggestions for the client. + + Returns trending and popular queries to help users discover content. + """ + try: + search_service = SearchService(db) + + suggestions = await search_service.get_popular_queries( + client_id=client_info["client_id"], + limit=limit, + ) + + return {"suggestions": suggestions} + + except Exception as e: + raise ServiceUnavailableError(f"Suggestions service error: {str(e)}") \ No newline at end of file diff --git a/packages/api/app/api/routes.py b/packages/api/app/api/routes.py new file mode 100644 index 0000000..7e08c64 --- /dev/null +++ b/packages/api/app/api/routes.py @@ -0,0 +1,33 @@ +"""Main API router configuration.""" + +from fastapi import APIRouter + +from app.api.endpoints import search, documents, analytics, health + +# Create main API router +api_router = APIRouter() + +# Include endpoint routers +api_router.include_router( + search.router, + prefix="/search", + tags=["search"], +) + +api_router.include_router( + documents.router, + prefix="/documents", + tags=["documents"], +) + +api_router.include_router( + analytics.router, + prefix="/analytics", + tags=["analytics"], +) + +api_router.include_router( + health.router, + prefix="/health", + tags=["health"], +) \ No newline at end of file diff --git a/packages/api/app/core/config.py b/packages/api/app/core/config.py new file mode 100644 index 0000000..2ad832a --- /dev/null +++ b/packages/api/app/core/config.py @@ -0,0 +1,146 @@ +""" +Application configuration management. +""" + +import os +from functools import lru_cache +from typing import List, Optional + +from pydantic import Field +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + """Application settings.""" + + # Environment + DEBUG: bool = Field(default=False, env="DEBUG") + ENVIRONMENT: str = Field(default="development", env="ENVIRONMENT") + + # API Configuration + API_HOST: str = Field(default="0.0.0.0", env="API_HOST") + API_PORT: int = Field(default=8000, env="API_PORT") + API_WORKERS: int = Field(default=1, env="API_WORKERS") + + # Security + SECRET_KEY: str = Field(env="SECRET_KEY") + JWT_ALGORITHM: str = Field(default="HS256", env="JWT_ALGORITHM") + JWT_EXPIRE_MINUTES: int = Field(default=1440, env="JWT_EXPIRE_MINUTES") # 24 hours + + # CORS + ALLOWED_ORIGINS: List[str] = Field( + default=["http://localhost:3000", "http://localhost:8080"], + env="ALLOWED_ORIGINS" + ) + + # Database + DATABASE_URL: str = Field(env="DATABASE_URL") + DATABASE_POOL_SIZE: int = Field(default=10, env="DATABASE_POOL_SIZE") + DATABASE_MAX_OVERFLOW: int = Field(default=20, env="DATABASE_MAX_OVERFLOW") + + # Redis + REDIS_URL: str = Field(env="REDIS_URL") + REDIS_MAX_CONNECTIONS: int = Field(default=10, env="REDIS_MAX_CONNECTIONS") + + # OpenAI + OPENAI_API_KEY: Optional[str] = Field(default=None, env="OPENAI_API_KEY") + OPENAI_MODEL: str = Field(default="gpt-4-turbo-preview", env="OPENAI_MODEL") + OPENAI_EMBEDDING_MODEL: str = Field( + default="text-embedding-ada-002", env="OPENAI_EMBEDDING_MODEL" + ) + + # Anthropic + ANTHROPIC_API_KEY: Optional[str] = Field(default=None, env="ANTHROPIC_API_KEY") + ANTHROPIC_MODEL: str = Field( + default="claude-3-sonnet-20240229", env="ANTHROPIC_MODEL" + ) + + # HuggingFace + HUGGINGFACE_API_KEY: Optional[str] = Field(default=None, env="HUGGINGFACE_API_KEY") + HUGGINGFACE_MODEL: str = Field( + default="sentence-transformers/all-MiniLM-L6-v2", env="HUGGINGFACE_MODEL" + ) + + # Cohere + COHERE_API_KEY: Optional[str] = Field(default=None, env="COHERE_API_KEY") + COHERE_MODEL: str = Field(default="embed-english-v2.0", env="COHERE_MODEL") + + # Vector Databases + # Milvus + MILVUS_HOST: str = Field(default="localhost", env="MILVUS_HOST") + MILVUS_PORT: int = Field(default=19530, env="MILVUS_PORT") + MILVUS_USER: Optional[str] = Field(default=None, env="MILVUS_USER") + MILVUS_PASSWORD: Optional[str] = Field(default=None, env="MILVUS_PASSWORD") + + # Pinecone + PINECONE_API_KEY: Optional[str] = Field(default=None, env="PINECONE_API_KEY") + PINECONE_ENVIRONMENT: Optional[str] = Field(default=None, env="PINECONE_ENVIRONMENT") + PINECONE_INDEX_NAME: str = Field(default="ai-search", env="PINECONE_INDEX_NAME") + + # Weaviate + WEAVIATE_URL: Optional[str] = Field(default=None, env="WEAVIATE_URL") + WEAVIATE_API_KEY: Optional[str] = Field(default=None, env="WEAVIATE_API_KEY") + WEAVIATE_CLASS_NAME: str = Field(default="Document", env="WEAVIATE_CLASS_NAME") + + # Qdrant + QDRANT_URL: Optional[str] = Field(default=None, env="QDRANT_URL") + QDRANT_API_KEY: Optional[str] = Field(default=None, env="QDRANT_API_KEY") + QDRANT_COLLECTION_NAME: str = Field(default="documents", env="QDRANT_COLLECTION_NAME") + + # Vector Database Selection + VECTOR_DB_PROVIDER: str = Field(default="milvus", env="VECTOR_DB_PROVIDER") + VECTOR_DB_DIMENSIONS: int = Field(default=1536, env="VECTOR_DB_DIMENSIONS") + + # Content Processing + MAX_DOCUMENT_SIZE: int = Field(default=50 * 1024 * 1024, env="MAX_DOCUMENT_SIZE") # 50MB + CHUNK_SIZE: int = Field(default=1000, env="CHUNK_SIZE") + CHUNK_OVERLAP: int = Field(default=200, env="CHUNK_OVERLAP") + + # Rate Limiting + RATE_LIMIT_REQUESTS: int = Field(default=100, env="RATE_LIMIT_REQUESTS") + RATE_LIMIT_WINDOW: int = Field(default=60, env="RATE_LIMIT_WINDOW") # seconds + + # Celery (for background tasks) + CELERY_BROKER_URL: str = Field(env="CELERY_BROKER_URL") + CELERY_RESULT_BACKEND: str = Field(env="CELERY_RESULT_BACKEND") + + # Monitoring + SENTRY_DSN: Optional[str] = Field(default=None, env="SENTRY_DSN") + LOG_LEVEL: str = Field(default="INFO", env="LOG_LEVEL") + + # File Storage + UPLOAD_DIR: str = Field(default="uploads", env="UPLOAD_DIR") + + class Config: + env_file = ".env" + env_file_encoding = "utf-8" + case_sensitive = True + + +@lru_cache() +def get_settings() -> Settings: + """Get cached application settings.""" + return Settings() + + +# Environment-specific configurations +def get_database_url() -> str: + """Get database URL with proper configuration.""" + settings = get_settings() + return settings.DATABASE_URL + + +def get_redis_url() -> str: + """Get Redis URL with proper configuration.""" + settings = get_settings() + return settings.REDIS_URL + + +def is_development() -> bool: + """Check if running in development mode.""" + return get_settings().ENVIRONMENT == "development" + + +def is_production() -> bool: + """Check if running in production mode.""" + return get_settings().ENVIRONMENT == "production" \ No newline at end of file diff --git a/packages/api/app/core/database.py b/packages/api/app/core/database.py new file mode 100644 index 0000000..7e994a1 --- /dev/null +++ b/packages/api/app/core/database.py @@ -0,0 +1,125 @@ +""" +Database configuration and connection management. +""" + +import asyncio +from typing import AsyncGenerator + +from sqlalchemy import MetaData, create_engine, pool +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +from app.core.config import get_settings + +# Create base class for models +Base = declarative_base() + +# Naming convention for constraints +convention = { + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + +Base.metadata = MetaData(naming_convention=convention) + +# Database engines and sessions +engine = None +async_engine = None +SessionLocal = None +AsyncSessionLocal = None + + +def create_database_engine(): + """Create synchronous database engine.""" + settings = get_settings() + + # Convert async URL to sync URL for SQLAlchemy + sync_url = settings.DATABASE_URL.replace("postgresql+asyncpg://", "postgresql://") + + return create_engine( + sync_url, + pool_size=settings.DATABASE_POOL_SIZE, + max_overflow=settings.DATABASE_MAX_OVERFLOW, + pool_pre_ping=True, + pool_recycle=300, + echo=settings.DEBUG, + ) + + +def create_async_database_engine(): + """Create asynchronous database engine.""" + settings = get_settings() + + return create_async_engine( + settings.DATABASE_URL, + pool_size=settings.DATABASE_POOL_SIZE, + max_overflow=settings.DATABASE_MAX_OVERFLOW, + pool_pre_ping=True, + pool_recycle=300, + echo=settings.DEBUG, + ) + + +async def init_db(): + """Initialize database connections.""" + global engine, async_engine, SessionLocal, AsyncSessionLocal + + # Create engines + engine = create_database_engine() + async_engine = create_async_database_engine() + + # Create session makers + SessionLocal = sessionmaker( + bind=engine, + autocommit=False, + autoflush=False, + ) + + AsyncSessionLocal = async_sessionmaker( + bind=async_engine, + class_=AsyncSession, + autocommit=False, + autoflush=False, + expire_on_commit=False, + ) + + +async def get_db() -> AsyncGenerator[AsyncSession, None]: + """Get async database session.""" + async with AsyncSessionLocal() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + finally: + await session.close() + + +def get_sync_db(): + """Get synchronous database session.""" + db = SessionLocal() + try: + yield db + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() + + +async def close_db(): + """Close database connections.""" + global engine, async_engine + + if async_engine: + await async_engine.dispose() + + if engine: + engine.dispose() \ No newline at end of file diff --git a/packages/api/app/core/exceptions.py b/packages/api/app/core/exceptions.py new file mode 100644 index 0000000..8c64656 --- /dev/null +++ b/packages/api/app/core/exceptions.py @@ -0,0 +1,133 @@ +""" +Custom exception classes for the API. +""" + +from typing import Any, Dict, Optional + + +class APIException(Exception): + """Base exception class for API errors.""" + + def __init__( + self, + message: str, + status_code: int = 500, + error_code: str = "INTERNAL_SERVER_ERROR", + details: Optional[Dict[str, Any]] = None, + ): + self.message = message + self.status_code = status_code + self.error_code = error_code + self.details = details + super().__init__(message) + + +class ValidationError(APIException): + """Validation error exception.""" + + def __init__(self, message: str, details: Optional[Dict[str, Any]] = None): + super().__init__( + message=message, + status_code=422, + error_code="VALIDATION_ERROR", + details=details, + ) + + +class AuthenticationError(APIException): + """Authentication error exception.""" + + def __init__(self, message: str = "Authentication failed"): + super().__init__( + message=message, + status_code=401, + error_code="AUTHENTICATION_ERROR", + ) + + +class AuthorizationError(APIException): + """Authorization error exception.""" + + def __init__(self, message: str = "Insufficient permissions"): + super().__init__( + message=message, + status_code=403, + error_code="AUTHORIZATION_ERROR", + ) + + +class NotFoundError(APIException): + """Resource not found exception.""" + + def __init__(self, message: str = "Resource not found"): + super().__init__( + message=message, + status_code=404, + error_code="NOT_FOUND", + ) + + +class ConflictError(APIException): + """Resource conflict exception.""" + + def __init__(self, message: str = "Resource conflict"): + super().__init__( + message=message, + status_code=409, + error_code="CONFLICT", + ) + + +class RateLimitError(APIException): + """Rate limit exceeded exception.""" + + def __init__(self, message: str = "Rate limit exceeded"): + super().__init__( + message=message, + status_code=429, + error_code="RATE_LIMIT_EXCEEDED", + ) + + +class ServiceUnavailableError(APIException): + """Service unavailable exception.""" + + def __init__(self, message: str = "Service temporarily unavailable"): + super().__init__( + message=message, + status_code=503, + error_code="SERVICE_UNAVAILABLE", + ) + + +class ExternalServiceError(APIException): + """External service error exception.""" + + def __init__(self, service: str, message: str): + super().__init__( + message=f"{service} service error: {message}", + status_code=502, + error_code="EXTERNAL_SERVICE_ERROR", + details={"service": service}, + ) + + +class EmbeddingServiceError(ExternalServiceError): + """Embedding service error exception.""" + + def __init__(self, message: str): + super().__init__("Embedding", message) + + +class VectorDatabaseError(ExternalServiceError): + """Vector database error exception.""" + + def __init__(self, message: str): + super().__init__("Vector Database", message) + + +class LLMServiceError(ExternalServiceError): + """LLM service error exception.""" + + def __init__(self, message: str): + super().__init__("LLM", message) \ No newline at end of file diff --git a/packages/api/app/core/security.py b/packages/api/app/core/security.py new file mode 100644 index 0000000..d423c46 --- /dev/null +++ b/packages/api/app/core/security.py @@ -0,0 +1,209 @@ +""" +Security utilities for authentication and authorization. +""" + +import secrets +from datetime import datetime, timedelta +from typing import Any, Dict, Optional, Union + +import jwt +from fastapi import Depends, HTTPException, Request, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from passlib.context import CryptContext + +from app.core.config import get_settings +from app.core.exceptions import AuthenticationError, AuthorizationError + +# Password hashing +pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") + +# JWT token scheme +security = HTTPBearer() + + +def create_access_token( + subject: Union[str, Any], expires_delta: Optional[timedelta] = None +) -> str: + """Create JWT access token.""" + settings = get_settings() + + if expires_delta: + expire = datetime.utcnow() + expires_delta + else: + expire = datetime.utcnow() + timedelta(minutes=settings.JWT_EXPIRE_MINUTES) + + to_encode = {"exp": expire, "sub": str(subject)} + + encoded_jwt = jwt.encode( + to_encode, settings.SECRET_KEY, algorithm=settings.JWT_ALGORITHM + ) + + return encoded_jwt + + +def verify_token(token: str) -> Dict[str, Any]: + """Verify and decode JWT token.""" + settings = get_settings() + + try: + payload = jwt.decode( + token, settings.SECRET_KEY, algorithms=[settings.JWT_ALGORITHM] + ) + return payload + except jwt.ExpiredSignatureError: + raise AuthenticationError("Token has expired") + except jwt.JWTError: + raise AuthenticationError("Invalid token") + + +def get_password_hash(password: str) -> str: + """Hash a password.""" + return pwd_context.hash(password) + + +def verify_password(plain_password: str, hashed_password: str) -> bool: + """Verify a password against its hash.""" + return pwd_context.verify(plain_password, hashed_password) + + +def generate_api_key() -> str: + """Generate a secure API key.""" + return secrets.token_urlsafe(32) + + +async def get_current_user( + credentials: HTTPAuthorizationCredentials = Depends(security), +) -> Dict[str, Any]: + """Get current authenticated user from JWT token.""" + try: + payload = verify_token(credentials.credentials) + user_id = payload.get("sub") + + if user_id is None: + raise AuthenticationError("Invalid token payload") + + # In a real application, you would fetch user data from database + return {"id": user_id, "is_active": True} + + except AuthenticationError: + raise + except Exception: + raise AuthenticationError("Could not validate credentials") + + +async def get_api_key(request: Request) -> str: + """Extract and validate API key from request.""" + # Check Authorization header + auth_header = request.headers.get("Authorization") + if auth_header and auth_header.startswith("Bearer "): + api_key = auth_header[7:] # Remove "Bearer " prefix + return api_key + + # Check X-API-Key header + api_key = request.headers.get("X-API-Key") + if api_key: + return api_key + + # Check query parameter + api_key = request.query_params.get("api_key") + if api_key: + return api_key + + raise AuthenticationError("API key required") + + +async def validate_api_key(api_key: str) -> Dict[str, Any]: + """Validate API key and return associated client info.""" + # In a real application, you would validate against database + # For now, we'll do basic validation + + if not api_key or len(api_key) < 20: + raise AuthenticationError("Invalid API key format") + + # Mock validation - replace with actual database lookup + return { + "api_key": api_key, + "client_id": "client_123", + "is_active": True, + "rate_limit": 1000, + "features": ["search", "analytics", "enhanced_answers"], + } + + +class RateLimiter: + """Rate limiting utility.""" + + def __init__(self, redis_client=None): + self.redis_client = redis_client + + async def is_allowed( + self, + key: str, + limit: int, + window: int + ) -> bool: + """Check if request is within rate limit.""" + if not self.redis_client: + return True # No rate limiting if Redis not available + + try: + current = await self.redis_client.get(key) + if current is None: + await self.redis_client.setex(key, window, 1) + return True + + if int(current) >= limit: + return False + + await self.redis_client.incr(key) + return True + + except Exception: + # If Redis fails, allow the request + return True + + +# Dependency to check API key and rate limits +async def require_api_key(request: Request) -> Dict[str, Any]: + """Dependency to require and validate API key.""" + api_key = await get_api_key(request) + client_info = await validate_api_key(api_key) + + # Add rate limiting + rate_limiter = RateLimiter() # Initialize with Redis client + client_id = client_info["client_id"] + + settings = get_settings() + is_allowed = await rate_limiter.is_allowed( + f"rate_limit:{client_id}", + settings.RATE_LIMIT_REQUESTS, + settings.RATE_LIMIT_WINDOW, + ) + + if not is_allowed: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail="Rate limit exceeded", + ) + + return client_info + + +# Permission decorators +def require_permission(permission: str): + """Decorator to require specific permission.""" + def decorator(func): + async def wrapper(*args, **kwargs): + # Get client info from dependency + client_info = kwargs.get("client_info") + if not client_info: + raise AuthorizationError("Client information not available") + + # Check if client has required permission + features = client_info.get("features", []) + if permission not in features: + raise AuthorizationError(f"Permission '{permission}' required") + + return await func(*args, **kwargs) + return wrapper + return decorator \ No newline at end of file diff --git a/packages/api/app/models/__init__.py b/packages/api/app/models/__init__.py new file mode 100644 index 0000000..69a5685 --- /dev/null +++ b/packages/api/app/models/__init__.py @@ -0,0 +1,14 @@ +"""Database models for the AI Search Platform.""" + +from .document import Document, DocumentChunk +from .analytics import AnalyticsEvent, SearchSession +from .client import Client, APIKey + +__all__ = [ + "Document", + "DocumentChunk", + "AnalyticsEvent", + "SearchSession", + "Client", + "APIKey", +] \ No newline at end of file diff --git a/packages/api/app/models/analytics.py b/packages/api/app/models/analytics.py new file mode 100644 index 0000000..be0e637 --- /dev/null +++ b/packages/api/app/models/analytics.py @@ -0,0 +1,170 @@ +"""Analytics models for tracking search behavior and performance.""" + +from datetime import datetime +from typing import Dict, Optional + +from sqlalchemy import ( + Column, + DateTime, + Enum, + ForeignKey, + Integer, + JSON, + String, + Float, + Index, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +import uuid + +from app.core.database import Base + + +class SearchSession(Base): + """Search session model for tracking user interactions.""" + + __tablename__ = "search_sessions" + + # Primary key + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + # Session info + session_id = Column(String(255), nullable=False, unique=True, index=True) + user_id = Column(String(255), nullable=True, index=True) + + # Client association + client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True) + + # Session metadata + user_agent = Column(Text, nullable=True) + ip_address = Column(String(45), nullable=True) # IPv6 compatible + referrer = Column(String(2048), nullable=True) + country = Column(String(2), nullable=True) + device_type = Column(String(50), nullable=True) + + # Session metrics + total_queries = Column(Integer, nullable=False, default=0) + total_clicks = Column(Integer, nullable=False, default=0) + session_duration = Column(Integer, nullable=True) # in seconds + + # Timestamps + started_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + ended_at = Column(DateTime(timezone=True), nullable=True) + last_activity = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False) + + # Relationships + client = relationship("Client", back_populates="search_sessions") + events = relationship("AnalyticsEvent", back_populates="session", cascade="all, delete-orphan") + + # Indexes + __table_args__ = ( + Index("ix_search_sessions_client_started", "client_id", "started_at"), + Index("ix_search_sessions_user_started", "user_id", "started_at"), + ) + + def __repr__(self) -> str: + return f"" + + +class AnalyticsEvent(Base): + """Analytics event model for detailed tracking.""" + + __tablename__ = "analytics_events" + + # Primary key + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + # Session reference + session_id = Column(UUID(as_uuid=True), ForeignKey("search_sessions.id"), nullable=False, index=True) + + # Event details + event_type = Column( + Enum( + "search_query", + "search_result_click", + "widget_load", + "widget_interaction", + "document_view", + "feedback_positive", + "feedback_negative", + "auto_complete", + "enhanced_answer_view", + "source_click", + name="event_type_enum" + ), + nullable=False, + index=True, + ) + + # Event data + data = Column(JSON, nullable=False) + + # Performance metrics + processing_time = Column(Float, nullable=True) # in milliseconds + + # Timestamps + timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False, index=True) + + # Relationships + session = relationship("SearchSession", back_populates="events") + + # Indexes + __table_args__ = ( + Index("ix_analytics_events_session_type", "session_id", "event_type"), + Index("ix_analytics_events_type_timestamp", "event_type", "timestamp"), + Index("ix_analytics_events_timestamp", "timestamp"), + ) + + def __repr__(self) -> str: + return f"" + + +class SearchQuery(Base): + """Search query model for tracking and analysis.""" + + __tablename__ = "search_queries" + + # Primary key + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + # Query details + query_text = Column(Text, nullable=False, index=True) + query_hash = Column(String(64), nullable=False, index=True) # MD5 hash for deduplication + + # Search configuration + search_type = Column( + Enum("semantic", "keyword", "hybrid", name="search_type_enum"), + nullable=False, + default="semantic", + ) + + # Results + results_count = Column(Integer, nullable=False, default=0) + processing_time = Column(Float, nullable=False) # in milliseconds + + # Enhanced answer + enhanced_answer_generated = Column(Boolean, nullable=False, default=False) + enhanced_answer_id = Column(UUID(as_uuid=True), nullable=True) + + # Session and client + session_id = Column(UUID(as_uuid=True), ForeignKey("search_sessions.id"), nullable=False, index=True) + client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True) + + # Timestamps + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + # Relationships + session = relationship("SearchSession") + client = relationship("Client") + + # Indexes + __table_args__ = ( + Index("ix_search_queries_client_created", "client_id", "created_at"), + Index("ix_search_queries_hash_client", "query_hash", "client_id"), + Index("ix_search_queries_text_trgm", "query_text", postgresql_using="gin"), # Full-text search + ) + + def __repr__(self) -> str: + return f"" \ No newline at end of file diff --git a/packages/api/app/models/client.py b/packages/api/app/models/client.py new file mode 100644 index 0000000..347b866 --- /dev/null +++ b/packages/api/app/models/client.py @@ -0,0 +1,163 @@ +"""Client and API key models.""" + +from datetime import datetime +from typing import Dict, List, Optional + +from sqlalchemy import ( + Column, + DateTime, + Boolean, + ForeignKey, + Integer, + JSON, + String, + Text, + Index, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +import uuid + +from app.core.database import Base + + +class Client(Base): + """Client model for API access management.""" + + __tablename__ = "clients" + + # Primary key + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + # Client info + name = Column(String(255), nullable=False) + email = Column(String(255), nullable=False, unique=True, index=True) + company = Column(String(255), nullable=True) + website = Column(String(255), nullable=True) + + # Status + is_active = Column(Boolean, nullable=False, default=True, index=True) + is_verified = Column(Boolean, nullable=False, default=False) + + # Subscription and limits + plan = Column(String(50), nullable=False, default="free") # free, pro, enterprise + monthly_quota = Column(Integer, nullable=False, default=1000) + monthly_usage = Column(Integer, nullable=False, default=0) + + # Configuration + settings = Column(JSON, nullable=True) + allowed_domains = Column(JSON, nullable=True) # Array of allowed domains + + # Timestamps + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False) + last_login = Column(DateTime(timezone=True), nullable=True) + + # Relationships + api_keys = relationship("APIKey", back_populates="client", cascade="all, delete-orphan") + documents = relationship("Document", back_populates="client", cascade="all, delete-orphan") + search_sessions = relationship("SearchSession", back_populates="client", cascade="all, delete-orphan") + + # Indexes + __table_args__ = ( + Index("ix_clients_active_plan", "is_active", "plan"), + Index("ix_clients_created_at", "created_at"), + ) + + def __repr__(self) -> str: + return f"" + + +class APIKey(Base): + """API key model for authentication.""" + + __tablename__ = "api_keys" + + # Primary key + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + # Key details + key_hash = Column(String(255), nullable=False, unique=True, index=True) + key_prefix = Column(String(20), nullable=False) # First few characters for identification + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # Client reference + client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True) + + # Status and permissions + is_active = Column(Boolean, nullable=False, default=True, index=True) + permissions = Column(JSON, nullable=True) # Array of permissions + + # Rate limiting + rate_limit_per_minute = Column(Integer, nullable=False, default=60) + rate_limit_per_hour = Column(Integer, nullable=False, default=1000) + rate_limit_per_day = Column(Integer, nullable=False, default=10000) + + # Usage tracking + total_requests = Column(Integer, nullable=False, default=0) + last_used = Column(DateTime(timezone=True), nullable=True) + + # Timestamps + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False) + expires_at = Column(DateTime(timezone=True), nullable=True) + + # Relationships + client = relationship("Client", back_populates="api_keys") + + # Indexes + __table_args__ = ( + Index("ix_api_keys_client_active", "client_id", "is_active"), + Index("ix_api_keys_expires_at", "expires_at"), + ) + + def __repr__(self) -> str: + return f"" + + +class Usage(Base): + """Usage tracking model for billing and analytics.""" + + __tablename__ = "usage" + + # Primary key + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + # Client and API key + client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True) + api_key_id = Column(UUID(as_uuid=True), ForeignKey("api_keys.id"), nullable=False, index=True) + + # Usage metrics + endpoint = Column(String(255), nullable=False, index=True) + method = Column(String(10), nullable=False) + requests_count = Column(Integer, nullable=False, default=0) + + # Billing metrics + tokens_used = Column(Integer, nullable=False, default=0) + compute_time = Column(Float, nullable=False, default=0.0) # in seconds + + # Time period + date = Column(DateTime(timezone=True), nullable=False, index=True) + hour = Column(Integer, nullable=False) # 0-23 + + # Timestamps + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False) + + # Relationships + client = relationship("Client") + api_key = relationship("APIKey") + + # Indexes + __table_args__ = ( + Index("ix_usage_client_date", "client_id", "date"), + Index("ix_usage_api_key_date", "api_key_id", "date"), + Index("ix_usage_endpoint_date", "endpoint", "date"), + # Unique constraint for deduplication + Index("uq_usage_client_endpoint_date_hour", "client_id", "endpoint", "date", "hour", unique=True), + ) + + def __repr__(self) -> str: + return f"" \ No newline at end of file diff --git a/packages/api/app/models/document.py b/packages/api/app/models/document.py new file mode 100644 index 0000000..cd79be7 --- /dev/null +++ b/packages/api/app/models/document.py @@ -0,0 +1,125 @@ +"""Document and document chunk models.""" + +from datetime import datetime +from typing import Dict, List, Optional + +from sqlalchemy import ( + Column, + DateTime, + Enum, + ForeignKey, + Integer, + JSON, + String, + Text, + Index, +) +from sqlalchemy.dialects.postgresql import UUID, ARRAY +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +import uuid + +from app.core.database import Base + + +class Document(Base): + """Document model for storing indexed content.""" + + __tablename__ = "documents" + + # Primary key + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + # Basic document info + url = Column(String(2048), nullable=True, index=True) + title = Column(String(500), nullable=False, index=True) + content = Column(Text, nullable=False) + content_type = Column( + Enum( + "text", "html", "markdown", "pdf", "doc", "docx", + name="content_type_enum" + ), + nullable=False, + default="text", + ) + + # Metadata + metadata = Column(JSON, nullable=True) + language = Column(String(10), nullable=False, default="en", index=True) + tags = Column(ARRAY(String), nullable=True, index=True) + source_id = Column(String(255), nullable=True, index=True) + + # Status and processing + status = Column( + Enum( + "pending", "indexed", "failed", "archived", + name="document_status_enum" + ), + nullable=False, + default="pending", + index=True, + ) + + # Timestamps + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False) + indexed_at = Column(DateTime(timezone=True), nullable=True) + + # Client association + client_id = Column(UUID(as_uuid=True), ForeignKey("clients.id"), nullable=False, index=True) + + # Relationships + chunks = relationship("DocumentChunk", back_populates="document", cascade="all, delete-orphan") + client = relationship("Client", back_populates="documents") + + # Indexes + __table_args__ = ( + Index("ix_documents_client_status", "client_id", "status"), + Index("ix_documents_client_language", "client_id", "language"), + Index("ix_documents_created_at", "created_at"), + Index("ix_documents_url_hash", func.md5("url")), # For faster URL lookups + ) + + def __repr__(self) -> str: + return f"" + + +class DocumentChunk(Base): + """Document chunk model for vector storage.""" + + __tablename__ = "document_chunks" + + # Primary key + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + # Document reference + document_id = Column(UUID(as_uuid=True), ForeignKey("documents.id"), nullable=False, index=True) + + # Chunk content + content = Column(Text, nullable=False) + chunk_index = Column(Integer, nullable=False) + start_offset = Column(Integer, nullable=False) + end_offset = Column(Integer, nullable=False) + + # Vector embedding (stored as JSON array) + embedding = Column(JSON, nullable=True) + embedding_model = Column(String(255), nullable=True) + + # Metadata + metadata = Column(JSON, nullable=True) + + # Timestamps + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False) + + # Relationships + document = relationship("Document", back_populates="chunks") + + # Indexes + __table_args__ = ( + Index("ix_document_chunks_document_id", "document_id"), + Index("ix_document_chunks_document_chunk", "document_id", "chunk_index"), + ) + + def __repr__(self) -> str: + return f"" \ No newline at end of file diff --git a/packages/api/app/services/__init__.py b/packages/api/app/services/__init__.py new file mode 100644 index 0000000..c66a0b2 --- /dev/null +++ b/packages/api/app/services/__init__.py @@ -0,0 +1 @@ +# Services package \ No newline at end of file diff --git a/packages/api/app/services/embedding_service.py b/packages/api/app/services/embedding_service.py new file mode 100644 index 0000000..e42554d --- /dev/null +++ b/packages/api/app/services/embedding_service.py @@ -0,0 +1,176 @@ +""" +Embedding service for generating vector embeddings from text. +""" + +import asyncio +import httpx +from typing import List, Dict, Any, Optional +import numpy as np + +from app.core.config import get_settings +from app.core.exceptions import EmbeddingServiceError + + +class EmbeddingService: + """Service for generating text embeddings using various providers.""" + + def __init__(self): + self.settings = get_settings() + self.client = httpx.AsyncClient(timeout=30.0) + + async def generate_embeddings( + self, + texts: List[str], + provider: str = "openai", + model: Optional[str] = None, + ) -> List[List[float]]: + """Generate embeddings for a list of texts.""" + if provider == "openai": + return await self._generate_openai_embeddings(texts, model) + elif provider == "huggingface": + return await self._generate_huggingface_embeddings(texts, model) + elif provider == "cohere": + return await self._generate_cohere_embeddings(texts, model) + else: + raise EmbeddingServiceError(f"Unsupported embedding provider: {provider}") + + async def _generate_openai_embeddings( + self, + texts: List[str], + model: Optional[str] = None + ) -> List[List[float]]: + """Generate embeddings using OpenAI API.""" + if not self.settings.OPENAI_API_KEY: + raise EmbeddingServiceError("OpenAI API key not configured") + + model = model or self.settings.OPENAI_EMBEDDING_MODEL + + try: + # Process in batches to avoid rate limits + batch_size = 100 + all_embeddings = [] + + for i in range(0, len(texts), batch_size): + batch = texts[i:i + batch_size] + + response = await self.client.post( + "https://api.openai.com/v1/embeddings", + headers={ + "Authorization": f"Bearer {self.settings.OPENAI_API_KEY}", + "Content-Type": "application/json", + }, + json={ + "input": batch, + "model": model, + }, + ) + + if response.status_code != 200: + error_data = response.json() + raise EmbeddingServiceError( + f"OpenAI API error: {error_data.get('error', {}).get('message', 'Unknown error')}" + ) + + data = response.json() + batch_embeddings = [item["embedding"] for item in data["data"]] + all_embeddings.extend(batch_embeddings) + + return all_embeddings + + except httpx.RequestError as e: + raise EmbeddingServiceError(f"OpenAI API request failed: {str(e)}") + except Exception as e: + raise EmbeddingServiceError(f"OpenAI embedding generation failed: {str(e)}") + + async def _generate_huggingface_embeddings( + self, + texts: List[str], + model: Optional[str] = None + ) -> List[List[float]]: + """Generate embeddings using HuggingFace API.""" + model = model or self.settings.HUGGINGFACE_MODEL + + try: + # For HuggingFace, we'll use the sentence-transformers library locally + # In production, you might want to use their API or host your own model + from sentence_transformers import SentenceTransformer + + # Load model (cache it in production) + transformer_model = SentenceTransformer(model) + + # Generate embeddings + embeddings = transformer_model.encode(texts, convert_to_tensor=False) + + return embeddings.tolist() + + except ImportError: + raise EmbeddingServiceError("sentence-transformers library not installed") + except Exception as e: + raise EmbeddingServiceError(f"HuggingFace embedding generation failed: {str(e)}") + + async def _generate_cohere_embeddings( + self, + texts: List[str], + model: Optional[str] = None + ) -> List[List[float]]: + """Generate embeddings using Cohere API.""" + if not self.settings.COHERE_API_KEY: + raise EmbeddingServiceError("Cohere API key not configured") + + model = model or self.settings.COHERE_MODEL + + try: + response = await self.client.post( + "https://api.cohere.ai/v1/embed", + headers={ + "Authorization": f"Bearer {self.settings.COHERE_API_KEY}", + "Content-Type": "application/json", + }, + json={ + "texts": texts, + "model": model, + }, + ) + + if response.status_code != 200: + error_data = response.json() + raise EmbeddingServiceError( + f"Cohere API error: {error_data.get('message', 'Unknown error')}" + ) + + data = response.json() + return data["embeddings"] + + except httpx.RequestError as e: + raise EmbeddingServiceError(f"Cohere API request failed: {str(e)}") + except Exception as e: + raise EmbeddingServiceError(f"Cohere embedding generation failed: {str(e)}") + + async def get_embedding_dimensions(self, provider: str = "openai", model: Optional[str] = None) -> int: + """Get the dimensions of embeddings for a given provider/model.""" + if provider == "openai": + model = model or self.settings.OPENAI_EMBEDDING_MODEL + if "ada-002" in model: + return 1536 + elif "3-small" in model: + return 1536 + elif "3-large" in model: + return 3072 + else: + return 1536 # Default + elif provider == "huggingface": + model = model or self.settings.HUGGINGFACE_MODEL + if "all-MiniLM-L6-v2" in model: + return 384 + elif "all-mpnet-base-v2" in model: + return 768 + else: + return 384 # Default + elif provider == "cohere": + return 4096 # Cohere embed-english-v2.0 + else: + raise EmbeddingServiceError(f"Unknown provider: {provider}") + + async def close(self): + """Close the HTTP client.""" + await self.client.aclose() \ No newline at end of file diff --git a/packages/api/app/services/llm_service.py b/packages/api/app/services/llm_service.py new file mode 100644 index 0000000..4cdb2a5 --- /dev/null +++ b/packages/api/app/services/llm_service.py @@ -0,0 +1,338 @@ +""" +LLM service for generating enhanced answers and content. +""" + +import asyncio +import httpx +from typing import Dict, Any, Optional, List +import json + +from app.core.config import get_settings +from app.core.exceptions import LLMServiceError + + +class LLMService: + """Service for LLM-powered content generation.""" + + def __init__(self): + self.settings = get_settings() + self.client = httpx.AsyncClient(timeout=60.0) + + async def generate_answer( + self, + query: str, + context: str, + max_length: int = 500, + provider: str = "openai", + model: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Generate an enhanced answer based on query and context.""" + try: + if provider == "openai": + return await self._generate_openai_answer(query, context, max_length, model) + elif provider == "anthropic": + return await self._generate_anthropic_answer(query, context, max_length, model) + else: + raise LLMServiceError(f"Unsupported LLM provider: {provider}") + except Exception as e: + raise LLMServiceError(f"Answer generation failed: {str(e)}") + + async def _generate_openai_answer( + self, + query: str, + context: str, + max_length: int, + model: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Generate answer using OpenAI API.""" + if not self.settings.OPENAI_API_KEY: + raise LLMServiceError("OpenAI API key not configured") + + model = model or self.settings.OPENAI_MODEL + + # Construct the prompt + prompt = self._build_answer_prompt(query, context, max_length) + + try: + response = await self.client.post( + "https://api.openai.com/v1/chat/completions", + headers={ + "Authorization": f"Bearer {self.settings.OPENAI_API_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": model, + "messages": [ + { + "role": "system", + "content": "You are a helpful AI assistant that provides accurate, concise answers based on the given context. Always cite your sources and be honest about limitations." + }, + { + "role": "user", + "content": prompt + } + ], + "max_tokens": max_length, + "temperature": 0.3, + "top_p": 0.9, + }, + ) + + if response.status_code != 200: + error_data = response.json() + raise LLMServiceError( + f"OpenAI API error: {error_data.get('error', {}).get('message', 'Unknown error')}" + ) + + data = response.json() + + if not data.get("choices"): + return None + + content = data["choices"][0]["message"]["content"].strip() + + return { + "content": content, + "confidence": 0.8, # Could be calculated based on various factors + "model": model, + "provider": "openai", + "usage": data.get("usage", {}), + } + + except httpx.RequestError as e: + raise LLMServiceError(f"OpenAI API request failed: {str(e)}") + + async def _generate_anthropic_answer( + self, + query: str, + context: str, + max_length: int, + model: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Generate answer using Anthropic API.""" + if not self.settings.ANTHROPIC_API_KEY: + raise LLMServiceError("Anthropic API key not configured") + + model = model or self.settings.ANTHROPIC_MODEL + + # Construct the prompt + prompt = self._build_answer_prompt(query, context, max_length) + + try: + response = await self.client.post( + "https://api.anthropic.com/v1/messages", + headers={ + "x-api-key": self.settings.ANTHROPIC_API_KEY, + "Content-Type": "application/json", + "anthropic-version": "2023-06-01", + }, + json={ + "model": model, + "max_tokens": max_length, + "temperature": 0.3, + "messages": [ + { + "role": "user", + "content": prompt + } + ], + }, + ) + + if response.status_code != 200: + error_data = response.json() + raise LLMServiceError( + f"Anthropic API error: {error_data.get('error', {}).get('message', 'Unknown error')}" + ) + + data = response.json() + + if not data.get("content"): + return None + + content = data["content"][0]["text"].strip() + + return { + "content": content, + "confidence": 0.8, + "model": model, + "provider": "anthropic", + "usage": data.get("usage", {}), + } + + except httpx.RequestError as e: + raise LLMServiceError(f"Anthropic API request failed: {str(e)}") + + def _build_answer_prompt(self, query: str, context: str, max_length: int) -> str: + """Build a prompt for answer generation.""" + return f"""Based on the following context, please provide a comprehensive and accurate answer to the user's question. + +Context: +{context} + +Question: {query} + +Instructions: +1. Provide a clear, concise answer based solely on the information in the context +2. If the context doesn't contain enough information to fully answer the question, say so +3. Keep your response under {max_length} characters +4. Use a professional but friendly tone +5. Structure your answer with clear paragraphs if needed + +Answer:""" + + async def generate_summary( + self, + content: str, + max_length: int = 200, + provider: str = "openai", + ) -> Optional[str]: + """Generate a summary of the given content.""" + try: + prompt = f"""Please provide a concise summary of the following content in no more than {max_length} characters: + +{content} + +Summary:""" + + if provider == "openai": + result = await self._generate_openai_completion(prompt, max_length) + elif provider == "anthropic": + result = await self._generate_anthropic_completion(prompt, max_length) + else: + raise LLMServiceError(f"Unsupported provider: {provider}") + + return result.get("content") if result else None + + except Exception as e: + raise LLMServiceError(f"Summary generation failed: {str(e)}") + + async def _generate_openai_completion( + self, + prompt: str, + max_tokens: int, + ) -> Optional[Dict[str, Any]]: + """Generate a completion using OpenAI API.""" + try: + response = await self.client.post( + "https://api.openai.com/v1/chat/completions", + headers={ + "Authorization": f"Bearer {self.settings.OPENAI_API_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": self.settings.OPENAI_MODEL, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": 0.3, + }, + ) + + if response.status_code == 200: + data = response.json() + if data.get("choices"): + return { + "content": data["choices"][0]["message"]["content"].strip(), + "usage": data.get("usage", {}), + } + + return None + + except Exception: + return None + + async def _generate_anthropic_completion( + self, + prompt: str, + max_tokens: int, + ) -> Optional[Dict[str, Any]]: + """Generate a completion using Anthropic API.""" + try: + response = await self.client.post( + "https://api.anthropic.com/v1/messages", + headers={ + "x-api-key": self.settings.ANTHROPIC_API_KEY, + "Content-Type": "application/json", + "anthropic-version": "2023-06-01", + }, + json={ + "model": self.settings.ANTHROPIC_MODEL, + "max_tokens": max_tokens, + "temperature": 0.3, + "messages": [{"role": "user", "content": prompt}], + }, + ) + + if response.status_code == 200: + data = response.json() + if data.get("content"): + return { + "content": data["content"][0]["text"].strip(), + "usage": data.get("usage", {}), + } + + return None + + except Exception: + return None + + async def validate_content( + self, + content: str, + guidelines: Optional[List[str]] = None, + ) -> Dict[str, Any]: + """Validate content against guidelines and detect potential issues.""" + try: + # Basic content validation + issues = [] + + # Check for potential inappropriate content (basic keywords) + inappropriate_keywords = [ + "spam", "scam", "fraud", "illegal", "hate", "violence" + ] + + content_lower = content.lower() + for keyword in inappropriate_keywords: + if keyword in content_lower: + issues.append({ + "type": "inappropriate_content", + "severity": "medium", + "message": f"Potentially inappropriate content detected: {keyword}", + }) + + # Check content length + if len(content) < 10: + issues.append({ + "type": "content_too_short", + "severity": "low", + "message": "Content appears to be too short to be meaningful", + }) + + # Check for excessive repetition + words = content.split() + if len(words) > 10: + unique_words = set(words) + repetition_ratio = len(unique_words) / len(words) + if repetition_ratio < 0.3: + issues.append({ + "type": "excessive_repetition", + "severity": "medium", + "message": "Content contains excessive repetition", + }) + + return { + "is_valid": len([i for i in issues if i["severity"] == "high"]) == 0, + "issues": issues, + "confidence": 0.7, # Basic validation confidence + } + + except Exception as e: + return { + "is_valid": True, # Default to valid if validation fails + "issues": [{"type": "validation_error", "severity": "low", "message": str(e)}], + "confidence": 0.1, + } + + async def close(self): + """Close the HTTP client.""" + await self.client.aclose() \ No newline at end of file diff --git a/packages/api/app/services/search_service.py b/packages/api/app/services/search_service.py new file mode 100644 index 0000000..d04ad6a --- /dev/null +++ b/packages/api/app/services/search_service.py @@ -0,0 +1,458 @@ +""" +Search service for semantic and keyword search functionality. +""" + +import asyncio +from typing import List, Dict, Any, Optional, Tuple +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select, func, text +import time + +from app.models.document import Document, DocumentChunk +from app.services.embedding_service import EmbeddingService +from app.services.vector_service import VectorService +from app.services.llm_service import LLMService +from app.core.exceptions import ValidationError, ServiceUnavailableError + + +class SearchService: + """Service for performing semantic and keyword searches.""" + + def __init__(self, db: AsyncSession): + self.db = db + self.embedding_service = EmbeddingService() + self.vector_service = VectorService() + self.llm_service = LLMService() + + async def search( + self, + query: str, + client_id: str, + filters: Optional[Dict[str, Any]] = None, + options: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """ + Perform comprehensive search across documents. + + Supports semantic, keyword, and hybrid search modes. + """ + start_time = time.time() + + # Parse options + search_options = options or {} + search_type = search_options.get("search_type", "semantic") + limit = search_options.get("limit", 10) + offset = search_options.get("offset", 0) + min_score = search_options.get("min_score", 0.1) + enhance_with_llm = search_options.get("enhance_with_llm", False) + + # Validate search type + if search_type not in ["semantic", "keyword", "hybrid"]: + raise ValidationError("Invalid search_type. Must be 'semantic', 'keyword', or 'hybrid'") + + results = [] + total_results = 0 + + if search_type == "semantic": + results, total_results = await self._semantic_search( + query, client_id, filters, limit, offset, min_score + ) + elif search_type == "keyword": + results, total_results = await self._keyword_search( + query, client_id, filters, limit, offset + ) + elif search_type == "hybrid": + results, total_results = await self._hybrid_search( + query, client_id, filters, limit, offset, min_score + ) + + processing_time = (time.time() - start_time) * 1000 + + # Generate enhanced answer if requested + enhanced_answer = None + if enhance_with_llm and results: + try: + enhanced_answer = await self._generate_enhanced_answer(query, results[:5]) + except Exception as e: + # Don't fail the entire search if LLM enhancement fails + pass + + # Generate suggestions + suggestions = await self._generate_suggestions(query, client_id) + + return { + "results": results, + "total": total_results, + "processing_time": processing_time, + "enhanced_answer": enhanced_answer, + "suggestions": suggestions, + } + + async def _semantic_search( + self, + query: str, + client_id: str, + filters: Optional[Dict[str, Any]], + limit: int, + offset: int, + min_score: float, + ) -> Tuple[List[Dict[str, Any]], int]: + """Perform semantic search using vector embeddings.""" + try: + # Generate query embedding + query_embeddings = await self.embedding_service.generate_embeddings([query]) + query_vector = query_embeddings[0] + + # Search vector database + vector_results = await self.vector_service.search_vectors( + query_vector=query_vector, + limit=limit + offset, # Get more to handle offset + filters=self._build_vector_filters(client_id, filters), + ) + + # Filter by minimum score and apply offset + filtered_results = [ + r for r in vector_results + if r["score"] >= min_score + ][offset:offset + limit] + + # Get document information + results = [] + for vector_result in filtered_results: + # Get document chunk info + chunk_query = select(DocumentChunk).where( + DocumentChunk.id == vector_result["id"] + ).join(Document).where( + Document.client_id == client_id, + Document.status == "indexed" + ) + + chunk_result = await self.db.execute(chunk_query) + chunk = chunk_result.scalar_one_or_none() + + if chunk and chunk.document: + results.append({ + "id": str(chunk.id), + "document_id": str(chunk.document_id), + "title": chunk.document.title, + "content": chunk.content, + "url": chunk.document.url, + "score": vector_result["score"], + "metadata": chunk.metadata, + "chunk_index": chunk.chunk_index, + }) + + return results, len(vector_results) + + except Exception as e: + raise ServiceUnavailableError(f"Semantic search failed: {str(e)}") + + async def _keyword_search( + self, + query: str, + client_id: str, + filters: Optional[Dict[str, Any]], + limit: int, + offset: int, + ) -> Tuple[List[Dict[str, Any]], int]: + """Perform keyword search using full-text search.""" + try: + # Build the base query + base_query = select(Document).where( + Document.client_id == client_id, + Document.status == "indexed" + ) + + # Add text search + search_query = base_query.where( + text("to_tsvector('english', title || ' ' || content) @@ plainto_tsquery('english', :query)") + ).params(query=query) + + # Apply filters + if filters: + search_query = self._apply_document_filters(search_query, filters) + + # Add ranking and ordering + ranked_query = search_query.add_columns( + text("ts_rank(to_tsvector('english', title || ' ' || content), plainto_tsquery('english', :query)) as rank") + ).params(query=query).order_by(text("rank DESC")) + + # Get total count + count_query = select(func.count()).select_from( + ranked_query.subquery() + ) + count_result = await self.db.execute(count_query) + total_results = count_result.scalar() + + # Apply pagination + paginated_query = ranked_query.offset(offset).limit(limit) + + # Execute search + search_result = await self.db.execute(paginated_query) + documents = search_result.fetchall() + + # Format results + results = [] + for doc, rank in documents: + results.append({ + "id": str(doc.id), + "document_id": str(doc.id), + "title": doc.title, + "content": doc.content[:1000], # Truncate for display + "url": doc.url, + "score": float(rank) if rank else 0.0, + "metadata": doc.metadata, + "chunk_index": 0, # Full document + }) + + return results, total_results + + except Exception as e: + raise ServiceUnavailableError(f"Keyword search failed: {str(e)}") + + async def _hybrid_search( + self, + query: str, + client_id: str, + filters: Optional[Dict[str, Any]], + limit: int, + offset: int, + min_score: float, + ) -> Tuple[List[Dict[str, Any]], int]: + """Perform hybrid search combining semantic and keyword search.""" + try: + # Perform both searches in parallel + semantic_task = asyncio.create_task( + self._semantic_search(query, client_id, filters, limit * 2, 0, min_score * 0.5) + ) + keyword_task = asyncio.create_task( + self._keyword_search(query, client_id, filters, limit * 2, 0) + ) + + semantic_results, semantic_total = await semantic_task + keyword_results, keyword_total = await keyword_task + + # Combine and re-rank results + combined_results = self._combine_search_results( + semantic_results, keyword_results, query + ) + + # Apply offset and limit + paginated_results = combined_results[offset:offset + limit] + + return paginated_results, len(combined_results) + + except Exception as e: + raise ServiceUnavailableError(f"Hybrid search failed: {str(e)}") + + def _combine_search_results( + self, + semantic_results: List[Dict[str, Any]], + keyword_results: List[Dict[str, Any]], + query: str, + ) -> List[Dict[str, Any]]: + """Combine and re-rank semantic and keyword search results.""" + # Create a mapping of document_id to results + result_map = {} + + # Add semantic results with higher weight + for result in semantic_results: + doc_id = result["document_id"] + result["combined_score"] = result["score"] * 0.7 # Semantic weight + result_map[doc_id] = result + + # Add keyword results, combining scores if document already exists + for result in keyword_results: + doc_id = result["document_id"] + keyword_score = result["score"] * 0.3 # Keyword weight + + if doc_id in result_map: + # Combine scores + result_map[doc_id]["combined_score"] += keyword_score + # Use the better content (longer usually means more context) + if len(result["content"]) > len(result_map[doc_id]["content"]): + result_map[doc_id]["content"] = result["content"] + else: + result["combined_score"] = keyword_score + result_map[doc_id] = result + + # Sort by combined score + combined_results = list(result_map.values()) + combined_results.sort(key=lambda x: x["combined_score"], reverse=True) + + # Update the score field to reflect combined score + for result in combined_results: + result["score"] = result["combined_score"] + del result["combined_score"] + + return combined_results + + async def _generate_enhanced_answer( + self, + query: str, + top_results: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + """Generate an enhanced answer using LLM.""" + try: + if not top_results: + return None + + # Prepare context from search results + context_pieces = [] + sources = [] + + for result in top_results: + context_pieces.append(f"Title: {result['title']}\nContent: {result['content']}") + sources.append({ + "id": result["document_id"], + "title": result["title"], + "url": result.get("url"), + }) + + context = "\n\n---\n\n".join(context_pieces) + + # Generate enhanced answer + answer = await self.llm_service.generate_answer( + query=query, + context=context, + max_length=500, + ) + + if answer: + return { + "content": answer["content"], + "sources": [s["id"] for s in sources], + "confidence": answer.get("confidence", 0.8), + } + + return None + + except Exception as e: + # Log error but don't fail the search + return None + + async def _generate_suggestions( + self, + query: str, + client_id: str, + limit: int = 5, + ) -> List[str]: + """Generate search suggestions based on query.""" + try: + # Get popular queries for this client + popular_queries = await self.get_popular_queries(client_id, limit * 2) + + # Simple similarity matching (in production, use more sophisticated methods) + query_words = set(query.lower().split()) + suggestions = [] + + for popular_query in popular_queries: + popular_words = set(popular_query.lower().split()) + # Calculate Jaccard similarity + intersection = len(query_words & popular_words) + union = len(query_words | popular_words) + similarity = intersection / union if union > 0 else 0 + + if similarity > 0.2 and popular_query.lower() != query.lower(): + suggestions.append(popular_query) + + return suggestions[:limit] + + except Exception: + return [] + + async def get_popular_queries( + self, + client_id: str, + limit: int = 10, + ) -> List[str]: + """Get popular search queries for a client.""" + try: + # This would typically query an analytics table + # For now, return some mock data + return [ + "getting started guide", + "API documentation", + "troubleshooting", + "best practices", + "configuration", + ][:limit] + + except Exception: + return [] + + async def get_autocomplete_suggestions( + self, + query: str, + client_id: str, + limit: int = 5, + ) -> List[Dict[str, Any]]: + """Get autocomplete suggestions for a partial query.""" + try: + suggestions = [] + + # Get document titles that match the query + title_query = select(Document.title).where( + Document.client_id == client_id, + Document.status == "indexed", + Document.title.ilike(f"%{query}%") + ).limit(limit) + + title_result = await self.db.execute(title_query) + titles = title_result.scalars().all() + + for title in titles: + suggestions.append({ + "text": title, + "score": 0.9, + "type": "document_title", + }) + + # Add popular queries if we need more suggestions + if len(suggestions) < limit: + popular = await self.get_popular_queries(client_id, limit - len(suggestions)) + for query_text in popular: + if query.lower() in query_text.lower(): + suggestions.append({ + "text": query_text, + "score": 0.7, + "type": "query", + }) + + return suggestions[:limit] + + except Exception: + return [] + + def _build_vector_filters( + self, + client_id: str, + filters: Optional[Dict[str, Any]], + ) -> Dict[str, Any]: + """Build filters for vector database queries.""" + vector_filters = {"client_id": client_id} + + if filters: + if "content_type" in filters: + vector_filters["content_type"] = {"$in": filters["content_type"]} + if "tags" in filters: + vector_filters["tags"] = {"$in": filters["tags"]} + if "language" in filters: + vector_filters["language"] = filters["language"] + if "source_id" in filters: + vector_filters["source_id"] = filters["source_id"] + + return vector_filters + + def _apply_document_filters(self, query, filters: Dict[str, Any]): + """Apply filters to a document query.""" + if "content_type" in filters: + query = query.where(Document.content_type.in_(filters["content_type"])) + if "tags" in filters: + query = query.where(Document.tags.overlap(filters["tags"])) + if "language" in filters: + query = query.where(Document.language == filters["language"]) + if "source_id" in filters: + query = query.where(Document.source_id == filters["source_id"]) + + return query \ No newline at end of file diff --git a/packages/api/app/services/vector_service.py b/packages/api/app/services/vector_service.py new file mode 100644 index 0000000..10d1d9d --- /dev/null +++ b/packages/api/app/services/vector_service.py @@ -0,0 +1,357 @@ +""" +Vector database service for storing and searching embeddings. +""" + +import asyncio +from typing import List, Dict, Any, Optional, Tuple +import numpy as np + +from app.core.config import get_settings +from app.core.exceptions import VectorDatabaseError + + +class VectorService: + """Service for vector database operations.""" + + def __init__(self): + self.settings = get_settings() + self.client = None + self.provider = self.settings.VECTOR_DB_PROVIDER + + async def initialize(self): + """Initialize the vector database connection.""" + if self.provider == "milvus": + await self._init_milvus() + elif self.provider == "pinecone": + await self._init_pinecone() + elif self.provider == "weaviate": + await self._init_weaviate() + elif self.provider == "qdrant": + await self._init_qdrant() + else: + raise VectorDatabaseError(f"Unsupported vector database: {self.provider}") + + async def _init_milvus(self): + """Initialize Milvus connection.""" + try: + from pymilvus import connections, Collection, CollectionSchema, FieldSchema, DataType + + # Connect to Milvus + connections.connect( + alias="default", + host=self.settings.MILVUS_HOST, + port=self.settings.MILVUS_PORT, + user=self.settings.MILVUS_USER, + password=self.settings.MILVUS_PASSWORD, + ) + + # Define collection schema + fields = [ + FieldSchema(name="id", dtype=DataType.VARCHAR, max_length=100, is_primary=True), + FieldSchema(name="document_id", dtype=DataType.VARCHAR, max_length=100), + FieldSchema(name="chunk_index", dtype=DataType.INT64), + FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=self.settings.VECTOR_DB_DIMENSIONS), + FieldSchema(name="content", dtype=DataType.VARCHAR, max_length=65535), + FieldSchema(name="metadata", dtype=DataType.JSON), + ] + + schema = CollectionSchema(fields=fields, description="Document chunks collection") + + # Create or get collection + collection_name = "document_chunks" + try: + self.collection = Collection(name=collection_name, schema=schema) + except Exception: + # Collection might already exist + self.collection = Collection(name=collection_name) + + # Create index + index_params = { + "metric_type": "COSINE", + "index_type": "IVF_FLAT", + "params": {"nlist": 1024} + } + + try: + self.collection.create_index(field_name="embedding", index_params=index_params) + except Exception: + pass # Index might already exist + + self.collection.load() + + except ImportError: + raise VectorDatabaseError("pymilvus library not installed") + except Exception as e: + raise VectorDatabaseError(f"Milvus initialization failed: {str(e)}") + + async def _init_pinecone(self): + """Initialize Pinecone connection.""" + try: + import pinecone + + pinecone.init( + api_key=self.settings.PINECONE_API_KEY, + environment=self.settings.PINECONE_ENVIRONMENT, + ) + + # Create index if it doesn't exist + index_name = self.settings.PINECONE_INDEX_NAME + + if index_name not in pinecone.list_indexes(): + pinecone.create_index( + name=index_name, + dimension=self.settings.VECTOR_DB_DIMENSIONS, + metric="cosine", + ) + + self.index = pinecone.Index(index_name) + + except ImportError: + raise VectorDatabaseError("pinecone-client library not installed") + except Exception as e: + raise VectorDatabaseError(f"Pinecone initialization failed: {str(e)}") + + async def _init_weaviate(self): + """Initialize Weaviate connection.""" + try: + import weaviate + + auth_config = None + if self.settings.WEAVIATE_API_KEY: + auth_config = weaviate.AuthApiKey(api_key=self.settings.WEAVIATE_API_KEY) + + self.client = weaviate.Client( + url=self.settings.WEAVIATE_URL, + auth_client_secret=auth_config, + ) + + # Create class schema if it doesn't exist + class_name = self.settings.WEAVIATE_CLASS_NAME + + if not self.client.schema.exists(class_name): + schema = { + "class": class_name, + "vectorizer": "none", # We provide our own vectors + "properties": [ + { + "name": "document_id", + "dataType": ["string"], + }, + { + "name": "chunk_index", + "dataType": ["int"], + }, + { + "name": "content", + "dataType": ["text"], + }, + { + "name": "metadata", + "dataType": ["object"], + }, + ], + } + + self.client.schema.create_class(schema) + + except ImportError: + raise VectorDatabaseError("weaviate-client library not installed") + except Exception as e: + raise VectorDatabaseError(f"Weaviate initialization failed: {str(e)}") + + async def _init_qdrant(self): + """Initialize Qdrant connection.""" + try: + from qdrant_client import QdrantClient + from qdrant_client.http import models + + self.client = QdrantClient( + url=self.settings.QDRANT_URL, + api_key=self.settings.QDRANT_API_KEY, + ) + + collection_name = self.settings.QDRANT_COLLECTION_NAME + + # Create collection if it doesn't exist + try: + self.client.create_collection( + collection_name=collection_name, + vectors_config=models.VectorParams( + size=self.settings.VECTOR_DB_DIMENSIONS, + distance=models.Distance.COSINE, + ), + ) + except Exception: + pass # Collection might already exist + + except ImportError: + raise VectorDatabaseError("qdrant-client library not installed") + except Exception as e: + raise VectorDatabaseError(f"Qdrant initialization failed: {str(e)}") + + async def insert_vectors( + self, + vectors: List[Dict[str, Any]], + ) -> bool: + """Insert vectors into the database.""" + try: + if self.provider == "milvus": + return await self._insert_milvus(vectors) + elif self.provider == "pinecone": + return await self._insert_pinecone(vectors) + elif self.provider == "weaviate": + return await self._insert_weaviate(vectors) + elif self.provider == "qdrant": + return await self._insert_qdrant(vectors) + else: + raise VectorDatabaseError(f"Unsupported provider: {self.provider}") + except Exception as e: + raise VectorDatabaseError(f"Vector insertion failed: {str(e)}") + + async def _insert_milvus(self, vectors: List[Dict[str, Any]]) -> bool: + """Insert vectors into Milvus.""" + data = [ + [v["id"] for v in vectors], + [v["document_id"] for v in vectors], + [v["chunk_index"] for v in vectors], + [v["embedding"] for v in vectors], + [v["content"] for v in vectors], + [v["metadata"] for v in vectors], + ] + + self.collection.insert(data) + self.collection.flush() + return True + + async def _insert_pinecone(self, vectors: List[Dict[str, Any]]) -> bool: + """Insert vectors into Pinecone.""" + upsert_data = [ + ( + v["id"], + v["embedding"], + { + "document_id": v["document_id"], + "chunk_index": v["chunk_index"], + "content": v["content"], + "metadata": v["metadata"], + } + ) + for v in vectors + ] + + self.index.upsert(vectors=upsert_data) + return True + + async def search_vectors( + self, + query_vector: List[float], + limit: int = 10, + filters: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + """Search for similar vectors.""" + try: + if self.provider == "milvus": + return await self._search_milvus(query_vector, limit, filters) + elif self.provider == "pinecone": + return await self._search_pinecone(query_vector, limit, filters) + elif self.provider == "weaviate": + return await self._search_weaviate(query_vector, limit, filters) + elif self.provider == "qdrant": + return await self._search_qdrant(query_vector, limit, filters) + else: + raise VectorDatabaseError(f"Unsupported provider: {self.provider}") + except Exception as e: + raise VectorDatabaseError(f"Vector search failed: {str(e)}") + + async def _search_milvus( + self, + query_vector: List[float], + limit: int, + filters: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + """Search vectors in Milvus.""" + search_params = {"metric_type": "COSINE", "params": {"nprobe": 10}} + + results = self.collection.search( + data=[query_vector], + anns_field="embedding", + param=search_params, + limit=limit, + output_fields=["id", "document_id", "chunk_index", "content", "metadata"], + ) + + formatted_results = [] + for hit in results[0]: + formatted_results.append({ + "id": hit.entity.get("id"), + "document_id": hit.entity.get("document_id"), + "chunk_index": hit.entity.get("chunk_index"), + "content": hit.entity.get("content"), + "metadata": hit.entity.get("metadata"), + "score": hit.score, + }) + + return formatted_results + + async def _search_pinecone( + self, + query_vector: List[float], + limit: int, + filters: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + """Search vectors in Pinecone.""" + results = self.index.query( + vector=query_vector, + top_k=limit, + include_metadata=True, + filter=filters, + ) + + formatted_results = [] + for match in results["matches"]: + formatted_results.append({ + "id": match["id"], + "document_id": match["metadata"]["document_id"], + "chunk_index": match["metadata"]["chunk_index"], + "content": match["metadata"]["content"], + "metadata": match["metadata"]["metadata"], + "score": match["score"], + }) + + return formatted_results + + async def delete_vectors(self, document_id: str) -> bool: + """Delete all vectors for a document.""" + try: + if self.provider == "milvus": + return await self._delete_milvus(document_id) + elif self.provider == "pinecone": + return await self._delete_pinecone(document_id) + elif self.provider == "weaviate": + return await self._delete_weaviate(document_id) + elif self.provider == "qdrant": + return await self._delete_qdrant(document_id) + else: + raise VectorDatabaseError(f"Unsupported provider: {self.provider}") + except Exception as e: + raise VectorDatabaseError(f"Vector deletion failed: {str(e)}") + + async def _delete_milvus(self, document_id: str) -> bool: + """Delete vectors from Milvus.""" + expr = f'document_id == "{document_id}"' + self.collection.delete(expr) + return True + + async def _delete_pinecone(self, document_id: str) -> bool: + """Delete vectors from Pinecone.""" + self.index.delete(filter={"document_id": document_id}) + return True + + async def close(self): + """Close database connections.""" + if self.provider == "milvus": + from pymilvus import connections + connections.disconnect("default") + elif hasattr(self, 'client') and self.client: + if hasattr(self.client, 'close'): + self.client.close() \ No newline at end of file diff --git a/packages/api/main.py b/packages/api/main.py new file mode 100644 index 0000000..384d6dd --- /dev/null +++ b/packages/api/main.py @@ -0,0 +1,250 @@ +""" +AI Search Platform API +Production-grade FastAPI application for AI-powered search and content discovery. +""" + +import logging +import time +from contextlib import asynccontextmanager +from typing import AsyncGenerator + +import structlog +from fastapi import FastAPI, Request, Response +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware +from fastapi.responses import JSONResponse +from prometheus_client import Counter, Histogram, generate_latest +from starlette.middleware.base import BaseHTTPMiddleware + +from app.api.routes import api_router +from app.core.config import get_settings +from app.core.database import init_db +from app.core.exceptions import APIException +from app.services.vector_service import VectorService +from app.services.embedding_service import EmbeddingService +from app.services.llm_service import LLMService + +# Configure structured logging +structlog.configure( + processors=[ + structlog.stdlib.filter_by_level, + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.TimeStamper(fmt="iso"), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.processors.UnicodeDecoder(), + structlog.processors.JSONRenderer(), + ], + context_class=dict, + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=True, +) + +logger = structlog.get_logger() + +# Metrics +REQUEST_COUNT = Counter( + "http_requests_total", "Total HTTP requests", ["method", "endpoint", "status"] +) +REQUEST_DURATION = Histogram( + "http_request_duration_seconds", "HTTP request duration", ["method", "endpoint"] +) +SEARCH_COUNT = Counter("search_requests_total", "Total search requests") +EMBEDDING_COUNT = Counter("embedding_requests_total", "Total embedding requests") + + +class MetricsMiddleware(BaseHTTPMiddleware): + """Middleware to collect Prometheus metrics.""" + + async def dispatch(self, request: Request, call_next): + start_time = time.time() + + response = await call_next(request) + + duration = time.time() - start_time + endpoint = request.url.path + method = request.method + status = str(response.status_code) + + REQUEST_COUNT.labels(method=method, endpoint=endpoint, status=status).inc() + REQUEST_DURATION.labels(method=method, endpoint=endpoint).observe(duration) + + return response + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: + """Application lifespan events.""" + settings = get_settings() + + logger.info("Starting AI Search Platform API", version="1.0.0") + + # Initialize database + await init_db() + + # Initialize services + try: + # Initialize vector service + vector_service = VectorService() + await vector_service.initialize() + app.state.vector_service = vector_service + + # Initialize embedding service + embedding_service = EmbeddingService() + app.state.embedding_service = embedding_service + + # Initialize LLM service + llm_service = LLMService() + app.state.llm_service = llm_service + + logger.info("All services initialized successfully") + + except Exception as e: + logger.error("Failed to initialize services", error=str(e)) + raise + + yield + + # Cleanup + logger.info("Shutting down AI Search Platform API") + + # Close vector service connections + if hasattr(app.state, "vector_service"): + await app.state.vector_service.close() + + +# Create FastAPI app +app = FastAPI( + title="AI Search Platform API", + description="Production-grade AI-powered website search and content discovery platform", + version="1.0.0", + docs_url="/docs" if get_settings().DEBUG else None, + redoc_url="/redoc" if get_settings().DEBUG else None, + lifespan=lifespan, +) + +# Add middleware +app.add_middleware( + CORSMiddleware, + allow_origins=get_settings().ALLOWED_ORIGINS, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.add_middleware(GZipMiddleware, minimum_size=1000) +app.add_middleware(MetricsMiddleware) + + +# Exception handlers +@app.exception_handler(APIException) +async def api_exception_handler(request: Request, exc: APIException) -> JSONResponse: + """Handle custom API exceptions.""" + logger.error( + "API exception occurred", + error_code=exc.error_code, + message=exc.message, + details=exc.details, + path=request.url.path, + ) + + return JSONResponse( + status_code=exc.status_code, + content={ + "success": False, + "error": { + "code": exc.error_code, + "message": exc.message, + "details": exc.details, + }, + "metadata": { + "requestId": getattr(request.state, "request_id", None), + "timestamp": time.time(), + "version": "1.0.0", + }, + }, + ) + + +@app.exception_handler(Exception) +async def general_exception_handler(request: Request, exc: Exception) -> JSONResponse: + """Handle unexpected exceptions.""" + logger.error( + "Unexpected exception occurred", + error=str(exc), + path=request.url.path, + exc_info=True, + ) + + return JSONResponse( + status_code=500, + content={ + "success": False, + "error": { + "code": "INTERNAL_SERVER_ERROR", + "message": "An unexpected error occurred", + "details": None, + }, + "metadata": { + "requestId": getattr(request.state, "request_id", None), + "timestamp": time.time(), + "version": "1.0.0", + }, + }, + ) + + +# Health check endpoints +@app.get("/health") +async def health_check(): + """Basic health check endpoint.""" + return {"status": "healthy", "timestamp": time.time()} + + +@app.get("/health/ready") +async def readiness_check(): + """Readiness check endpoint.""" + # Check if all services are ready + checks = { + "database": True, # Add actual database check + "vector_service": hasattr(app.state, "vector_service"), + "embedding_service": hasattr(app.state, "embedding_service"), + "llm_service": hasattr(app.state, "llm_service"), + } + + all_ready = all(checks.values()) + + return { + "status": "ready" if all_ready else "not_ready", + "checks": checks, + "timestamp": time.time(), + } + + +@app.get("/metrics") +async def metrics(): + """Prometheus metrics endpoint.""" + return Response( + generate_latest(), + media_type="text/plain; version=0.0.4; charset=utf-8", + ) + + +# Include API routes +app.include_router(api_router, prefix="/api/v1") + + +if __name__ == "__main__": + import uvicorn + + settings = get_settings() + uvicorn.run( + "main:app", + host="0.0.0.0", + port=8000, + reload=settings.DEBUG, + log_config=None, # Use structlog instead + ) \ No newline at end of file diff --git a/packages/api/package.json b/packages/api/package.json new file mode 100644 index 0000000..b6cdfc1 --- /dev/null +++ b/packages/api/package.json @@ -0,0 +1,17 @@ +{ + "name": "@ai-search/api", + "version": "1.0.0", + "description": "FastAPI backend for AI search platform", + "scripts": { + "dev": "python -m uvicorn main:app --reload --host 0.0.0.0 --port 8000", + "start": "python -m uvicorn main:app --host 0.0.0.0 --port 8000", + "test": "python -m pytest tests/ -v", + "lint": "python -m flake8 . && python -m black --check .", + "format": "python -m black . && python -m isort .", + "type-check": "python -m mypy .", + "clean": "find . -type d -name __pycache__ -exec rm -rf {} + || true" + }, + "dependencies": { + "@ai-search/shared": "workspace:*" + } +} \ No newline at end of file diff --git a/packages/api/pyproject.toml b/packages/api/pyproject.toml new file mode 100644 index 0000000..f2662a2 --- /dev/null +++ b/packages/api/pyproject.toml @@ -0,0 +1,47 @@ +[tool.black] +line-length = 88 +target-version = ['py311'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +multi_line_output = 3 +line_length = 88 +known_first_party = ["app"] + +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra -q --strict-markers" +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +asyncio_mode = "auto" \ No newline at end of file diff --git a/packages/api/requirements.txt b/packages/api/requirements.txt new file mode 100644 index 0000000..edf3665 --- /dev/null +++ b/packages/api/requirements.txt @@ -0,0 +1,55 @@ +# FastAPI and web framework +fastapi==0.104.1 +uvicorn[standard]==0.24.0 +python-multipart==0.0.6 +python-jose[cryptography]==3.3.0 +passlib[bcrypt]==1.7.4 + +# Database +sqlalchemy==2.0.23 +alembic==1.12.1 +psycopg2-binary==2.9.9 +redis==5.0.1 + +# AI/ML +openai==1.3.5 +anthropic==0.7.7 +transformers==4.35.2 +torch==2.1.1 +sentence-transformers==2.2.2 +numpy==1.24.4 + +# Vector databases +pymilvus==2.3.4 +pinecone-client==2.2.4 +weaviate-client==3.25.3 +qdrant-client==1.7.0 + +# Text processing +beautifulsoup4==4.12.2 +pypdf==3.17.1 +python-docx==1.1.0 +markdown==3.5.1 +html2text==2020.1.16 + +# Utilities +pydantic==2.5.0 +pydantic-settings==2.1.0 +httpx==0.25.2 +aiofiles==23.2.1 +celery==5.3.4 +python-dotenv==1.0.0 + +# Monitoring and logging +structlog==23.2.0 +sentry-sdk[fastapi]==1.38.0 +prometheus-client==0.19.0 + +# Development +pytest==7.4.3 +pytest-asyncio==0.21.1 +pytest-cov==4.1.0 +black==23.11.0 +flake8==6.1.0 +mypy==1.7.1 +isort==5.12.0 \ No newline at end of file diff --git a/packages/shared/package.json b/packages/shared/package.json new file mode 100644 index 0000000..d645428 --- /dev/null +++ b/packages/shared/package.json @@ -0,0 +1,24 @@ +{ + "name": "@ai-search/shared", + "version": "1.0.0", + "description": "Shared types, utilities, and configurations", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "clean": "rm -rf dist", + "type-check": "tsc --noEmit", + "lint": "eslint src --ext .ts,.tsx" + }, + "dependencies": { + "zod": "^3.22.4" + }, + "devDependencies": { + "typescript": "^5.2.2", + "@types/node": "^20.8.0", + "eslint": "^8.51.0", + "@typescript-eslint/eslint-plugin": "^6.7.4", + "@typescript-eslint/parser": "^6.7.4" + } +} \ No newline at end of file diff --git a/packages/shared/src/config/constants.ts b/packages/shared/src/config/constants.ts new file mode 100644 index 0000000..bb23333 --- /dev/null +++ b/packages/shared/src/config/constants.ts @@ -0,0 +1,229 @@ +/** + * Application constants and configuration + */ + +// API Configuration +export const API_CONFIG = { + DEFAULT_TIMEOUT: 30000, + MAX_RETRIES: 3, + RETRY_DELAY: 1000, + MAX_REQUEST_SIZE: 10 * 1024 * 1024, // 10MB + DEFAULT_PAGE_SIZE: 20, + MAX_PAGE_SIZE: 100, +} as const; + +// Search Configuration +export const SEARCH_CONFIG = { + DEFAULT_LIMIT: 10, + MAX_LIMIT: 100, + MIN_QUERY_LENGTH: 1, + MAX_QUERY_LENGTH: 1000, + DEFAULT_MIN_SCORE: 0.1, + MAX_RESULTS_FOR_ENHANCEMENT: 5, + CHUNK_SIZE: 1000, + CHUNK_OVERLAP: 200, +} as const; + +// Embedding Configuration +export const EMBEDDING_CONFIG = { + OPENAI: { + MODEL: 'text-embedding-ada-002', + DIMENSIONS: 1536, + MAX_TOKENS: 8191, + BATCH_SIZE: 100, + }, + HUGGINGFACE: { + MODEL: 'sentence-transformers/all-MiniLM-L6-v2', + DIMENSIONS: 384, + MAX_TOKENS: 512, + BATCH_SIZE: 32, + }, + COHERE: { + MODEL: 'embed-english-v2.0', + DIMENSIONS: 4096, + MAX_TOKENS: 512, + BATCH_SIZE: 96, + }, +} as const; + +// LLM Configuration +export const LLM_CONFIG = { + OPENAI: { + MODELS: { + GPT_4: 'gpt-4', + GPT_4_TURBO: 'gpt-4-turbo-preview', + GPT_3_5_TURBO: 'gpt-3.5-turbo', + }, + MAX_TOKENS: 4096, + DEFAULT_TEMPERATURE: 0.7, + }, + ANTHROPIC: { + MODELS: { + CLAUDE_3_OPUS: 'claude-3-opus-20240229', + CLAUDE_3_SONNET: 'claude-3-sonnet-20240229', + CLAUDE_3_HAIKU: 'claude-3-haiku-20240307', + }, + MAX_TOKENS: 4096, + DEFAULT_TEMPERATURE: 0.7, + }, +} as const; + +// Vector Database Configuration +export const VECTOR_DB_CONFIG = { + MILVUS: { + DEFAULT_COLLECTION: 'documents', + INDEX_TYPE: 'IVF_FLAT', + METRIC_TYPE: 'COSINE', + NLIST: 1024, + }, + PINECONE: { + DEFAULT_NAMESPACE: 'default', + METRIC: 'cosine', + PODS: 1, + }, + WEAVIATE: { + DEFAULT_CLASS: 'Document', + DISTANCE_METRIC: 'cosine', + }, +} as const; + +// Widget Configuration +export const WIDGET_CONFIG = { + DEFAULT_THEME: { + PRIMARY_COLOR: '#007bff', + SECONDARY_COLOR: '#6c757d', + BACKGROUND_COLOR: '#ffffff', + TEXT_COLOR: '#212529', + BORDER_RADIUS: 8, + FONT_FAMILY: 'system-ui, -apple-system, sans-serif', + FONT_SIZE: 14, + }, + DEFAULT_LAYOUT: { + POSITION: 'bottom-right', + WIDTH: 400, + HEIGHT: 500, + Z_INDEX: 9999, + OFFSET: { X: 20, Y: 20 }, + }, + RATE_LIMITS: { + SEARCHES_PER_MINUTE: 60, + SEARCHES_PER_HOUR: 1000, + SEARCHES_PER_DAY: 10000, + }, +} as const; + +// Analytics Configuration +export const ANALYTICS_CONFIG = { + BATCH_SIZE: 100, + FLUSH_INTERVAL: 5000, // 5 seconds + MAX_QUEUE_SIZE: 1000, + RETENTION_DAYS: 90, + AGGREGATION_INTERVALS: ['hour', 'day', 'week', 'month'] as const, +} as const; + +// Content Processing +export const CONTENT_CONFIG = { + MAX_DOCUMENT_SIZE: 50 * 1024 * 1024, // 50MB + SUPPORTED_FORMATS: ['text', 'html', 'markdown', 'pdf', 'doc', 'docx'] as const, + MAX_CHUNKS_PER_DOCUMENT: 1000, + MIN_CHUNK_SIZE: 100, + MAX_CHUNK_SIZE: 2000, +} as const; + +// Security Configuration +export const SECURITY_CONFIG = { + JWT_EXPIRY: '24h', + API_KEY_LENGTH: 32, + MAX_LOGIN_ATTEMPTS: 5, + LOCKOUT_DURATION: 15 * 60 * 1000, // 15 minutes + BCRYPT_ROUNDS: 12, + CORS_MAX_AGE: 86400, // 24 hours +} as const; + +// Error Codes +export const ERROR_CODES = { + // Authentication + INVALID_API_KEY: 'INVALID_API_KEY', + EXPIRED_TOKEN: 'EXPIRED_TOKEN', + INSUFFICIENT_PERMISSIONS: 'INSUFFICIENT_PERMISSIONS', + + // Validation + INVALID_INPUT: 'INVALID_INPUT', + MISSING_REQUIRED_FIELD: 'MISSING_REQUIRED_FIELD', + INVALID_FORMAT: 'INVALID_FORMAT', + + // Search + QUERY_TOO_SHORT: 'QUERY_TOO_SHORT', + QUERY_TOO_LONG: 'QUERY_TOO_LONG', + NO_RESULTS_FOUND: 'NO_RESULTS_FOUND', + SEARCH_TIMEOUT: 'SEARCH_TIMEOUT', + + // Content + DOCUMENT_NOT_FOUND: 'DOCUMENT_NOT_FOUND', + DOCUMENT_TOO_LARGE: 'DOCUMENT_TOO_LARGE', + UNSUPPORTED_FORMAT: 'UNSUPPORTED_FORMAT', + INDEXING_FAILED: 'INDEXING_FAILED', + + // Rate Limiting + RATE_LIMIT_EXCEEDED: 'RATE_LIMIT_EXCEEDED', + QUOTA_EXCEEDED: 'QUOTA_EXCEEDED', + + // External Services + EMBEDDING_SERVICE_ERROR: 'EMBEDDING_SERVICE_ERROR', + LLM_SERVICE_ERROR: 'LLM_SERVICE_ERROR', + VECTOR_DB_ERROR: 'VECTOR_DB_ERROR', + + // System + INTERNAL_SERVER_ERROR: 'INTERNAL_SERVER_ERROR', + SERVICE_UNAVAILABLE: 'SERVICE_UNAVAILABLE', + TIMEOUT: 'TIMEOUT', +} as const; + +// HTTP Status Codes +export const HTTP_STATUS = { + OK: 200, + CREATED: 201, + NO_CONTENT: 204, + BAD_REQUEST: 400, + UNAUTHORIZED: 401, + FORBIDDEN: 403, + NOT_FOUND: 404, + METHOD_NOT_ALLOWED: 405, + CONFLICT: 409, + UNPROCESSABLE_ENTITY: 422, + TOO_MANY_REQUESTS: 429, + INTERNAL_SERVER_ERROR: 500, + BAD_GATEWAY: 502, + SERVICE_UNAVAILABLE: 503, + GATEWAY_TIMEOUT: 504, +} as const; + +// Environment Variables +export const ENV_VARS = { + NODE_ENV: 'NODE_ENV', + PORT: 'PORT', + DATABASE_URL: 'DATABASE_URL', + REDIS_URL: 'REDIS_URL', + + // API Keys + OPENAI_API_KEY: 'OPENAI_API_KEY', + ANTHROPIC_API_KEY: 'ANTHROPIC_API_KEY', + HUGGINGFACE_API_KEY: 'HUGGINGFACE_API_KEY', + COHERE_API_KEY: 'COHERE_API_KEY', + + // Vector Databases + PINECONE_API_KEY: 'PINECONE_API_KEY', + PINECONE_ENVIRONMENT: 'PINECONE_ENVIRONMENT', + MILVUS_HOST: 'MILVUS_HOST', + MILVUS_PORT: 'MILVUS_PORT', + WEAVIATE_URL: 'WEAVIATE_URL', + WEAVIATE_API_KEY: 'WEAVIATE_API_KEY', + + // Security + JWT_SECRET: 'JWT_SECRET', + ENCRYPTION_KEY: 'ENCRYPTION_KEY', + + // Monitoring + SENTRY_DSN: 'SENTRY_DSN', + LOG_LEVEL: 'LOG_LEVEL', +} as const; \ No newline at end of file diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts new file mode 100644 index 0000000..f8fe567 --- /dev/null +++ b/packages/shared/src/index.ts @@ -0,0 +1,9 @@ +// Export all types +export * from './types'; + +// Export utilities +export * from './utils/validation'; +export * from './utils/text-processing'; + +// Export configuration +export * from './config/constants'; \ No newline at end of file diff --git a/packages/shared/src/types/analytics.ts b/packages/shared/src/types/analytics.ts new file mode 100644 index 0000000..6fddc7a --- /dev/null +++ b/packages/shared/src/types/analytics.ts @@ -0,0 +1,114 @@ +import { z } from 'zod'; + +// Analytics event base schema +export const AnalyticsEventSchema = z.object({ + id: z.string().uuid(), + sessionId: z.string(), + userId: z.string().optional(), + timestamp: z.date(), + eventType: z.enum([ + 'search_query', + 'search_result_click', + 'widget_load', + 'widget_interaction', + 'document_view', + 'feedback_positive', + 'feedback_negative', + 'auto_complete', + 'enhanced_answer_view', + 'source_click' + ]), + data: z.record(z.any()), + metadata: z.object({ + userAgent: z.string().optional(), + referrer: z.string().optional(), + ip: z.string().optional(), + country: z.string().optional(), + device: z.string().optional(), + widgetVersion: z.string().optional(), + }).optional(), +}); + +// Specific event schemas +export const SearchQueryEventSchema = AnalyticsEventSchema.extend({ + eventType: z.literal('search_query'), + data: z.object({ + query: z.string(), + searchType: z.enum(['semantic', 'keyword', 'hybrid']), + resultsCount: z.number().int().min(0), + processingTime: z.number().min(0), + filters: z.record(z.any()).optional(), + enhancedAnswer: z.boolean().optional(), + }), +}); + +export const SearchResultClickEventSchema = AnalyticsEventSchema.extend({ + eventType: z.literal('search_result_click'), + data: z.object({ + query: z.string(), + documentId: z.string().uuid(), + resultPosition: z.number().int().min(0), + score: z.number().min(0).max(1), + url: z.string().url().optional(), + }), +}); + +export const FeedbackEventSchema = AnalyticsEventSchema.extend({ + eventType: z.enum(['feedback_positive', 'feedback_negative']), + data: z.object({ + query: z.string().optional(), + documentId: z.string().uuid().optional(), + enhancedAnswerId: z.string().uuid().optional(), + comment: z.string().optional(), + rating: z.number().int().min(1).max(5).optional(), + }), +}); + +// Analytics aggregation schemas +export const SearchAnalyticsSchema = z.object({ + period: z.enum(['hour', 'day', 'week', 'month']), + startDate: z.date(), + endDate: z.date(), + metrics: z.object({ + totalSearches: z.number().int().min(0), + uniqueUsers: z.number().int().min(0), + avgResultsPerQuery: z.number().min(0), + avgProcessingTime: z.number().min(0), + clickThroughRate: z.number().min(0).max(1), + zeroResultsRate: z.number().min(0).max(1), + enhancedAnswerUsage: z.number().min(0).max(1), + }), + topQueries: z.array(z.object({ + query: z.string(), + count: z.number().int().min(0), + avgScore: z.number().min(0).max(1), + })), + topDocuments: z.array(z.object({ + documentId: z.string().uuid(), + title: z.string(), + clicks: z.number().int().min(0), + avgPosition: z.number().min(0), + })), +}); + +// User engagement metrics +export const UserEngagementSchema = z.object({ + userId: z.string(), + sessionId: z.string(), + startTime: z.date(), + endTime: z.date().optional(), + totalQueries: z.number().int().min(0), + totalClicks: z.number().int().min(0), + avgQueryLength: z.number().min(0), + sessionDuration: z.number().min(0), // in seconds + bounceRate: z.number().min(0).max(1), + conversionEvents: z.array(z.string()), +}); + +// Export types +export type AnalyticsEvent = z.infer; +export type SearchQueryEvent = z.infer; +export type SearchResultClickEvent = z.infer; +export type FeedbackEvent = z.infer; +export type SearchAnalytics = z.infer; +export type UserEngagement = z.infer; \ No newline at end of file diff --git a/packages/shared/src/types/document.ts b/packages/shared/src/types/document.ts new file mode 100644 index 0000000..429a2d6 --- /dev/null +++ b/packages/shared/src/types/document.ts @@ -0,0 +1,57 @@ +import { z } from 'zod'; + +// Base document schema +export const DocumentSchema = z.object({ + id: z.string().uuid(), + url: z.string().url(), + title: z.string().min(1).max(500), + content: z.string().min(1), + metadata: z.record(z.any()).optional(), + createdAt: z.date(), + updatedAt: z.date(), + indexedAt: z.date().optional(), + status: z.enum(['pending', 'indexed', 'failed', 'archived']), + contentType: z.enum(['text', 'html', 'markdown', 'pdf', 'doc']), + language: z.string().default('en'), + tags: z.array(z.string()).default([]), + sourceId: z.string().optional(), // For tracking content source +}); + +// Document chunk for vector storage +export const DocumentChunkSchema = z.object({ + id: z.string().uuid(), + documentId: z.string().uuid(), + content: z.string().min(1), + embedding: z.array(z.number()).optional(), + chunkIndex: z.number().int().min(0), + startOffset: z.number().int().min(0), + endOffset: z.number().int().min(0), + metadata: z.record(z.any()).optional(), +}); + +// Document ingestion request +export const DocumentIngestionRequestSchema = z.object({ + url: z.string().url().optional(), + content: z.string().min(1).optional(), + title: z.string().min(1).max(500), + contentType: z.enum(['text', 'html', 'markdown', 'pdf', 'doc']).default('text'), + metadata: z.record(z.any()).optional(), + tags: z.array(z.string()).default([]), + sourceId: z.string().optional(), +}).refine(data => data.url || data.content, { + message: "Either URL or content must be provided" +}); + +// Export types +export type Document = z.infer; +export type DocumentChunk = z.infer; +export type DocumentIngestionRequest = z.infer; + +// Document status update +export const DocumentStatusUpdateSchema = z.object({ + id: z.string().uuid(), + status: z.enum(['pending', 'indexed', 'failed', 'archived']), + error: z.string().optional(), +}); + +export type DocumentStatusUpdate = z.infer; \ No newline at end of file diff --git a/packages/shared/src/types/embedding.ts b/packages/shared/src/types/embedding.ts new file mode 100644 index 0000000..399d759 --- /dev/null +++ b/packages/shared/src/types/embedding.ts @@ -0,0 +1,90 @@ +import { z } from 'zod'; + +// Embedding provider configuration +export const EmbeddingProviderSchema = z.object({ + provider: z.enum(['openai', 'huggingface', 'cohere', 'custom']), + model: z.string(), + apiKey: z.string().optional(), + apiUrl: z.string().url().optional(), + dimensions: z.number().int().min(1), + maxTokens: z.number().int().min(1).optional(), + batchSize: z.number().int().min(1).max(100).default(10), +}); + +// Embedding request +export const EmbeddingRequestSchema = z.object({ + texts: z.array(z.string().min(1)), + model: z.string().optional(), + provider: z.enum(['openai', 'huggingface', 'cohere', 'custom']).optional(), + options: z.object({ + normalize: z.boolean().default(true), + truncate: z.boolean().default(true), + }).optional(), +}); + +// Embedding response +export const EmbeddingResponseSchema = z.object({ + embeddings: z.array(z.array(z.number())), + model: z.string(), + provider: z.string(), + usage: z.object({ + totalTokens: z.number().int().min(0), + promptTokens: z.number().int().min(0), + }).optional(), +}); + +// Vector search request +export const VectorSearchRequestSchema = z.object({ + queryVector: z.array(z.number()), + limit: z.number().int().min(1).max(100).default(10), + minScore: z.number().min(0).max(1).optional(), + filters: z.record(z.any()).optional(), + includeMetadata: z.boolean().default(false), + includeVectors: z.boolean().default(false), +}); + +// Vector search result +export const VectorSearchResultSchema = z.object({ + id: z.string(), + score: z.number().min(0).max(1), + metadata: z.record(z.any()).optional(), + vector: z.array(z.number()).optional(), +}); + +// Vector database configuration +export const VectorDatabaseConfigSchema = z.object({ + provider: z.enum(['milvus', 'pinecone', 'weaviate', 'qdrant', 'chroma']), + connectionString: z.string(), + collectionName: z.string(), + dimensions: z.number().int().min(1), + metricType: z.enum(['cosine', 'euclidean', 'dot_product']).default('cosine'), + indexType: z.string().optional(), + apiKey: z.string().optional(), +}); + +// Batch embedding job +export const BatchEmbeddingJobSchema = z.object({ + id: z.string().uuid(), + status: z.enum(['pending', 'processing', 'completed', 'failed']), + documentIds: z.array(z.string().uuid()), + provider: z.string(), + model: z.string(), + createdAt: z.date(), + startedAt: z.date().optional(), + completedAt: z.date().optional(), + progress: z.object({ + total: z.number().int().min(0), + processed: z.number().int().min(0), + failed: z.number().int().min(0), + }), + error: z.string().optional(), +}); + +// Export types +export type EmbeddingProvider = z.infer; +export type EmbeddingRequest = z.infer; +export type EmbeddingResponse = z.infer; +export type VectorSearchRequest = z.infer; +export type VectorSearchResult = z.infer; +export type VectorDatabaseConfig = z.infer; +export type BatchEmbeddingJob = z.infer; \ No newline at end of file diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts new file mode 100644 index 0000000..5f393b8 --- /dev/null +++ b/packages/shared/src/types/index.ts @@ -0,0 +1,56 @@ +// Document types +export * from './document'; + +// Search types +export * from './search'; + +// Analytics types +export * from './analytics'; + +// Embedding types +export * from './embedding'; + +// LLM types +export * from './llm'; + +// Widget types +export * from './widget'; + +// Common API response types +import { z } from 'zod'; + +export const ApiResponseSchema = z.object({ + success: z.boolean(), + data: z.any().optional(), + error: z.object({ + code: z.string(), + message: z.string(), + details: z.any().optional(), + }).optional(), + metadata: z.object({ + requestId: z.string().uuid(), + timestamp: z.date(), + version: z.string(), + processingTime: z.number().min(0).optional(), + }), +}); + +export const PaginatedResponseSchema = z.object({ + items: z.array(z.any()), + pagination: z.object({ + page: z.number().int().min(1), + limit: z.number().int().min(1), + total: z.number().int().min(0), + totalPages: z.number().int().min(0), + hasNext: z.boolean(), + hasPrev: z.boolean(), + }), +}); + +export type ApiResponse = z.infer & { + data?: T; +}; + +export type PaginatedResponse = z.infer & { + items: T[]; +}; \ No newline at end of file diff --git a/packages/shared/src/types/llm.ts b/packages/shared/src/types/llm.ts new file mode 100644 index 0000000..43134b2 --- /dev/null +++ b/packages/shared/src/types/llm.ts @@ -0,0 +1,142 @@ +import { z } from 'zod'; + +// LLM provider configuration +export const LLMProviderSchema = z.object({ + provider: z.enum(['openai', 'anthropic', 'huggingface', 'custom']), + model: z.string(), + apiKey: z.string().optional(), + apiUrl: z.string().url().optional(), + maxTokens: z.number().int().min(1).max(32000).default(2000), + temperature: z.number().min(0).max(2).default(0.7), + topP: z.number().min(0).max(1).default(1), + frequencyPenalty: z.number().min(-2).max(2).default(0), + presencePenalty: z.number().min(-2).max(2).default(0), +}); + +// Enhanced answer request +export const EnhancedAnswerRequestSchema = z.object({ + query: z.string().min(1), + searchResults: z.array(z.object({ + id: z.string(), + title: z.string(), + content: z.string(), + url: z.string().url().optional(), + score: z.number().min(0).max(1), + })), + options: z.object({ + maxLength: z.number().int().min(50).max(2000).default(500), + includeSourceCitations: z.boolean().default(true), + tone: z.enum(['professional', 'casual', 'technical', 'friendly']).default('professional'), + language: z.string().default('en'), + customInstructions: z.string().optional(), + }).optional(), + context: z.object({ + userId: z.string().optional(), + sessionId: z.string().optional(), + previousQueries: z.array(z.string()).optional(), + }).optional(), +}); + +// Enhanced answer response +export const EnhancedAnswerResponseSchema = z.object({ + id: z.string().uuid(), + content: z.string(), + sources: z.array(z.object({ + id: z.string(), + title: z.string(), + url: z.string().url().optional(), + relevanceScore: z.number().min(0).max(1), + })), + confidence: z.number().min(0).max(1), + model: z.string(), + provider: z.string(), + usage: z.object({ + promptTokens: z.number().int().min(0), + completionTokens: z.number().int().min(0), + totalTokens: z.number().int().min(0), + }).optional(), + processingTime: z.number().min(0), + createdAt: z.date(), +}); + +// Content guardrails +export const ContentGuardrailsSchema = z.object({ + enableProfanityFilter: z.boolean().default(true), + enableToxicityFilter: z.boolean().default(true), + enableFactualityCheck: z.boolean().default(false), + enableSourceAttribution: z.boolean().default(true), + enableHallucinationDetection: z.boolean().default(true), + customFilters: z.array(z.object({ + name: z.string(), + pattern: z.string(), + action: z.enum(['block', 'warn', 'flag']), + })).optional(), + maxResponseLength: z.number().int().min(10).max(5000).default(1000), + requireSourceCitation: z.boolean().default(false), +}); + +// Guardrail violation +export const GuardrailViolationSchema = z.object({ + type: z.enum(['profanity', 'toxicity', 'factuality', 'hallucination', 'custom']), + severity: z.enum(['low', 'medium', 'high']), + message: z.string(), + confidence: z.number().min(0).max(1), + action: z.enum(['blocked', 'warned', 'flagged']), + details: z.record(z.any()).optional(), +}); + +// LLM chat message +export const ChatMessageSchema = z.object({ + id: z.string().uuid(), + role: z.enum(['system', 'user', 'assistant']), + content: z.string(), + timestamp: z.date(), + metadata: z.object({ + sources: z.array(z.string()).optional(), + confidence: z.number().min(0).max(1).optional(), + processingTime: z.number().min(0).optional(), + model: z.string().optional(), + }).optional(), +}); + +// Chat session +export const ChatSessionSchema = z.object({ + id: z.string().uuid(), + userId: z.string().optional(), + messages: z.array(ChatMessageSchema), + createdAt: z.date(), + updatedAt: z.date(), + metadata: z.object({ + title: z.string().optional(), + tags: z.array(z.string()).optional(), + language: z.string().optional(), + }).optional(), +}); + +// A/B test configuration for LLM responses +export const LLMTestConfigSchema = z.object({ + id: z.string().uuid(), + name: z.string(), + description: z.string().optional(), + variants: z.array(z.object({ + id: z.string(), + name: z.string(), + weight: z.number().min(0).max(1), + config: LLMProviderSchema, + promptTemplate: z.string().optional(), + })), + enabled: z.boolean().default(false), + startDate: z.date().optional(), + endDate: z.date().optional(), + targetMetrics: z.array(z.enum(['response_quality', 'response_time', 'user_satisfaction'])), +}); + +// Export types +export type LLMProvider = z.infer; +export type EnhancedAnswerRequest = z.infer; +export type EnhancedAnswerResponse = z.infer; +export type ContentGuardrails = z.infer; +export type GuardrailViolation = z.infer; +export type ChatMessage = z.infer; +export type ChatSession = z.infer; +export type LLMTestConfig = z.infer; \ No newline at end of file diff --git a/packages/shared/src/types/search.ts b/packages/shared/src/types/search.ts new file mode 100644 index 0000000..ae5e5f5 --- /dev/null +++ b/packages/shared/src/types/search.ts @@ -0,0 +1,87 @@ +import { z } from 'zod'; + +// Search query schema +export const SearchQuerySchema = z.object({ + query: z.string().min(1).max(1000), + filters: z.object({ + contentType: z.array(z.string()).optional(), + tags: z.array(z.string()).optional(), + dateRange: z.object({ + from: z.date().optional(), + to: z.date().optional(), + }).optional(), + language: z.string().optional(), + sourceId: z.string().optional(), + }).optional(), + options: z.object({ + limit: z.number().int().min(1).max(100).default(10), + offset: z.number().int().min(0).default(0), + includeContent: z.boolean().default(true), + includeMetadata: z.boolean().default(false), + minScore: z.number().min(0).max(1).optional(), + searchType: z.enum(['semantic', 'keyword', 'hybrid']).default('semantic'), + enhanceWithLLM: z.boolean().default(false), + }).optional(), + sessionId: z.string().optional(), + userId: z.string().optional(), +}); + +// Search result item +export const SearchResultItemSchema = z.object({ + id: z.string().uuid(), + documentId: z.string().uuid(), + title: z.string(), + content: z.string(), + url: z.string().url().optional(), + score: z.number().min(0).max(1), + metadata: z.record(z.any()).optional(), + highlights: z.array(z.object({ + field: z.string(), + fragments: z.array(z.string()), + })).optional(), + chunkIndex: z.number().int().optional(), +}); + +// Search response +export const SearchResponseSchema = z.object({ + results: z.array(SearchResultItemSchema), + total: z.number().int().min(0), + query: z.string(), + processingTime: z.number().min(0), + enhancedAnswer: z.object({ + content: z.string(), + sources: z.array(z.string().uuid()), + confidence: z.number().min(0).max(1), + }).optional(), + suggestions: z.array(z.string()).optional(), + facets: z.record(z.array(z.object({ + value: z.string(), + count: z.number().int().min(0), + }))).optional(), +}); + +// Auto-complete suggestion +export const AutoCompleteRequestSchema = z.object({ + query: z.string().min(1).max(100), + limit: z.number().int().min(1).max(20).default(5), + filters: z.object({ + contentType: z.array(z.string()).optional(), + tags: z.array(z.string()).optional(), + language: z.string().optional(), + }).optional(), +}); + +export const AutoCompleteResponseSchema = z.object({ + suggestions: z.array(z.object({ + text: z.string(), + score: z.number().min(0).max(1), + type: z.enum(['query', 'document_title', 'tag']), + })), +}); + +// Export types +export type SearchQuery = z.infer; +export type SearchResultItem = z.infer; +export type SearchResponse = z.infer; +export type AutoCompleteRequest = z.infer; +export type AutoCompleteResponse = z.infer; \ No newline at end of file diff --git a/packages/shared/src/types/widget.ts b/packages/shared/src/types/widget.ts new file mode 100644 index 0000000..b7d88ad --- /dev/null +++ b/packages/shared/src/types/widget.ts @@ -0,0 +1,149 @@ +import { z } from 'zod'; + +// Widget configuration +export const WidgetConfigSchema = z.object({ + id: z.string().uuid(), + name: z.string().min(1).max(100), + apiKey: z.string(), + theme: z.object({ + primaryColor: z.string().regex(/^#[0-9A-F]{6}$/i).default('#007bff'), + secondaryColor: z.string().regex(/^#[0-9A-F]{6}$/i).default('#6c757d'), + backgroundColor: z.string().regex(/^#[0-9A-F]{6}$/i).default('#ffffff'), + textColor: z.string().regex(/^#[0-9A-F]{6}$/i).default('#212529'), + borderRadius: z.number().min(0).max(50).default(8), + fontFamily: z.string().default('system-ui, -apple-system, sans-serif'), + fontSize: z.number().min(10).max(24).default(14), + }).optional(), + layout: z.object({ + position: z.enum(['bottom-right', 'bottom-left', 'top-right', 'top-left', 'center']).default('bottom-right'), + width: z.number().min(200).max(800).default(400), + height: z.number().min(300).max(600).default(500), + offset: z.object({ + x: z.number().default(20), + y: z.number().default(20), + }).optional(), + zIndex: z.number().min(1).max(999999).default(9999), + }).optional(), + behavior: z.object({ + autoOpen: z.boolean().default(false), + showWelcomeMessage: z.boolean().default(true), + enableVoiceInput: z.boolean().default(false), + enableFileUpload: z.boolean().default(false), + maxFileSize: z.number().min(1).max(50).default(10), // MB + allowedFileTypes: z.array(z.string()).default(['pdf', 'doc', 'docx', 'txt']), + enableFeedback: z.boolean().default(true), + enableAnalytics: z.boolean().default(true), + }).optional(), + content: z.object({ + welcomeMessage: z.string().default('Hi! How can I help you find what you\'re looking for?'), + placeholder: z.string().default('Ask me anything...'), + noResultsMessage: z.string().default('I couldn\'t find any relevant results. Try rephrasing your question.'), + errorMessage: z.string().default('Something went wrong. Please try again.'), + loadingMessage: z.string().default('Searching...'), + poweredByText: z.string().default('Powered by AI Search'), + feedbackPrompt: z.string().default('Was this helpful?'), + }).optional(), + features: z.object({ + enableSemanticSearch: z.boolean().default(true), + enableEnhancedAnswers: z.boolean().default(true), + enableAutoComplete: z.boolean().default(true), + enableSearchSuggestions: z.boolean().default(true), + enableResultHighlighting: z.boolean().default(true), + enableSourceCitations: z.boolean().default(true), + maxResults: z.number().min(1).max(50).default(10), + enableFilters: z.boolean().default(false), + availableFilters: z.array(z.enum(['contentType', 'tags', 'dateRange', 'language'])).optional(), + }).optional(), + security: z.object({ + allowedDomains: z.array(z.string().url()).optional(), + enableCSP: z.boolean().default(true), + enableCORS: z.boolean().default(true), + rateLimitPerMinute: z.number().min(1).max(1000).default(60), + }).optional(), + createdAt: z.date(), + updatedAt: z.date(), +}); + +// Widget installation code +export const WidgetInstallationSchema = z.object({ + widgetId: z.string().uuid(), + scriptTag: z.string(), + htmlSnippet: z.string(), + customization: z.record(z.any()).optional(), + version: z.string().default('1.0.0'), +}); + +// Widget analytics event +export const WidgetAnalyticsEventSchema = z.object({ + widgetId: z.string().uuid(), + eventType: z.enum([ + 'widget_loaded', + 'widget_opened', + 'widget_closed', + 'search_initiated', + 'result_clicked', + 'feedback_given', + 'error_occurred', + 'file_uploaded' + ]), + timestamp: z.date(), + sessionId: z.string(), + userId: z.string().optional(), + data: z.record(z.any()), + metadata: z.object({ + userAgent: z.string().optional(), + referrer: z.string().optional(), + viewport: z.object({ + width: z.number(), + height: z.number(), + }).optional(), + widgetVersion: z.string().optional(), + }).optional(), +}); + +// Widget performance metrics +export const WidgetPerformanceSchema = z.object({ + widgetId: z.string().uuid(), + period: z.enum(['hour', 'day', 'week', 'month']), + metrics: z.object({ + totalLoads: z.number().int().min(0), + uniqueUsers: z.number().int().min(0), + totalSearches: z.number().int().min(0), + avgResponseTime: z.number().min(0), + errorRate: z.number().min(0).max(1), + conversionRate: z.number().min(0).max(1), + userSatisfaction: z.number().min(0).max(5).optional(), + bounceRate: z.number().min(0).max(1), + }), + topQueries: z.array(z.object({ + query: z.string(), + count: z.number().int().min(0), + })), + deviceBreakdown: z.object({ + desktop: z.number().min(0).max(1), + mobile: z.number().min(0).max(1), + tablet: z.number().min(0).max(1), + }), +}); + +// Widget customization template +export const WidgetTemplateSchema = z.object({ + id: z.string().uuid(), + name: z.string(), + description: z.string().optional(), + category: z.enum(['ecommerce', 'documentation', 'support', 'blog', 'corporate']), + config: WidgetConfigSchema.omit({ id: true, apiKey: true, createdAt: true, updatedAt: true }), + preview: z.object({ + thumbnail: z.string().url(), + demo: z.string().url().optional(), + }), + isPublic: z.boolean().default(false), + createdBy: z.string().optional(), +}); + +// Export types +export type WidgetConfig = z.infer; +export type WidgetInstallation = z.infer; +export type WidgetAnalyticsEvent = z.infer; +export type WidgetPerformance = z.infer; +export type WidgetTemplate = z.infer; \ No newline at end of file diff --git a/packages/shared/src/utils/text-processing.ts b/packages/shared/src/utils/text-processing.ts new file mode 100644 index 0000000..8aca6cb --- /dev/null +++ b/packages/shared/src/utils/text-processing.ts @@ -0,0 +1,201 @@ +/** + * Text processing utilities for search and content analysis + */ + +/** + * Split text into chunks for embedding + */ +export function chunkText( + text: string, + chunkSize: number = 1000, + overlap: number = 200 +): Array<{ content: string; startOffset: number; endOffset: number }> { + const chunks: Array<{ content: string; startOffset: number; endOffset: number }> = []; + + if (text.length <= chunkSize) { + return [{ content: text, startOffset: 0, endOffset: text.length }]; + } + + let start = 0; + let chunkIndex = 0; + + while (start < text.length) { + let end = Math.min(start + chunkSize, text.length); + + // Try to break at sentence boundaries + if (end < text.length) { + const sentenceEnd = text.lastIndexOf('.', end); + const questionEnd = text.lastIndexOf('?', end); + const exclamationEnd = text.lastIndexOf('!', end); + + const lastSentenceEnd = Math.max(sentenceEnd, questionEnd, exclamationEnd); + + if (lastSentenceEnd > start + chunkSize * 0.5) { + end = lastSentenceEnd + 1; + } + } + + const content = text.substring(start, end).trim(); + if (content.length > 0) { + chunks.push({ + content, + startOffset: start, + endOffset: end, + }); + } + + start = Math.max(start + chunkSize - overlap, end); + chunkIndex++; + } + + return chunks; +} + +/** + * Extract keywords from text + */ +export function extractKeywords(text: string, maxKeywords: number = 10): string[] { + // Simple keyword extraction - in production, use more sophisticated NLP + const words = text + .toLowerCase() + .replace(/[^\w\s]/g, '') + .split(/\s+/) + .filter(word => word.length > 3); + + // Remove common stop words + const stopWords = new Set([ + 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', + 'by', 'from', 'up', 'about', 'into', 'through', 'during', 'before', + 'after', 'above', 'below', 'between', 'among', 'this', 'that', 'these', + 'those', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', + 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should' + ]); + + const filteredWords = words.filter(word => !stopWords.has(word)); + + // Count word frequency + const wordCount = new Map(); + filteredWords.forEach(word => { + wordCount.set(word, (wordCount.get(word) || 0) + 1); + }); + + // Sort by frequency and return top keywords + return Array.from(wordCount.entries()) + .sort(([, a], [, b]) => b - a) + .slice(0, maxKeywords) + .map(([word]) => word); +} + +/** + * Calculate text similarity using Jaccard index + */ +export function calculateTextSimilarity(text1: string, text2: string): number { + const words1 = new Set(text1.toLowerCase().split(/\s+/)); + const words2 = new Set(text2.toLowerCase().split(/\s+/)); + + const intersection = new Set([...words1].filter(x => words2.has(x))); + const union = new Set([...words1, ...words2]); + + return intersection.size / union.size; +} + +/** + * Highlight search terms in text + */ +export function highlightSearchTerms( + text: string, + searchTerms: string[], + highlightTag: string = 'mark' +): string { + let highlightedText = text; + + searchTerms.forEach(term => { + const regex = new RegExp(`\\b${term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'gi'); + highlightedText = highlightedText.replace( + regex, + `<${highlightTag}>$&` + ); + }); + + return highlightedText; +} + +/** + * Generate text summary + */ +export function generateSummary(text: string, maxLength: number = 200): string { + if (text.length <= maxLength) return text; + + // Split into sentences + const sentences = text.split(/[.!?]+/).filter(s => s.trim().length > 0); + + if (sentences.length === 0) return text.substring(0, maxLength) + '...'; + + let summary = ''; + for (const sentence of sentences) { + const trimmedSentence = sentence.trim(); + if (summary.length + trimmedSentence.length + 1 <= maxLength) { + summary += (summary ? '. ' : '') + trimmedSentence; + } else { + break; + } + } + + return summary || text.substring(0, maxLength) + '...'; +} + +/** + * Clean HTML content for indexing + */ +export function cleanHtmlContent(html: string): string { + // Remove script and style tags + let cleaned = html.replace(/]*>[\s\S]*?<\/script>/gi, ''); + cleaned = cleaned.replace(/]*>[\s\S]*?<\/style>/gi, ''); + + // Remove HTML tags + cleaned = cleaned.replace(/<[^>]*>/g, ' '); + + // Decode HTML entities + cleaned = cleaned + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'"); + + // Normalize whitespace + cleaned = cleaned.replace(/\s+/g, ' ').trim(); + + return cleaned; +} + +/** + * Detect language of text (simple heuristic) + */ +export function detectLanguage(text: string): string { + // Simple language detection - in production, use proper language detection library + const sample = text.toLowerCase().substring(0, 1000); + + // English indicators + if (/\b(the|and|or|but|in|on|at|to|for|of|with)\b/.test(sample)) { + return 'en'; + } + + // Spanish indicators + if (/\b(el|la|los|las|y|o|pero|en|de|con|para)\b/.test(sample)) { + return 'es'; + } + + // French indicators + if (/\b(le|la|les|et|ou|mais|dans|de|avec|pour)\b/.test(sample)) { + return 'fr'; + } + + // German indicators + if (/\b(der|die|das|und|oder|aber|in|von|mit|für)\b/.test(sample)) { + return 'de'; + } + + return 'en'; // Default to English +} \ No newline at end of file diff --git a/packages/shared/src/utils/validation.ts b/packages/shared/src/utils/validation.ts new file mode 100644 index 0000000..53d4244 --- /dev/null +++ b/packages/shared/src/utils/validation.ts @@ -0,0 +1,108 @@ +import { z } from 'zod'; + +/** + * Utility function to validate data against a Zod schema + */ +export function validateData(schema: z.ZodSchema, data: unknown): T { + try { + return schema.parse(data); + } catch (error) { + if (error instanceof z.ZodError) { + throw new Error(`Validation error: ${error.errors.map(e => e.message).join(', ')}`); + } + throw error; + } +} + +/** + * Safe validation that returns result with success flag + */ +export function safeValidateData( + schema: z.ZodSchema, + data: unknown +): { success: true; data: T } | { success: false; error: string } { + try { + const result = schema.parse(data); + return { success: true, data: result }; + } catch (error) { + if (error instanceof z.ZodError) { + return { + success: false, + error: error.errors.map(e => `${e.path.join('.')}: ${e.message}`).join(', ') + }; + } + return { success: false, error: 'Unknown validation error' }; + } +} + +/** + * Sanitize text content for search and display + */ +export function sanitizeText(text: string): string { + return text + .trim() + .replace(/\s+/g, ' ') + .replace(/[^\w\s\-_.!?,:;()]/g, '') + .substring(0, 10000); // Limit length +} + +/** + * Validate URL format + */ +export function isValidUrl(url: string): boolean { + try { + new URL(url); + return true; + } catch { + return false; + } +} + +/** + * Generate UUID v4 + */ +export function generateUUID(): string { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = Math.random() * 16 | 0; + const v = c == 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); +} + +/** + * Validate and normalize email + */ +export function normalizeEmail(email: string): string { + return email.toLowerCase().trim(); +} + +/** + * Check if string is valid JSON + */ +export function isValidJSON(str: string): boolean { + try { + JSON.parse(str); + return true; + } catch { + return false; + } +} + +/** + * Truncate text to specified length with ellipsis + */ +export function truncateText(text: string, maxLength: number): string { + if (text.length <= maxLength) return text; + return text.substring(0, maxLength - 3) + '...'; +} + +/** + * Extract domain from URL + */ +export function extractDomain(url: string): string | null { + try { + return new URL(url).hostname; + } catch { + return null; + } +} \ No newline at end of file diff --git a/packages/shared/tsconfig.json b/packages/shared/tsconfig.json new file mode 100644 index 0000000..a604885 --- /dev/null +++ b/packages/shared/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2020", + "lib": ["ES2020"], + "allowJs": true, + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "module": "ESNext", + "moduleResolution": "node", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": false, + "declaration": true, + "declarationMap": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"], + "exclude": ["node_modules", "dist"] +} \ No newline at end of file diff --git a/packages/widget/package.json b/packages/widget/package.json new file mode 100644 index 0000000..fd96897 --- /dev/null +++ b/packages/widget/package.json @@ -0,0 +1,46 @@ +{ + "name": "@ai-search/widget", + "version": "1.0.0", + "description": "Embeddable AI search widget", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "preview": "vite preview", + "test": "vitest", + "test:ui": "vitest --ui", + "lint": "eslint src --ext .ts,.tsx", + "type-check": "tsc --noEmit", + "clean": "rm -rf dist" + }, + "dependencies": { + "@ai-search/shared": "workspace:*", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "lucide-react": "^0.292.0", + "clsx": "^2.0.0", + "zustand": "^4.4.4" + }, + "devDependencies": { + "@types/react": "^18.2.37", + "@types/react-dom": "^18.2.15", + "@vitejs/plugin-react": "^4.1.1", + "typescript": "^5.2.2", + "vite": "^4.5.0", + "vite-plugin-dts": "^3.6.3", + "vitest": "^0.34.6", + "@testing-library/react": "^13.4.0", + "@testing-library/jest-dom": "^6.1.4", + "eslint": "^8.51.0", + "@typescript-eslint/eslint-plugin": "^6.7.4", + "@typescript-eslint/parser": "^6.7.4" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } +} \ No newline at end of file diff --git a/packages/widget/src/components/ChatInterface.tsx b/packages/widget/src/components/ChatInterface.tsx new file mode 100644 index 0000000..e99b661 --- /dev/null +++ b/packages/widget/src/components/ChatInterface.tsx @@ -0,0 +1,168 @@ +import React, { useRef, useEffect } from 'react'; +import { WidgetConfig, SearchResultItem } from '@ai-search/shared'; +import { useSearchStore } from '../store/search-store'; +import { useSearch } from '../hooks/useSearch'; +import { MessageBubble } from './MessageBubble'; +import { SearchBox } from './SearchBox'; +import { ThumbsUp, ThumbsDown } from 'lucide-react'; + +interface ChatInterfaceProps { + config: WidgetConfig; + onResultClick?: (result: SearchResultItem) => void; + onFeedback?: (feedback: { type: 'positive' | 'negative'; messageId: string; comment?: string }) => void; +} + +export function ChatInterface({ config, onResultClick, onFeedback }: ChatInterfaceProps) { + const { messages, isLoading } = useSearchStore(); + const { search } = useSearch(); + const messagesEndRef = useRef(null); + + // Auto-scroll to bottom when new messages arrive + useEffect(() => { + messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + }, [messages, isLoading]); + + const handleFeedback = (messageId: string, type: 'positive' | 'negative') => { + onFeedback?.({ type, messageId }); + }; + + return ( +
+ {/* Messages area */} +
+ {messages.map((message) => ( +
+ { + if (source.url) { + window.open(source.url, '_blank', 'noopener,noreferrer'); + } + }} + /> + + {/* Feedback buttons for assistant messages */} + {message.type === 'assistant' && config.behavior?.enableFeedback && ( +
+ + + +
+ )} +
+ ))} + + {/* Loading indicator */} + {isLoading && ( + + )} + +
+
+ + {/* Input area */} +
+ +
+
+ ); +} \ No newline at end of file diff --git a/packages/widget/src/components/MessageBubble.tsx b/packages/widget/src/components/MessageBubble.tsx new file mode 100644 index 0000000..cc7a649 --- /dev/null +++ b/packages/widget/src/components/MessageBubble.tsx @@ -0,0 +1,190 @@ +import React from 'react'; +import { WidgetConfig } from '@ai-search/shared'; +import { User, Bot, ExternalLink, Loader2 } from 'lucide-react'; +import { clsx } from 'clsx'; + +interface MessageBubbleProps { + message: { + id: string; + type: 'user' | 'assistant'; + content: string; + timestamp: Date; + sources?: Array<{ id: string; title: string; url?: string }>; + }; + config: WidgetConfig; + isLoading?: boolean; + onSourceClick?: (source: { id: string; title: string; url?: string }) => void; +} + +export function MessageBubble({ message, config, isLoading, onSourceClick }: MessageBubbleProps) { + const isUser = message.type === 'user'; + + return ( +
+ {/* Avatar */} +
+ {isUser ? : } +
+ + {/* Message content */} +
+ {/* Message bubble */} +
+ {isLoading ? ( +
+ + {message.content} +
+ ) : ( +
+ {message.content} +
+ )} +
+ + {/* Sources */} + {message.sources && message.sources.length > 0 && config.features?.enableSourceCitations && ( +
+
+ Sources: +
+ + {message.sources.map((source, index) => ( + + ))} +
+ )} + + {/* Timestamp */} +
+ {message.timestamp.toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit' + })} +
+
+
+ ); +} \ No newline at end of file diff --git a/packages/widget/src/components/ResultsList.tsx b/packages/widget/src/components/ResultsList.tsx new file mode 100644 index 0000000..78d3c9a --- /dev/null +++ b/packages/widget/src/components/ResultsList.tsx @@ -0,0 +1,275 @@ +import React from 'react'; +import { WidgetConfig, SearchResultItem } from '@ai-search/shared'; +import { useSearchStore } from '../store/search-store'; +import { ExternalLink, FileText, Clock } from 'lucide-react'; +import { clsx } from 'clsx'; + +interface ResultsListProps { + config: WidgetConfig; + onResultClick?: (result: SearchResultItem) => void; +} + +export function ResultsList({ config, onResultClick }: ResultsListProps) { + const { results, error } = useSearchStore(); + + if (error) { + return ( +
+
+ Something went wrong +
+
+ {error} +
+
+ ); + } + + if (!results || results.results.length === 0) { + return null; + } + + const handleResultClick = (result: SearchResultItem) => { + onResultClick?.(result); + + if (result.url) { + window.open(result.url, '_blank', 'noopener,noreferrer'); + } + }; + + return ( +
+
+ + {results.total} result{results.total !== 1 ? 's' : ''} found + + +
+ + {results.processingTime.toFixed(0)}ms +
+
+ +
+ {results.results.map((result, index) => ( +
handleResultClick(result)} + onMouseEnter={(e) => { + if (result.url) { + e.currentTarget.style.borderColor = 'var(--primary-color)'; + e.currentTarget.style.boxShadow = '0 2px 8px rgba(0, 0, 0, 0.1)'; + } + }} + onMouseLeave={(e) => { + e.currentTarget.style.borderColor = '#e1e5e9'; + e.currentTarget.style.boxShadow = 'none'; + }} + > + {/* Result header */} +
+
+

+ {result.title} +

+ + {result.url && ( +
+ {new URL(result.url).hostname} +
+ )} +
+ +
+
+ {Math.round(result.score * 100)}% +
+ + {result.url ? ( + + ) : ( + + )} +
+
+ + {/* Result content */} +
+ {config.features?.enableResultHighlighting && result.highlights ? ( +
h.fragments.join(' ... ')) + .join(' ... ') + }} + /> + ) : ( + result.content + )} +
+ + {/* Result position indicator */} +
+ {index + 1} +
+
+ ))} +
+ + {/* Load more button */} + {results.results.length < results.total && ( + + )} +
+ ); +} \ No newline at end of file diff --git a/packages/widget/src/components/SearchBox.tsx b/packages/widget/src/components/SearchBox.tsx new file mode 100644 index 0000000..c955549 --- /dev/null +++ b/packages/widget/src/components/SearchBox.tsx @@ -0,0 +1,244 @@ +import React, { useState, useRef, useEffect } from 'react'; +import { WidgetConfig } from '@ai-search/shared'; +import { Search, Loader2, Mic } from 'lucide-react'; +import { useSearchStore } from '../store/search-store'; +import { useSearch } from '../hooks/useSearch'; +import { clsx } from 'clsx'; + +interface SearchBoxProps { + config: WidgetConfig; + onSearch?: (query: string) => void; +} + +export function SearchBox({ config, onSearch }: SearchBoxProps) { + const { + query, + isLoading, + suggestions, + setQuery, + } = useSearchStore(); + + const { search, getSuggestions } = useSearch(); + const [showSuggestions, setShowSuggestions] = useState(false); + const [selectedSuggestion, setSelectedSuggestion] = useState(-1); + const inputRef = useRef(null); + const suggestionsRef = useRef(null); + + const handleInputChange = (e: React.ChangeEvent) => { + const value = e.target.value; + setQuery(value); + + if (value.trim() && config.features?.enableAutoComplete) { + getSuggestions(value); + setShowSuggestions(true); + setSelectedSuggestion(-1); + } else { + setShowSuggestions(false); + } + }; + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (query.trim()) { + search(); + onSearch?.(query); + setShowSuggestions(false); + } + }; + + const handleSuggestionClick = (suggestion: string) => { + setQuery(suggestion); + search(suggestion); + onSearch?.(suggestion); + setShowSuggestions(false); + inputRef.current?.focus(); + }; + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (!showSuggestions || suggestions.length === 0) return; + + switch (e.key) { + case 'ArrowDown': + e.preventDefault(); + setSelectedSuggestion(prev => + prev < suggestions.length - 1 ? prev + 1 : prev + ); + break; + case 'ArrowUp': + e.preventDefault(); + setSelectedSuggestion(prev => prev > 0 ? prev - 1 : -1); + break; + case 'Enter': + if (selectedSuggestion >= 0) { + e.preventDefault(); + handleSuggestionClick(suggestions[selectedSuggestion]); + } + break; + case 'Escape': + setShowSuggestions(false); + setSelectedSuggestion(-1); + break; + } + }; + + // Close suggestions when clicking outside + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + suggestionsRef.current && + !suggestionsRef.current.contains(event.target as Node) && + !inputRef.current?.contains(event.target as Node) + ) { + setShowSuggestions(false); + } + }; + + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + }, []); + + return ( +
+
+
{ + e.currentTarget.style.borderColor = 'var(--primary-color)'; + }} + onBlurCapture={(e) => { + if (!e.currentTarget.contains(e.relatedTarget as Node)) { + e.currentTarget.style.borderColor = '#e1e5e9'; + } + }} + > + + + + + {config.behavior?.enableVoiceInput && ( + + )} + + {isLoading && ( +
+ +
+ )} +
+
+ + {/* Suggestions dropdown */} + {showSuggestions && suggestions.length > 0 && ( +
+ {suggestions.map((suggestion, index) => ( +
handleSuggestionClick(suggestion)} + onMouseEnter={() => setSelectedSuggestion(index)} + > + + {suggestion} +
+ ))} +
+ )} + + +
+ ); +} \ No newline at end of file diff --git a/packages/widget/src/components/SearchWidget.tsx b/packages/widget/src/components/SearchWidget.tsx new file mode 100644 index 0000000..604aff9 --- /dev/null +++ b/packages/widget/src/components/SearchWidget.tsx @@ -0,0 +1,163 @@ +import React, { useEffect, useRef } from 'react'; +import { useSearchStore } from '../store/search-store'; +import { WidgetConfig } from '@ai-search/shared'; +import { AnalyticsTracker } from '../utils/api'; +import { ChatInterface } from './ChatInterface'; +import { SearchBox } from './SearchBox'; +import { ResultsList } from './ResultsList'; +import { WidgetButton } from './WidgetButton'; +import { WidgetContainer } from './WidgetContainer'; +import { clsx } from 'clsx'; + +interface SearchWidgetProps { + config: WidgetConfig; + apiKey: string; + userId?: string; + className?: string; + onError?: (error: Error) => void; +} + +export function SearchWidget({ + config, + apiKey, + userId, + className, + onError +}: SearchWidgetProps) { + const { + isOpen, + isMinimized, + messages, + sessionId, + setConfig, + setApiKey, + toggleOpen, + } = useSearchStore(); + + const analyticsRef = useRef(null); + + // Initialize widget + useEffect(() => { + setConfig(config); + setApiKey(apiKey); + + // Initialize analytics + if (config.behavior?.enableAnalytics) { + analyticsRef.current = new AnalyticsTracker(apiKey, sessionId, userId); + analyticsRef.current.track('widget_load', { + widgetId: config.id, + config: { + theme: config.theme?.primaryColor, + position: config.layout?.position, + features: Object.keys(config.features || {}), + }, + }); + } + + // Auto-open if configured + if (config.behavior?.autoOpen && !isOpen) { + setTimeout(() => toggleOpen(), 1000); + } + + return () => { + if (analyticsRef.current) { + analyticsRef.current.destroy(); + } + }; + }, [config, apiKey, userId, sessionId, setConfig, setApiKey, isOpen, toggleOpen]); + + // Track widget interactions + const trackEvent = (eventType: string, data: Record = {}) => { + if (analyticsRef.current) { + analyticsRef.current.track(eventType as any, { + widgetId: config.id, + ...data, + }); + } + }; + + const handleToggleOpen = () => { + const newState = !isOpen; + toggleOpen(); + trackEvent(newState ? 'widget_opened' : 'widget_closed'); + }; + + const widgetStyle = { + '--primary-color': config.theme?.primaryColor || '#007bff', + '--secondary-color': config.theme?.secondaryColor || '#6c757d', + '--background-color': config.theme?.backgroundColor || '#ffffff', + '--text-color': config.theme?.textColor || '#212529', + '--border-radius': `${config.theme?.borderRadius || 8}px`, + '--font-family': config.theme?.fontFamily || 'system-ui, -apple-system, sans-serif', + '--font-size': `${config.theme?.fontSize || 14}px`, + } as React.CSSProperties; + + if (!isOpen) { + return ( +
+ +
+ ); + } + + return ( +
+ { + toggleOpen(); + trackEvent('widget_closed'); + }} + onMinimize={() => { + useSearchStore.getState().toggleMinimized(); + trackEvent('widget_minimized'); + }} + > + {messages.length === 0 ? ( +
+ {config.content?.welcomeMessage && ( +
+ {config.content.welcomeMessage} +
+ )} + trackEvent('search_initiated', { query })} + /> +
+ ) : ( + trackEvent('result_clicked', { + documentId: result.documentId, + position: result.score, + url: result.url, + })} + onFeedback={(feedback) => trackEvent('feedback_given', feedback)} + /> + )} + + {!isMinimized && ( + trackEvent('result_clicked', { + documentId: result.documentId, + position: result.score, + url: result.url, + })} + /> + )} + +
+ ); +} \ No newline at end of file diff --git a/packages/widget/src/components/WidgetButton.tsx b/packages/widget/src/components/WidgetButton.tsx new file mode 100644 index 0000000..0422cf1 --- /dev/null +++ b/packages/widget/src/components/WidgetButton.tsx @@ -0,0 +1,94 @@ +import React from 'react'; +import { WidgetConfig } from '@ai-search/shared'; +import { MessageCircle, Search } from 'lucide-react'; +import { clsx } from 'clsx'; + +interface WidgetButtonProps { + config: WidgetConfig; + onClick: () => void; +} + +export function WidgetButton({ config, onClick }: WidgetButtonProps) { + const position = config.layout?.position || 'bottom-right'; + const offset = config.layout?.offset || { x: 20, y: 20 }; + + const positionStyles: Record = { + 'bottom-right': { + position: 'fixed', + bottom: offset.y, + right: offset.x, + zIndex: config.layout?.zIndex || 9999, + }, + 'bottom-left': { + position: 'fixed', + bottom: offset.y, + left: offset.x, + zIndex: config.layout?.zIndex || 9999, + }, + 'top-right': { + position: 'fixed', + top: offset.y, + right: offset.x, + zIndex: config.layout?.zIndex || 9999, + }, + 'top-left': { + position: 'fixed', + top: offset.y, + left: offset.x, + zIndex: config.layout?.zIndex || 9999, + }, + 'center': { + position: 'fixed', + top: '50%', + left: '50%', + transform: 'translate(-50%, -50%)', + zIndex: config.layout?.zIndex || 9999, + }, + }; + + return ( + + ); +} \ No newline at end of file diff --git a/packages/widget/src/components/WidgetContainer.tsx b/packages/widget/src/components/WidgetContainer.tsx new file mode 100644 index 0000000..9159d13 --- /dev/null +++ b/packages/widget/src/components/WidgetContainer.tsx @@ -0,0 +1,182 @@ +import React, { ReactNode } from 'react'; +import { WidgetConfig } from '@ai-search/shared'; +import { X, Minus, Maximize2 } from 'lucide-react'; +import { clsx } from 'clsx'; + +interface WidgetContainerProps { + config: WidgetConfig; + isMinimized: boolean; + onClose: () => void; + onMinimize: () => void; + children: ReactNode; +} + +export function WidgetContainer({ + config, + isMinimized, + onClose, + onMinimize, + children +}: WidgetContainerProps) { + const position = config.layout?.position || 'bottom-right'; + const offset = config.layout?.offset || { x: 20, y: 20 }; + const width = config.layout?.width || 400; + const height = config.layout?.height || 500; + + const positionStyles: Record = { + 'bottom-right': { + position: 'fixed', + bottom: offset.y, + right: offset.x, + zIndex: config.layout?.zIndex || 9999, + }, + 'bottom-left': { + position: 'fixed', + bottom: offset.y, + left: offset.x, + zIndex: config.layout?.zIndex || 9999, + }, + 'top-right': { + position: 'fixed', + top: offset.y, + right: offset.x, + zIndex: config.layout?.zIndex || 9999, + }, + 'top-left': { + position: 'fixed', + top: offset.y, + left: offset.x, + zIndex: config.layout?.zIndex || 9999, + }, + 'center': { + position: 'fixed', + top: '50%', + left: '50%', + transform: 'translate(-50%, -50%)', + zIndex: config.layout?.zIndex || 9999, + }, + }; + + return ( +
+ {/* Header */} +
+
+ {config.name || 'AI Search'} +
+ +
+ + + +
+
+ + {/* Content */} + {!isMinimized && ( +
+ {children} +
+ )} + + {/* Footer */} + {!isMinimized && config.content?.poweredByText && ( +
+ {config.content.poweredByText} +
+ )} +
+ ); +} \ No newline at end of file diff --git a/packages/widget/src/hooks/useSearch.ts b/packages/widget/src/hooks/useSearch.ts new file mode 100644 index 0000000..1c13ba6 --- /dev/null +++ b/packages/widget/src/hooks/useSearch.ts @@ -0,0 +1,155 @@ +import { useCallback, useEffect, useRef } from 'react'; +import { useSearchStore } from '../store/search-store'; +import { SearchQuery, SearchResponse, AutoCompleteRequest } from '@ai-search/shared'; +import { searchApi } from '../utils/api'; + +export function useSearch() { + const { + config, + apiKey, + query, + setLoading, + setResults, + setSuggestions, + setError, + addMessage, + sessionId, + } = useSearchStore(); + + const abortControllerRef = useRef(null); + const debounceTimeoutRef = useRef(null); + + // Perform search + const search = useCallback(async (searchQuery?: string) => { + const queryToSearch = searchQuery || query; + if (!queryToSearch.trim() || !apiKey || !config) return; + + // Cancel previous request + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + } + + abortControllerRef.current = new AbortController(); + setLoading(true); + setError(null); + + try { + // Add user message + addMessage({ + type: 'user', + content: queryToSearch, + }); + + const searchRequest: SearchQuery = { + query: queryToSearch, + options: { + limit: config.features?.maxResults || 10, + includeContent: true, + includeMetadata: config.features?.enableSourceCitations || false, + searchType: config.features?.enableSemanticSearch ? 'semantic' : 'keyword', + enhanceWithLLM: config.features?.enableEnhancedAnswers || false, + }, + sessionId, + }; + + const response = await searchApi.search(searchRequest, apiKey, { + signal: abortControllerRef.current.signal, + }); + + setResults(response); + + // Add assistant message + if (response.enhancedAnswer) { + addMessage({ + type: 'assistant', + content: response.enhancedAnswer.content, + sources: response.enhancedAnswer.sources.map(sourceId => { + const result = response.results.find(r => r.id === sourceId); + return { + id: sourceId, + title: result?.title || 'Unknown', + url: result?.url, + }; + }), + }); + } else if (response.results.length > 0) { + const resultsList = response.results + .slice(0, 3) + .map(r => `• ${r.title}`) + .join('\n'); + + addMessage({ + type: 'assistant', + content: `I found ${response.total} results for "${queryToSearch}":\n\n${resultsList}`, + }); + } else { + addMessage({ + type: 'assistant', + content: config.content?.noResultsMessage || "I couldn't find any relevant results. Try rephrasing your question.", + }); + } + } catch (error) { + if (error instanceof Error && error.name !== 'AbortError') { + const errorMessage = error.message || 'An error occurred while searching'; + setError(errorMessage); + + addMessage({ + type: 'assistant', + content: config.content?.errorMessage || 'Something went wrong. Please try again.', + }); + } + } finally { + setLoading(false); + abortControllerRef.current = null; + } + }, [query, apiKey, config, sessionId, setLoading, setResults, setError, addMessage]); + + // Get autocomplete suggestions + const getSuggestions = useCallback(async (searchQuery: string) => { + if (!searchQuery.trim() || !apiKey || !config?.features?.enableAutoComplete) { + setSuggestions([]); + return; + } + + try { + const request: AutoCompleteRequest = { + query: searchQuery, + limit: 5, + }; + + const response = await searchApi.autocomplete(request, apiKey); + setSuggestions(response.suggestions.map(s => s.text)); + } catch (error) { + // Silently fail for autocomplete + setSuggestions([]); + } + }, [apiKey, config, setSuggestions]); + + // Debounced autocomplete + const debouncedGetSuggestions = useCallback((searchQuery: string) => { + if (debounceTimeoutRef.current) { + clearTimeout(debounceTimeoutRef.current); + } + + debounceTimeoutRef.current = setTimeout(() => { + getSuggestions(searchQuery); + }, 300); + }, [getSuggestions]); + + // Cleanup on unmount + useEffect(() => { + return () => { + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + } + if (debounceTimeoutRef.current) { + clearTimeout(debounceTimeoutRef.current); + } + }; + }, []); + + return { + search, + getSuggestions: debouncedGetSuggestions, + }; +} \ No newline at end of file diff --git a/packages/widget/src/index.ts b/packages/widget/src/index.ts new file mode 100644 index 0000000..3b60040 --- /dev/null +++ b/packages/widget/src/index.ts @@ -0,0 +1,250 @@ +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import { SearchWidget } from './components/SearchWidget'; +import { WidgetConfig } from '@ai-search/shared'; + +export { SearchWidget } from './components/SearchWidget'; +export { useSearchStore } from './store/search-store'; +export { useSearch } from './hooks/useSearch'; +export * from './utils/api'; + +// Widget initialization interface +export interface AISearchWidgetOptions { + apiKey: string; + config?: Partial; + userId?: string; + container?: HTMLElement | string; + onError?: (error: Error) => void; + onReady?: () => void; +} + +// Default widget configuration +const defaultConfig: Partial = { + name: 'AI Search', + theme: { + primaryColor: '#007bff', + secondaryColor: '#6c757d', + backgroundColor: '#ffffff', + textColor: '#212529', + borderRadius: 8, + fontFamily: 'system-ui, -apple-system, sans-serif', + fontSize: 14, + }, + layout: { + position: 'bottom-right', + width: 400, + height: 500, + offset: { x: 20, y: 20 }, + zIndex: 9999, + }, + behavior: { + autoOpen: false, + showWelcomeMessage: true, + enableVoiceInput: false, + enableFileUpload: false, + enableFeedback: true, + enableAnalytics: true, + }, + content: { + welcomeMessage: "Hi! How can I help you find what you're looking for?", + placeholder: 'Ask me anything...', + noResultsMessage: "I couldn't find any relevant results. Try rephrasing your question.", + errorMessage: 'Something went wrong. Please try again.', + loadingMessage: 'Searching...', + poweredByText: 'Powered by AI Search', + feedbackPrompt: 'Was this helpful?', + }, + features: { + enableSemanticSearch: true, + enableEnhancedAnswers: true, + enableAutoComplete: true, + enableSearchSuggestions: true, + enableResultHighlighting: true, + enableSourceCitations: true, + maxResults: 10, + enableFilters: false, + }, + security: { + enableCSP: true, + enableCORS: true, + rateLimitPerMinute: 60, + }, + createdAt: new Date(), + updatedAt: new Date(), +}; + +// Widget instance class +export class AISearchWidget { + private root: ReactDOM.Root | null = null; + private container: HTMLElement | null = null; + private config: WidgetConfig; + private apiKey: string; + private userId?: string; + private onError?: (error: Error) => void; + + constructor(options: AISearchWidgetOptions) { + this.apiKey = options.apiKey; + this.userId = options.userId; + this.onError = options.onError; + + // Merge user config with defaults + this.config = { + ...defaultConfig, + ...options.config, + id: this.generateWidgetId(), + apiKey: options.apiKey, + } as WidgetConfig; + + // Setup container + this.setupContainer(options.container); + + // Initialize widget + this.init(); + + // Call ready callback + options.onReady?.(); + } + + private generateWidgetId(): string { + return 'widget_' + Math.random().toString(36).substr(2, 9) + Date.now().toString(36); + } + + private setupContainer(container?: HTMLElement | string) { + if (typeof container === 'string') { + this.container = document.querySelector(container); + if (!this.container) { + throw new Error(`Container element not found: ${container}`); + } + } else if (container instanceof HTMLElement) { + this.container = container; + } else { + // Create default container + this.container = document.createElement('div'); + this.container.id = `ai-search-widget-${this.config.id}`; + document.body.appendChild(this.container); + } + + // Add CSS classes + this.container.classList.add('ai-search-widget-container'); + } + + private init() { + if (!this.container) { + throw new Error('Widget container not found'); + } + + try { + this.root = ReactDOM.createRoot(this.container); + this.render(); + } catch (error) { + const err = error instanceof Error ? error : new Error('Failed to initialize widget'); + this.onError?.(err); + throw err; + } + } + + private render() { + if (!this.root) return; + + this.root.render( + React.createElement(SearchWidget, { + config: this.config, + apiKey: this.apiKey, + userId: this.userId, + onError: this.onError, + }) + ); + } + + // Public methods + public updateConfig(newConfig: Partial) { + this.config = { ...this.config, ...newConfig, updatedAt: new Date() }; + this.render(); + } + + public updateApiKey(apiKey: string) { + this.apiKey = apiKey; + this.config.apiKey = apiKey; + this.render(); + } + + public destroy() { + if (this.root) { + this.root.unmount(); + this.root = null; + } + + if (this.container && this.container.parentNode) { + this.container.parentNode.removeChild(this.container); + } + } + + public getConfig(): WidgetConfig { + return { ...this.config }; + } + + public open() { + // This would trigger the widget to open + // Implementation depends on how the store is accessed + } + + public close() { + // This would trigger the widget to close + // Implementation depends on how the store is accessed + } +} + +// Global initialization function +export function initAISearchWidget(options: AISearchWidgetOptions): AISearchWidget { + return new AISearchWidget(options); +} + +// Auto-initialization from script tag +declare global { + interface Window { + AISearchWidget?: typeof AISearchWidget; + initAISearchWidget?: typeof initAISearchWidget; + } +} + +// Expose to global scope for script tag usage +if (typeof window !== 'undefined') { + window.AISearchWidget = AISearchWidget; + window.initAISearchWidget = initAISearchWidget; + + // Auto-init from data attributes + document.addEventListener('DOMContentLoaded', () => { + const scripts = document.querySelectorAll('script[data-ai-search-api-key]'); + + scripts.forEach((script) => { + const apiKey = script.getAttribute('data-ai-search-api-key'); + const userId = script.getAttribute('data-ai-search-user-id'); + const containerId = script.getAttribute('data-ai-search-container'); + + if (apiKey) { + try { + new AISearchWidget({ + apiKey, + userId: userId || undefined, + container: containerId || undefined, + config: { + // Parse any data attributes as config + theme: { + primaryColor: script.getAttribute('data-ai-search-primary-color') || undefined, + secondaryColor: script.getAttribute('data-ai-search-secondary-color') || undefined, + }, + layout: { + position: (script.getAttribute('data-ai-search-position') as any) || undefined, + }, + behavior: { + autoOpen: script.getAttribute('data-ai-search-auto-open') === 'true', + }, + }, + }); + } catch (error) { + console.error('Failed to initialize AI Search Widget:', error); + } + } + }); + }); +} \ No newline at end of file diff --git a/packages/widget/src/store/search-store.ts b/packages/widget/src/store/search-store.ts new file mode 100644 index 0000000..0604dba --- /dev/null +++ b/packages/widget/src/store/search-store.ts @@ -0,0 +1,113 @@ +import { create } from 'zustand'; +import { SearchQuery, SearchResponse, WidgetConfig } from '@ai-search/shared'; + +interface SearchState { + // Configuration + config: WidgetConfig | null; + apiKey: string | null; + + // UI State + isOpen: boolean; + isLoading: boolean; + isMinimized: boolean; + + // Search State + query: string; + results: SearchResponse | null; + suggestions: string[]; + error: string | null; + + // Chat State + messages: Array<{ + id: string; + type: 'user' | 'assistant'; + content: string; + timestamp: Date; + sources?: Array<{ id: string; title: string; url?: string }>; + }>; + + // Session + sessionId: string; + + // Actions + setConfig: (config: WidgetConfig) => void; + setApiKey: (apiKey: string) => void; + toggleOpen: () => void; + toggleMinimized: () => void; + setQuery: (query: string) => void; + setLoading: (loading: boolean) => void; + setResults: (results: SearchResponse | null) => void; + setSuggestions: (suggestions: string[]) => void; + setError: (error: string | null) => void; + addMessage: (message: Omit) => void; + clearMessages: () => void; + reset: () => void; +} + +const generateSessionId = () => { + return 'session_' + Math.random().toString(36).substr(2, 9) + Date.now().toString(36); +}; + +export const useSearchStore = create((set, get) => ({ + // Initial state + config: null, + apiKey: null, + isOpen: false, + isLoading: false, + isMinimized: false, + query: '', + results: null, + suggestions: [], + error: null, + messages: [], + sessionId: generateSessionId(), + + // Actions + setConfig: (config) => set({ config }), + + setApiKey: (apiKey) => set({ apiKey }), + + toggleOpen: () => set((state) => ({ + isOpen: !state.isOpen, + isMinimized: false + })), + + toggleMinimized: () => set((state) => ({ + isMinimized: !state.isMinimized + })), + + setQuery: (query) => set({ query }), + + setLoading: (isLoading) => set({ isLoading }), + + setResults: (results) => set({ results, error: null }), + + setSuggestions: (suggestions) => set({ suggestions }), + + setError: (error) => set({ error, isLoading: false }), + + addMessage: (message) => set((state) => ({ + messages: [ + ...state.messages, + { + ...message, + id: 'msg_' + Math.random().toString(36).substr(2, 9), + timestamp: new Date(), + }, + ], + })), + + clearMessages: () => set({ messages: [], sessionId: generateSessionId() }), + + reset: () => set({ + isOpen: false, + isLoading: false, + isMinimized: false, + query: '', + results: null, + suggestions: [], + error: null, + messages: [], + sessionId: generateSessionId(), + }), +})); \ No newline at end of file diff --git a/packages/widget/src/styles/widget.css b/packages/widget/src/styles/widget.css new file mode 100644 index 0000000..6a81a2e --- /dev/null +++ b/packages/widget/src/styles/widget.css @@ -0,0 +1,509 @@ +/* AI Search Widget Styles */ +.ai-search-widget { + --primary-color: #007bff; + --secondary-color: #6c757d; + --background-color: #ffffff; + --text-color: #212529; + --border-radius: 8px; + --font-family: system-ui, -apple-system, sans-serif; + --font-size: 14px; + --shadow-sm: 0 2px 4px rgba(0, 0, 0, 0.1); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.15); + --shadow-lg: 0 8px 32px rgba(0, 0, 0, 0.12); +} + +/* Reset and base styles */ +.ai-search-widget * { + box-sizing: border-box; +} + +.ai-search-widget button { + font-family: inherit; + font-size: inherit; +} + +.ai-search-widget input { + font-family: inherit; + font-size: inherit; +} + +/* Widget container */ +.ai-search-widget-container { + position: relative; + z-index: 999999; +} + +/* Floating button */ +.ai-search-button-floating { + position: fixed; + width: 60px; + height: 60px; + border-radius: 50%; + border: none; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s ease; + box-shadow: var(--shadow-md); +} + +.ai-search-button-floating:hover { + transform: scale(1.1); + box-shadow: var(--shadow-lg); +} + +.ai-search-button-floating:active { + transform: scale(0.95); +} + +/* Widget container */ +.ai-search-container { + position: fixed; + background: var(--background-color); + border: 1px solid #e1e5e9; + border-radius: var(--border-radius); + box-shadow: var(--shadow-lg); + font-family: var(--font-family); + font-size: var(--font-size); + color: var(--text-color); + overflow: hidden; + transition: all 0.3s ease; +} + +.ai-search-container-minimized { + height: auto !important; +} + +/* Header */ +.ai-search-header { + background: var(--primary-color); + color: white; + padding: 12px 16px; + display: flex; + align-items: center; + justify-content: space-between; + border-radius: var(--border-radius) var(--border-radius) 0 0; +} + +.ai-search-title { + font-weight: 600; + font-size: 16px; +} + +.ai-search-controls { + display: flex; + gap: 8px; +} + +.ai-search-control-btn { + background: none; + border: none; + color: white; + cursor: pointer; + padding: 4px; + border-radius: 4px; + display: flex; + align-items: center; + justify-content: center; + opacity: 0.8; + transition: opacity 0.2s ease; +} + +.ai-search-control-btn:hover { + opacity: 1; + background: rgba(255, 255, 255, 0.1); +} + +/* Content area */ +.ai-search-content { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +/* Initial view */ +.ai-search-initial { + padding: 24px 16px; + display: flex; + flex-direction: column; + gap: 16px; + height: 100%; +} + +.ai-search-welcome { + font-size: 16px; + color: var(--text-color); + text-align: center; + margin-bottom: 8px; + line-height: 1.4; +} + +/* Search box */ +.ai-search-box-container { + position: relative; +} + +.ai-search-input-wrapper { + display: flex; + align-items: center; + border: 2px solid #e1e5e9; + border-radius: var(--border-radius); + padding: 12px 16px; + background: white; + transition: border-color 0.2s ease; +} + +.ai-search-input-wrapper:focus-within { + border-color: var(--primary-color); +} + +.ai-search-input { + flex: 1; + border: none; + outline: none; + background: transparent; + font-size: inherit; + font-family: inherit; + color: inherit; +} + +.ai-search-input::placeholder { + color: #9ca3af; +} + +/* Suggestions */ +.ai-search-suggestions { + position: absolute; + top: 100%; + left: 0; + right: 0; + background: white; + border: 1px solid #e1e5e9; + border-radius: var(--border-radius); + box-shadow: var(--shadow-md); + z-index: 1000; + max-height: 200px; + overflow-y: auto; + margin-top: 4px; +} + +.ai-search-suggestion { + padding: 12px 16px; + cursor: pointer; + border-bottom: 1px solid #f1f3f4; + display: flex; + align-items: center; + font-size: inherit; + color: inherit; + transition: background-color 0.1s ease; +} + +.ai-search-suggestion:last-child { + border-bottom: none; +} + +.ai-search-suggestion:hover, +.ai-search-suggestion-selected { + background-color: #f8f9fa; +} + +/* Chat interface */ +.ai-search-chat { + display: flex; + flex-direction: column; + height: 100%; +} + +.ai-search-messages { + flex: 1; + overflow-y: auto; + padding: 16px; + display: flex; + flex-direction: column; + gap: 16px; +} + +.ai-search-input-area { + padding: 16px; + border-top: 1px solid #e1e5e9; + background: #f8f9fa; +} + +/* Messages */ +.ai-search-message { + display: flex; + align-items: flex-start; + gap: 12px; +} + +.ai-search-message-user { + flex-direction: row-reverse; +} + +.ai-search-avatar { + width: 32px; + height: 32px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + color: white; + flex-shrink: 0; +} + +.ai-search-message-content { + max-width: 80%; + display: flex; + flex-direction: column; + gap: 8px; +} + +.ai-search-bubble { + padding: 12px 16px; + border-radius: 18px; + line-height: 1.4; + word-wrap: break-word; +} + +.ai-search-message-user .ai-search-bubble { + background: var(--primary-color); + color: white; + border-radius: 18px 18px 4px 18px; +} + +.ai-search-message-assistant .ai-search-bubble { + background: #f1f3f4; + color: var(--text-color); + border-radius: 18px 18px 18px 4px; +} + +/* Sources */ +.ai-search-sources { + display: flex; + flex-direction: column; + gap: 4px; +} + +.ai-search-source { + background: none; + border: 1px solid #e1e5e9; + border-radius: 8px; + padding: 8px 12px; + cursor: pointer; + text-align: left; + font-size: 12px; + display: flex; + align-items: center; + justify-content: space-between; + transition: all 0.2s ease; +} + +.ai-search-source:hover:not(:disabled) { + border-color: var(--primary-color); + background: #f8f9fa; +} + +.ai-search-source:disabled { + cursor: default; + opacity: 0.6; +} + +/* Timestamp */ +.ai-search-timestamp { + font-size: 11px; + color: var(--secondary-color); + opacity: 0.7; +} + +/* Feedback */ +.ai-search-feedback { + display: flex; + gap: 8px; + margin-top: 8px; +} + +.ai-search-feedback-btn { + background: none; + border: 1px solid #e1e5e9; + border-radius: 16px; + padding: 4px 8px; + cursor: pointer; + display: flex; + align-items: center; + gap: 4px; + font-size: 12px; + color: var(--secondary-color); + transition: all 0.2s ease; +} + +.ai-search-feedback-btn:hover { + border-color: var(--primary-color); + color: var(--primary-color); +} + +/* Results */ +.ai-search-results { + padding: 16px; + border-top: 1px solid #e1e5e9; + max-height: 300px; + overflow-y: auto; +} + +.ai-search-results-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 12px; + font-size: 14px; + color: var(--secondary-color); +} + +.ai-search-results-list { + display: flex; + flex-direction: column; + gap: 12px; +} + +.ai-search-result-item { + position: relative; + border: 1px solid #e1e5e9; + border-radius: var(--border-radius); + padding: 12px; + background: white; + transition: all 0.2s ease; +} + +.ai-search-result-clickable { + cursor: pointer; +} + +.ai-search-result-clickable:hover { + border-color: var(--primary-color); + box-shadow: var(--shadow-sm); +} + +.ai-search-result-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + margin-bottom: 8px; +} + +.ai-search-result-title { + margin: 0; + font-size: 16px; + font-weight: 600; + line-height: 1.3; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.ai-search-result-url { + font-size: 12px; + color: var(--secondary-color); + margin-top: 2px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.ai-search-result-meta { + display: flex; + align-items: center; + gap: 8px; + margin-left: 12px; + flex-shrink: 0; +} + +.ai-search-result-score { + font-size: 12px; + color: var(--secondary-color); + padding: 2px 6px; + background: #f1f3f4; + border-radius: 4px; +} + +.ai-search-result-content { + font-size: 14px; + line-height: 1.4; + overflow: hidden; + display: -webkit-box; + -webkit-line-clamp: 3; + -webkit-box-orient: vertical; +} + +.ai-search-result-position { + position: absolute; + top: 8px; + left: 8px; + width: 20px; + height: 20px; + background: var(--primary-color); + color: white; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + font-size: 11px; + font-weight: 600; +} + +/* Load more button */ +.ai-search-load-more { + width: 100%; + padding: 12px; + margin-top: 16px; + border: 1px solid var(--primary-color); + border-radius: var(--border-radius); + background: transparent; + color: var(--primary-color); + cursor: pointer; + transition: all 0.2s ease; +} + +.ai-search-load-more:hover { + background: var(--primary-color); + color: white; +} + +/* Error states */ +.ai-search-error { + padding: 16px; + text-align: center; + color: #dc3545; +} + +/* Footer */ +.ai-search-footer { + padding: 8px 16px; + border-top: 1px solid #e1e5e9; + font-size: 12px; + color: var(--secondary-color); + text-align: center; +} + +/* Animations */ +@keyframes spin { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } +} + +/* Responsive */ +@media (max-width: 480px) { + .ai-search-container { + width: calc(100vw - 20px) !important; + height: calc(100vh - 20px) !important; + top: 10px !important; + left: 10px !important; + right: 10px !important; + bottom: 10px !important; + } + + .ai-search-button-floating { + width: 50px; + height: 50px; + } +} \ No newline at end of file diff --git a/packages/widget/src/utils/api.ts b/packages/widget/src/utils/api.ts new file mode 100644 index 0000000..828c94e --- /dev/null +++ b/packages/widget/src/utils/api.ts @@ -0,0 +1,196 @@ +import { + SearchQuery, + SearchResponse, + AutoCompleteRequest, + AutoCompleteResponse, + AnalyticsEvent +} from '@ai-search/shared'; + +const DEFAULT_API_BASE_URL = 'https://api.aisearch.com/v1'; + +interface RequestOptions { + signal?: AbortSignal; + timeout?: number; +} + +class SearchAPI { + private baseUrl: string; + + constructor(baseUrl: string = DEFAULT_API_BASE_URL) { + this.baseUrl = baseUrl; + } + + private async request( + endpoint: string, + options: RequestInit & { apiKey: string }, + requestOptions?: RequestOptions + ): Promise { + const { apiKey, ...fetchOptions } = options; + + const controller = new AbortController(); + const timeoutId = requestOptions?.timeout + ? setTimeout(() => controller.abort(), requestOptions.timeout) + : null; + + try { + const response = await fetch(`${this.baseUrl}${endpoint}`, { + ...fetchOptions, + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + 'X-Widget-Version': '1.0.0', + ...fetchOptions.headers, + }, + signal: requestOptions?.signal || controller.signal, + }); + + if (timeoutId) { + clearTimeout(timeoutId); + } + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error?.message || `HTTP ${response.status}: ${response.statusText}`); + } + + const data = await response.json(); + return data.data || data; + } catch (error) { + if (timeoutId) { + clearTimeout(timeoutId); + } + throw error; + } + } + + async search( + query: SearchQuery, + apiKey: string, + options?: RequestOptions + ): Promise { + return this.request('/search', { + method: 'POST', + body: JSON.stringify(query), + apiKey, + }, options); + } + + async autocomplete( + request: AutoCompleteRequest, + apiKey: string, + options?: RequestOptions + ): Promise { + return this.request('/autocomplete', { + method: 'POST', + body: JSON.stringify(request), + apiKey, + }, options); + } + + async trackEvent( + event: AnalyticsEvent, + apiKey: string, + options?: RequestOptions + ): Promise { + try { + await this.request('/analytics/events', { + method: 'POST', + body: JSON.stringify(event), + apiKey, + }, options); + } catch (error) { + // Silently fail for analytics to not break user experience + console.warn('Failed to track analytics event:', error); + } + } + + async submitFeedback( + feedback: { + query?: string; + documentId?: string; + enhancedAnswerId?: string; + rating: number; + comment?: string; + }, + apiKey: string, + options?: RequestOptions + ): Promise { + return this.request('/feedback', { + method: 'POST', + body: JSON.stringify(feedback), + apiKey, + }, options); + } +} + +export const searchApi = new SearchAPI(); + +// Analytics helper +export class AnalyticsTracker { + private apiKey: string; + private sessionId: string; + private userId?: string; + private eventQueue: AnalyticsEvent[] = []; + private flushTimer?: NodeJS.Timeout; + + constructor(apiKey: string, sessionId: string, userId?: string) { + this.apiKey = apiKey; + this.sessionId = sessionId; + this.userId = userId; + } + + track(eventType: AnalyticsEvent['eventType'], data: Record) { + const event: AnalyticsEvent = { + id: this.generateEventId(), + sessionId: this.sessionId, + userId: this.userId, + timestamp: new Date(), + eventType, + data, + metadata: { + userAgent: navigator.userAgent, + referrer: document.referrer, + widgetVersion: '1.0.0', + }, + }; + + this.eventQueue.push(event); + this.scheduleFlush(); + } + + private generateEventId(): string { + return 'evt_' + Math.random().toString(36).substr(2, 9) + Date.now().toString(36); + } + + private scheduleFlush() { + if (this.flushTimer) { + clearTimeout(this.flushTimer); + } + + this.flushTimer = setTimeout(() => { + this.flush(); + }, 1000); + } + + private async flush() { + if (this.eventQueue.length === 0) return; + + const events = [...this.eventQueue]; + this.eventQueue = []; + + try { + await Promise.all( + events.map(event => searchApi.trackEvent(event, this.apiKey)) + ); + } catch (error) { + console.warn('Failed to flush analytics events:', error); + } + } + + destroy() { + if (this.flushTimer) { + clearTimeout(this.flushTimer); + } + this.flush(); + } +} \ No newline at end of file diff --git a/packages/widget/tsconfig.json b/packages/widget/tsconfig.json new file mode 100644 index 0000000..b9a514d --- /dev/null +++ b/packages/widget/tsconfig.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "allowJs": false, + "skipLibCheck": true, + "esModuleInterop": false, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "module": "ESNext", + "moduleResolution": "node", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "declaration": true, + "declarationMap": true, + "outDir": "dist" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../shared" + } + ] +} \ No newline at end of file diff --git a/packages/widget/vite.config.ts b/packages/widget/vite.config.ts new file mode 100644 index 0000000..9738cf5 --- /dev/null +++ b/packages/widget/vite.config.ts @@ -0,0 +1,33 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; +import dts from 'vite-plugin-dts'; +import { resolve } from 'path'; + +export default defineConfig({ + plugins: [ + react(), + dts({ + insertTypesEntry: true, + }), + ], + build: { + lib: { + entry: resolve(__dirname, 'src/index.ts'), + name: 'AISearchWidget', + formats: ['es', 'umd'], + fileName: (format) => `ai-search-widget.${format}.js`, + }, + rollupOptions: { + external: ['react', 'react-dom'], + output: { + globals: { + react: 'React', + 'react-dom': 'ReactDOM', + }, + }, + }, + }, + define: { + 'process.env.NODE_ENV': JSON.stringify(process.env.NODE_ENV || 'development'), + }, +}); \ No newline at end of file diff --git a/turbo.json b/turbo.json new file mode 100644 index 0000000..379bd5f --- /dev/null +++ b/turbo.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://turbo.build/schema.json", + "globalDependencies": ["**/.env.*local"], + "pipeline": { + "build": { + "dependsOn": ["^build"], + "outputs": ["dist/**", ".next/**", "!.next/cache/**"] + }, + "dev": { + "cache": false, + "persistent": true + }, + "test": { + "dependsOn": ["build"], + "outputs": ["coverage/**"] + }, + "lint": { + "dependsOn": ["^lint"] + }, + "type-check": { + "dependsOn": ["^type-check"] + }, + "clean": { + "cache": false + } + } +} \ No newline at end of file