Build ai powered website search platform

.env.example

@@ -0,0 +1,83 @@
+# Environment Configuration
+NODE_ENV=development
+DEBUG=true
+ENVIRONMENT=development
+
+# API Configuration
+API_HOST=0.0.0.0
+API_PORT=8000
+SECRET_KEY=your-secret-key-change-in-production
+
+# Database Configuration
+DATABASE_URL=postgresql+asyncpg://aisearch:aisearch_password@localhost:5432/aisearch
+DATABASE_POOL_SIZE=10
+DATABASE_MAX_OVERFLOW=20
+
+# Redis Configuration
+REDIS_URL=redis://localhost:6379
+REDIS_MAX_CONNECTIONS=10
+
+# OpenAI Configuration
+OPENAI_API_KEY=your-openai-api-key
+OPENAI_MODEL=gpt-4-turbo-preview
+OPENAI_EMBEDDING_MODEL=text-embedding-ada-002
+
+# Anthropic Configuration
+ANTHROPIC_API_KEY=your-anthropic-api-key
+ANTHROPIC_MODEL=claude-3-sonnet-20240229
+
+# HuggingFace Configuration
+HUGGINGFACE_API_KEY=your-huggingface-api-key
+HUGGINGFACE_MODEL=sentence-transformers/all-MiniLM-L6-v2
+
+# Cohere Configuration
+COHERE_API_KEY=your-cohere-api-key
+COHERE_MODEL=embed-english-v2.0
+
+# Vector Database Configuration (Milvus)
+MILVUS_HOST=localhost
+MILVUS_PORT=19530
+MILVUS_USER=
+MILVUS_PASSWORD=
+
+# Vector Database Configuration (Pinecone)
+PINECONE_API_KEY=your-pinecone-api-key
+PINECONE_ENVIRONMENT=your-pinecone-environment
+PINECONE_INDEX_NAME=ai-search
+
+# Vector Database Configuration (Weaviate)
+WEAVIATE_URL=http://localhost:8080
+WEAVIATE_API_KEY=your-weaviate-api-key
+WEAVIATE_CLASS_NAME=Document
+
+# Vector Database Configuration (Qdrant)
+QDRANT_URL=http://localhost:6333
+QDRANT_API_KEY=your-qdrant-api-key
+QDRANT_COLLECTION_NAME=documents
+
+# Vector Database Selection
+VECTOR_DB_PROVIDER=milvus
+VECTOR_DB_DIMENSIONS=1536
+
+# Content Processing
+MAX_DOCUMENT_SIZE=52428800  # 50MB
+CHUNK_SIZE=1000
+CHUNK_OVERLAP=200
+
+# Rate Limiting
+RATE_LIMIT_REQUESTS=100
+RATE_LIMIT_WINDOW=60
+
+# Celery Configuration
+CELERY_BROKER_URL=redis://localhost:6379/0
+CELERY_RESULT_BACKEND=redis://localhost:6379/0
+
+# Monitoring
+SENTRY_DSN=your-sentry-dsn
+LOG_LEVEL=INFO
+
+# File Storage
+UPLOAD_DIR=uploads
+
+# CORS Configuration
+ALLOWED_ORIGINS=["http://localhost:3000","http://localhost:8080","https://yourdomain.com"]

.github/workflows/ci-cd.yml

@@ -0,0 +1,200 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main ]
+
+env:
+  NODE_VERSION: '18'
+  PYTHON_VERSION: '3.11'
+
+jobs:
+  # Frontend Tests
+  frontend-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run linting
+        run: npm run lint
+
+      - name: Run type checking
+        run: npm run type-check
+
+      - name: Run tests
+        run: npm test
+
+      - name: Build packages
+        run: npm run build
+
+  # Backend Tests
+  backend-tests:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test_aisearch
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        working-directory: ./packages/api
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run linting
+        working-directory: ./packages/api
+        run: |
+          flake8 .
+          black --check .
+          mypy .
+
+      - name: Run tests
+        working-directory: ./packages/api
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_aisearch
+          REDIS_URL: redis://localhost:6379
+          SECRET_KEY: test-secret-key
+        run: pytest tests/ -v --cov=app --cov-report=xml
+
+      - name: Upload coverage
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./packages/api/coverage.xml
+          flags: backend
+
+  # Security Scanning
+  security-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+
+      - name: Upload Trivy scan results
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+  # Build and Deploy
+  build-deploy:
+    needs: [frontend-tests, backend-tests]
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push API image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./packages/api
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}/api:latest
+            ghcr.io/${{ github.repository }}/api:${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build and push Demo image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./apps/demo
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}/demo:latest
+            ghcr.io/${{ github.repository }}/demo:${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Deploy to staging
+        if: github.ref == 'refs/heads/main'
+        run: |
+          echo "Deploying to staging environment..."
+          # Add your deployment commands here
+          # e.g., kubectl apply, terraform apply, etc.
+
+      - name: Run integration tests
+        run: |
+          echo "Running integration tests..."
+          # Add integration test commands here
+
+      - name: Deploy to production
+        if: github.ref == 'refs/heads/main'
+        run: |
+          echo "Deploying to production environment..."
+          # Add production deployment commands here
+
+  # Performance Tests
+  performance-tests:
+    needs: [build-deploy]
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run load tests
+        run: |
+          echo "Running performance tests..."
+          # Add load testing commands here (e.g., k6, artillery)
+
+      - name: Upload performance results
+        uses: actions/upload-artifact@v3
+        with:
+          name: performance-results
+          path: performance-results/