diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..920c08b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +# Nothing is COPY'd into the image — the Dockerfile clones from GitHub. +# This file is kept as a safeguard in case a COPY is added later. + +# Secrets — never bake certs or keys into the image +secrets/ + +# OS +.DS_Store +Thumbs.db diff --git a/.gitignore b/.gitignore index d4bb158..9a4722b 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ experiments/ # Local databases NAS_Database/ Django_database/ +.claudeignore diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..05fc2ba --- /dev/null +++ b/Dockerfile @@ -0,0 +1,87 @@ +# ============================================================ +# SpikesortingLabHub — Production Image +# +# Self-contained: downloads the repo from GitHub so anyone can +# reproduce this image without needing the local source tree. +# ============================================================ + +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# ------------------------------------------------------------ +# 1. System packages +# ------------------------------------------------------------ +RUN apt-get update && apt-get upgrade -y --no-install-recommends +RUN apt-get install -y --no-install-recommends \ + python3 \ + python3-venv \ + python3-pip \ + build-essential \ + libpq-dev \ + openssl \ + ca-certificates \ + curl \ + git \ + nodejs \ + npm \ + wget \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# ------------------------------------------------------------ +# 2. Download repository (docker branch) and unpack +# ------------------------------------------------------------ +WORKDIR /app +RUN wget -q https://github.com/UserFriendlySpikesorting/SpikesortingLabHub-server/archive/refs/heads/developing_branch.zip \ + && unzip -q developing_branch.zip \ + && mv SpikesortingLabHub-server-developing_branch/* . \ + && rm -rf developing_branch.zip SpikesortingLabHub-server-developing_branch + +# ------------------------------------------------------------ +# 3. Set up Python virtual environment and install requirements +# ------------------------------------------------------------ +RUN python3 -m venv /app/venv +RUN /app/venv/bin/pip install --no-cache-dir -r requirements.txt + +# Install SpikesortingLabHub-CLI (patch setup.py for Python 3.12) +RUN wget https://github.com/UserFriendlySpikesorting/SpikesortingLabHub-CLI/archive/refs/heads/main.zip \ + && unzip main.zip \ + && sed -i 's|3.13|3.12|' SpikesortingLabHub-CLI-main/setup.py \ + && /app/venv/bin/pip install --no-cache-dir SpikesortingLabHub-CLI-main/ + +ENV PATH="/app/venv/bin:$PATH" + +# ------------------------------------------------------------ +# 4. Build the React frontend +# my-app/build/ is gitignored so it must be compiled here. +# ------------------------------------------------------------ +RUN cd my-app && npm ci --omit=dev && npm run build + +# ------------------------------------------------------------ +# 5. Placeholder directories for bind mounts +# Docker Compose overlays the real host paths at runtime. +# /data ← trurnasdata (read-only NAS database) +# /django_db ← persistentdata (Django SQLite DB + logs) +# /experiments ← binary recording files (read-only) +# ------------------------------------------------------------ +RUN mkdir -p /data /app/django_db /app/experiments /app/secrets + +# ------------------------------------------------------------ +# 6. Entrypoint — already in the repo after download. +# Runs pre-flight checks, collectstatic, migrate, then Gunicorn. +# ------------------------------------------------------------ +RUN chmod +x /app/entrypoint.sh +ENTRYPOINT ["/app/entrypoint.sh"] + +# ------------------------------------------------------------ +# 7. Expose ports +# 9000 — plain HTTP +# 9443 — HTTPS (Gunicorn with --certfile / --keyfile) +# ------------------------------------------------------------ +EXPOSE 9000 9443 + +# ------------------------------------------------------------ +# 8. Default command — passed to entrypoint.sh via exec "$@". +# ------------------------------------------------------------ +CMD ["gunicorn", "-c", "gunicorn.conf.py", "labhub.wsgi:application"] diff --git a/INSTRUCTIONS.md b/INSTRUCTIONS.md new file mode 100644 index 0000000..92be5ac --- /dev/null +++ b/INSTRUCTIONS.md @@ -0,0 +1,108 @@ +# SpikesortingLabHub — TrueNAS Deployment Instructions + +Steps tested on the test rig. Replicate these exactly on the main TrueNAS. + +> **Path difference:** test rig used `user_home`, main TrueNAS uses `users`. +> Every path below already reflects the main TrueNAS convention. + +--- + +## 1. Enable SSH on TrueNAS + +TrueNAS UI → System → Services → start **SSH** and set it to auto-start. + +--- + +## 2. Copy the init script from your Mac to TrueNAS + +Run from your Mac terminal (not SSH): + +```bash +scp /Users/kajalpatel/SpikesortingLabHub-server/truenas_init.sh \ + kajal@128.164.33.182:/mnt/root_data_storage/users/kajal/ +``` + +--- + +## 3. SSH into TrueNAS + +```bash +ssh truenas_admin@128.164.33.182 +``` + +--- + +## 4. Move the script into the sslh folder + +```bash +sudo cp /mnt/root_data_storage/users/kajal/truenas_init.sh \ + /mnt/root_data_storage/users/sslh/truenas_init.sh + +sudo chmod +x /mnt/root_data_storage/users/sslh/truenas_init.sh +``` + +--- + +## 5. Test-run the script manually + +```bash +sudo /mnt/root_data_storage/users/sslh/truenas_init.sh +``` + +Check the log it generates: + +```bash +cat /mnt/root_data_storage/users/sslh/sslh_init.log +``` + +--- + +## 6. Pull the Docker image and start the container (first time only) + +```bash +docker pull ikajalpatel21/spikesorting-labhub-latestimg:latest + +export DJANGO_SECRET_KEY="$(cat /mnt/root_data_storage/users/sslh/secrets/django_secret.key)" + +docker compose -f /mnt/root_data_storage/users/sslh/docker-compose.yml up -d +``` + +> If `secrets/django_secret.key` does not exist yet, generate it first: +> ```bash +> mkdir -p /mnt/root_data_storage/users/sslh/secrets +> openssl rand -hex 50 > /mnt/root_data_storage/users/sslh/secrets/django_secret.key +> chmod 600 /mnt/root_data_storage/users/sslh/secrets/django_secret.key +> ``` + +--- + +## 7. Verify the container is running + +```bash +docker ps +docker logs spikesorting-labhub-server-spikesorting-labhub-1 +``` + +Open in browser: `https://128.164.33.182:9443` +(self-signed cert warning is expected — click through) + +--- + +## 8. Register the init script in TrueNAS UI + +System → Advanced Settings → Init/Shutdown Scripts → **Add** + +| Field | Value | +|---------|---------------------------------------------------------| +| Type | Script | +| Script | `/mnt/root_data_storage/users/sslh/truenas_init.sh` | +| When | Pre Init | +| Timeout | 30 | + +--- + +## What happens on every reboot after this + +1. TrueNAS kernel starts → ZFS pool mounts automatically +2. Pre Init script runs → verifies/creates bind-mount directories → bind-mounts experiments and trurnasdata +3. Docker daemon starts → `restart: unless-stopped` brings the container back up with all mounts in place diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 3cdaf51..0000000 --- a/deploy.sh +++ /dev/null @@ -1,299 +0,0 @@ -#!/bin/bash - -# ============================================================================= -# QModel Django Deployment Script -# ============================================================================= -# This script sets up the complete environment for the qmodel branch including: -# - GitHub repository setup -# - Virtual environment creation -# - Dependencies installation -# - Database initialization -# - SSL certificate generation -# - Static files collection -# - Server startup options -# ============================================================================= - -set -e # Exit on any error - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Configuration -REPO_URL="https://github.com/iKajalpatel21/spikesorting-labhub-try-error.git" -BRANCH_NAME="qmodel" -PROJECT_DIR="spikesorting-labhub-try-error" -VENV_NAME=".djangovenv" - -# Function to print colored output -print_status() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -print_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} - -print_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# Function to check if command exists -command_exists() { - command -v "$1" >/dev/null 2>&1 -} - -# ============================================================================= -# 1. System Requirements Check -# ============================================================================= -print_status "Checking system requirements..." - -if ! command_exists python3; then - print_error "Python3 is not installed. Please install Python3 first." - exit 1 -fi - -if ! command_exists git; then - print_error "Git is not installed. Please install Git first." - exit 1 -fi - -if ! command_exists openssl; then - print_error "OpenSSL is not installed. Please install OpenSSL first." - exit 1 -fi - -print_success "All system requirements are met." - -# ============================================================================= -# 2. Repository Setup -# ============================================================================= -print_status "Setting up repository..." - -# Check if we're already in the project directory -if [[ $(basename "$PWD") == "$PROJECT_DIR" ]]; then - print_status "Already in project directory. Pulling latest changes..." - git fetch origin - git checkout $BRANCH_NAME || git checkout -b $BRANCH_NAME origin/$BRANCH_NAME - git pull origin $BRANCH_NAME -else - # Check if project directory exists - if [ -d "$PROJECT_DIR" ]; then - print_status "Project directory exists. Updating..." - cd "$PROJECT_DIR" - git fetch origin - git checkout $BRANCH_NAME || git checkout -b $BRANCH_NAME origin/$BRANCH_NAME - git pull origin $BRANCH_NAME - else - print_status "Cloning repository..." - git clone -b $BRANCH_NAME $REPO_URL $PROJECT_DIR - cd "$PROJECT_DIR" - fi -fi - -print_success "Repository setup complete." - -# ============================================================================= -# 3. Virtual Environment Setup -# ============================================================================= -print_status "Setting up Python virtual environment..." - -# Create virtual environment if it doesn't exist -if [ ! -d "$VENV_NAME" ]; then - python3 -m venv $VENV_NAME || { - print_error "Failed to create virtual environment" - exit 1 - } -fi - -# Activate virtual environment -source $VENV_NAME/bin/activate || { - print_error "Failed to activate virtual environment" - exit 1 -} - -print_success "Virtual environment activated." - -# ============================================================================= -# 4. Dependencies Installation -# ============================================================================= -print_status "Installing/updating dependencies..." - -# Update pip -pip install -U pip || { - print_error "Failed to update pip" - exit 1 -} - -# Install requirements -if [ -f "requirements.txt" ]; then - pip install -U -r requirements.txt || { - print_error "Failed to install requirements from requirements.txt" - exit 1 - } -else - # Fallback to manual installation - print_warning "requirements.txt not found. Installing core dependencies manually..." - pip install -U django djangorestframework requests gunicorn urllib3 || { - print_error "Failed to install core dependencies" - exit 1 - } -fi - -print_success "Dependencies installed successfully." - -# ============================================================================= -# 5. Database Setup -# ============================================================================= -print_status "Setting up database..." - -# Ask user if they want to reset the database -read -p "Do you want to reset the database? This will clear all existing data. (y/N): " -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]]; then - print_warning "Resetting database..." - echo -n > db.sqlite3 -fi - -# Run migrations -print_status "Running database migrations..." -python manage.py makemigrations || { - print_error "Failed to create migrations" - exit 1 -} - -python manage.py migrate || { - print_error "Failed to run migrations" - exit 1 -} - -print_success "Database setup complete." - -# ============================================================================= -# 6. Create Superuser (Optional) -# ============================================================================= -read -p "Do you want to create a superuser? (y/N): " -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]]; then - print_status "Creating superuser..." - python manage.py createsuperuser --username admin || { - print_warning "Superuser creation failed or was skipped" - } -fi - -# ============================================================================= -# 7. SSL Certificate Generation -# ============================================================================= -print_status "Checking SSL certificates..." - -mkdir -p secrets -if [ ! -f "secrets/cert.crt" ] || [ ! -f "secrets/cert.key" ]; then - print_status "Generating self-signed SSL certificate..." - # Use the machine's actual IP/hostname as CN so clients can verify it. - # SAN (subjectAltName) covers both the hostname and IP address. - SERVER_HOST=$(hostname -f 2>/dev/null || hostname) - SERVER_IP=$(hostname -I 2>/dev/null | awk '{print $1}') - print_status "Certificate CN: ${SERVER_HOST} (IP: ${SERVER_IP})" - - openssl req -x509 -newkey rsa:4096 -keyout secrets/cert.key -out secrets/cert.crt -days 365 -nodes \ - -subj "/CN=${SERVER_HOST}" \ - -addext "subjectAltName=DNS:${SERVER_HOST},IP:${SERVER_IP},DNS:localhost,IP:127.0.0.1" || { - print_error "Failed to generate SSL certificates" - exit 1 - } - print_success "SSL certificate generated: secrets/cert.crt / secrets/cert.key (CN=${SERVER_HOST})" -else - print_success "SSL certificates already exist (secrets/cert.crt / secrets/cert.key)." -fi - -# ============================================================================= -# 8. Static Files Collection -# ============================================================================= -print_status "Collecting static files..." -yes yes | python manage.py collectstatic || { - print_warning "Static files collection failed or was skipped" -} - -print_success "Static files collected." - -# ============================================================================= -# 9. Server Startup Options -# ============================================================================= -print_success "Deployment complete! Choose how to run the server:" -echo -echo "Available options:" -echo "1) Development server (HTTP on port 8000)" -echo "2) Gunicorn server (HTTP on port 8000)" -echo "3) Gunicorn server with HTTPS (port 8443)" -echo "4) Just setup - don't start server" -echo "5) Start worker only" -echo - -read -p "Enter your choice (1-5): " -n 1 -r -echo - -case $REPLY in - 1) - print_status "Starting Django development server..." - python manage.py runserver - ;; - 2) - print_status "Starting Gunicorn HTTP server..." - gunicorn -c gunicorn.conf.py labhub.wsgi:application - ;; - 3) - print_status "Starting Gunicorn HTTPS server (port 443)..." - gunicorn -c gunicorn.conf.py \ - --certfile=secrets/cert.crt \ - --keyfile=secrets/cert.key \ - -b 0.0.0.0:443 \ - labhub.wsgi:application - ;; - 4) - print_success "Setup complete. You can manually start the server when ready." - echo - echo "To start the development server: python manage.py runserver" - echo "To start Gunicorn HTTP: gunicorn -c gunicorn.conf.py labhub.wsgi:application" - echo "To start Gunicorn HTTPS: gunicorn -c gunicorn.conf.py --certfile=secrets/cert.crt --keyfile=secrets/cert.key -b 0.0.0.0:443 labhub.wsgi:application" - echo "To start worker: python qmodel_worker.py" - ;; - 5) - print_status "Starting qmodel worker..." - python qmodel_worker.py - ;; - *) - print_warning "Invalid choice. Setup complete but no server started." - ;; -esac - -# ============================================================================= -# 10. Final Instructions -# ============================================================================= -echo -print_success "=== Deployment Summary ===" -echo "Project: $PROJECT_DIR" -echo "Branch: $BRANCH_NAME" -echo "Virtual Environment: $VENV_NAME" -echo "Database: SQLite (db.sqlite3)" -echo "SSL Certificates: cert.pem, key.pem" -echo -echo "=== Usage Instructions ===" -echo "• HTTP — Django admin: http://localhost:8000/admin/" -echo "• HTTPS — Django admin: https://localhost:443/admin/" -echo "• HTTP — Worker fetch: http://localhost:8000/job-queue/next-job/" -echo "• HTTPS — Worker fetch: https://localhost:443/job-queue/next-job/" -echo "• Worker (HTTP): python qmodel_worker.py" -echo "• Worker (HTTPS): LABHUB_BASE_URL=https://localhost LABHUB_SSL_VERIFY=false python qmodel_worker.py" -echo -echo "=== Multiple Terminal Setup ===" -echo "Terminal 1 (Server): ./deploy.sh (choose option 2 or 3)" -echo "Terminal 2 (Worker): source $VENV_NAME/bin/activate && python qmodel_worker.py" -echo -print_success "Setup complete! Enjoy your deployment and run the server as needed." diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c06e25f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,73 @@ +# ============================================================ +# SpikesortingLabHub — Docker Compose +# +# Bind-mount layout (host path → container path): +# +# trurnasdata /mnt/root_data_storage/users/sslh/trurnasdata → /data (read-only — NAS database) +# persistentdata /mnt/root_data_storage/users/sslh/persistentdata → /django_db (read/write — Django DB + logs) +# experiments /mnt/root_data_storage/experiments → /experiments (read-only — binary recordings) +# +# Ports 8000, 8080, and 8443 are used by the NAS itself — avoid them. +# +# Usage: +# export DJANGO_SECRET_KEY="your-secret-key" +# docker compose build +# docker compose up -d +# (entrypoint.sh generates SSL certs automatically on first start) +# ============================================================ + +services: + spikesorting-labhub: + build: . + image: spikesorting-labhub:latest + restart: unless-stopped + + command: + - gunicorn + - -c + - gunicorn.conf.py + - --certfile=/app/secrets/cert.crt + - --keyfile=/app/secrets/cert.key + - -b + - 0.0.0.0:9443 + - labhub.wsgi:application + + ports: + - "9443:9443" # HTTPS — Gunicorn binds here via command: override above + + environment: + # Django core + DJANGO_SECRET_KEY: "${DJANGO_SECRET_KEY}" + DJANGO_DEBUG: "False" + + # Database lives on the persistentdata mount + DATABASE_PATH: "/django_db/db.sqlite3" + + # Tell Django where the NAS root is so $NAS$ placeholders resolve + NAS_ROOT: "/data" + + # Scan these directories for .bin / .prb data files + DATA_DIRS: "/experiments,/experiments/probes" + + volumes: + # trurnasdata — NAS database (read-only) + - type: bind + source: /mnt/root_data_storage/users/sslh/trurnasdata + target: /data + read_only: true + + # persistentdata — Django SQLite DB + logs (read/write) + - type: bind + source: /mnt/root_data_storage/users/sslh/persistentdata + target: /django_db + + # experiments — binary recording files (read-only) + - type: bind + source: /mnt/root_data_storage/experiments + target: /experiments + read_only: true + + # secrets — SSL certs generated by entrypoint.sh on first start (read-write) + - type: bind + source: /mnt/root_data_storage/users/sslh/secrets + target: /app/secrets diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..089125f --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -e + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +ok() { echo -e "${GREEN}[OK]${NC} $1"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +info() { echo "$1"; } + + +# ============================================================================= +# 1. Check the database directory (mounted from persistentdata on the host) +# ============================================================================= +DB_PATH="${DATABASE_PATH:-/app/django_db/db.sqlite3}" +DB_DIR="$(dirname "$DB_PATH")" + +# mkdir -p "$DB_DIR" + +if [ -f "$DB_PATH" ]; then + ok "SQLite database exists: $DB_PATH" +else + warn "SQLite database not found: $DB_PATH" + info "Creating SQLite DB via Django migrations..." + + python manage.py migrate --noinput + + ok "SQLite database created and migrations applied." +fi + +# ============================================================================= +# 2. Generate SSL certificate if not already present +# ============================================================================= +# mkdir -p /app/secrets + +if [ -f "/app/secrets/cert.crt" ] && [ -f "/app/secrets/cert.key" ]; then + ok "SSL certificates already exist — leaving untouched." +else + echo "Generating self-signed TLS certificate into /app/secrets/..." + + SERVER_HOST=$(hostname -f 2>/dev/null || hostname) + SERVER_IP=$(hostname -I 2>/dev/null | awk '{print $1}') + [ -z "$SERVER_IP" ] && SERVER_IP="127.0.0.1" + + openssl req -x509 -newkey rsa:4096 \ + -keyout /app/secrets/cert.key \ + -out /app/secrets/cert.crt \ + -days 365 -nodes \ + -subj "/CN=${SERVER_HOST}" \ + -addext "subjectAltName=DNS:${SERVER_HOST},IP:${SERVER_IP},DNS:localhost,IP:127.0.0.1" + + ok "Certificate created (CN=${SERVER_HOST}, IP=${SERVER_IP})" +fi + +# ============================================================================= +# 3. Collect static files — needs DJANGO_SECRET_KEY from the environment. +# ============================================================================= +python manage.py collectstatic --noinput + +# ============================================================================= +# 4. Run database migrations — safe to run repeatedly. +# ============================================================================= +python manage.py migrate --noinput + +# ============================================================================= +# 5. Hand off to the main process (Gunicorn). +# ============================================================================= +exec "$@" diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 932bc8e..0b739de 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -6,7 +6,7 @@ Usage: HTTP: gunicorn -c gunicorn.conf.py labhub.wsgi:application - HTTPS: gunicorn -c gunicorn.conf.py --certfile=secrets/cert.crt --keyfile=secrets/cert.key -b 0.0.0.0:443 labhub.wsgi:application + HTTPS: gunicorn -c gunicorn.conf.py --certfile=cert.crt --keyfile=cert.key -b 0.0.0.0:9443 labhub.wsgi:application On Linux, binding to port 443 requires one of: sudo gunicorn ... @@ -18,7 +18,7 @@ # ----------------------------------------------------------------------------- # Server socket (overridden by -b on the command line when using HTTPS) # ----------------------------------------------------------------------------- -bind = "0.0.0.0:8000" +bind = "0.0.0.0:9000" backlog = 2048 # ----------------------------------------------------------------------------- diff --git a/labhub/templates/index.html b/labhub/templates/index.html index a26a976..9d23568 100644 --- a/labhub/templates/index.html +++ b/labhub/templates/index.html @@ -1 +1 @@ -Spike Sorting Lab Hub
\ No newline at end of file +Spike Sorting Lab Hub
\ No newline at end of file diff --git a/my-app/src/App.js b/my-app/src/App.js index e97cd35..135b478 100644 --- a/my-app/src/App.js +++ b/my-app/src/App.js @@ -14,7 +14,7 @@ function DashboardLayout() { {/* App nav bar */}
- {/* Push content below both bars */} -
+ {/* Push content below the fixed navbar */} +
diff --git a/my-app/src/pages/CombineAndDownsample.js b/my-app/src/pages/CombineAndDownsample.js new file mode 100644 index 0000000..0de12c5 --- /dev/null +++ b/my-app/src/pages/CombineAndDownsample.js @@ -0,0 +1,316 @@ +import { useState } from 'react'; +import FileBrowser from '../components/FileBrowser'; +import '../styles/CombineAndDownsample.css'; + +const MODES = [ + { + id: 'both', + label: 'Combine and downsample', + description: 'Run both operations in a single pass — reads each file once, writes raw .dat and downsampled .mat simultaneously.', + icon: ( + + + + + + ), + }, + { + id: 'combine', + label: 'Combine', + description: 'Merge multiple recordings into one continuous .dat stream. No downsampling.', + icon: ( + + + + + + ), + }, + { + id: 'downsample', + label: 'Downsample', + description: 'Reduce the sampling rate to produce an LFP-band .mat file. Input is a single combined .dat.', + icon: ( + + + + + + + ), + }, +]; + +export default function CombineAndDownsample({ onBack }) { + const [mode, setMode] = useState('both'); + const [inputFiles, setInputFiles] = useState([]); // array of absolute server paths + const [browserOpen, setBrowserOpen] = useState(false); + const [numChannels, setChannels] = useState('64'); + const [dsFactor, setDsFactor] = useState('30'); + const [outputName, setOutName] = useState(''); + const [outputFolder, setOutDir] = useState(''); + const [loading, setLoading] = useState(false); + const [response, setResponse] = useState(null); // { ok, data, error } + + function selectMode(newMode) { + if (newMode === mode) return; + setMode(newMode); + setInputFiles([]); + setResponse(null); + } + + function handleFileSelected(f) { + setInputFiles(prev => + prev.some(p => p === f.path) ? prev : [...prev, f.path] + ); + } + + function removeFile(path) { + setInputFiles(prev => prev.filter(p => p !== path)); + } + + function isValid() { + if (inputFiles.length === 0) return false; + if (!numChannels || parseInt(numChannels) < 1) return false; + if (mode !== 'combine' && (!dsFactor || parseInt(dsFactor) < 2)) return false; + return true; + } + + async function handleSubmit() { + if (!isValid()) return; + setLoading(true); + setResponse(null); + + try { + const token = window.localStorage.getItem('token'); + + const body = { + input_files: inputFiles, + num_channels: parseInt(numChannels), + downsample_factor: parseInt(dsFactor), + mode, + }; + if (outputName.trim()) body.output_name = outputName.trim(); + if (outputFolder.trim()) body.output_folder = outputFolder.trim(); + + const resp = await fetch('/submit-jobs/combine-downsample/', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(token ? { Authorization: `Token ${token}` } : {}), + }, + body: JSON.stringify(body), + }); + + const ct = resp.headers.get('content-type') || ''; + if (resp.ok && ct.includes('application/json')) { + const data = await resp.json(); + setResponse({ ok: true, data }); + } else { + const text = ct.includes('application/json') + ? JSON.stringify(await resp.json(), null, 2) + : await resp.text(); + setResponse({ ok: false, error: text }); + } + } catch (err) { + setResponse({ ok: false, error: err.message || String(err) }); + } finally { + setLoading(false); + } + } + + const selectedMode = MODES.find(m => m.id === mode); + + return ( +
+ {/* Back */} + + + {/* Header */} +
+

Pipeline

+

Combine & Downsample

+
+ + {/* Mode selector */} +
+ {MODES.map(m => ( +
selectMode(m.id)} + > +
{m.icon}
+

{m.label}

+

{m.description}

+
+ ))} +
+ + {/* Form */} +
+

+ Configure — {selectedMode.label} +

+ + {/* File picker */} +
+ + + {/* Accumulated file list */} + {inputFiles.length > 0 && ( +
+ {inputFiles.map((path, i) => ( +
+ {i + 1} + + {path.split('/').pop()} + + {path} + +
+ ))} +
+ )} + + + + {inputFiles.length > 0 && ( +

{inputFiles.length} file{inputFiles.length > 1 ? 's' : ''} selected. Files will be concatenated in the order listed.

+ )} +
+ + {/* Number of channels */} +
+ + setChannels(e.target.value)} + placeholder="e.g. 64" + /> +
+ + {/* Downsample factor — hidden for combine-only */} + {mode !== 'combine' && ( +
+ + setDsFactor(e.target.value)} + placeholder="e.g. 30" + /> +

+ Factor of {dsFactor || '?'} → {numChannels ? Math.round(30000 / parseInt(dsFactor || 1)) + ' Hz output' : '—'} (assuming 30 kHz input) +

+
+ )} + +
+ + {/* Optional fields */} +
+ + setOutName(e.target.value)} + placeholder="Defaults to experiment folder name" + /> +
+ +
+ + setOutDir(e.target.value)} + placeholder="Defaults to combined_ next to input" + /> +
+ + {/* Submit */} + + + {/* Response */} + {response && !response.ok && ( + 
{response.error}
+ )} + {response && response.ok && response.data && ( +
+

{response.data.operation} — submitted

+ +
+ Input files +
    + {response.data.input_files.map((f, i) => ( +
  • + {i + 1} + {f} +
    • + ))} +
+
+ +
+ Output folder + {response.data.output_folder} +
+ +
+ Output files +
    + {response.data.output_files.map(f => ( +
  • + {f.name} + {f.description} +
    • + ))} +
+
+
+ )} +
+ + {/* File browser modal */} + {browserOpen && ( + setBrowserOpen(false)} + /> + )} +
+ ); +} diff --git a/my-app/src/pages/Dashboard.js b/my-app/src/pages/Dashboard.js index 33a72a4..07f55d2 100644 --- a/my-app/src/pages/Dashboard.js +++ b/my-app/src/pages/Dashboard.js @@ -3,6 +3,7 @@ import { useAuth } from '../context/AuthContext'; import CreateSortingJobWizard from './CreateSortingJobWizard'; import AddNewPipeline from './AddNewPipeline'; import ManageJobs from './ManageJobs'; +import CombineAndDownsample from './CombineAndDownsample'; import '../styles/Dashboard.css'; function getGreeting() { @@ -28,6 +29,10 @@ export default function Dashboard() { return setActiveSection('home')} />; } + if (activeSection === 'combineDs') { + return setActiveSection('home')} />; + } + return (
{/* Greeting */} @@ -101,6 +106,27 @@ export default function Dashboard() { Progress
+ +
setActiveSection('combineDs')} + > +
+ + + + + + Preprocessing +
+

Combine & Downsample

+

Merge multiple .dat recordings into one file and downsample to LFP in a single pass.

+
+ Combine + Downsample + .mat +
+
{/* Your Workspace */} diff --git a/my-app/src/styles/CombineAndDownsample.css b/my-app/src/styles/CombineAndDownsample.css new file mode 100644 index 0000000..24a7873 --- /dev/null +++ b/my-app/src/styles/CombineAndDownsample.css @@ -0,0 +1,405 @@ +/* ── Combine & Downsample Page ── */ +.cds-container { + min-height: 100vh; + background: #f0efe8; + padding: 48px 56px; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; + box-sizing: border-box; +} + +/* Back button */ +.cds-back { + display: inline-flex; + align-items: center; + gap: 6px; + background: none; + border: none; + color: #888; + font-size: 0.88em; + font-weight: 500; + cursor: pointer; + padding: 0; + margin-bottom: 36px; + transition: color 0.15s; +} +.cds-back:hover { color: #444; } +.cds-back svg { width: 14px; height: 14px; } + +/* Page header */ +.cds-header { + margin-bottom: 40px; +} +.cds-header .cds-label { + font-size: 0.78em; + font-weight: 600; + letter-spacing: 1.2px; + text-transform: uppercase; + color: #aaa; + margin: 0 0 8px 0; +} +.cds-header h1 { + font-size: 2.2em; + font-weight: 400; + color: #1e1e1e; + margin: 0; + letter-spacing: -0.4px; +} + +/* Mode selector */ +.cds-mode-grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 14px; + margin-bottom: 40px; +} + +.cds-mode-card { + background: #fff; + border: 1.5px solid #e4e3dc; + border-radius: 14px; + padding: 22px 20px; + cursor: pointer; + display: flex; + flex-direction: column; + gap: 8px; + transition: border-color 0.15s, box-shadow 0.15s; +} +.cds-mode-card:hover { + border-color: #bbb; +} +.cds-mode-card.selected { + border-color: #1e1e1e; + box-shadow: 0 0 0 1px #1e1e1e; +} + +.cds-mode-icon { + width: 36px; + height: 36px; + border-radius: 8px; + background: #f0efe8; + display: flex; + align-items: center; + justify-content: center; + margin-bottom: 4px; +} +.cds-mode-icon svg { + width: 18px; + height: 18px; + stroke: #555; +} +.cds-mode-card.selected .cds-mode-icon { + background: #1e1e1e; +} +.cds-mode-card.selected .cds-mode-icon svg { + stroke: #fff; +} + +.cds-mode-card h3 { + margin: 0; + font-size: 1em; + font-weight: 500; + color: #1e1e1e; +} +.cds-mode-card p { + margin: 0; + font-size: 0.83em; + color: #999; + line-height: 1.5; +} + +/* Form */ +.cds-form { + background: #fff; + border: 1px solid #e4e3dc; + border-radius: 16px; + padding: 36px; + max-width: 760px; +} + +.cds-form-title { + font-size: 1em; + font-weight: 500; + color: #1e1e1e; + margin: 0 0 28px 0; +} + +.cds-field { + margin-bottom: 24px; +} +.cds-field label { + display: block; + font-size: 0.85em; + font-weight: 500; + color: #555; + margin-bottom: 8px; + letter-spacing: -0.1px; +} +.cds-field input[type="number"], +.cds-field input[type="text"] { + width: 100%; + padding: 10px 14px; + border: 1px solid #e4e3dc; + border-radius: 8px; + font-size: 0.9em; + color: #1e1e1e; + background: #fafaf8; + box-sizing: border-box; + transition: border-color 0.15s; + font-family: inherit; +} +.cds-field input:focus, +.cds-field textarea:focus { + outline: none; + border-color: #aaa; + background: #fff; +} + +.cds-field .cds-hint { + font-size: 0.78em; + color: #bbb; + margin-top: 5px; +} + +/* File drop zone */ +.cds-dropzone { + border: 1.5px dashed #d4d3cc; + border-radius: 10px; + padding: 28px 20px; + text-align: center; + cursor: pointer; + transition: border-color 0.15s, background 0.15s; + background: #fafaf8; +} +.cds-dropzone:hover { + border-color: #aaa; + background: #f5f4ed; +} +.cds-dropzone input[type="file"] { + display: none; +} +.cds-dropzone-icon { + margin-bottom: 10px; + color: #bbb; +} +.cds-dropzone-icon svg { + width: 28px; + height: 28px; + stroke: #bbb; +} +.cds-dropzone p { + margin: 0 0 4px; + font-size: 0.88em; + color: #777; + font-weight: 500; +} +.cds-dropzone span { + font-size: 0.78em; + color: #bbb; +} + +/* File list */ +.cds-file-list { + margin-bottom: 12px; + display: flex; + flex-direction: column; + gap: 4px; +} +.cds-file-item { + display: grid; + grid-template-columns: 22px 1fr auto; + grid-template-rows: auto auto; + column-gap: 10px; + row-gap: 1px; + align-items: start; + background: #f5f4ed; + border-radius: 8px; + padding: 10px 12px; + font-size: 0.82em; + color: #444; + border: 1px solid #e9e8e0; +} +.cds-file-idx { + grid-row: 1 / 3; + align-self: center; + font-size: 0.78em; + font-weight: 600; + color: #bbb; + text-align: right; + flex-shrink: 0; +} +.cds-file-name { + font-weight: 500; + color: #1e1e1e; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.cds-file-path { + grid-column: 2; + font-size: 0.9em; + color: #aaa; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + font-family: 'SF Mono', 'Fira Code', monospace; +} +.cds-file-remove { + grid-row: 1 / 3; + align-self: center; + background: none; + border: none; + color: #ccc; + cursor: pointer; + font-size: 1em; + padding: 0; + line-height: 1; + flex-shrink: 0; + transition: color 0.15s; +} +.cds-file-remove:hover { color: #e05555; } + +/* Browse button */ +.cds-browse-btn { + display: inline-flex; + align-items: center; + gap: 7px; + padding: 9px 16px; + background: #f5f4ed; + border: 1px solid #d4d3cc; + border-radius: 8px; + font-size: 0.85em; + font-weight: 500; + color: #555; + cursor: pointer; + font-family: inherit; + transition: background 0.15s, border-color 0.15s; +} +.cds-browse-btn:hover { + background: #eceae0; + border-color: #bbb; +} +.cds-browse-btn svg { + width: 14px; + height: 14px; + flex-shrink: 0; +} + +/* Divider */ +.cds-divider { + border: none; + border-top: 1px solid #ece9e0; + margin: 28px 0; +} + +/* Submit */ +.cds-submit { + display: inline-flex; + align-items: center; + gap: 8px; + padding: 12px 28px; + background: #1e1e1e; + color: #fff; + border: none; + border-radius: 9px; + font-size: 0.9em; + font-weight: 500; + cursor: pointer; + font-family: inherit; + transition: opacity 0.15s; +} +.cds-submit:hover:not(:disabled) { opacity: 0.82; } +.cds-submit:disabled { opacity: 0.4; cursor: not-allowed; } + +/* Error response */ +.cds-response { + margin-top: 24px; + background: #f5f4ed; + border-radius: 10px; + padding: 18px; + font-size: 0.82em; + color: #444; + font-family: 'SF Mono', 'Fira Code', monospace; + white-space: pre-wrap; + word-break: break-all; + border: 1px solid #e4e3dc; +} +.cds-response.error { border-left: 3px solid #e07a5a; } + +/* Success result card */ +.cds-result { + margin-top: 24px; + border: 1px solid #d6f0d6; + border-left: 3px solid #7aad7a; + border-radius: 10px; + background: #f6fbf6; + padding: 20px 22px; + display: flex; + flex-direction: column; + gap: 16px; +} +.cds-result-op { + margin: 0; + font-size: 0.88em; + font-weight: 600; + color: #4a7a4a; + letter-spacing: -0.1px; +} +.cds-result-block { + display: flex; + flex-direction: column; + gap: 6px; +} +.cds-result-label { + font-size: 0.75em; + font-weight: 600; + letter-spacing: 0.8px; + text-transform: uppercase; + color: #aaa; +} +.cds-result-list { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 4px; +} +.cds-result-list li { + display: flex; + align-items: baseline; + gap: 10px; + font-size: 0.83em; +} +.cds-result-idx { + font-size: 0.78em; + font-weight: 600; + color: #bbb; + min-width: 14px; + text-align: right; + flex-shrink: 0; +} +.cds-result-path { + font-family: 'SF Mono', 'Fira Code', monospace; + color: #555; + word-break: break-all; +} +.cds-result-fname { + font-family: 'SF Mono', 'Fira Code', monospace; + font-weight: 600; + color: #1e1e1e; +} +.cds-result-desc { + color: #999; + font-size: 0.9em; +} +.cds-result-code { + font-family: 'SF Mono', 'Fira Code', monospace; + font-size: 0.83em; + color: #555; + word-break: break-all; +} + +@media (max-width: 900px) { + .cds-container { padding: 32px 24px; } + .cds-mode-grid { grid-template-columns: 1fr; } +} diff --git a/my-app/src/styles/Dashboard.css b/my-app/src/styles/Dashboard.css index 41596bc..350cbd4 100644 --- a/my-app/src/styles/Dashboard.css +++ b/my-app/src/styles/Dashboard.css @@ -39,7 +39,7 @@ /* Action cards grid */ .dashboard-actions { display: grid; - grid-template-columns: repeat(3, 1fr); + grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); gap: 20px; margin-bottom: 56px; } @@ -75,6 +75,10 @@ background: #c5ca85; } +.combine-ds { + background: #7a9eb5; +} + /* Card icon */ .card-icon-label { font-size: 0.8em; diff --git a/my-app/src/styles/WizardSteps.css b/my-app/src/styles/WizardSteps.css index 6a258c6..c131317 100644 --- a/my-app/src/styles/WizardSteps.css +++ b/my-app/src/styles/WizardSteps.css @@ -115,6 +115,8 @@ padding: 12px; background: #f9f9f9; border-radius: 4px; + max-height: 220px; + overflow-y: auto; } .checkbox-label { @@ -250,6 +252,8 @@ grid-template-columns: repeat(auto-fill, minmax(50px, 1fr)); gap: 8px; margin-bottom: 15px; + max-height: 220px; + overflow-y: auto; } .channel-btn { diff --git a/secrets/.gitkeep b/secrets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/submit_jobs/serializers.py b/submit_jobs/serializers.py index 1ec79a2..484fde6 100644 --- a/submit_jobs/serializers.py +++ b/submit_jobs/serializers.py @@ -155,6 +155,35 @@ class Meta: read_only_fields = ["job_id", "status", "created_at"] +class CombineDownsampleSerializer(serializers.Serializer): + """ + Validates a combine-and-downsample job request. + All input_files must be absolute server-side paths to .dat files. + """ + + input_files = serializers.ListField( + child=serializers.CharField(min_length=1), + min_length=1, + ) + num_channels = serializers.IntegerField(min_value=1) + downsample_factor = serializers.IntegerField(min_value=2, required=False, default=30) + mode = serializers.ChoiceField( + choices=['both', 'combine', 'downsample'], + required=False, + default='both', + ) + output_name = serializers.CharField(required=False, allow_blank=True, default='') + output_folder = serializers.CharField(required=False, allow_blank=True, default='') + + def validate_input_files(self, paths): + for p in paths: + if not p.startswith('/'): + raise serializers.ValidationError( + f"Each path must be absolute (start with /): {p!r}" + ) + return paths + + class JobCreationLogSerializer(serializers.ModelSerializer): """ Serializes a JobCreationLog audit record for inspection and debugging. diff --git a/submit_jobs/urls.py b/submit_jobs/urls.py index e87aa99..067f1e6 100644 --- a/submit_jobs/urls.py +++ b/submit_jobs/urls.py @@ -6,4 +6,5 @@ urlpatterns = [ path("create-sorting-job/", views.create_sorting_job, name="create_sorting_job"), path("browse/", views.browse_data_files, name="browse_data_files"), + path("combine-downsample/", views.combine_downsample_job, name="combine_downsample_job"), ] diff --git a/submit_jobs/views.py b/submit_jobs/views.py index 4937d3c..1e68ec9 100644 --- a/submit_jobs/views.py +++ b/submit_jobs/views.py @@ -12,7 +12,7 @@ resolve_placeholder_dependencies, build_job_env_config, ) -from .serializers import CreateSortingJobSerializer +from .serializers import CreateSortingJobSerializer, CombineDownsampleSerializer def strip_nas_root(path: str) -> str: @@ -155,3 +155,119 @@ def browse_data_files(request): cursor = os.path.dirname(cursor) return Response({"current_path": requested, "parents": parents, "dirs": dirs, "files": files}) + + +# ============================================================================ +# Combine & Downsample Job Endpoint +# ============================================================================ + + +@api_view(["POST"]) +@permission_classes([IsAuthenticated]) +def combine_downsample_job(request): + """ + POST: Create a standalone combine-and-downsample job. + + Expected JSON body: + { + "input_files": ["/abs/path/to/recording1/continuous.dat", ...], + "num_channels": 64, + "downsample_factor": 30, // ignored when mode == "combine" + "mode": "both", // "both" | "combine" | "downsample" + "output_name": "session01", // optional + "output_folder": "/mnt/nas/out" // optional + } + """ + serializer = CombineDownsampleSerializer(data=request.data) + if not serializer.is_valid(): + return Response({"errors": serializer.errors}, status=status.HTTP_400_BAD_REQUEST) + + vd = serializer.validated_data + mode = vd["mode"] + + step_config = { + "input files": vd["input_files"], + "number of channels": vd["num_channels"], + "downsample factor": vd["downsample_factor"], + "mode": mode, + } + if vd.get("output_name"): + step_config["output name"] = vd["output_name"] + if vd.get("output_folder"): + step_config["output folder"] = strip_nas_root(vd["output_folder"]) + + # Paths on the NAS should be stored with the $NAS$ prefix so the worker + # can substitute its own mount point at runtime. + step_config["input files"] = [strip_nas_root(p) for p in step_config["input files"]] + + try: + identifier = get_or_create_step_configs("combine_and_downsample", step_config) + except RuntimeError as e: + return Response({"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + job_env = { + "base directory": "$LOCAL$/$JOB_ID$", + "log_level": "DEBUG", + "REDIRECT": { + "log": "$NAS$/SORTING_LOGS/$JOB_ID$/run.log", + "out": "$NAS$/SORTING_LOGS/$JOB_ID$/run.out", + "err": "$NAS$/SORTING_LOGS/$JOB_ID$/run.err", + }, + } + + job_steps = [ + {"function": "combine_and_downsample", "identifier": identifier, "depends": []} + ] + + try: + job = create_a_job(job_env, job_steps) + except RuntimeError as e: + return Response({"error": str(e)}, status=status.HTTP_400_BAD_REQUEST) + + # Infer expected output location from the first input file. + # Open Ephys layout: /RecordNode/experiment/recording/continuous//continuous.dat + # Walking 6 levels up reaches the session root. + first_input = vd["input_files"][0] + p = os.path.abspath(first_input) + for _ in range(6): + p = os.path.dirname(p) + session_name = os.path.basename(p) + session_parent = os.path.dirname(p) + + if vd.get("output_folder"): + out_dir = vd["output_folder"] + else: + out_dir = os.path.join(session_parent, f"combined_{session_name}") + + name = vd.get("output_name") or session_name + + mode_labels = { + "both": "Combine + Downsample", + "combine": "Combine", + "downsample": "Downsample", + } + output_files = [] + if mode in ("both", "combine"): + fname = f"combined_raw_{name}.dat" + output_files.append({ + "name": fname, + "path": os.path.join(out_dir, fname), + "description": "full-rate int16 binary", + }) + if mode in ("both", "downsample"): + fname = f"combined_ds{vd['downsample_factor']}_{name}.h5" + output_files.append({ + "name": fname, + "path": os.path.join(out_dir, fname), + "description": f"downsampled LFP at {30000 // vd['downsample_factor']} Hz", + }) + + return Response( + { + "operation": mode_labels[mode], + "input_files": vd["input_files"], + "output_folder": out_dir, + "output_files": output_files, + }, + status=status.HTTP_201_CREATED, + ) diff --git a/truenas_init.sh b/truenas_init.sh new file mode 100644 index 0000000..0490344 --- /dev/null +++ b/truenas_init.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# ============================================================ +# SpikesortingLabHub — TrueNAS Pre-Init Script +# +# Runs BEFORE Docker daemon starts. Its only job is to ensure +# all bind-mount directories exist on the ZFS pool so that +# Docker's restart policy can bring up the container cleanly. +# +# Store this file on the NAS at: +# /mnt/root_data_storage/users/sslh/truenas_init.sh +# +# TrueNAS UI: +# System → Advanced Settings → Init/Shutdown Scripts → Add +# Type: Script When: Pre Init Timeout: 30 +# ============================================================ + +set -euo pipefail + +LOG="/var/log/sslh_init.log" +exec >> "$LOG" 2>&1 + +echo "=== $(date) SpikesortingLabHub pre-init starting ===" + +# ── Required bind-mount directories ─────────────────────────── +REQUIRED_DIRS=( + "/mnt/root_data_storage/users/sslh/trurnasdata" + "/mnt/root_data_storage/users/sslh/persistentdata" + "/mnt/root_data_storage/users/sslh/secrets" + "/mnt/root_data_storage/users/sslh/experiments" +) + +for DIR in "${REQUIRED_DIRS[@]}"; do + if [ ! -d "$DIR" ]; then + mkdir "$DIR" && chown -R sslh "$DIR" + fi + echo "OK: $DIR" +done +mount -o bind,ro /data /mnt/root_data_storage/users/sslh/trurnasdata +mount -o bind,ro /mnt/root_data_storage/experiments /mnt/root_data_storage/users/sslh/experiments +echo "=== $(date) All directories present — Docker may start ==="