diff --git a/fdm-app/app/components/blocks/soil/bulk-upload-form.tsx b/fdm-app/app/components/blocks/soil/bulk-upload-form.tsx index fcb20215a..1a4d94003 100644 --- a/fdm-app/app/components/blocks/soil/bulk-upload-form.tsx +++ b/fdm-app/app/components/blocks/soil/bulk-upload-form.tsx @@ -1,5 +1,4 @@ -import { useState, useEffect } from "react" -import { useFetcher } from "react-router" +import { useState } from "react" import { FileText, Upload, Trash2, X, FileUp } from "lucide-react" import { Dropzone } from "~/components/custom/dropzone" import { Button } from "~/components/ui/button" @@ -13,42 +12,148 @@ import { import { Spinner } from "~/components/ui/spinner" import { ScrollArea } from "~/components/ui/scroll-area" import { cn } from "~/lib/utils" +import { Progress } from "~/components/ui/progress" +import { toast } from "sonner" +import type { ProcessedAnalysis } from "./bulk-upload-review" export function BulkSoilAnalysisUploadForm({ onSuccess, }: { - onSuccess: (data: any[]) => void + onSuccess: (data: ProcessedAnalysis[]) => void }) { const [files, setFiles] = useState([]) - const fetcher = useFetcher() - const isUploading = fetcher.state !== "idle" + const [isUploading, setIsUploading] = useState(false) + const [uploadProgress, setUploadProgress] = useState(0) + const [currentFile, setCurrentFile] = useState(null) const MAX_FILE_SIZE = 5 * 1024 * 1024 const handleFilesChange = (newFiles: File[]) => { setFiles(newFiles) } - const handleUpload = () => { + const handleUpload = async () => { if (files.length === 0) return + setIsUploading(true) + setUploadProgress(0) + + const allResults: any[] = [] + const totalFiles = files.length + let completedFiles = 0 + let errorOccurred = false + const formData = new FormData() for (const file of files) { formData.append("soilAnalysisFile", file) } - fetcher.submit(formData, { - method: "POST", - encType: "multipart/form-data", - }) - } + try { + const response = await fetch("/api/soil-analysis/extract", { + method: "POST", + body: formData, + credentials: "same-origin", + }) + + if (!response.ok) { + let errorMessage = "Fout bij starten van analyse" + try { + const errorData = await response.json() + errorMessage = + errorData.message || errorData.error || errorMessage + } catch { + try { + const textError = await response.text() + if (textError) errorMessage = textError + } catch { + // Ignore text parsing errors + } + } + throw new Error(`${errorMessage} (Status: ${response.status})`) + } + + if (!response.body) { + throw new Error("Geen stream response ontvangen") + } + + const reader = response.body.getReader() + const decoder = new TextDecoder() + let buffer = "" + + while (true) { + const { done, value } = await reader.read() + if (done) break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split("\n") + // Keep the last partial line in the buffer + buffer = lines.pop() || "" + + for (const line of lines) { + if (!line.trim()) continue + try { + const result = JSON.parse(line) + completedFiles++ + + if (result.success && result.analyses) { + allResults.push(...result.analyses) + } else if (result.error) { + toast.error( + `Fout bij ${result.filename}: ${result.error}`, + ) + } - useEffect(() => { - if (isUploading) return + setCurrentFile(result.filename) + setUploadProgress( + Math.round((completedFiles / totalFiles) * 100), + ) + } catch (e) { + console.error("Error parsing NDJSON line:", e) + } + } + } - if (fetcher.data?.analyses) { - onSuccess(fetcher.data.analyses) + // Process final partial line if exists + if (buffer.trim()) { + try { + const result = JSON.parse(buffer) + completedFiles++ + + if (result.success && result.analyses) { + allResults.push(...result.analyses) + } else if (result.error) { + toast.error( + `Fout bij ${result.filename}: ${result.error}`, + ) + } + + setCurrentFile(result.filename) + setUploadProgress( + Math.round((completedFiles / totalFiles) * 100), + ) + } catch (e) { + console.error("Error parsing final NDJSON line:", e) + } + } + } catch (error) { + console.error("Bulk upload error:", error) + toast.error( + error instanceof Error ? error.message : "Upload mislukt", + ) + errorOccurred = true + } finally { + setIsUploading(false) + setCurrentFile(null) + + if (allResults.length > 0) { + toast.success( + `${allResults.length} analyses succesvol verwerkt`, + ) + onSuccess(allResults) + } else if (!errorOccurred && totalFiles > 0) { + toast.error("Geen analyses kunnen verwerken") + } } - }, [fetcher.data, isUploading, onSuccess]) + } const removeFile = (index: number) => { const newFiles = [...files] @@ -90,6 +195,7 @@ export function BulkSoilAnalysisUploadForm({ onFilesChange={handleFilesChange} allowReset={false} maxSize={MAX_FILE_SIZE} + disabled={isUploading} className={cn( "w-full transition-all duration-200 border-2", files.length > 0 @@ -197,6 +303,25 @@ export function BulkSoilAnalysisUploadForm({ + {isUploading && ( +
+
+ + {currentFile + ? `Analyseert: ${currentFile}` + : "Bestanden verwerken..."} + + + {uploadProgress}% + +
+ +
+ )} +
+ + + + + +
+ ) } export type ProcessedAnalysis = { @@ -49,8 +172,13 @@ export type ProcessedAnalysis = { a_p_al?: number a_p_cc?: number a_nmin_cc?: number + a_nh4_cc?: number + a_no3_cc?: number + a_depth_upper?: number + a_depth_lower?: number a_source: string matchedFieldId?: string + matchReason?: "geometry" | "name" | "both" data: Record // Raw parsed data } @@ -69,29 +197,56 @@ export function BulkSoilAnalysisReview({ analyses: ProcessedAnalysis[] fields: Field[] soilParameterDescription: SoilParameterDescription - onSave: (matches: { analysisId: string; fieldId: string }[]) => void + onSave: ( + matches: { analysisId: string; fieldId: string }[], + updatedAnalyses: ProcessedAnalysis[], + ) => void onCancel: () => void }) { const [matches, setMatches] = useState>( Object.fromEntries(analyses.map((a) => [a.id, a.matchedFieldId || ""])), ) + const [dates, setDates] = useState>( + Object.fromEntries( + analyses.map((a) => { + const date = a.b_sampling_date + ? new Date(a.b_sampling_date) + : null + const validDateStr = + date && isValid(date) ? format(date, "yyyy-MM-dd") : "" + return [a.id, validDateStr] + }), + ), + ) const handleFieldChange = (analysisId: string, fieldId: string) => { setMatches((prev) => ({ ...prev, [analysisId]: fieldId })) } + const handleDateChange = (analysisId: string, date: string) => { + setDates((prev) => ({ ...prev, [analysisId]: date })) + } + const validMatches = useMemo( () => Object.entries(matches) .filter(([analysisId, fieldId]) => { if (fieldId === "" || fieldId === "none") return false - const analysis = analyses.find((a) => a.id === analysisId) - return isValidDate(analysis?.b_sampling_date) + const date = dates[analysisId] + return isValidDate(date) }) .map(([analysisId, fieldId]) => ({ analysisId, fieldId })), - [matches, analyses], + [matches, dates], ) + const handleSave = () => { + const updatedAnalyses = analyses.map((a) => ({ + ...a, + b_sampling_date: dates[a.id], + })) + onSave(validMatches, updatedAnalyses) + } + const columns: ColumnDef[] = [ { accessorKey: "filename", @@ -111,9 +266,22 @@ export function BulkSoilAnalysisReview({ {row.original.filename} -
- - {sourceLabel} +
+
+ + {sourceLabel} +
+ {(row.original.a_depth_upper !== undefined || + row.original.a_depth_lower !== undefined) && ( +
+ + + Diepte:{" "} + {row.original.a_depth_upper ?? 0} -{" "} + {row.original.a_depth_lower ?? "?"} cm + +
+ )}
) @@ -122,12 +290,13 @@ export function BulkSoilAnalysisReview({ { accessorKey: "b_sampling_date", header: "Datum", - cell: ({ row }) => { - if (!isValidDate(row.original.b_sampling_date)) return "-" - return format(new Date(row.original.b_sampling_date), "P", { - locale: nl, - }) - }, + cell: ({ row }) => ( + + ), }, { id: "parameters", @@ -141,7 +310,7 @@ export function BulkSoilAnalysisReview({ )} {row.original.a_p_al != null && ( - P-AL: {row.original.a_p_al} + P-Al: {row.original.a_p_al} )} {row.original.a_p_cc != null && ( @@ -154,56 +323,103 @@ export function BulkSoilAnalysisReview({ Nmin: {row.original.a_nmin_cc} )} + {row.original.a_nh4_cc != null && ( + + NH₄: {row.original.a_nh4_cc} + + )} + {row.original.a_no3_cc != null && ( + + NO₃: {row.original.a_no3_cc} + + )} ), }, { id: "match", header: "Perceel", - cell: ({ row }) => ( - + handleFieldChange(row.original.id, value) + } + > + + + + + + -- Geen perceel -- - ))} - - - ), + {fields.map((field) => ( + + {field.b_name} + + ))} + + + ) + }, }, { id: "status", header: "Status", cell: ({ row }) => { - const match = matches[row.original.id] - const isMatched = match && match !== "none" - const isValid = isValidDate(row.original.b_sampling_date) + const matchId = matches[row.original.id] + const isMatched = matchId && matchId !== "none" + const date = dates[row.original.id] + const isDateValid = isValidDate(date) + const reason = row.original.matchReason + const initialMatchId = row.original.matchedFieldId - if (!isValid) { + if (!isDateValid) { return (
- Ongeldige pdf + Datum ontbreekt
) } - return isMatched ? ( -
- - Gekoppeld -
- ) : ( + if (isMatched) { + const isAutomatic = matchId === initialMatchId + let tooltipText = "Handmatig gekoppeld" + + if (isAutomatic && reason) { + if (reason === "geometry") + tooltipText = + "Automatisch gekoppeld op basis van geometrie" + else if (reason === "name") + tooltipText = + "Automatisch gekoppeld op basis van naam" + else if (reason === "both") + tooltipText = + "Automatisch gekoppeld op basis van geometrie en naam" + } + + return ( + + +
+ + Gekoppeld +
+
+ +

{tooltipText}

+
+
+ ) + } + + return (
Niet gekoppeld @@ -219,110 +435,88 @@ export function BulkSoilAnalysisReview({ getCoreRowModel: getCoreRowModel(), }) - const handleSave = () => { - onSave(validMatches) - } - return ( - - - Controleer en koppel - - Controleer de gegevens uit de pdf's en koppel ze aan het - juiste perceel. Analyses met ontbrekende datum of zonder - gekoppeld perceel worden overgeslagen. - - - -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - - {header.isPlaceholder - ? null - : flexRender( - header.column.columnDef - .header, - header.getContext(), - )} - - ))} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => { - const isValid = isValidDate( - row.original.b_sampling_date, - ) - - return ( + + + + Controleer en koppel + + Controleer de gegevens uit de pdf's en koppel ze aan het + juiste perceel. Analyses met ontbrekende datum of zonder + gekoppeld perceel worden overgeslagen. + + + +
+
+ + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column + .columnDef.header, + header.getContext(), + )} + + ))} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( {row .getVisibleCells() .map((cell) => ( - {cell.column.id === - "match" && - !isValid ? ( -
- Niet koppelbaar -
- ) : ( - flexRender( - cell.column - .columnDef - .cell, - cell.getContext(), - ) + {flexRender( + cell.column + .columnDef.cell, + cell.getContext(), )}
))}
- ) - }) - ) : ( - - - Geen resultaten. - - - )} -
-
-
-
- - - - -
+ )) + ) : ( + + + Geen resultaten. + + + )} + + +
+ + + + + + + ) } diff --git a/fdm-app/app/components/custom/dropzone.tsx b/fdm-app/app/components/custom/dropzone.tsx index e44aab578..c91d8d8dc 100644 --- a/fdm-app/app/components/custom/dropzone.tsx +++ b/fdm-app/app/components/custom/dropzone.tsx @@ -106,6 +106,22 @@ export const Dropzone = ({ else if (inputRef.current) inputRef.current.value = "" } + const syncFilesToInput = (filesToSync: File[]) => { + if (!inputRef.current) return + try { + const container = new DataTransfer() + for (const f of filesToSync) { + if (f instanceof File) { + container.items.add(f) + } + } + inputRef.current.files = container.files + } catch (err) { + // Fallback or silent ignore if DataTransfer is restricted + console.warn("Could not sync files to hidden input:", err) + } + } + const handleDragOver = (e: React.DragEvent) => { e.preventDefault() } @@ -156,13 +172,7 @@ export const Dropzone = ({ inputFiles = await handleFilesSet(files, validNewFiles) } - if (inputRef.current?.files) { - const container = new DataTransfer() - inputFiles.forEach((f) => { - container.items.add(f) - }) - inputRef.current.files = container.files - } + syncFilesToInput(inputFiles) } } @@ -176,14 +186,14 @@ export const Dropzone = ({ const finalFiles = await handleFilesSet(files, validNewFiles) - if (inputRef.current) { - const container = new DataTransfer() - finalFiles.forEach((f) => { - container.items.add(f) - }) - inputRef.current.files = container.files + syncFilesToInput(finalFiles) + + try { + e.dataTransfer.clearData() + } catch (err) { + // clearData may throw in some browsers after drop + console.warn("Could not clear dataTransfer:", err) } - e.dataTransfer.clearData() } } diff --git a/fdm-app/app/integrations/nmi.ts b/fdm-app/app/integrations/nmi.ts index fd9bbca34..ec23f61b3 100644 --- a/fdm-app/app/integrations/nmi.ts +++ b/fdm-app/app/integrations/nmi.ts @@ -3,8 +3,18 @@ import type { Feature, Geometry, Polygon } from "geojson" import { z } from "zod" import { serverConfig } from "~/lib/config.server" import { fileTypeFromBuffer } from "file-type" +import proj4 from "proj4" + const MAX_PDF_SIZE = 5 * 1024 * 1024 +// Register the projection for RD New (EPSG:28992) +if (!proj4.defs("EPSG:28992")) { + proj4.defs( + "EPSG:28992", + "+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +towgs84=565.2369,50.0087,465.658,-0.406857330322398,0.350732676542563,-1.8703473836068,4.0812 +units=m +no_defs", + ) +} + export function getNmiApiKey() { if (!serverConfig.integrations.nmi) { return undefined @@ -206,7 +216,11 @@ export async function extractSoilAnalysis(formData: FormData) { const field = response.fields[0] // Select the a_* parameters - const soilAnalysis: { [key: string]: string | number | Date } = {} + const soilAnalysis: { + [key: string]: string | number | Date | any + location?: { type: "Point"; coordinates: [number, number] } + } = {} + soilAnalysis.b_name = field.b_fieldname // Map b_fieldname for name matching for (const key of Object.keys(field).filter((key) => key.startsWith("a_"), )) { @@ -214,9 +228,8 @@ export async function extractSoilAnalysis(formData: FormData) { } // Check if soil parameters are returned - if (Object.keys(soilAnalysis).length <= 1) { - // a_source is returned with invalid soil analysis - throw new Error("Invalid soil analysis") + if (!soilAnalysis.a_source) { + throw new Error("Invalid soil analysis: laboratory source missing") } // Process the other parameters @@ -227,7 +240,14 @@ export async function extractSoilAnalysis(formData: FormData) { const day = Number.parseInt(dateParts[0], 10) const month = Number.parseInt(dateParts[1], 10) - 1 // Month is 0-indexed const year = Number.parseInt(dateParts[2], 10) - soilAnalysis.b_sampling_date = new Date(year, month, day) + + if ( + !Number.isNaN(day) && + !Number.isNaN(month) && + !Number.isNaN(year) + ) { + soilAnalysis.b_sampling_date = new Date(year, month, day) + } } } if (field.b_soiltype_agr) { @@ -248,6 +268,31 @@ export async function extractSoilAnalysis(formData: FormData) { throw new Error(`Invalid numeric depth values: ${field.b_depth}`) } } + + // Add coordinates for geometry matching + const x_rd = field.b_loc_x + const y_rd = field.b_loc_y + + if (x_rd && y_rd) { + const numericX = Number(x_rd) + const numericY = Number(y_rd) + + if (!Number.isNaN(numericX) && !Number.isNaN(numericY)) { + try { + const [lon, lat] = proj4("EPSG:28992", "EPSG:4326", [ + numericX, + numericY, + ]) + soilAnalysis.location = { + type: "Point", + coordinates: [lon, lat], + } + } catch (e) { + console.error("Coordinate transformation failed:", e) + } + } + } + return soilAnalysis } @@ -258,102 +303,188 @@ export async function extractBulkSoilAnalyses(formData: FormData) { throw new Error("NMI API key not configured") } + // Filter out potential non-File objects or empty slots const files = formData.getAll("soilAnalysisFile") as File[] - if (files.length === 0) { - throw new Error("Geen bestanden gevonden in FormData") + const validFiles = files.filter((file) => file instanceof File && file.name) + if (validFiles.length === 0) { + throw new Error("Geen geldige bestanden gevonden in FormData") } - for (const file of files) { + for (const file of validFiles) { await validatePdfMagicBytes(file) } - const responseApi = await fetch("https://api.nmi-agro.nl/soilreader", { - method: "POST", - headers: { - Authorization: `Bearer ${nmiApiKey}`, - }, - body: formData, - }) + const BATCH_SIZE = 10 + const MAX_BATCH_BYTES = 20 * 1024 * 1024 // 20MB - if (!responseApi.ok) { - const text = await responseApi.text() - console.error( - `NMI API Error: ${responseApi.status} ${responseApi.statusText}`, - text, - ) - throw new Error( - `Request to NMI API failed: ${responseApi.status} ${responseApi.statusText}`, - ) + // Group files into batches based on count and total size + let currentBatchFiles: File[] = [] + let currentBatchSize = 0 + const batches: File[][] = [] + + for (const file of validFiles) { + if ( + currentBatchFiles.length >= BATCH_SIZE || + (currentBatchFiles.length > 0 && + currentBatchSize + file.size > MAX_BATCH_BYTES) + ) { + batches.push(currentBatchFiles) + currentBatchFiles = [] + currentBatchSize = 0 + } + currentBatchFiles.push(file) + currentBatchSize += file.size + } + if (currentBatchFiles.length > 0) { + batches.push(currentBatchFiles) } - const text = await responseApi.text() - try { - const result = JSON.parse(text) - const response = result?.data + // Process each batch sequentially to avoid overwhelming the API + const allBatchFields: any[] = [] + for (let i = 0; i < batches.length; i++) { + const batchFiles = batches[i] + const batchFormData = new FormData() + for (const file of batchFiles) { + batchFormData.append("soilAnalysisFile", file) + } + + const responseApi = await fetch("https://api.nmi-agro.nl/soilreader", { + method: "POST", + headers: { + Authorization: `Bearer ${nmiApiKey}`, + }, + body: batchFormData, + }) + + if (!responseApi.ok) { + const text = await responseApi.text() + console.error( + `NMI API Error (Batch ${i + 1}/${batches.length}): ${responseApi.status} ${responseApi.statusText}`, + text, + ) + + if (responseApi.status === 413) { + throw new Error( + `invalid: Groep ${i + 1} van de bestanden is te groot voor de NMI API. Verklein de PDF's of upload ze in nog kleinere groepen.`, + ) + } - if (!response?.fields || !Array.isArray(response.fields)) { + throw new Error( + `Request to NMI API failed: ${responseApi.status} ${responseApi.statusText}`, + ) + } + + const result = await responseApi.json() + const responseData = result?.data + + if (!responseData?.fields || !Array.isArray(responseData.fields)) { console.error( - "Invalid NMI API response structure:", + `Invalid NMI API response structure in batch ${i + 1}:`, JSON.stringify(result, null, 2), ) - throw new Error("Invalid API response: no fields found") + throw new Error( + `Invalid API response in batch ${i + 1}: no fields found`, + ) } - return response.fields.map((field: any, index: number) => { - const soilAnalysis: { [key: string]: any } = { - id: crypto.randomUUID(), // Used for UI matching - filename: field.filename || `Analyse ${index + 1}`, - } + allBatchFields.push(...responseData.fields) + } - // Safely map known soil parameters (starting with a_) - for (const key of Object.keys(field).filter((key) => - key.startsWith("a_"), - )) { - soilAnalysis[key] = field[key] - } + return allBatchFields.map((field: any, index: number) => { + const soilAnalysis: { [key: string]: any } = { + id: crypto.randomUUID(), // Used for UI matching + filename: field.filename || `Analyse ${index + 1}`, + b_name: field.b_fieldname, // Map b_fieldname for name matching + } - if (field.b_date) { - const dateParts = field.b_date.split("-") - if (dateParts.length === 3) { - const day = Number.parseInt(dateParts[0], 10) - const month = Number.parseInt(dateParts[1], 10) - 1 - const year = Number.parseInt(dateParts[2], 10) + // Safely map known soil parameters (starting with a_) + for (const key of Object.keys(field).filter((key) => + key.startsWith("a_"), + )) { + soilAnalysis[key] = field[key] + } + + if (field.b_date) { + const dateParts = field.b_date.split("-") + if (dateParts.length === 3) { + const day = Number.parseInt(dateParts[0], 10) + const month = Number.parseInt(dateParts[1], 10) - 1 + const year = Number.parseInt(dateParts[2], 10) + + if ( + !Number.isNaN(day) && + !Number.isNaN(month) && + !Number.isNaN(year) + ) { soilAnalysis.b_sampling_date = new Date(year, month, day) } } + } + + if (field.b_soiltype_agr) { + soilAnalysis.b_soil_type = field.b_soiltype_agr + } - if (field.b_soiltype_agr) { - soilAnalysis.b_soil_type = field.b_soiltype_agr + if (field.b_depth) { + const depthParts = field.b_depth.split("-") + if (depthParts.length !== 2) { + throw new Error(`Invalid depth format: ${field.b_depth}`) } + const upper = Number(depthParts[0]) + const lower = Number(depthParts[1]) + if (Number.isNaN(upper) || Number.isNaN(lower)) { + throw new Error( + `Invalid numeric depth values: ${field.b_depth}`, + ) + } + soilAnalysis.a_depth_upper = upper + soilAnalysis.a_depth_lower = lower + } - if (field.b_depth) { - const depthParts = field.b_depth.split("-") - if (depthParts.length !== 2) { - throw new Error(`Invalid depth format: ${field.b_depth}`) + // Add coordinates for geometry matching, but keep them separate from the main analysis data + // NMI API uses RD New (EPSG:28992) with keys b_loc_x and b_loc_y + const x_rd = field.b_loc_x + const y_rd = field.b_loc_y + + if (x_rd && y_rd) { + const numericX = Number(x_rd) + const numericY = Number(y_rd) + + if (!Number.isNaN(numericX) && !Number.isNaN(numericY)) { + try { + // Transform from RD New to WGS84 + const [lon, lat] = proj4("EPSG:28992", "EPSG:4326", [ + numericX, + numericY, + ]) + soilAnalysis.location = { + type: "Point", + coordinates: [lon, lat], + } + } catch (e) { + console.error("Coordinate transformation failed:", e) } - const upper = Number(depthParts[0]) - const lower = Number(depthParts[1]) - if (Number.isNaN(upper) || Number.isNaN(lower)) { - throw new Error( - `Invalid numeric depth values: ${field.b_depth}`, - ) - } - soilAnalysis.a_depth_upper = upper - soilAnalysis.a_depth_lower = lower } + } - // Add coordinates for geometry matching, but keep them separate from the main analysis data - if (field.a_lat && field.a_lon) { - soilAnalysis.location = { - type: "Point", - coordinates: [Number(field.a_lon), Number(field.a_lat)], + // Fallback for WGS84 if provided directly under other keys + if (!soilAnalysis.location) { + const lat = field.a_lat ?? field.latitude ?? field.lat + const lon = field.a_lon ?? field.longitude ?? field.lon + + if (lat != null && lon != null) { + const numericLat = Number(lat) + const numericLon = Number(lon) + + if (!Number.isNaN(numericLat) && !Number.isNaN(numericLon)) { + soilAnalysis.location = { + type: "Point", + coordinates: [numericLon, numericLat], + } } } + } - return soilAnalysis - }) - } catch (e) { - console.error("Failed to parse NMI API response:", text) - throw e - } + return soilAnalysis + }) } diff --git a/fdm-app/app/routes/api.soil-analysis.extract.ts b/fdm-app/app/routes/api.soil-analysis.extract.ts new file mode 100644 index 000000000..5819ca9ef --- /dev/null +++ b/fdm-app/app/routes/api.soil-analysis.extract.ts @@ -0,0 +1,103 @@ +import { type ActionFunctionArgs } from "react-router" +import { getSession } from "~/lib/auth.server" +import { extractBulkSoilAnalyses } from "~/integrations/nmi" + +/** + * API Route: Bulk Soil Analysis Extraction + * + * WHY THIS EXISTS: + * This is a standalone API route instead of being handled in the page action because + * bulk uploads (50+ files) require high concurrency. Sending many small parallel + * requests to a standard React Router action causes "Session Locking" in the database + * (Better Auth), leading to 401/302 redirects. + * + * THIS SOLUTION: + * 1. The browser sends ALL files in a single POST request (one session check). + * 2. The server processes these files in parallel (concurrency limit 10). + * 3. Results are streamed back as NDJSON (Newline Delimited JSON). + * 4. This allows the frontend to update the progress bar and file name for every + * single file in real-time while maintaining maximum performance. + */ + +export async function action({ request }: ActionFunctionArgs) { + // Single session check for the entire bulk operation + const session = await getSession(request) + if (!session) { + return new Response("Unauthorized", { status: 401 }) + } + + const formData = await request.formData() + const files = (formData.getAll("soilAnalysisFile") as File[]).filter( + (f) => f instanceof File && f.name, + ) + + if (files.length === 0) { + return new Response(JSON.stringify({ error: "No files uploaded" }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }) + } + + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + let nextIndex = 0 + const concurrency = 10 + + const workers = Array.from( + { length: Math.min(concurrency, files.length) }, + async () => { + while (nextIndex < files.length) { + const file = files[nextIndex++] + if (!file) continue + + try { + // Create a minimal FormData for a single file extraction + const singleFileFormData = new FormData() + singleFileFormData.append("soilAnalysisFile", file) + + const analyses = + await extractBulkSoilAnalyses( + singleFileFormData, + ) + + controller.enqueue( + encoder.encode( + `${JSON.stringify({ + success: true, + filename: file.name, + analyses, + })}\n`, + ), + ) + } catch (err) { + controller.enqueue( + encoder.encode( + `${JSON.stringify({ + success: false, + filename: file.name, + error: + err instanceof Error + ? err.message + : "Analyse mislukt", + })}\n`, + ), + ) + } + } + }, + ) + + await Promise.all(workers) + controller.close() + }, + }) + + return new Response(stream, { + headers: { + "Content-Type": "application/x-ndjson", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }, + }) +} diff --git a/fdm-app/app/routes/farm.$b_id_farm.soil-analysis.bulk.tsx b/fdm-app/app/routes/farm.$b_id_farm.soil-analysis.bulk.tsx index 20efea63e..8d7797ab2 100644 --- a/fdm-app/app/routes/farm.$b_id_farm.soil-analysis.bulk.tsx +++ b/fdm-app/app/routes/farm.$b_id_farm.soil-analysis.bulk.tsx @@ -4,7 +4,6 @@ import { HeaderFarm } from "~/components/blocks/header/farm" import { FarmTitle } from "~/components/blocks/farm/farm-title" import { FarmContent } from "~/components/blocks/farm/farm-content" import { - getFarm, getFarms, getFields, getSoilParametersDescription, @@ -27,10 +26,9 @@ import { BulkSoilAnalysisReview, type ProcessedAnalysis, } from "~/components/blocks/soil/bulk-upload-review" -import { extractBulkSoilAnalyses } from "~/integrations/nmi" -import { booleanPointInPolygon } from "@turf/turf" import { Spinner } from "~/components/ui/spinner" -import { redirectWithSuccess, dataWithSuccess } from "remix-toast" +import { redirectWithSuccess } from "remix-toast" +import { matchAnalysesToFields } from "~/components/blocks/soil/bulk-upload-match" export async function loader({ request, params }: LoaderFunctionArgs) { try { @@ -77,56 +75,26 @@ export default function BulkSoilAnalysisUploadPage() { const submit = useSubmit() const isSaving = - navigation.state === "submitting" && navigation.formData?.has("matches") + navigation.state !== "idle" && + navigation.formMethod?.toLowerCase() === "post" const handleUploadSuccess = (analyses: any[]) => { - // Perform geometry matching - const matchedAnalyses = analyses.map((analysis) => { - let matchedFieldId = "" - - if (analysis.location) { - const fieldMatch = fields.find((field) => { - if (!field.geometry) return false - try { - return booleanPointInPolygon( - analysis.location, - field.geometry, - ) - } catch (e) { - return false - } - }) - if (fieldMatch) matchedFieldId = fieldMatch.b_id - } - - // If no geometry match, try name match - if (!matchedFieldId) { - const fieldMatch = fields.find( - (field) => - field.b_name.toLowerCase() === - analysis.filename.replace(/\.pdf$/i, "").toLowerCase(), - ) - if (fieldMatch) matchedFieldId = fieldMatch.b_id - } - - return { - ...analysis, - matchedFieldId, - } - }) - + const matchedAnalyses = matchAnalysesToFields(analyses, fields) setProcessedAnalyses(matchedAnalyses) setStep("review") } - const handleSave = (matches: { analysisId: string; fieldId: string }[]) => { + const handleSave = ( + matches: { analysisId: string; fieldId: string }[], + updatedAnalyses: ProcessedAnalysis[], + ) => { const formData = new FormData() // Filter out "none" selections const validMatches = matches.filter( (m) => m.fieldId !== "none" && m.fieldId !== "", ) formData.append("matches", JSON.stringify(validMatches)) - formData.append("analysesData", JSON.stringify(processedAnalyses)) + formData.append("analysesData", JSON.stringify(updatedAnalyses)) submit(formData, { method: "post" }) } @@ -190,17 +158,6 @@ export async function action({ request, params }: ActionFunctionArgs) { const session = await getSession(request) const formData = await request.formData() - // Handle initial upload to NMI - if (formData.has("soilAnalysisFile")) { - const analyses = await extractBulkSoilAnalyses(formData) - return dataWithSuccess( - { analyses }, - { - message: `${analyses.length} analyses succesvol verwerkt`, - }, - ) - } - // Handle final save if (formData.has("matches")) { const matchesRaw = formData.get("matches") as string @@ -241,8 +198,21 @@ export async function action({ request, params }: ActionFunctionArgs) { `Analysis ${match.analysisId}: invalid b_sampling_date (${analysis.b_sampling_date})`, ) } - // Strip UI-only properties before saving to DB - const { id, location, ...dbAnalysis } = analysis + // Strip UI-only and redundant properties before saving to DB + const { + id, + location, + a_source, + matchedFieldId, + matchReason, + filename, + b_name, + b_sampling_date, + a_depth_upper, + a_depth_lower, + data: _data, // Strip raw data + ...dbAnalysis + } = analysis return addSoilAnalysis( fdm, @@ -267,6 +237,6 @@ export async function action({ request, params }: ActionFunctionArgs) { return data({ message: "Invalid request" }, { status: 400 }) } catch (error) { - throw handleActionError(error) + return handleActionError(error) } } diff --git a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.soil-analysis.bulk.tsx b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.soil-analysis.bulk.tsx index b7f1f87f8..87bc5fe8c 100644 --- a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.soil-analysis.bulk.tsx +++ b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.soil-analysis.bulk.tsx @@ -22,13 +22,13 @@ import { type ProcessedAnalysis, } from "~/components/blocks/soil/bulk-upload-review" import { extractBulkSoilAnalyses } from "~/integrations/nmi" -import { booleanPointInPolygon } from "@turf/turf" import { Header } from "~/components/blocks/header/base" import { HeaderFarmCreate } from "~/components/blocks/header/create-farm" import { SidebarInset } from "~/components/ui/sidebar" import { getCalendar, getTimeframe } from "~/lib/calendar" import { Spinner } from "~/components/ui/spinner" import { redirectWithSuccess, dataWithSuccess } from "remix-toast" +import { matchAnalysesToFields } from "~/components/blocks/soil/bulk-upload-match" export async function loader({ request, params }: LoaderFunctionArgs) { try { @@ -77,54 +77,26 @@ export default function BulkSoilAnalysisUploadWizardPage() { const submit = useSubmit() const isSaving = - navigation.state === "submitting" && navigation.formData?.has("matches") + navigation.state !== "idle" && + navigation.formMethod?.toLowerCase() === "post" const handleUploadSuccess = (analyses: any[]) => { - const matchedAnalyses = analyses.map((analysis) => { - let matchedFieldId = "" - - if (analysis.location) { - const fieldMatch = fields.find((field) => { - if (!field.geometry) return false - try { - return booleanPointInPolygon( - analysis.location, - field.geometry, - ) - } catch (e) { - return false - } - }) - if (fieldMatch) matchedFieldId = fieldMatch.b_id - } - - if (!matchedFieldId) { - const fieldMatch = fields.find( - (field) => - field.b_name.toLowerCase() === - analysis.filename.replace(/\.pdf$/i, "").toLowerCase(), - ) - if (fieldMatch) matchedFieldId = fieldMatch.b_id - } - - return { - ...analysis, - matchedFieldId, - } - }) - + const matchedAnalyses = matchAnalysesToFields(analyses, fields) setProcessedAnalyses(matchedAnalyses) setStep("review") } - const handleSave = (matches: { analysisId: string; fieldId: string }[]) => { + const handleSave = ( + matches: { analysisId: string; fieldId: string }[], + updatedAnalyses: ProcessedAnalysis[], + ) => { const formData = new FormData() // Filter out "none" selections const validMatches = matches.filter( (m) => m.fieldId !== "none" && m.fieldId !== "", ) formData.append("matches", JSON.stringify(validMatches)) - formData.append("analysesData", JSON.stringify(processedAnalyses)) + formData.append("analysesData", JSON.stringify(updatedAnalyses)) submit(formData, { method: "post" }) } @@ -185,16 +157,6 @@ export async function action({ request, params }: ActionFunctionArgs) { const calendar = getCalendar(params) const formData = await request.formData() - if (formData.has("soilAnalysisFile")) { - const analyses = await extractBulkSoilAnalyses(formData) - return dataWithSuccess( - { analyses }, - { - message: `${analyses.length} analyses succesvol verwerkt`, - }, - ) - } - if (formData.has("matches")) { const matches = JSON.parse(formData.get("matches") as string) const analysesData = JSON.parse( @@ -233,8 +195,21 @@ export async function action({ request, params }: ActionFunctionArgs) { `Analysis ${match.analysisId}: invalid b_sampling_date (${analysis.b_sampling_date})`, ) } - // Strip UI-only properties before saving to DB - const { id, location, a_source, ...dbAnalysis } = analysis + // Strip UI-only and redundant properties before saving to DB + const { + id, + location, + a_source, + matchedFieldId, + matchReason, + filename, + b_name, + b_sampling_date, + a_depth_upper, + a_depth_lower, + data: _data, // Strip raw data + ...dbAnalysis + } = analysis return addSoilAnalysis( fdm, @@ -262,6 +237,6 @@ export async function action({ request, params }: ActionFunctionArgs) { return data({ message: "Invalid request" }, { status: 400 }) } catch (error) { - throw handleActionError(error) + return handleActionError(error) } }