From 62d3f0edfaba4847f24eac32a835a020c319d647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Tue, 31 Mar 2026 09:11:34 +0200 Subject: [PATCH 01/44] Move shapefile parsing and handling code to fdm-rvo --- .../blocks/mijnpercelen/form-upload.tsx | 20 +- ...arm.create.$b_id_farm.$calendar.upload.tsx | 514 +++++++++++------- fdm-rvo/package.json | 11 + fdm-rvo/src/index.ts | 3 +- fdm-rvo/src/shapefile.test.ts | 46 ++ fdm-rvo/src/shapefile.ts | 158 ++++++ fdm-rvo/src/shpjs.d.ts | 1 + pnpm-lock.yaml | 45 +- 8 files changed, 584 insertions(+), 214 deletions(-) create mode 100644 fdm-rvo/src/shapefile.test.ts create mode 100644 fdm-rvo/src/shapefile.ts create mode 100644 fdm-rvo/src/shpjs.d.ts diff --git a/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx b/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx index 2b4f37088..ec5797c02 100644 --- a/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx +++ b/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx @@ -1,4 +1,5 @@ import { zodResolver } from "@hookform/resolvers/zod" +import { parseShapefileGeoJsonProperties } from "@nmi-agro/fdm-rvo/shapefile" import { AlertCircle, CheckCircle, @@ -7,9 +8,9 @@ import { FlaskConical, } from "lucide-react" import { useEffect, useRef, useState } from "react" +import { useWatch } from "react-hook-form" import { Form, NavLink, useActionData, useNavigation } from "react-router" import { RemixFormProvider, useRemixForm } from "remix-hook-form" -import { parseDbf } from "shpjs" import { toast as notify } from "sonner" import { z } from "zod" import { cn } from "@/app/lib/utils" @@ -36,7 +37,6 @@ import { FormMessage, } from "~/components/ui/form" import { Spinner } from "~/components/ui/spinner" - import { MijnPercelenUploadAnimation } from "./upload-animation" type UploadState = "idle" | "animating" | "success" | "error" @@ -60,6 +60,7 @@ export function MijnPercelenUploadForm({ mode: "onTouched", resolver: zodResolver(FormSchema), defaultValues: { + intent: "upload", shapefile: [], }, }) @@ -111,7 +112,11 @@ export function MijnPercelenUploadForm({ } }, [uploadState, form.reset]) - const selectedFiles = form.watch("shapefile") + const selectedFiles = useWatch({ + control: form.control, + name: "shapefile", + defaultValue: [], + }) const selectedFileExtensions = selectedFiles.map((file) => getFileExtension(file.name), @@ -135,8 +140,7 @@ export function MijnPercelenUploadForm({ ) if (dbfFile) { try { - const dbfBuffer = await dbfFile.arrayBuffer() - const dbfData = parseDbf(dbfBuffer) as any[] + const dbfData = await parseShapefileGeoJsonProperties(dbfFile) let unnamedCount = 0 const names = dbfData.map((row) => { const trimmedNaam = @@ -271,6 +275,11 @@ export function MijnPercelenUploadForm({ encType="multipart/form-data" >
+
() + const navigation = useNavigation() + const location = useLocation() + + const [rvoImportReviewData, setRvoImportReviewData] = + useState[]>() + const [userChoices, setUserChoices] = useState({}) + + const actionData = useActionData() + + const handleChoiceChange = (id: string, action: ImportReviewAction) => { + setUserChoices((prev: UserChoiceMap) => ({ ...prev, [id]: action })) + } + + const actionRvoImportReviewData = actionData?.RvoImportReviewData + + const isSaving = + navigation.state === "submitting" && + navigation.formData?.get("intent") === "save_fields" + + useEffect(() => { + if (actionRvoImportReviewData) { + setRvoImportReviewData(actionRvoImportReviewData) + } + }, [actionRvoImportReviewData]) + + useEffect(() => { + // Initialize user choices with defaults + const initialChoices: UserChoiceMap = {} + + if (rvoImportReviewData) { + rvoImportReviewData.forEach((item) => { + const id = getItemId(item) + let defaultAction: ImportReviewAction + + switch (item.status) { + case "NEW_REMOTE": + defaultAction = "ADD_REMOTE" + break + // In creation wizard, other statuses are unlikely but good to handle defaults + default: + defaultAction = "NO_ACTION" + break + } + initialChoices[id] = defaultAction + }) + } + setUserChoices(initialChoices) + }, [rvoImportReviewData]) + + // Warn the user before refreshing or leaving when data is present + const expectedRedirectPath = `/farm/create/${b_id_farm}/${calendar}/fields` + useEffect(() => { + if (rvoImportReviewData && rvoImportReviewData.length > 0) { + const handleBeforeUnload = (e: BeforeUnloadEvent) => { + if (location.pathname.startsWith(expectedRedirectPath)) return + e.preventDefault() + e.returnValue = + "Als u de pagina ververst, wordt de verbinding met RVO verbroken en moet u opnieuw inloggen met eHerkenning. Wilt u doorgaan?" + return e.returnValue + } + window.addEventListener("beforeunload", handleBeforeUnload) + return () => + window.removeEventListener("beforeunload", handleBeforeUnload) + } + }, [location.pathname, expectedRedirectPath, rvoImportReviewData]) return (
-
-
- -
+
+ {!rvoImportReviewData ? ( +
+ +
+ ) : ( + <> + {actionData?.message && ( +
+ + + {actionData.success ? "Succes" : "Fout"} + + + {actionData.message} + + +
+ )} + + + +
+
+
+ + + + +
+
+
+ +
+
+ + + )}
) } -interface RvoProperties { - SECTORID: string - SECTORVER: number - NEN3610ID: string - VOLGNR: number - NAAM: string | null | undefined - BEGINDAT: number - EINDDAT: number - GEWASCODE: string - GEWASOMSCH: string - TITEL: string - TITELOMSCH: string -} - -export async function action({ request, params }: ActionFunctionArgs) { - const fileStorage = createFsFileStorage("./uploads/shapefiles") +export async function action({ request, params }: ActionFunctionArgs): Promise< + | Response + | { + success?: boolean + message?: string + RvoImportReviewData?: RvoImportReviewItem[] + } +> { const storageKeys: string[] = [] - + const fileStorage = createFsFileStorage("./uploads/shapefiles") try { - // Get the Id and name of the farm - const b_id_farm = params.b_id_farm - if (!b_id_farm) { - throw data("Farm ID is required", { - status: 400, - statusText: "Farm ID is required", - }) + const { b_id_farm, calendar: yearString } = params + if (!b_id_farm || !yearString) { + throw data( + { + message: "b_id_farm and calendar are required", + success: false, + }, + { + status: 400, + }, + ) + } + const year = Number(yearString) + if (!Number.isInteger(year)) { + throw data( + { message: "Ongeldig kalenderjaar", success: false }, + { status: 400 }, + ) } const session = await getSession(request) - const calendar = await getCalendar(params) - const nmiApiKey = getNmiApiKey() + // Parse form data with streaming const uploadHandler = async (fileUpload: FileUpload) => { const storageKey = crypto.randomUUID() storageKeys.push(storageKey) @@ -132,169 +298,149 @@ export async function action({ request, params }: ActionFunctionArgs) { } return file } - const formData = await parseFormData( request, { maxFileSize: 5 * 1024 * 1024 }, uploadHandler, ) - const files = formData.getAll("shapefile") as File[] + const intent = formData.get("intent") + + if (intent === "upload") { + // Prepare existing fields for comparison + const fields = await getFields(fdm, session.principal_id, b_id_farm) + const fieldsExtended = await Promise.all( + fields.map(async (field) => ({ + ...field, + cultivations: await getCultivations( + fdm, + session.principal_id, + field.b_id, + ), + })), + ) + const cultivationsCatalogue = await getCultivationsFromCatalogue( + fdm, + session.principal_id, + b_id_farm, + ) - const shp_file = files.find((f) => f.name.endsWith(".shp")) - const shx_file = files.find((f) => f.name.endsWith(".shx")) - const dbf_file = files.find((f) => f.name.endsWith(".dbf")) - const prj_file = files.find((f) => f.name.endsWith(".prj")) + const files = formData.getAll("shapefile") as File[] - if (!shp_file || !shx_file || !dbf_file || !prj_file) { - return dataWithWarning( - {}, - "Een .shp, .shx, .dbf en .prj bestand zijn verplicht.", + const shp_file = files.find((f) => f.name.endsWith(".shp")) + const shx_file = files.find((f) => f.name.endsWith(".shx")) + const dbf_file = files.find((f) => f.name.endsWith(".dbf")) + const prj_file = files.find((f) => f.name.endsWith(".prj")) + + if (!shp_file || !shx_file || !dbf_file || !prj_file) { + const message = + "Een .shp, .shx, .dbf en .prj bestand zijn verplicht." + return { + message: message, + success: false, + RvoImportReviewData: undefined, + } + } + + const rvoFields = await getRvoFieldsFromShapefile( + shp_file, + shx_file, + dbf_file, + prj_file, ) - } - const shpBuffer = await shp_file.arrayBuffer() - const shxBuffer = await shx_file.arrayBuffer() - const dbfBuffer = await dbf_file.arrayBuffer() - const prj_text = await prj_file.text() - - let shapefile: FeatureCollection - try { - shapefile = (await combine([ - parseShp(shpBuffer, shxBuffer), - parseDbf(dbfBuffer), - ])) as FeatureCollection - } catch (_error) { - return dataWithWarning({}, "Shapefile is ongeldig.") - } + const RvoImportReviewData = compareFields( + fieldsExtended, + rvoFields, + year, + cultivationsCatalogue, + ) - if (shapefile.features.length === 0) { - return dataWithWarning({}, "Shapefile bevat geen percelen.") + return { + RvoImportReviewData: RvoImportReviewData, + message: "Percelen zijn klaar voor beeordeling! 🎉", + success: true, + } } - const source_proj = prj_text - const dest_proj = "EPSG:4326" - - const converter = proj4(source_proj, dest_proj) - - const features = shapefile.features.map( - (feature: Feature) => { - const new_coords = feature.geometry.coordinates.map( - (ring: number[][]) => { - return ring.map((coord: number[]) => { - return converter.forward(coord) - }) - }, - ) - feature.geometry.coordinates = new_coords - return feature - }, - ) + if (intent === "save_fields") { + const RvoImportReviewDataJson = formData.get( + "RvoImportReviewDataJson", + ) + const userChoicesJson = formData.get("userChoices") + + let rvoImportReviewData: RvoImportReviewItem[] = [] + let userChoices: UserChoiceMap = {} - let unnamedCount = 0 - for (const feature of features) { - const { properties, geometry } = feature - const { - SECTORID, - SECTORVER, - NEN3610ID, - VOLGNR, - NAAM, - BEGINDAT, - EINDDAT, - GEWASCODE, - GEWASOMSCH, - TITEL, - TITELOMSCH, - } = properties - - if ( - !SECTORID || - !SECTORVER || - !NEN3610ID || - !VOLGNR || - NAAM === undefined || - !BEGINDAT || - !EINDDAT || - !GEWASCODE || - !GEWASOMSCH || - !TITEL || - !TITELOMSCH - ) { - return dataWithWarning( - {}, - "De shapefile bevat niet de vereiste RVO attributen.", - ) + if (!RvoImportReviewDataJson || !userChoicesJson) { + return { + success: false, + message: + "Geen data gevonden om te verwerken. Start de RVO import opnieuw.", + RvoImportReviewData: undefined, + } } - const b_geometry = turf.polygon(geometry.coordinates) - const trimmedNaam = typeof NAAM === "string" ? NAAM.trim() : "" - const b_name = trimmedNaam || `Naamloos perceel ${++unnamedCount}` - const b_start = new Date(BEGINDAT) - const b_end = EINDDAT === 253402297199 ? null : new Date(EINDDAT) - const b_lu_catalogue = `nl_${GEWASCODE}` - const b_acquiring_method = `nl_${TITEL}` - const b_id_source = SECTORID + rvoImportReviewData = JSON.parse(String(RvoImportReviewDataJson)) + userChoices = JSON.parse(String(userChoicesJson)) - const fieldId = await addField( - fdm, - session.principal_id, - b_id_farm, - b_name, - b_id_source, - b_geometry.geometry, - b_start, - b_acquiring_method, - b_end, - ) + if (!Array.isArray(rvoImportReviewData)) { + throw new Error("Invalid review data format") + } + + const onFieldAdded = async ( + tx: FdmType, + b_id: string, + geometry: any, + ) => { + const nmiApiKey = getNmiApiKey() + if (nmiApiKey) { + try { + const soilEstimates = await getSoilParameterEstimates( + geometry, + nmiApiKey, + ) + await addSoilAnalysis( + tx, + session.principal_id, + undefined, + "nl-other-nmi", + b_id, + soilEstimates.a_depth_lower ?? 30, + undefined, + soilEstimates, + soilEstimates.a_depth_upper, + ) + } catch (e) { + console.warn( + `Failed to fetch soil estimates for field ${b_id}:`, + e, + ) + } + } + } - const cultivationDefaultDates = await getDefaultDatesOfCultivation( + await processRvoImport( fdm, session.principal_id, b_id_farm, - b_lu_catalogue, - Number(calendar), - ) - const b_lu_start = cultivationDefaultDates.b_lu_start - const b_lu_end = cultivationDefaultDates.b_lu_end - await addCultivation( - fdm, - session.principal_id, - b_lu_catalogue, - fieldId, - b_lu_start, - b_lu_end, + rvoImportReviewData, + userChoices, + year, + onFieldAdded, ) - - if (nmiApiKey) { - const estimates = await getSoilParameterEstimates( - b_geometry, - nmiApiKey, - ) - - await addSoilAnalysis( - fdm, - session.principal_id, - undefined, - estimates.a_source, - fieldId, - estimates.a_depth_lower, - undefined, - estimates, - ) - } + return redirect(`/farm/create/${b_id_farm}/${yearString}/fields`) } - return redirectWithSuccess( - `/farm/create/${b_id_farm}/${calendar}/fields`, - { - message: "Percelen zijn succesvol geïmporteerd! 🎉", - }, - ) - } catch (error) { - throw handleActionError(error) + return {} + } catch (e: any) { + console.error("Error at saving RVO fields: ", e) + return { + success: false, + message: `Error at saving RVO fields: ${await extractErrorMessage(e)}`, + } } finally { - for (const key of storageKeys) { - await fileStorage.remove(key) + for (const storageKey of storageKeys) { + fileStorage.remove(storageKey) } } } diff --git a/fdm-rvo/package.json b/fdm-rvo/package.json index 25bcb15be..b4733fd15 100644 --- a/fdm-rvo/package.json +++ b/fdm-rvo/package.json @@ -37,6 +37,12 @@ "types": "./dist/utils.d.ts", "default": "./dist/utils.js" } + }, + "./shapefile": { + "import": { + "types": "./dist/shapefile.d.ts", + "default": "./dist/shapefile.js" + } } }, "files": [ @@ -57,13 +63,18 @@ "@turf/bbox": "^7.3.4", "@turf/helpers": "^7.3.4", "@turf/intersect": "^7.3.4", + "@turf/turf": "^7.3.4", "@turf/union": "^7.3.4", + "geojson": "^0.5.0", + "proj4": "^2.20.4", + "shpjs": "^6.2.0", "zod": "^4.3.6" }, "devDependencies": { "@nmi-agro/fdm-core": "workspace:*", "@rollup/plugin-commonjs": "catalog:", "@rollup/plugin-node-resolve": "catalog:", + "@types/geojson": "^7946.0.16", "@types/node": "catalog:", "@vitest/coverage-v8": "catalog:", "rollup": "catalog:", diff --git a/fdm-rvo/src/index.ts b/fdm-rvo/src/index.ts index ad1433521..c8c962806 100644 --- a/fdm-rvo/src/index.ts +++ b/fdm-rvo/src/index.ts @@ -29,6 +29,7 @@ export * from "./auth" export * from "./compare" export * from "./data" -export * from "./types" export * from "./process" +export * from "./shapefile" +export * from "./types" export * from "./utils" diff --git a/fdm-rvo/src/shapefile.test.ts b/fdm-rvo/src/shapefile.test.ts new file mode 100644 index 000000000..05807dd48 --- /dev/null +++ b/fdm-rvo/src/shapefile.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, test } from "vitest" +import { parseShapefileGeometry } from "./shapefile" + +describe("parseShapefileGeometry", () => { + it("should throw an error for an invalid shapefile", async () => { + const shp = new File([new Uint8Array()], "invalid.shp") + expect(parseShapefileGeometry(shp)).rejects.toThrow( + "Shapefile is not valid", + ) + }) + + it("should return an empty feature collection on empty glob", async () => { + const bounds = ( + minX: number, + minY: number, + maxX: number, + maxY: number, + ) => new Uint32Array(new Float64Array([minX, minY, maxX, maxY]).buffer) // 16 words + const data = new Uint32Array([ + 0x0a270000, // magic number + 0, + 0, + 0, + 0, + 0, + 50 << 24, // file length in big endian, only the header + 0, + 5, // each feature is a polygon + ...bounds(5, 50, 7, 55), // bounding box + ...bounds(0, 0, 0, 0), // third and fourth dimension bounds + ]) + const shp = new File([data], "empty.shp") + expect(await parseShapefileGeometry(shp)).toHaveLength(0) + }) +}) + +describe("", () => { + it("should throw an error for an invalid dbf file", async () => { + const dbf = new File([new Uint8Array()], "invalid.dbf") + expect(parseShapefileGeometry(dbf)).rejects.toThrow( + "Shapefile is not valid", + ) + }) + + it("should map the data fields correctly", async () => {}) +}) diff --git a/fdm-rvo/src/shapefile.ts b/fdm-rvo/src/shapefile.ts new file mode 100644 index 000000000..31df0cec6 --- /dev/null +++ b/fdm-rvo/src/shapefile.ts @@ -0,0 +1,158 @@ +import type { + Feature, + FeatureCollection, + GeoJsonProperties, + Geometry, + Polygon, +} from "geojson" +import proj4 from "proj4" +import { combine, parseDbf, parseShp } from "shpjs" +import type { RvoField } from "./types" + +interface RvoProperties { + SECTORID: string + SECTORVER: number + NEN3610ID: string + VOLGNR: number + NAAM: string | null | undefined + BEGINDAT: number + EINDDAT: number + GEWASCODE: string + GEWASOMSCH: string + TITEL: string + TITELOMSCH: string +} + +export async function parseShapefileGeometry( + shp_file: Blob, + shx_file?: Blob, +): Promise { + try { + return parseShp( + await shp_file.arrayBuffer(), + await shx_file?.arrayBuffer(), + ) + } catch (_error) { + throw new Error("Shapefile is not valid", { cause: _error }) + } +} + +export async function parseShapefileGeoJsonProperties( + dbf_file: Blob, +): Promise { + try { + return parseDbf(await dbf_file.arrayBuffer(), undefined) + } catch (_error) { + throw new Error("DBF file is not valid", { cause: _error }) + } +} + +export async function getRvoFieldsFromShapefile( + shp_file: Blob, + shx_file: Blob, + dbf_file: Blob, + prj_file: Blob, +) { + const prj_text = await prj_file.text() + + let shapefile: FeatureCollection + const shapefileGeometry = await parseShapefileGeometry(shp_file, shx_file) + const shapefileGeoJsonProperties = + await parseShapefileGeoJsonProperties(dbf_file) + try { + shapefile = combine([ + shapefileGeometry, + shapefileGeoJsonProperties, + ]) as FeatureCollection + } catch (_error) { + throw new Error("Shapefile is not valid", { cause: _error }) + } + + if (shapefile.features.length === 0) { + throw new Error("Shapefile does not contain any fields") + } + + const source_proj = prj_text + const dest_proj = "EPSG:4326" + + const converter = + source_proj !== null ? proj4(source_proj, dest_proj) : null + + const features = shapefile.features.map( + (feature: Feature) => { + const new_coords = feature.geometry.coordinates.map( + (ring: number[][]) => { + return ring.map((coord: number[]) => { + return converter !== null + ? converter.forward(coord) + : coord + }) + }, + ) + feature.geometry.coordinates = new_coords + return feature + }, + ) + + const fields: RvoField[] = [] + for (const feature of features) { + const { properties, geometry } = feature + const { + SECTORID, + SECTORVER, + NEN3610ID, + VOLGNR, + NAAM, + BEGINDAT, + EINDDAT, + GEWASCODE, + GEWASOMSCH, + TITEL, + TITELOMSCH, + } = properties + + if ( + !SECTORID || + !SECTORVER || + !NEN3610ID || + !VOLGNR || + NAAM === undefined || + !BEGINDAT || + !EINDDAT || + !GEWASCODE || + !GEWASOMSCH || + !TITEL || + !TITELOMSCH + ) { + throw new Error("Field does not have the required attributes") + } + + const trimmedNaam = typeof NAAM === "string" ? NAAM.trim() : "" + + fields.push({ + type: "Feature", + geometry: geometry, + properties: { + CropFieldID: SECTORID, // b_id_source + CropFieldVersion: "1.0.0", // not needed + CropFieldDesignator: trimmedNaam, // b_name + BeginDate: new Date(BEGINDAT).toISOString(), // b_start + Country: "nl", // b_lu_catalogue[0] + CropTypeCode: GEWASCODE, // b_lu_catalogue[1] + UseTitleCode: TITEL, // b_acquiring_method + ThirdPartyCropFieldID: undefined, // not needed + EndDate: + EINDDAT !== 253402297199 + ? new Date(EINDDAT).toISOString() + : undefined, // b_end + VarietyCode: undefined, // not needed + CropProductionPurposeCode: undefined, // not needed + FieldUseCode: undefined, // not needed + RegulatorySoiltypeCode: undefined, // not needed + CropFieldCause: undefined, // not needed + }, + }) + } + + return fields +} diff --git a/fdm-rvo/src/shpjs.d.ts b/fdm-rvo/src/shpjs.d.ts new file mode 100644 index 000000000..f7f7c9ba2 --- /dev/null +++ b/fdm-rvo/src/shpjs.d.ts @@ -0,0 +1 @@ +declare module "shpjs" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0689d5ce6..8f35f7b3a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,15 +6,9 @@ settings: catalogs: default: - '@dotenvx/dotenvx': - specifier: ^1.54.1 - version: 1.57.2 '@rollup/plugin-commonjs': specifier: ^29.0.2 version: 29.0.2 - '@rollup/plugin-json': - specifier: ^6.1.0 - version: 6.1.0 '@rollup/plugin-node-resolve': specifier: ^16.0.3 version: 16.0.3 @@ -24,36 +18,18 @@ catalogs: '@vitest/coverage-v8': specifier: 4.1.0 version: 4.1.0 - better-auth: - specifier: ^1.5.4 - version: 1.5.5 - drizzle-kit: - specifier: ^0.31.9 - version: 0.31.10 - drizzle-orm: - specifier: ^0.45.1 - version: 0.45.1 rollup: specifier: ^4.59.0 version: 4.60.0 rollup-plugin-esbuild: specifier: ^6.2.1 version: 6.2.1 - rollup-plugin-polyfill-node: - specifier: ^0.13.0 - version: 0.13.0 typedoc: specifier: ^0.28.17 version: 0.28.18 - typedoc-plugin-missing-exports: - specifier: ^4.1.2 - version: 4.1.2 typescript: specifier: ^5.9.3 version: 5.9.3 - vite: - specifier: ^7.3.1 - version: 7.3.1 vitest: specifier: ^4.1.0 version: 4.1.1 @@ -603,9 +579,21 @@ importers: '@turf/intersect': specifier: ^7.3.4 version: 7.3.4 + '@turf/turf': + specifier: ^7.3.4 + version: 7.3.4 '@turf/union': specifier: ^7.3.4 version: 7.3.4 + geojson: + specifier: ^0.5.0 + version: 0.5.0 + proj4: + specifier: ^2.20.4 + version: 2.20.4 + shpjs: + specifier: ^6.2.0 + version: 6.2.0 zod: specifier: ^4.3.6 version: 4.3.6 @@ -616,6 +604,9 @@ importers: '@rollup/plugin-node-resolve': specifier: 'catalog:' version: 16.0.3(rollup@4.60.0) + '@types/geojson': + specifier: ^7946.0.16 + version: 7946.0.16 '@types/node': specifier: 'catalog:' version: 25.5.0 @@ -7420,6 +7411,10 @@ packages: geojson-polygon-self-intersections@1.2.2: resolution: {integrity: sha512-6XRNF4CsRHYmR9z5YuIk5f/aOototnDf0dgMqYGcS7y1l57ttt6MAIAxl3rXyas6lq1HEbTuLMh4PgvO+OV42w==} + geojson@0.5.0: + resolution: {integrity: sha512-/Bx5lEn+qRF4TfQ5aLu6NH+UKtvIv7Lhc487y/c8BdludrCTpiWf9wyI0RTyqg49MFefIAvFDuEi5Dfd/zgNxQ==} + engines: {node: '>= 0.10'} + geokdbush@2.0.1: resolution: {integrity: sha512-0M8so1Qx6+jJ1xpirpCNrgUsWAzIcQ3LrLmh0KJPBYI3gH7vy70nY5zEEjSp9Tn0nBt6Q2Fh922oL08lfib4Zg==} @@ -19942,6 +19937,8 @@ snapshots: dependencies: rbush: 2.0.2 + geojson@0.5.0: {} + geokdbush@2.0.1: dependencies: tinyqueue: 2.0.3 From 865a264658dfc7fa48b2944b2a4847a36c169ec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Tue, 31 Mar 2026 09:18:16 +0200 Subject: [PATCH 02/44] Remove shpjs from fdm-app --- ...arm.create.$b_id_farm.$calendar.upload.tsx | 4 +-- fdm-rvo/package.json | 1 - pnpm-lock.yaml | 27 ++++++++++++++++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx index 6d69931cb..4069a1cc8 100644 --- a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx +++ b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx @@ -1,4 +1,3 @@ -import { MijnPercelenUploadForm } from "@/app/components/blocks/mijnpercelen/form-upload" import { addSoilAnalysis, type FdmType, @@ -8,12 +7,12 @@ import { getFarm, getFields, } from "@nmi-agro/fdm-core" -import { getItemId } from "@nmi-agro/fdm-rvo/utils" import type { ImportReviewAction, RvoImportReviewItem, UserChoiceMap, } from "@nmi-agro/fdm-rvo/types" +import { getItemId } from "@nmi-agro/fdm-rvo/utils" import { createFsFileStorage } from "@remix-run/file-storage/fs" import { type FileUpload, parseFormData } from "@remix-run/form-data-parser" import { Loader2 } from "lucide-react" @@ -36,6 +35,7 @@ import { FarmContent } from "~/components/blocks/farm/farm-content" import { FarmTitle } from "~/components/blocks/farm/farm-title" import { Header } from "~/components/blocks/header/base" import { HeaderFarmCreate } from "~/components/blocks/header/create-farm" +import { MijnPercelenUploadForm } from "~/components/blocks/mijnpercelen/form-upload" import { RvoImportReviewTable } from "~/components/blocks/rvo/import-review-table" import { Alert, AlertDescription, AlertTitle } from "~/components/ui/alert" import { Button } from "~/components/ui/button" diff --git a/fdm-rvo/package.json b/fdm-rvo/package.json index b4733fd15..18a9e6922 100644 --- a/fdm-rvo/package.json +++ b/fdm-rvo/package.json @@ -67,7 +67,6 @@ "@turf/union": "^7.3.4", "geojson": "^0.5.0", "proj4": "^2.20.4", - "shpjs": "^6.2.0", "zod": "^4.3.6" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8f35f7b3a..5d5a3f4df 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,9 +6,15 @@ settings: catalogs: default: + '@dotenvx/dotenvx': + specifier: ^1.54.1 + version: 1.57.2 '@rollup/plugin-commonjs': specifier: ^29.0.2 version: 29.0.2 + '@rollup/plugin-json': + specifier: ^6.1.0 + version: 6.1.0 '@rollup/plugin-node-resolve': specifier: ^16.0.3 version: 16.0.3 @@ -18,18 +24,36 @@ catalogs: '@vitest/coverage-v8': specifier: 4.1.0 version: 4.1.0 + better-auth: + specifier: ^1.5.4 + version: 1.5.5 + drizzle-kit: + specifier: ^0.31.9 + version: 0.31.10 + drizzle-orm: + specifier: ^0.45.1 + version: 0.45.1 rollup: specifier: ^4.59.0 version: 4.60.0 rollup-plugin-esbuild: specifier: ^6.2.1 version: 6.2.1 + rollup-plugin-polyfill-node: + specifier: ^0.13.0 + version: 0.13.0 typedoc: specifier: ^0.28.17 version: 0.28.18 + typedoc-plugin-missing-exports: + specifier: ^4.1.2 + version: 4.1.2 typescript: specifier: ^5.9.3 version: 5.9.3 + vite: + specifier: ^7.3.1 + version: 7.3.1 vitest: specifier: ^4.1.0 version: 4.1.1 @@ -591,9 +615,6 @@ importers: proj4: specifier: ^2.20.4 version: 2.20.4 - shpjs: - specifier: ^6.2.0 - version: 6.2.0 zod: specifier: ^4.3.6 version: 4.3.6 From 54d54c0dfdc7e4a225fb6a20fb358ca58c46d7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Tue, 31 Mar 2026 09:24:08 +0200 Subject: [PATCH 03/44] Add shpjs import back --- fdm-rvo/package.json | 1 + pnpm-lock.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/fdm-rvo/package.json b/fdm-rvo/package.json index 18a9e6922..b4733fd15 100644 --- a/fdm-rvo/package.json +++ b/fdm-rvo/package.json @@ -67,6 +67,7 @@ "@turf/union": "^7.3.4", "geojson": "^0.5.0", "proj4": "^2.20.4", + "shpjs": "^6.2.0", "zod": "^4.3.6" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5d5a3f4df..b83aaa006 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -615,6 +615,9 @@ importers: proj4: specifier: ^2.20.4 version: 2.20.4 + shpjs: + specifier: ^6.2.0 + version: 6.2.0 zod: specifier: ^4.3.6 version: 4.3.6 From d86e15a8974546ae9a5e6c00cba4a5e7bfe59769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Fri, 3 Apr 2026 11:21:18 +0200 Subject: [PATCH 04/44] Remove shpjs dependency from fdm-app --- fdm-app/package.json | 1 - pnpm-lock.yaml | 3 --- 2 files changed, 4 deletions(-) diff --git a/fdm-app/package.json b/fdm-app/package.json index 678725138..38dab851f 100644 --- a/fdm-app/package.json +++ b/fdm-app/package.json @@ -78,7 +78,6 @@ "remix-hook-form": "7.1.1", "remix-toast": "^4.0.0", "remix-utils": "^9.3.1", - "shpjs": "^6.2.0", "sonner": "^2.0.7", "tailwind-merge": "^3.5.0", "tailwindcss-animate": "^1.0.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 684ecc59a..73fb53ecb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -323,9 +323,6 @@ importers: remix-utils: specifier: ^9.3.1 version: 9.3.1(@standard-schema/spec@1.1.0)(react-router@7.13.2(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4) - shpjs: - specifier: ^6.2.0 - version: 6.2.0 sonner: specifier: ^2.0.7 version: 2.0.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) From 7b562d71bbfe9fdd2b89592a86d25a19979155d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Tue, 7 Apr 2026 11:55:40 +0200 Subject: [PATCH 05/44] Test shapefile.ts --- .../blocks/mijnpercelen/form-upload.tsx | 2 +- fdm-rvo/src/shapefile.test.ts | 179 ++++++++++++++---- fdm-rvo/src/shapefile.ts | 23 +-- 3 files changed, 154 insertions(+), 50 deletions(-) diff --git a/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx b/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx index ec5797c02..352b12787 100644 --- a/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx +++ b/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx @@ -1,5 +1,5 @@ import { zodResolver } from "@hookform/resolvers/zod" -import { parseShapefileGeoJsonProperties } from "@nmi-agro/fdm-rvo/shapefile" +import { parseShapefileAttributes } from "@nmi-agro/fdm-rvo/shapefile" import { AlertCircle, CheckCircle, diff --git a/fdm-rvo/src/shapefile.test.ts b/fdm-rvo/src/shapefile.test.ts index 05807dd48..fb30166e8 100644 --- a/fdm-rvo/src/shapefile.test.ts +++ b/fdm-rvo/src/shapefile.test.ts @@ -1,46 +1,153 @@ -import { describe, expect, it, test } from "vitest" -import { parseShapefileGeometry } from "./shapefile" +import { geometry } from "@turf/helpers" +import * as shpjs from "shpjs" +import { beforeEach, describe, expect, it, vi } from "vitest" +import { getRvoFieldsFromShapefile } from "./shapefile" +import type { Geometry, Polygon } from "geojson" +import proj4 from "proj4" -describe("parseShapefileGeometry", () => { - it("should throw an error for an invalid shapefile", async () => { - const shp = new File([new Uint8Array()], "invalid.shp") - expect(parseShapefileGeometry(shp)).rejects.toThrow( - "Shapefile is not valid", +vi.mock("shpjs", async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + combine: vi.fn(actual.combine), + parseShp: vi.fn(actual.parseShp), + parseDbf: vi.fn(actual.parseDbf), + } +}) + +describe("getRvoFieldsFromShapefile", () => { + beforeEach(async () => { + vi.mocked(shpjs.combine).mockReset() + vi.mocked(shpjs.parseDbf).mockReset() + vi.mocked(shpjs.parseShp).mockReset() + }) + + it("should throw an error with invalid geometry", async () => { + vi.mocked(shpjs.parseShp).mockRejectedValueOnce( + new Error("Failed to parse shp"), ) + + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([]) + + await expect( + getRvoFieldsFromShapefile( + new File([], "invalid.shp"), + undefined, + new File([], "invalid.dbf"), + new File([], "invalid.prj"), + ), + ).rejects.toThrow("Shapefile is not valid") }) - it("should return an empty feature collection on empty glob", async () => { - const bounds = ( - minX: number, - minY: number, - maxX: number, - maxY: number, - ) => new Uint32Array(new Float64Array([minX, minY, maxX, maxY]).buffer) // 16 words - const data = new Uint32Array([ - 0x0a270000, // magic number - 0, - 0, - 0, - 0, - 0, - 50 << 24, // file length in big endian, only the header - 0, - 5, // each feature is a polygon - ...bounds(5, 50, 7, 55), // bounding box - ...bounds(0, 0, 0, 0), // third and fourth dimension bounds - ]) - const shp = new File([data], "empty.shp") - expect(await parseShapefileGeometry(shp)).toHaveLength(0) + it("should throw an error with invalid attributes", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([]) + + vi.mocked(shpjs.parseDbf).mockRejectedValueOnce( + new Error("Failed to parse dbf"), + ) + + await expect( + getRvoFieldsFromShapefile( + new File([], "invalid.shp"), + undefined, + new File([], "invalid.dbf"), + new File([], "invalid.prj"), + ), + ).rejects.toThrow("Shapefile is not valid") }) -}) -describe("", () => { - it("should throw an error for an invalid dbf file", async () => { - const dbf = new File([new Uint8Array()], "invalid.dbf") - expect(parseShapefileGeometry(dbf)).rejects.toThrow( - "Shapefile is not valid", + it("should throw an error with no fields", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([]) + + await expect( + getRvoFieldsFromShapefile( + new File([], "invalid.shp"), + undefined, + new File([], "invalid.shx"), + new File([], "invalid.prj"), + ), + ).rejects.toThrow("Shapefile does not contain any fields") + }) + + const createMockGeometry = () => + geometry("Polygon", [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 1], + [0, 0], + ], + ]) + + const MOCK_PROPERTIES = { + // Relevant + SECTORID: "test_b_id_source", // b_id_source + NAAM: " Field 1 ", // b_name + BEGINDAT: 1704067200000, // b_start + EINDDAT: 1706659200000, // b_end + GEWASCODE: "02", // b_lu_catalogue[1] + TITEL: "Geliberaliseerde pacht, 6 jaar of korter", // b_acquiring_method + + // Irrelevant + SECTORVER: "1.0.0", + NEN3610ID: "unique", + VOLGNR: 1, + GEWASOMSCH: "Krokus, bloembollen en -knollen", + TITELOMSCH: "Purchased by the test farm", + } + + it("should map the data fields correctly", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) + + const parsed = await getRvoFieldsFromShapefile( + new File([], "shapefile.shp"), + undefined, + new File([], "shapefile.dbf"), + new File(["EPSG:4326"], "shapefile.prj"), + ) + + expect(parsed).toHaveLength(1) + + expect(parsed[0].properties.CropFieldID).toBe("test_b_id_source") + expect(parsed[0].properties.CropFieldDesignator).toBe("Field 1") + expect(new Date(parsed[0].properties.BeginDate).getTime()).toBe( + 1704067200000, + ) + expect(new Date(parsed[0].properties.EndDate ?? "").getTime()).toBe( + 1706659200000, + ) + expect(parsed[0].properties.CropTypeCode).toBe("02") + expect(parsed[0].properties.UseTitleCode).toBe( + "Geliberaliseerde pacht, 6 jaar of korter", ) }) - it("should map the data fields correctly", async () => {}) + it("should project field geometry", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) + + const parsed = await getRvoFieldsFromShapefile( + new File([], "shapefile.shp"), + undefined, + new File([], "shapefile.dbf"), + // Identity transform + new File(["EPSG:3785"], "shapefile.prj"), + ) + + const projector = proj4("EPSG:3785", "EPSG:4326") + const expectedCoords = [ + createMockGeometry().coordinates[0].map((coord) => + projector.forward(coord), + ), + ] + + expect(parsed).toHaveLength(1) + expect((parsed[0].geometry as Geometry).type).toBe("Polygon") + expect((parsed[0].geometry as Polygon).coordinates).toStrictEqual( + expectedCoords, + ) + }) }) diff --git a/fdm-rvo/src/shapefile.ts b/fdm-rvo/src/shapefile.ts index 31df0cec6..c7b4dfa6d 100644 --- a/fdm-rvo/src/shapefile.ts +++ b/fdm-rvo/src/shapefile.ts @@ -28,7 +28,7 @@ export async function parseShapefileGeometry( shx_file?: Blob, ): Promise { try { - return parseShp( + return await parseShp( await shp_file.arrayBuffer(), await shx_file?.arrayBuffer(), ) @@ -37,32 +37,32 @@ export async function parseShapefileGeometry( } } -export async function parseShapefileGeoJsonProperties( +export async function parseShapefileAttributes( dbf_file: Blob, ): Promise { try { - return parseDbf(await dbf_file.arrayBuffer(), undefined) + return await parseDbf(await dbf_file.arrayBuffer(), undefined) } catch (_error) { - throw new Error("DBF file is not valid", { cause: _error }) + throw new Error("Shapefile is not valid", { cause: _error }) } } export async function getRvoFieldsFromShapefile( shp_file: Blob, - shx_file: Blob, + shx_file: Blob | undefined, dbf_file: Blob, - prj_file: Blob, + prj_file: Blob | undefined, ) { - const prj_text = await prj_file.text() + const source_proj = prj_file ? await prj_file.text() : null + const dest_proj = "EPSG:4326" let shapefile: FeatureCollection const shapefileGeometry = await parseShapefileGeometry(shp_file, shx_file) - const shapefileGeoJsonProperties = - await parseShapefileGeoJsonProperties(dbf_file) + const shapefileAttributes = await parseShapefileAttributes(dbf_file) try { shapefile = combine([ shapefileGeometry, - shapefileGeoJsonProperties, + shapefileAttributes, ]) as FeatureCollection } catch (_error) { throw new Error("Shapefile is not valid", { cause: _error }) @@ -72,9 +72,6 @@ export async function getRvoFieldsFromShapefile( throw new Error("Shapefile does not contain any fields") } - const source_proj = prj_text - const dest_proj = "EPSG:4326" - const converter = source_proj !== null ? proj4(source_proj, dest_proj) : null From e59d08f8ecee2167ae42c6f8e686de91ca67b898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Tue, 7 Apr 2026 12:27:56 +0200 Subject: [PATCH 06/44] Improve coverage --- .../blocks/mijnpercelen/form-upload.tsx | 4 +- fdm-rvo/src/shapefile.test.ts | 59 +++++++++++++++++-- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx b/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx index 352b12787..c08df502a 100644 --- a/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx +++ b/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx @@ -1,5 +1,5 @@ import { zodResolver } from "@hookform/resolvers/zod" -import { parseShapefileAttributes } from "@nmi-agro/fdm-rvo/shapefile" +import { parseShapefileAttributes } from "@nmi-agro/fdm-rvo" import { AlertCircle, CheckCircle, @@ -140,7 +140,7 @@ export function MijnPercelenUploadForm({ ) if (dbfFile) { try { - const dbfData = await parseShapefileGeoJsonProperties(dbfFile) + const dbfData = await parseShapefileAttributes(dbfFile) let unnamedCount = 0 const names = dbfData.map((row) => { const trimmedNaam = diff --git a/fdm-rvo/src/shapefile.test.ts b/fdm-rvo/src/shapefile.test.ts index fb30166e8..0e3f40e28 100644 --- a/fdm-rvo/src/shapefile.test.ts +++ b/fdm-rvo/src/shapefile.test.ts @@ -1,9 +1,9 @@ import { geometry } from "@turf/helpers" +import type { Geometry, Polygon } from "geojson" +import proj4 from "proj4" import * as shpjs from "shpjs" import { beforeEach, describe, expect, it, vi } from "vitest" import { getRvoFieldsFromShapefile } from "./shapefile" -import type { Geometry, Polygon } from "geojson" -import proj4 from "proj4" vi.mock("shpjs", async (importOriginal) => { const actual = await importOriginal() @@ -17,9 +17,7 @@ vi.mock("shpjs", async (importOriginal) => { describe("getRvoFieldsFromShapefile", () => { beforeEach(async () => { - vi.mocked(shpjs.combine).mockReset() - vi.mocked(shpjs.parseDbf).mockReset() - vi.mocked(shpjs.parseShp).mockReset() + vi.resetAllMocks() }) it("should throw an error with invalid geometry", async () => { @@ -106,7 +104,7 @@ describe("getRvoFieldsFromShapefile", () => { new File([], "shapefile.shp"), undefined, new File([], "shapefile.dbf"), - new File(["EPSG:4326"], "shapefile.prj"), + undefined, ) expect(parsed).toHaveLength(1) @@ -125,6 +123,23 @@ describe("getRvoFieldsFromShapefile", () => { ) }) + it("should handle null ending date", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([ + { ...MOCK_PROPERTIES, EINDDAT: 253402297199 }, + ]) + + const parsed = await getRvoFieldsFromShapefile( + new File([], "shapefile.shp"), + undefined, + new File([], "shapefile.dbf"), + undefined, + ) + + expect(parsed).toHaveLength(1) + expect(parsed[0].properties.EndDate).toBeUndefined() + }) + it("should project field geometry", async () => { vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) @@ -150,4 +165,36 @@ describe("getRvoFieldsFromShapefile", () => { expectedCoords, ) }) + + it("should throw an error if there are missing but required properties", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) + const { NAAM, ...otherProps } = MOCK_PROPERTIES + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([otherProps]) + + await expect( + getRvoFieldsFromShapefile( + new File([], "shapefile.shp"), + undefined, + new File([], "shapefile.dbf"), + new File(["EPSG:4326"], "shapefile.prj"), + ), + ).rejects.toThrow("Field does not have the required attributes") + }) + + it("should throw an error if the shapefile does not match the attributes file", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([ + createMockGeometry(), + createMockGeometry(), + ]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) + + await expect( + getRvoFieldsFromShapefile( + new File([], "shapefile.shp"), + undefined, + new File([], "shapefile.dbf"), + new File(["EPSG:4326"], "shapefile.prj"), + ), + ).rejects.toThrow("Field does not have the required attributes") + }) }) From 8f7a21daa8520147ea75758d713f764fe0df7837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Tue, 7 Apr 2026 15:12:02 +0200 Subject: [PATCH 07/44] Separate functionality better in shapefile.ts --- fdm-rvo/src/shapefile.test.ts | 224 +++++++++++++++++------------- fdm-rvo/src/shapefile.ts | 250 ++++++++++++++++++++-------------- 2 files changed, 276 insertions(+), 198 deletions(-) diff --git a/fdm-rvo/src/shapefile.test.ts b/fdm-rvo/src/shapefile.test.ts index 0e3f40e28..43468735c 100644 --- a/fdm-rvo/src/shapefile.test.ts +++ b/fdm-rvo/src/shapefile.test.ts @@ -3,7 +3,10 @@ import type { Geometry, Polygon } from "geojson" import proj4 from "proj4" import * as shpjs from "shpjs" import { beforeEach, describe, expect, it, vi } from "vitest" -import { getRvoFieldsFromShapefile } from "./shapefile" +import { + convertShapefileFeatureIntoRvoField, + getRvoFieldsFromShapefile, +} from "./shapefile" vi.mock("shpjs", async (importOriginal) => { const actual = await importOriginal() @@ -15,87 +18,35 @@ vi.mock("shpjs", async (importOriginal) => { } }) -describe("getRvoFieldsFromShapefile", () => { - beforeEach(async () => { - vi.resetAllMocks() - }) - - it("should throw an error with invalid geometry", async () => { - vi.mocked(shpjs.parseShp).mockRejectedValueOnce( - new Error("Failed to parse shp"), - ) - - vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([]) - - await expect( - getRvoFieldsFromShapefile( - new File([], "invalid.shp"), - undefined, - new File([], "invalid.dbf"), - new File([], "invalid.prj"), - ), - ).rejects.toThrow("Shapefile is not valid") - }) - - it("should throw an error with invalid attributes", async () => { - vi.mocked(shpjs.parseShp).mockResolvedValueOnce([]) - - vi.mocked(shpjs.parseDbf).mockRejectedValueOnce( - new Error("Failed to parse dbf"), - ) - - await expect( - getRvoFieldsFromShapefile( - new File([], "invalid.shp"), - undefined, - new File([], "invalid.dbf"), - new File([], "invalid.prj"), - ), - ).rejects.toThrow("Shapefile is not valid") - }) - - it("should throw an error with no fields", async () => { - vi.mocked(shpjs.parseShp).mockResolvedValueOnce([]) - vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([]) - - await expect( - getRvoFieldsFromShapefile( - new File([], "invalid.shp"), - undefined, - new File([], "invalid.shx"), - new File([], "invalid.prj"), - ), - ).rejects.toThrow("Shapefile does not contain any fields") - }) - - const createMockGeometry = () => - geometry("Polygon", [ - [ - [0, 0], - [1, 0], - [1, 1], - [0, 1], - [0, 0], - ], - ]) - - const MOCK_PROPERTIES = { - // Relevant - SECTORID: "test_b_id_source", // b_id_source - NAAM: " Field 1 ", // b_name - BEGINDAT: 1704067200000, // b_start - EINDDAT: 1706659200000, // b_end - GEWASCODE: "02", // b_lu_catalogue[1] - TITEL: "Geliberaliseerde pacht, 6 jaar of korter", // b_acquiring_method - - // Irrelevant - SECTORVER: "1.0.0", - NEN3610ID: "unique", - VOLGNR: 1, - GEWASOMSCH: "Krokus, bloembollen en -knollen", - TITELOMSCH: "Purchased by the test farm", - } +const createMockGeometry = () => + geometry("Polygon", [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 1], + [0, 0], + ], + ]) + +const MOCK_PROPERTIES = { + // Relevant + SECTORID: "test_b_id_source", // b_id_source + NAAM: " Field 1 ", // b_name + BEGINDAT: 1704067200000, // b_start + EINDDAT: 1706659200000, // b_end + GEWASCODE: "02", // b_lu_catalogue[1] + TITEL: "Geliberaliseerde pacht, 6 jaar of korter", // b_acquiring_method + + // Irrelevant + SECTORVER: 1, + NEN3610ID: "unique", + VOLGNR: 1, + GEWASOMSCH: "Krokus, bloembollen en -knollen", + TITELOMSCH: "Purchased by the test farm", +} +describe("getRvoFieldsFromShapefile", () => { it("should map the data fields correctly", async () => { vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) @@ -160,25 +111,49 @@ describe("getRvoFieldsFromShapefile", () => { ] expect(parsed).toHaveLength(1) + expect((parsed[0].geometry as Geometry).type).toBe("Polygon") expect((parsed[0].geometry as Polygon).coordinates).toStrictEqual( expectedCoords, ) }) - it("should throw an error if there are missing but required properties", async () => { - vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) - const { NAAM, ...otherProps } = MOCK_PROPERTIES - vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([otherProps]) + it("should handle multi-polygon geometry", async () => { + const MOCK_MULTIPOLYGON = geometry("MultiPolygon", [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 1], + [0, 0], + ], + [ + [1, 0], + [2, 0], + [2, 1], + [1, 1], + [1, 0], + ], + ]) + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([MOCK_MULTIPOLYGON]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([ + MOCK_PROPERTIES, + { ...MOCK_PROPERTIES, NAAM: "Field 2" }, + ]) - await expect( - getRvoFieldsFromShapefile( - new File([], "shapefile.shp"), - undefined, - new File([], "shapefile.dbf"), - new File(["EPSG:4326"], "shapefile.prj"), - ), - ).rejects.toThrow("Field does not have the required attributes") + const parsed = await getRvoFieldsFromShapefile( + new File([], "shapefile.shp"), + undefined, + new File([], "shapefile.dbf"), + undefined, + ) + + expect(parsed).toHaveLength(1) + + expect((parsed[0].geometry as Geometry).type).toBe("MultiPolygon") + expect((parsed[0].geometry as Polygon).coordinates).toStrictEqual( + MOCK_MULTIPOLYGON.coordinates, + ) }) it("should throw an error if the shapefile does not match the attributes file", async () => { @@ -197,4 +172,67 @@ describe("getRvoFieldsFromShapefile", () => { ), ).rejects.toThrow("Field does not have the required attributes") }) + + it("should accept array buffers and strings", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) + + await expect( + getRvoFieldsFromShapefile( + new ArrayBuffer(), + new ArrayBuffer(), + new ArrayBuffer(), + "EPSG:3785", + ), + ).resolves.toBeDefined() + }) +}) + +describe("convertShapefileFeatureIntoRvoField", () => { + beforeEach(async () => { + vi.resetAllMocks() + }) + + it("should map the data fields correctly", () => { + const parsed = convertShapefileFeatureIntoRvoField({ + type: "Feature", + geometry: createMockGeometry(), + properties: MOCK_PROPERTIES, + }) + + expect(parsed.properties.CropFieldID).toBe("test_b_id_source") + expect(parsed.properties.CropFieldDesignator).toBe("Field 1") + expect(new Date(parsed.properties.BeginDate).getTime()).toBe( + 1704067200000, + ) + expect(new Date(parsed.properties.EndDate ?? "").getTime()).toBe( + 1706659200000, + ) + expect(parsed.properties.CropTypeCode).toBe("02") + expect(parsed.properties.UseTitleCode).toBe( + "Geliberaliseerde pacht, 6 jaar of korter", + ) + }) + + it("should handle null ending date", () => { + const parsed = convertShapefileFeatureIntoRvoField({ + type: "Feature", + geometry: createMockGeometry(), + properties: { ...MOCK_PROPERTIES, EINDDAT: 253402297199 }, + }) + + expect(parsed.properties.EndDate).toBeUndefined() + }) + + it("should throw an error if there are missing but required properties", () => { + const { NAAM, ...otherProps } = MOCK_PROPERTIES + + expect(() => + convertShapefileFeatureIntoRvoField({ + type: "Feature", + geometry: createMockGeometry(), + properties: otherProps, + }), + ).toThrow("Field does not have the required attributes") + }) }) diff --git a/fdm-rvo/src/shapefile.ts b/fdm-rvo/src/shapefile.ts index c7b4dfa6d..5c0a8364b 100644 --- a/fdm-rvo/src/shapefile.ts +++ b/fdm-rvo/src/shapefile.ts @@ -1,9 +1,10 @@ +import { multiPolygon, polygon } from "@turf/helpers" import type { Feature, FeatureCollection, - GeoJsonProperties, Geometry, Polygon, + Position, } from "geojson" import proj4 from "proj4" import { combine, parseDbf, parseShp } from "shpjs" @@ -23,47 +24,165 @@ interface RvoProperties { TITELOMSCH: string } +type FileInterface = Blob | ArrayBuffer + +/** + * Parses the files found in a MijnPercelen Shapefile export and compiles a GeoJSON feature collection where each feature's properties represent the field properties registered by RVO. + * @param shp_file Shapefile or ArrayBuffer to parse + * @param _shx_file Shapefile index or ArrayBuffer file to parse, the library might be able to optimize lookups in the shp file using this + * @param prj_file Projection definition file or ArrayBuffer for coordinates found in the shp file, if not provided EPSG:4326 is assumed + * @returns an array of geometries which can be passed to the shpjs combine function + */ export async function parseShapefileGeometry( - shp_file: Blob, - shx_file?: Blob, + shp_file: FileInterface, + shx_file: FileInterface | undefined, + prj_file: Blob | string | undefined, ): Promise { try { - return await parseShp( - await shp_file.arrayBuffer(), - await shx_file?.arrayBuffer(), - ) + const [shpData, shxData, projection] = await Promise.all([ + shp_file instanceof Blob ? await shp_file.arrayBuffer() : shp_file, + shx_file instanceof Blob ? await shx_file.arrayBuffer() : shx_file, + prj_file instanceof Blob ? await prj_file.text() : prj_file, + ]) + + const geometries: Geometry[] = await parseShp(shpData, shxData) + + const projector = projection + ? proj4(projection, "EPSG:4326") + : undefined + + return geometries.map((geometry) => { + const transformRing = (ring: Position[][]) => + projector + ? ring.map((coords) => + coords.map((coord) => projector.forward(coord)), + ) + : ring + + if (geometry.type === "MultiPolygon") { + return multiPolygon(geometry.coordinates.map(transformRing)) + .geometry + } + + if (geometry.type === "Polygon") { + return polygon(transformRing(geometry.coordinates)).geometry + } + + throw new Error("Non-polygonal geometry encountered") + }) } catch (_error) { throw new Error("Shapefile is not valid", { cause: _error }) } } +/** + * Parses the dbf file that is part of the MijnPercelen Shapefile export + * @param dbf_file DBase file or ArrayBuffer to parse + * @returns an array of objects representing the rows in the dbf file + */ export async function parseShapefileAttributes( - dbf_file: Blob, -): Promise { + dbf_file: FileInterface, +): Promise[]> { try { - return await parseDbf(await dbf_file.arrayBuffer(), undefined) + return await parseDbf( + dbf_file instanceof Blob ? await dbf_file.arrayBuffer() : dbf_file, + undefined, + ) } catch (_error) { throw new Error("Shapefile is not valid", { cause: _error }) } } +/** + * Converts a feature found in a Shapefile to a RvoField object to be used with the RVO import system + * @param shapefileFeatures + * @returns a RvoField object + * @throws if any of the required properties are missing + */ +export function convertShapefileFeatureIntoRvoField( + feature: Feature>, +): RvoField { + const { properties, geometry } = feature + const { + SECTORID, + SECTORVER, + NEN3610ID, + VOLGNR, + NAAM, + BEGINDAT, + EINDDAT, + GEWASCODE, + GEWASOMSCH, + TITEL, + TITELOMSCH, + } = properties + + if ( + !SECTORID || + !SECTORVER || + !NEN3610ID || + !VOLGNR || + NAAM === undefined || + !BEGINDAT || + !EINDDAT || + !GEWASCODE || + !GEWASOMSCH || + !TITEL || + !TITELOMSCH + ) { + throw new Error("Field does not have the required attributes") + } + + const trimmedNaam = typeof NAAM === "string" ? NAAM.trim() : "" + + return { + type: "Feature", + geometry: geometry, + properties: { + CropFieldID: SECTORID, // b_id_source + CropFieldVersion: "1.0.0", // not needed + CropFieldDesignator: trimmedNaam, // b_name + BeginDate: new Date(BEGINDAT).toISOString(), // b_start + Country: "nl", // b_lu_catalogue[0] + CropTypeCode: GEWASCODE, // b_lu_catalogue[1] + UseTitleCode: TITEL, // b_acquiring_method + ThirdPartyCropFieldID: undefined, // not needed + EndDate: + EINDDAT !== 253402297199 + ? new Date(EINDDAT).toISOString() + : undefined, // b_end + VarietyCode: undefined, // not needed + CropProductionPurposeCode: undefined, // not needed + FieldUseCode: undefined, // not needed + RegulatorySoiltypeCode: undefined, // not needed + CropFieldCause: undefined, // not needed + }, + } +} + +/** + * Parses the files found in a MijnPercelen Shapefile export and compiles a GeoJSON feature collection where each feature's properties represent the field properties registered by RVO. + * @param shp_file Shapefile or ArrayBuffer to parse + * @param shx_file Shapefile index or ArrayBuffer file to parse, the library might be able to optimize lookups in the shp file using this + * @param dbf_file DBase file or ArrayBuffer to parse containing field properties registered by RVO + * @param prj_file Projection definition file or ArrayBuffer for coordinates found in the shp file, if not provided EPSG:4326 is assumed + * @returns + */ export async function getRvoFieldsFromShapefile( - shp_file: Blob, - shx_file: Blob | undefined, - dbf_file: Blob, - prj_file: Blob | undefined, -) { - const source_proj = prj_file ? await prj_file.text() : null - const dest_proj = "EPSG:4326" - - let shapefile: FeatureCollection - const shapefileGeometry = await parseShapefileGeometry(shp_file, shx_file) - const shapefileAttributes = await parseShapefileAttributes(dbf_file) + shp_file: FileInterface, + shx_file: FileInterface | undefined, + dbf_file: FileInterface, + prj_file: Blob | string | undefined, +): Promise { + let shapefile: FeatureCollection> + const geometries = await parseShapefileGeometry( + shp_file, + shx_file, + prj_file, + ) + const attributes = await parseShapefileAttributes(dbf_file) try { - shapefile = combine([ - shapefileGeometry, - shapefileAttributes, - ]) as FeatureCollection + shapefile = combine([geometries, attributes]) } catch (_error) { throw new Error("Shapefile is not valid", { cause: _error }) } @@ -72,84 +191,5 @@ export async function getRvoFieldsFromShapefile( throw new Error("Shapefile does not contain any fields") } - const converter = - source_proj !== null ? proj4(source_proj, dest_proj) : null - - const features = shapefile.features.map( - (feature: Feature) => { - const new_coords = feature.geometry.coordinates.map( - (ring: number[][]) => { - return ring.map((coord: number[]) => { - return converter !== null - ? converter.forward(coord) - : coord - }) - }, - ) - feature.geometry.coordinates = new_coords - return feature - }, - ) - - const fields: RvoField[] = [] - for (const feature of features) { - const { properties, geometry } = feature - const { - SECTORID, - SECTORVER, - NEN3610ID, - VOLGNR, - NAAM, - BEGINDAT, - EINDDAT, - GEWASCODE, - GEWASOMSCH, - TITEL, - TITELOMSCH, - } = properties - - if ( - !SECTORID || - !SECTORVER || - !NEN3610ID || - !VOLGNR || - NAAM === undefined || - !BEGINDAT || - !EINDDAT || - !GEWASCODE || - !GEWASOMSCH || - !TITEL || - !TITELOMSCH - ) { - throw new Error("Field does not have the required attributes") - } - - const trimmedNaam = typeof NAAM === "string" ? NAAM.trim() : "" - - fields.push({ - type: "Feature", - geometry: geometry, - properties: { - CropFieldID: SECTORID, // b_id_source - CropFieldVersion: "1.0.0", // not needed - CropFieldDesignator: trimmedNaam, // b_name - BeginDate: new Date(BEGINDAT).toISOString(), // b_start - Country: "nl", // b_lu_catalogue[0] - CropTypeCode: GEWASCODE, // b_lu_catalogue[1] - UseTitleCode: TITEL, // b_acquiring_method - ThirdPartyCropFieldID: undefined, // not needed - EndDate: - EINDDAT !== 253402297199 - ? new Date(EINDDAT).toISOString() - : undefined, // b_end - VarietyCode: undefined, // not needed - CropProductionPurposeCode: undefined, // not needed - FieldUseCode: undefined, // not needed - RegulatorySoiltypeCode: undefined, // not needed - CropFieldCause: undefined, // not needed - }, - }) - } - - return fields + return shapefile.features.map(convertShapefileFeatureIntoRvoField) } From c8401dd6e7c378f5eb635d62d60158f652e72733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Tue, 7 Apr 2026 16:34:21 +0200 Subject: [PATCH 08/44] Add back deleted tests --- fdm-rvo/src/shapefile.test.ts | 107 +++++++++++++++++++++------------- fdm-rvo/src/shapefile.ts | 10 ++-- 2 files changed, 72 insertions(+), 45 deletions(-) diff --git a/fdm-rvo/src/shapefile.test.ts b/fdm-rvo/src/shapefile.test.ts index 43468735c..75996de64 100644 --- a/fdm-rvo/src/shapefile.test.ts +++ b/fdm-rvo/src/shapefile.test.ts @@ -47,38 +47,13 @@ const MOCK_PROPERTIES = { } describe("getRvoFieldsFromShapefile", () => { - it("should map the data fields correctly", async () => { - vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) - vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) - - const parsed = await getRvoFieldsFromShapefile( - new File([], "shapefile.shp"), - undefined, - new File([], "shapefile.dbf"), - undefined, - ) - - expect(parsed).toHaveLength(1) - - expect(parsed[0].properties.CropFieldID).toBe("test_b_id_source") - expect(parsed[0].properties.CropFieldDesignator).toBe("Field 1") - expect(new Date(parsed[0].properties.BeginDate).getTime()).toBe( - 1704067200000, - ) - expect(new Date(parsed[0].properties.EndDate ?? "").getTime()).toBe( - 1706659200000, - ) - expect(parsed[0].properties.CropTypeCode).toBe("02") - expect(parsed[0].properties.UseTitleCode).toBe( - "Geliberaliseerde pacht, 6 jaar of korter", - ) + beforeEach(async () => { + vi.resetAllMocks() }) - it("should handle null ending date", async () => { + it("should get RvoField objects from Shapefile", async () => { vi.mocked(shpjs.parseShp).mockResolvedValueOnce([createMockGeometry()]) - vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([ - { ...MOCK_PROPERTIES, EINDDAT: 253402297199 }, - ]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) const parsed = await getRvoFieldsFromShapefile( new File([], "shapefile.shp"), @@ -88,7 +63,6 @@ describe("getRvoFieldsFromShapefile", () => { ) expect(parsed).toHaveLength(1) - expect(parsed[0].properties.EndDate).toBeUndefined() }) it("should project field geometry", async () => { @@ -118,6 +92,54 @@ describe("getRvoFieldsFromShapefile", () => { ) }) + it("should throw an error with invalid geometry", async () => { + vi.mocked(shpjs.parseShp).mockRejectedValueOnce( + new Error("Failed to parse shp"), + ) + + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([]) + + await expect( + getRvoFieldsFromShapefile( + new File([], "invalid.shp"), + undefined, + new File([], "invalid.dbf"), + new File([], "invalid.prj"), + ), + ).rejects.toThrow("Shapefile is not valid") + }) + + it("should throw an error with invalid attributes", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([]) + + vi.mocked(shpjs.parseDbf).mockRejectedValueOnce( + new Error("Failed to parse shp"), + ) + + await expect( + getRvoFieldsFromShapefile( + new File([], "invalid.shp"), + undefined, + new File([], "invalid.dbf"), + new File([], "invalid.prj"), + ), + ).rejects.toThrow("Shapefile is not valid") + }) + + it("should throw an error with no fields", async () => { + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([]) + + await expect( + getRvoFieldsFromShapefile( + new File([], "invalid.shp"), + undefined, + new File([], "invalid.shx"), + new File([], "invalid.prj"), + ), + ).rejects.toThrow("Shapefile does not contain any fields") + }) + it("should handle multi-polygon geometry", async () => { const MOCK_MULTIPOLYGON = geometry("MultiPolygon", [ [ @@ -136,10 +158,7 @@ describe("getRvoFieldsFromShapefile", () => { ], ]) vi.mocked(shpjs.parseShp).mockResolvedValueOnce([MOCK_MULTIPOLYGON]) - vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([ - MOCK_PROPERTIES, - { ...MOCK_PROPERTIES, NAAM: "Field 2" }, - ]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) const parsed = await getRvoFieldsFromShapefile( new File([], "shapefile.shp"), @@ -156,10 +175,10 @@ describe("getRvoFieldsFromShapefile", () => { ) }) - it("should throw an error if the shapefile does not match the attributes file", async () => { + it("should handle not-supported geometry", async () => { + const MOCK_UNSUPPORTED_GEOMETRY = geometry("Point", [0, 0]) vi.mocked(shpjs.parseShp).mockResolvedValueOnce([ - createMockGeometry(), - createMockGeometry(), + MOCK_UNSUPPORTED_GEOMETRY, ]) vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) @@ -168,9 +187,9 @@ describe("getRvoFieldsFromShapefile", () => { new File([], "shapefile.shp"), undefined, new File([], "shapefile.dbf"), - new File(["EPSG:4326"], "shapefile.prj"), + undefined, ), - ).rejects.toThrow("Field does not have the required attributes") + ).rejects.toThrow("Shapefile is not valid") }) it("should accept array buffers and strings", async () => { @@ -214,6 +233,16 @@ describe("convertShapefileFeatureIntoRvoField", () => { ) }) + it("should trim NAAM", () => { + const parsed = convertShapefileFeatureIntoRvoField({ + type: "Feature", + geometry: createMockGeometry(), + properties: { ...MOCK_PROPERTIES, NAAM: " Test Name " }, + }) + + expect(parsed.properties.CropFieldDesignator).toBe("Test Name") + }) + it("should handle null ending date", () => { const parsed = convertShapefileFeatureIntoRvoField({ type: "Feature", diff --git a/fdm-rvo/src/shapefile.ts b/fdm-rvo/src/shapefile.ts index 5c0a8364b..81fdc9981 100644 --- a/fdm-rvo/src/shapefile.ts +++ b/fdm-rvo/src/shapefile.ts @@ -174,18 +174,16 @@ export async function getRvoFieldsFromShapefile( dbf_file: FileInterface, prj_file: Blob | string | undefined, ): Promise { - let shapefile: FeatureCollection> const geometries = await parseShapefileGeometry( shp_file, shx_file, prj_file, ) const attributes = await parseShapefileAttributes(dbf_file) - try { - shapefile = combine([geometries, attributes]) - } catch (_error) { - throw new Error("Shapefile is not valid", { cause: _error }) - } + const shapefile: FeatureCollection< + Polygon, + Partial + > = combine([geometries, attributes]) if (shapefile.features.length === 0) { throw new Error("Shapefile does not contain any fields") From 4d355b42a0e7afae87936213d74b2a397f032ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Tue, 7 Apr 2026 16:45:34 +0200 Subject: [PATCH 09/44] Fix fdm-app imports --- fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx | 2 +- .../app/routes/farm.create.$b_id_farm.$calendar.upload.tsx | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx b/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx index c08df502a..4ca09e4d2 100644 --- a/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx +++ b/fdm-app/app/components/blocks/mijnpercelen/form-upload.tsx @@ -1,5 +1,5 @@ import { zodResolver } from "@hookform/resolvers/zod" -import { parseShapefileAttributes } from "@nmi-agro/fdm-rvo" +import { parseShapefileAttributes } from "@nmi-agro/fdm-rvo/shapefile" import { AlertCircle, CheckCircle, diff --git a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx index 4069a1cc8..e093d3ea1 100644 --- a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx +++ b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx @@ -7,6 +7,7 @@ import { getFarm, getFields, } from "@nmi-agro/fdm-core" +import { getRvoFieldsFromShapefile } from "@nmi-agro/fdm-rvo/shapefile" import type { ImportReviewAction, RvoImportReviewItem, @@ -49,11 +50,7 @@ import { getCalendar } from "~/lib/calendar" import { clientConfig } from "~/lib/config" import { extractErrorMessage } from "~/lib/error" import { fdm } from "~/lib/fdm.server" -import { - compareFields, - getRvoFieldsFromShapefile, - processRvoImport, -} from "~/lib/rvo.server" +import { compareFields, processRvoImport } from "~/lib/rvo.server" export const handle = { hideNavigationProgress: true } From f40f64364016be907f351e71f481a2b2db49a211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Fri, 10 Apr 2026 11:04:27 +0200 Subject: [PATCH 10/44] Make the rvo feature flag true by default just like the gerrit flag --- fdm-app/app/root.tsx | 1 + fdm-app/app/routes/farm.$b_id_farm.$calendar.rvo.tsx | 2 +- fdm-app/app/routes/farm.$b_id_farm._index.tsx | 2 +- fdm-app/app/routes/farm.create.$b_id_farm.$calendar._index.tsx | 2 +- fdm-app/app/routes/farm.create.$b_id_farm.$calendar.rvo.tsx | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fdm-app/app/root.tsx b/fdm-app/app/root.tsx index c1b6102b5..6ff2e471b 100644 --- a/fdm-app/app/root.tsx +++ b/fdm-app/app/root.tsx @@ -98,6 +98,7 @@ export function Layout() { bootstrap: { featureFlags: { gerrit: false, + rvo: false, }, }, loaded: () => {}, diff --git a/fdm-app/app/routes/farm.$b_id_farm.$calendar.rvo.tsx b/fdm-app/app/routes/farm.$b_id_farm.$calendar.rvo.tsx index be51d7894..e67d8596e 100644 --- a/fdm-app/app/routes/farm.$b_id_farm.$calendar.rvo.tsx +++ b/fdm-app/app/routes/farm.$b_id_farm.$calendar.rvo.tsx @@ -231,7 +231,7 @@ export default function RvoImportReviewPage() { const navigation = useNavigation() const location = useLocation() - const isRvoEnabled = useFeatureFlagEnabled("rvo") + const isRvoEnabled = useFeatureFlagEnabled("rvo") ?? true const isImporting = navigation.state === "submitting" && diff --git a/fdm-app/app/routes/farm.$b_id_farm._index.tsx b/fdm-app/app/routes/farm.$b_id_farm._index.tsx index 7a784bb25..121bf9889 100644 --- a/fdm-app/app/routes/farm.$b_id_farm._index.tsx +++ b/fdm-app/app/routes/farm.$b_id_farm._index.tsx @@ -156,7 +156,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { export default function FarmDashboardIndex() { const loaderData = useLoaderData() - const isRvoEnabled = useFeatureFlagEnabled("rvo") + const isRvoEnabled = useFeatureFlagEnabled("rvo") ?? true const calendar = useCalendarStore((state) => state.calendar) const setCalendar = useCalendarStore((state) => state.setCalendar) diff --git a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar._index.tsx b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar._index.tsx index 5723853b3..96c4a7d44 100644 --- a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar._index.tsx +++ b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar._index.tsx @@ -64,7 +64,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { export default function ChooseFieldImportMethod() { const { farm, isRvoConfigured } = useLoaderData() - const isRvoEnabled = useFeatureFlagEnabled("rvo") + const isRvoEnabled = useFeatureFlagEnabled("rvo") ?? true const showRvoOption = isRvoConfigured && isRvoEnabled !== false return ( diff --git a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.rvo.tsx b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.rvo.tsx index 846bc8a2b..a2ad5cde9 100644 --- a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.rvo.tsx +++ b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.rvo.tsx @@ -205,7 +205,7 @@ export default function RvoImportCreatePage() { const navigation = useNavigation() const location = useLocation() - const isRvoEnabled = useFeatureFlagEnabled("rvo") + const isRvoEnabled = useFeatureFlagEnabled("rvo") ?? true const isImporting = navigation.state === "submitting" && From fa72a25b112d9a8aabb7d2a43db6019a5076c12b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Fri, 10 Apr 2026 16:37:11 +0200 Subject: [PATCH 11/44] Add buffer strip checkboxes to the Shapefile import --- .../blocks/rvo/import-review-table.tsx | 130 +++++++++++++++--- ...arm.create.$b_id_farm.$calendar.upload.tsx | 94 +++++++++++-- fdm-core/src/index.ts | 1 + fdm-rvo/package.json | 1 + fdm-rvo/src/shapefile.test.ts | 72 +++++----- fdm-rvo/src/shapefile.ts | 41 +++++- pnpm-lock.yaml | 5 +- 7 files changed, 279 insertions(+), 65 deletions(-) diff --git a/fdm-app/app/components/blocks/rvo/import-review-table.tsx b/fdm-app/app/components/blocks/rvo/import-review-table.tsx index 868340718..02d55cf05 100644 --- a/fdm-app/app/components/blocks/rvo/import-review-table.tsx +++ b/fdm-app/app/components/blocks/rvo/import-review-table.tsx @@ -8,14 +8,24 @@ import { type ColumnDef, flexRender, getCoreRowModel, + type RowData, useReactTable, } from "@tanstack/react-table" import { area } from "@turf/turf" import { format, parseISO } from "date-fns" -import { Archive, ArrowLeftRight, Check, Plus, Trash2, X } from "lucide-react" -import { useMemo } from "react" +import { + Archive, + ArrowLeftRight, + Check, + Info, + Plus, + Trash2, + X, +} from "lucide-react" +import { type ReactNode, useMemo } from "react" import { clientConfig } from "@/app/lib/config" import { Badge } from "~/components/ui/badge" +import { Checkbox } from "~/components/ui/checkbox" import { Select, SelectContent, @@ -41,15 +51,25 @@ import { acquiringMethodOptions } from "~/lib/constants" import { cn } from "~/lib/utils" declare module "@tanstack/react-table" { - interface TableMeta { + interface TableMeta { userChoices: UserChoiceMap + flags?: ImportReviewFlags + /** Function to replace a review item. `getItemId(replacement)` will return the same value as the original. */ + onItemChange?: (id: string, item: RvoImportReviewItem) => void onChoiceChange: (id: string, action: ImportReviewAction) => void } } +export interface ImportReviewFlags { + b_bufferstrip_info_available?: boolean +} + interface RvoImportReviewTableProps { data: RvoImportReviewItem[] userChoices: UserChoiceMap + flags?: ImportReviewFlags + /** Function to replace a review item. `getItemId(replacement)` will return the same value as the original. Replacements won't work if this is not provided. */ + onItemChange?: (id: string, action: RvoImportReviewItem) => void onChoiceChange: (id: string, action: ImportReviewAction) => void } @@ -77,31 +97,39 @@ const DiffCell = ({ status, action, formatter = (v: any) => v, + rvoFormatter, + debug, }: { local?: any remote?: any status: string action: ImportReviewAction - formatter?: (v: any) => React.ReactNode + formatter?: (v: any) => ReactNode + rvoFormatter?: (v: any) => ReactNode + debug?: boolean }) => { // If MATCH, just show one value - if (status === "MATCH") { + if (!rvoFormatter && status === "MATCH") { + if (debug) console.debug("MATCH") return ( {formatter(local)} ) } // NEW REMOTE -> Show remote without badge - if (status === "NEW_REMOTE") { + if (!rvoFormatter && status === "NEW_REMOTE") { return ( - {formatter(remote)} + {(rvoFormatter ?? formatter)(remote)} ) } // NEW LOCAL -> Show local without badge - if (status === "NEW_LOCAL" || status === "EXPIRED_LOCAL") { + if ( + !rvoFormatter && + (status === "NEW_LOCAL" || status === "EXPIRED_LOCAL") + ) { return ( {formatter(local)} @@ -110,9 +138,11 @@ const DiffCell = ({ } // CONFLICT - if (status === "CONFLICT") { + if (rvoFormatter || status === "CONFLICT") { + if (debug) console.debug("CONFLICT") // If values are effectively equal (deep check), show one - if (JSON.stringify(local) === JSON.stringify(remote)) { + if (!rvoFormatter && JSON.stringify(local) === JSON.stringify(remote)) { + if (debug) console.debug("CONFLICT SAME") return ( {formatter(local)} @@ -181,7 +211,7 @@ const DiffCell = ({ useRemote && "font-bold", )} > - {formatter(remote)} + {(rvoFormatter ?? formatter)(remote)}
)} @@ -485,18 +515,36 @@ export const columns: ColumnDef>[] = [ }, { id: "bufferstrook", - header: () => ( + header: ({ table }) => ( - Bufferstrook + + Bufferstrook + {table.options.meta?.flags?.b_bufferstrip_info_available === + false && } + - Geeft aan of het perceel bij RVO geregistreerd staat als - bufferstrook. +
+

+ Geeft aan of het perceel bij RVO geregistreerd staat + als bufferstrook. +

+ {table.options.meta?.flags + ?.b_bufferstrip_info_available === false && ( +

+ Geen info uit het Shapefile is beschikbaar, dus + deze waarden zijn de oude waarden of + schattingen. +

+ )} +
), cell: ({ row, table }) => { const item = row.original - const { userChoices } = table.options.meta! + const { userChoices, flags } = table.options.meta! + const bufferstripInfoAvailable = + flags?.b_bufferstrip_info_available !== false const action = userChoices[getItemId(item)] as ImportReviewAction const rvoBufferstrip = @@ -515,13 +563,57 @@ export const columns: ColumnDef>[] = [ ? "Ja" : "Nee" + function handleUpdateValue(newValue: boolean) { + if (table.options.meta?.onItemChange) { + table.options.meta.onItemChange(getItemId(row.original), { + ...row.original, + rvoField: { + ...row.original.rvoField, + properties: { + ...row.original.rvoField?.properties, + mestData: { + ...row.original.rvoField?.properties + .mestData, + IndBufferstrook: newValue ? "J" : "N", + }, + }, + }, + } as RvoImportReviewItem) + } + } + return ( val ?? "-"} + formatter={(val) => ( +
+ {val ?? "-"} +
+ )} + rvoFormatter={ + bufferstripInfoAvailable + ? undefined + : (originalValue: "Ja" | "Nee" | undefined) => { + const value = originalValue === "Ja" + return ( + // biome-ignore lint/a11y/noLabelWithoutControl: the input is nested inside the label + + ) + } + } /> ) }, @@ -632,6 +724,8 @@ export const columns: ColumnDef>[] = [ export function RvoImportReviewTable({ data, userChoices, + flags, + onItemChange, onChoiceChange, }: RvoImportReviewTableProps) { const sortedData = useMemo(() => { @@ -660,6 +754,8 @@ export function RvoImportReviewTable({ getCoreRowModel: getCoreRowModel(), meta: { userChoices, + flags, + onItemChange, onChoiceChange, }, }) diff --git a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx index e093d3ea1..e2a5cf483 100644 --- a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx +++ b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx @@ -7,7 +7,6 @@ import { getFarm, getFields, } from "@nmi-agro/fdm-core" -import { getRvoFieldsFromShapefile } from "@nmi-agro/fdm-rvo/shapefile" import type { ImportReviewAction, RvoImportReviewItem, @@ -50,7 +49,13 @@ import { getCalendar } from "~/lib/calendar" import { clientConfig } from "~/lib/config" import { extractErrorMessage } from "~/lib/error" import { fdm } from "~/lib/fdm.server" -import { compareFields, processRvoImport } from "~/lib/rvo.server" +import { + compareFields, + getRvoFieldsFromShapefile, + processRvoImport, + RvoImportReviewStatus, + UserRvoImportReviewDecision, +} from "~/lib/rvo.server" export const handle = { hideNavigationProgress: true } @@ -104,6 +109,26 @@ export default function UploadMijnPercelenPage() { const actionData = useActionData() + const handleItemChange = (id: string, item: RvoImportReviewItem) => { + // Note: there is the assumption that getItemId will keep returning the same id. + // Therefore, make sure that `item` doesn't have a different rvoField id and localField id. + setRvoImportReviewData((data) => { + if (!data) return + const index = data.findIndex( + (originalItem) => getItemId(originalItem) === id, + ) + if (index === -1) { + console.warn( + `Item with id ${id} not found so nothing is modified.`, + ) + return + } + const newData = [...data] + newData[index] = item + return newData + }) + } + const handleChoiceChange = (id: string, action: ImportReviewAction) => { setUserChoices((prev: UserChoiceMap) => ({ ...prev, [id]: action })) } @@ -117,15 +142,10 @@ export default function UploadMijnPercelenPage() { useEffect(() => { if (actionRvoImportReviewData) { setRvoImportReviewData(actionRvoImportReviewData) - } - }, [actionRvoImportReviewData]) - - useEffect(() => { - // Initialize user choices with defaults - const initialChoices: UserChoiceMap = {} - if (rvoImportReviewData) { - rvoImportReviewData.forEach((item) => { + // Initialize user choices with defaults + const initialChoices: UserChoiceMap = {} + actionRvoImportReviewData.forEach((item) => { const id = getItemId(item) let defaultAction: ImportReviewAction @@ -140,9 +160,9 @@ export default function UploadMijnPercelenPage() { } initialChoices[id] = defaultAction }) + setUserChoices(initialChoices) } - setUserChoices(initialChoices) - }, [rvoImportReviewData]) + }, [actionRvoImportReviewData]) // Warn the user before refreshing or leaving when data is present const expectedRedirectPath = `/farm/create/${b_id_farm}/${calendar}/fields` @@ -196,7 +216,7 @@ export default function UploadMijnPercelenPage() { )} @@ -239,6 +259,10 @@ export default function UploadMijnPercelenPage() {
@@ -352,6 +376,22 @@ export async function action({ request, params }: ActionFunctionArgs): Promise< cultivationsCatalogue, ) + // Override buffer strip status from existing fields + for (const item of RvoImportReviewData) { + if (item.localField && item.rvoField?.properties.mestData) { + item.rvoField.properties.mestData.IndBufferstrook = item + .localField.b_bufferstrip + ? "J" + : "N" + item.diffs = item.diffs.filter( + (diff) => diff !== "b_bufferstrip", + ) + if (item.diffs.length === 0 && item.status === "CONFLICT") { + item.status = RvoImportReviewStatus.MATCH + } + } + } + return { RvoImportReviewData: RvoImportReviewData, message: "Percelen zijn klaar voor beeordeling! 🎉", @@ -384,6 +424,34 @@ export async function action({ request, params }: ActionFunctionArgs): Promise< throw new Error("Invalid review data format") } + // If the user wants to change buffer strip status but the row is marked as MATCH, change it to update with RVO + for (const item of rvoImportReviewData) { + const id = getItemId(item) + const originalRvoBufferstrip = + item.rvoField?.properties.mestData?.IndBufferstrook + const userChoice = userChoices[id] + + if ( + originalRvoBufferstrip && + item.localField && + (originalRvoBufferstrip === "J") !== + item.localField.b_bufferstrip + ) { + if ( + item.status === "MATCH" || + (item.status === "CONFLICT" && + userChoice === "NO_ACTION") + ) { + // User was not asked at all + item.status = RvoImportReviewStatus.CONFLICT + item.diffs = ["b_bufferstrip"] + userChoices[id] = "UPDATE_FROM_REMOTE" + } else if (item.status === "CONFLICT") { + item.diffs.push("b_bufferstrip") + } + } + } + const onFieldAdded = async ( tx: FdmType, b_id: string, diff --git a/fdm-core/src/index.ts b/fdm-core/src/index.ts index a0c4650c4..8869cd5bf 100644 --- a/fdm-core/src/index.ts +++ b/fdm-core/src/index.ts @@ -123,6 +123,7 @@ export type { } from "./fertilizer.d" export { addField, + determineIfFieldIsBuffer, getField, getFields, listAvailableAcquiringMethods, diff --git a/fdm-rvo/package.json b/fdm-rvo/package.json index 08a851a70..24e1e6ed2 100644 --- a/fdm-rvo/package.json +++ b/fdm-rvo/package.json @@ -63,6 +63,7 @@ "@turf/bbox": "^7.3.4", "@turf/helpers": "^7.3.4", "@turf/intersect": "^7.3.4", + "@turf/length": "^7.3.4", "@turf/turf": "^7.3.4", "@turf/union": "^7.3.4", "geojson": "^0.5.0", diff --git a/fdm-rvo/src/shapefile.test.ts b/fdm-rvo/src/shapefile.test.ts index 75996de64..aab85f2bb 100644 --- a/fdm-rvo/src/shapefile.test.ts +++ b/fdm-rvo/src/shapefile.test.ts @@ -141,38 +141,46 @@ describe("getRvoFieldsFromShapefile", () => { }) it("should handle multi-polygon geometry", async () => { - const MOCK_MULTIPOLYGON = geometry("MultiPolygon", [ - [ - [0, 0], - [1, 0], - [1, 1], - [0, 1], - [0, 0], - ], - [ - [1, 0], - [2, 0], - [2, 1], - [1, 1], - [1, 0], - ], - ]) - vi.mocked(shpjs.parseShp).mockResolvedValueOnce([MOCK_MULTIPOLYGON]) - vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) - - const parsed = await getRvoFieldsFromShapefile( - new File([], "shapefile.shp"), - undefined, - new File([], "shapefile.dbf"), - undefined, - ) - - expect(parsed).toHaveLength(1) - - expect((parsed[0].geometry as Geometry).type).toBe("MultiPolygon") - expect((parsed[0].geometry as Polygon).coordinates).toStrictEqual( - MOCK_MULTIPOLYGON.coordinates, - ) + try { + const MOCK_MULTIPOLYGON = geometry("MultiPolygon", [ + [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 1], + [0, 0], + ], + [ + [1, 0], + [2, 0], + [2, 1], + [1, 1], + [1, 0], + ], + ], + ]) + + vi.mocked(shpjs.parseShp).mockResolvedValueOnce([MOCK_MULTIPOLYGON]) + vi.mocked(shpjs.parseDbf).mockResolvedValueOnce([MOCK_PROPERTIES]) + + const parsed = await getRvoFieldsFromShapefile( + new File([], "shapefile.shp"), + undefined, + new File([], "shapefile.dbf"), + undefined, + ) + + expect(parsed).toHaveLength(1) + + expect((parsed[0].geometry as Geometry).type).toBe("MultiPolygon") + expect((parsed[0].geometry as Polygon).coordinates).toStrictEqual( + MOCK_MULTIPOLYGON.coordinates, + ) + } catch (e) { + console.error(e) + throw e + } }) it("should handle not-supported geometry", async () => { diff --git a/fdm-rvo/src/shapefile.ts b/fdm-rvo/src/shapefile.ts index 81fdc9981..32a4fb618 100644 --- a/fdm-rvo/src/shapefile.ts +++ b/fdm-rvo/src/shapefile.ts @@ -1,8 +1,12 @@ -import { multiPolygon, polygon } from "@turf/helpers" +import { determineIfFieldIsBuffer } from "@nmi-agro/fdm-core" +import area from "@turf/area" +import { lineString, multiPolygon, polygon } from "@turf/helpers" +import length from "@turf/length" import type { Feature, FeatureCollection, Geometry, + MultiPolygon, Polygon, Position, } from "geojson" @@ -26,6 +30,22 @@ interface RvoProperties { type FileInterface = Blob | ArrayBuffer +function perimeter(geometry: Polygon | MultiPolygon) { + if (geometry.type === "Polygon") { + return geometry.coordinates + .map((ring) => length(lineString(ring))) + .reduce((a, b) => a + b) + } + if (geometry.type === "MultiPolygon") { + return geometry.coordinates + .flatMap((polygon) => + polygon.flatMap((ring) => length(lineString(ring))), + ) + .reduce((a, b) => a + b) + } + return 0 +} + /** * Parses the files found in a MijnPercelen Shapefile export and compiles a GeoJSON feature collection where each feature's properties represent the field properties registered by RVO. * @param shp_file Shapefile or ArrayBuffer to parse @@ -37,7 +57,7 @@ export async function parseShapefileGeometry( shp_file: FileInterface, shx_file: FileInterface | undefined, prj_file: Blob | string | undefined, -): Promise { +): Promise<(Polygon | MultiPolygon)[]> { try { const [shpData, shxData, projection] = await Promise.all([ shp_file instanceof Blob ? await shp_file.arrayBuffer() : shp_file, @@ -95,6 +115,10 @@ export async function parseShapefileAttributes( /** * Converts a feature found in a Shapefile to a RvoField object to be used with the RVO import system + * + * `properties.mestData.IndBufferstrook` will be estimated using fdm-core's buffer strip estimation. + * Callers must be aware that this is just an estimate. + * * @param shapefileFeatures * @returns a RvoField object * @throws if any of the required properties are missing @@ -146,6 +170,15 @@ export function convertShapefileFeatureIntoRvoField( Country: "nl", // b_lu_catalogue[0] CropTypeCode: GEWASCODE, // b_lu_catalogue[1] UseTitleCode: TITEL, // b_acquiring_method + mestData: { + IndBufferstrook: determineIfFieldIsBuffer( + area(feature), + perimeter(feature.geometry), + trimmedNaam, + ) + ? "J" + : "N", + }, ThirdPartyCropFieldID: undefined, // not needed EndDate: EINDDAT !== 253402297199 @@ -162,6 +195,10 @@ export function convertShapefileFeatureIntoRvoField( /** * Parses the files found in a MijnPercelen Shapefile export and compiles a GeoJSON feature collection where each feature's properties represent the field properties registered by RVO. + * + * For each field `properties.mestData.IndBufferstrook` will be estimated using fdm-core's buffer strip estimation. + * Callers must be aware that this is just an estimate. + * * @param shp_file Shapefile or ArrayBuffer to parse * @param shx_file Shapefile index or ArrayBuffer file to parse, the library might be able to optimize lookups in the shp file using this * @param dbf_file DBase file or ArrayBuffer to parse containing field properties registered by RVO diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fbbd92a7f..eafb42073 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,7 +20,7 @@ catalogs: version: 16.0.3 '@types/node': specifier: ^25.5.2 - version: 25.5.0 + version: 25.5.2 '@vitest/coverage-v8': specifier: 4.1.2 version: 4.1.2 @@ -658,6 +658,9 @@ importers: '@turf/intersect': specifier: ^7.3.4 version: 7.3.4 + '@turf/length': + specifier: ^7.3.4 + version: 7.3.4 '@turf/turf': specifier: ^7.3.4 version: 7.3.4 From c07e1c6fd7148f5284fde6c38047466ddecef0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Wed, 15 Apr 2026 12:28:37 +0200 Subject: [PATCH 12/44] Simplify buffer strip checkbox implementation --- .../blocks/rvo/import-review-table.tsx | 148 ++++++++---------- ...arm.create.$b_id_farm.$calendar.upload.tsx | 68 +++----- pnpm-lock.yaml | 2 +- 3 files changed, 87 insertions(+), 131 deletions(-) diff --git a/fdm-app/app/components/blocks/rvo/import-review-table.tsx b/fdm-app/app/components/blocks/rvo/import-review-table.tsx index 02d55cf05..83eb85120 100644 --- a/fdm-app/app/components/blocks/rvo/import-review-table.tsx +++ b/fdm-app/app/components/blocks/rvo/import-review-table.tsx @@ -13,16 +13,8 @@ import { } from "@tanstack/react-table" import { area } from "@turf/turf" import { format, parseISO } from "date-fns" -import { - Archive, - ArrowLeftRight, - Check, - Info, - Plus, - Trash2, - X, -} from "lucide-react" -import { type ReactNode, useMemo } from "react" +import { Archive, ArrowLeftRight, Check, Plus, Trash2, X } from "lucide-react" +import { useMemo } from "react" import { clientConfig } from "@/app/lib/config" import { Badge } from "~/components/ui/badge" import { Checkbox } from "~/components/ui/checkbox" @@ -97,39 +89,31 @@ const DiffCell = ({ status, action, formatter = (v: any) => v, - rvoFormatter, - debug, }: { local?: any remote?: any status: string action: ImportReviewAction - formatter?: (v: any) => ReactNode - rvoFormatter?: (v: any) => ReactNode - debug?: boolean + formatter?: (v: any) => React.ReactNode }) => { // If MATCH, just show one value - if (!rvoFormatter && status === "MATCH") { - if (debug) console.debug("MATCH") + if (status === "MATCH") { return ( {formatter(local)} ) } // NEW REMOTE -> Show remote without badge - if (!rvoFormatter && status === "NEW_REMOTE") { + if (status === "NEW_REMOTE") { return ( - {(rvoFormatter ?? formatter)(remote)} + {formatter(remote)} ) } // NEW LOCAL -> Show local without badge - if ( - !rvoFormatter && - (status === "NEW_LOCAL" || status === "EXPIRED_LOCAL") - ) { + if (status === "NEW_LOCAL" || status === "EXPIRED_LOCAL") { return ( {formatter(local)} @@ -138,11 +122,9 @@ const DiffCell = ({ } // CONFLICT - if (rvoFormatter || status === "CONFLICT") { - if (debug) console.debug("CONFLICT") + if (status === "CONFLICT") { // If values are effectively equal (deep check), show one - if (!rvoFormatter && JSON.stringify(local) === JSON.stringify(remote)) { - if (debug) console.debug("CONFLICT SAME") + if (JSON.stringify(local) === JSON.stringify(remote)) { return ( {formatter(local)} @@ -211,7 +193,7 @@ const DiffCell = ({ useRemote && "font-bold", )} > - {(rvoFormatter ?? formatter)(remote)} + {formatter(remote)} )} @@ -515,36 +497,18 @@ export const columns: ColumnDef>[] = [ }, { id: "bufferstrook", - header: ({ table }) => ( + header: () => ( - - Bufferstrook - {table.options.meta?.flags?.b_bufferstrip_info_available === - false && } - + Bufferstrook -
-

- Geeft aan of het perceel bij RVO geregistreerd staat - als bufferstrook. -

- {table.options.meta?.flags - ?.b_bufferstrip_info_available === false && ( -

- Geen info uit het Shapefile is beschikbaar, dus - deze waarden zijn de oude waarden of - schattingen. -

- )} -
+ Geeft aan of het perceel bij RVO geregistreerd staat als + bufferstrook. ), cell: ({ row, table }) => { const item = row.original - const { userChoices, flags } = table.options.meta! - const bufferstripInfoAvailable = - flags?.b_bufferstrip_info_available !== false + const { userChoices } = table.options.meta! const action = userChoices[getItemId(item)] as ImportReviewAction const rvoBufferstrip = @@ -563,6 +527,40 @@ export const columns: ColumnDef>[] = [ ? "Ja" : "Nee" + return ( + val ?? "-"} + /> + ) + }, + }, + { + id: "bufferstrook_editable", + header: () => ( + + Bufferstrook + +
+

+ Geeft aan of het perceel bij RVO geregistreerd staat + als bufferstrook. +

+

+ Geen info uit het Shapefile is beschikbaar, dus deze + waarden zijn de oude waarden of schattingen. +

+
+ + + ), + cell: ({ row, table }) => { + const value = + row.original.rvoField?.properties.mestData?.IndBufferstrook === + "J" function handleUpdateValue(newValue: boolean) { if (table.options.meta?.onItemChange) { table.options.meta.onItemChange(getItemId(row.original), { @@ -583,38 +581,14 @@ export const columns: ColumnDef>[] = [ } return ( - ( -
- {val ?? "-"} -
- )} - rvoFormatter={ - bufferstripInfoAvailable - ? undefined - : (originalValue: "Ja" | "Nee" | undefined) => { - const value = originalValue === "Ja" - return ( - // biome-ignore lint/a11y/noLabelWithoutControl: the input is nested inside the label - - ) - } - } - /> + // biome-ignore lint/a11y/noLabelWithoutControl: input is nested inside the label + ) }, }, @@ -748,6 +722,11 @@ export function RvoImportReviewTable({ }) }, [data]) + const columnVisibility = { + bufferstrook: !flags?.b_bufferstrip_info_available, + bufferstrook_editable: !!flags?.b_bufferstrip_info_available, + } + const table = useReactTable({ data: sortedData, columns, @@ -758,6 +737,9 @@ export function RvoImportReviewTable({ onItemChange, onChoiceChange, }, + state: { + columnVisibility: columnVisibility, + }, }) return ( diff --git a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx index e2a5cf483..7baa5835a 100644 --- a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx +++ b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx @@ -6,6 +6,7 @@ import { getCultivationsFromCatalogue, getFarm, getFields, + updateField, } from "@nmi-agro/fdm-core" import type { ImportReviewAction, @@ -53,8 +54,6 @@ import { compareFields, getRvoFieldsFromShapefile, processRvoImport, - RvoImportReviewStatus, - UserRvoImportReviewDecision, } from "~/lib/rvo.server" export const handle = { hideNavigationProgress: true } @@ -104,7 +103,7 @@ export default function UploadMijnPercelenPage() { const location = useLocation() const [rvoImportReviewData, setRvoImportReviewData] = - useState[]>() + useState[]>() const [userChoices, setUserChoices] = useState({}) const actionData = useActionData() @@ -376,22 +375,6 @@ export async function action({ request, params }: ActionFunctionArgs): Promise< cultivationsCatalogue, ) - // Override buffer strip status from existing fields - for (const item of RvoImportReviewData) { - if (item.localField && item.rvoField?.properties.mestData) { - item.rvoField.properties.mestData.IndBufferstrook = item - .localField.b_bufferstrip - ? "J" - : "N" - item.diffs = item.diffs.filter( - (diff) => diff !== "b_bufferstrip", - ) - if (item.diffs.length === 0 && item.status === "CONFLICT") { - item.status = RvoImportReviewStatus.MATCH - } - } - } - return { RvoImportReviewData: RvoImportReviewData, message: "Percelen zijn klaar voor beeordeling! 🎉", @@ -424,34 +407,6 @@ export async function action({ request, params }: ActionFunctionArgs): Promise< throw new Error("Invalid review data format") } - // If the user wants to change buffer strip status but the row is marked as MATCH, change it to update with RVO - for (const item of rvoImportReviewData) { - const id = getItemId(item) - const originalRvoBufferstrip = - item.rvoField?.properties.mestData?.IndBufferstrook - const userChoice = userChoices[id] - - if ( - originalRvoBufferstrip && - item.localField && - (originalRvoBufferstrip === "J") !== - item.localField.b_bufferstrip - ) { - if ( - item.status === "MATCH" || - (item.status === "CONFLICT" && - userChoice === "NO_ACTION") - ) { - // User was not asked at all - item.status = RvoImportReviewStatus.CONFLICT - item.diffs = ["b_bufferstrip"] - userChoices[id] = "UPDATE_FROM_REMOTE" - } else if (item.status === "CONFLICT") { - item.diffs.push("b_bufferstrip") - } - } - } - const onFieldAdded = async ( tx: FdmType, b_id: string, @@ -493,6 +448,25 @@ export async function action({ request, params }: ActionFunctionArgs): Promise< year, onFieldAdded, ) + + // Override field properties for columns that are always "updated from RVO" + for (const item of rvoImportReviewData) { + if (item.localField && item.rvoField?.properties.mestData) { + await updateField( + fdm, + session.principal_id, + item.localField.b_id, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + item.rvoField.properties.mestData.IndBufferstrook === + "J", // b_bufferstrip + ) + } + } return redirect(`/farm/create/${b_id_farm}/${yearString}/fields`) } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eafb42073..a9bacdebc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,7 +20,7 @@ catalogs: version: 16.0.3 '@types/node': specifier: ^25.5.2 - version: 25.5.2 + version: 25.5.0 '@vitest/coverage-v8': specifier: 4.1.2 version: 4.1.2 From 70534e1f358c9788794bfcd7055dbedf0c1d6b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Wed, 15 Apr 2026 14:20:30 +0200 Subject: [PATCH 13/44] Fix client-side bundling issue with fdm-rvo/shapefile --- ...arm.create.$b_id_farm.$calendar.upload.tsx | 55 +++++++++++++++++++ fdm-app/package.json | 1 + fdm-rvo/package.json | 2 +- fdm-rvo/src/shapefile.ts | 35 +----------- fdm-rvo/tsdown.config.ts | 2 +- pnpm-lock.yaml | 9 ++- 6 files changed, 67 insertions(+), 37 deletions(-) diff --git a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx index 7baa5835a..b01899534 100644 --- a/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx +++ b/fdm-app/app/routes/farm.create.$b_id_farm.$calendar.upload.tsx @@ -1,5 +1,6 @@ import { addSoilAnalysis, + determineIfFieldIsBuffer, type FdmType, type Field, getCultivations, @@ -16,6 +17,10 @@ import type { import { getItemId } from "@nmi-agro/fdm-rvo/utils" import { createFsFileStorage } from "@remix-run/file-storage/fs" import { type FileUpload, parseFormData } from "@remix-run/form-data-parser" +import area from "@turf/area" +import { lineString } from "@turf/helpers" +import { length } from "@turf/length" +import type { MultiPolygon, Polygon } from "geojson" import { Loader2 } from "lucide-react" import { useEffect, useState } from "react" import type { @@ -54,6 +59,7 @@ import { compareFields, getRvoFieldsFromShapefile, processRvoImport, + RvoImportReviewStatus, } from "~/lib/rvo.server" export const handle = { hideNavigationProgress: true } @@ -368,6 +374,22 @@ export async function action({ request, params }: ActionFunctionArgs): Promise< prj_file, ) + function perimeter(geometry: Polygon | MultiPolygon) { + if (geometry.type === "Polygon") { + return geometry.coordinates + .map((ring) => length(lineString(ring))) + .reduce((a, b) => a + b) + } + if (geometry.type === "MultiPolygon") { + return geometry.coordinates + .flatMap((polygon) => + polygon.flatMap((ring) => length(lineString(ring))), + ) + .reduce((a, b) => a + b) + } + return 0 + } + const RvoImportReviewData = compareFields( fieldsExtended, rvoFields, @@ -375,6 +397,39 @@ export async function action({ request, params }: ActionFunctionArgs): Promise< cultivationsCatalogue, ) + // Determine if any imported field might be a buffer strip and add it since Shapefiles don't have this information + for (const item of rvoFields) { + item.properties.mestData = { + IndBufferstrook: determineIfFieldIsBuffer( + area(item.geometry), + perimeter(item.geometry), + item.properties.CropFieldDesignator, + ) + ? "J" + : "N", + } + } + + // Override each local field's corresponding RVO field buffer strip status + for (const item of RvoImportReviewData) { + item.diffs = item.diffs.filter( + (diff) => diff !== "b_bufferstrip", + ) + // b_bufferstrip should no longer be considered a difference + if ( + item.diffs.length === 0 && + item.status === RvoImportReviewStatus.CONFLICT + ) { + item.status = RvoImportReviewStatus.MATCH + } + if (item.localField && item.rvoField?.properties.mestData) { + item.rvoField.properties.mestData.IndBufferstrook = item + .localField.b_bufferstrip + ? "J" + : "N" + } + } + return { RvoImportReviewData: RvoImportReviewData, message: "Percelen zijn klaar voor beeordeling! 🎉", diff --git a/fdm-app/package.json b/fdm-app/package.json index cda0eb013..c5f25c18a 100644 --- a/fdm-app/package.json +++ b/fdm-app/package.json @@ -41,6 +41,7 @@ "@turf/boolean-point-in-polygon": "^7.3.4", "@turf/centroid": "^7.3.4", "@turf/helpers": "^7.3.4", + "@turf/length": "^7.3.4", "@turf/simplify": "^7.3.4", "better-auth": "catalog:", "chrono-node": "^2.9.0", diff --git a/fdm-rvo/package.json b/fdm-rvo/package.json index 9b7e3d25b..4e6f31f3c 100644 --- a/fdm-rvo/package.json +++ b/fdm-rvo/package.json @@ -63,7 +63,6 @@ "@turf/bbox": "^7.3.4", "@turf/helpers": "^7.3.4", "@turf/intersect": "^7.3.4", - "@turf/length": "^7.3.4", "@turf/union": "^7.3.4", "geojson": "^0.5.0", "proj4": "^2.20.4", @@ -72,6 +71,7 @@ }, "devDependencies": { "@nmi-agro/fdm-core": "workspace:*", + "@types/geojson": "^7946.0.16", "@types/node": "catalog:", "@vitest/coverage-v8": "catalog:", "tsdown": "catalog:", diff --git a/fdm-rvo/src/shapefile.ts b/fdm-rvo/src/shapefile.ts index 32a4fb618..82b62436c 100644 --- a/fdm-rvo/src/shapefile.ts +++ b/fdm-rvo/src/shapefile.ts @@ -1,7 +1,4 @@ -import { determineIfFieldIsBuffer } from "@nmi-agro/fdm-core" -import area from "@turf/area" -import { lineString, multiPolygon, polygon } from "@turf/helpers" -import length from "@turf/length" +import { multiPolygon, polygon } from "@turf/helpers" import type { Feature, FeatureCollection, @@ -30,22 +27,6 @@ interface RvoProperties { type FileInterface = Blob | ArrayBuffer -function perimeter(geometry: Polygon | MultiPolygon) { - if (geometry.type === "Polygon") { - return geometry.coordinates - .map((ring) => length(lineString(ring))) - .reduce((a, b) => a + b) - } - if (geometry.type === "MultiPolygon") { - return geometry.coordinates - .flatMap((polygon) => - polygon.flatMap((ring) => length(lineString(ring))), - ) - .reduce((a, b) => a + b) - } - return 0 -} - /** * Parses the files found in a MijnPercelen Shapefile export and compiles a GeoJSON feature collection where each feature's properties represent the field properties registered by RVO. * @param shp_file Shapefile or ArrayBuffer to parse @@ -170,15 +151,6 @@ export function convertShapefileFeatureIntoRvoField( Country: "nl", // b_lu_catalogue[0] CropTypeCode: GEWASCODE, // b_lu_catalogue[1] UseTitleCode: TITEL, // b_acquiring_method - mestData: { - IndBufferstrook: determineIfFieldIsBuffer( - area(feature), - perimeter(feature.geometry), - trimmedNaam, - ) - ? "J" - : "N", - }, ThirdPartyCropFieldID: undefined, // not needed EndDate: EINDDAT !== 253402297199 @@ -196,14 +168,13 @@ export function convertShapefileFeatureIntoRvoField( /** * Parses the files found in a MijnPercelen Shapefile export and compiles a GeoJSON feature collection where each feature's properties represent the field properties registered by RVO. * - * For each field `properties.mestData.IndBufferstrook` will be estimated using fdm-core's buffer strip estimation. - * Callers must be aware that this is just an estimate. + * `mestData` is not available in Shapefiles and will not be available in the result, thus no buffer strip information * * @param shp_file Shapefile or ArrayBuffer to parse * @param shx_file Shapefile index or ArrayBuffer file to parse, the library might be able to optimize lookups in the shp file using this * @param dbf_file DBase file or ArrayBuffer to parse containing field properties registered by RVO * @param prj_file Projection definition file or ArrayBuffer for coordinates found in the shp file, if not provided EPSG:4326 is assumed - * @returns + * @returns List of RvoField objects */ export async function getRvoFieldsFromShapefile( shp_file: FileInterface, diff --git a/fdm-rvo/tsdown.config.ts b/fdm-rvo/tsdown.config.ts index 83dcf6a12..2fcdc196a 100644 --- a/fdm-rvo/tsdown.config.ts +++ b/fdm-rvo/tsdown.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsdown" export default defineConfig({ - entry: ["src/index.ts", "src/types.ts", "src/utils.ts"], + entry: ["src/index.ts", "src/types.ts", "src/utils.ts", "src/shapefile.ts"], format: "esm", outDir: "dist", dts: true, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cae72d541..f6e683927 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -182,6 +182,9 @@ importers: '@turf/helpers': specifier: ^7.3.4 version: 7.3.4 + '@turf/length': + specifier: ^7.3.4 + version: 7.3.4 '@turf/simplify': specifier: ^7.3.4 version: 7.3.4 @@ -592,9 +595,6 @@ importers: '@turf/intersect': specifier: ^7.3.4 version: 7.3.4 - '@turf/length': - specifier: ^7.3.4 - version: 7.3.4 '@turf/union': specifier: ^7.3.4 version: 7.3.4 @@ -611,6 +611,9 @@ importers: specifier: ^4.3.6 version: 4.3.6 devDependencies: + '@types/geojson': + specifier: ^7946.0.16 + version: 7946.0.16 '@types/node': specifier: 'catalog:' version: 25.5.2 From ec1a2c89217816a810e22dfbd861cfcc72c2c11e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mert=20Bora=20=C4=B0nevi?= Date: Wed, 15 Apr 2026 14:40:06 +0200 Subject: [PATCH 14/44] Fix hidden column logic --- fdm-app/app/components/blocks/rvo/import-review-table.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fdm-app/app/components/blocks/rvo/import-review-table.tsx b/fdm-app/app/components/blocks/rvo/import-review-table.tsx index f6facb4e0..a5071f40a 100644 --- a/fdm-app/app/components/blocks/rvo/import-review-table.tsx +++ b/fdm-app/app/components/blocks/rvo/import-review-table.tsx @@ -582,12 +582,12 @@ export const columns: ColumnDef>[] = [ return ( // biome-ignore lint/a11y/noLabelWithoutControl: input is nested inside the label -