diff --git a/.changeset/frank-boxes-join.md b/.changeset/frank-boxes-join.md new file mode 100644 index 000000000..a85f7fff1 --- /dev/null +++ b/.changeset/frank-boxes-join.md @@ -0,0 +1,5 @@ +--- +"@svenvw/fdm-core": minor +--- + +For withCalculationCache add the option to provide which sensitive keys should be redacted in the cache diff --git a/.changeset/old-papayas-retire.md b/.changeset/old-papayas-retire.md new file mode 100644 index 000000000..78b522580 --- /dev/null +++ b/.changeset/old-papayas-retire.md @@ -0,0 +1,5 @@ +--- +"@svenvw/fdm-calculator": minor +--- + +For balance calculation cache per field instead of per farm and thus replace getNitrogenBalance with getNitrogenBalanceField and getOrganicMatterBalance with getOrganicMatterBalanceField diff --git a/.changeset/young-dolls-cheat.md b/.changeset/young-dolls-cheat.md new file mode 100644 index 000000000..9515dedd6 --- /dev/null +++ b/.changeset/young-dolls-cheat.md @@ -0,0 +1,5 @@ +--- +"@svenvw/fdm-calculator": patch +--- + +Refactor Nitrogen and Organic Matter balance calculations to use a bottom-up (Field -> Farm) approach diff --git a/fdm-app/app/components/blocks/balance/nitrogen-chart.tsx b/fdm-app/app/components/blocks/balance/nitrogen-chart.tsx index 1377c0b86..6d8b6eb6f 100644 --- a/fdm-app/app/components/blocks/balance/nitrogen-chart.tsx +++ b/fdm-app/app/components/blocks/balance/nitrogen-chart.tsx @@ -1,7 +1,7 @@ import type { - collectInputForNitrogenBalance, - getNitrogenBalance, - NitrogenBalanceFieldResultNumeric, + FieldInput as FdmFieldInput, + NitrogenBalanceFieldNumeric, + NitrogenBalanceNumeric, } from "@svenvw/fdm-calculator" import { format } from "date-fns/format" import { useId, useMemo, useState } from "react" @@ -16,9 +16,9 @@ import { ChartTooltip, } from "~/components/ui/chart" -type FieldInput = Awaited> -type FarmBalanceData = Awaited> -type FieldBalanceData = NitrogenBalanceFieldResultNumeric +type FieldInput = FdmFieldInput +type FarmBalanceData = NitrogenBalanceNumeric +type FieldBalanceData = NitrogenBalanceFieldNumeric type ApplicationChartConfigItem = { label: string @@ -206,7 +206,7 @@ function buildChartDataAndLegend({ label: label, unit: unit, detail: [ - application.p_name_nl, + application.p_name_nl ?? "Naam onbekend", format(application.p_app_date, "PP", { locale: nl, }), @@ -243,14 +243,14 @@ function buildChartDataAndLegend({ if (!styles[styleId]) { styles[styleId] = { ...styles[""], - color: getCultivationColor(cultivation.b_lu_croprotation), + color: getCultivationColor(cultivation.b_lu_croprotation ?? undefined), } } chartData[dataKey] = Math.abs(cultivationResult.value) ;(chartConfig as ExtendedChartConfig)[dataKey] = { label: label, unit: unit, - detail: [cultivation.b_lu_name], + detail: [cultivation.b_lu_name ?? "Naam onbekend"], styleId: styleId, } bar.push(dataKey) @@ -330,10 +330,16 @@ function buildChartDataAndLegend({ styleId: "removalHarvest", detail: cultivationDetails ? [ - cultivationDetails.b_lu_name, - format(harvestDetails.b_lu_harvest_date, "PP", { - locale: nl, - }), + cultivationDetails.b_lu_name ?? "Naam onbekend", + harvestDetails.b_lu_harvest_date + ? format( + harvestDetails.b_lu_harvest_date, + "PP", + { + locale: nl, + }, + ) + : "Datum onbekend", ] : [], } diff --git a/fdm-app/app/components/blocks/balance/nitrogen-details.tsx b/fdm-app/app/components/blocks/balance/nitrogen-details.tsx index 76d6e7569..552163e26 100644 --- a/fdm-app/app/components/blocks/balance/nitrogen-details.tsx +++ b/fdm-app/app/components/blocks/balance/nitrogen-details.tsx @@ -1,6 +1,6 @@ import type { FieldInput, - NitrogenBalanceNumeric, + NitrogenBalanceFieldNumeric, NitrogenEmissionNumeric, NitrogenRemovalHarvestsNumeric, NitrogenRemovalNumeric, @@ -23,7 +23,7 @@ import { } from "~/components/ui/accordion" interface NitrogenBalanceDetailsProps { - balanceData: NitrogenBalanceNumeric + balanceData: NitrogenBalanceFieldNumeric fieldInput: FieldInput } diff --git a/fdm-app/app/components/blocks/fertilizer-applications/metrics.tsx b/fdm-app/app/components/blocks/fertilizer-applications/metrics.tsx index 2be77fd33..539cb0839 100644 --- a/fdm-app/app/components/blocks/fertilizer-applications/metrics.tsx +++ b/fdm-app/app/components/blocks/fertilizer-applications/metrics.tsx @@ -1,7 +1,7 @@ import type { Dose, GebruiksnormResult, - NitrogenBalanceNumeric, + NitrogenBalanceFieldResultNumeric, NormFilling, NutrientAdvice, } from "@svenvw/fdm-calculator" @@ -46,7 +46,7 @@ interface FertilizerApplicationMetricsData { nitrogen: NormFilling } }> - nitrogenBalance: Promise | undefined + nitrogenBalance: Promise | undefined nutrientAdvice: NutrientAdvice dose: Dose b_id: string @@ -327,11 +327,19 @@ export function FertilizerApplicationMetricsCard({ resolve={nitrogenBalance} > {(resolvedNitrogenBalance) => { + const balance = + resolvedNitrogenBalance?.balance + if (!balance) { + return ( +
+ Geen balans + beschikbaar +
+ ) + } const task = - resolvedNitrogenBalance - .balance.target - - resolvedNitrogenBalance - .balance.balance + balance.target - + balance.balance return (
{/* Simplified Flow (Top Section) */} @@ -360,10 +368,10 @@ export function FertilizerApplicationMetricsCard({ {Math.round( - resolvedNitrogenBalance - .balance + balance .supply - .total, + ?.total ?? + 0, )}{" "} kg N @@ -389,10 +397,10 @@ export function FertilizerApplicationMetricsCard({ {Math.round( - resolvedNitrogenBalance - .balance + balance .removal - .total, + ?.total ?? + 0, )}{" "} kg N @@ -420,11 +428,11 @@ export function FertilizerApplicationMetricsCard({ {Math.round( - resolvedNitrogenBalance - .balance + balance .emission - .ammonia - .total, + ?.ammonia + ?.total ?? + 0, )}{" "} kg N @@ -438,9 +446,7 @@ export function FertilizerApplicationMetricsCard({

{Math.round( - resolvedNitrogenBalance - .balance - .balance, + balance.balance, )}{" "} kg N @@ -451,9 +457,7 @@ export function FertilizerApplicationMetricsCard({

{Math.round( - resolvedNitrogenBalance - .balance - .target, + balance.target, )}{" "} kg N diff --git a/fdm-app/app/integrations/calculator.ts b/fdm-app/app/integrations/calculator.ts index e865aa8ee..964c644bb 100644 --- a/fdm-app/app/integrations/calculator.ts +++ b/fdm-app/app/integrations/calculator.ts @@ -1,13 +1,20 @@ import { + calculateNitrogenBalance, + calculateOrganicMatterBalance, collectInputForNitrogenBalance, + collectInputForOrganicMatterBalance, createFunctionsForFertilizerApplicationFilling, createFunctionsForNorms, - getNitrogenBalance, + getNitrogenBalanceField, getNutrientAdvice, + getOrganicMatterBalanceField, + type FieldInput, + type NitrogenBalanceFieldResultNumeric, type NitrogenBalanceNumeric, + type OrganicMatterBalanceFieldResultNumeric, + type OrganicMatterBalanceNumeric, } from "@svenvw/fdm-calculator" import { - type Cultivation, type FdmType, type Field, type fdmSchema, @@ -18,8 +25,8 @@ import { } from "@svenvw/fdm-core" import { getNmiApiKey } from "./nmi" -// Get nitrogen balance for the field -export async function getNitrogenBalanceforField({ +// Get nitrogen balance for a field +export async function getNitrogenBalanceForField({ fdm, principal_id, b_id_farm, @@ -29,10 +36,10 @@ export async function getNitrogenBalanceforField({ fdm: FdmType principal_id: PrincipalId b_id_farm: fdmSchema.farmsTypeSelect["b_id_farm"] - b_id: Field.b_id + b_id: Field["b_id"] timeframe: Timeframe -}): Promise { - const nitrogenBalanceInput = await collectInputForNitrogenBalance( +}): Promise { + const { fields, ...rest } = await collectInputForNitrogenBalance( fdm, principal_id, b_id_farm, @@ -40,18 +47,104 @@ export async function getNitrogenBalanceforField({ b_id, ) - const nitrogenBalanceResult = await getNitrogenBalance( + if (fields.length === 0) { + throw new Error(`Field ${b_id} not found for farm ${b_id_farm}`) + } + + const nitrogenBalanceResult = await getNitrogenBalanceField(fdm, { + fieldInput: fields[0], + ...rest, + }) + return { + b_id: b_id, + b_area: fields[0].field.b_area ?? 0, + balance: nitrogenBalanceResult, + } +} + +export async function getNitrogenBalanceForFarm({ + fdm, + principal_id, + b_id_farm, + timeframe, +}: { + fdm: FdmType + principal_id: PrincipalId + b_id_farm: fdmSchema.farmsTypeSelect["b_id_farm"] + timeframe: Timeframe +}): Promise { + const input = await collectInputForNitrogenBalance( fdm, - nitrogenBalanceInput, + principal_id, + b_id_farm, + timeframe, ) - const nitrogenBalance = nitrogenBalanceResult.fields.find( - (field: { b_id: string }) => field.b_id === b_id, + + return calculateNitrogenBalance(fdm, input) +} + +// Get organic matter balance for a field +export async function getOrganicMatterBalanceForField({ + fdm, + principal_id, + b_id_farm, + b_id, + timeframe, +}: { + fdm: FdmType + principal_id: PrincipalId + b_id_farm: fdmSchema.farmsTypeSelect["b_id_farm"] + b_id: Field["b_id"] + timeframe: Timeframe +}): Promise<{ + fieldResult: OrganicMatterBalanceFieldResultNumeric + fieldInput: FieldInput +}> { + const { fields, ...rest } = await collectInputForOrganicMatterBalance( + fdm, + principal_id, + b_id_farm, + timeframe, + b_id, ) - if (!nitrogenBalance) { - throw new Error(`Nitrogen balance not found for field ${b_id}`) + + if (fields.length === 0) { + throw new Error(`Field ${b_id} not found for farm ${b_id_farm}`) } - return nitrogenBalance + const organicMatterBalanceResult = await getOrganicMatterBalanceField(fdm, { + fieldInput: fields[0], + ...rest, + }) + return { + fieldResult: { + b_id: b_id, + b_area: fields[0].field.b_area ?? 0, + balance: organicMatterBalanceResult, + }, + fieldInput: fields[0], + } +} + +export async function getOrganicMatterBalanceForFarm({ + fdm, + principal_id, + b_id_farm, + timeframe, +}: { + fdm: FdmType + principal_id: PrincipalId + b_id_farm: fdmSchema.farmsTypeSelect["b_id_farm"] + timeframe: Timeframe +}): Promise { + const input = await collectInputForOrganicMatterBalance( + fdm, + principal_id, + b_id_farm, + timeframe, + ) + + return calculateOrganicMatterBalance(fdm, input) } export async function getNutrientAdviceForField({ @@ -63,8 +156,8 @@ export async function getNutrientAdviceForField({ }: { fdm: FdmType principal_id: PrincipalId - b_id: Field.b_id - b_centroid: Field.b_centroid + b_id: Field["b_id"] + b_centroid: Field["b_centroid"] timeframe: Timeframe }) { const nmiApiKey = getNmiApiKey() @@ -77,7 +170,7 @@ export async function getNutrientAdviceForField({ b_id, timeframe, ) - let b_lu_catalogue: Cultivation.b_lu_catalogue + let b_lu_catalogue: string | null if (!cultivations.length) { b_lu_catalogue = null @@ -104,7 +197,7 @@ export async function getNorms({ }: { fdm: FdmType principal_id: PrincipalId - b_id: Field.b_id + b_id: Field["b_id"] }) { const functionsForNorms = createFunctionsForNorms("NL", "2025") const functionsForFilling = createFunctionsForFertilizerApplicationFilling( diff --git a/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.nitrogen.$b_id.tsx b/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.nitrogen.$b_id.tsx index 585dad9c5..5d50f9165 100644 --- a/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.nitrogen.$b_id.tsx +++ b/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.nitrogen.$b_id.tsx @@ -1,6 +1,6 @@ import { + calculateNitrogenBalance, collectInputForNitrogenBalance, - getNitrogenBalance, } from "@svenvw/fdm-calculator" import { getFarm, getField } from "@svenvw/fdm-core" import { @@ -99,7 +99,10 @@ export async function loader({ request, params }: LoaderFunctionArgs) { timeframe, b_id, ).then(async (input) => { - const nitrogenBalanceResult = await getNitrogenBalance(fdm, input) + const nitrogenBalanceResult = await calculateNitrogenBalance( + fdm, + input, + ) let fieldResult = nitrogenBalanceResult.fields.find( (field: { b_id: string }) => field.b_id === b_id, ) diff --git a/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.nitrogen._index.tsx b/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.nitrogen._index.tsx index 06aaf4f9e..4327fb680 100644 --- a/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.nitrogen._index.tsx +++ b/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.nitrogen._index.tsx @@ -1,6 +1,4 @@ import { - collectInputForNitrogenBalance, - getNitrogenBalance, type NitrogenBalanceFieldResultNumeric, } from "@svenvw/fdm-calculator" import { getFarm, getFields } from "@svenvw/fdm-core" @@ -37,6 +35,7 @@ import { TooltipContent, TooltipTrigger, } from "~/components/ui/tooltip" +import { getNitrogenBalanceForFarm } from "~/integrations/calculator" import { getSession } from "~/lib/auth.server" import { getTimeframe } from "~/lib/calendar" import { clientConfig } from "~/lib/config" @@ -87,18 +86,12 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const fields = await getFields(fdm, session.principal_id, b_id_farm) const asyncData = (async () => { - // Collect input data for nutrient balance calculation - const nitrogenBalanceInput = await collectInputForNitrogenBalance( + const nitrogenBalanceResult = await getNitrogenBalanceForFarm({ fdm, - session.principal_id, + principal_id: session.principal_id, b_id_farm, timeframe, - ) - - const nitrogenBalanceResult = await getNitrogenBalance( - fdm, - nitrogenBalanceInput, - ) + }) if (nitrogenBalanceResult.hasErrors) { reportError( diff --git a/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.organic-matter.$b_id.tsx b/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.organic-matter.$b_id.tsx index 413ab4088..291cf5417 100644 --- a/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.organic-matter.$b_id.tsx +++ b/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.organic-matter.$b_id.tsx @@ -1,7 +1,3 @@ -import { - collectInputForOrganicMatterBalance, - getOrganicMatterBalance, -} from "@svenvw/fdm-calculator" import { getFarm, getField } from "@svenvw/fdm-core" import { ArrowDownToLine, @@ -31,6 +27,7 @@ import { CardHeader, CardTitle, } from "~/components/ui/card" +import { getOrganicMatterBalanceForField } from "~/integrations/calculator" import { getSession } from "~/lib/auth.server" import { getTimeframe } from "~/lib/calendar" import { clientConfig } from "~/lib/config" @@ -69,20 +66,15 @@ export async function loader({ request, params }: LoaderFunctionArgs) { } const field = await getField(fdm, session.principal_id, b_id) - const organicMatterBalancePromise = collectInputForOrganicMatterBalance( - fdm, - session.principal_id, - b_id_farm, - timeframe, - b_id, - ).then(async (input) => { - const organicMatterBalanceResult = await getOrganicMatterBalance( + const organicMatterBalancePromise = (async () => { + const result = await getOrganicMatterBalanceForField({ fdm, - input, - ) - let fieldResult = organicMatterBalanceResult.fields.find( - (field: { b_id: string }) => field.b_id === b_id, - ) + principal_id: session.principal_id, + b_id_farm, + b_id, + timeframe, + }) + let { fieldResult, fieldInput } = result if (!fieldResult) { throw new Error( @@ -106,16 +98,12 @@ export async function loader({ request, params }: LoaderFunctionArgs) { ) fieldResult = { ...fieldResult, errorId } } - const inputForField = input.fields.find( - (field: { field: { b_id: string } }) => - field.field.b_id === b_id, - ) return { fieldResult: fieldResult, - fieldInput: inputForField, + fieldInput: fieldInput, } - }) + })() return { organicMatterBalanceResult: organicMatterBalancePromise, diff --git a/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.organic-matter._index.tsx b/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.organic-matter._index.tsx index 64c68576f..1dc1b5430 100644 --- a/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.organic-matter._index.tsx +++ b/fdm-app/app/routes/farm.$b_id_farm.$calendar.balance.organic-matter._index.tsx @@ -1,8 +1,4 @@ -import { - collectInputForOrganicMatterBalance, - getOrganicMatterBalance, - type OrganicMatterBalanceFieldResultNumeric, -} from "@svenvw/fdm-calculator" +import { type OrganicMatterBalanceFieldResultNumeric } from "@svenvw/fdm-calculator" import { getFarm, getFields } from "@svenvw/fdm-core" import { ArrowDownToLine, @@ -30,6 +26,7 @@ import { CardHeader, CardTitle, } from "~/components/ui/card" +import { getOrganicMatterBalanceForFarm } from "~/integrations/calculator" import { getSession } from "~/lib/auth.server" import { getTimeframe } from "~/lib/calendar" import { clientConfig } from "~/lib/config" @@ -73,18 +70,13 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const fields = await getFields(fdm, session.principal_id, b_id_farm) const asyncData = (async () => { - const organicMatterBalanceInput = - await collectInputForOrganicMatterBalance( + const organicMatterBalanceResult = + await getOrganicMatterBalanceForFarm({ fdm, - session.principal_id, + principal_id: session.principal_id, b_id_farm, timeframe, - ) - - const organicMatterBalanceResult = await getOrganicMatterBalance( - fdm, - organicMatterBalanceInput, - ) + }) if (organicMatterBalanceResult.hasErrors) { reportError( diff --git a/fdm-app/app/routes/farm.$b_id_farm.$calendar.field.$b_id.fertilizer._index.tsx b/fdm-app/app/routes/farm.$b_id_farm.$calendar.field.$b_id.fertilizer._index.tsx index 758698f90..5d3808184 100644 --- a/fdm-app/app/routes/farm.$b_id_farm.$calendar.field.$b_id.fertilizer._index.tsx +++ b/fdm-app/app/routes/farm.$b_id_farm.$calendar.field.$b_id.fertilizer._index.tsx @@ -35,7 +35,7 @@ import { handleActionError, handleLoaderError } from "~/lib/error" import { fdm } from "~/lib/fdm.server" import { extractFormValuesFromRequest } from "~/lib/form" import { - getNitrogenBalanceforField, + getNitrogenBalanceForField, getNorms, } from "../integrations/calculator" @@ -186,7 +186,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { principal_id, b_id, }), - nitrogenBalance: getNitrogenBalanceforField({ + nitrogenBalance: getNitrogenBalanceForField({ fdm, principal_id, b_id_farm, diff --git a/fdm-calculator/src/balance/nitrogen/emission/ammonia/fertilizers.test.ts b/fdm-calculator/src/balance/nitrogen/emission/ammonia/fertilizers.test.ts index 70802e6c4..e900e2ec9 100644 --- a/fdm-calculator/src/balance/nitrogen/emission/ammonia/fertilizers.test.ts +++ b/fdm-calculator/src/balance/nitrogen/emission/ammonia/fertilizers.test.ts @@ -86,6 +86,8 @@ describe("calculateNitrogenEmissionViaAmmoniaByFertilizers", () => { b_lu_start: new Date("2024-01-01"), b_lu_end: new Date("2024-02-29"), // Ends before cropland starts m_cropresidue: null, + b_lu_name: "Grasland", + b_lu_croprotation: "grass", }, { b_lu: "cult-2", @@ -93,6 +95,8 @@ describe("calculateNitrogenEmissionViaAmmoniaByFertilizers", () => { b_lu_start: new Date("2024-03-01"), b_lu_end: new Date("2024-10-31"), m_cropresidue: null, + b_lu_name: "Maïs", + b_lu_croprotation: "maize", }, ] diff --git a/fdm-calculator/src/balance/nitrogen/emission/ammonia/residues.test.ts b/fdm-calculator/src/balance/nitrogen/emission/ammonia/residues.test.ts index 7bc5885ac..d9b685f65 100644 --- a/fdm-calculator/src/balance/nitrogen/emission/ammonia/residues.test.ts +++ b/fdm-calculator/src/balance/nitrogen/emission/ammonia/residues.test.ts @@ -27,6 +27,8 @@ describe("calculateNitrogenEmissionViaAmmoniaByResidues", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [] @@ -66,6 +68,8 @@ describe("calculateNitrogenEmissionViaAmmoniaByResidues", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ @@ -133,6 +137,8 @@ describe("calculateNitrogenEmissionViaAmmoniaByResidues", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [] @@ -157,6 +163,8 @@ describe("calculateNitrogenEmissionViaAmmoniaByResidues", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: false, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [] @@ -194,6 +202,8 @@ describe("calculateNitrogenEmissionViaAmmoniaByResidues", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: null, // null residue handling + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [] @@ -232,6 +242,8 @@ describe("calculateNitrogenEmissionViaAmmoniaByResidues", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ @@ -280,6 +292,8 @@ describe("calculateNitrogenEmissionViaAmmoniaByResidues", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ diff --git a/fdm-calculator/src/balance/nitrogen/emission/nitrate/index.test.ts b/fdm-calculator/src/balance/nitrogen/emission/nitrate/index.test.ts index 80b3fa280..774f0ea9e 100644 --- a/fdm-calculator/src/balance/nitrogen/emission/nitrate/index.test.ts +++ b/fdm-calculator/src/balance/nitrogen/emission/nitrate/index.test.ts @@ -41,6 +41,8 @@ describe("calculateNitrogenEmissionViaNitrate", () => { b_lu_start: null, b_lu_end: null, m_cropresidue: null, + b_lu_name: "Grassland", + b_lu_croprotation: "grass", }, ] const soilAnalysis: SoilAnalysisPicked = { @@ -72,6 +74,8 @@ describe("calculateNitrogenEmissionViaNitrate", () => { b_lu_start: null, b_lu_end: null, m_cropresidue: null, + b_lu_name: "Maize", + b_lu_croprotation: "maize", }, ] const soilAnalysis: SoilAnalysisPicked = { @@ -103,6 +107,8 @@ describe("calculateNitrogenEmissionViaNitrate", () => { b_lu_start: null, b_lu_end: null, m_cropresidue: null, + b_lu_name: "Other", + b_lu_croprotation: "other", }, ] const soilAnalysis: SoilAnalysisPicked = { @@ -135,6 +141,8 @@ describe("calculateNitrogenEmissionViaNitrate", () => { b_lu_start: null, b_lu_end: null, m_cropresidue: null, + b_lu_name: "Grassland", + b_lu_croprotation: "grass", }, ] const soilAnalysis: SoilAnalysisPicked = { @@ -166,6 +174,8 @@ describe("calculateNitrogenEmissionViaNitrate", () => { b_lu_start: null, b_lu_end: null, m_cropresidue: null, + b_lu_name: "Grassland", + b_lu_croprotation: "grass", }, { b_lu_catalogue: "nl_1019", @@ -173,6 +183,8 @@ describe("calculateNitrogenEmissionViaNitrate", () => { b_lu_start: null, b_lu_end: null, m_cropresidue: null, + b_lu_name: "Maize", + b_lu_croprotation: "maize", }, ] const soilAnalysis: SoilAnalysisPicked = { diff --git a/fdm-calculator/src/balance/nitrogen/index.test.ts b/fdm-calculator/src/balance/nitrogen/index.test.ts index 06958685b..21d627a96 100644 --- a/fdm-calculator/src/balance/nitrogen/index.test.ts +++ b/fdm-calculator/src/balance/nitrogen/index.test.ts @@ -1,6 +1,19 @@ -import { describe, expect, it } from "vitest" +import { describe, expect, it, vi } from "vitest" import { calculateNitrogenBalance } from "." import type { NitrogenBalanceInput } from "./types" +import type { FdmType } from "@svenvw/fdm-core" +import Decimal from "decimal.js" + +// Mock FdmType +const mockFdm = { + select: vi.fn().mockReturnThis(), + from: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + limit: vi.fn().mockReturnThis(), + then: vi.fn((resolve) => resolve ? Promise.resolve(resolve([])) : Promise.resolve([])), // Simulate cache miss + insert: vi.fn().mockReturnThis(), + values: vi.fn().mockResolvedValue(undefined), +} as unknown as FdmType describe("calculateNitrogenBalance", () => { it("should calculate nitrogen balance correctly with mock input", async () => { @@ -21,6 +34,8 @@ describe("calculateNitrogenBalance", () => { m_cropresidue: true, b_lu_start: new Date("2023-01-01"), b_lu_end: new Date("2023-12-31"), + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ], harvests: [ @@ -76,6 +91,9 @@ describe("calculateNitrogenBalance", () => { p_app_date: new Date("2025-03-15"), }, ], + depositionSupply: { + total: new Decimal(0), + }, }, ], fertilizerDetails: [ @@ -106,7 +124,10 @@ describe("calculateNitrogenBalance", () => { }, } - const result = await calculateNitrogenBalance(mockNitrogenBalanceInput) + const result = await calculateNitrogenBalance( + mockFdm, + mockNitrogenBalanceInput, + ) function assertValidFertilizerBreakdown( obj: { total: number } & Record< @@ -156,6 +177,9 @@ describe("calculateNitrogenBalance", () => { harvests: [], soilAnalyses: [], fertilizerApplications: [], + depositionSupply: { + total: new Decimal(0), + }, }, ], fertilizerDetails: [], @@ -166,7 +190,10 @@ describe("calculateNitrogenBalance", () => { }, } - const result = await calculateNitrogenBalance(mockNitrogenBalanceInput) + const result = await calculateNitrogenBalance( + mockFdm, + mockNitrogenBalanceInput, + ) expect(result.hasErrors).toBe(true) expect(result.fieldErrorMessages.length).toBeGreaterThan(0) diff --git a/fdm-calculator/src/balance/nitrogen/index.ts b/fdm-calculator/src/balance/nitrogen/index.ts index c816a2c95..88d11acf8 100644 --- a/fdm-calculator/src/balance/nitrogen/index.ts +++ b/fdm-calculator/src/balance/nitrogen/index.ts @@ -1,137 +1,89 @@ -import { withCalculationCache } from "@svenvw/fdm-core" +import { type FdmType, withCalculationCache } from "@svenvw/fdm-core" import Decimal from "decimal.js" import pkg from "../../package" -import { getFdmPublicDataUrl } from "../../shared/public-data-url" -import { convertNitrogenBalanceToNumeric } from "../shared/conversion" import { combineSoilAnalyses } from "../shared/soil" import { calculateNitrogenEmission } from "./emission" import { calculateNitrogenEmissionViaNitrate } from "./emission/nitrate" import { calculateNitrogenRemoval } from "./removal" import { calculateNitrogenSupply } from "./supply" -import { calculateAllFieldsNitrogenSupplyByDeposition } from "./supply/deposition" import { calculateTargetForNitrogenBalance } from "./target" import type { - CultivationDetail, FertilizerDetail, - FieldInput, - NitrogenBalance, - NitrogenBalanceField, - NitrogenBalanceFieldResult, + NitrogenBalanceFieldInput, + NitrogenBalanceFieldNumeric, + NitrogenBalanceFieldResultNumeric, NitrogenBalanceInput, NitrogenBalanceNumeric, SoilAnalysisPicked, } from "./types" +import { convertDecimalToNumberRecursive } from "../shared/conversion" /** - * Calculates the nitrogen balance for a set of fields, considering nitrogen supply, removal, and emission. + * Calculates the nitrogen balance for an entire farm. * - * This function takes comprehensive input data, including field details, fertilizer information, - * and cultivation practices, to provide a detailed nitrogen balance analysis. It processes each field - * individually and then aggregates the results to provide an overall farm-level balance. + * This function orchestrates the nitrogen balance calculation for all fields on a farm. + * It calls `getNitrogenBalanceField` for each field and then aggregates the results + * using `calculateNitrogenBalancesFieldToFarm`. * - * @param nitrogenBalanceInput - The input data for the nitrogen balance calculation, including fields, fertilizer details, and cultivation details. - * @returns A promise that resolves with the calculated nitrogen balance, with numeric values as numbers. - * @throws Throws an error if any of the calculations fail. + * @param fdm - The FDM instance for database access (caching). + * @param nitrogenBalanceInput - The input data for the nitrogen balance calculation, including all fields. + * @returns A promise that resolves with the aggregated nitrogen balance for the farm. */ export async function calculateNitrogenBalance( + fdm: FdmType, nitrogenBalanceInput: NitrogenBalanceInput, ): Promise { - // Destructure input directly const { fields, fertilizerDetails, cultivationDetails, timeFrame } = nitrogenBalanceInput - // Set the link to location of FDM public data - const fdmPublicDataUrl = getFdmPublicDataUrl() - - // Pre-process details into Maps for efficient lookups - const fertilizerDetailsMap = new Map( - fertilizerDetails.map((detail) => [detail.p_id_catalogue, detail]), - ) - const cultivationDetailsMap = new Map( - cultivationDetails.map((detail) => [detail.b_lu_catalogue, detail]), - ) - - // Fetch all deposition data in a single, batched request to avoid requesting the GeoTIIF for every field - const depositionByField = - await calculateAllFieldsNitrogenSupplyByDeposition( - fields, - timeFrame, - fdmPublicDataUrl, - ) - - // Process fields in batches to control concurrency. - // Instead of running all fields in parallel with Promise.all, which can - // overwhelm the server for farms with many fields, we process them in - // smaller, manageable chunks. This provides more stable performance. - const batchSize = 50 // A sensible default, can be tuned based on profiling. - const fieldsWithBalanceResults: NitrogenBalanceFieldResult[] = [] - let hasErrors = false - const fieldErrorMessages: string[] = [] + const fieldsWithBalanceResults: NitrogenBalanceFieldResultNumeric[] = [] + const batchSize = 50 for (let i = 0; i < fields.length; i += batchSize) { const batch = fields.slice(i, i + batchSize) - const batchPromises = batch.map(async (field: FieldInput) => { - const depositionSupply = depositionByField.get(field.field.b_id) - if (!depositionSupply) { - return { - b_id: field.field.b_id, - b_area: field.field.b_area ?? 0, - errorMessage: `Deposition data not found for field ${field.field.b_id}`, + const batchResults = await Promise.all( + batch.map(async (fieldInput) => { + try { + const balance = await getNitrogenBalanceField(fdm, { + fieldInput, + fertilizerDetails, + cultivationDetails, + timeFrame, + }) + return { + b_id: fieldInput.field.b_id, + b_area: fieldInput.field.b_area ?? 0, + balance, + } + } catch (error) { + return { + b_id: fieldInput.field.b_id, + b_area: fieldInput.field.b_area ?? 0, + errorMessage: + error instanceof Error + ? error.message + : String(error), + } } - } - - return calculateNitrogenBalanceField( - field.field, - field.cultivations, - field.harvests, - field.fertilizerApplications, - field.soilAnalyses, - fertilizerDetailsMap, - cultivationDetailsMap, - timeFrame, - depositionSupply, - ) - }) - - const batchResults = await Promise.all(batchPromises) - for (const r of batchResults) { - if (r.errorMessage) { - hasErrors = true - fieldErrorMessages.push(`[${r.b_id}] ${r.errorMessage}`) - } - } + }), + ) fieldsWithBalanceResults.push(...batchResults) } - // Aggregate the field balances to farm level - const farmWithBalanceDecimal = calculateNitrogenBalancesFieldToFarm( + const hasErrors = fieldsWithBalanceResults.some( + (result) => result.errorMessage !== undefined, + ) + const fieldErrorMessages = fieldsWithBalanceResults + .filter((result) => result.errorMessage !== undefined) + .map((result) => result.errorMessage as string) + + return calculateNitrogenBalancesFieldToFarm( fieldsWithBalanceResults, - fields, hasErrors, fieldErrorMessages, ) - - // Convert the final result to use numbers instead of Decimals - return convertNitrogenBalanceToNumeric(farmWithBalanceDecimal) } -/** - * A cached version of the `calculateNitrogenBalance` function. - * - * This function provides the same functionality as `calculateNitrogenBalance` but - * includes a caching mechanism to improve performance for repeated calls with the - * same input. The cache is managed by `withCalculationCache` and uses the - * `pkg.calculatorVersion` as part of its cache key. - * - * @param nitrogenBalanceInput - The input data for the nitrogen balance calculation. - * @returns A promise that resolves with the calculated nitrogen balance, with numeric values as numbers. - */ -export const getNitrogenBalance = withCalculationCache( - calculateNitrogenBalance, - "calculateNitrogenBalance", - pkg.calculatorVersion, -) - /** * Calculates the nitrogen balance for a single field, considering nitrogen supply, removal, and emission. * @@ -146,142 +98,149 @@ export const getNitrogenBalance = withCalculationCache( * - fertilizer applications and their nitrogen contributions * - soil analysis data * - * @param field - The field to calculate the nitrogen balance for. - * @param cultivations - The cultivations on the field. - * @param harvests - The harvests from the field. - * @param fertilizerApplications - The fertilizer applications on the field. - * @param soilAnalyses - The soil analyses for the field. - * @param fertilizerDetailsMap - A map containing details for each fertilizer. - * @param cultivationDetailsMap - A map containing details for each cultivation. - * @param timeFrame - The time frame for the calculation. - * @param depositionSupply - The pre-calculated nitrogen supply from deposition. - * @returns The calculated nitrogen balance for the field, or an error message if the calculation fails. + * @param nitrogenBalanceInput - The input data for the nitrogen balance calculation for a single field. + * @returns The calculated nitrogen balance for the field. */ export function calculateNitrogenBalanceField( - field: FieldInput["field"], - cultivations: FieldInput["cultivations"], - harvests: FieldInput["harvests"], - fertilizerApplications: FieldInput["fertilizerApplications"], - soilAnalyses: FieldInput["soilAnalyses"], - fertilizerDetailsMap: Map, - cultivationDetailsMap: Map, - timeFrame: NitrogenBalanceInput["timeFrame"], - depositionSupply: NitrogenBalanceField["supply"]["deposition"], -): NitrogenBalanceFieldResult { - try { - if (!timeFrame.start || !timeFrame.end) { - throw new Error("Timeframe start and end dates must be provided.") - } - // Get the details of the field - const fieldDetails = field - - // Combine soil analyses - const soilAnalysis = combineSoilAnalyses( - soilAnalyses, - [ - "b_soiltype_agr", - "a_n_rt", - "a_c_of", - "a_cn_fr", - "a_density_sa", - "a_som_loi", - "b_gwl_class", - ], - true, - ) + nitrogenBalanceInput: NitrogenBalanceFieldInput, +): NitrogenBalanceFieldNumeric { + const { fieldInput, fertilizerDetails, cultivationDetails, timeFrame } = + nitrogenBalanceInput - // Use a field-local timeframe (intersection with input timeframe) - const timeFrameStartTime = timeFrame.start.getTime() - const timeFrameEndTime = timeFrame.end.getTime() + // Get the details of the field + const { + field, + harvests, + cultivations, + soilAnalyses, + fertilizerApplications, + depositionSupply, + } = fieldInput + + if (!timeFrame.start || !timeFrame.end) { + throw new Error("Timeframe start and end dates must be provided.") + } - const fieldStartTime = field.b_start - ? field.b_start.getTime() - : Number.NEGATIVE_INFINITY - const fieldEndTime = field.b_end - ? field.b_end.getTime() - : Number.POSITIVE_INFINITY + const fertilizerDetailsMap = new Map( + fertilizerDetails.map((detail) => [detail.p_id_catalogue, detail]), + ) + const cultivationDetailsMap = new Map( + cultivationDetails.map((detail) => [detail.b_lu_catalogue, detail]), + ) - const timeFrameField = { - start: new Date(Math.max(fieldStartTime, timeFrameStartTime)), - end: new Date(Math.min(fieldEndTime, timeFrameEndTime)), - } - // Normalize: ensure start <= end - if (timeFrameField.end.getTime() < timeFrameField.start.getTime()) { - // Clamp to an empty interval at the boundary to signal “no overlap” - timeFrameField.end = timeFrameField.start - } + // Combine soil analyses + const soilAnalysis = combineSoilAnalyses( + soilAnalyses, + [ + "b_soiltype_agr", + "a_n_rt", + "a_c_of", + "a_cn_fr", + "a_density_sa", + "a_som_loi", + "b_gwl_class", + ], + true, + ) - // Calculate the amount of Nitrogen supplied - const supply = calculateNitrogenSupply( - cultivations, - fertilizerApplications, - soilAnalysis, - cultivationDetailsMap, - fertilizerDetailsMap, - depositionSupply, - timeFrameField, - ) + // Use a field-local timeframe (intersection with input timeframe) + const timeFrameStartTime = timeFrame.start.getTime() + const timeFrameEndTime = timeFrame.end.getTime() - // Calculate the amount of Nitrogen removed - const removal = calculateNitrogenRemoval( - cultivations, - harvests, - cultivationDetailsMap, - ) + const fieldStartTime = field.b_start + ? field.b_start.getTime() + : Number.NEGATIVE_INFINITY + const fieldEndTime = field.b_end + ? field.b_end.getTime() + : Number.POSITIVE_INFINITY - // Calculate the amount of Nitrogen that is volatilized - const emission = calculateNitrogenEmission( - cultivations, - harvests, - fertilizerApplications, - cultivationDetailsMap, - fertilizerDetailsMap, - ) + const timeFrameField = { + start: new Date(Math.max(fieldStartTime, timeFrameStartTime)), + end: new Date(Math.min(fieldEndTime, timeFrameEndTime)), + } + // Normalize: ensure start <= end + if (timeFrameField.end.getTime() < timeFrameField.start.getTime()) { + // Clamp to an empty interval at the boundary to signal “no overlap” + timeFrameField.end = timeFrameField.start + } - // Calculate the balance - const balance = supply.total - .add(removal.total) - .add(emission.ammonia.total) - - // Calculate the Nitrogen Emssion via Nitrate as the surplus of nitrogen balance that is leached out - const nitrateEmission = calculateNitrogenEmissionViaNitrate( - balance, - cultivations, - soilAnalysis, - cultivationDetailsMap, - ) - emission.nitrate = nitrateEmission - emission.total = emission.total.add(nitrateEmission.total) - - // Calculate the target for the Nitrogen balance - const target = calculateTargetForNitrogenBalance( - cultivations, - soilAnalysis, - cultivationDetailsMap, - timeFrameField, - ) + // Calculate the amount of Nitrogen supplied + const supply = calculateNitrogenSupply( + cultivations, + fertilizerApplications, + soilAnalysis, + cultivationDetailsMap, + fertilizerDetailsMap, + depositionSupply, + timeFrameField, + ) - return { - b_id: fieldDetails.b_id, - b_area: fieldDetails.b_area ?? 0, - balance: { - b_id: fieldDetails.b_id, - balance: balance, - supply: supply, - removal: removal, - emission: emission, - target: target, - }, - } - } catch (error) { - return { - b_id: field.b_id, - b_area: field.b_area ?? 0, - errorMessage: String(error).replace("Error: ", ""), - } - } + // Calculate the amount of Nitrogen removed + const removal = calculateNitrogenRemoval( + cultivations, + harvests, + cultivationDetailsMap, + ) + + // Calculate the amount of Nitrogen that is volatilized + const emission = calculateNitrogenEmission( + cultivations, + harvests, + fertilizerApplications, + cultivationDetailsMap, + fertilizerDetailsMap, + ) + + // Calculate the balance + const balance = supply.total.add(removal.total).add(emission.ammonia.total) + + // Calculate the Nitrogen Emssion via Nitrate as the surplus of nitrogen balance that is leached out + const nitrateEmission = calculateNitrogenEmissionViaNitrate( + balance, + cultivations, + soilAnalysis, + cultivationDetailsMap, + ) + emission.nitrate = nitrateEmission + emission.total = emission.total.add(nitrateEmission.total) + + // Calculate the target for the Nitrogen balance + const target = calculateTargetForNitrogenBalance( + cultivations, + soilAnalysis, + cultivationDetailsMap, + timeFrameField, + ) + + const balanceNumeric = convertDecimalToNumberRecursive({ + b_id: field.b_id, + balance: balance, + supply: supply, + removal: removal, + emission: emission, + target: target, + }) as NitrogenBalanceFieldNumeric + + return balanceNumeric } +/** + * A cached version of the `calculateNitrogenBalanceField` function. + * + * This function provides the same functionality as `calculateNitrogenBalanceField` but + * includes a caching mechanism to improve performance for repeated calls with the + * same input. The cache is managed by `withCalculationCache` and uses the + * `pkg.calculatorVersion` as part of its cache key. + * + * @param nitrogenBalanceInput - The input data for the nitrogen balance calculation. + * @returns A promise that resolves with the calculated nitrogen balance, with numeric values as numbers. + */ +export const getNitrogenBalanceField = withCalculationCache( + calculateNitrogenBalanceField, + "calculateNitrogenBalanceField", + pkg.calculatorVersion, +) + /** * Aggregates nitrogen balances from individual fields to the farm level. * @@ -292,21 +251,21 @@ export function calculateNitrogenBalanceField( * The function returns a comprehensive nitrogen balance for the farm, including total supply, * removal, emission, and the overall balance. * @param fieldsWithBalanceResults - An array of nitrogen balance results for individual fields, potentially including errors. - * @param fields - All field inputs, used to get original field data like area. * @param hasErrors - Indicates if any field calculations failed. * @param fieldErrorMessages - A list of error messages for fields that failed to calculate. * @returns The aggregated nitrogen balance for the farm. */ export function calculateNitrogenBalancesFieldToFarm( - fieldsWithBalanceResults: NitrogenBalanceFieldResult[], - fields: FieldInput[], + fieldsWithBalanceResults: NitrogenBalanceFieldResultNumeric[], hasErrors: boolean, fieldErrorMessages: string[], -): NitrogenBalance { +): NitrogenBalanceNumeric { // Filter out fields that have errors for aggregation const successfulFieldBalances = fieldsWithBalanceResults.filter( (result) => result.balance !== undefined, - ) as (NitrogenBalanceFieldResult & { balance: NitrogenBalanceField })[] + ) as (NitrogenBalanceFieldResultNumeric & { + balance: NitrogenBalanceFieldNumeric + })[] // Calculate total weighted supply, removal, and emission across the farm const fertilizerTypes = ["mineral", "manure", "compost", "other"] as const @@ -322,7 +281,7 @@ export function calculateNitrogenBalancesFieldToFarm( }, [] as [(typeof fertilizerTypes)[number], Decimal][], ), - ) as Omit + ) as Record<(typeof fertilizerTypes)[number], Decimal> let totalFarmRemoval = new Decimal(0) let totalFarmRemovalHarvest = new Decimal(0) let totalFarmRemovalResidue = new Decimal(0) @@ -338,66 +297,71 @@ export function calculateNitrogenBalancesFieldToFarm( }, [] as [(typeof fertilizerTypes)[number], Decimal][], ), - ) as Omit + ) as Record<(typeof fertilizerTypes)[number], Decimal> let totalFarmEmissionAmmoniaResidue = new Decimal(0) let totalFarmEmissionNitrate = new Decimal(0) let totalFarmTarget = new Decimal(0) let totalFarmArea = new Decimal(0) for (const fieldResult of successfulFieldBalances) { - const fieldInput = fields.find((f) => f.field.b_id === fieldResult.b_id) - - if (!fieldInput) { - console.warn( - `Could not find field input for field balance ${fieldResult.b_id}`, - ) - continue - } - const fieldArea = new Decimal(fieldInput.field.b_area ?? 0) + const fieldArea = new Decimal(fieldResult.b_area ?? 0) totalFarmArea = totalFarmArea.add(fieldArea) totalFarmSupply = totalFarmSupply.add( - fieldResult.balance.supply.total.times(fieldArea), + new Decimal(fieldResult.balance.supply.total).times(fieldArea), ) totalFarmSupplyDeposition = totalFarmSupplyDeposition.add( - fieldResult.balance.supply.deposition.total.times(fieldArea), + new Decimal(fieldResult.balance.supply.deposition.total).times( + fieldArea, + ), ) totalFarmSupplyFixation = totalFarmSupplyFixation.add( - fieldResult.balance.supply.fixation.total.times(fieldArea), + new Decimal(fieldResult.balance.supply.fixation.total).times( + fieldArea, + ), ) totalFarmSupplyMineralization = totalFarmSupplyMineralization.add( - fieldResult.balance.supply.mineralisation.total.times(fieldArea), + new Decimal(fieldResult.balance.supply.mineralisation.total).times( + fieldArea, + ), ) for (const fertilizerType of fertilizerTypes) { totalFarmSupplyFertilizers[fertilizerType] = totalFarmSupplyFertilizers[fertilizerType].add( - fieldResult.balance.supply.fertilizers[ - fertilizerType - ].total.times(fieldArea), + new Decimal( + fieldResult.balance.supply.fertilizers[fertilizerType] + .total, + ).times(fieldArea), ) } totalFarmRemoval = totalFarmRemoval.add( - fieldResult.balance.removal.total.times(fieldArea), + new Decimal(fieldResult.balance.removal.total).times(fieldArea), ) totalFarmRemovalHarvest = totalFarmRemovalHarvest.add( - fieldResult.balance.removal.harvests.total.times(fieldArea), + new Decimal(fieldResult.balance.removal.harvests.total).times( + fieldArea, + ), ) totalFarmRemovalResidue = totalFarmRemovalResidue.add( - fieldResult.balance.removal.residues.total.times(fieldArea), + new Decimal(fieldResult.balance.removal.residues.total).times( + fieldArea, + ), ) totalFarmEmission = totalFarmEmission.add( - fieldResult.balance.emission.total.times(fieldArea), + new Decimal(fieldResult.balance.emission.total).times(fieldArea), ) totalFarmEmissionAmmonia = totalFarmEmissionAmmonia.add( - fieldResult.balance.emission.ammonia.total.times(fieldArea), + new Decimal(fieldResult.balance.emission.ammonia.total).times( + fieldArea, + ), ) for (const fertilizerType of fertilizerTypes) { - const fieldTotal = - fieldResult.balance.emission.ammonia.fertilizers[ - fertilizerType - ].total.mul(fieldArea) + const fieldTotal = new Decimal( + fieldResult.balance.emission.ammonia.fertilizers[fertilizerType] + .total, + ).times(fieldArea) ammoniaByFertilizerType[fertilizerType] = ammoniaByFertilizerType[fertilizerType].add(fieldTotal) totalFarmEmissionAmmoniaFertilizer = @@ -405,17 +369,19 @@ export function calculateNitrogenBalancesFieldToFarm( } totalFarmEmissionAmmoniaResidue = totalFarmEmissionAmmoniaResidue.add( - fieldResult.balance.emission.ammonia.residues.total.times( - fieldArea, - ), + new Decimal( + fieldResult.balance.emission.ammonia.residues.total, + ).times(fieldArea), ) totalFarmEmissionNitrate = totalFarmEmissionNitrate.add( - fieldResult.balance.emission.nitrate.total.times(fieldArea), + new Decimal(fieldResult.balance.emission.nitrate.total).times( + fieldArea, + ), ) totalFarmTarget = totalFarmTarget.add( - fieldResult.balance.target.times(fieldArea), + new Decimal(fieldResult.balance.target).times(fieldArea), ) } @@ -483,7 +449,7 @@ export function calculateNitrogenBalancesFieldToFarm( .add(avgFarmEmission) // Return the farm with average balances per hectare - const farmWithBalance: NitrogenBalance = { + const farmWithBalance = { balance: avgFarmBalance, supply: { total: avgFarmSupply, @@ -520,5 +486,7 @@ export function calculateNitrogenBalancesFieldToFarm( fieldErrorMessages: fieldErrorMessages, } - return farmWithBalance + return convertDecimalToNumberRecursive( + farmWithBalance, + ) as NitrogenBalanceNumeric } diff --git a/fdm-calculator/src/balance/nitrogen/input.test.ts b/fdm-calculator/src/balance/nitrogen/input.test.ts index 8f4488825..50877f65d 100644 --- a/fdm-calculator/src/balance/nitrogen/input.test.ts +++ b/fdm-calculator/src/balance/nitrogen/input.test.ts @@ -1,3 +1,12 @@ +import { + getCultivations, + getCultivationsFromCatalogue, + getFertilizerApplications, + getFertilizers, + getFields, + getHarvests, + getSoilAnalyses, +} from "@svenvw/fdm-core" import type { Cultivation, CultivationCatalogue, @@ -13,6 +22,8 @@ import type { import { beforeEach, describe, expect, it, vi } from "vitest" import { collectInputForNitrogenBalance } from "./input" import type { FieldInput, NitrogenBalanceInput } from "./types" +import { calculateAllFieldsNitrogenSupplyByDeposition } from "./supply/deposition" +import Decimal from "decimal.js" // Mock the @svenvw/fdm-core module vi.mock("@svenvw/fdm-core", async () => { @@ -29,18 +40,23 @@ vi.mock("@svenvw/fdm-core", async () => { } }) +// Mock the deposition supply calculation +vi.mock("./supply/deposition", () => ({ + calculateAllFieldsNitrogenSupplyByDeposition: vi.fn(), +})) + // Import mocks after vi.mock call -const fdmCoreMocks = await import("@svenvw/fdm-core") -const mockedGetFields = vi.mocked(fdmCoreMocks.getFields) -const mockedGetCultivations = vi.mocked(fdmCoreMocks.getCultivations) -const mockedGetHarvests = vi.mocked(fdmCoreMocks.getHarvests) -const mockedGetSoilAnalyses = vi.mocked(fdmCoreMocks.getSoilAnalyses) -const mockedGetFertilizerApplications = vi.mocked( - fdmCoreMocks.getFertilizerApplications, -) -const mockedGetFertilizers = vi.mocked(fdmCoreMocks.getFertilizers) +const mockedGetFields = vi.mocked(getFields) +const mockedGetCultivations = vi.mocked(getCultivations) +const mockedGetHarvests = vi.mocked(getHarvests) +const mockedGetSoilAnalyses = vi.mocked(getSoilAnalyses) +const mockedGetFertilizerApplications = vi.mocked(getFertilizerApplications) +const mockedGetFertilizers = vi.mocked(getFertilizers) const mockedGetCultivationsFromCatalogue = vi.mocked( - fdmCoreMocks.getCultivationsFromCatalogue, + getCultivationsFromCatalogue, +) +const mockedCalculateAllFieldsNitrogenSupplyByDeposition = vi.mocked( + calculateAllFieldsNitrogenSupplyByDeposition, ) describe("collectInputForNitrogenBalance", () => { @@ -175,6 +191,10 @@ describe("collectInputForNitrogenBalance", () => { b_n_fixation: 0, }, ] as unknown as CultivationCatalogue[] + const mockDepositionSupplyMap = new Map([ + ["field-1", { total: new Decimal(10) }], + ["field-2", { total: new Decimal(20) }], + ]) // Setup mocks mockedGetFields.mockResolvedValue(mockFieldsData) @@ -188,6 +208,9 @@ describe("collectInputForNitrogenBalance", () => { mockedGetCultivationsFromCatalogue.mockResolvedValue( mockCultivationDetailsData, ) + mockedCalculateAllFieldsNitrogenSupplyByDeposition.mockResolvedValue( + mockDepositionSupplyMap, + ) const result = await collectInputForNitrogenBalance( mockFdm, @@ -197,12 +220,13 @@ describe("collectInputForNitrogenBalance", () => { ) const expectedFieldInputs: FieldInput[] = mockFieldsData.map( - (field) => ({ - field: field, + (fieldData) => ({ + field: fieldData, cultivations: mockCultivationsData, harvests: mockHarvestsData, soilAnalyses: mockSoilAnalysesData, fertilizerApplications: mockFertilizerApplicationsData, + depositionSupply: mockDepositionSupplyMap.get(fieldData.b_id)!, }), ) @@ -261,6 +285,13 @@ describe("collectInputForNitrogenBalance", () => { principal_id, b_id_farm, ) + expect( + mockedCalculateAllFieldsNitrogenSupplyByDeposition, + ).toHaveBeenCalledWith( + expect.anything(), + timeframe, + expect.any(String), + ) }) it("should throw an error if getFields fails", async () => { diff --git a/fdm-calculator/src/balance/nitrogen/input.ts b/fdm-calculator/src/balance/nitrogen/input.ts index 977690190..4b13cfdb6 100644 --- a/fdm-calculator/src/balance/nitrogen/input.ts +++ b/fdm-calculator/src/balance/nitrogen/input.ts @@ -15,6 +15,8 @@ import { getSoilAnalyses, } from "@svenvw/fdm-core" import type { NitrogenBalanceInput } from "./types" +import { calculateAllFieldsNitrogenSupplyByDeposition } from "./supply/deposition" +import { getFdmPublicDataUrl } from "../../shared/public-data-url" /** * Collects necessary input data from a FDM instance for calculating the nitrogen balance. @@ -59,6 +61,17 @@ export async function collectInputForNitrogenBalance( ) } + // Set the link to location of FDM public data + const fdmPublicDataUrl = getFdmPublicDataUrl() + + // Fetch all deposition data in a single, batched request to avoid requesting the GeoTIIF for every field + const depositionByField = + await calculateAllFieldsNitrogenSupplyByDeposition( + farmFields, + timeframe, + fdmPublicDataUrl, + ) + // Collect the details per field const fields = await Promise.all( farmFields.map(async (field) => { @@ -113,6 +126,7 @@ export async function collectInputForNitrogenBalance( harvests: harvestsFiltered, fertilizerApplications: fertilizerApplications, soilAnalyses: soilAnalyses, + depositionSupply: depositionByField.get(field.b_id), } }), ) diff --git a/fdm-calculator/src/balance/nitrogen/performance.test.ts b/fdm-calculator/src/balance/nitrogen/performance.test.ts index 1ac9f6d8e..9dfdd00cb 100644 --- a/fdm-calculator/src/balance/nitrogen/performance.test.ts +++ b/fdm-calculator/src/balance/nitrogen/performance.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest" +import { describe, expect, it, vi } from "vitest" import { calculateNitrogenBalance } from "./index" import type { CultivationDetail, @@ -6,6 +6,20 @@ import type { FieldInput, NitrogenBalanceInput, } from "./types" +import type { FdmType } from "@svenvw/fdm-core" +import Decimal from "decimal.js" + +// Mock FdmType +const mockFdm = { + select: vi.fn().mockReturnThis(), + from: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + limit: vi.fn().mockReturnThis(), + // biome-ignore lint/suspicious/noThenProperty: Simulate cache miss + then: vi.fn((resolve) => resolve ? Promise.resolve(resolve([])) : Promise.resolve([])), + insert: vi.fn().mockReturnThis(), + values: vi.fn().mockResolvedValue(undefined), +} as unknown as FdmType /** * Utility function to generate mock data for performance testing. @@ -81,6 +95,8 @@ function generateMockData(numberOfFields: number): NitrogenBalanceInput { m_cropresidue: false, b_lu_start: new Date(2023, 3, 1), b_lu_end: new Date(2023, 8, 1), + b_lu_name: "Mock Cultivation", + b_lu_croprotation: "maize", }, ] @@ -147,6 +163,9 @@ function generateMockData(numberOfFields: number): NitrogenBalanceInput { harvests, soilAnalyses, fertilizerApplications, + depositionSupply: { + total: new Decimal(0), + }, }) } @@ -179,7 +198,7 @@ describe("Nitrogen Balance Performance", () => { // Measure execution time const startTime = process.hrtime.bigint() - const result = await calculateNitrogenBalance(mockInput) + const result = await calculateNitrogenBalance(mockFdm, mockInput) const endTime = process.hrtime.bigint() const durationMs = Number(endTime - startTime) / 1_000_000 @@ -190,8 +209,10 @@ describe("Nitrogen Balance Performance", () => { expect(result).toBeDefined() expect(result.fields.length).toBe(numberOfFields) - // Add more specific assertions if needed, e.g., checking total balance values - // expect(result.balance).toBeCloseTo(...) + expect(typeof result.balance).toBe("number") + expect(typeof result.supply.total).toBe("number") + expect(typeof result.emission.total).toBe("number") + expect(typeof result.removal.total).toBe("number") // Assert that the calculation completed within the desired timeout expect(durationMs).toBeLessThan(30000) // 30 seconds diff --git a/fdm-calculator/src/balance/nitrogen/removal/harvest.test.ts b/fdm-calculator/src/balance/nitrogen/removal/harvest.test.ts index be8bb513e..0b5f1ccb3 100644 --- a/fdm-calculator/src/balance/nitrogen/removal/harvest.test.ts +++ b/fdm-calculator/src/balance/nitrogen/removal/harvest.test.ts @@ -12,6 +12,8 @@ describe("calculateNitrogenRemovalByHarvests", () => { m_cropresidue: true, b_lu_start: null, b_lu_end: null, + b_lu_name: "test", + b_lu_croprotation: null }, ] const harvests: FieldInput["harvests"] = [ @@ -79,6 +81,8 @@ describe("calculateNitrogenRemovalByHarvests", () => { m_cropresidue: true, b_lu_start: null, b_lu_end: null, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ @@ -193,6 +197,8 @@ describe("calculateNitrogenRemovalByHarvests", () => { m_cropresidue: true, b_lu_start: null, b_lu_end: null, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ @@ -304,6 +310,8 @@ describe("calculateNitrogenRemovalByHarvests", () => { m_cropresidue: true, b_lu_start: null, b_lu_end: null, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ diff --git a/fdm-calculator/src/balance/nitrogen/removal/index.test.ts b/fdm-calculator/src/balance/nitrogen/removal/index.test.ts index 9ea7478bc..97444a5f3 100644 --- a/fdm-calculator/src/balance/nitrogen/removal/index.test.ts +++ b/fdm-calculator/src/balance/nitrogen/removal/index.test.ts @@ -26,6 +26,8 @@ describe("calculateNitrogenRemoval", () => { m_cropresidue: true, b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ @@ -77,6 +79,8 @@ describe("calculateNitrogenRemoval", () => { m_cropresidue: false, b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [] diff --git a/fdm-calculator/src/balance/nitrogen/removal/residue.test.ts b/fdm-calculator/src/balance/nitrogen/removal/residue.test.ts index e1620a635..eca76cc85 100644 --- a/fdm-calculator/src/balance/nitrogen/removal/residue.test.ts +++ b/fdm-calculator/src/balance/nitrogen/removal/residue.test.ts @@ -39,6 +39,8 @@ describe("calculateNitrogenRemovalByResidue", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: false, // No residue left + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [] @@ -63,6 +65,8 @@ describe("calculateNitrogenRemovalByResidue", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ @@ -113,6 +117,8 @@ describe("calculateNitrogenRemovalByResidue", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ @@ -180,6 +186,8 @@ describe("calculateNitrogenRemovalByResidue", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [] // No harvest data @@ -202,6 +210,8 @@ describe("calculateNitrogenRemovalByResidue", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [] @@ -226,6 +236,8 @@ describe("calculateNitrogenRemovalByResidue", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: null, // null residue handling + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [] @@ -250,6 +262,8 @@ describe("calculateNitrogenRemovalByResidue", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] const harvests: FieldInput["harvests"] = [ @@ -282,6 +296,8 @@ describe("calculateNitrogenRemovalByResidue", () => { b_lu_start: new Date("2022-01-01"), b_lu_end: new Date("2022-12-31"), m_cropresidue: true, + b_lu_name: "test", + b_lu_croprotation: null }, ] const harvests: FieldInput["harvests"] = [] diff --git a/fdm-calculator/src/balance/nitrogen/supply/deposition.test.ts b/fdm-calculator/src/balance/nitrogen/supply/deposition.test.ts index 877c4a33e..b627d8df0 100644 --- a/fdm-calculator/src/balance/nitrogen/supply/deposition.test.ts +++ b/fdm-calculator/src/balance/nitrogen/supply/deposition.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest" import { getFdmPublicDataUrl } from "../../../shared/public-data-url" import type { FieldInput, NitrogenBalanceInput } from "../types" import { calculateAllFieldsNitrogenSupplyByDeposition } from "./deposition" +import Decimal from "decimal.js" +import type { Field } from "@svenvw/fdm-core" describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { const fdmPublicDataUrl = getFdmPublicDataUrl() @@ -19,6 +21,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { harvests: [], soilAnalyses: [], fertilizerApplications: [], + depositionSupply: { total: new Decimal(0) }, } const timeFrame: NitrogenBalanceInput["timeFrame"] = { start: new Date("2025-01-01"), @@ -26,7 +29,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { } const resultMap = await calculateAllFieldsNitrogenSupplyByDeposition( - [field], + [field.field as unknown as Field], timeFrame, fdmPublicDataUrl, ) @@ -49,6 +52,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { harvests: [], soilAnalyses: [], fertilizerApplications: [], + depositionSupply: { total: new Decimal(0) }, } // Test with a full year @@ -57,7 +61,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { end: new Date("2024-01-01"), } let resultMap = await calculateAllFieldsNitrogenSupplyByDeposition( - [field], + [field.field as unknown as Field], timeFrame, fdmPublicDataUrl, ) @@ -71,7 +75,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { end: new Date("2023-07-01"), } resultMap = await calculateAllFieldsNitrogenSupplyByDeposition( - [field], + [field.field as unknown as Field], timeFrame, fdmPublicDataUrl, ) @@ -93,6 +97,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { harvests: [], soilAnalyses: [], fertilizerApplications: [], + depositionSupply: { total: new Decimal(0) }, } const timeFrame: NitrogenBalanceInput["timeFrame"] = { start: new Date("2023-01-01"), @@ -100,7 +105,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { } const resultMap = await calculateAllFieldsNitrogenSupplyByDeposition( - [field], + [field.field as unknown as Field], timeFrame, fdmPublicDataUrl, ) @@ -124,6 +129,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { harvests: [], soilAnalyses: [], fertilizerApplications: [], + depositionSupply: { total: new Decimal(0) }, }, { field: { @@ -137,6 +143,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { harvests: [], soilAnalyses: [], fertilizerApplications: [], + depositionSupply: { total: new Decimal(0) }, }, ] const timeFrame: NitrogenBalanceInput["timeFrame"] = { @@ -145,7 +152,7 @@ describe("calculateAllFieldsNitrogenSupplyByDeposition", () => { } const resultMap = await calculateAllFieldsNitrogenSupplyByDeposition( - fields, + fields.map(f => f.field as unknown as Field), timeFrame, fdmPublicDataUrl, ) diff --git a/fdm-calculator/src/balance/nitrogen/supply/deposition.ts b/fdm-calculator/src/balance/nitrogen/supply/deposition.ts index 2ca370f6e..220d3ee8c 100644 --- a/fdm-calculator/src/balance/nitrogen/supply/deposition.ts +++ b/fdm-calculator/src/balance/nitrogen/supply/deposition.ts @@ -1,7 +1,8 @@ import { differenceInCalendarDays } from "date-fns" import Decimal from "decimal.js" import { getGeoTiffValue } from "../../../shared/geotiff" -import type { FieldInput, NitrogenBalanceInput, NitrogenSupply } from "../types" +import type { NitrogenSupply } from "../types" +import type { Field, Timeframe } from "@svenvw/fdm-core" /** * Calculates the nitrogen deposition for a batch of fields from a GeoTIFF file. @@ -16,8 +17,8 @@ import type { FieldInput, NitrogenBalanceInput, NitrogenSupply } from "../types" * the calculated nitrogen deposition supply for that field. */ export async function calculateAllFieldsNitrogenSupplyByDeposition( - fields: FieldInput[], - timeFrame: NitrogenBalanceInput["timeFrame"], + fields: Field[], + timeFrame: Timeframe, fdmPublicDataUrl: string, ): Promise> { if (fields.length === 0) { @@ -34,20 +35,25 @@ export async function calculateAllFieldsNitrogenSupplyByDeposition( // Step 1: Create an array of promises to calculate deposition for each field concurrently. const depositionPromises = fields.map(async (field) => { // Compute per-field effective timeframe (intersection with field existence) - const fStart = field.field.b_start ?? timeFrame.start - const fEnd = field.field.b_end ?? timeFrame.end - const effectiveStart = new Date( - Math.max(fStart.getTime(), timeFrame.start.getTime()), - ) - const effectiveEnd = new Date( - Math.min(fEnd.getTime(), timeFrame.end.getTime()), - ) - const days = differenceInCalendarDays(effectiveEnd, effectiveStart) - const fraction = - days >= 0 ? new Decimal(days).add(1).dividedBy(365) : new Decimal(0) + let fraction = new Decimal(1) + if (timeFrame.start && timeFrame.end) { + const fStart = field.b_start ?? timeFrame.start + const fEnd = field.b_end ?? timeFrame.end + const effectiveStart = new Date( + Math.max(fStart.getTime(), timeFrame.start.getTime()), + ) + const effectiveEnd = new Date( + Math.min(fEnd.getTime(), timeFrame.end.getTime()), + ) + const days = differenceInCalendarDays(effectiveEnd, effectiveStart) + fraction = + days >= 0 + ? new Decimal(days).add(1).dividedBy(365) + : new Decimal(0) + } // Get the deposition value from the GeoTIFF using the new getTiffValue function. - const [longitude, latitude] = field.field.b_centroid + const [longitude, latitude] = field.b_centroid const value = await getGeoTiffValue(url, longitude, latitude) let depositionValue = new Decimal(0) @@ -56,7 +62,7 @@ export async function calculateAllFieldsNitrogenSupplyByDeposition( } return { - fieldId: field.field.b_id, + fieldId: field.b_id, deposition: { total: depositionValue }, } }) diff --git a/fdm-calculator/src/balance/nitrogen/supply/fixation.test.ts b/fdm-calculator/src/balance/nitrogen/supply/fixation.test.ts index 0157f9a3a..0a696d4b2 100644 --- a/fdm-calculator/src/balance/nitrogen/supply/fixation.test.ts +++ b/fdm-calculator/src/balance/nitrogen/supply/fixation.test.ts @@ -25,6 +25,8 @@ describe("calculateNitrogenFixation", () => { m_cropresidue: false, b_lu_start: null, b_lu_end: null, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] @@ -62,6 +64,8 @@ describe("calculateNitrogenFixation", () => { m_cropresidue: true, b_lu_start: null, b_lu_end: null, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, { b_lu: "cultivation2", @@ -69,6 +73,8 @@ describe("calculateNitrogenFixation", () => { m_cropresidue: false, b_lu_start: null, b_lu_end: null, + b_lu_name: "Cultivation 2", + b_lu_croprotation: "other", }, ] @@ -119,6 +125,8 @@ describe("calculateNitrogenFixation", () => { m_cropresidue: true, b_lu_start: null, b_lu_end: null, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, { b_lu: "cultivation2", @@ -126,6 +134,8 @@ describe("calculateNitrogenFixation", () => { m_cropresidue: false, b_lu_start: null, b_lu_end: null, + b_lu_name: "Cultivation 2", + b_lu_croprotation: "other", }, ] @@ -176,6 +186,8 @@ describe("calculateNitrogenFixation", () => { m_cropresidue: true, b_lu_start: null, b_lu_end: null, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "cereal", }, ] diff --git a/fdm-calculator/src/balance/nitrogen/supply/mineralization.test.ts b/fdm-calculator/src/balance/nitrogen/supply/mineralization.test.ts index c4fbc9906..59b732563 100644 --- a/fdm-calculator/src/balance/nitrogen/supply/mineralization.test.ts +++ b/fdm-calculator/src/balance/nitrogen/supply/mineralization.test.ts @@ -65,6 +65,8 @@ describe("calculateNitrogenSupplyBySoilMineralization", () => { b_lu_end: new Date("2023-12-31"), b_lu_catalogue: "3", m_cropresidue: false, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "grass", }, ] const timeFrame: NitrogenBalanceInput["timeFrame"] = { @@ -90,6 +92,8 @@ describe("calculateNitrogenSupplyBySoilMineralization", () => { b_lu_end: new Date("2023-12-31"), b_lu_catalogue: "1", m_cropresidue: false, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "grass", }, ] const timeFrame: NitrogenBalanceInput["timeFrame"] = { @@ -119,6 +123,8 @@ describe("calculateNitrogenSupplyBySoilMineralization", () => { b_lu_end: new Date("2023-12-31"), b_lu_catalogue: "1", m_cropresidue: false, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "grass", }, ] const timeFrame: NitrogenBalanceInput["timeFrame"] = { @@ -148,6 +154,8 @@ describe("calculateNitrogenSupplyBySoilMineralization", () => { b_lu_end: new Date("2023-05-14"), b_lu_catalogue: "1", m_cropresidue: false, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "grass", }, ] const timeFrame: NitrogenBalanceInput["timeFrame"] = { @@ -177,6 +185,8 @@ describe("calculateNitrogenSupplyBySoilMineralization", () => { b_lu_end: null, b_lu_catalogue: "1", m_cropresidue: false, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "grass", }, ] const timeFrame: NitrogenBalanceInput["timeFrame"] = { @@ -206,6 +216,8 @@ describe("calculateNitrogenSupplyBySoilMineralization", () => { b_lu_end: new Date("2023-12-31"), b_lu_catalogue: "1", m_cropresidue: false, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "grass", }, ] const timeFrame: NitrogenBalanceInput["timeFrame"] = { @@ -235,6 +247,8 @@ describe("calculateNitrogenSupplyBySoilMineralization", () => { b_lu_end: new Date("2024-12-31"), b_lu_catalogue: "1", m_cropresidue: false, + b_lu_name: "Cultivation 1", + b_lu_croprotation: "grass", }, ] const timeFrame: NitrogenBalanceInput["timeFrame"] = { diff --git a/fdm-calculator/src/balance/nitrogen/target.test.ts b/fdm-calculator/src/balance/nitrogen/target.test.ts index 677f4685a..2126dce8c 100644 --- a/fdm-calculator/src/balance/nitrogen/target.test.ts +++ b/fdm-calculator/src/balance/nitrogen/target.test.ts @@ -15,12 +15,15 @@ describe("calculateTargetForNitrogenBalance", () => { const createCultivation = ( b_lu_catalogue: string, + b_lu_croprotation: "cereal" | "grass" | "maize" | "other", ): FieldInput["cultivations"][0] => ({ b_lu: "test_lu", b_lu_catalogue, m_cropresidue: true, b_lu_start: new Date("2023-01-01"), b_lu_end: new Date("2023-12-31"), + b_lu_name: "Test Cultivation", + b_lu_croprotation, }) const createSoilAnalysis = ( @@ -56,7 +59,7 @@ describe("calculateTargetForNitrogenBalance", () => { ]) it("should calculate target for grassland on dry sandy soil", () => { - const cultivations = [createCultivation("grass1")] + const cultivations = [createCultivation("grass1", "grass")] const soilAnalysis = createSoilAnalysis("duinzand", "VIIo") const cultivationDetailsMap = createCultivationDetailsMap( "grass1", @@ -72,7 +75,7 @@ describe("calculateTargetForNitrogenBalance", () => { }) it("should calculate target for grassland on other soil types", () => { - const cultivations = [createCultivation("grass1")] + const cultivations = [createCultivation("grass1", "grass")] const soilAnalysis = createSoilAnalysis("zeeklei", "V") const cultivationDetailsMap = createCultivationDetailsMap( "grass1", @@ -89,7 +92,7 @@ describe("calculateTargetForNitrogenBalance", () => { it("should calculate target for arable land on dry sandy soil", () => { const cultivations: FieldInput["cultivations"] = [ - createCultivation("crop1"), + createCultivation("crop1", "cereal"), ] const soilAnalysis = createSoilAnalysis("dekzand", "VIIo") const cultivationDetailsMap = createCultivationDetailsMap( @@ -107,7 +110,7 @@ describe("calculateTargetForNitrogenBalance", () => { it("should calculate target for arable land on average sandy soil", () => { const cultivations: FieldInput["cultivations"] = [ - createCultivation("crop1"), + createCultivation("crop1", "cereal"), ] const soilAnalysis = createSoilAnalysis("loess", "Vb") const cultivationDetailsMap = createCultivationDetailsMap( @@ -125,7 +128,7 @@ describe("calculateTargetForNitrogenBalance", () => { it("should calculate target for arable land on wet sandy soil", () => { const cultivations: FieldInput["cultivations"] = [ - createCultivation("crop1"), + createCultivation("crop1", "cereal"), ] const soilAnalysis = createSoilAnalysis("dalgrond", "III") const cultivationDetailsMap = createCultivationDetailsMap( @@ -143,7 +146,7 @@ describe("calculateTargetForNitrogenBalance", () => { it("should calculate target for arable land on dry clay soil", () => { const cultivations: FieldInput["cultivations"] = [ - createCultivation("crop1"), + createCultivation("crop1", "cereal"), ] const soilAnalysis = createSoilAnalysis("rivierklei", "VIIo") const cultivationDetailsMap = createCultivationDetailsMap( @@ -161,7 +164,7 @@ describe("calculateTargetForNitrogenBalance", () => { it("should calculate target for arable land on other clay soil", () => { const cultivations: FieldInput["cultivations"] = [ - createCultivation("crop1"), + createCultivation("crop1", "cereal"), ] const soilAnalysis = createSoilAnalysis("veen", "Va") const cultivationDetailsMap = createCultivationDetailsMap( @@ -179,7 +182,7 @@ describe("calculateTargetForNitrogenBalance", () => { it("should adjust target value based on time frame", () => { const cultivations: FieldInput["cultivations"] = [ - createCultivation("grass1"), + createCultivation("grass1", "grass"), ] const soilAnalysis = createSoilAnalysis("duinzand", "VIIo") const cultivationDetailsMap = createCultivationDetailsMap( diff --git a/fdm-calculator/src/balance/nitrogen/types.d.ts b/fdm-calculator/src/balance/nitrogen/types.d.ts index 7a922fb87..5e661e32a 100644 --- a/fdm-calculator/src/balance/nitrogen/types.d.ts +++ b/fdm-calculator/src/balance/nitrogen/types.d.ts @@ -92,6 +92,17 @@ export type NitrogenSupplyFixation = { cultivations: { id: string; value: Decimal }[] } +/** + * Represents the nitrogen supply derived from atmospheric deposition. + * All values are in kilograms of nitrogen per hectare (kg N / ha). + */ +export type NitrogenSupplyDeposition = { + /** + * The total amount of nitrogen deposited on the field. + */ + total: Decimal +} + /** * Represents the amount of nitrogen supply derived from soil mineralization. * All values are in kilograms of nitrogen per hectare (kg N / ha). @@ -127,7 +138,7 @@ export type NitrogenSupply = { /** * The amount of nitrogen supplied through atmospheric deposition. */ - deposition: { total: Decimal } + deposition: NitrogenSupplyDeposition /** * The amount of nitrogen supplied through mineralization of organic matter in the soil during a cultivation */ @@ -461,7 +472,13 @@ export type FieldInput = { field: Pick cultivations: Pick< Cultivation, - "b_lu" | "b_lu_start" | "b_lu_end" | "b_lu_catalogue" | "m_cropresidue" + | "b_lu" + | "b_lu_start" + | "b_lu_end" + | "b_lu_catalogue" + | "m_cropresidue" + | "b_lu_name" + | "b_lu_croprotation" >[] harvests: Harvest[] soilAnalyses: Pick< @@ -477,6 +494,7 @@ export type FieldInput = { | "b_gwl_class" >[] fertilizerApplications: FertilizerApplication[] + depositionSupply: NitrogenSupplyDeposition } /** @@ -509,7 +527,7 @@ export type FertilizerDetail = Pick< > /** - * Represents the overall input structure required for nitrogen balance calculation. + * Represents the overall input structure required for nitrogen balance calculation for an entire farm. */ export type NitrogenBalanceInput = { fields: FieldInput[] @@ -521,6 +539,19 @@ export type NitrogenBalanceInput = { } } +/** + * Represents the input structure required for nitrogen balance calculation for a single field. + */ +export type NitrogenBalanceFieldInput = { + fieldInput: FieldInput + fertilizerDetails: FertilizerDetail[] + cultivationDetails: CultivationDetail[] + timeFrame: { + start: Date + end: Date + } +} + // Numeric version of NitrogenSupplyFertilizers export type NitrogenSupplyFertilizersNumeric = { total: number diff --git a/fdm-calculator/src/balance/organic-matter/index.test.ts b/fdm-calculator/src/balance/organic-matter/index.test.ts index 47c52ab09..4f0dbf336 100644 --- a/fdm-calculator/src/balance/organic-matter/index.test.ts +++ b/fdm-calculator/src/balance/organic-matter/index.test.ts @@ -9,11 +9,9 @@ import { } from "./index" import * as supply from "./supply" import type { - CultivationDetail, - FertilizerDetail, FieldInput, - OrganicMatterBalanceField, - OrganicMatterBalanceFieldResult, + OrganicMatterBalanceFieldNumeric, + OrganicMatterBalanceFieldResultNumeric, OrganicMatterBalanceInput, } from "./types" @@ -33,8 +31,6 @@ describe("Organic Matter Balance Calculation", () => { const mockCultivations: FieldInput["cultivations"] = [] const mockFertilizerApplications: FieldInput["fertilizerApplications"] = [] const mockSoilAnalyses: FieldInput["soilAnalyses"] = [] - const mockCultivationDetailsMap = new Map() - const mockFertilizerDetailsMap = new Map() describe("calculateOrganicMatterBalanceField", () => { it("should calculate balance as supply - degradation", () => { @@ -49,115 +45,85 @@ describe("Organic Matter Balance Calculation", () => { }) vi.spyOn(shared, "combineSoilAnalyses").mockReturnValue({} as any) - const result = calculateOrganicMatterBalanceField( - mockField, - mockCultivations, - mockFertilizerApplications, - mockSoilAnalyses, - mockFertilizerDetailsMap, - mockCultivationDetailsMap, - timeFrame, - ) - - expect(result.balance?.balance.toNumber()).toBe(300) - expect(result.balance?.supply.total.toNumber()).toBe(500) - expect(result.balance?.degradation.total.toNumber()).toBe(-200) - }) - - it("should return an error message if a sub-calculation fails", () => { - vi.spyOn(supply, "calculateOrganicMatterSupply").mockImplementation( - () => { - throw new Error("Supply calculation failed") + const result = calculateOrganicMatterBalanceField({ + fieldInput: { + field: mockField, + cultivations: mockCultivations, + fertilizerApplications: mockFertilizerApplications, + soilAnalyses: mockSoilAnalyses, }, - ) - vi.spyOn(shared, "combineSoilAnalyses").mockReturnValue({} as any) - - const result = calculateOrganicMatterBalanceField( - mockField, - mockCultivations, - mockFertilizerApplications, - mockSoilAnalyses, - mockFertilizerDetailsMap, - mockCultivationDetailsMap, + fertilizerDetails: [], + cultivationDetails: [], timeFrame, - ) + }) - expect(result.errorMessage).toBe("Supply calculation failed") - expect(result.balance).toBeUndefined() + expect(result.balance).toBe(300) + expect(result.supply.total).toBe(500) + expect(result.degradation.total).toBe(-200) }) }) describe("calculateOrganicMatterBalancesFieldToFarm", () => { it("should aggregate field results to a weighted farm average", () => { - const results: OrganicMatterBalanceFieldResult[] = [ + const results: OrganicMatterBalanceFieldResultNumeric[] = [ { b_id: "field1", b_area: 10, balance: { - supply: { total: new Decimal(500) }, - degradation: { total: new Decimal(-200) }, - balance: new Decimal(300), - } as OrganicMatterBalanceField, + supply: { total: 500 }, + degradation: { total: -200 }, + balance: 300, + } as OrganicMatterBalanceFieldNumeric, }, { b_id: "field2", b_area: 5, balance: { - supply: { total: new Decimal(400) }, - degradation: { total: new Decimal(-300) }, - balance: new Decimal(100), - } as OrganicMatterBalanceField, + supply: { total: 400 }, + degradation: { total: -300 }, + balance: 100, + } as OrganicMatterBalanceFieldNumeric, }, ] - const fields: FieldInput[] = [ - { field: { b_id: "field1", b_area: 10 } } as FieldInput, - { field: { b_id: "field2", b_area: 5 } } as FieldInput, - ] const farmBalance = calculateOrganicMatterBalancesFieldToFarm( results, - fields, false, [], ) - // Total Supply = (500*10 + 400*5) / (10+5) = 7000 / 15 = 466.67 - // Total Degradation = (200*10 + 300*5) / (10+5) = 3500 / 15 = 233.33 - // Total Balance = 466.67 - 233.33 = 233.34 - expect(farmBalance.supply.toNumber()).toBeCloseTo(466.67, 2) - expect(farmBalance.degradation.toNumber()).toBeCloseTo(-233.33, 2) - expect(farmBalance.balance.toNumber()).toBeCloseTo(233.33, 2) + // Total Supply = (500*10 + 400*5) / (10+5) = 7000 / 15 = 466.67 -> 467 + // Total Degradation = ((-200)*10 + (-300)*5) / (10+5) = -3500 / 15 = -233.33 -> -233 + // Total Balance = 466.67 + (-233.33) = 233.34 -> 233 (supply + degradation, since degradation is negative) + expect(farmBalance.supply).toBe(467) + expect(farmBalance.degradation).toBe(-233) + expect(farmBalance.balance).toBe(233) }) it("should handle cases with calculation errors", () => { - const results: OrganicMatterBalanceFieldResult[] = [ + const results: OrganicMatterBalanceFieldResultNumeric[] = [ { b_id: "field1", b_area: 10, balance: { - balance: new Decimal(300), - supply: { total: new Decimal(500) }, - degradation: { total: new Decimal(-200) }, - } as OrganicMatterBalanceField, + balance: 300, + supply: { total: 500 }, + degradation: { total: -200 }, + } as OrganicMatterBalanceFieldNumeric, }, { b_id: "field2", b_area: 5, errorMessage: "Failed" }, ] - const fields: FieldInput[] = [ - { field: { b_id: "field1", b_area: 10 } } as FieldInput, - { field: { b_id: "field2", b_area: 5 } } as FieldInput, - ] const farmBalance = calculateOrganicMatterBalancesFieldToFarm( results, - fields, true, ["Error"], ) expect(farmBalance.hasErrors).toBe(true) expect(farmBalance.fieldErrorMessages).toEqual(["Error"]) // Check that only the successful field is aggregated - expect(farmBalance.supply.toNumber()).toBeCloseTo(500) - expect(farmBalance.degradation.toNumber()).toBeCloseTo(-200) - expect(farmBalance.balance.toNumber()).toBeCloseTo(300) + expect(farmBalance.supply).toBeCloseTo(500) + expect(farmBalance.degradation).toBeCloseTo(-200) + expect(farmBalance.balance).toBeCloseTo(300) }) }) diff --git a/fdm-calculator/src/balance/organic-matter/index.ts b/fdm-calculator/src/balance/organic-matter/index.ts index 69fd936bf..3061cddc6 100644 --- a/fdm-calculator/src/balance/organic-matter/index.ts +++ b/fdm-calculator/src/balance/organic-matter/index.ts @@ -1,17 +1,16 @@ -import { withCalculationCache } from "@svenvw/fdm-core" +import { type FdmType, withCalculationCache } from "@svenvw/fdm-core" import Decimal from "decimal.js" import pkg from "../../package" -import { convertOrganicMatterBalanceToNumeric } from "../shared/conversion" +import { convertDecimalToNumberRecursive } from "../shared/conversion" import { combineSoilAnalyses } from "../shared/soil" import { calculateOrganicMatterDegradation } from "./degradation" import { calculateOrganicMatterSupply } from "./supply" import type { CultivationDetail, FertilizerDetail, - FieldInput, - OrganicMatterBalance, - OrganicMatterBalanceField, - OrganicMatterBalanceFieldResult, + OrganicMatterBalanceFieldInput, + OrganicMatterBalanceFieldNumeric, + OrganicMatterBalanceFieldResultNumeric, OrganicMatterBalanceInput, OrganicMatterBalanceNumeric, SoilAnalysisPicked, @@ -26,96 +25,72 @@ import type { * farm-level balance. The final output is a numeric representation of the balance, * suitable for display or further analysis. * + * @param fdm - The FDM instance for database access (caching). * @param organicMatterBalanceInput - The complete dataset required for the calculation, including all fields, * fertilizer catalogues, and cultivation catalogues for the farm. * @returns A promise that resolves to the aggregated `OrganicMatterBalanceNumeric` object for the farm. * @throws {Error} Throws an error if the calculation process fails for any reason. */ export async function calculateOrganicMatterBalance( + fdm: FdmType, organicMatterBalanceInput: OrganicMatterBalanceInput, ): Promise { // Destructure input for easier access. const { fields, fertilizerDetails, cultivationDetails, timeFrame } = organicMatterBalanceInput - // Pre-process catalogue details into Maps for efficient lookups within the calculation functions. - const fertilizerDetailsMap = new Map( - fertilizerDetails.map((detail: FertilizerDetail) => [ - detail.p_id_catalogue, - detail, - ]), - ) - const cultivationDetailsMap = new Map( - cultivationDetails.map((detail: CultivationDetail) => [ - detail.b_lu_catalogue, - detail, - ]), - ) - // Process fields in batches to avoid overwhelming the system with concurrent promises, // especially for farms with a large number of fields. - const batchSize = 50 // This can be adjusted based on performance testing. - const fieldsWithBalanceResults: OrganicMatterBalanceFieldResult[] = [] - let hasErrors = false - const fieldErrorMessages: string[] = [] + const fieldsWithBalanceResults: OrganicMatterBalanceFieldResultNumeric[] = + [] + const batchSize = 50 for (let i = 0; i < fields.length; i += batchSize) { const batch = fields.slice(i, i + batchSize) - const batchPromises = batch.map(async (field: FieldInput) => { - // Calculate the balance for each field individually. - return calculateOrganicMatterBalanceField( - field.field, - field.cultivations, - field.fertilizerApplications, - field.soilAnalyses, - fertilizerDetailsMap, - cultivationDetailsMap, - timeFrame, - ) - }) - - // Wait for the current batch to complete. - const batchResults = await Promise.all(batchPromises) - for (const r of batchResults) { - // Collect any errors that occurred during field calculations. - if (r.errorMessage) { - hasErrors = true - fieldErrorMessages.push(`[${r.b_id}] ${r.errorMessage}`) - } - } + const batchResults = await Promise.all( + batch.map(async (fieldInput) => { + try { + const balance = await getOrganicMatterBalanceField(fdm, { + fieldInput, + fertilizerDetails, + cultivationDetails, + timeFrame, + }) + return { + b_id: fieldInput.field.b_id, + b_area: fieldInput.field.b_area ?? 0, + balance, + } + } catch (error) { + return { + b_id: fieldInput.field.b_id, + b_area: fieldInput.field.b_area ?? 0, + errorMessage: + error instanceof Error + ? error.message + : String(error), + } + } + }), + ) fieldsWithBalanceResults.push(...batchResults) } + const hasErrors = fieldsWithBalanceResults.some( + (result) => result.errorMessage !== undefined, + ) + const fieldErrorMessages = fieldsWithBalanceResults + .filter((result) => result.errorMessage !== undefined) + .map((result) => result.errorMessage as string) + // Aggregate the results from all individual fields into a single farm-level balance. - const farmWithBalanceDecimal = calculateOrganicMatterBalancesFieldToFarm( + return calculateOrganicMatterBalancesFieldToFarm( fieldsWithBalanceResults, - fields, hasErrors, fieldErrorMessages, ) - - // Convert the final `Decimal`-based result to a plain `number`-based object. - return convertOrganicMatterBalanceToNumeric(farmWithBalanceDecimal) } -/** - * A cached version of the `calculateOrganicMatterBalance` function. - * - * This wrapper provides caching capabilities to the main calculation function, - * returning a stored result if the same input has been processed before. This can - * significantly improve performance for repeated requests with identical data. - * The cache is versioned using the calculator's package version to ensure data integrity - * after updates. - * - * @param organicMatterBalanceInput - The input data for the organic matter balance calculation. - * @returns A promise that resolves with the calculated `OrganicMatterBalanceNumeric`. - */ -export const getOrganicMatterBalance = withCalculationCache( - calculateOrganicMatterBalance, - "calculateOrganicMatterBalance", - pkg.calculatorVersion, -) - /** * Calculates the organic matter balance for a single field. * @@ -124,73 +99,83 @@ export const getOrganicMatterBalance = withCalculationCache( * `calculateOrganicMatterSupply` and `calculateOrganicMatterDegradation` to get the two * main components of the balance. * - * @param field - The core details of the field. - * @param cultivations - An array of cultivation records for the field. - * @param fertilizerApplications - An array of fertilizer application records. - * @param soilAnalyses - An array of soil analysis records. - * @param fertilizerDetailsMap - A map of available fertilizer details. - * @param cultivationDetailsMap - A map of available cultivation details. - * @param timeFrame - The calculation period. + * @param organicMatterBalanceFieldInput - The input data for the organic matter balance calculation for a single field. * @returns A `OrganicMatterBalanceFieldResult` object containing the detailed balance or an error message. */ export function calculateOrganicMatterBalanceField( - field: FieldInput["field"], - cultivations: FieldInput["cultivations"], - fertilizerApplications: FieldInput["fertilizerApplications"], - soilAnalyses: FieldInput["soilAnalyses"], - fertilizerDetailsMap: Map, - cultivationDetailsMap: Map, - timeFrame: { start: Date; end: Date }, -): OrganicMatterBalanceFieldResult { - try { - const fieldDetails = field + organicMatterBalanceFieldInput: OrganicMatterBalanceFieldInput, +): OrganicMatterBalanceFieldNumeric { + const { fieldInput, fertilizerDetails, cultivationDetails, timeFrame } = + organicMatterBalanceFieldInput - // 1. Combine multiple soil analyses into a single representative record for the field. - // We need 'a_som_loi' and 'a_density_sa' for the degradation calculation. - const soilAnalysis = combineSoilAnalyses( - soilAnalyses, - ["a_som_loi", "a_density_sa", "b_soiltype_agr"], - true, // Enable estimation of missing values if possible - ) + const { field, cultivations, fertilizerApplications, soilAnalyses } = + fieldInput - // 2. Calculate the total supply of effective organic matter (EOM). - const supply = calculateOrganicMatterSupply( - cultivations, - fertilizerApplications, - cultivationDetailsMap, - fertilizerDetailsMap, - timeFrame, - ) + const fertilizerDetailsMap = new Map( + fertilizerDetails.map((detail: FertilizerDetail) => [ + detail.p_id_catalogue, + detail, + ]), + ) + const cultivationDetailsMap = new Map( + cultivationDetails.map((detail: CultivationDetail) => [ + detail.b_lu_catalogue, + detail, + ]), + ) + const fieldDetails = field + + // 1. Combine multiple soil analyses into a single representative record for the field. + // We need 'a_som_loi' and 'a_density_sa' for the degradation calculation. + const soilAnalysis = combineSoilAnalyses( + soilAnalyses, + ["a_som_loi", "a_density_sa", "b_soiltype_agr"], + true, // Enable estimation of missing values if possible + ) - // 3. Calculate the total degradation of soil organic matter (SOM). - const degradation = calculateOrganicMatterDegradation( - soilAnalysis, - cultivations, - cultivationDetailsMap, - timeFrame, - ) + // 2. Calculate the total supply of effective organic matter (EOM). + const supply = calculateOrganicMatterSupply( + cultivations, + fertilizerApplications, + cultivationDetailsMap, + fertilizerDetailsMap, + timeFrame, + ) - // 4. Calculate the final balance: EOM Supply - SOM Degradation. - return { - b_id: fieldDetails.b_id, - b_area: fieldDetails.b_area ?? 0, - balance: { - b_id: fieldDetails.b_id, - balance: supply.total.plus(degradation.total), - supply: supply, - degradation: degradation, - }, - } - } catch (error) { - // If any step fails, return a result object with an error message. - return { - b_id: field.b_id, - b_area: field.b_area ?? 0, - errorMessage: String(error).replace("Error: ", ""), - } - } + // 3. Calculate the total degradation of soil organic matter (SOM). + const degradation = calculateOrganicMatterDegradation( + soilAnalysis, + cultivations, + cultivationDetailsMap, + timeFrame, + ) + + // 4. Calculate the final balance: EOM Supply - SOM Degradation. + return convertDecimalToNumberRecursive({ + b_id: fieldDetails.b_id, + balance: supply.total.plus(degradation.total), + supply: supply, + degradation: degradation, + }) as OrganicMatterBalanceFieldNumeric } +/** + * A cached version of the `calculateOrganicMatterBalanceField` function. + * + * This function provides the same functionality as `calculateOrganicMatterBalanceField` but + * includes a caching mechanism to improve performance for repeated calls with the + * same input. The cache is managed by `withCalculationCache` and uses the + * `pkg.calculatorVersion` as part of its cache key. + * + * @param organicMatterBalanceFieldInput - The input data for the organic matter balance calculation for a single field. + * @returns A promise that resolves with the calculated organic matter balance, with numeric values as numbers. + */ +export const getOrganicMatterBalanceField = withCalculationCache( + calculateOrganicMatterBalanceField, + "calculateOrganicMatterBalanceField", + pkg.calculatorVersion, +) + /** * Aggregates the organic matter balances from individual fields to a farm-level summary. * @@ -198,23 +183,21 @@ export function calculateOrganicMatterBalanceField( * and calculates a weighted average for the farm's overall supply, degradation, and balance, * using the area of each field as the weight. * - * @param fieldsWithBalanceResults - An array of `OrganicMatterBalanceFieldResult` objects. - * @param fields - The original array of `FieldInput` objects, used to retrieve field areas. + * @param fieldsWithBalanceResults - An array of `OrganicMatterBalanceFieldResultNumeric` objects. * @param hasErrors - A boolean flag indicating if any field calculations failed. * @param fieldErrorMessages - An array of error messages from failed calculations. - * @returns A single `OrganicMatterBalance` object representing the aggregated farm-level results. + * @returns A single `OrganicMatterBalanceNumeric` object representing the aggregated farm-level results. */ export function calculateOrganicMatterBalancesFieldToFarm( - fieldsWithBalanceResults: OrganicMatterBalanceFieldResult[], - fields: FieldInput[], + fieldsWithBalanceResults: OrganicMatterBalanceFieldResultNumeric[], hasErrors: boolean, fieldErrorMessages: string[], -): OrganicMatterBalance { +): OrganicMatterBalanceNumeric { // Filter out fields that have errors to ensure they are not included in the aggregation. const successfulFieldBalances = fieldsWithBalanceResults.filter( (result) => result.balance !== undefined, - ) as (OrganicMatterBalanceFieldResult & { - balance: OrganicMatterBalanceField + ) as (OrganicMatterBalanceFieldResultNumeric & { + balance: OrganicMatterBalanceFieldNumeric })[] let totalFarmSupply = new Decimal(0) @@ -223,24 +206,15 @@ export function calculateOrganicMatterBalancesFieldToFarm( // Calculate the total supply and degradation across the farm, weighted by field area. for (const fieldResult of successfulFieldBalances) { - const fieldInput = fields.find((f) => f.field.b_id === fieldResult.b_id) - - if (!fieldInput) { - // This should not happen in a normal flow but is a safeguard. - console.warn( - `Could not find field input for field balance ${fieldResult.b_id}`, - ) - continue - } - const fieldArea = new Decimal(fieldInput.field.b_area ?? 0) + const fieldArea = new Decimal(fieldResult.b_area ?? 0) totalFarmArea = totalFarmArea.add(fieldArea) // Add the area-weighted supply and degradation to the farm totals. totalFarmSupply = totalFarmSupply.add( - fieldResult.balance.supply.total.times(fieldArea), + new Decimal(fieldResult.balance.supply.total).times(fieldArea), ) totalFarmDegradation = totalFarmDegradation.add( - fieldResult.balance.degradation.total.times(fieldArea), + new Decimal(fieldResult.balance.degradation.total).times(fieldArea), ) } @@ -255,17 +229,12 @@ export function calculateOrganicMatterBalancesFieldToFarm( // The final farm balance is the difference between the average supply and average degradation. const avgFarmBalance = avgFarmSupply.plus(avgFarmDegradation) - // Construct the final farm-level balance object. - const farmWithBalance: OrganicMatterBalance = { + return convertDecimalToNumberRecursive({ balance: avgFarmBalance, supply: avgFarmSupply, degradation: avgFarmDegradation, - fields: fieldsWithBalanceResults, // Include results for all fields, even those with errors. - hasErrors: - hasErrors || - fieldsWithBalanceResults.length !== successfulFieldBalances.length, - fieldErrorMessages: fieldErrorMessages, - } - - return farmWithBalance + fields: fieldsWithBalanceResults, + hasErrors, + fieldErrorMessages, + }) as OrganicMatterBalanceNumeric } diff --git a/fdm-calculator/src/balance/organic-matter/performance.test.ts b/fdm-calculator/src/balance/organic-matter/performance.test.ts new file mode 100644 index 000000000..3c9ef7e40 --- /dev/null +++ b/fdm-calculator/src/balance/organic-matter/performance.test.ts @@ -0,0 +1,160 @@ +import type { FdmType } from "@svenvw/fdm-core" +import { describe, expect, it, vi } from "vitest" +import { calculateOrganicMatterBalance } from "./index" +import type { + CultivationDetail, + FertilizerDetail, + FieldInput, + OrganicMatterBalanceInput, +} from "./types" + +// Mock FdmType +const mockFdm = { + select: vi.fn().mockReturnThis(), + from: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + limit: vi.fn().mockReturnThis(), + // biome-ignore lint/suspicious/noThenProperty: Simulate cache miss + then: vi.fn((resolve) => + resolve ? Promise.resolve(resolve([])) : Promise.resolve([]), + ), + insert: vi.fn().mockReturnThis(), + values: vi.fn().mockResolvedValue(undefined), +} as unknown as FdmType + +/** + * Utility function to generate mock data for performance testing. + * This function creates a specified number of fields with realistic, but simplified, + * associated data for cultivations, fertilizer applications, and soil analyses. + * + * @param numberOfFields - The number of fields to generate. + * @returns A OrganicMatterBalanceInput object with dynamically generated data. + */ +function generateMockData(numberOfFields: number): OrganicMatterBalanceInput { + const fields: FieldInput[] = [] + const fertilizerDetails: FertilizerDetail[] = [ + { + p_id_catalogue: "fert-cat-1", + p_eom: 100, // 100 kg EOM/ton (simplified unit) + p_type: "manure", + }, + { + p_id_catalogue: "fert-cat-2", + p_eom: 500, // 500 kg EOM/ton + p_type: "compost", + }, + ] + const cultivationDetails: CultivationDetail[] = [ + { + b_lu_catalogue: "cat-cult-1", + b_lu_croprotation: "maize", + b_lu_eom: 1500, + b_lu_eom_residues: 200, + }, + { + b_lu_catalogue: "cat-cult-2", + b_lu_croprotation: "grass", + b_lu_eom: 800, + b_lu_eom_residues: 100, + }, + ] + + for (let i = 0; i < numberOfFields; i++) { + const fieldId = `field-${i}` + const fieldStart = new Date(2023, 0, 1) + const fieldEnd = new Date(2023, 11, 31) + + const field: FieldInput["field"] = { + b_id: fieldId, + b_centroid: [ + Math.random() * 10 + 4, // Random longitude between 4 and 14 + Math.random() * 5 + 50, // Random latitude between 50 and 55 + ], + b_area: Math.floor(Math.random() * 50 + 10), // Random area between 10 and 60 + b_start: fieldStart, + b_end: fieldEnd, + } + + const cultivations: FieldInput["cultivations"] = [ + { + b_lu: `cult-${fieldId}-1`, + b_lu_catalogue: "cat-cult-1", + m_cropresidue: true, + b_lu_start: new Date(2023, 3, 1), + b_lu_end: new Date(2023, 8, 1), + b_lu_name: "Mock Cultivation", + }, + ] + + const soilAnalyses: FieldInput["soilAnalyses"] = [ + { + a_id: `sa-${fieldId}-1`, + b_sampling_date: new Date(2023, 2, 1), + a_som_loi: Math.random() * 2 + 3, // 3-5% + a_density_sa: Math.random() * 0.5 + 1.2, // 1.2-1.7 + b_soiltype_agr: "zand", + }, + ] + + const fertilizerApplications: FieldInput["fertilizerApplications"] = [ + { + p_app_id: `fa-${fieldId}-1`, + // Randomly pick one of the available fertilizer catalogue IDs + p_id_catalogue: "fert-cat-1", + p_app_amount: Math.floor(Math.random() * 20 + 10), // 10-30 tons + p_app_date: new Date(2023, 4, 1), + p_app_method: "broadcasting", + p_name_nl: "Mock Fertilizer", + p_id: `mock-fert-${i}`, + }, + ] + + fields.push({ + field, + cultivations, + soilAnalyses, + fertilizerApplications, + }) + } + + return { + fields, + fertilizerDetails, + cultivationDetails, + timeFrame: { + start: new Date(2023, 0, 1), + end: new Date(2023, 11, 31), + }, + } +} + +describe("Organic Matter Balance Performance", () => { + // This test is designed to measure the performance of the organic matter balance calculation + // for a large number of fields. + // The timeout is set to 30 seconds (30000 ms). + it("should calculate organic matter balance for a large farm (~300 fields) within 30 seconds", async () => { + const numberOfFields = 300 + const mockInput = generateMockData(numberOfFields) + + // Measure execution time + const startTime = process.hrtime.bigint() + + const result = await calculateOrganicMatterBalance(mockFdm, mockInput) + + const endTime = process.hrtime.bigint() + const durationMs = Number(endTime - startTime) / 1_000_000 + + console.log( + `Calculated organic matter balance for ${numberOfFields} fields in ${durationMs.toFixed(2)} ms`, + ) + + expect(result).toBeDefined() + expect(result.fields.length).toBe(numberOfFields) + expect(typeof result.balance).toBe("number") + expect(typeof result.supply).toBe("number") + expect(typeof result.degradation).toBe("number") + + // Assert that the calculation completed within the desired timeout + expect(durationMs).toBeLessThan(30000) // 30 seconds + }, 35000) // Set Vitest timeout slightly higher than the expected test duration +}) diff --git a/fdm-calculator/src/balance/organic-matter/types.ts b/fdm-calculator/src/balance/organic-matter/types.ts index bf21d4eba..a7c9c309f 100644 --- a/fdm-calculator/src/balance/organic-matter/types.ts +++ b/fdm-calculator/src/balance/organic-matter/types.ts @@ -254,6 +254,24 @@ export type OrganicMatterBalanceInput = { } } +/** + * Represents the necessary input data for a single field for the organic matter balance calculation. + * This includes the field-specific data as well as the shared catalogue details. + */ +export type OrganicMatterBalanceFieldInput = { + /** The input data for the specific field. */ + fieldInput: FieldInput + /** A list of all available fertilizer details from the farm's catalogue. */ + fertilizerDetails: FertilizerDetail[] + /** A list of all available cultivation details from the farm's catalogue. */ + cultivationDetails: CultivationDetail[] + /** The calculation period. */ + timeFrame: { + start: Date + end: Date + } +} + // --- Numeric Types --- // The following types are numeric-only versions of the types above, // intended for final outputs where `Decimal` objects are converted to numbers. diff --git a/fdm-calculator/src/balance/shared/conversion.ts b/fdm-calculator/src/balance/shared/conversion.ts index 07244a961..9de35276a 100644 --- a/fdm-calculator/src/balance/shared/conversion.ts +++ b/fdm-calculator/src/balance/shared/conversion.ts @@ -1,9 +1,4 @@ import Decimal from "decimal.js" -import type { - NitrogenBalance, - NitrogenBalanceFieldNumeric, - NitrogenBalanceNumeric, -} from "../nitrogen/types" import type { OrganicMatterBalance, OrganicMatterBalanceFieldNumeric, @@ -12,8 +7,14 @@ import type { // Helper function to convert Decimal to number recursively export function convertDecimalToNumberRecursive(data: unknown): unknown { - if (data instanceof Decimal) { - return data.round().toNumber() + if ( + data instanceof Decimal || + (typeof data === "object" && data !== null && (data as any).isDecimal) + ) { + return (data as Decimal).round().toNumber() + } + if (typeof data === "number") { + return data } if (Array.isArray(data)) { return data.map(convertDecimalToNumberRecursive) @@ -22,9 +23,10 @@ export function convertDecimalToNumberRecursive(data: unknown): unknown { const newData: { [key: string]: unknown } = {} for (const key in data) { if (Object.hasOwn(data, key)) { - newData[key] = convertDecimalToNumberRecursive( + const converted = convertDecimalToNumberRecursive( (data as Record)[key], ) + newData[key] = converted } } return newData @@ -32,34 +34,6 @@ export function convertDecimalToNumberRecursive(data: unknown): unknown { return data } -// Main conversion function for NitrogenBalance -export function convertNitrogenBalanceToNumeric( - balance: NitrogenBalance, -): NitrogenBalanceNumeric { - const numericBalance = convertDecimalToNumberRecursive( - balance, - ) as NitrogenBalanceNumeric - - numericBalance.fields = balance.fields.map((fieldResult) => { - if (fieldResult.balance) { - return { - b_id: fieldResult.b_id, - b_area: fieldResult.b_area, - balance: convertDecimalToNumberRecursive( - fieldResult.balance, - ) as NitrogenBalanceFieldNumeric, - } - } - return { - b_id: fieldResult.b_id, - b_area: fieldResult.b_area, - errorMessage: fieldResult.errorMessage, - } - }) - - return numericBalance -} - // Main conversion function for OrganicMatterBalance export function convertOrganicMatterBalanceToNumeric( balance: OrganicMatterBalance, diff --git a/fdm-calculator/src/index.ts b/fdm-calculator/src/index.ts index 2ca0b47bc..27ed81d3f 100644 --- a/fdm-calculator/src/index.ts +++ b/fdm-calculator/src/index.ts @@ -2,11 +2,13 @@ import pkg from "./package" export const fdmCalculator = pkg export { calculateNitrogenBalance, - getNitrogenBalance, + calculateNitrogenBalanceField, + getNitrogenBalanceField, } from "./balance/nitrogen/index" export { collectInputForNitrogenBalance } from "./balance/nitrogen/input" export type { FieldInput, + NitrogenBalanceFieldInput, NitrogenBalanceFieldNumeric, NitrogenBalanceFieldResultNumeric, NitrogenBalanceInput, @@ -14,6 +16,7 @@ export type { NitrogenEmissionAmmoniaFertilizersNumeric, NitrogenEmissionAmmoniaNumeric, NitrogenEmissionAmmoniaResiduesNumeric, + NitrogenEmissionNumeric, NitrogenRemovalHarvestsNumeric, NitrogenRemovalNumeric, NitrogenRemovalResiduesNumeric, @@ -24,7 +27,8 @@ export type { } from "./balance/nitrogen/types" export { calculateOrganicMatterBalance, - getOrganicMatterBalance, + calculateOrganicMatterBalanceField, + getOrganicMatterBalanceField, } from "./balance/organic-matter/index" export { collectInputForOrganicMatterBalance } from "./balance/organic-matter/input" export type { diff --git a/fdm-calculator/src/nutrient-advice/index.ts b/fdm-calculator/src/nutrient-advice/index.ts index 15b427472..5e46c5815 100644 --- a/fdm-calculator/src/nutrient-advice/index.ts +++ b/fdm-calculator/src/nutrient-advice/index.ts @@ -105,4 +105,5 @@ export const getNutrientAdvice = withCalculationCache( requestNutrientAdvice, "requestNutrientAdvice", pkg.calculatorVersion, + ["nmiApiKey"], ) diff --git a/fdm-core/src/calculator.test.ts b/fdm-core/src/calculator.test.ts index bfff8e057..c8611c1b7 100644 --- a/fdm-core/src/calculator.test.ts +++ b/fdm-core/src/calculator.test.ts @@ -275,4 +275,92 @@ describe("withCalculationCache", () => { .where(eq(calculationCache.calculation_hash, expectedHash)) expect(cached).toHaveLength(0) // Should NOT cache the result if cache read failed }) + + it("should redact sensitive keys from cache key and storage", async () => { + const calculate = vi.fn( + async (inputs: { data: string; apiKey: string }) => { + return `result for ${inputs.data}` + }, + ) + const calculatorVersion = "1.0.0" + const input = { data: "public data", apiKey: "secret-key" } + const getCalculation = withCalculationCache( + calculate, + "calculateWithSecrets", + calculatorVersion, + ["apiKey"], + ) + + // Call the function + await expect(getCalculation(fdm, input)).resolves.toBe( + "result for public data", + ) + + // Verify original function received full input including secret + expect(calculate).toHaveBeenCalledWith(input) + + // Expected input for cache (with REDACTED secret) + const expectedCacheInput = { data: "public data", apiKey: "REDACTED" } + const expectedHash = generateCalculationHash( + "calculateWithSecrets", + calculatorVersion, + expectedCacheInput, + ) + + // Verify cache entry exists with the redacted hash + const cached = await fdm + .select() + .from(calculationCache) + .where(eq(calculationCache.calculation_hash, expectedHash)) + expect(cached).toHaveLength(1) + expect(cached[0].result).toBe("result for public data") + + // Verify stored input in DB is redacted + // Note: The input column type in DB might be JSON, drizzle handles it. + const storedInput = cached[0].input as any + expect(storedInput.apiKey).toBe("REDACTED") + expect(storedInput.data).toBe("public data") + }) + + it("should redact nested sensitive keys from cache key and storage", async () => { + const calculate = vi.fn( + async (inputs: { data: { apiKey: string; value: string } }) => { + return `result for ${inputs.data.value}` + }, + ) + const calculatorVersion = "1.0.0" + const input = { data: { apiKey: "nested-secret", value: "public" } } + const getCalculation = withCalculationCache( + calculate, + "calculateWithNestedSecrets", + calculatorVersion, + ["apiKey"], + ) + + await expect(getCalculation(fdm, input)).resolves.toBe( + "result for public", + ) + + // Verify original function received full input + expect(calculate).toHaveBeenCalledWith(input) + + // Expected input for cache (with nested REDACTED secret) + const expectedCacheInput = { + data: { apiKey: "REDACTED", value: "public" }, + } + const expectedHash = generateCalculationHash( + "calculateWithNestedSecrets", + calculatorVersion, + expectedCacheInput, + ) + + const cached = await fdm + .select() + .from(calculationCache) + .where(eq(calculationCache.calculation_hash, expectedHash)) + expect(cached).toHaveLength(1) + + const storedInput = cached[0].input as any + expect(storedInput.data.apiKey).toBe("REDACTED") + }) }) diff --git a/fdm-core/src/calculator.ts b/fdm-core/src/calculator.ts index fb0a28e14..5f5adbbbe 100644 --- a/fdm-core/src/calculator.ts +++ b/fdm-core/src/calculator.ts @@ -164,6 +164,7 @@ export function withCalculationCache( calculationFunction: (inputs: T_Input) => T_Output | Promise, calculationFunctionName: string, calculatorVersion: string, + sensitiveKeys: string[] = [], ) { return async (fdm: FdmType, input: T_Input) => { if (!calculationFunctionName) { @@ -178,11 +179,40 @@ export function withCalculationCache( ) } + // Sanitize input if sensitive keys are provided + let inputForCache = input + if (sensitiveKeys.length > 0) { + const redact = (obj: unknown): unknown => { + if (typeof obj !== "object" || obj === null) { + return obj + } + if (Array.isArray(obj)) { + return obj.map(redact) + } + // Check if it's a plain object or similar to avoid breaking classes/Dates if they shouldn't be touched + // Ideally input is a plain object for hashing/json. + if (obj instanceof Date) { + return obj + } + + const newObj = { ...(obj as object) } as Record + for (const key of Object.keys(newObj)) { + if (sensitiveKeys.includes(key)) { + newObj[key] = "REDACTED" + } else { + newObj[key] = redact(newObj[key]) + } + } + return newObj + } + inputForCache = redact(input) as T_Input + } + // Generate a unique hash for the current calculation based on function name, version, and input. const calculationHash = generateCalculationHash( calculationFunctionName, calculatorVersion, - input, + inputForCache, ) let cachedResult: T_Output | null = null @@ -230,7 +260,7 @@ export function withCalculationCache( calculationHash, calculationFunctionName, calculatorVersion, - input, + inputForCache, result, ) } catch (e: unknown) { @@ -263,7 +293,7 @@ export function withCalculationCache( fdm, calculationFunctionName, calculatorVersion, - input, + inputForCache, errorMessage, stackTrace, )