From be349e10212dc8f1748f028ffbfd6178e36f47ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ana=20Garc=C3=ADa?= Date: Thu, 22 Oct 2026 12:19:23 +0200 Subject: [PATCH 1/2] fix(import): send one data values request per period (#419) DHIS2 only accepts the first period of a dataValueSets request when the data set has data input periods (DHIS2-22277), so imports and deletes with several periods were partially rejected as "not open". Group the data values by period before chunking them by size. --- src/data/Dhis2DataValueChunks.ts | 13 +++++ src/data/InstanceDhisRepository.ts | 3 +- .../__tests__/Dhis2DataValueChunks.spec.ts | 56 +++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 src/data/Dhis2DataValueChunks.ts create mode 100644 src/data/__tests__/Dhis2DataValueChunks.spec.ts diff --git a/src/data/Dhis2DataValueChunks.ts b/src/data/Dhis2DataValueChunks.ts new file mode 100644 index 00000000..9caf80de --- /dev/null +++ b/src/data/Dhis2DataValueChunks.ts @@ -0,0 +1,13 @@ +import _ from "lodash"; + +// DHIS2 only accepts the first period of a dataValueSets request when the data set has data input +// periods (https://dhis2.atlassian.net/browse/DHIS2-22277). Once it is fixed upstream, chunk by size only. +export function chunkDataValuesByPeriod( + dataValues: ReadonlyArray, + chunkSize: number +): T[][] { + const dataValuesByPeriod = _.groupBy(dataValues, dataValue => dataValue.period); + const periods = _.uniq(dataValues.map(dataValue => dataValue.period)); + + return periods.flatMap(period => _.chunk(dataValuesByPeriod[period] ?? [], chunkSize)); +} diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index d2cf59f7..42030ab8 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -76,6 +76,7 @@ import { resolveRegistrations, resolveRequestedRegistrationKeys, } from "./Dhis2DataSetCompletion"; +import { chunkDataValuesByPeriod } from "./Dhis2DataValueChunks"; import { Maybe } from "../types/utils"; const bulkOperationChunkSize = 1000; @@ -520,7 +521,7 @@ export class InstanceDhisRepository implements InstanceRepository { }; } - const chunks = _.chunk(dataValues, bulkOperationChunkSize); + const chunks = chunkDataValuesByPeriod(dataValues, bulkOperationChunkSize); const chunkResults = await promiseMap(chunks, async chunk => { const { response } = await this.api.dataValues diff --git a/src/data/__tests__/Dhis2DataValueChunks.spec.ts b/src/data/__tests__/Dhis2DataValueChunks.spec.ts new file mode 100644 index 00000000..a8df4471 --- /dev/null +++ b/src/data/__tests__/Dhis2DataValueChunks.spec.ts @@ -0,0 +1,56 @@ +import { chunkDataValuesByPeriod } from "../Dhis2DataValueChunks"; + +const PERIOD_2024 = "2024"; +const PERIOD_2025 = "2025"; +const PERIOD_2026 = "2026"; + +type TestDataValue = Readonly<{ period: string; orgUnit: string; value: string }>; + +function buildValue(period: string, orgUnit = "ou1", value = "1"): TestDataValue { + return { period, orgUnit, value }; +} + +describe("chunkDataValuesByPeriod", () => { + describe("when the data values have several periods", () => { + it("returns one chunk per period, in the order the periods first appear", () => { + const value2025 = buildValue(PERIOD_2025); + const value2024 = buildValue(PERIOD_2024); + const value2026 = buildValue(PERIOD_2026); + + const chunks = chunkDataValuesByPeriod([value2025, value2024, value2026], 1000); + + expect(chunks).toEqual([[value2025], [value2024], [value2026]]); + }); + + it("keeps the values of several org units of the same period in the same chunk", () => { + const ou1In2024 = buildValue(PERIOD_2024, "ou1"); + const ou2In2024 = buildValue(PERIOD_2024, "ou2"); + const ou1In2025 = buildValue(PERIOD_2025, "ou1"); + const ou2In2025 = buildValue(PERIOD_2025, "ou2"); + + const chunks = chunkDataValuesByPeriod([ou1In2024, ou1In2025, ou2In2024, ou2In2025], 1000); + + expect(chunks).toEqual([ + [ou1In2024, ou2In2024], + [ou1In2025, ou2In2025], + ]); + }); + }); + + describe("when a period has more values than the chunk size", () => { + it("splits that period into chunks of the given size", () => { + const values2024 = ["1", "2", "3"].map(value => buildValue(PERIOD_2024, "ou1", value)); + const value2025 = buildValue(PERIOD_2025); + + const chunks = chunkDataValuesByPeriod([...values2024, value2025], 2); + + expect(chunks).toEqual([[values2024[0], values2024[1]], [values2024[2]], [value2025]]); + }); + }); + + describe("when there are no data values", () => { + it("returns no chunks", () => { + expect(chunkDataValuesByPeriod([], 1000)).toEqual([]); + }); + }); +}); From 86de5c02453e25ded7a88d2a9ad67d9d78f592b8 Mon Sep 17 00:00:00 2001 From: Miquel Adell Date: Fri, 23 Oct 2026 12:00:21 +0200 Subject: [PATCH 2/2] chore(release): bump Bulk Load to 3.38.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2221819b..64252159 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "bulk-load", "description": "Bulk importing made easy", - "version": "3.38.0", + "version": "3.38.1", "license": "GPL-3.0", "author": "EyeSeeTea team", "homepage": ".",