diff --git a/package.json b/package.json index 2221819b..64252159 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "bulk-load", "description": "Bulk importing made easy", - "version": "3.38.0", + "version": "3.38.1", "license": "GPL-3.0", "author": "EyeSeeTea team", "homepage": ".", diff --git a/src/data/Dhis2DataValueChunks.ts b/src/data/Dhis2DataValueChunks.ts new file mode 100644 index 00000000..9caf80de --- /dev/null +++ b/src/data/Dhis2DataValueChunks.ts @@ -0,0 +1,13 @@ +import _ from "lodash"; + +// DHIS2 only accepts the first period of a dataValueSets request when the data set has data input +// periods (https://dhis2.atlassian.net/browse/DHIS2-22277). Once it is fixed upstream, chunk by size only. +export function chunkDataValuesByPeriod( + dataValues: ReadonlyArray, + chunkSize: number +): T[][] { + const dataValuesByPeriod = _.groupBy(dataValues, dataValue => dataValue.period); + const periods = _.uniq(dataValues.map(dataValue => dataValue.period)); + + return periods.flatMap(period => _.chunk(dataValuesByPeriod[period] ?? [], chunkSize)); +} diff --git a/src/data/InstanceDhisRepository.ts b/src/data/InstanceDhisRepository.ts index d2cf59f7..42030ab8 100644 --- a/src/data/InstanceDhisRepository.ts +++ b/src/data/InstanceDhisRepository.ts @@ -76,6 +76,7 @@ import { resolveRegistrations, resolveRequestedRegistrationKeys, } from "./Dhis2DataSetCompletion"; +import { chunkDataValuesByPeriod } from "./Dhis2DataValueChunks"; import { Maybe } from "../types/utils"; const bulkOperationChunkSize = 1000; @@ -520,7 +521,7 @@ export class InstanceDhisRepository implements InstanceRepository { }; } - const chunks = _.chunk(dataValues, bulkOperationChunkSize); + const chunks = chunkDataValuesByPeriod(dataValues, bulkOperationChunkSize); const chunkResults = await promiseMap(chunks, async chunk => { const { response } = await this.api.dataValues diff --git a/src/data/__tests__/Dhis2DataValueChunks.spec.ts b/src/data/__tests__/Dhis2DataValueChunks.spec.ts new file mode 100644 index 00000000..a8df4471 --- /dev/null +++ b/src/data/__tests__/Dhis2DataValueChunks.spec.ts @@ -0,0 +1,56 @@ +import { chunkDataValuesByPeriod } from "../Dhis2DataValueChunks"; + +const PERIOD_2024 = "2024"; +const PERIOD_2025 = "2025"; +const PERIOD_2026 = "2026"; + +type TestDataValue = Readonly<{ period: string; orgUnit: string; value: string }>; + +function buildValue(period: string, orgUnit = "ou1", value = "1"): TestDataValue { + return { period, orgUnit, value }; +} + +describe("chunkDataValuesByPeriod", () => { + describe("when the data values have several periods", () => { + it("returns one chunk per period, in the order the periods first appear", () => { + const value2025 = buildValue(PERIOD_2025); + const value2024 = buildValue(PERIOD_2024); + const value2026 = buildValue(PERIOD_2026); + + const chunks = chunkDataValuesByPeriod([value2025, value2024, value2026], 1000); + + expect(chunks).toEqual([[value2025], [value2024], [value2026]]); + }); + + it("keeps the values of several org units of the same period in the same chunk", () => { + const ou1In2024 = buildValue(PERIOD_2024, "ou1"); + const ou2In2024 = buildValue(PERIOD_2024, "ou2"); + const ou1In2025 = buildValue(PERIOD_2025, "ou1"); + const ou2In2025 = buildValue(PERIOD_2025, "ou2"); + + const chunks = chunkDataValuesByPeriod([ou1In2024, ou1In2025, ou2In2024, ou2In2025], 1000); + + expect(chunks).toEqual([ + [ou1In2024, ou2In2024], + [ou1In2025, ou2In2025], + ]); + }); + }); + + describe("when a period has more values than the chunk size", () => { + it("splits that period into chunks of the given size", () => { + const values2024 = ["1", "2", "3"].map(value => buildValue(PERIOD_2024, "ou1", value)); + const value2025 = buildValue(PERIOD_2025); + + const chunks = chunkDataValuesByPeriod([...values2024, value2025], 2); + + expect(chunks).toEqual([[values2024[0], values2024[1]], [values2024[2]], [value2025]]); + }); + }); + + describe("when there are no data values", () => { + it("returns no chunks", () => { + expect(chunkDataValuesByPeriod([], 1000)).toEqual([]); + }); + }); +});