Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "bulk-load",
"description": "Bulk importing made easy",
"version": "3.38.0",
"version": "3.38.1",
"license": "GPL-3.0",
"author": "EyeSeeTea team",
"homepage": ".",
Expand Down
13 changes: 13 additions & 0 deletions src/data/Dhis2DataValueChunks.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import _ from "lodash";

// DHIS2 only accepts the first period of a dataValueSets request when the data set has data input
// periods (https://dhis2.atlassian.net/browse/DHIS2-22277). Once it is fixed upstream, chunk by size only.
export function chunkDataValuesByPeriod<T extends { period: string }>(
dataValues: ReadonlyArray<T>,
chunkSize: number
): T[][] {
const dataValuesByPeriod = _.groupBy(dataValues, dataValue => dataValue.period);
const periods = _.uniq(dataValues.map(dataValue => dataValue.period));

return periods.flatMap(period => _.chunk(dataValuesByPeriod[period] ?? [], chunkSize));
}
3 changes: 2 additions & 1 deletion src/data/InstanceDhisRepository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ import {
resolveRegistrations,
resolveRequestedRegistrationKeys,
} from "./Dhis2DataSetCompletion";
import { chunkDataValuesByPeriod } from "./Dhis2DataValueChunks";
import { Maybe } from "../types/utils";

const bulkOperationChunkSize = 1000;
Expand Down Expand Up @@ -520,7 +521,7 @@ export class InstanceDhisRepository implements InstanceRepository {
};
}

const chunks = _.chunk(dataValues, bulkOperationChunkSize);
const chunks = chunkDataValuesByPeriod(dataValues, bulkOperationChunkSize);

const chunkResults = await promiseMap(chunks, async chunk => {
const { response } = await this.api.dataValues
Expand Down
56 changes: 56 additions & 0 deletions src/data/__tests__/Dhis2DataValueChunks.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import { chunkDataValuesByPeriod } from "../Dhis2DataValueChunks";

const PERIOD_2024 = "2024";
const PERIOD_2025 = "2025";
const PERIOD_2026 = "2026";

type TestDataValue = Readonly<{ period: string; orgUnit: string; value: string }>;

function buildValue(period: string, orgUnit = "ou1", value = "1"): TestDataValue {
return { period, orgUnit, value };
}

describe("chunkDataValuesByPeriod", () => {
describe("when the data values have several periods", () => {
it("returns one chunk per period, in the order the periods first appear", () => {
const value2025 = buildValue(PERIOD_2025);
const value2024 = buildValue(PERIOD_2024);
const value2026 = buildValue(PERIOD_2026);

const chunks = chunkDataValuesByPeriod([value2025, value2024, value2026], 1000);

expect(chunks).toEqual([[value2025], [value2024], [value2026]]);
});

it("keeps the values of several org units of the same period in the same chunk", () => {
const ou1In2024 = buildValue(PERIOD_2024, "ou1");
const ou2In2024 = buildValue(PERIOD_2024, "ou2");
const ou1In2025 = buildValue(PERIOD_2025, "ou1");
const ou2In2025 = buildValue(PERIOD_2025, "ou2");

const chunks = chunkDataValuesByPeriod([ou1In2024, ou1In2025, ou2In2024, ou2In2025], 1000);

expect(chunks).toEqual([
[ou1In2024, ou2In2024],
[ou1In2025, ou2In2025],
]);
});
});

describe("when a period has more values than the chunk size", () => {
it("splits that period into chunks of the given size", () => {
const values2024 = ["1", "2", "3"].map(value => buildValue(PERIOD_2024, "ou1", value));
const value2025 = buildValue(PERIOD_2025);

const chunks = chunkDataValuesByPeriod([...values2024, value2025], 2);

expect(chunks).toEqual([[values2024[0], values2024[1]], [values2024[2]], [value2025]]);
});
});

describe("when there are no data values", () => {
it("returns no chunks", () => {
expect(chunkDataValuesByPeriod([], 1000)).toEqual([]);
});
});
});
Loading