Skip to content

Fix large data downloads #1863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
becky-gilbert merged 12 commits into from
Apr 15, 2026
Merged

Fix large data downloads #1863

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
3b91d94
write all-responses json iteratively to tmp file in psychds download …
becky-gilbert Apr 2, 2026
29f93fa
write psychds zip to temp file instead of bytesIO to reduce peak memo…
becky-gilbert Apr 2, 2026
9535f79
stream the frame data into psychds zip instead of accumulating all re…
becky-gilbert Apr 2, 2026
b47c0e3
write psychds overview CSVs to temp files instead of building strings…
becky-gilbert Apr 2, 2026
16969b9
store variables measured as a set to de-duplicate immediately, instea…
becky-gilbert Apr 2, 2026
d188bac
add tests for psychds zip download contents
becky-gilbert Apr 3, 2026
bf4404d
switch from io.BytesIO to tempfile in the frame data csv/zip download…
becky-gilbert Apr 3, 2026
96080b8
response overview csv: use a named temp file to avoid in-memory strin…
becky-gilbert Apr 3, 2026
a693aa8
same issue with child summary csv: write to temp file to avoid in-mem…
becky-gilbert Apr 3, 2026
baaead4
demographics JSON download: stream content with a json generator inst…
becky-gilbert Apr 3, 2026
2f23af6
demographics CSV: create a temp file and write rows to disk on each i…
becky-gilbert Apr 3, 2026
f895b74
update tests to be able to handle streaming responses
becky-gilbert Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 112 additions & 16 deletions exp/tests/test_response_views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import io
import json
import re
import zipfile

from django.test import Client, TestCase, override_settings
from django.urls import reverse
Expand Down Expand Up @@ -399,6 +400,12 @@ def test_edit_modifiable_fields_in_response(self):


class ResponseDataDownloadTestCase(TestCase):
def _decode_response(self, response):
"""Read content from either a streaming or regular response."""
if hasattr(response, "streaming_content"):
return b"".join(response.streaming_content).decode("utf-8")
return response.content.decode("utf-8")

def setUp(self):
self.client = Force2FAClient()

Expand Down Expand Up @@ -688,7 +695,7 @@ def test_get_appropriate_fields_in_csv_downloads_set1(self):
self.client.force_login(self.study_reader)
query_string = urlencode({"data_options": self.optionset_1}, doseq=True)
response = self.client.get(f"{self.response_summary_url}?{query_string}")
content = response.content.decode("utf-8")
content = self._decode_response(response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_headers = csv_body.pop(0)
Expand Down Expand Up @@ -799,7 +806,7 @@ def test_get_appropriate_fields_in_csv_downloads_set2(self):
self.client.force_login(self.study_reader)
query_string = urlencode({"data_options": self.optionset_2}, doseq=True)
response = self.client.get(f"{self.response_summary_url}?{query_string}")
content = response.content.decode("utf-8")
content = self._decode_response(response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_headers = csv_body.pop(0)
Expand Down Expand Up @@ -932,7 +939,7 @@ def test_get_exit_survey_fields_in_summary_csv(self):
]

csv_response = self.client.get(self.response_summary_url)
content = csv_response.content.decode("utf-8")
content = self._decode_response(csv_response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_headers = csv_body.pop(0)
Expand Down Expand Up @@ -1106,7 +1113,7 @@ def test_get_researcher_editable_fields_in_csv_downloads(self):
self.client.force_login(self.study_reader)
query_string = urlencode({"data_options": self.optionset_1}, doseq=True)
response = self.client.get(f"{self.response_summary_url}?{query_string}")
content = response.content.decode("utf-8")
content = self._decode_response(response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_headers = csv_body.pop(0)
Expand Down Expand Up @@ -1137,7 +1144,7 @@ def test_get_appropriate_children_in_child_csv_as_previewer(self):
"exp:study-responses-children-summary-csv", kwargs={"pk": self.study.pk}
)
)
content = csv_response.content.decode("utf-8")
content = self._decode_response(csv_response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_body.pop(0)
Expand All @@ -1160,7 +1167,7 @@ def test_get_appropriate_children_in_child_csv_as_researcher(self):
"exp:study-responses-children-summary-csv", kwargs={"pk": self.study.pk}
)
)
content = csv_response.content.decode("utf-8")
content = self._decode_response(csv_response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_body.pop(0)
Expand All @@ -1177,7 +1184,7 @@ def test_get_study_demographics_view_as_researcher(self):
response = self.client.get(
reverse("exp:study-demographics", kwargs={"pk": self.study.pk})
)
content = response.content.decode("utf-8")
content = self._decode_response(response)
self.assertIn(
f"{self.n_previews + self.n_responses} snapshot",
content,
Expand All @@ -1190,7 +1197,7 @@ def test_get_study_demographics_view_as_previewer(self):
response = self.client.get(
reverse("exp:study-demographics", kwargs={"pk": self.study.pk})
)
content = response.content.decode("utf-8")
content = self._decode_response(response)
self.assertIn(
f"{self.n_previews} snapshot",
content,
Expand All @@ -1207,7 +1214,7 @@ def test_get_appropriate_participants_in_demographic_csv_as_researcher(self):
csv_response = self.client.get(
reverse("exp:study-demographics-download-csv", kwargs={"pk": self.study.pk})
)
content = csv_response.content.decode("utf-8")
content = self._decode_response(csv_response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_headers = csv_body.pop(0)
Expand All @@ -1234,7 +1241,7 @@ def test_get_appropriate_participants_in_demographic_csv_as_previewer(self):
csv_response = self.client.get(
reverse("exp:study-demographics-download-csv", kwargs={"pk": self.study.pk})
)
content = csv_response.content.decode("utf-8")
content = self._decode_response(csv_response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_headers = csv_body.pop(0)
Expand Down Expand Up @@ -1267,7 +1274,7 @@ def test_get_appropriate_fields_in_demographic_csv(self):
{"demo_options": ["participant__global_id"]}, doseq=True
)
csv_response = self.client.get(f"{demographic_csv_url}?{query_string}")
content = csv_response.content.decode("utf-8")
content = self._decode_response(csv_response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_headers = csv_body.pop(0)
Expand All @@ -1277,7 +1284,7 @@ def test_get_appropriate_fields_in_demographic_csv(self):
# Without participant__global_id selected, should not be in header and data should not be included
query_string = urlencode({"demo_options": []}, doseq=True)
csv_response = self.client.get(f"{demographic_csv_url}?{query_string}")
content = csv_response.content.decode("utf-8")
content = self._decode_response(csv_response)
csv_reader = csv.reader(io.StringIO(content), quoting=csv.QUOTE_ALL)
csv_body = list(csv_reader)
csv_headers = csv_body.pop(0)
Expand All @@ -1295,7 +1302,7 @@ def test_get_appropriate_fields_in_demographic_json(self):
{"demo_options": ["participant__global_id"]}, doseq=True
)
response = self.client.get(f"{demographic_json_url}?{query_string}")
content = response.content.decode("utf-8")
content = self._decode_response(response)
data = json.loads(content)
for demo in data:
self.assertIn("global_id", demo["participant"])
Expand All @@ -1309,7 +1316,7 @@ def test_get_appropriate_fields_in_demographic_json(self):
# Without participant__global_id selected, this info is absent
query_string = urlencode({"demo_options": []}, doseq=True)
response = self.client.get(f"{demographic_json_url}?{query_string}")
content = response.content.decode("utf-8")
content = self._decode_response(response)
data = json.loads(content)
for demo in data:
self.assertNotIn("global_id", demo["participant"])
Expand All @@ -1327,7 +1334,7 @@ def test_get_appropriate_individual_responses_as_researcher(self):
response = self.client.get(
f"{reverse('exp:study-responses-list', kwargs={'pk': self.study.pk})}"
)
content = response.content.decode("utf-8")
content = self._decode_response(response)
matches = re.finditer('data-response-uuid="(.*)"', content)
for m in matches:
n_matches += 1
Expand Down Expand Up @@ -1358,7 +1365,7 @@ def test_get_appropriate_individual_responses_as_previewer(self):
response = self.client.get(
reverse("exp:study-responses-list", kwargs={"pk": self.study.pk})
)
content = response.content.decode("utf-8")
content = self._decode_response(response)

matches = re.finditer('data-response-uuid="(.*)"', content)
n_matches = 0
Expand All @@ -1373,6 +1380,95 @@ def test_get_appropriate_individual_responses_as_previewer(self):
# Assumes n_previews fit on one page
self.assertEqual(n_matches, self.n_previews)

def _get_psychds_zip(self, query_string=""):
Copy link
Copy Markdown
Contributor Author

@becky-gilbert becky-gilbert Apr 3, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bleonar5 I was worried about modifying the content in psychds downloads, so I added some tests (starting here) to check that it's still correct. Can you let me know what you think, and if I should add any others? I want to make sure that I'm not missing any edge cases.

url = reverse(
"exp:study-responses-download-frame-data-zip-psychds",
kwargs={"pk": self.study.pk},
)
response = self.client.get(f"{url}?{query_string}" if query_string else url)
zip_bytes = b"".join(response.streaming_content)
return response, zip_bytes

def test_psychds_download_returns_zip(self):
self.client.force_login(self.study_reader)
response, zip_bytes = self._get_psychds_zip()
self.assertEqual(response.status_code, 200)
self.assertRegex(
response.get("Content-Disposition"),
r'^attachment; filename=".*--psychds\.zip"',
)
# Verify content is a valid zip file
zipfile.ZipFile(io.BytesIO(zip_bytes))

def test_psychds_download_zip_structure(self):
self.client.force_login(self.study_reader)
_, zip_bytes = self._get_psychds_zip()
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
names = zf.namelist()
self.assertIn("dataset_description.json", names)
self.assertIn("README.md", names)
self.assertIn(".psychds-ignore", names)
self.assertTrue(
any(n.startswith("data/framedata-per-response/") for n in names)
)
self.assertTrue(any(n.startswith("data/overview/") for n in names))
self.assertTrue(
any(n.startswith("data/raw/") and n.endswith(".json") for n in names)
)

def test_psychds_download_dataset_description_has_variables_measured(self):
self.client.force_login(self.study_reader)
_, zip_bytes = self._get_psychds_zip()
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
metadata = json.loads(zf.read("dataset_description.json"))
self.assertIn("variableMeasured", metadata)
self.assertIsInstance(metadata["variableMeasured"], list)
self.assertGreater(len(metadata["variableMeasured"]), 0)

def test_psychds_download_frame_data_file_per_response(self):
self.client.force_login(self.study_reader)
_, zip_bytes = self._get_psychds_zip()
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
framedata_csv_files = [
n
for n in zf.namelist()
if n.startswith("data/framedata-per-response/") and n.endswith(".csv")
]
self.assertEqual(
len(framedata_csv_files),
self.n_responses + self.n_previews,
"Expected one frame data CSV file per consented response",
)

def test_psychds_download_all_responses_json_count(self):
self.client.force_login(self.study_reader)
_, zip_bytes = self._get_psychds_zip()
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
all_responses_json = [
n
for n in zf.namelist()
if n.startswith("data/raw/") and n.endswith(".json")
]
self.assertEqual(len(all_responses_json), 1)
all_responses = json.loads(zf.read(all_responses_json[0]))
self.assertEqual(
len(all_responses),
self.n_responses + self.n_previews,
"Unexpected number of responses in all_responses JSON",
)

def test_psychds_download_excludes_unconsented_responses(self):
self.client.force_login(self.study_reader)
_, zip_bytes = self._get_psychds_zip()
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
for name in zf.namelist():
content = zf.read(name).decode("utf-8", errors="replace")
self.assertNotIn(
self.poison_string,
content,
f"Data from unconsented response found in {name}",
)


class ResponseViewResearcherUpdateFieldsTestCase(TestCase):
def setUp(self):
Expand Down
Loading