Group coproducts by source dataset, and stream exports as raw bytes

CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## [Unreleased]
 
+### Changed
+- Database exports download as raw bytes instead of base64-encoded JSON,
+  matching how uploads already work — a third less data on the wire and far
+  less memory on both ends for big files. Any export approximation warnings
+  now travel in the `X-Volca-Export-Warnings` response header.
+
+### Fixed
+- A database holding activities with several products can now be exported to
+  ILCD. Each product becomes its own ILCD process, instead of the whole export
+  being refused. This unblocks exporting databases read from SimaPro CSV, where
+  two unrelated processes can share a name and so look like one multi-product
+  activity.
+- Exporting a large database to a zipped format (ILCD, EcoSpold 2) no longer
+  stalls. The time spent packing the archive grew with the square of the number
+  of files, so a full Agribalyse ILCD export — some fifty thousand files —
+  exhausted memory and never returned.
+
 ## [0.9.0] - 2026-07-06
 
 A characterization-accuracy release. EF 3.1 scores on Agribalyse and ecoinvent

app/Main.hs

@@ -31,7 +31,7 @@ import CLI.Types
 import Config (ClassificationPreset, Config (..), DatabaseConfig (..), HostingConfig (..), ServerConfig (..), loadConfig)
 import Control.Concurrent.STM (readTVarIO)
 import Database.Manager (DatabaseManager (..), initDatabaseManager)
-import Network.HTTP.Client (defaultManagerSettings, newManager)
+import Network.HTTP.Client (Manager, defaultManagerSettings, managerResponseTimeout, newManager, responseTimeoutNone)
 import Progress
 
 -- For server mode
@@ -110,26 +110,33 @@ runCLIWithConfig cliConfig cmd cfgFile = do
     dbManager <- initDatabaseManager config (noCache (globalOptions cliConfig)) (Just cfgFile)
     executeCommand cliConfig cmd dbManager
 
+{- | HTTP manager for client-mode commands, with the 30 s default response
+timeout lifted: the server legitimately computes for minutes before the first
+byte of a large database export or batch scoring arrives.
+-}
+newClientManager :: IO Manager
+newClientManager = newManager defaultManagerSettings{managerResponseTimeout = responseTimeoutNone}
+
 -- | Run CLI commands via HTTP against a running server (lightweight, no DB loading)
 runCLIViaAPI :: CLIConfig -> Command -> FilePath -> IO ()
 runCLIViaAPI cliConfig cmd cfgFile = do
     config <- loadConfigOrDie cfgFile
-    mgr <- newManager defaultManagerSettings
+    mgr <- newClientManager
     rc <- resolveRemoteConfig (globalOptions cliConfig) (Just config)
     executeRemoteCommand mgr rc (globalOptions cliConfig) cmd
 
 -- | Run interactive REPL over HTTP (auto-starts server if needed)
 runReplMode :: CLIConfig -> FilePath -> IO ()
 runReplMode cliConfig cfgFile = do
     config <- loadConfigOrDie cfgFile
-    mgr <- newManager defaultManagerSettings
+    mgr <- newClientManager
     rc <- resolveRemoteConfig (globalOptions cliConfig) (Just config)
     runRepl mgr rc (globalOptions cliConfig) cfgFile
 
 -- | Run stop without config — resolveRemoteConfig falls back to env vars / defaults
 runStopWithoutConfig :: CLIConfig -> IO ()
 runStopWithoutConfig cliConfig = do
-    mgr <- newManager defaultManagerSettings
+    mgr <- newClientManager
     rc <- resolveRemoteConfig (globalOptions cliConfig) Nothing
     executeRemoteCommand mgr rc (globalOptions cliConfig) Stop
 

pyvolca/README.md

@@ -437,9 +437,10 @@ validated client-side; an unknown value raises VoLCAError before any
 request. Single-file formats carry their bytes directly; EcoSpold 2 /
 ILCD multi-file trees come back zipped.
 
-The engine returns the payload base64-encoded in the ``data`` field;
-this method base64-decodes it and returns the raw bytes. Raises
-VoLCAError on ``success=false`` or a missing ``data`` field.
+The engine streams the payload as raw bytes. Best-effort approximation
+warnings arrive in the ``X-Volca-Export-Warnings`` response header
+(percent-encoded, newline-joined) and are surfaced through
+:mod:`warnings`. Raises VoLCAError on an HTTP error.
 
 ##### `Client.export_to_file(fmt: str, out_path: str, db_name: str | None = None) -> None`
 

pyvolca/src/volca/client.py

@@ -39,7 +39,8 @@
 
 from __future__ import annotations
 
-import base64
+import urllib.parse
+import warnings
 from enum import Enum
 from pathlib import Path
 from typing import Any
@@ -771,9 +772,10 @@ def export_database(self, fmt: str, db_name: str | None = None) -> bytes:
         request. Single-file formats carry their bytes directly; EcoSpold 2 /
         ILCD multi-file trees come back zipped.
 
-        The engine returns the payload base64-encoded in the ``data`` field;
-        this method base64-decodes it and returns the raw bytes. Raises
-        VoLCAError on ``success=false`` or a missing ``data`` field.
+        The engine streams the payload as raw bytes. Best-effort approximation
+        warnings arrive in the ``X-Volca-Export-Warnings`` response header
+        (percent-encoded, newline-joined) and are surfaced through
+        :mod:`warnings`. Raises VoLCAError on an HTTP error.
         """
         fmt_norm = fmt.strip().lower()
         if fmt_norm not in _EXPORT_FORMATS:
@@ -782,21 +784,20 @@ def export_database(self, fmt: str, db_name: str | None = None) -> bytes:
                 f"(expected {'|'.join(sorted(_EXPORT_FORMATS))})"
             )
         target = self._db(db_name)
-        payload = self._require_success(
-            self._json(
-                self._session.post(
-                    f"{self.base_url}/api/v1/db/{target}/export",
-                    json={"format": fmt_norm},
-                )
-            ),
-            "export_database",
+        resp = self._session.post(
+            f"{self.base_url}/api/v1/db/{target}/export",
+            json={"format": fmt_norm},
         )
-        data = payload.get("data")
-        if data is None:
+        if resp.status_code >= 400:
             raise VoLCAError(
-                "export_database succeeded but the response carried no data field."
+                f"export_database failed (HTTP {resp.status_code}): "
+                f"{resp.text[:500]}"
             )
-        return base64.b64decode(data)
+        header = resp.headers.get("X-Volca-Export-Warnings", "")
+        for line in urllib.parse.unquote(header).split("\n"):
+            if line:
+                warnings.warn(line, stacklevel=2)
+        return resp.content
 
     def export_to_file(
         self, fmt: str, out_path: str, db_name: str | None = None

pyvolca/tests/conftest.py

@@ -145,6 +145,7 @@ def _make_response(json_body: Any, status: int = 200) -> MagicMock:
     r.content = json.dumps(json_body).encode()
     r.text = json.dumps(json_body)
     r.json.return_value = json_body
+    r.headers = {}
     r.history = []
     r.raise_for_status = MagicMock()
     return r

pyvolca/tests/test_db_write.py

@@ -4,13 +4,13 @@
 carry no operationId — they bypass the OpenAPI dispatcher and build their
 URLs directly. They also do not exist in any released engine binary, so these
 tests never touch a live engine: they mock ``Client._session`` and assert on
-the wire shape (URL, JSON body, base64 decoding, format validation, error
+the wire shape (URL, JSON body, raw-bytes handling, format validation, error
 surfacing).
 """
 
 from __future__ import annotations
 
-import base64
+import urllib.parse
 
 import pytest
 
@@ -190,12 +190,22 @@ def test_remove_dependency_url(self, mocked_client):
 # ---------------------------------------------------------------------------
 
 
+def _raw(session, body: bytes, warnings_header: str | None = None) -> None:
+    """Wire the mocked session's POST to return raw export bytes."""
+    from tests.conftest import _make_response
+
+    r = _make_response({})
+    r.content = body
+    if warnings_header is not None:
+        r.headers = {"X-Volca-Export-Warnings": warnings_header}
+    session.post.return_value = r
+
+
 class TestExport:
-    def test_base64_decode_returns_raw_bytes(self, mocked_client):
+    def test_returns_raw_bytes(self, mocked_client):
         client, session = mocked_client
         raw = b"PK\x03\x04 zipped db bytes"
-        _ok(session, {"success": True, "message": "ok",
-                      "data": base64.b64encode(raw).decode()})
+        _raw(session, raw)
         out = client.export_database("ecospold2")
         assert out == raw
         url = session.post.call_args[0][0]
@@ -204,8 +214,7 @@ def test_base64_decode_returns_raw_bytes(self, mocked_client):
 
     def test_format_normalized_before_send(self, mocked_client):
         client, session = mocked_client
-        _ok(session, {"success": True, "message": "ok",
-                      "data": base64.b64encode(b"x").decode()})
+        _raw(session, b"x")
         client.export_database("  SimaPro  ")
         assert session.post.call_args[1]["json"] == {"format": "simapro"}
 
@@ -215,23 +224,31 @@ def test_unknown_format_raises_before_request(self, mocked_client):
             client.export_database("parquet")
         session.post.assert_not_called()
 
-    def test_in_band_failure_raises(self, mocked_client):
+    def test_http_error_raises(self, mocked_client):
         client, session = mocked_client
-        _ok(session, {"success": False, "message": "not loaded", "data": None})
+        from tests.conftest import _make_response
+
+        session.post.return_value = _make_response(
+            {"error": "Database not loaded: testdb"}, status=404
+        )
         with pytest.raises(VoLCAError, match="not loaded"):
             client.export_database("simapro")
 
-    def test_missing_data_raises(self, mocked_client):
+    def test_warnings_header_surfaced(self, mocked_client):
         client, session = mocked_client
-        _ok(session, {"success": True, "message": "ok", "data": None})
-        with pytest.raises(VoLCAError, match="no data field"):
-            client.export_database("simapro")
+        header = urllib.parse.quote("orphan waste in Café crème\nsecond warning")
+        _raw(session, b"x", warnings_header=header)
+        with pytest.warns(UserWarning) as caught:
+            client.export_database("brightway")
+        assert [str(w.message) for w in caught] == [
+            "orphan waste in Café crème",
+            "second warning",
+        ]
 
-    def test_to_file_writes_decoded_bytes(self, mocked_client, tmp_path):
+    def test_to_file_writes_bytes(self, mocked_client, tmp_path):
         client, session = mocked_client
         raw = b"hello bytes"
-        _ok(session, {"success": True, "message": "ok",
-                      "data": base64.b64encode(raw).decode()})
+        _raw(session, raw)
         out = tmp_path / "export.csv"
         client.export_to_file("simapro", str(out))
         assert out.read_bytes() == raw

src/API/DatabaseHandlers.hs

@@ -55,7 +55,6 @@ import Control.Monad (mfilter, void)
 import Control.Monad.Catch (finally)
 import Control.Monad.IO.Class (liftIO)
 import qualified Data.ByteString as BS
-import qualified Data.ByteString.Base64 as B64
 import qualified Data.ByteString.Lazy as BSL
 import Data.List (isPrefixOf)
 import qualified Data.Map as M
@@ -64,6 +63,7 @@ import qualified Data.Text as T
 import qualified Data.Text.Encoding as T
 import qualified Data.Text.IO as T
 import Data.Word (Word64)
+import Network.HTTP.Types.URI (urlEncode)
 import Servant (Header, Headers, ServerError, SourceIO, addHeader, err400, err404, err500, errBody, throwError)
 import qualified Servant.Types.SourceT as S
 import qualified System.Directory
@@ -85,7 +85,6 @@ import API.Types (
     DeleteSelectionRequest (..),
     DeleteSelectionResponse (..),
     ExportRequest (..),
-    ExportResponse (..),
     LoadDatabaseResponse (..),
     RefDataListResponse (..),
     RefDataStatusAPI (..),
@@ -276,21 +275,26 @@ deleteActivitiesHandler dbName req = do
                     ("Deleted " <> T.pack (show deleted) <> " activities from " <> dbName)
                     deleted
 
-{- | Export a loaded database, returning the serialized bytes base64-encoded.
-EcoSpold 2 / ILCD multi-file trees are zipped; single-file formats carry their
-bytes directly. Mirrors the upload endpoint's base64 convention. Failures surface
-as HTTP errors — 400 for an unknown format or data the target format cannot
-represent, 404 for a database that is not loaded — never a 200 with a failure flag.
+{- | Export a loaded database as a raw octet-stream body — the same shape the
+upload endpoint reads, and the only response cheap enough for a multi-hundred-MB
+archive (a base64 JSON envelope costs +33% and four full copies before the
+first byte leaves). EcoSpold 2 / ILCD multi-file trees are zipped; single-file
+formats carry their bytes directly. Best-effort approximation warnings ride the
+@X-Volca-Export-Warnings@ header, percent-encoded because activity names are
+arbitrary Unicode and joined with newlines. Failures surface as HTTP errors —
+400 for an unknown format or data the target format cannot represent, 404 for a
+database that is not loaded — never a 200 with a failure flag.
 -}
-exportDatabaseHandler :: Text -> ExportRequest -> AppM ExportResponse
+exportDatabaseHandler :: Text -> ExportRequest -> AppM (Headers '[Header "X-Volca-Export-Warnings" Text] BinaryContent)
 exportDatabaseHandler dbName req = do
     dbManager <- asks aeDbManager
     fmt <- either (httpErr err400) pure (parseExportFormat (exrFormat req))
     mLoaded <- liftIO (getDatabase dbManager dbName)
     ld <- maybe (httpErr err404 ("Database not loaded: " <> dbName)) pure mLoaded
     (bytes, warnings) <- either (httpErr err400) pure (serializeDatabase fmt (ldDatabase ld))
-    pure (ExportResponse (T.decodeUtf8 (B64.encode (BSL.toStrict bytes))) warnings)
+    pure (addHeader (encodeWarnings warnings) (BinaryContent bytes))
   where
+    encodeWarnings = T.decodeUtf8 . urlEncode False . T.encodeUtf8 . T.intercalate "\n"
     httpErr :: ServerError -> Text -> AppM a
     httpErr status msg = throwError status{errBody = BSL.fromStrict (T.encodeUtf8 msg)}
 

src/API/Routes.hs

@@ -10,7 +10,7 @@ module API.Routes where
 import API.DatabaseHandlers (simpleAction)
 import qualified API.DatabaseHandlers as DBHandlers
 import qualified API.OpenApi
-import API.Types (ActivateResponse (..), ActivityContribution (..), ActivityInfo (..), ActivitySummary (..), Aggregation (..), BatchImpactsEntry (..), BatchImpactsRequest (..), BatchImpactsResponse (..), BinaryContent (..), CharacterizationEntry (..), CharacterizationResult (..), ClassificationEntryInfo (..), ClassificationPresetInfo (..), ClassificationSystem (..), ConsumersResponse (..), ContributingActivitiesResult (..), ContributingFlowsResult (..), CutoffWasteFlow (..), DatabaseListResponse (..), DeleteSelectionRequest (..), DeleteSelectionResponse (..), ExchangeDetail (..), ExportRequest (..), ExportResponse (..), FlowCFEntry (..), FlowCFMapping (..), FlowContributionEntry (..), FlowDetail (..), FlowSearchResult (..), FlowSummary (..), GraphExport (..), InventoryExport (..), LCIABatchResult (..), LCIAResult (..), LoadDatabaseResponse (..), MappingStatus (..), MethodCollectionListResponse (..), MethodCollectionStatusAPI (..), MethodDetail (..), MethodFactorAPI (..), MethodSummary (..), PerturbedEntry (..), RefDataListResponse (..), RelinkRequest (..), RelinkResponse (..), ScoringIndicator (..), SearchResults (..), SensitivityRequest (..), SensitivityResponse (..), SubstitutionRequest (..), SupplyChainResponse (..), SynonymGroupsResponse (..), TreeExport (..), UnmappedFlowAPI (..), UploadChunk (..), UploadResponse (..), apiFlowOfKind)
+import API.Types (ActivateResponse (..), ActivityContribution (..), ActivityInfo (..), ActivitySummary (..), Aggregation (..), BatchImpactsEntry (..), BatchImpactsRequest (..), BatchImpactsResponse (..), BinaryContent (..), CharacterizationEntry (..), CharacterizationResult (..), ClassificationEntryInfo (..), ClassificationPresetInfo (..), ClassificationSystem (..), ConsumersResponse (..), ContributingActivitiesResult (..), ContributingFlowsResult (..), CutoffWasteFlow (..), DatabaseListResponse (..), DeleteSelectionRequest (..), DeleteSelectionResponse (..), ExchangeDetail (..), ExportRequest (..), FlowCFEntry (..), FlowCFMapping (..), FlowContributionEntry (..), FlowDetail (..), FlowSearchResult (..), FlowSummary (..), GraphExport (..), InventoryExport (..), LCIABatchResult (..), LCIAResult (..), LoadDatabaseResponse (..), MappingStatus (..), MethodCollectionListResponse (..), MethodCollectionStatusAPI (..), MethodDetail (..), MethodFactorAPI (..), MethodSummary (..), PerturbedEntry (..), RefDataListResponse (..), RelinkRequest (..), RelinkResponse (..), ScoringIndicator (..), SearchResults (..), SensitivityRequest (..), SensitivityResponse (..), SubstitutionRequest (..), SupplyChainResponse (..), SynonymGroupsResponse (..), TreeExport (..), UnmappedFlowAPI (..), UploadChunk (..), UploadResponse (..), apiFlowOfKind)
 import App.Env (AppEnv (..), AppM, runApp)
 import qualified Config
 import Control.Concurrent.Async (mapConcurrently)
@@ -124,8 +124,9 @@ type LCAAPI =
                 :<|> "db" :> Capture "dbName" Text :> Delete '[JSON] ActivateResponse
                 -- Delete the whole filtered set of activities (selection in JSON body)
                 :<|> "db" :> Capture "dbName" Text :> "delete" :> ReqBody '[JSON] DeleteSelectionRequest :> Post '[JSON] DeleteSelectionResponse
-                -- Export a loaded database to a serialized (base64) file in the requested format
-                :<|> "db" :> Capture "dbName" Text :> "export" :> ReqBody '[JSON] ExportRequest :> Post '[JSON] ExportResponse
+                -- Export a loaded database as raw bytes in the requested format;
+                -- approximation warnings travel percent-encoded in a response header
+                :<|> "db" :> Capture "dbName" Text :> "export" :> ReqBody '[JSON] ExportRequest :> Post '[OctetStream] (Headers '[Header "X-Volca-Export-Warnings" Text] BinaryContent)
                 -- Upload endpoint (streamed octet-stream body; metadata in query params)
                 :<|> "db" :> "upload" :> QueryParam "name" Text :> QueryParam "description" Text :> StreamBody NoFraming OctetStream (SourceIO UploadChunk) :> Post '[JSON] UploadResponse
                 -- Database setup endpoints (for cross-DB linking configuration)

src/API/Types.hs

@@ -797,19 +797,6 @@ newtype ExportRequest = ExportRequest
     deriving (Generic)
     deriving (ToJSON, FromJSON, ToSchema) via (Stripped ExportRequest)
 
-{- | Response for a successful database export: the serialized database,
-base64-encoded (single-file formats carry their bytes directly; EcoSpold 2 /
-ILCD are zipped), mirroring the upload endpoint's base64 convention. Failures are
-HTTP errors (400 bad format / unexportable data, 404 not loaded), never a
-success flag in a 200 body — so the type cannot represent "success with no data".
--}
-data ExportResponse = ExportResponse
-    { exrespData :: Text -- Base64-encoded serialized database
-    , exrespWarnings :: [Text] -- Best-effort approximations; empty on a faithful export
-    }
-    deriving (Generic)
-    deriving (ToJSON, FromJSON, ToSchema) via (Stripped ExportResponse)
-
 -- | Response for database upload
 data UploadResponse = UploadResponse
     { uprSuccess :: Bool

src/BrightwayExcel/Parser.hs

@@ -335,6 +335,7 @@ rawToActivity cfg ra =
             , activityAllocationPercent = Nothing
             , activityAllocationFormula = Nothing
             , activityNativeType = Nothing
+            , activityNativeId = Nothing
             }
 
 {- | Build the reference-product (or coproduct) exchange from a @production@ row,