Improve file reading logic for large database files

README.md

@@ -8,6 +8,7 @@
 A C++ local database library with cross language bindings. Aiming to be a fast, lightweight, and easy-to-use data communication solution for RPC and coupled modeling in scientific computing.
 
 ## What's new
+- **2026-03-04 (Release 0.1.12)**: Fixed a critical issue where loading large database files (> 2GB) on Linux/Unix systems would fail to read the complete file, leading to missing tables or data corruption. The file reading logic has been improved to correctly handle partial reads for large files. (PR #23)
 - **2026-03-04 (Memory Overflow Improvement)**: Enhanced the `MemoryStream` implementation to handle large data sizes exceeding 4GB without causing size overflow in `chunk_data_t.size` (u32). This improvement allows for more robust handling of large datasets in memory. (PR #22)
 - **2026-02-28 (Release Improvement)**: Fix bugs related to build process in Windows. (PR #20)
 - **2025-12-31(Bug Fix)**: Fixed an issue where shared memory segments were not being properly unregistered from the resource tracker upon closing, which could lead to resource leaks. (PR #17)

fastcarto/fastdb/src/FastVectorDb.cpp

@@ -146,7 +146,20 @@ printf("loading [%s] ...",filename);
         }
         size_t size = fileStat.st_size;
         void* pdata = malloc(sizeof(u8)*size+64);
-        _read(fd, pdata, size);
+
+        size_t total_read = 0;
+        while(total_read < size) {
+            unsigned int chunk = (size - total_read > 0x40000000) ? 0x40000000 : (unsigned int)(size - total_read);
+            int r = _read(fd, (char*)pdata + total_read, chunk);
+            if (r == -1) {
+                printf("Error reading file: %s\n", strerror(errno));
+                _close(fd);
+                free(pdata);
+                return NULL;
+            }
+            if (r == 0) break;
+            total_read += r;
+        }
         _close(fd);
 #else
         int fd = open(filename, O_RDONLY);
@@ -165,7 +178,20 @@ printf("loading [%s] ...",filename);
         }
         size_t size =  fileStat.st_size;
         void* pdata = malloc(sizeof(u8)*size+64);
-        read(fd,pdata,size);
+
+        size_t total_read = 0;
+        while(total_read < size) {
+            ssize_t r = read(fd, (char*)pdata + total_read, size - total_read);
+            if (r == -1) {
+                if (errno == EINTR) continue;
+                printf("Error reading file: %s\n", strerror(errno));
+                close(fd);
+                free(pdata);
+                return NULL;
+            }
+            if (r == 0) break; // EOF
+            total_read += r;
+        }
         close(fd);
 #endif
         auto db = load(pdata,size,free_data_buffer,0);

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fastdb4py"
-version = "0.1.11"
+version = "0.1.12"
 description = "FastCarto database bindings"
 readme = "README.md"
 requires-python = ">=3.10"