From 70ba11a506d5a56624ecbe98202063ee8a65a7c3 Mon Sep 17 00:00:00 2001 From: arturmakoev-positronic Date: Wed, 9 Sep 2026 14:22:55 +0200 Subject: [PATCH 1/4] add mtime logic to file difference checking --- CHANGELOG.md | 19 +++++ README.md | 2 +- pos3/__init__.py | 64 ++++++++++++++- tests/test_s3.py | 206 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 287 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eaecfe..1849bf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## [Unreleased] + +### Fixed +- Change detection now compares modification time in addition to size, so a + file rewritten in place with the same byte count (an overwritten + checkpoint, a fixed-shape array, a same-length text edit) is transferred + instead of being silently skipped. Applies to `download`, `upload`, + `sync`, the background interval loop, `plan_*`, and the CLI dry-run. + Rule: copy when missing, size differs, or the source is newer than the + target by more than 1 s (S3 `LastModified` is whole-second). + +### Added +- `FileInfo.mtime` (POSIX timestamp, `None` for directories or when the + backend reports none; falls back to size-only for that entry). + +### Changed +- Downloaded files are stamped with the S3 `LastModified` time (rsync `-t` + style), so a `sync()` does not re-upload the tree it just downloaded. + ## [0.3.1] - 2026-05-21 ### Added diff --git a/README.md b/README.md index e3d5477..0618119 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ - **Enter**: Initializes the sync environment (threads, cache). - **Body**: You explicitly call `pos3.download()` to fetch files and `pos3.upload()` to register outputs. - **Exit**: Uploads registered output paths (mirroring local to S3). -- **Lazy & Efficient**: Only transfers files that have changed (based on size/presence). +- **Lazy & Efficient**: Only transfers files that have changed (based on size/presence/timestamp). - **Local Paths**: All API calls return a `pathlib.Path` to the local file/directory. If you pass a local path instead of an S3 URL, it is passed through unchanged (no copy). - **Background Sync**: Can optionally upload changes in the background (e.g., every 60s) for long-running jobs. diff --git a/pos3/__init__.py b/pos3/__init__.py index 121256b..cfa7dd8 100644 --- a/pos3/__init__.py +++ b/pos3/__init__.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import os import shutil import threading import time @@ -11,6 +12,7 @@ from contextlib import contextmanager, nullcontext from contextvars import ContextVar from dataclasses import dataclass, field +from datetime import datetime from functools import wraps from pathlib import Path, PurePosixPath from typing import Any @@ -139,6 +141,28 @@ class FileInfo: relative_path: str # Relative path from root (empty string for root file/dir) size: int # File size in bytes, 0 for directories is_dir: bool # True if this represents a directory + # Last-modification time as a POSIX timestamp (UTC seconds). ``None`` for + # directories and for backends that do not report one; the diff then + # falls back to size-only comparison for that entry. + mtime: float | None = None + + +# S3 reports ``LastModified`` at whole-second resolution, while local +# filesystems keep sub-second mtimes. Immediately after an upload the local +# file can therefore look "newer" than its S3 twin by a fraction of a second. +# Treat a source as newer only when it is ahead by more than this margin, so +# a freshly synced tree does not re-transfer on every tick. +_MTIME_TOLERANCE_SECONDS = 2.0 + + +def _s3_mtime(obj: dict) -> float | None: + """Extract a POSIX timestamp from an S3 object's ``LastModified``, if present.""" + last_modified = obj.get("LastModified") + if isinstance(last_modified, datetime): + return last_modified.timestamp() + if isinstance(last_modified, (int, float)): + return float(last_modified) + return None def _scan_local(path: Path) -> Iterator[FileInfo]: @@ -159,7 +183,8 @@ def _scan_local(path: Path) -> Iterator[FileInfo]: yield FileInfo(relative_path=relative, size=0, is_dir=True) stack.extend(p.iterdir()) else: - yield FileInfo(relative_path=relative, size=p.stat().st_size, is_dir=False) + st = p.stat() + yield FileInfo(relative_path=relative, size=st.st_size, is_dir=False, mtime=st.st_mtime) def _filter_fileinfo(fileinfo_iter: Iterator[FileInfo], exclude: list[str] | None) -> Iterator[FileInfo]: @@ -192,7 +217,26 @@ def _filter_fileinfo(fileinfo_iter: Iterator[FileInfo], exclude: list[str] | Non yield info +def _is_newer(source: FileInfo, target: FileInfo) -> bool: + """True when ``source`` was modified after ``target`` beyond the S3 rounding tolerance. + + Unknown timestamps on either side never count as "newer" -- the caller + falls back to size comparison in that case. + """ + if source.mtime is None or target.mtime is None: + return False + return source.mtime > target.mtime + _MTIME_TOLERANCE_SECONDS + + def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) -> tuple[list[FileInfo], list[FileInfo]]: + """Return ``(to_copy, to_delete)`` to make ``target`` mirror ``source``. + + A file is copied when it is missing on the target, differs in size, or + when the source's modification time is newer than the target's. The + mtime check is what catches an in-place edit that + leaves the byte count unchanged (a rewritten checkpoint, a fixed-shape + array, a same-length text edit); size alone cannot see it. + """ source_map: dict[str, FileInfo] = {info.relative_path: info for info in source} target_map: dict[str, FileInfo] = {info.relative_path: info for info in target} @@ -206,7 +250,11 @@ def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) - elif source_info.is_dir != target_info.is_dir: to_delete.append(target_info) to_copy.append(source_info) - elif not source_info.is_dir and source_info.size != target_info.size: + elif source_info.is_dir: + continue + elif source_info.size != target_info.size: + to_copy.append(source_info) + elif _is_newer(source_info, target_info): to_copy.append(source_info) for relative_path, target_info in target_map.items(): @@ -929,7 +977,7 @@ def _scan_s3(self, bucket: str, prefix: str, profile: Profile | None = None) -> else: if relative == "": has_root_file = True - yield FileInfo(relative_path=relative, size=obj["Size"], is_dir=False) + yield FileInfo(relative_path=relative, size=obj["Size"], is_dir=False, mtime=_s3_mtime(obj)) if "/" in relative: parts = relative.split("/") @@ -982,6 +1030,16 @@ def _put_locally( target.parent.mkdir(parents=True, exist_ok=True) client = self._get_client(profile) client.download_file(bucket, key, str(target), Callback=pbar.update) + # Stamp the local copy with the S3 LastModified time (rsync -t + # style). Without this the fresh download carries "now" as + # its mtime, so a following upload of the same tree (sync()) + # would see every local file as newer and re-upload it all. + # With it, the local file is newer only after a real edit. + if info.mtime is not None: + try: + os.utime(target, (info.mtime, info.mtime)) + except OSError as exc: + logger.warning("Could not set mtime on %s: %s", target, exc) except Exception as exc: logger.error("Failed to put %s locally: %s", key, exc) raise diff --git a/tests/test_s3.py b/tests/test_s3.py index 10736a8..94c527e 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -1922,3 +1922,209 @@ def test_url_unknown_profile_hard_error(self, mock_boto_client): with s3.mirror(cache_root=tmpdir, show_progress=False): with pytest.raises(ValueError, match="Unknown profile"): s3.download("s3://ghost@bucket/data") + + +class TestMtimeComparison: + """Size-plus-mtime change detection. + + Regression coverage for the case size-only comparison cannot see: a + file rewritten in place with the same byte count. + """ + + NOW = 1_700_000_000.0 + + def _pair(self, src_mtime, tgt_mtime, size=5): + src = [s3.FileInfo("", 0, True), s3.FileInfo("f.bin", size, False, mtime=src_mtime)] + tgt = [s3.FileInfo("", 0, True), s3.FileInfo("f.bin", size, False, mtime=tgt_mtime)] + return iter(src), iter(tgt) + + def test_same_size_source_newer_is_copied(self): + to_copy, to_delete = s3._compute_sync_diff(*self._pair(self.NOW + 10, self.NOW)) + assert [i.relative_path for i in to_copy] == ["f.bin"] + assert to_delete == [] + + def test_same_size_source_older_is_not_copied(self): + to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW, self.NOW + 10)) + assert to_copy == [] + + def test_same_size_within_tolerance_is_not_copied(self): + # S3 LastModified is whole-second; a local file uploaded at x.7s must + # not look "newer" than its S3 twin stamped at x.0s. + to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + 0.7, self.NOW)) + assert to_copy == [] + + def test_same_size_just_over_tolerance_is_copied(self): + to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + 1.01, self.NOW)) + assert [i.relative_path for i in to_copy] == ["f.bin"] + + def test_missing_mtime_falls_back_to_size_only(self): + assert s3._compute_sync_diff(*self._pair(self.NOW + 10, None))[0] == [] + assert s3._compute_sync_diff(*self._pair(None, self.NOW))[0] == [] + assert s3._compute_sync_diff(*self._pair(None, None))[0] == [] + + def test_size_difference_still_wins_regardless_of_mtime(self): + src = iter([s3.FileInfo("f.bin", 6, False, mtime=self.NOW)]) + tgt = iter([s3.FileInfo("f.bin", 5, False, mtime=self.NOW + 100)]) + to_copy, _ = s3._compute_sync_diff(src, tgt) + assert [i.relative_path for i in to_copy] == ["f.bin"] + + def test_scan_local_reports_mtime(self): + with tempfile.TemporaryDirectory() as tmpdir: + f = Path(tmpdir) / "a.txt" + f.write_bytes(b"12345") + os.utime(f, (self.NOW, self.NOW)) + infos = {i.relative_path: i for i in s3._scan_local(Path(tmpdir))} + assert infos[""].mtime is None # directories carry no mtime + assert infos["a.txt"].mtime == pytest.approx(self.NOW, abs=1.0) + + @patch(BOTO3_PATCH_TARGET) + def test_scan_s3_parses_last_modified(self, mock_boto_client): + from datetime import datetime, timezone + + stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc) + paginate = [ + { + "Contents": [ + {"Key": "data/with.txt", "Size": 5, "LastModified": stamp}, + {"Key": "data/without.txt", "Size": 5}, + ] + } + ] + _setup_s3_mock(mock_boto_client, paginate) + with s3.mirror(show_progress=False): + mirror_obj = s3._require_active_mirror() + infos = {i.relative_path: i for i in mirror_obj._scan_s3("bucket", "data")} + assert infos["with.txt"].mtime == pytest.approx(self.NOW) + assert infos["without.txt"].mtime is None + + @patch(BOTO3_PATCH_TARGET) + def test_upload_same_size_but_newer_local_file(self, mock_boto_client): + """The headline bug: an in-place edit that keeps the size must upload.""" + from datetime import datetime, timezone + + stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc) + paginate = [{"Contents": [{"Key": "out/ckpt.bin", "Size": 5, "LastModified": stamp}]}] + mock_s3 = _setup_s3_mock(mock_boto_client, paginate) + + with tempfile.TemporaryDirectory() as tmpdir: + local_dir = Path(tmpdir) / "out" + local_dir.mkdir() + f = local_dir / "ckpt.bin" + f.write_bytes(b"NEW!!") # same size as S3 + os.utime(f, (self.NOW + 60, self.NOW + 60)) + + with s3.mirror(cache_root=tmpdir, show_progress=False): + s3.upload("s3://bucket/out", local=local_dir, interval=None, delete=False) + + assert mock_s3.upload_file.call_count == 1 + assert "ckpt.bin" in str(mock_s3.upload_file.call_args[0][0]) + + @patch(BOTO3_PATCH_TARGET) + def test_upload_same_size_local_not_newer_is_skipped(self, mock_boto_client): + from datetime import datetime, timezone + + stamp = datetime.fromtimestamp(self.NOW + 60, tz=timezone.utc) + paginate = [{"Contents": [{"Key": "out/ckpt.bin", "Size": 5, "LastModified": stamp}]}] + mock_s3 = _setup_s3_mock(mock_boto_client, paginate) + + with tempfile.TemporaryDirectory() as tmpdir: + local_dir = Path(tmpdir) / "out" + local_dir.mkdir() + f = local_dir / "ckpt.bin" + f.write_bytes(b"12345") + os.utime(f, (self.NOW, self.NOW)) + + with s3.mirror(cache_root=tmpdir, show_progress=False): + s3.upload("s3://bucket/out", local=local_dir, interval=None, delete=False) + + assert mock_s3.upload_file.call_count == 0 + + @patch(BOTO3_PATCH_TARGET) + def test_download_same_size_but_newer_remote_refreshes_and_stamps_mtime(self, mock_boto_client): + from datetime import datetime, timezone + + stamp = datetime.fromtimestamp(self.NOW + 60, tz=timezone.utc) + paginate = [{"Contents": [{"Key": "data/f.txt", "Size": 5, "LastModified": stamp}]}] + mock_s3 = _setup_s3_mock(mock_boto_client, paginate) + + def fake_download(bucket, key, filename, Callback=None): + Path(filename).write_bytes(b"fresh") + + mock_s3.download_file.side_effect = fake_download + + with tempfile.TemporaryDirectory() as tmpdir: + local_dir = Path(tmpdir) / "data" + local_dir.mkdir() + f = local_dir / "f.txt" + f.write_bytes(b"stale") # same size as S3 + os.utime(f, (self.NOW, self.NOW)) + + with s3.mirror(cache_root=tmpdir, show_progress=False): + s3.download("s3://bucket/data", local=local_dir) + + assert mock_s3.download_file.call_count == 1 + assert f.read_bytes() == b"fresh" + # Local copy carries the S3 LastModified, not "now". + assert f.stat().st_mtime == pytest.approx(self.NOW + 60, abs=1.0) + + @patch(BOTO3_PATCH_TARGET) + def test_download_same_size_remote_not_newer_is_skipped(self, mock_boto_client): + from datetime import datetime, timezone + + stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc) + paginate = [{"Contents": [{"Key": "data/f.txt", "Size": 5, "LastModified": stamp}]}] + mock_s3 = _setup_s3_mock(mock_boto_client, paginate) + + with tempfile.TemporaryDirectory() as tmpdir: + local_dir = Path(tmpdir) / "data" + local_dir.mkdir() + f = local_dir / "f.txt" + f.write_bytes(b"12345") + os.utime(f, (self.NOW + 60, self.NOW + 60)) + + with s3.mirror(cache_root=tmpdir, show_progress=False): + s3.download("s3://bucket/data", local=local_dir) + + assert mock_s3.download_file.call_count == 0 + + @patch(BOTO3_PATCH_TARGET) + def test_sync_does_not_reupload_what_it_just_downloaded(self, mock_boto_client): + """Downloaded files inherit the S3 timestamp, so the upload leg is a no-op.""" + from datetime import datetime, timezone + + stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc) + paginate = [{"Contents": [{"Key": "data/f.txt", "Size": 5, "LastModified": stamp}]}] + mock_s3 = _setup_s3_mock(mock_boto_client, paginate) + + def fake_download(bucket, key, filename, Callback=None): + Path(filename).write_bytes(b"12345") + + mock_s3.download_file.side_effect = fake_download + + with tempfile.TemporaryDirectory() as tmpdir: + local_dir = Path(tmpdir) / "data" + with s3.mirror(cache_root=tmpdir, show_progress=False): + s3.sync("s3://bucket/data", local=local_dir, interval=None) + + assert mock_s3.download_file.call_count == 1 + assert mock_s3.upload_file.call_count == 0 + + @patch(BOTO3_PATCH_TARGET) + def test_plan_upload_reports_same_size_newer_file(self, mock_boto_client): + from datetime import datetime, timezone + + stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc) + paginate = [{"Contents": [{"Key": "out/ckpt.bin", "Size": 5, "LastModified": stamp}]}] + _setup_s3_mock(mock_boto_client, paginate) + + with tempfile.TemporaryDirectory() as tmpdir: + local_dir = Path(tmpdir) / "out" + local_dir.mkdir() + f = local_dir / "ckpt.bin" + f.write_bytes(b"NEW!!") + os.utime(f, (self.NOW + 60, self.NOW + 60)) + + with s3.mirror(cache_root=tmpdir, show_progress=False): + plan = s3.plan_upload("s3://bucket/out", local=local_dir) + + assert [dst for _, dst in plan.to_copy] == ["s3://bucket/out/ckpt.bin"] From b3d07f44004d75e11f627f25d9aabf417eca1e37 Mon Sep 17 00:00:00 2001 From: arturmakoev-positronic Date: Wed, 9 Sep 2026 15:03:49 +0200 Subject: [PATCH 2/4] tune tests to use time tolerance constant --- tests/test_s3.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/test_s3.py b/tests/test_s3.py index 94c527e..649247a 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -1932,6 +1932,9 @@ class TestMtimeComparison: """ NOW = 1_700_000_000.0 + TOL = s3._MTIME_TOLERANCE_SECONDS + # An offset that is unambiguously "newer" no matter how the tolerance is tuned. + WELL_OVER = 10 * s3._MTIME_TOLERANCE_SECONDS def _pair(self, src_mtime, tgt_mtime, size=5): src = [s3.FileInfo("", 0, True), s3.FileInfo("f.bin", size, False, mtime=src_mtime)] @@ -1939,32 +1942,37 @@ def _pair(self, src_mtime, tgt_mtime, size=5): return iter(src), iter(tgt) def test_same_size_source_newer_is_copied(self): - to_copy, to_delete = s3._compute_sync_diff(*self._pair(self.NOW + 10, self.NOW)) + to_copy, to_delete = s3._compute_sync_diff(*self._pair(self.NOW + self.WELL_OVER, self.NOW)) assert [i.relative_path for i in to_copy] == ["f.bin"] assert to_delete == [] def test_same_size_source_older_is_not_copied(self): - to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW, self.NOW + 10)) + to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW, self.NOW + self.WELL_OVER)) assert to_copy == [] def test_same_size_within_tolerance_is_not_copied(self): - # S3 LastModified is whole-second; a local file uploaded at x.7s must - # not look "newer" than its S3 twin stamped at x.0s. - to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + 0.7, self.NOW)) + # S3 LastModified is whole-second; a local file uploaded a fraction of + # a second after its S3 twin's stamp must not look "newer". + to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + 0.7 * self.TOL, self.NOW)) + assert to_copy == [] + + def test_same_size_exactly_at_tolerance_is_not_copied(self): + # The comparison is strict: a source ahead by exactly the tolerance is not newer. + to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + self.TOL, self.NOW)) assert to_copy == [] def test_same_size_just_over_tolerance_is_copied(self): - to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + 1.01, self.NOW)) + to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + self.TOL + 0.01, self.NOW)) assert [i.relative_path for i in to_copy] == ["f.bin"] def test_missing_mtime_falls_back_to_size_only(self): - assert s3._compute_sync_diff(*self._pair(self.NOW + 10, None))[0] == [] + assert s3._compute_sync_diff(*self._pair(self.NOW + self.WELL_OVER, None))[0] == [] assert s3._compute_sync_diff(*self._pair(None, self.NOW))[0] == [] assert s3._compute_sync_diff(*self._pair(None, None))[0] == [] def test_size_difference_still_wins_regardless_of_mtime(self): src = iter([s3.FileInfo("f.bin", 6, False, mtime=self.NOW)]) - tgt = iter([s3.FileInfo("f.bin", 5, False, mtime=self.NOW + 100)]) + tgt = iter([s3.FileInfo("f.bin", 5, False, mtime=self.NOW + self.WELL_OVER)]) to_copy, _ = s3._compute_sync_diff(src, tgt) assert [i.relative_path for i in to_copy] == ["f.bin"] @@ -2011,7 +2019,7 @@ def test_upload_same_size_but_newer_local_file(self, mock_boto_client): local_dir.mkdir() f = local_dir / "ckpt.bin" f.write_bytes(b"NEW!!") # same size as S3 - os.utime(f, (self.NOW + 60, self.NOW + 60)) + os.utime(f, (self.NOW + self.WELL_OVER, self.NOW + self.WELL_OVER)) with s3.mirror(cache_root=tmpdir, show_progress=False): s3.upload("s3://bucket/out", local=local_dir, interval=None, delete=False) @@ -2023,7 +2031,7 @@ def test_upload_same_size_but_newer_local_file(self, mock_boto_client): def test_upload_same_size_local_not_newer_is_skipped(self, mock_boto_client): from datetime import datetime, timezone - stamp = datetime.fromtimestamp(self.NOW + 60, tz=timezone.utc) + stamp = datetime.fromtimestamp(self.NOW + self.WELL_OVER, tz=timezone.utc) paginate = [{"Contents": [{"Key": "out/ckpt.bin", "Size": 5, "LastModified": stamp}]}] mock_s3 = _setup_s3_mock(mock_boto_client, paginate) @@ -2043,7 +2051,7 @@ def test_upload_same_size_local_not_newer_is_skipped(self, mock_boto_client): def test_download_same_size_but_newer_remote_refreshes_and_stamps_mtime(self, mock_boto_client): from datetime import datetime, timezone - stamp = datetime.fromtimestamp(self.NOW + 60, tz=timezone.utc) + stamp = datetime.fromtimestamp(self.NOW + self.WELL_OVER, tz=timezone.utc) paginate = [{"Contents": [{"Key": "data/f.txt", "Size": 5, "LastModified": stamp}]}] mock_s3 = _setup_s3_mock(mock_boto_client, paginate) @@ -2065,7 +2073,7 @@ def fake_download(bucket, key, filename, Callback=None): assert mock_s3.download_file.call_count == 1 assert f.read_bytes() == b"fresh" # Local copy carries the S3 LastModified, not "now". - assert f.stat().st_mtime == pytest.approx(self.NOW + 60, abs=1.0) + assert f.stat().st_mtime == pytest.approx(self.NOW + self.WELL_OVER, abs=1.0) @patch(BOTO3_PATCH_TARGET) def test_download_same_size_remote_not_newer_is_skipped(self, mock_boto_client): @@ -2080,7 +2088,7 @@ def test_download_same_size_remote_not_newer_is_skipped(self, mock_boto_client): local_dir.mkdir() f = local_dir / "f.txt" f.write_bytes(b"12345") - os.utime(f, (self.NOW + 60, self.NOW + 60)) + os.utime(f, (self.NOW + self.WELL_OVER, self.NOW + self.WELL_OVER)) with s3.mirror(cache_root=tmpdir, show_progress=False): s3.download("s3://bucket/data", local=local_dir) @@ -2122,7 +2130,7 @@ def test_plan_upload_reports_same_size_newer_file(self, mock_boto_client): local_dir.mkdir() f = local_dir / "ckpt.bin" f.write_bytes(b"NEW!!") - os.utime(f, (self.NOW + 60, self.NOW + 60)) + os.utime(f, (self.NOW + self.WELL_OVER, self.NOW + self.WELL_OVER)) with s3.mirror(cache_root=tmpdir, show_progress=False): plan = s3.plan_upload("s3://bucket/out", local=local_dir) From a8dd33542d516ffd1c1414229802db275b36d54d Mon Sep 17 00:00:00 2001 From: arturmakoev-positronic Date: Thu, 10 Sep 2026 15:48:31 +0200 Subject: [PATCH 3/4] Release v0.3.2 Bump version to 0.3.2 in pyproject.toml and uv.lock, and move the mtime change-detection entries from [Unreleased] to [0.3.2]. Also correct the documented tolerance (2 s, matching _MTIME_TOLERANCE_SECONDS, not 1 s) and add an upgrade note: trees downloaded by <= 0.3.1 carry download-time mtimes, so their first upload/sync after upgrading re-uploads once. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 25 +++++++++++++++---------- pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1849bf9..30e153f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,6 @@ # Changelog -## [Unreleased] - -### Fixed -- Change detection now compares modification time in addition to size, so a - file rewritten in place with the same byte count (an overwritten - checkpoint, a fixed-shape array, a same-length text edit) is transferred - instead of being silently skipped. Applies to `download`, `upload`, - `sync`, the background interval loop, `plan_*`, and the CLI dry-run. - Rule: copy when missing, size differs, or the source is newer than the - target by more than 1 s (S3 `LastModified` is whole-second). +## [0.3.2] - 2026-09-10 ### Added - `FileInfo.mtime` (POSIX timestamp, `None` for directories or when the @@ -18,6 +9,20 @@ ### Changed - Downloaded files are stamped with the S3 `LastModified` time (rsync `-t` style), so a `sync()` does not re-upload the tree it just downloaded. +- **Upgrade note:** files downloaded by pos3 0.3.1 or earlier carry their + download time as mtime, which is newer than the S3 `LastModified`. The + first `upload()` / `sync()` of such a tree after upgrading will therefore + re-upload it once; subsequent syncs are incremental again. + +### Fixed +- Change detection now compares modification time in addition to size, so a + file rewritten in place with the same byte count (an overwritten + checkpoint, a fixed-shape array, a same-length text edit) is transferred + instead of being silently skipped. Applies to `download`, `upload`, + `sync`, the background interval loop, `plan_*`, and the CLI dry-run. + Rule: copy when missing, size differs, or the source is newer than the + target by more than 2 s (`_MTIME_TOLERANCE_SECONDS`; S3 `LastModified` is + whole-second, so a small margin keeps freshly synced trees stable). ## [0.3.1] - 2026-05-21 diff --git a/pyproject.toml b/pyproject.toml index 4d6627e..fe0bca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pos3" -version = "0.3.1" +version = "0.3.2" description = "S3 Simple Sync - Make using S3 as simple as using local files" readme = "README.md" requires-python = ">=3.11" diff --git a/uv.lock b/uv.lock index 9a2d586..8394c97 100644 --- a/uv.lock +++ b/uv.lock @@ -223,7 +223,7 @@ wheels = [ [[package]] name = "pos3" -version = "0.3.0" +version = "0.3.2" source = { editable = "." } dependencies = [ { name = "boto3" }, From c34176b2ea1884cff16476b54990e0a7ed1e358b Mon Sep 17 00:00:00 2001 From: arturmakoev-positronic Date: Thu, 10 Sep 2026 16:01:49 +0200 Subject: [PATCH 4/4] fix verbose docstring --- pos3/__init__.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/pos3/__init__.py b/pos3/__init__.py index cfa7dd8..0f0dfab 100644 --- a/pos3/__init__.py +++ b/pos3/__init__.py @@ -218,25 +218,14 @@ def _filter_fileinfo(fileinfo_iter: Iterator[FileInfo], exclude: list[str] | Non def _is_newer(source: FileInfo, target: FileInfo) -> bool: - """True when ``source`` was modified after ``target`` beyond the S3 rounding tolerance. - - Unknown timestamps on either side never count as "newer" -- the caller - falls back to size comparison in that case. - """ + """True if ``source`` is newer than ``target`` beyond the tolerance; False if either mtime is unknown.""" if source.mtime is None or target.mtime is None: return False return source.mtime > target.mtime + _MTIME_TOLERANCE_SECONDS def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) -> tuple[list[FileInfo], list[FileInfo]]: - """Return ``(to_copy, to_delete)`` to make ``target`` mirror ``source``. - - A file is copied when it is missing on the target, differs in size, or - when the source's modification time is newer than the target's. The - mtime check is what catches an in-place edit that - leaves the byte count unchanged (a rewritten checkpoint, a fixed-shape - array, a same-length text edit); size alone cannot see it. - """ + """Return ``(to_copy, to_delete)``; copy files that are missing, differ in size, or are newer on the source.""" source_map: dict[str, FileInfo] = {info.relative_path: info for info in source} target_map: dict[str, FileInfo] = {info.relative_path: info for info in target}