add mtime logic to file difference checking

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## [0.3.2] - 2026-09-10
+
+### Added
+- `FileInfo.mtime` (POSIX timestamp, `None` for directories or when the
+  backend reports none; falls back to size-only for that entry).
+
+### Changed
+- Downloaded files are stamped with the S3 `LastModified` time (rsync `-t`
+  style), so a `sync()` does not re-upload the tree it just downloaded.
+- **Upgrade note:** files downloaded by pos3 0.3.1 or earlier carry their
+  download time as mtime, which is newer than the S3 `LastModified`. The
+  first `upload()` / `sync()` of such a tree after upgrading will therefore
+  re-upload it once; subsequent syncs are incremental again.
+
+### Fixed
+- Change detection now compares modification time in addition to size, so a
+  file rewritten in place with the same byte count (an overwritten
+  checkpoint, a fixed-shape array, a same-length text edit) is transferred
+  instead of being silently skipped. Applies to `download`, `upload`,
+  `sync`, the background interval loop, `plan_*`, and the CLI dry-run.
+  Rule: copy when missing, size differs, or the source is newer than the
+  target by more than 2 s (`_MTIME_TOLERANCE_SECONDS`; S3 `LastModified` is
+  whole-second, so a small margin keeps freshly synced trees stable).
+
 ## [0.3.1] - 2026-05-21
 
 ### Added

README.md

@@ -12,7 +12,7 @@
     - **Enter**: Initializes the sync environment (threads, cache).
     - **Body**: You explicitly call `pos3.download()` to fetch files and `pos3.upload()` to register outputs.
     - **Exit**: Uploads registered output paths (mirroring local to S3).
-- **Lazy & Efficient**: Only transfers files that have changed (based on size/presence).
+- **Lazy & Efficient**: Only transfers files that have changed (based on size/presence/timestamp).
 - **Local Paths**: All API calls return a `pathlib.Path` to the local file/directory. If you pass a local path instead of an S3 URL, it is passed through unchanged (no copy).
 - **Background Sync**: Can optionally upload changes in the background (e.g., every 60s) for long-running jobs.
 

pos3/__init__.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import shutil
 import threading
 import time
@@ -11,6 +12,7 @@
 from contextlib import contextmanager, nullcontext
 from contextvars import ContextVar
 from dataclasses import dataclass, field
+from datetime import datetime
 from functools import wraps
 from pathlib import Path, PurePosixPath
 from typing import Any
@@ -139,6 +141,28 @@ class FileInfo:
     relative_path: str  # Relative path from root (empty string for root file/dir)
     size: int  # File size in bytes, 0 for directories
     is_dir: bool  # True if this represents a directory
+    # Last-modification time as a POSIX timestamp (UTC seconds). ``None`` for
+    # directories and for backends that do not report one; the diff then
+    # falls back to size-only comparison for that entry.
+    mtime: float | None = None
+
+
+# S3 reports ``LastModified`` at whole-second resolution, while local
+# filesystems keep sub-second mtimes. Immediately after an upload the local
+# file can therefore look "newer" than its S3 twin by a fraction of a second.
+# Treat a source as newer only when it is ahead by more than this margin, so
+# a freshly synced tree does not re-transfer on every tick.
+_MTIME_TOLERANCE_SECONDS = 2.0
+
+
+def _s3_mtime(obj: dict) -> float | None:
+    """Extract a POSIX timestamp from an S3 object's ``LastModified``, if present."""
+    last_modified = obj.get("LastModified")
+    if isinstance(last_modified, datetime):
+        return last_modified.timestamp()
+    if isinstance(last_modified, (int, float)):
+        return float(last_modified)
+    return None
 
 
 def _scan_local(path: Path) -> Iterator[FileInfo]:
@@ -159,7 +183,8 @@ def _scan_local(path: Path) -> Iterator[FileInfo]:
             yield FileInfo(relative_path=relative, size=0, is_dir=True)
             stack.extend(p.iterdir())
         else:
-            yield FileInfo(relative_path=relative, size=p.stat().st_size, is_dir=False)
+            st = p.stat()
+            yield FileInfo(relative_path=relative, size=st.st_size, is_dir=False, mtime=st.st_mtime)
 
 
 def _filter_fileinfo(fileinfo_iter: Iterator[FileInfo], exclude: list[str] | None) -> Iterator[FileInfo]:
@@ -192,7 +217,15 @@ def _filter_fileinfo(fileinfo_iter: Iterator[FileInfo], exclude: list[str] | Non
             yield info
 
 
+def _is_newer(source: FileInfo, target: FileInfo) -> bool:
+    """True if ``source`` is newer than ``target`` beyond the tolerance; False if either mtime is unknown."""
+    if source.mtime is None or target.mtime is None:
+        return False
+    return source.mtime > target.mtime + _MTIME_TOLERANCE_SECONDS
+
+
 def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) -> tuple[list[FileInfo], list[FileInfo]]:
+    """Return ``(to_copy, to_delete)``; copy files that are missing, differ in size, or are newer on the source."""
     source_map: dict[str, FileInfo] = {info.relative_path: info for info in source}
     target_map: dict[str, FileInfo] = {info.relative_path: info for info in target}
 
@@ -206,7 +239,11 @@ def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) -
         elif source_info.is_dir != target_info.is_dir:
             to_delete.append(target_info)
             to_copy.append(source_info)
-        elif not source_info.is_dir and source_info.size != target_info.size:
+        elif source_info.is_dir:
+            continue
+        elif source_info.size != target_info.size:
+            to_copy.append(source_info)
+        elif _is_newer(source_info, target_info):
             to_copy.append(source_info)
 
     for relative_path, target_info in target_map.items():
@@ -929,7 +966,7 @@ def _scan_s3(self, bucket: str, prefix: str, profile: Profile | None = None) ->
             else:
                 if relative == "":
                     has_root_file = True
-                yield FileInfo(relative_path=relative, size=obj["Size"], is_dir=False)
+                yield FileInfo(relative_path=relative, size=obj["Size"], is_dir=False, mtime=_s3_mtime(obj))
 
                 if "/" in relative:
                     parts = relative.split("/")
@@ -982,6 +1019,16 @@ def _put_locally(
                 target.parent.mkdir(parents=True, exist_ok=True)
                 client = self._get_client(profile)
                 client.download_file(bucket, key, str(target), Callback=pbar.update)
+                # Stamp the local copy with the S3 LastModified time (rsync -t
+                # style). Without this the fresh download carries "now" as
+                # its mtime, so a following upload of the same tree (sync())
+                # would see every local file as newer and re-upload it all.
+                # With it, the local file is newer only after a real edit.
+                if info.mtime is not None:
+                    try:
+                        os.utime(target, (info.mtime, info.mtime))
+                    except OSError as exc:
+                        logger.warning("Could not set mtime on %s: %s", target, exc)
         except Exception as exc:
             logger.error("Failed to put %s locally: %s", key, exc)
             raise

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pos3"
-version = "0.3.1"
+version = "0.3.2"
 description = "S3 Simple Sync - Make using S3 as simple as using local files"
 readme = "README.md"
 requires-python = ">=3.11"

tests/test_s3.py

@@ -1922,3 +1922,217 @@ def test_url_unknown_profile_hard_error(self, mock_boto_client):
             with s3.mirror(cache_root=tmpdir, show_progress=False):
                 with pytest.raises(ValueError, match="Unknown profile"):
                     s3.download("s3://ghost@bucket/data")
+
+
+class TestMtimeComparison:
+    """Size-plus-mtime change detection.
+
+    Regression coverage for the case size-only comparison cannot see: a
+    file rewritten in place with the same byte count.
+    """
+
+    NOW = 1_700_000_000.0
+    TOL = s3._MTIME_TOLERANCE_SECONDS
+    # An offset that is unambiguously "newer" no matter how the tolerance is tuned.
+    WELL_OVER = 10 * s3._MTIME_TOLERANCE_SECONDS
+
+    def _pair(self, src_mtime, tgt_mtime, size=5):
+        src = [s3.FileInfo("", 0, True), s3.FileInfo("f.bin", size, False, mtime=src_mtime)]
+        tgt = [s3.FileInfo("", 0, True), s3.FileInfo("f.bin", size, False, mtime=tgt_mtime)]
+        return iter(src), iter(tgt)
+
+    def test_same_size_source_newer_is_copied(self):
+        to_copy, to_delete = s3._compute_sync_diff(*self._pair(self.NOW + self.WELL_OVER, self.NOW))
+        assert [i.relative_path for i in to_copy] == ["f.bin"]
+        assert to_delete == []
+
+    def test_same_size_source_older_is_not_copied(self):
+        to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW, self.NOW + self.WELL_OVER))
+        assert to_copy == []
+
+    def test_same_size_within_tolerance_is_not_copied(self):
+        # S3 LastModified is whole-second; a local file uploaded a fraction of
+        # a second after its S3 twin's stamp must not look "newer".
+        to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + 0.7 * self.TOL, self.NOW))
+        assert to_copy == []
+
+    def test_same_size_exactly_at_tolerance_is_not_copied(self):
+        # The comparison is strict: a source ahead by exactly the tolerance is not newer.
+        to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + self.TOL, self.NOW))
+        assert to_copy == []
+
+    def test_same_size_just_over_tolerance_is_copied(self):
+        to_copy, _ = s3._compute_sync_diff(*self._pair(self.NOW + self.TOL + 0.01, self.NOW))
+        assert [i.relative_path for i in to_copy] == ["f.bin"]
+
+    def test_missing_mtime_falls_back_to_size_only(self):
+        assert s3._compute_sync_diff(*self._pair(self.NOW + self.WELL_OVER, None))[0] == []
+        assert s3._compute_sync_diff(*self._pair(None, self.NOW))[0] == []
+        assert s3._compute_sync_diff(*self._pair(None, None))[0] == []
+
+    def test_size_difference_still_wins_regardless_of_mtime(self):
+        src = iter([s3.FileInfo("f.bin", 6, False, mtime=self.NOW)])
+        tgt = iter([s3.FileInfo("f.bin", 5, False, mtime=self.NOW + self.WELL_OVER)])
+        to_copy, _ = s3._compute_sync_diff(src, tgt)
+        assert [i.relative_path for i in to_copy] == ["f.bin"]
+
+    def test_scan_local_reports_mtime(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            f = Path(tmpdir) / "a.txt"
+            f.write_bytes(b"12345")
+            os.utime(f, (self.NOW, self.NOW))
+            infos = {i.relative_path: i for i in s3._scan_local(Path(tmpdir))}
+            assert infos[""].mtime is None  # directories carry no mtime
+            assert infos["a.txt"].mtime == pytest.approx(self.NOW, abs=1.0)
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_scan_s3_parses_last_modified(self, mock_boto_client):
+        from datetime import datetime, timezone
+
+        stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc)
+        paginate = [
+            {
+                "Contents": [
+                    {"Key": "data/with.txt", "Size": 5, "LastModified": stamp},
+                    {"Key": "data/without.txt", "Size": 5},
+                ]
+            }
+        ]
+        _setup_s3_mock(mock_boto_client, paginate)
+        with s3.mirror(show_progress=False):
+            mirror_obj = s3._require_active_mirror()
+            infos = {i.relative_path: i for i in mirror_obj._scan_s3("bucket", "data")}
+        assert infos["with.txt"].mtime == pytest.approx(self.NOW)
+        assert infos["without.txt"].mtime is None
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_upload_same_size_but_newer_local_file(self, mock_boto_client):
+        """The headline bug: an in-place edit that keeps the size must upload."""
+        from datetime import datetime, timezone
+
+        stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc)
+        paginate = [{"Contents": [{"Key": "out/ckpt.bin", "Size": 5, "LastModified": stamp}]}]
+        mock_s3 = _setup_s3_mock(mock_boto_client, paginate)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            local_dir = Path(tmpdir) / "out"
+            local_dir.mkdir()
+            f = local_dir / "ckpt.bin"
+            f.write_bytes(b"NEW!!")  # same size as S3
+            os.utime(f, (self.NOW + self.WELL_OVER, self.NOW + self.WELL_OVER))
+
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                s3.upload("s3://bucket/out", local=local_dir, interval=None, delete=False)
+
+            assert mock_s3.upload_file.call_count == 1
+            assert "ckpt.bin" in str(mock_s3.upload_file.call_args[0][0])
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_upload_same_size_local_not_newer_is_skipped(self, mock_boto_client):
+        from datetime import datetime, timezone
+
+        stamp = datetime.fromtimestamp(self.NOW + self.WELL_OVER, tz=timezone.utc)
+        paginate = [{"Contents": [{"Key": "out/ckpt.bin", "Size": 5, "LastModified": stamp}]}]
+        mock_s3 = _setup_s3_mock(mock_boto_client, paginate)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            local_dir = Path(tmpdir) / "out"
+            local_dir.mkdir()
+            f = local_dir / "ckpt.bin"
+            f.write_bytes(b"12345")
+            os.utime(f, (self.NOW, self.NOW))
+
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                s3.upload("s3://bucket/out", local=local_dir, interval=None, delete=False)
+
+            assert mock_s3.upload_file.call_count == 0
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_download_same_size_but_newer_remote_refreshes_and_stamps_mtime(self, mock_boto_client):
+        from datetime import datetime, timezone
+
+        stamp = datetime.fromtimestamp(self.NOW + self.WELL_OVER, tz=timezone.utc)
+        paginate = [{"Contents": [{"Key": "data/f.txt", "Size": 5, "LastModified": stamp}]}]
+        mock_s3 = _setup_s3_mock(mock_boto_client, paginate)
+
+        def fake_download(bucket, key, filename, Callback=None):
+            Path(filename).write_bytes(b"fresh")
+
+        mock_s3.download_file.side_effect = fake_download
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            local_dir = Path(tmpdir) / "data"
+            local_dir.mkdir()
+            f = local_dir / "f.txt"
+            f.write_bytes(b"stale")  # same size as S3
+            os.utime(f, (self.NOW, self.NOW))
+
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                s3.download("s3://bucket/data", local=local_dir)
+
+            assert mock_s3.download_file.call_count == 1
+            assert f.read_bytes() == b"fresh"
+            # Local copy carries the S3 LastModified, not "now".
+            assert f.stat().st_mtime == pytest.approx(self.NOW + self.WELL_OVER, abs=1.0)
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_download_same_size_remote_not_newer_is_skipped(self, mock_boto_client):
+        from datetime import datetime, timezone
+
+        stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc)
+        paginate = [{"Contents": [{"Key": "data/f.txt", "Size": 5, "LastModified": stamp}]}]
+        mock_s3 = _setup_s3_mock(mock_boto_client, paginate)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            local_dir = Path(tmpdir) / "data"
+            local_dir.mkdir()
+            f = local_dir / "f.txt"
+            f.write_bytes(b"12345")
+            os.utime(f, (self.NOW + self.WELL_OVER, self.NOW + self.WELL_OVER))
+
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                s3.download("s3://bucket/data", local=local_dir)
+
+            assert mock_s3.download_file.call_count == 0
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_sync_does_not_reupload_what_it_just_downloaded(self, mock_boto_client):
+        """Downloaded files inherit the S3 timestamp, so the upload leg is a no-op."""
+        from datetime import datetime, timezone
+
+        stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc)
+        paginate = [{"Contents": [{"Key": "data/f.txt", "Size": 5, "LastModified": stamp}]}]
+        mock_s3 = _setup_s3_mock(mock_boto_client, paginate)
+
+        def fake_download(bucket, key, filename, Callback=None):
+            Path(filename).write_bytes(b"12345")
+
+        mock_s3.download_file.side_effect = fake_download
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            local_dir = Path(tmpdir) / "data"
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                s3.sync("s3://bucket/data", local=local_dir, interval=None)
+
+            assert mock_s3.download_file.call_count == 1
+            assert mock_s3.upload_file.call_count == 0
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_plan_upload_reports_same_size_newer_file(self, mock_boto_client):
+        from datetime import datetime, timezone
+
+        stamp = datetime.fromtimestamp(self.NOW, tz=timezone.utc)
+        paginate = [{"Contents": [{"Key": "out/ckpt.bin", "Size": 5, "LastModified": stamp}]}]
+        _setup_s3_mock(mock_boto_client, paginate)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            local_dir = Path(tmpdir) / "out"
+            local_dir.mkdir()
+            f = local_dir / "ckpt.bin"
+            f.write_bytes(b"NEW!!")
+            os.utime(f, (self.NOW + self.WELL_OVER, self.NOW + self.WELL_OVER))
+
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                plan = s3.plan_upload("s3://bucket/out", local=local_dir)
+
+            assert [dst for _, dst in plan.to_copy] == ["s3://bucket/out/ckpt.bin"]