From d8a498926d5ca6f231889be509d16c5449125e45 Mon Sep 17 00:00:00 2001 From: Mathieu Masson Date: Wed, 13 May 2026 17:05:13 +0200 Subject: [PATCH 1/2] state: Store the packfile size. * Add a new field inside the ET_PACKFILE type to store the packfile size. * This needs a format version bump. --- repository/repositorywriter.go | 4 +-- repository/state/state.go | 45 ++++++++++++++++++++++------------ repository/state/state_test.go | 4 +-- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/repository/repositorywriter.go b/repository/repositorywriter.go index c96f3d8e7..491d43412 100644 --- a/repository/repositorywriter.go +++ b/repository/repositorywriter.go @@ -292,7 +292,7 @@ func (r *RepositoryWriter) PutPackfile(pfile packfile.Packfile) error { db.Commit() - return r.currentDeltaState().PutPackfile(r.currentStateID, mac) + return r.currentDeltaState().PutPackfile(r.currentStateID, mac, uint64(nbytes)) } func (r *RepositoryWriter) PutPtarPackfile(packfile *packer.PackWriter) error { @@ -342,5 +342,5 @@ func (r *RepositoryWriter) PutPtarPackfile(packfile *packer.PackWriter) error { } } - return r.currentDeltaState().PutPackfile(r.currentStateID, mac) + return r.currentDeltaState().PutPackfile(r.currentStateID, mac, uint64(nbytes)) } diff --git a/repository/state/state.go b/repository/state/state.go index 8510b9f88..13547d71c 100644 --- a/repository/state/state.go +++ b/repository/state/state.go @@ -32,7 +32,7 @@ import ( "github.com/vmihailenco/msgpack/v5" ) -const VERSION = "1.1.0" +const VERSION = "1.2.0" func init() { versioning.Register(resources.RT_STATE, versioning.FromString(VERSION)) @@ -92,9 +92,11 @@ type PackfileEntry struct { Packfile objects.MAC StateID objects.MAC Timestamp time.Time + Size uint64 } -const PackfileEntrySerializedSize = 32 + 32 + 8 +const PackfileEntrySerializedSizeV1 = 32 + 32 + 8 +const PackfileEntrySerializedSizeV2 = 32 + 32 + 8 + 8 type ConfigurationEntry struct { Key string @@ -257,10 +259,10 @@ func (ls *LocalState) MergeState(stateID objects.MAC, rd io.Reader, ver versioni // This implicitely sets the parent, see the note about refactoring, and // since we Derive() to construct Delta streams this will set the correct // parent. This is all way too intricated and will be fixed by a refacto. - if ver.Equals(versioning.FromString("1.1.0")) { - err = ls.deserializeFromStream(rd) - } else { + if ver.Equals(versioning.FromString("1.0.0")) { err = ls.deserializeFromStreamv100(rd) + } else { + err = ls.deserializeFromStream(rd) } if err != nil { return err @@ -379,7 +381,7 @@ func (ls *LocalState) SerializeToStream(w io.Writer) error { return fmt.Errorf("failed to write packfile entry type: %w", err) } - if err := writeUint32(PackfileEntrySerializedSize); err != nil { + if err := writeUint32(PackfileEntrySerializedSizeV2); err != nil { return fmt.Errorf("failed to write packfile entry length: %w", err) } @@ -551,6 +553,10 @@ func PackfileEntryFromBytes(buf []byte) (pe PackfileEntry, err error) { timestamp := binary.LittleEndian.Uint64(bbuf.Next(8)) pe.Timestamp = time.Unix(0, int64(timestamp)) + if len(buf) == PackfileEntrySerializedSizeV2 { + pe.Size = binary.LittleEndian.Uint64(bbuf.Next(8)) + } + return } @@ -559,10 +565,12 @@ func (pe *PackfileEntry) _toBytes(buf []byte) { pos += copy(buf[pos:], pe.Packfile[:]) pos += copy(buf[pos:], pe.StateID[:]) binary.LittleEndian.PutUint64(buf[pos:], uint64(pe.Timestamp.UnixNano())) + pos += 8 + binary.LittleEndian.PutUint64(buf[pos:], uint64(pe.Size)) } func (pe *PackfileEntry) ToBytes() (ret []byte) { - ret = make([]byte, PackfileEntrySerializedSize) + ret = make([]byte, PackfileEntrySerializedSizeV2) pe._toBytes(ret) return } @@ -674,7 +682,6 @@ func (ls *LocalState) deserializeFromStream(r io.Reader) error { de_buf := make([]byte, DeltaEntrySerializedSize) del_buf := make([]byte, DeleteEntrySerializedSize) coloured_buf := make([]byte, ColouredEntrySerializedSize) - pe_buf := make([]byte, PackfileEntrySerializedSize) for { n, err := r.Read(et_buf) if err != nil || n != len(et_buf) { @@ -755,10 +762,11 @@ func (ls *LocalState) deserializeFromStream(r io.Reader) error { return err } case ET_PACKFILE: - if length != PackfileEntrySerializedSize { - return fmt.Errorf("failed to read packfile entry wrong length got(%d)/expected(%d)", length, PackfileEntrySerializedSize) + if length != PackfileEntrySerializedSizeV1 && length != PackfileEntrySerializedSizeV2 { + return fmt.Errorf("failed to read packfile entry wrong length got(%d)/expected(%d or %d)", length, PackfileEntrySerializedSizeV1, PackfileEntrySerializedSizeV2) } + pe_buf := make([]byte, length) if n, err := io.ReadFull(r, pe_buf); err != nil { return fmt.Errorf("failed to read packfile entry %w, read(%d)/expected(%d)", err, n, length) } @@ -768,7 +776,9 @@ func (ls *LocalState) deserializeFromStream(r io.Reader) error { return fmt.Errorf("failed to deserialize packfile entry %w", err) } - if err := ls.cache.PutPackfile(pe.Packfile, pe_buf); err != nil { + // We have to re-encode here to force normalize in-db to have the V2 + // version. + if err := ls.cache.PutPackfile(pe.Packfile, pe.ToBytes()); err != nil { return err } @@ -837,7 +847,7 @@ func (ls *LocalState) deserializeFromStreamv100(r io.Reader) error { et_buf := make([]byte, 1) de_buf := make([]byte, DeltaEntrySerializedSize) coloured_buf := make([]byte, ColouredEntrySerializedSize) - pe_buf := make([]byte, PackfileEntrySerializedSize) + pe_buf := make([]byte, PackfileEntrySerializedSizeV1) for { n, err := r.Read(et_buf) if err != nil || n != len(et_buf) { @@ -894,8 +904,8 @@ func (ls *LocalState) deserializeFromStreamv100(r io.Reader) error { return err } case ET_PACKFILE: - if length != PackfileEntrySerializedSize { - return fmt.Errorf("failed to read packfile entry wrong length got(%d)/expected(%d)", length, PackfileEntrySerializedSize) + if length != PackfileEntrySerializedSizeV1 { + return fmt.Errorf("failed to read packfile entry wrong length got(%d)/expected(%d)", length, PackfileEntrySerializedSizeV1) } if n, err := io.ReadFull(r, pe_buf); err != nil { @@ -907,7 +917,9 @@ func (ls *LocalState) deserializeFromStreamv100(r io.Reader) error { return fmt.Errorf("failed to deserialize packfile entry %w", err) } - if err := ls.cache.PutPackfile(pe.Packfile, pe_buf); err != nil { + // We have to re-encode here to force normalize in-db to have the V2 + // version. + if err := ls.cache.PutPackfile(pe.Packfile, pe.ToBytes()); err != nil { return err } @@ -1080,11 +1092,12 @@ func (ls *LocalState) GetSubpartForBlob(Type resources.Type, blobMAC objects.MAC } } -func (ls *LocalState) PutPackfile(stateId, packfile objects.MAC) error { +func (ls *LocalState) PutPackfile(stateId, packfile objects.MAC, size uint64) error { pe := PackfileEntry{ StateID: stateId, Packfile: packfile, Timestamp: time.Now(), + Size: size, } return ls.cache.PutPackfile(pe.Packfile, pe.ToBytes()) diff --git a/repository/state/state_test.go b/repository/state/state_test.go index f24b716a8..d078d5352 100644 --- a/repository/state/state_test.go +++ b/repository/state/state_test.go @@ -475,7 +475,7 @@ func TestSerializeToStream(t *testing.T) { StateID: objects.MAC{17, 18, 19, 20}, Timestamp: time.Now(), } - state.PutPackfile(packfileEntry.StateID, packfileEntry.Packfile) + state.PutPackfile(packfileEntry.StateID, packfileEntry.Packfile, 0) configEntry := &ConfigurationEntry{ Key: "test_key", @@ -531,7 +531,7 @@ func TestPackfileEntrySerialization(t *testing.T) { // Serialize data := original.ToBytes() - require.Len(t, data, PackfileEntrySerializedSize) + require.Len(t, data, PackfileEntrySerializedSizeV2) // Deserialize deserialized, err := PackfileEntryFromBytes(data) From f24fa76f9daad8addbafd8ac59858ed8de336a49 Mon Sep 17 00:00:00 2001 From: Mathieu Masson Date: Wed, 13 May 2026 17:06:00 +0200 Subject: [PATCH 2/2] repository: Change the way we compute repo's size. * Now that we store each packfile sizes in the state, let's use that to compute the size of the repository. * If any of the packfile entry has a 0 size fallback to the old codepath. Let's see if it's worth implementing a self healing solution (it might be a bit heavy as we need to regen a full state...) * Note that this misses the state size, which will be tackled in a subsequent branch as they do not need a state file change. --- repository/repository.go | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/repository/repository.go b/repository/repository.go index 956f8bcef..1790f129b 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -549,14 +549,39 @@ func (r *Repository) Store() storage.Store { func (r *Repository) StorageSize() (int64, error) { if r.storageSizeDirty { - size, err := r.store.Size(r.appContext) - if err != nil { - return 0, err + var totSize uint64 + var old bool + + if r.state != nil { + for p, err := range r.state.ListPackfileEntries() { + if err != nil { + return 0, err + } + + // We have an old repo, fallback to slow method + if p.Size == 0 { + old = true + break + } + + totSize += p.Size + } } - r.storageSize = size - r.storageSizeDirty = false + if old { + size, err := r.store.Size(r.appContext) + if err != nil { + return 0, err + } + + r.storageSize = size + r.storageSizeDirty = false + } else { + r.storageSize = int64(totSize) + r.storageSizeDirty = false + } } + return r.storageSize, nil }