From 8a3cafad3e15b6e6a6076b936713e9d4e4164d9a Mon Sep 17 00:00:00 2001 From: Andrey Core Date: Thu, 4 Dec 2025 00:11:16 +0300 Subject: [PATCH 1/4] feature(notifier): Add enhanced progress --- Cargo.lock | 779 +++++++++++++++++++-- Cargo.toml | 23 +- src/extracted_image.rs | 136 ++-- src/main.rs | 32 +- src/notifier.rs | 222 +++++- src/processor.rs | 267 ++++++- src/sources/docker.rs | 361 +++++++++- src/sources/nerdctl.rs | 14 +- src/sources/source.rs | 13 +- src/sources/tar.rs | 14 +- src/tar_extractor.rs | 2 +- tests/extracted_image_test.rs | 9 +- tests/integration/common/tar_processing.rs | 11 +- tests/integration/tar/mod.rs | 48 +- 14 files changed, 1724 insertions(+), 207 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15bb1f6..201b2d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,12 +82,29 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "2.10.0" @@ -100,6 +117,24 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + [[package]] name = "cc" version = "1.2.48" @@ -127,6 +162,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-link", ] @@ -162,7 +198,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -179,15 +215,15 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "console" -version = "0.16.1" +version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ "encode_unicode", "libc", "once_cell", "unicode-width", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -246,7 +282,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.111", ] [[package]] @@ -257,7 +293,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -278,7 +314,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -288,7 +324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn", + "syn 2.0.111", ] [[package]] @@ -299,7 +335,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -381,6 +417,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -390,6 +441,107 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "futures_codec" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce54d63f8b0c75023ed920d46fd71d0cbbb830b0ee012726b5b4f506fb6dea5b" +dependencies = [ + "bytes 0.5.6", + "futures", + "memchr", + "pin-project 0.4.30", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -411,7 +563,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -435,6 +587,109 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes 1.11.0", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes 1.11.0", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes 1.11.0", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-openssl" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ee5d7a8f718585d1c3c61dfde28ef5b0bb14734b4db13f5ada856cdc6c612b" +dependencies = [ + "http", + "hyper", + "linked_hash_set", + "once_cell", + "openssl", + "openssl-sys", + "parking_lot", + "tokio", + "tokio-openssl", + "tower-layer", +] + +[[package]] +name = "hyperlocal" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" +dependencies = [ + "futures-util", + "hex", + "hyper", + "pin-project 1.1.10", + "tokio", +] + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -569,14 +824,14 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.18.3" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" dependencies = [ "console", + "number_prefix", "portable-atomic", "unicode-width", - "unit-prefix", "web-time", ] @@ -613,7 +868,7 @@ checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -693,6 +948,21 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linked_hash_set" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "984fb35d06508d1e69fc91050cceba9c0b748f983e6739fa2c7a9237154c52c8" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -705,11 +975,20 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "memchr" @@ -717,6 +996,12 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -727,6 +1012,17 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mio" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -736,6 +1032,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "oci-spec" version = "0.8.3" @@ -755,21 +1057,27 @@ dependencies = [ [[package]] name = "oci2git" -version = "0.2.4" +version = "0.2.3" dependencies = [ "anyhow", + "atty", "chrono", "clap", + "console", "env_logger", "flate2", + "futures-util", "git2", "indicatif", "log", "oci-spec", "serde", "serde_json", + "shiplift", "tar", "tempfile", + "tokio", + "walkdir", ] [[package]] @@ -784,6 +1092,32 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "openssl-probe" version = "0.1.6" @@ -802,12 +1136,87 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef0f924a5ee7ea9cbcea77529dba45f8a9ba9f622419fe3386ca581a3ae9d5a" +dependencies = [ + "pin-project-internal 0.4.30", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal 1.1.10", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851c8d0ce9bebe43790dedfc86614c23494ac9f423dd618d3a61fc693eafe61e" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.32" @@ -857,7 +1266,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -947,6 +1356,21 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.228" @@ -974,7 +1398,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -990,6 +1414,33 @@ dependencies = [ "serde_core", ] +[[package]] +name = "shiplift" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e468265908f45299c26571dad9a2e5cb3656eceb51cd58f1441cf61aa71aad6" +dependencies = [ + "base64", + "byteorder", + "bytes 1.11.0", + "chrono", + "flate2", + "futures-util", + "futures_codec", + "hyper", + "hyper-openssl", + "hyperlocal", + "log", + "mime", + "openssl", + "pin-project 1.1.10", + "serde", + "serde_json", + "tar", + "tokio", + "url", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1002,12 +1453,38 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1035,7 +1512,18 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.111", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] @@ -1057,7 +1545,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1101,7 +1589,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1114,6 +1602,79 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "libc", + "mio", + "pin-project-lite", + "socket2 0.6.1", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "tokio-openssl" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd" +dependencies = [ + "openssl", + "openssl-sys", + "tokio", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -1132,12 +1693,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unit-prefix" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" - [[package]] name = "url" version = "2.5.7" @@ -1168,6 +1723,31 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" @@ -1209,7 +1789,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.111", "wasm-bindgen-shared", ] @@ -1232,6 +1812,37 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -1253,7 +1864,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1264,7 +1875,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1291,13 +1902,31 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets", + "windows-targets 0.53.5", ] [[package]] @@ -1309,6 +1938,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + [[package]] name = "windows-targets" version = "0.53.5" @@ -1316,58 +1961,106 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ "windows-link", - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_aarch64_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + [[package]] name = "windows_i686_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_i686_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "windows_x86_64_msvc" version = "0.53.1" @@ -1415,7 +2108,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", "synstructure", ] @@ -1436,7 +2129,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", "synstructure", ] @@ -1470,5 +2163,5 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] diff --git a/Cargo.toml b/Cargo.toml index 103d130..2c0c2b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oci2git" -version = "0.2.4" +version = "0.2.3" edition = "2021" authors = ["Dmitry Rubinstein "] description = "A tool to convert OCI images to Git repositories" @@ -12,17 +12,6 @@ readme = "README.md" keywords = ["oci", "git", "container", "docker", "image"] categories = ["command-line-utilities"] -exclude = [ - ".github", - ".claude", - "assets/*", - "tests/*", - ".gitignore", - "CLAUDE.md", - "CONTRIBUTING.md", - "CODE_OF_CONDUCT.md", -] - [dependencies] clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } @@ -30,13 +19,19 @@ serde_json = "1.0" anyhow = "1.0" tempfile = "3.20" flate2 = "1.0" +walkdir = "2.5" git2 = "0.20" chrono = "0.4" oci-spec = { version = "0.8.1", features = ["image"] } -indicatif = "0.18" +indicatif = "0.17" log = "0.4" env_logger = "0.11" -tar-rs = { package = "tar", version = "0.4" } +console = "0.15.11" +futures-util = "0.3.31" +tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread"] } +atty = "0.2.14" +tar = "0.4.44" +shiplift = "0.7.0" [features] # default = ["nerdctl", "docker"] diff --git a/src/extracted_image.rs b/src/extracted_image.rs index 21f4117..f86cf6a 100644 --- a/src/extracted_image.rs +++ b/src/extracted_image.rs @@ -30,12 +30,14 @@ //! Temporary extraction is scoped to the instance lifetime via `tempfile::TempDir`. use crate::metadata::{self, ImageMetadata}; -use crate::notifier::Notifier; -use crate::tar_extractor; +use crate::notifier::AnyNotifier; use anyhow::{anyhow, Context, Result}; use chrono::{DateTime, Utc}; -use std::fs; +use std::fs::{self, File}; +use std::io::{BufReader, Read}; use std::path::{Path, PathBuf}; +use std::process::Command; +use tar::Archive; #[derive(Debug, Clone)] pub struct Layer { @@ -43,7 +45,7 @@ pub struct Layer { pub command: String, pub created_at: DateTime, pub is_empty: bool, - pub tarball_path: Option, // Some for non-empty layers, None for empty layers + pub tarball_path: Option, // Some for non-empty layers, None for empty layers pub digest: String, // Always present - either tarball digest or "empty" for empty layers pub comment: Option, // Comment from image layer history } @@ -56,34 +58,93 @@ pub struct ExtractedImage { } impl ExtractedImage { - pub fn from_tarball>(tarball_path: P, notifier: &Notifier) -> Result { - let tarball_path = tarball_path.as_ref(); + // Quiet helper used by from_tarball (no bar unless you pass Some(&pb)) + fn extract_tar_file(tar_path: &Path, extract_dir: &Path) -> Result<()> { + // Try to detect if the file is gzip compressed by checking the magic bytes + let mut file_for_detection = File::open(tar_path)?; + let mut magic_bytes = [0u8; 2]; + file_for_detection.read_exact(&mut magic_bytes)?; + + let mut cmd = Command::new("tar"); + + if magic_bytes == [0x1f, 0x8b] { + // This is a gzip file + cmd.arg("-xzf"); + } else { + // This is a plain tar file + cmd.arg("-xf"); + } + + cmd.arg(tar_path).arg("-C").arg(extract_dir); + + let output = cmd.output().context(format!( + "Failed to run tar command for file: {:?}", + tar_path + ))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(anyhow::anyhow!( + "Failed to extract tar file {:?}: {}", + tar_path, + stderr + )); + } + + Ok(()) + } + + pub fn unpack_tar_with_progress( + tar_path: &Path, + out: &Path, + notifier: &AnyNotifier, + ) -> anyhow::Result<()> { + let file = File::open(tar_path)?; + let total = file.metadata()?.len(); + + let pb = notifier.create_progress_bar(total, &format!("Image {}", tar_path.display())); + // // wrap the reader so reads advance the bar + let reader = BufReader::new(file); + let reader: Box = if let Some(ref pb) = pb { + Box::new(pb.wrap_read(reader)) + } else { + Box::new(reader) + }; + + let mut ar = Archive::new(reader); + ar.unpack(out)?; + + if let Some(pb) = pb { + pb.finish_and_clear(); + } - notifier.debug(&format!("Extracting image tarball: {tarball_path:?}")); + Ok(()) + } + + pub fn from_tarball>(tarball_path: P, notifier: &AnyNotifier) -> Result { + let tarball_path = tarball_path.as_ref(); - // Create a temporary directory for extraction let temp_dir = tempfile::tempdir().context("Failed to create temporary directory")?; let extract_dir = temp_dir.path().join("extracted"); fs::create_dir_all(&extract_dir)?; - // Extract the tarball - Self::extract_tar_file(tarball_path, &extract_dir)?; + match notifier { + AnyNotifier::Enhanced(_) => { + Self::unpack_tar_with_progress(tarball_path, &extract_dir, notifier)?; + } + AnyNotifier::Simple(_) => { + Self::unpack_tar_with_progress(tarball_path, &extract_dir, notifier)?; + } + } - // Verify the extracted content has the expected OCI structure let manifest_path = extract_dir.join("manifest.json"); if !manifest_path.exists() { - return Err(anyhow!( - "Invalid image tarball: manifest.json not found. This does not appear to be a valid OCI/Docker image tarball." - )); + return Err(anyhow!("Invalid image tarball: manifest.json not found.")); } - - // Load metadata and layers using static helper methods notifier.debug("Loading image metadata..."); let metadata = Self::load_metadata_from_dir(&extract_dir, "temp")?; - notifier.debug("Loading image layers..."); let layers = Self::load_layers_from_dir(&extract_dir)?; - notifier.info(&format!("Successfully loaded {} layers", layers.len())); Ok(ExtractedImage { @@ -94,6 +155,16 @@ impl ExtractedImage { }) } + pub fn extract_layer_to>( + &self, + layer_tarball: &Path, + output_dir: P, + ) -> Result<()> { + let output_dir = output_dir.as_ref(); + fs::create_dir_all(output_dir)?; + + Self::extract_tar_file(layer_tarball, output_dir) + } pub fn metadata(&self, _image_name: &str) -> Result { // Return the metadata as-is, keeping the proper SHA digest as ID Ok(self.metadata.clone()) @@ -111,25 +182,10 @@ impl ExtractedImage { Ok(self.layers.clone()) } - pub fn extract_layer_to>( - &self, - layer_tarball: &Path, - output_dir: P, - ) -> Result<()> { - let output_dir = output_dir.as_ref(); - fs::create_dir_all(output_dir)?; - Self::extract_tar_file(layer_tarball, output_dir) - } - pub fn extract_dir(&self) -> &Path { &self.extract_dir } - fn extract_tar_file(tar_path: &Path, extract_dir: &Path) -> Result<()> { - tar_extractor::extract_tar(tar_path, extract_dir) - .context(format!("Failed to extract tar file: {tar_path:?}")) - } - fn load_metadata_from_dir(extract_dir: &Path, image_name: &str) -> Result { // Parse the manifest to get the config file path let manifest_path = extract_dir.join("manifest.json"); @@ -151,7 +207,7 @@ impl ExtractedImage { // Read the config file as JSON let config_path = extract_dir.join(config_file); let config_content = fs::read_to_string(&config_path) - .context(format!("Failed to read config file: {config_file}"))?; + .context(format!("Failed to read config file: {}", config_file))?; // Parse as OCI ImageConfiguration let config: oci_spec::image::ImageConfiguration = @@ -180,9 +236,9 @@ impl ExtractedImage { // Fallback: Extract digest from config file path (format: blobs/sha256/HASH) if metadata.id.is_empty() { if let Some(digest_hash) = config_file.strip_prefix("blobs/sha256/") { - metadata.id = format!("sha256:{digest_hash}"); + metadata.id = format!("sha256:{}", digest_hash); } else if let Some(digest_hash) = config_file.strip_suffix(".json") { - metadata.id = format!("sha256:{digest_hash}"); + metadata.id = format!("sha256:{}", digest_hash); } } @@ -199,7 +255,7 @@ impl ExtractedImage { let path = PathBuf::from(image_name); if let Some(filename) = path.file_stem() { if let Some(name) = filename.to_str() { - metadata.repo_tags.push(format!("{name}:latest")); + metadata.repo_tags.push(format!("{}:latest", name)); } } } @@ -228,7 +284,7 @@ impl ExtractedImage { // Read the config file as JSON let config_path = extract_dir.join(config_file); let config_content = fs::read_to_string(&config_path) - .context(format!("Failed to read config file: {config_file}"))?; + .context(format!("Failed to read config file: {}", config_file))?; let config: serde_json::Value = serde_json::from_str(&config_content).context("Failed to parse image configuration")?; @@ -302,7 +358,7 @@ impl ExtractedImage { let id = tarball .file_name() .map(|name| name.to_string_lossy().to_string()) - .unwrap_or_else(|| format!("layer-{i}")); + .unwrap_or_else(|| format!("layer-{}", i)); // Extract digest from tarball path let digest = @@ -311,7 +367,7 @@ impl ExtractedImage { (id, Some(tarball.clone()), digest) } else { // Empty layer or no tarball available - let id = format!(""); + let id = format!("", i); let digest = if is_empty { "empty".to_string() } else { diff --git a/src/main.rs b/src/main.rs index c66ea95..b7ea904 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,10 @@ +use crate::ProgressStyle::{Enhanced, Simple}; use anyhow::{anyhow, Result}; use clap::{Parser, ValueEnum}; +use oci2git::notifier::{AnyNotifier, NotifierFlavor}; +use oci2git::{DockerSource, ImageProcessor, NerdctlSource, TarSource}; use std::path::PathBuf; -use oci2git::{DockerSource, ImageProcessor, NerdctlSource, Notifier, TarSource}; - #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] enum Engine { Docker, @@ -11,6 +12,12 @@ enum Engine { Tar, } +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +enum ProgressStyle { + Simple, + Enhanced, +} + #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Cli { @@ -36,6 +43,15 @@ struct Cli { )] engine: Engine, + #[arg( + short, + long, + value_enum, + default_value = "enhanced", + help = "Progress style mode (default is enhanced, use `simple` for returning to simple mode)" + )] + progress: ProgressStyle, + #[arg( short, long, @@ -47,16 +63,16 @@ struct Cli { fn main() -> Result<()> { let cli = Cli::parse(); + let notifier_flavor = match cli.progress { + Simple => NotifierFlavor::Simple, + Enhanced => NotifierFlavor::Enhanced, + }; - // Create notifier with verbosity level - let notifier = Notifier::new(cli.verbose); + let notifier = AnyNotifier::new(notifier_flavor, cli.verbose); + notifier.info(&format!("Progress style: {:?}", cli.progress)); notifier.debug(&format!("Output directory: {}", cli.output.display())); notifier.debug(&format!("Engine: {:?}", cli.engine)); - notifier.debug(&format!( - "Beautiful progress: {}", - notifier.use_beautiful_progress() - )); match cli.engine { Engine::Docker => { diff --git a/src/notifier.rs b/src/notifier.rs index 86b8bdf..2bd06a8 100644 --- a/src/notifier.rs +++ b/src/notifier.rs @@ -1,27 +1,29 @@ //! Unified logging and progress UI. //! -//! [`Notifier`] wraps `env_logger` (text logs) and `indicatif` (spinners/bars) under a single +//! [`SimpleNotifier`] wraps `env_logger` (text logs) and `indicatif` (spinners/bars) under a single //! verbosity switch: -//! - [`VerbosityLevel::Quiet`] → no text logs; shows a live spinner and optional progress bars. +//! - [`VerbosityLevel::Quiet`] → no text logs; shows a live spinner_style and optional progress bars. //! - [`VerbosityLevel::Info`]/[`VerbosityLevel::Debug`]/[`VerbosityLevel::Trace`] → standard logs. //! //! What you get: -//! - [`Notifier::info`]/[`Notifier::debug`]/[`Notifier::warn`]/[`Notifier::trace`] — emit logs -//! (or update the Quiet-mode spinner message for `info`). -//! - [`Notifier::create_progress_bar`] — add a pretty progress bar (Quiet mode only). -//! - [`Notifier::progress`] — periodic textual progress for non-Quiet modes. -//! - [`Notifier::use_beautiful_progress`] — check if UI bars are active. -//! - [`Notifier::verbosity_level`] — read the current level. +//! - [`SimpleNotifier::info`]/[`SimpleNotifier::debug`]/[`SimpleNotifier::warn`]/[`SimpleNotifier::trace`] — emit logs +//! (or update the Quiet-mode spinner_style message for `info`). +//! - [`SimpleNotifier::create_progress_bar`] — add a pretty progress bar (Quiet mode only). +//! - [`SimpleNotifier::progress`] — periodic textual progress for non-Quiet modes. +//! - [`SimpleNotifier::use_beautiful_progress`] — check if UI bars are active. +//! - [`SimpleNotifier::verbosity_level`] — read the current level. //! //! Levels map to `env_logger` filters; Quiet suppresses logs (≥ Warn) while rendering //! spinners/bars via an internal `MultiProgress`. +use atty::Stream; use env_logger::Env; -use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; use log::{Level, LevelFilter, Log, Record}; use std::cell::RefCell; use std::sync::Arc; use std::time::Duration; +use std::{collections::HashMap, sync::Mutex}; #[derive(Debug, Clone, Copy, PartialEq)] pub enum VerbosityLevel { @@ -52,15 +54,98 @@ impl VerbosityLevel { } } } +#[derive(Clone, Copy)] +pub enum NotifierFlavor { + Simple, + Enhanced, +} + +pub enum AnyNotifier { + Simple(SimpleNotifier), + Enhanced(EnhancedNotifier), +} -pub struct Notifier { +pub struct SimpleNotifier { verbosity: VerbosityLevel, logger: env_logger::Logger, multi_progress: Option>, active_spinner: RefCell>, } +pub struct EnhancedNotifier { + verbosity: VerbosityLevel, + logger: env_logger::Logger, + multi_progress: MultiProgress, + bars: Mutex>, +} +pub trait Notifier: Send + Sync { + fn info(&self, msg: &str); + fn debug(&self, msg: &str); + fn warn(&self, msg: &str); + fn error(&self, msg: &str); +} + +impl AnyNotifier { + pub fn new(flavor: NotifierFlavor, verbosity: u8) -> Self { + match flavor { + NotifierFlavor::Simple => AnyNotifier::Simple(SimpleNotifier::new(verbosity)), + NotifierFlavor::Enhanced => AnyNotifier::Enhanced(EnhancedNotifier::new(verbosity)), + } + } -impl Notifier { + pub fn finish_spinner(&self) { + if let AnyNotifier::Simple(n) = self { + n.finish_spinner(); + } + } + pub fn create_progress_bar(&self, length: u64, name: &str) -> Option { + match self { + // Enhanced expects (name, Option): use `message` as the name/prefix + AnyNotifier::Simple(n) => n.create_progress_bar(length, name), + AnyNotifier::Enhanced(n) => n.create_progress_bar_enhanced(length, name), + } + } + pub fn progress(&self, current: u64, total: u64, message: &str) { + match self { + AnyNotifier::Simple(n) => n.progress(current, total, message), + AnyNotifier::Enhanced(_) => {} + } + } + pub fn info(&self, msg: &str) { + match self { + AnyNotifier::Simple(n) => n.info(msg), + AnyNotifier::Enhanced(n) => n.info(msg), + } + } + pub fn debug(&self, msg: &str) { + match self { + AnyNotifier::Simple(n) => n.debug(msg), + AnyNotifier::Enhanced(n) => n.debug(msg), + } + } + + pub fn println_above(&self, msg: String) { + match self { + AnyNotifier::Enhanced(n) => n.println_above(msg), + AnyNotifier::Simple(_) => {} + } + } + + pub fn warn(&self, msg: &str) { + match self { + AnyNotifier::Simple(n) => n.warn(msg), + AnyNotifier::Enhanced(n) => n.warn(msg), + } + } + + pub fn error(&self, msg: &str) { + match self { + AnyNotifier::Simple(_) => {} + AnyNotifier::Enhanced(n) => n.error(msg), + } + } +} + +impl SimpleNotifier { pub fn new(verbosity_level: u8) -> Self { let verbosity = VerbosityLevel::from(verbosity_level); @@ -83,10 +168,16 @@ impl Notifier { } } + pub fn finish_spinner(&self) { + if let Some(spinner) = self.active_spinner.borrow_mut().take() { + spinner.finish_and_clear(); + } + } + pub fn info(&self, message: &str) { match self.verbosity { VerbosityLevel::Quiet => { - // Lazy initialize spinner on first info call + // Lazy initialize spinner_style on first info call if self.active_spinner.borrow().is_none() { if let Some(multi_progress) = &self.multi_progress { let spinner_style = ProgressStyle::default_spinner() @@ -101,7 +192,7 @@ impl Notifier { } } - // Update spinner message + // Update spinner_style message if let Some(spinner) = self.active_spinner.borrow().as_ref() { spinner.set_message(message.to_string()); } @@ -159,7 +250,7 @@ impl Notifier { if let Some(multi_progress) = &self.multi_progress { let progress_style = ProgressStyle::default_bar() .template( - "{spinner:.green} [{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + "{spinner:.green} [{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg} [{eta}]", ) .unwrap() .progress_chars("=> "); @@ -187,3 +278,106 @@ impl Notifier { self.verbosity } } + +impl EnhancedNotifier { + pub fn new(verbosity_level: u8) -> Self { + let verbosity = VerbosityLevel::from(verbosity_level); + let logger = env_logger::Builder::from_env(Env::default()) + .filter_level(verbosity.to_log_level()) + .build(); + let mp = if atty::is(Stream::Stderr) { + MultiProgress::with_draw_target(ProgressDrawTarget::stderr_with_hz(15)) + } else { + MultiProgress::with_draw_target(ProgressDrawTarget::hidden()) + }; + + Self { + verbosity, + logger, + multi_progress: mp, + bars: Mutex::new(HashMap::new()), + } + } + + pub fn finish_progress_bar(&self, human_label: &str) { + let key = format!("files:{}", human_label); + let mut map = self.bars.lock().unwrap(); + if let Some(pb) = map.remove(&key) { + pb.finish_and_clear(); + } + } + + pub fn create_progress_bar_enhanced(&self, length: u64, message: &str) -> Option { + if self.verbosity == VerbosityLevel::Quiet { + let multi_progress = &self.multi_progress; + let progress_style = ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {prefix} [{eta}]") + .unwrap() + .progress_chars("=> "); + + let progress_bar = multi_progress.add(ProgressBar::new(length)); + progress_bar.set_style(progress_style); + progress_bar.set_prefix(message.to_string()); // label goes in {prefix} + progress_bar.set_message(""); // keep {msg} free/empty + progress_bar.set_draw_target(ProgressDrawTarget::stderr_with_hz(30)); + return Some(progress_bar); + } + None + } + + fn level_enabled(&self, level: LevelFilter) -> bool { + // LevelFilter can be compared to Level directly (Info <= Debug, etc.) + // so we can reuse your VerbosityLevel mapping. + self.verbosity.to_log_level() >= level + } + pub fn println_above(&self, msg: impl AsRef) { + // prints a line above all bars without creating a new bar + println!("{}", msg.as_ref()) + } + + pub fn shutdown(&self) { + let mut map = self.bars.lock().unwrap(); + for pb in map.values() { + if !pb.is_finished() { + pb.finish_and_clear(); + } + } + self.multi_progress + .set_draw_target(ProgressDrawTarget::hidden()); + map.clear(); + } + + pub fn suspend R, R>(&self, f: F) -> R { + // temporarily suspends drawing, runs `f`, then resumes + self.multi_progress.suspend(f) + } +} + +impl Notifier for EnhancedNotifier { + fn info(&self, msg: &str) { + // show info only when enabled (i.e., -v or higher) + if self.level_enabled(LevelFilter::Info) { + // use MultiProgress::println so bars don’t jump + self.multi_progress.println(msg).ok(); + } + } + + fn debug(&self, msg: &str) { + // debug only when -vv or higher + if self.level_enabled(LevelFilter::Debug) { + self.logger.log( + &Record::builder() + .args(format_args!("{}", msg)) + .level(Level::Debug) + .target(module_path!()) + .build(), + ); + } + } + fn warn(&self, msg: &str) { + self.multi_progress.println(format!("⚠️ {msg}")).ok(); + } + fn error(&self, msg: &str) { + self.multi_progress.println(format!("❌ {msg}")).ok(); + } +} diff --git a/src/processor.rs b/src/processor.rs index 364e161..80a3772 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -16,12 +16,17 @@ use crate::digest_tracker::DigestTracker; use crate::extracted_image::ExtractedImage; use crate::git::GitRepo; use crate::image_metadata::ImageMetadata; -use crate::notifier::Notifier; +use crate::notifier::AnyNotifier; use crate::sources::Source; use crate::successor_navigator::SuccessorNavigator; use anyhow::{Context, Result}; +use console::{style, Emoji}; +use indicatif::{HumanDuration, ProgressBar, ProgressStyle}; use std::fs; use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use walkdir; /// Orchestrates the OCI image to Git repo conversion pipeline for a concrete [`Source`]. /// @@ -40,9 +45,12 @@ use std::path::Path; pub struct ImageProcessor { /// The concrete image source (registry/daemon/nerdctl/tar, etc.). source: S, - notifier: Notifier, + notifier: AnyNotifier, } +static SPARK: Emoji<'_, '_> = Emoji("✨ ", ":-)"); +static EXTRACT: Emoji<'_, '_> = Emoji("🧱 ", ":-)"); + impl ImageProcessor { /// Constructs a new processor that will use the given [`Source`] and [`Notifier`]. /// @@ -51,7 +59,7 @@ impl ImageProcessor { /// /// Check [`crate::notifier::VerbosityLevel`] for more verbosity levels params /// - pub fn new(source: S, notifier: Notifier) -> Self { + pub fn new(source: S, notifier: AnyNotifier) -> Self { Self { source, notifier } } /// Convert an image into a Git repository at `output_dir`. @@ -96,18 +104,20 @@ impl ImageProcessor { /// ### Examples /// ```no_run /// use std::path::Path; - /// use oci2git::{DockerSource, ImageProcessor, Notifier}; + /// use oci2git::notifier::{AnyNotifier, NotifierFlavor}; + /// use oci2git::{DockerSource, ImageProcessor }; /// /// // Choose your source (e.g., Docker daemon/registry, nerdctl, tar file, etc.) /// // let src = DockerSource; // or TarSource::new("image.tar")? /// let src = DockerSource; - /// let notifier = Notifier::new(1); + /// let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); /// /// let p = ImageProcessor::new(src, notifier); /// p.convert("ubuntu:latest", Path::new("./ubuntu-image-repo"))?; /// # anyhow::Ok(()) /// ``` pub fn convert(&self, image_name: &str, output_dir: &Path) -> Result<()> { + let started = Instant::now(); self.notifier.info(&format!( "Starting conversion of image with {} source: {}", self.source.name(), @@ -136,6 +146,12 @@ impl ImageProcessor { // Extract the tarball and create ExtractedImage self.notifier.info("Extracting image tarball..."); + self.notifier.println_above(format!( + "{} {}", + EXTRACT, + style("Extracting & committing layers").bold() + )); + let extracted_image = ExtractedImage::from_tarball(&tarball_path, &self.notifier)?; // Get the layers in chronological order (oldest to newest) @@ -160,7 +176,7 @@ impl ImageProcessor { )); let branch_name = self.source.branch_name(image_name, &os_arch, &metadata.id); self.notifier - .debug(&format!("Generated branch name: '{branch_name}'")); + .debug(&format!("Generated branch name: '{}'", branch_name)); // Initialize or open repository let repo = GitRepo::init_with_branch(output_dir, None)?; @@ -175,7 +191,8 @@ impl ImageProcessor { match branch_commit { Some(commit) => { self.notifier.info(&format!( - "Found optimal branch point at commit {commit}, skipping {matched_layers} matched layers" + "Found optimal branch point at commit {}, skipping {} matched layers", + commit, matched_layers )); (Some(commit), matched_layers) } @@ -191,11 +208,12 @@ impl ImageProcessor { (None, 0) }; - // Check if this is a duplicate image - if branch exists and we're skipping all layers, + // Check if this is a duplicate image - if branch exists, and we're skipping all layers, // it means we're processing the exact same image again if repo.branch_exists(&branch_name) && skip_layers == layers.len() { - self.notifier.info(&format!( - "Image '{image_name}' already exists as branch '{branch_name}' with identical content. Skipping duplicate processing." + self.notifier.println_above(format!( + "Image '{}' already exists as branch '{}' with identical content. Skipping duplicate processing.", + image_name, branch_name )); return Ok(()); } @@ -225,11 +243,14 @@ impl ImageProcessor { // Process each layer in order (oldest to newest) // We'll process all layers from the history, but only extract the real layer tarballs - // Extract directly to the rootfs directory in the target output + // Create a temporary directory for layer extraction self.notifier.info("Preparing layer extraction..."); - // Extract layers directly to the target rootfs directory - let rootfs_path = rootfs_dir.clone(); + // Create a temporary directory for layer extraction and keep a reference to its path + let temp_layer_dir = tempfile::tempdir()?; + let temp_layer_path = temp_layer_dir.path().to_path_buf(); + // Store the temp_dir to keep it alive until the end of the function + temp_dirs.push(temp_layer_dir); // Each layer now contains its own tarball path and digest information self.notifier.debug(&format!( @@ -243,9 +264,8 @@ impl ImageProcessor { // Load existing digest tracker from the start commit Image.md match repo.read_file_from_commit(start_commit, "Image.md") { Ok(content) => { - let image_metadata = - crate::image_metadata::ImageMetadata::parse_markdown(&content) - .context("Failed to parse existing Image.md")?; + let image_metadata = ImageMetadata::parse_markdown(&content) + .context("Failed to parse existing Image.md")?; DigestTracker { layer_digests: image_metadata.layer_digests, } @@ -267,9 +287,39 @@ impl ImageProcessor { // Now process layers starting from the first unmatched layer let layers_to_process = layers.len() - skip_layers; self.notifier.info(&format!( - "Processing {layers_to_process} layers (skipping {skip_layers} matched layers)..." + "Processing {} layers (skipping {} matched layers)...", + layers_to_process, skip_layers )); + let use_multi = matches!(&self.notifier, AnyNotifier::Enhanced(_)); + + let (m, pb1, sty2) = if use_multi && layers_to_process > 0 { + let m = Arc::new(indicatif::MultiProgress::new()); + + let sty = ProgressStyle::default_bar().template( + "{spinner:.green} [{elapsed_precise}] {bar:40.cyan/blue} \ + {pos:>10}/{len:6} {msg} [ETA:{eta}]", + )?; + + let sty2 = ProgressStyle::with_template( + "{spinner:.green} {msg} {prefix:.bold.dim} {bar:40.cyan/blue} \ + {pos:>10}/{len:7} [{eta}]", + )? + .progress_chars("=> "); + + let pb1 = m.add(ProgressBar::new(layers_to_process as u64)); + pb1.set_style(sty); + pb1.enable_steady_tick(Duration::from_millis(80)); + + (Some(m), Some(pb1), Some(sty2)) + } else { + (None, None, None) + }; + + if let Some(ref pb1) = pb1 { + pb1.inc(1); + } + for (i, layer) in layers.iter().enumerate().skip(skip_layers) { self.notifier.info(&format!( "Layer {}/{}: {}", @@ -326,16 +376,166 @@ impl ImageProcessor { let layer_tarball = layer.tarball_path.as_ref().unwrap(); // Extract this layer to the temporary directory - self.notifier - .info(&format!("Extracting layer {}/{}", i + 1, layers.len())); + self.notifier.info(&format!("{}/{}", i + 1, layers.len())); self.notifier - .debug(&format!("Extracting tarball: {layer_tarball:?}")); - fs::create_dir_all(&rootfs_path)?; + .debug(&format!("Extracting tarball: {:?}", layer_tarball)); + fs::create_dir_all(&temp_layer_path)?; + + // Extract the layer tarball to the temp directory + extracted_image.extract_layer_to(layer_tarball, &temp_layer_path)?; + + // Recursively copy all files from the temp layer directory to rootfs + // ---- count only what we'll actually process (files & symlinks, no whiteouts) ---- + let entry_count = walkdir::WalkDir::new(&temp_layer_path) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| { + let ft = e.file_type(); + if !(ft.is_file() || ft.is_symlink()) { + return false; + } + let name = e.file_name().to_string_lossy(); + // exclude overlayfs whiteouts from the progress denominator + name != ".wh..wh..opq" && !name.starts_with(".wh.") + }) + .count() as u64; + + let pb2 = if let (Some(m), Some(sty2)) = (m.as_ref(), sty2.as_ref()) { + let pb = m.add(ProgressBar::new(entry_count)); + pb.set_style(sty2.clone()); + pb.set_message("|________"); + Some(pb) + } else { + None + }; + + for entry in walkdir::WalkDir::new(&temp_layer_path).follow_links(false) { + let entry = entry.context("Failed to read directory entry")?; + let source_path = entry.path(); + + if source_path == temp_layer_path { + continue; + } - // Extract the layer tarball directly to rootfs - // tar_extractor now handles: whiteouts, hardlinks, permission fixing, overlay behavior - extracted_image.extract_layer_to(layer_tarball, &rootfs_path)?; + let relative_path = source_path + .strip_prefix(&temp_layer_path) + .context("Failed to get relative path")?; + + // Skip whiteouts/opaque; do NOT tick for these + if let Some(fname) = relative_path.file_name().and_then(|s| s.to_str()) { + if fname == ".wh..wh..opq" { + let parent_dir = relative_path.parent().unwrap_or_else(|| Path::new("")); + let opaque_dir = rootfs_dir.join(parent_dir); + if opaque_dir.exists() && opaque_dir.is_dir() { + for path in fs::read_dir(&opaque_dir) + .unwrap_or_else(|_| fs::read_dir(".").unwrap()) + .flatten() + .map(|e| e.path()) + { + if path.is_dir() { + fs::remove_dir_all(&path).ok(); + } else { + fs::remove_file(&path).ok(); + } + } + } + continue; + } else if let Some(deleted_name) = fname.strip_prefix(".wh.") { + let parent_dir = relative_path.parent().unwrap_or_else(|| Path::new("")); + let deleted_path = rootfs_dir.join(parent_dir).join(deleted_name); + if deleted_path.exists() { + if deleted_path.is_dir() { + fs::remove_dir_all(&deleted_path).ok(); + } else { + fs::remove_file(&deleted_path).ok(); + } + } + continue; + } + } + + let target_path = rootfs_dir.join(relative_path); + + // Directories: ensure exist; do NOT tick + if entry.file_type().is_dir() { + if let Err(err) = fs::create_dir_all(&target_path) { + if err.kind() == std::io::ErrorKind::PermissionDenied { + self.notifier.warn(&format!( + "Permission denied creating directory {:?} - skipping", + target_path + )); + } + } + continue; + } + + // Symlinks (counted) + if entry.file_type().is_symlink() { + let link_target = fs::read_link(source_path)?; + if target_path.exists() { + if target_path.is_dir() && !target_path.is_symlink() { + fs::remove_dir_all(&target_path).ok(); + } else { + fs::remove_file(&target_path).ok(); + } + } + if let Some(parent) = target_path.parent() { + fs::create_dir_all(parent).ok(); + } + if let Err(err) = std::os::unix::fs::symlink(&link_target, &target_path) { + if err.kind() == std::io::ErrorKind::PermissionDenied { + self.notifier.warn(&format!( + "Permission denied creating symlink {:?} -> {:?} - skipping", + target_path, link_target + )); + } + } + if let Some(ref pb2) = pb2 { + pb2.inc(1); + } + continue; + } + + // Regular files (counted) + if entry.file_type().is_file() { + if target_path.exists() { + if target_path.is_dir() && !target_path.is_symlink() { + fs::remove_dir_all(&target_path).ok(); + } else { + fs::remove_file(&target_path).ok(); + } + } + if let Some(parent) = target_path.parent() { + fs::create_dir_all(parent).ok(); + } + // Copy attempt (we tick regardless to avoid a stuck bar if copy is skipped/denied) + let copy_res = fs::copy(source_path, &target_path); + if let Err(err) = copy_res { + if err.kind() == std::io::ErrorKind::PermissionDenied { + self.notifier.warn(&format!( + "Permission denied copying {:?} - skipping", + source_path + )); + } + } + if let Some(ref pb2) = pb2 { + pb2.inc(1); + } + continue; + } + } + + if let Some(pb2) = pb2 { + pb2.finish_and_clear(); + } + if let Some(ref pb1) = pb1 { + pb1.inc(1); + } + // Clear the temp directory for the next layer + fs::remove_dir_all(&temp_layer_path).ok(); + fs::create_dir_all(&temp_layer_path)?; // Track non-empty layer with digest // Use the current length of the digest tracker as the new position @@ -379,6 +579,25 @@ impl ImageProcessor { ); self.notifier.info(&msg); + if let Some(pb1) = pb1 { + pb1.finish_and_clear(); + } + + match &self.notifier { + AnyNotifier::Enhanced(_) => { + self.notifier.println_above(format!( + "{} {} {}", + SPARK, + style("Done in").bold(), + style(HumanDuration(started.elapsed())).bold() + )); + } + AnyNotifier::Simple(_) => { + // Plain text for simple mode + println!("Done!"); + } + } + Ok(()) } } diff --git a/src/sources/docker.rs b/src/sources/docker.rs index 6ae99db..682a7e2 100644 --- a/src/sources/docker.rs +++ b/src/sources/docker.rs @@ -1,10 +1,20 @@ +use super::{naming, Source}; +use crate::notifier::{AnyNotifier, EnhancedNotifier, SimpleNotifier}; use anyhow::{anyhow, Context, Result}; -use std::path::PathBuf; +use console::{style, Emoji}; +use futures_util::TryStreamExt; +use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; +use serde_json::Value; +use shiplift::{builder::PullOptions, Docker}; +use std::fs; +use std::io::{BufRead, BufReader, BufWriter, Read, Write}; +use std::path::{Path, PathBuf}; use std::process::Command; +use std::process::Stdio; +use std::{collections::HashMap, time::Duration}; use tempfile::TempDir; -use super::{naming, Source}; -use crate::notifier::Notifier; +static LOOK: Emoji<'_, '_> = Emoji("🔍 ", ""); /// Docker implementation of the Source trait pub struct DockerSource; @@ -18,7 +28,7 @@ impl DockerSource { let output = Command::new("docker") .args(args) .output() - .context(format!("Failed to execute docker command: {args:?}"))?; + .context(format!("Failed to execute docker command: {:?}", args))?; if !output.status.success() { let error = String::from_utf8_lossy(&output.stderr); @@ -29,16 +39,8 @@ impl DockerSource { Ok(stdout) } - fn image_exists(&self, image_name: &str) -> bool { - Command::new("docker") - .args(["image", "inspect", image_name]) - .output() - .map(|output| output.status.success()) - .unwrap_or(false) - } - - fn pull_image(&self, image_name: &str, notifier: &Notifier) -> Result<()> { - notifier.info(&format!("Pulling Docker image '{image_name}'...")); + fn pull_image(&self, image_name: &str, notifier: &SimpleNotifier) -> Result<()> { + notifier.info(&format!("Pulling Docker image '{}'...", image_name)); let output = Command::new("docker") .args(["pull", image_name]) @@ -50,7 +52,177 @@ impl DockerSource { return Err(anyhow!("Docker pull failed: {}", error)); } - notifier.info(&format!("Successfully pulled Docker image '{image_name}'")); + notifier.info(&format!( + "Successfully pulled Docker image '{}'", + image_name + )); + Ok(()) + } + #[tokio::main] + pub async fn pull_image_enhanced( + &self, + image_name: &str, + notifier: &EnhancedNotifier, + ) -> Result<()> { + notifier.println_above(format!("Pull Docker image '{}'...", image_name)); + let docker = Docker::new(); + + let tag = image_name + .rsplit_once(':') + .map(|(_, t)| t) // take the part after the last ':' + .unwrap_or("latest"); + + let opts = PullOptions::builder().image(image_name).tag(tag).build(); + let mut stream = docker.images().pull(&opts); + + let m = MultiProgress::with_draw_target(ProgressDrawTarget::stderr_with_hz(15)); + + let overall = m.add(ProgressBar::new(0)); + overall.set_style(ProgressStyle::with_template( + "{prefix:.bold} [{bar:60.cyan/blue}] {bytes}/{total_bytes} [ETA:{eta}]", + )?); + + let host = "docker.io"; + let pull_from_message = format!("Pull from {}", host); + + overall.set_prefix(pull_from_message); + overall.enable_steady_tick(Duration::from_millis(80)); + + let layer_style = ProgressStyle::with_template( + "{prefix:.dim} {spinner} [{bar:40.cyan/blue}] {bytes}/{total_bytes} {wide_msg}", + )? + .tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "); + + struct Layer { + pb: ProgressBar, + total: u64, + current: u64, + finished: bool, + } + let mut layers: HashMap = HashMap::new(); + let mut overall_total: u64 = 0; + + let update_overall = + |layers: &HashMap, overall: &ProgressBar, total_ref: &mut u64| { + let mut sum_total = 0u64; + let mut sum_curr = 0u64; + for l in layers.values() { + if l.total > 0 { + sum_total = sum_total.saturating_add(l.total); + } + sum_curr = sum_curr.saturating_add(l.current); + } + if sum_total > *total_ref { + *total_ref = sum_total; + overall.set_length(sum_total); + } + if *total_ref > 0 { + overall.set_position(sum_curr.min(*total_ref)); + overall.set_message(format!( + "({} layers downloaded)", + layers.values().filter(|l| l.finished).count() + )); + } + }; + + let important_msg = |status: &str| -> bool { + matches!( + status, + s if s.starts_with("Pulling from") + || s.starts_with("Digest:") + || s.starts_with("Status:") + || s.starts_with("Downloaded newer image for") + ) + }; + + while let Some(chunk) = stream.try_next().await? { + let v: &Value = &chunk; + let status = v.get("status").and_then(Value::as_str).unwrap_or(""); + let id_full = v.get("id").and_then(Value::as_str).unwrap_or_default(); + let id_short = if id_full.len() > 12 { + &id_full[..12] + } else { + id_full + }; + let pd = v.get("progressDetail").and_then(Value::as_object); + let total = pd + .and_then(|o| o.get("total")) + .and_then(Value::as_u64) + .unwrap_or(0); + let current = pd + .and_then(|o| o.get("current")) + .and_then(Value::as_u64) + .unwrap_or(0); + + // global messages + if id_full.is_empty() { + if important_msg(status) { + m.println(status)?; + } + continue; + } + + // We want bars only while Downloading. + match status { + "Downloading" => { + let entry = layers.entry(id_full.to_string()).or_insert_with(|| { + let pb = m.insert_after(&overall, ProgressBar::new(0)); + pb.set_style(layer_style.clone()); + pb.set_prefix(id_short.to_string()); + pb.enable_steady_tick(Duration::from_millis(80)); + Layer { + pb, + total: 0, + current: 0, + finished: false, + } + }); + + if total > 0 && entry.total == 0 { + entry.total = total; + entry.pb.set_length(total); + } + if total > 0 { + entry.current = current; + entry.pb.set_position(current); + } + entry.pb.set_message("Downloading"); + + update_overall(&layers, &overall, &mut overall_total); + } + + // As soon as a layer is no longer Downloading, remove its bar + "Extracting" | "Verifying Checksum" | "Download complete" | "Pull complete" + | "Already exists" | "Mounted from" => { + if let Some(entry) = layers.get_mut(id_full) { + // Snap to full if we know total, then clear bar + if entry.total > 0 { + entry.current = entry.total.max(entry.current); + entry.pb.set_position(entry.total); + } + if !entry.finished { + entry.finished = true; + entry.pb.finish_and_clear(); + } + } + update_overall(&layers, &overall, &mut overall_total); + } + + _ => {} + } + } + // overall.finish_with_message("waiting...."); + + // Finish whatever remains + for (_, l) in layers { + if !l.finished { + l.pb.finish_and_clear(); + } + } + if !overall.is_finished() { + overall.finish_and_clear(); + } + Ok(()) } } @@ -60,23 +232,138 @@ impl Source for DockerSource { "docker" } + /// Best-effort estimate using `docker images inspect {{.Size}}`. + fn estimate_image_size(image: &str) -> Option { + let _ = Command::new("docker") + .args(["images", image, "--format", "{{.Size}}"]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .output() + .ok()?; + + let raw_out = Command::new("docker") + .args(["image", "inspect", image, "--format", "{{.Size}}"]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .output() + .ok()?; + + if !raw_out.status.success() { + return None; + } + + let bytes_size: u64 = String::from_utf8_lossy(&raw_out.stdout) + .trim() + .parse() + .ok()?; + + Some(bytes_size) + } + + /// Stream `docker save` -> file with live progress. + /// - No deadlocks, no giant buffering + /// - Safe (writes to .partial then renames) + /// + fn image_save_with_progress(image: &str, tar_path: &Path) -> Result<()> { + let tag = image + .rsplit_once(':') + .map(|(_, t)| t) // take the part after the last ':' + .unwrap_or("latest"); + + let image = format!("{image}:{tag}"); + + // --- progress bar + let pb = ProgressBar::new(Self::estimate_image_size(image.as_str()).unwrap_or(0)); + pb.set_style(ProgressStyle::with_template( + "{prefix:.dim} [{bar:40.cyan/blue}] {bytes}/{total_bytes} [ETA:{eta}]", + )?); + + pb.set_prefix(format!("Exporting Docker image '{}' to tarball...", image)); + + // --- create temp file (atomic rename later) + let tmp_path = tar_path.with_extension("partial"); + let tmp_file = fs::File::create(&tmp_path) + .with_context(|| format!("create temp file at {}", tmp_path.display()))?; + let mut writer = BufWriter::new(tmp_file); + + // --- spawn docker save (to stdout) + let mut child = Command::new("docker") + .args(["save", image.as_str()]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) // capture errors + .spawn() + .context("failed to spawn `docker save`")?; + + // --- continuously drain stderr to avoid pipe blocking + keep logs + let mut err_reader = BufReader::new(child.stderr.take().expect("no stderr")); + let err_collector = std::thread::spawn(move || { + let mut buf = String::new(); + let mut acc = String::new(); + while let Ok(n) = err_reader.read_line(&mut buf) { + if n == 0 { + break; + } + acc.push_str(&buf); + buf.clear(); + } + acc + }); + + // --- stream stdout to file and update progress + let mut out = child.stdout.take().context("child has no stdout")?; + let mut buf = vec![0u8; 1024 * 1024]; // 1 MiB chunks + let mut written: u64 = 0; + + loop { + let n = out.read(&mut buf)?; + if n == 0 { + break; + } + writer.write_all(&buf[..n])?; + written += n as u64; + pb.set_position(written); + } + writer.flush()?; + + // --- wait and check status + let status = child.wait()?; + let stderr_text = err_collector.join().unwrap_or_default(); + + if !status.success() { + pb.finish_and_clear(); + // leave the .partial file for forensics or remove it: + let _ = fs::remove_file(&tmp_path); + return Err(anyhow!( + "`docker save` failed (exit {:?}). stderr:\n{}", + status.code(), + stderr_text + )); + } + + // --- rename atomically + fs::rename(&tmp_path, tar_path) + .with_context(|| format!("rename {} -> {}", tmp_path.display(), tar_path.display()))?; + + // pb.finish_with_message("✅"); + pb.finish_and_clear(); + + Ok(()) + } + fn get_image_tarball( &self, image_name: &str, - notifier: &Notifier, + notifier: &AnyNotifier, ) -> Result<(PathBuf, Option)> { // Create a temporary directory to save the image let temp_dir = TempDir::new().context("Failed to create temporary directory")?; let tarball_path = temp_dir.path().join("image.tar"); - // Use docker save to export the full image with all layers - notifier.info(&format!( - "Exporting Docker image '{image_name}' to tarball..." - )); + notifier.println_above(format!("{} {}", LOOK, style("Resolving image").bold())); + + notifier.finish_spinner(); - // Try to save the image first - let save_result = - self.run_command(&["save", "-o", tarball_path.to_str().unwrap(), image_name]); + let save_result = Self::image_save_with_progress(image_name, &tarball_path); match save_result { Ok(_) => { @@ -84,26 +371,38 @@ impl Source for DockerSource { Ok((tarball_path, Some(temp_dir))) } Err(e) => { - // Save failed - check if it's because the image doesn't exist - if !self.image_exists(image_name) { + let error_msg = e.to_string(); + // Check if the error is about missing image + if error_msg.contains("No such image") + || error_msg.contains("pull access denied") + || error_msg.contains("reference does not exist") + { notifier.info(&format!( - "Image '{image_name}' not found locally, attempting to pull..." + "Image '{}' not found locally, attempting to pull...", + image_name )); // Try to pull the image - self.pull_image(image_name, notifier) - .context(format!("Failed to pull image '{image_name}'"))?; + match notifier { + AnyNotifier::Simple(notifier) => self + .pull_image(image_name, notifier) + .context(format!("Failed to pull image '{}'", image_name))?, + AnyNotifier::Enhanced(notifier) => self + .pull_image_enhanced(image_name, notifier) + .context(format!("Failed to pull image '{}'", image_name))?, + } // Retry the save command after successful pull notifier.info(&format!( - "Retrying export of Docker image '{image_name}' to tarball..." + "Retrying export of Docker image '{}' to tarball...", + image_name )); self.run_command(&["save", "-o", tarball_path.to_str().unwrap(), image_name]) - .context(format!("Failed to save image '{image_name}' after pull"))?; + .context(format!("Failed to save image '{}' after pull", image_name))?; Ok((tarball_path, Some(temp_dir))) } else { - // Image exists but save failed for another reason - propagate original error + // Different error - propagate it Err(e) } } diff --git a/src/sources/nerdctl.rs b/src/sources/nerdctl.rs index 2320ad3..fbcb2a3 100644 --- a/src/sources/nerdctl.rs +++ b/src/sources/nerdctl.rs @@ -1,10 +1,10 @@ use anyhow::{anyhow, Context, Result}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; use tempfile::TempDir; use super::Source; -use crate::notifier::Notifier; +use crate::notifier::AnyNotifier; pub struct NerdctlSource; @@ -28,10 +28,18 @@ impl Source for NerdctlSource { "nerdctl" } + fn estimate_image_size(_image: &str) -> Option { + todo!() + } + + fn image_save_with_progress(_image: &str, _tar_path: &Path) -> Result<()> { + todo!() + } + fn get_image_tarball( &self, _image_name: &str, - _notifier: &Notifier, + _notifier: &AnyNotifier, ) -> Result<(PathBuf, Option)> { // This will be implemented in the future unimplemented!("nerdctl support is not yet implemented") diff --git a/src/sources/source.rs b/src/sources/source.rs index f674be7..8f8c39a 100644 --- a/src/sources/source.rs +++ b/src/sources/source.rs @@ -1,13 +1,20 @@ use anyhow::Result; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use tempfile::TempDir; -use crate::notifier::Notifier; +use crate::notifier::AnyNotifier; /// Source trait for getting OCI images from different container sources pub trait Source { /// Returns the name of the source for identification purposes fn name(&self) -> &str; + /// Best-effort estimate using `docker image inspect {{.Size}}`. + fn estimate_image_size(image: &str) -> Option; + /// Stream `docker save` -> file with live progress. + /// - No deadlocks, no giant buffering + /// - Safe (writes to .partial then renames) + /// + fn image_save_with_progress(image: &str, tar_path: &Path) -> Result<()>; /// Retrieves an OCI image tarball and returns the path to it along with temp directory if created /// The image_name parameter can be an image reference (for registry sources) @@ -18,7 +25,7 @@ pub trait Source { fn get_image_tarball( &self, image_name: &str, - notifier: &Notifier, + notifier: &AnyNotifier, ) -> Result<(PathBuf, Option)>; /// Generates a Git branch name from the image name/path diff --git a/src/sources/tar.rs b/src/sources/tar.rs index a8e12b8..9b58d57 100644 --- a/src/sources/tar.rs +++ b/src/sources/tar.rs @@ -1,9 +1,9 @@ use anyhow::{anyhow, Result}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use tempfile::TempDir; use super::Source; -use crate::notifier::Notifier; +use crate::notifier::AnyNotifier; /// Extracts filename from a tar path and sanitizes it for Git branch naming /// Removes file extension and sanitizes problematic characters @@ -31,10 +31,18 @@ impl Source for TarSource { "tar" } + fn estimate_image_size(_image: &str) -> Option { + todo!() + } + + fn image_save_with_progress(_image: &str, _tar_path: &Path) -> Result<()> { + todo!() + } + fn get_image_tarball( &self, image_path: &str, - notifier: &Notifier, + notifier: &AnyNotifier, ) -> Result<(PathBuf, Option)> { // For tar source, image_path is the path to the existing tarball let tarball_path = PathBuf::from(image_path); diff --git a/src/tar_extractor.rs b/src/tar_extractor.rs index cb57c82..7547af2 100644 --- a/src/tar_extractor.rs +++ b/src/tar_extractor.rs @@ -3,7 +3,7 @@ use flate2::read::GzDecoder; use std::fs::{self, File}; use std::io::{BufReader, Read}; use std::path::{Component, Path, PathBuf}; -use tar_rs as tar; +// use tar_rs as tar; /// Normalizes a path from a tar archive to be safe for extraction /// Removes any attempts to escape the root directory diff --git a/tests/extracted_image_test.rs b/tests/extracted_image_test.rs index 3db6ead..89e249b 100644 --- a/tests/extracted_image_test.rs +++ b/tests/extracted_image_test.rs @@ -1,5 +1,5 @@ use oci2git::extracted_image::ExtractedImage; -use oci2git::Notifier; +use oci2git::notifier::{AnyNotifier, NotifierFlavor}; use std::path::Path; #[test] @@ -12,8 +12,7 @@ fn test_extracted_image_eager_loading() { eprintln!("Skipping test: fixture file not found at {fixture_path:?}"); return; } - - let notifier = Notifier::new(0); // Verbosity level 0 for quiet + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); // Test that from_tarball does all the work upfront let extracted_image = ExtractedImage::from_tarball(fixture_path, ¬ifier) @@ -121,7 +120,7 @@ fn test_extracted_image_eager_loading() { #[test] fn test_extracted_image_validation() { - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); // Test with non-existent file let result = ExtractedImage::from_tarball("non-existent-file.tar", ¬ifier); @@ -141,7 +140,7 @@ fn test_extracted_image_multiple_calls() { return; } - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let extracted_image = ExtractedImage::from_tarball(fixture_path, ¬ifier) .expect("Failed to extract image from tarball"); diff --git a/tests/integration/common/tar_processing.rs b/tests/integration/common/tar_processing.rs index b39827d..4747474 100644 --- a/tests/integration/common/tar_processing.rs +++ b/tests/integration/common/tar_processing.rs @@ -5,7 +5,8 @@ //! functionality that's shared across all sources. use anyhow::Result; -use oci2git::notifier::Notifier; +use oci2git::notifier::AnyNotifier; +use oci2git::notifier::NotifierFlavor; use oci2git::processor::ImageProcessor; use oci2git::sources::Source; use std::fs; @@ -15,7 +16,7 @@ use tempfile::TempDir; /// Test helper: Process any tar file and verify basic structure pub fn test_basic_tar_processing(source: S, tar_path: &str) -> Result<()> { let output_dir = TempDir::new()?; - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let processor = ImageProcessor::new(source, notifier); // Process the tar file @@ -125,7 +126,7 @@ mod tests { let output_dir = TempDir::new()?; let tar_source = TarSource::new()?; - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let processor = ImageProcessor::new(tar_source, notifier); // Process the image @@ -152,7 +153,7 @@ mod tests { let output_dir = TempDir::new()?; let tar_source = TarSource::new()?; - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let processor = ImageProcessor::new(tar_source, notifier); // Process the image @@ -178,7 +179,7 @@ mod tests { let output_dir = TempDir::new()?; let tar_source = TarSource::new()?; - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let processor = ImageProcessor::new(tar_source, notifier); // Process the image diff --git a/tests/integration/tar/mod.rs b/tests/integration/tar/mod.rs index e604d71..0bfb82a 100644 --- a/tests/integration/tar/mod.rs +++ b/tests/integration/tar/mod.rs @@ -6,7 +6,6 @@ use crate::integration::common::tar_processing; use anyhow::Result; -use oci2git::notifier::Notifier; use oci2git::processor::ImageProcessor; use oci2git::sources::{Source, TarSource}; use std::io::Write; @@ -16,6 +15,7 @@ use tempfile::{NamedTempFile, TempDir}; #[cfg(test)] mod tests { use super::*; + use oci2git::notifier::{AnyNotifier, NotifierFlavor}; const FIXTURE_TAR_PATH: &str = "tests/integration/fixtures/oci2git-test.tar"; @@ -36,7 +36,7 @@ mod tests { let temp_path = temp_file.path().to_str().unwrap(); let tar_source = TarSource::new().expect("Should create TarSource"); - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let result = tar_source.get_image_tarball(temp_path, ¬ifier); assert!( @@ -66,7 +66,7 @@ mod tests { fn test_tar_source_with_nonexistent_file() { let tar_source = TarSource::new().expect("Should create TarSource"); let nonexistent_path = "/path/that/definitely/does/not/exist.tar"; - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let result = tar_source.get_image_tarball(nonexistent_path, ¬ifier); @@ -97,7 +97,7 @@ mod tests { let file_name = temp_file.path().file_name().unwrap().to_str().unwrap(); let tar_source = TarSource::new().expect("Should create TarSource"); - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let result = tar_source.get_image_tarball(file_name, ¬ifier); // The result will depend on TarSource implementation @@ -135,7 +135,7 @@ mod tests { let output_dir = TempDir::new()?; let tar_source = TarSource::new()?; - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let processor = ImageProcessor::new(tar_source, notifier); // Process through universal backend @@ -164,7 +164,7 @@ mod tests { let output_dir = TempDir::new()?; let tar_source = TarSource::new()?; - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let processor = ImageProcessor::new(tar_source, notifier); // Process through universal backend @@ -190,7 +190,7 @@ mod tests { let output_dir = TempDir::new()?; let tar_source = TarSource::new()?; - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let processor = ImageProcessor::new(tar_source, notifier); // Process through universal backend @@ -216,7 +216,7 @@ mod tests { let output_dir = TempDir::new()?; let tar_source = TarSource::new()?; - let notifier = Notifier::new(0); + let notifier = AnyNotifier::new(NotifierFlavor::Simple, 0); let processor = ImageProcessor::new(tar_source, notifier); println!("Converting image with hardlinks..."); @@ -335,12 +335,34 @@ mod tests { println!("✓ Multiple hardlinks to empty file work correctly"); // Verify file with multiple link types - let file1_content = std::fs::read_to_string(rootfs.join("app/data/file1.txt"))?; - let file1_hardlink = std::fs::read_to_string(rootfs.join("app/data/file1-hardlink.txt"))?; - let file1_symlink = std::fs::read_to_string(rootfs.join("app/data/file1-symlink.txt"))?; + let file1 = rootfs.join("app/data/file1.txt"); + let file1_hardlink = rootfs.join("app/data/file1-hardlink.txt"); + let file1_symlink_path = rootfs.join("app/data/file1-symlink.txt"); + + let file1_content = std::fs::read_to_string(&file1)?; + let file1_hardlink_content = std::fs::read_to_string(&file1_hardlink)?; assert_eq!(file1_content, "Data file 1\n"); - assert_eq!(file1_content, file1_hardlink); - assert_eq!(file1_content, file1_symlink); + assert_eq!(file1_content, file1_hardlink_content); + + #[cfg(unix)] + { + use std::path::{Path, PathBuf}; + + // Inspect the symlink itself + let link_target = std::fs::read_link(&file1_symlink_path)?; + // It should be an absolute container path + assert_eq!( + link_target, + Path::new("/app/data/file1.txt"), + "Unexpected target for file1-symlink.txt: {link_target:?}" + ); + + // Interpret container-absolute target under rootfs + let effective_target: PathBuf = rootfs.join(link_target.strip_prefix("/").unwrap()); + let effective_content = std::fs::read_to_string(effective_target)?; + assert_eq!(file1_content, effective_content); + } + println!("✓ Mixed hardlinks and symlinks work correctly"); println!("✅ All comprehensive link extraction tests passed!"); From d62e1fe96e6e2567ed04ac9592730eda25c5a309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=B9mi=C4=87er=20Rubin=C5=A1tejn?= Date: Mon, 22 Dec 2025 10:57:49 +0200 Subject: [PATCH 2/4] Add release pipelines (#44) * Add release pipelines * Fix pipeline --- .github/workflows/release.yml | 257 ++++++++++++++++++++++++++++++ PACKAGING.md | 283 ++++++++++++++++++++++++++++++++++ README.md | 68 +++++++- RELEASE.md | 161 +++++++++++++++++++ aur/.SRCINFO | 16 ++ aur/PKGBUILD | 24 +++ aur/update-aur.sh | 69 +++++++++ homebrew/oci2git.rb | 46 ++++++ homebrew/update-formula.sh | 56 +++++++ 9 files changed, 974 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 PACKAGING.md create mode 100644 RELEASE.md create mode 100644 aur/.SRCINFO create mode 100644 aur/PKGBUILD create mode 100755 aur/update-aur.sh create mode 100644 homebrew/oci2git.rb create mode 100755 homebrew/update-formula.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..da8762d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,257 @@ +name: Release + +on: + push: + tags: + - "v*" + workflow_dispatch: {} + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + create-release: + name: Create Release + runs-on: ubuntu-latest + outputs: + upload_url: ${{ steps.create_release.outputs.upload_url }} + version: ${{ steps.get_version.outputs.version }} + steps: + - name: Get version from tag + id: get_version + run: echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: false + prerelease: false + + build-release: + name: Build Release + needs: create-release + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + asset_name: oci2git-linux-x86_64 + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + asset_name: oci2git-linux-aarch64 + - target: x86_64-apple-darwin + os: macos-13 + asset_name: oci2git-darwin-x86_64 + - target: aarch64-apple-darwin + os: macos-14 + asset_name: oci2git-darwin-aarch64 + - target: x86_64-pc-windows-gnu + os: ubuntu-latest + asset_name: oci2git-windows-x86_64.exe + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross-compilation tools (Linux ARM64) + if: matrix.target == 'aarch64-unknown-linux-gnu' + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + + - name: Install cross-compilation tools (Windows) + if: matrix.target == 'x86_64-pc-windows-gnu' + run: | + sudo apt-get update + sudo apt-get install -y gcc-mingw-w64-x86-64 + + - name: Build release binary + run: cargo build --release --target ${{ matrix.target }} + env: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc + CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER: x86_64-w64-mingw32-gcc + + - name: Strip binary (Unix) + if: matrix.os != 'windows-latest' && matrix.target != 'aarch64-unknown-linux-gnu' + run: strip target/${{ matrix.target }}/release/oci2git + + - name: Prepare binary + run: | + mkdir -p dist + if [[ "${{ matrix.target }}" == *"windows"* ]]; then + cp target/${{ matrix.target }}/release/oci2git.exe dist/${{ matrix.asset_name }} + else + cp target/${{ matrix.target }}/release/oci2git dist/${{ matrix.asset_name }} + fi + + - name: Create tarball + run: | + cd dist + tar czf ${{ matrix.asset_name }}.tar.gz ${{ matrix.asset_name }} + sha256sum ${{ matrix.asset_name }}.tar.gz > ${{ matrix.asset_name }}.tar.gz.sha256 + + - name: Upload Release Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ./dist/${{ matrix.asset_name }}.tar.gz + asset_name: ${{ matrix.asset_name }}.tar.gz + asset_content_type: application/gzip + + - name: Upload Checksum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ./dist/${{ matrix.asset_name }}.tar.gz.sha256 + asset_name: ${{ matrix.asset_name }}.tar.gz.sha256 + asset_content_type: text/plain + + - name: Save artifacts for packaging + uses: actions/upload-artifact@v4 + with: + name: binary-${{ matrix.target }} + path: dist/ + + build-deb: + name: Build Debian Package + needs: [create-release, build-release] + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download Linux x86_64 binary + uses: actions/download-artifact@v4 + with: + name: binary-x86_64-unknown-linux-gnu + path: ./dist-x86_64 + + - name: Download Linux ARM64 binary + uses: actions/download-artifact@v4 + with: + name: binary-aarch64-unknown-linux-gnu + path: ./dist-aarch64 + + - name: Install packaging tools + run: sudo apt-get update && sudo apt-get install -y dpkg-dev + + - name: Build Debian package (amd64) + run: | + VERSION="${{ needs.create-release.outputs.version }}" + ARCH="amd64" + PKG_NAME="oci2git_${VERSION}_${ARCH}" + + mkdir -p "${PKG_NAME}/DEBIAN" + mkdir -p "${PKG_NAME}/usr/bin" + mkdir -p "${PKG_NAME}/usr/share/doc/oci2git" + + # Copy binary + cp dist-x86_64/oci2git-linux-x86_64 "${PKG_NAME}/usr/bin/oci2git" + chmod 755 "${PKG_NAME}/usr/bin/oci2git" + + # Copy README if exists + [ -f README.md ] && cp README.md "${PKG_NAME}/usr/share/doc/oci2git/" + + # Create control file + cat > "${PKG_NAME}/DEBIAN/control" << EOF + Package: oci2git + Version: ${VERSION} + Section: utils + Priority: optional + Architecture: ${ARCH} + Maintainer: Dmitry Rubinstein + Description: A tool to convert OCI images to Git repositories + oci2git converts OCI/Docker images into Git repositories, + making it easier to track and version container images. + Homepage: https://github.com/virviil/oci2git + EOF + + # Build package + dpkg-deb --build "${PKG_NAME}" + + - name: Build Debian package (arm64) + run: | + VERSION="${{ needs.create-release.outputs.version }}" + ARCH="arm64" + PKG_NAME="oci2git_${VERSION}_${ARCH}" + + mkdir -p "${PKG_NAME}/DEBIAN" + mkdir -p "${PKG_NAME}/usr/bin" + mkdir -p "${PKG_NAME}/usr/share/doc/oci2git" + + # Copy binary + cp dist-aarch64/oci2git-linux-aarch64 "${PKG_NAME}/usr/bin/oci2git" + chmod 755 "${PKG_NAME}/usr/bin/oci2git" + + # Copy README if exists + [ -f README.md ] && cp README.md "${PKG_NAME}/usr/share/doc/oci2git/" + + # Create control file + cat > "${PKG_NAME}/DEBIAN/control" << EOF + Package: oci2git + Version: ${VERSION} + Section: utils + Priority: optional + Architecture: ${ARCH} + Maintainer: Dmitry Rubinstein + Description: A tool to convert OCI images to Git repositories + oci2git converts OCI/Docker images into Git repositories, + making it easier to track and version container images. + Homepage: https://github.com/virviil/oci2git + EOF + + # Build package + dpkg-deb --build "${PKG_NAME}" + + - name: Generate checksums + run: | + sha256sum oci2git_*.deb > checksums.txt + + - name: Upload Debian package (amd64) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ./oci2git_${{ needs.create-release.outputs.version }}_amd64.deb + asset_name: oci2git_${{ needs.create-release.outputs.version }}_amd64.deb + asset_content_type: application/vnd.debian.binary-package + + - name: Upload Debian package (arm64) + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ./oci2git_${{ needs.create-release.outputs.version }}_arm64.deb + asset_name: oci2git_${{ needs.create-release.outputs.version }}_arm64.deb + asset_content_type: application/vnd.debian.binary-package + + - name: Upload checksums + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ./checksums.txt + asset_name: debian-checksums.txt + asset_content_type: text/plain diff --git a/PACKAGING.md b/PACKAGING.md new file mode 100644 index 0000000..eeb47de --- /dev/null +++ b/PACKAGING.md @@ -0,0 +1,283 @@ +# Packaging Guide for oci2git + +This document describes how to build and distribute oci2git binaries and packages for various platforms. + +## Overview + +The project uses GitHub Actions to automatically build releases for: +- **Binary releases**: Linux (x86_64, ARM64), macOS (x86_64, ARM64), Windows (x86_64) +- **Debian packages**: .deb packages for Ubuntu/Debian (amd64, arm64) +- **Homebrew**: macOS and Linux via Homebrew tap +- **AUR**: Arch Linux via the AUR (Arch User Repository) + +## Automated Release Process + +### Creating a New Release + +1. Update the version in `Cargo.toml`: + ```toml + version = "0.2.5" + ``` + +2. Commit the version change: + ```bash + git add Cargo.toml + git commit -m "Bump version to 0.2.5" + git push + ``` + +3. Create and push a version tag: + ```bash + git tag v0.2.5 + git push origin v0.2.5 + ``` + +4. GitHub Actions will automatically: + - Build binaries for all platforms + - Create Debian packages + - Create a GitHub release + - Upload all artifacts with checksums + +## Homebrew Tap Setup + +### One-Time Setup + +1. Create a new GitHub repository named `homebrew-oci2git`: + ```bash + gh repo create homebrew-oci2git --public + git clone https://github.com/virviil/homebrew-oci2git.git + cd homebrew-oci2git + mkdir -p Formula + ``` + +2. Copy the formula template: + ```bash + cp ../oci2git/homebrew/oci2git.rb Formula/ + ``` + +### Updating the Formula for Each Release + +After creating a new release, update the Homebrew formula: + +```bash +cd oci2git +./homebrew/update-formula.sh 0.2.5 +``` + +This script will: +- Download the checksums from the GitHub release +- Update the formula with the new version and checksums +- Display the updated formula + +Then copy to your tap repository: + +```bash +cp homebrew/oci2git.rb ../homebrew-oci2git/Formula/ +cd ../homebrew-oci2git +git add Formula/oci2git.rb +git commit -m "Update oci2git to 0.2.5" +git push +``` + +### Installing via Homebrew + +Users can then install with: + +```bash +brew tap virviil/oci2git +brew install oci2git +``` + +## Debian Package Distribution + +Debian packages are automatically built and uploaded to GitHub releases. + +### Manual Installation + +Users can download and install the appropriate .deb package: + +```bash +# For amd64 (x86_64) +wget https://github.com/virviil/oci2git/releases/download/v0.2.5/oci2git_0.2.5_amd64.deb +sudo dpkg -i oci2git_0.2.5_amd64.deb + +# For arm64 +wget https://github.com/virviil/oci2git/releases/download/v0.2.5/oci2git_0.2.5_arm64.deb +sudo dpkg -i oci2git_0.2.5_arm64.deb +``` + +### Setting Up a Debian Repository (Optional) + +For easier installation and updates, you can set up a Debian package repository using GitHub Pages or a hosting service. Tools like `aptly` or `reprepro` can help manage the repository. + +## AUR (Arch Linux) Setup + +### One-Time Setup + +1. Create an AUR account at https://aur.archlinux.org/ + +2. Set up SSH keys for AUR: + ```bash + ssh-keygen -t ed25519 -C "your_email@example.com" + # Add the public key to your AUR account settings + ``` + +3. Clone the AUR repository (first time): + ```bash + git clone ssh://aur@aur.archlinux.org/oci2git-bin.git + cd oci2git-bin + ``` + +### Updating the AUR Package for Each Release + +After creating a new release: + +```bash +cd oci2git/aur +./update-aur.sh 0.2.5 +``` + +This script will: +- Download checksums from GitHub release +- Update PKGBUILD with the new version and checksums +- Generate .SRCINFO + +Then publish to AUR: + +```bash +# Copy files to AUR repository +cp PKGBUILD .SRCINFO ../oci2git-bin/ + +cd ../oci2git-bin + +# Test the build (optional but recommended) +makepkg -si + +# Commit and push to AUR +git add PKGBUILD .SRCINFO +git commit -m "Update to 0.2.5" +git push +``` + +### Installing from AUR + +Users can install using an AUR helper: + +```bash +yay -S oci2git-bin +# or +paru -S oci2git-bin +``` + +Or manually: + +```bash +git clone https://aur.archlinux.org/oci2git-bin.git +cd oci2git-bin +makepkg -si +``` + +## Binary Releases + +Binary releases are automatically uploaded to GitHub Releases and include: + +- `oci2git-linux-x86_64.tar.gz` +- `oci2git-linux-aarch64.tar.gz` +- `oci2git-darwin-x86_64.tar.gz` (macOS Intel) +- `oci2git-darwin-aarch64.tar.gz` (macOS Apple Silicon) +- `oci2git-windows-x86_64.exe.tar.gz` + +Each archive includes SHA256 checksums for verification. + +### Manual Installation + +Users can download and install binaries manually: + +```bash +# Linux x86_64 example +wget https://github.com/virviil/oci2git/releases/download/v0.2.5/oci2git-linux-x86_64.tar.gz +tar xzf oci2git-linux-x86_64.tar.gz +sudo mv oci2git-linux-x86_64 /usr/local/bin/oci2git +chmod +x /usr/local/bin/oci2git +``` + +## Troubleshooting + +### GitHub Actions Failures + +If the release workflow fails: + +1. Check the Actions tab in GitHub for error logs +2. Common issues: + - Cross-compilation toolchain not installed + - Permission issues with GITHUB_TOKEN + - Network issues downloading dependencies + +### Homebrew Formula Issues + +If the formula doesn't work: + +1. Test locally: + ```bash + brew install --build-from-source ./Formula/oci2git.rb + ``` + +2. Check SHA256 mismatches: + ```bash + wget + shasum -a 256 oci2git-*.tar.gz + ``` + +### AUR Package Issues + +If the PKGBUILD fails: + +1. Test locally: + ```bash + makepkg -si + ``` + +2. Check for: + - Incorrect SHA256 sums + - Network issues downloading sources + - Missing dependencies + +## Release Checklist + +- [ ] Update version in Cargo.toml +- [ ] Update CHANGELOG.md (if exists) +- [ ] Commit version changes +- [ ] Create and push git tag (v0.2.5) +- [ ] Wait for GitHub Actions to complete +- [ ] Verify all artifacts in GitHub Release +- [ ] Update Homebrew formula +- [ ] Push Homebrew formula to tap +- [ ] Update AUR package +- [ ] Push AUR package update +- [ ] Test installations on each platform +- [ ] Announce release + +## Platform-Specific Notes + +### macOS Code Signing + +For production releases, consider code signing the macOS binaries: + +```bash +codesign --sign "Developer ID Application: Your Name" --timestamp oci2git +``` + +This requires an Apple Developer account. + +### Windows Signing + +For production releases, consider signing the Windows executable with a code signing certificate. + +### Linux AppImage (Future) + +Consider creating AppImage packages for broader Linux compatibility. + +## Support + +For issues with packaging or distribution, please open an issue at: +https://github.com/virviil/oci2git/issues diff --git a/README.md b/README.md index fb53647..f2834ae 100644 --- a/README.md +++ b/README.md @@ -133,15 +133,60 @@ This approach is particularly valuable for: ## Installation -### From Source +### Package Managers + +#### macOS / Linux (Homebrew) ```bash -# Clone the repository -git clone https://github.com/virviil/oci2git.git -cd oci2git +brew tap virviil/oci2git +brew install oci2git +``` -# Install locally -cargo install --path . +#### Debian / Ubuntu + +Download and install the .deb package from the [latest release](https://github.com/virviil/oci2git/releases/latest): + +```bash +# For amd64 (x86_64) +wget https://github.com/virviil/oci2git/releases/latest/download/oci2git_VERSION_amd64.deb +sudo dpkg -i oci2git_VERSION_amd64.deb + +# For arm64 +wget https://github.com/virviil/oci2git/releases/latest/download/oci2git_VERSION_arm64.deb +sudo dpkg -i oci2git_VERSION_arm64.deb +``` + +#### Arch Linux (AUR) + +```bash +# Using yay +yay -S oci2git-bin + +# Using paru +paru -S oci2git-bin + +# Manual installation +git clone https://aur.archlinux.org/oci2git-bin.git +cd oci2git-bin +makepkg -si +``` + +### Pre-built Binaries + +Download the appropriate binary for your platform from the [latest release](https://github.com/virviil/oci2git/releases/latest): + +```bash +# Linux x86_64 +wget https://github.com/virviil/oci2git/releases/latest/download/oci2git-linux-x86_64.tar.gz +tar xzf oci2git-linux-x86_64.tar.gz +sudo mv oci2git-linux-x86_64 /usr/local/bin/oci2git +chmod +x /usr/local/bin/oci2git + +# macOS (Apple Silicon) +wget https://github.com/virviil/oci2git/releases/latest/download/oci2git-darwin-aarch64.tar.gz +tar xzf oci2git-darwin-aarch64.tar.gz +sudo mv oci2git-darwin-aarch64 /usr/local/bin/oci2git +chmod +x /usr/local/bin/oci2git ``` ### From Crates.io @@ -150,6 +195,17 @@ cargo install --path . cargo install oci2git ``` +### From Source + +```bash +# Clone the repository +git clone https://github.com/virviil/oci2git.git +cd oci2git + +# Install locally +cargo install --path . +``` + ## Usage ```bash diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..448dee9 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,161 @@ +# Release Process Quick Reference + +This document provides a quick checklist for creating and publishing a new release of oci2git. + +## Pre-Release Checklist + +- [ ] All tests are passing (`cargo test`) +- [ ] Code is formatted (`cargo fmt`) +- [ ] No clippy warnings (`cargo clippy`) +- [ ] Documentation is up to date +- [ ] CHANGELOG.md is updated (if exists) + +## Creating a Release + +### 1. Update Version + +Edit `Cargo.toml`: + +```toml +version = "0.2.5" +``` + +### 2. Commit and Tag + +```bash +git add Cargo.toml +git commit -m "Bump version to 0.2.5" +git push + +git tag v0.2.5 +git push origin v0.2.5 +``` + +### 3. Wait for CI + +GitHub Actions will automatically: +- Build binaries for all platforms +- Create Debian packages +- Create a GitHub Release +- Upload all artifacts + +Monitor progress at: https://github.com/virviil/oci2git/actions + +### 4. Update Homebrew Tap + +```bash +./homebrew/update-formula.sh 0.2.5 + +# Copy to tap repository +cp homebrew/oci2git.rb ../homebrew-oci2git/Formula/ +cd ../homebrew-oci2git +git add Formula/oci2git.rb +git commit -m "Update oci2git to 0.2.5" +git push +``` + +### 5. Update AUR Package + +```bash +cd aur +./update-aur.sh 0.2.5 + +# Copy to AUR repository +cp PKGBUILD .SRCINFO ~/oci2git-bin/ +cd ~/oci2git-bin +git add PKGBUILD .SRCINFO +git commit -m "Update to 0.2.5" +git push +``` + +## Verification + +After release, verify installations work: + +```bash +# Homebrew +brew upgrade oci2git +oci2git --version + +# Debian +wget https://github.com/virviil/oci2git/releases/download/v0.2.5/oci2git_0.2.5_amd64.deb +sudo dpkg -i oci2git_0.2.5_amd64.deb +oci2git --version + +# AUR +yay -Syu oci2git-bin +oci2git --version +``` + +## Troubleshooting + +### GitHub Actions Failed + +1. Check the workflow logs +2. Re-run the failed jobs if it's a transient issue +3. Delete the tag and release if you need to fix code: + ```bash + git tag -d v0.2.5 + git push origin :v0.2.5 + # Fix the issue, then repeat steps 1-3 + ``` + +### Homebrew Formula Not Working + +Test locally: +```bash +brew install --build-from-source ./homebrew/oci2git.rb +``` + +### AUR Package Not Building + +Test locally: +```bash +cd aur +makepkg -si +``` + +## First-Time Setup + +### Homebrew Tap + +Create the tap repository once: + +```bash +gh repo create homebrew-oci2git --public +git clone https://github.com/virviil/homebrew-oci2git.git +cd homebrew-oci2git +mkdir -p Formula +# Then follow step 4 above for each release +``` + +### AUR + +Set up AUR access once: + +```bash +# Generate SSH key and add to AUR account +ssh-keygen -t ed25519 +# Add public key to https://aur.archlinux.org/account/ + +# Clone AUR repository +git clone ssh://aur@aur.archlinux.org/oci2git-bin.git +# Then follow step 5 above for each release +``` + +## Release Artifacts + +Each release creates: + +- `oci2git-linux-x86_64.tar.gz` + `.sha256` +- `oci2git-linux-aarch64.tar.gz` + `.sha256` +- `oci2git-darwin-x86_64.tar.gz` + `.sha256` +- `oci2git-darwin-aarch64.tar.gz` + `.sha256` +- `oci2git-windows-x86_64.exe.tar.gz` + `.sha256` +- `oci2git_VERSION_amd64.deb` +- `oci2git_VERSION_arm64.deb` +- `debian-checksums.txt` + +## Support + +For issues: https://github.com/virviil/oci2git/issues diff --git a/aur/.SRCINFO b/aur/.SRCINFO new file mode 100644 index 0000000..e0fab5c --- /dev/null +++ b/aur/.SRCINFO @@ -0,0 +1,16 @@ +pkgbase = oci2git-bin + pkgdesc = A tool to convert OCI images to Git repositories + pkgver = 0.2.4 + pkgrel = 1 + url = https://github.com/virviil/oci2git + arch = x86_64 + arch = aarch64 + license = MIT + provides = oci2git + conflicts = oci2git + source_x86_64 = https://github.com/virviil/oci2git/releases/download/v0.2.4/oci2git-linux-x86_64.tar.gz + sha256sums_x86_64 = REPLACE_WITH_X86_64_SHA256 + source_aarch64 = https://github.com/virviil/oci2git/releases/download/v0.2.4/oci2git-linux-aarch64.tar.gz + sha256sums_aarch64 = REPLACE_WITH_AARCH64_SHA256 + +pkgname = oci2git-bin diff --git a/aur/PKGBUILD b/aur/PKGBUILD new file mode 100644 index 0000000..aebb7f8 --- /dev/null +++ b/aur/PKGBUILD @@ -0,0 +1,24 @@ +# Maintainer: Dmitry Rubinstein +pkgname=oci2git-bin +pkgver=0.2.4 +pkgrel=1 +pkgdesc="A tool to convert OCI images to Git repositories" +arch=('x86_64' 'aarch64') +url="https://github.com/virviil/oci2git" +license=('MIT') +provides=('oci2git') +conflicts=('oci2git') +source_x86_64=("${url}/releases/download/v${pkgver}/oci2git-linux-x86_64.tar.gz") +source_aarch64=("${url}/releases/download/v${pkgver}/oci2git-linux-aarch64.tar.gz") +sha256sums_x86_64=('REPLACE_WITH_X86_64_SHA256') +sha256sums_aarch64=('REPLACE_WITH_AARCH64_SHA256') + +package() { + cd "$srcdir" + + if [ "$CARCH" = "x86_64" ]; then + install -Dm755 oci2git-linux-x86_64 "$pkgdir/usr/bin/oci2git" + elif [ "$CARCH" = "aarch64" ]; then + install -Dm755 oci2git-linux-aarch64 "$pkgdir/usr/bin/oci2git" + fi +} diff --git a/aur/update-aur.sh b/aur/update-aur.sh new file mode 100755 index 0000000..de8f18e --- /dev/null +++ b/aur/update-aur.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Script to update AUR package files with release checksums +# Usage: ./update-aur.sh + +set -e + +VERSION=${1:-$(cat ../Cargo.toml | grep "^version" | sed 's/version = "\(.*\)"/\1/')} + +echo "Updating AUR package for version $VERSION" + +# Download checksums from GitHub release +REPO="virviil/oci2git" +RELEASE_URL="https://github.com/${REPO}/releases/download/v${VERSION}" + +# Function to get SHA256 from GitHub release +get_sha256() { + local filename=$1 + curl -sL "${RELEASE_URL}/${filename}.sha256" | awk '{print $1}' +} + +echo "Downloading checksums..." +SHA256_LINUX_X86_64=$(get_sha256 "oci2git-linux-x86_64.tar.gz") +SHA256_LINUX_AARCH64=$(get_sha256 "oci2git-linux-aarch64.tar.gz") + +echo "Linux x86_64: $SHA256_LINUX_X86_64" +echo "Linux aarch64: $SHA256_LINUX_AARCH64" + +# Update PKGBUILD +PKGBUILD_FILE="PKGBUILD" + +cp "$PKGBUILD_FILE" "${PKGBUILD_FILE}.bak" + +# Update version +sed -i.tmp "s/pkgver=.*/pkgver=${VERSION}/" "$PKGBUILD_FILE" + +# Update SHA256 checksums +sed -i.tmp "s/sha256sums_x86_64=.*/sha256sums_x86_64=('${SHA256_LINUX_X86_64}')/" "$PKGBUILD_FILE" +sed -i.tmp "s/sha256sums_aarch64=.*/sha256sums_aarch64=('${SHA256_LINUX_AARCH64}')/" "$PKGBUILD_FILE" + +rm -f "${PKGBUILD_FILE}.tmp" + +# Generate .SRCINFO using makepkg +if command -v makepkg &> /dev/null; then + echo "Generating .SRCINFO..." + makepkg --printsrcinfo > .SRCINFO +else + echo "Warning: makepkg not found. Please generate .SRCINFO manually with: makepkg --printsrcinfo > .SRCINFO" + + # Manual update of .SRCINFO if makepkg is not available + sed -i.tmp "s/pkgver = .*/pkgver = ${VERSION}/" ".SRCINFO" + sed -i.tmp "s|source_x86_64 = .*|source_x86_64 = https://github.com/${REPO}/releases/download/v${VERSION}/oci2git-linux-x86_64.tar.gz|" ".SRCINFO" + sed -i.tmp "s|source_aarch64 = .*|source_aarch64 = https://github.com/${REPO}/releases/download/v${VERSION}/oci2git-linux-aarch64.tar.gz|" ".SRCINFO" + sed -i.tmp "s/sha256sums_x86_64 = .*/sha256sums_x86_64 = ${SHA256_LINUX_X86_64}/" ".SRCINFO" + sed -i.tmp "s/sha256sums_aarch64 = .*/sha256sums_aarch64 = ${SHA256_LINUX_AARCH64}/" ".SRCINFO" + rm -f ".SRCINFO.tmp" +fi + +echo "AUR package files updated successfully!" +echo "" +echo "Next steps to publish to AUR:" +echo " 1. Clone the AUR repository: git clone ssh://aur@aur.archlinux.org/oci2git-bin.git" +echo " 2. Copy PKGBUILD and .SRCINFO to the cloned repository" +echo " 3. Test the build: makepkg -si" +echo " 4. Commit and push: git add . && git commit -m 'Update to v${VERSION}' && git push" +echo "" +echo "Users can then install with:" +echo " yay -S oci2git-bin" +echo " # or" +echo " paru -S oci2git-bin" diff --git a/homebrew/oci2git.rb b/homebrew/oci2git.rb new file mode 100644 index 0000000..431b604 --- /dev/null +++ b/homebrew/oci2git.rb @@ -0,0 +1,46 @@ +class Oci2git < Formula + desc "A tool to convert OCI images to Git repositories" + homepage "https://github.com/virviil/oci2git" + version "0.2.4" + license "MIT" + + on_macos do + if Hardware::CPU.arm? + url "https://github.com/virviil/oci2git/releases/download/v#{version}/oci2git-darwin-aarch64.tar.gz" + sha256 "REPLACE_WITH_ARM64_MACOS_SHA256" + else + url "https://github.com/virviil/oci2git/releases/download/v#{version}/oci2git-darwin-x86_64.tar.gz" + sha256 "REPLACE_WITH_X86_64_MACOS_SHA256" + end + end + + on_linux do + if Hardware::CPU.arm? && Hardware::CPU.is_64_bit? + url "https://github.com/virviil/oci2git/releases/download/v#{version}/oci2git-linux-aarch64.tar.gz" + sha256 "REPLACE_WITH_ARM64_LINUX_SHA256" + else + url "https://github.com/virviil/oci2git/releases/download/v#{version}/oci2git-linux-x86_64.tar.gz" + sha256 "REPLACE_WITH_X86_64_LINUX_SHA256" + end + end + + def install + if OS.mac? + if Hardware::CPU.arm? + bin.install "oci2git-darwin-aarch64" => "oci2git" + else + bin.install "oci2git-darwin-x86_64" => "oci2git" + end + else + if Hardware::CPU.arm? + bin.install "oci2git-linux-aarch64" => "oci2git" + else + bin.install "oci2git-linux-x86_64" => "oci2git" + end + end + end + + test do + system "#{bin}/oci2git", "--help" + end +end diff --git a/homebrew/update-formula.sh b/homebrew/update-formula.sh new file mode 100755 index 0000000..1db7bc5 --- /dev/null +++ b/homebrew/update-formula.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Script to update Homebrew formula with release checksums +# Usage: ./update-formula.sh + +set -e + +VERSION=${1:-$(cat Cargo.toml | grep "^version" | sed 's/version = "\(.*\)"/\1/')} + +echo "Updating Homebrew formula for version $VERSION" + +# Download checksums from GitHub release +REPO="virviil/oci2git" +RELEASE_URL="https://github.com/${REPO}/releases/download/v${VERSION}" + +# Function to get SHA256 from GitHub release +get_sha256() { + local filename=$1 + curl -sL "${RELEASE_URL}/${filename}.sha256" | awk '{print $1}' +} + +echo "Downloading checksums..." +SHA256_DARWIN_ARM64=$(get_sha256 "oci2git-darwin-aarch64.tar.gz") +SHA256_DARWIN_X86_64=$(get_sha256 "oci2git-darwin-x86_64.tar.gz") +SHA256_LINUX_ARM64=$(get_sha256 "oci2git-linux-aarch64.tar.gz") +SHA256_LINUX_X86_64=$(get_sha256 "oci2git-linux-x86_64.tar.gz") + +echo "Darwin ARM64: $SHA256_DARWIN_ARM64" +echo "Darwin x86_64: $SHA256_DARWIN_X86_64" +echo "Linux ARM64: $SHA256_LINUX_ARM64" +echo "Linux x86_64: $SHA256_LINUX_X86_64" + +# Update formula file +FORMULA_FILE="homebrew/oci2git.rb" + +cp "$FORMULA_FILE" "${FORMULA_FILE}.bak" + +# Update version +sed -i.tmp "s/version \".*\"/version \"${VERSION}\"/" "$FORMULA_FILE" + +# Update SHA256 checksums +sed -i.tmp "s/REPLACE_WITH_ARM64_MACOS_SHA256/${SHA256_DARWIN_ARM64}/" "$FORMULA_FILE" +sed -i.tmp "s/REPLACE_WITH_X86_64_MACOS_SHA256/${SHA256_DARWIN_X86_64}/" "$FORMULA_FILE" +sed -i.tmp "s/REPLACE_WITH_ARM64_LINUX_SHA256/${SHA256_LINUX_ARM64}/" "$FORMULA_FILE" +sed -i.tmp "s/REPLACE_WITH_X86_64_LINUX_SHA256/${SHA256_LINUX_X86_64}/" "$FORMULA_FILE" + +rm -f "${FORMULA_FILE}.tmp" + +echo "Formula updated successfully!" +echo "Please review the changes and commit to your homebrew tap repository:" +echo " 1. Create a repository named 'homebrew-oci2git' on GitHub" +echo " 2. Copy the updated formula to Formula/oci2git.rb in that repository" +echo " 3. Commit and push" +echo "" +echo "Users can then install with:" +echo " brew tap virviil/oci2git" +echo " brew install oci2git" From ba2d950375be490bc7dd0d1d9cb785b4b3031f96 Mon Sep 17 00:00:00 2001 From: Dmitry Rubinstein Date: Mon, 22 Dec 2025 11:01:15 +0200 Subject: [PATCH 3/4] Fix release pipeline --- .github/workflows/release.yml | 169 ++++++++++++++-------------------- src/git.rs | 2 +- src/main.rs | 6 +- src/sources/docker.rs | 4 +- src/tar_extractor.rs | 4 +- 5 files changed, 79 insertions(+), 106 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index da8762d..92ccad3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,31 +11,8 @@ env: RUST_BACKTRACE: 1 jobs: - create-release: - name: Create Release - runs-on: ubuntu-latest - outputs: - upload_url: ${{ steps.create_release.outputs.upload_url }} - version: ${{ steps.get_version.outputs.version }} - steps: - - name: Get version from tag - id: get_version - run: echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - draft: false - prerelease: false - build-release: name: Build Release - needs: create-release strategy: fail-fast: false matrix: @@ -44,16 +21,16 @@ jobs: os: ubuntu-latest asset_name: oci2git-linux-x86_64 - target: aarch64-unknown-linux-gnu - os: ubuntu-latest + os: ubuntu-24.04-arm asset_name: oci2git-linux-aarch64 - target: x86_64-apple-darwin - os: macos-13 + os: macos-15-intel asset_name: oci2git-darwin-x86_64 - target: aarch64-apple-darwin - os: macos-14 + os: macos-latest asset_name: oci2git-darwin-aarch64 - - target: x86_64-pc-windows-gnu - os: ubuntu-latest + - target: x86_64-pc-windows-msvc + os: windows-latest asset_name: oci2git-windows-x86_64.exe runs-on: ${{ matrix.os }} @@ -67,64 +44,40 @@ jobs: with: targets: ${{ matrix.target }} - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - sudo apt-get install -y gcc-aarch64-linux-gnu - - - name: Install cross-compilation tools (Windows) - if: matrix.target == 'x86_64-pc-windows-gnu' - run: | - sudo apt-get update - sudo apt-get install -y gcc-mingw-w64-x86-64 - - name: Build release binary run: cargo build --release --target ${{ matrix.target }} - env: - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc - CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER: x86_64-w64-mingw32-gcc - name: Strip binary (Unix) - if: matrix.os != 'windows-latest' && matrix.target != 'aarch64-unknown-linux-gnu' + if: runner.os != 'Windows' run: strip target/${{ matrix.target }}/release/oci2git - - name: Prepare binary + - name: Prepare binary (Unix) + if: runner.os != 'Windows' run: | mkdir -p dist - if [[ "${{ matrix.target }}" == *"windows"* ]]; then - cp target/${{ matrix.target }}/release/oci2git.exe dist/${{ matrix.asset_name }} - else - cp target/${{ matrix.target }}/release/oci2git dist/${{ matrix.asset_name }} - fi + cp target/${{ matrix.target }}/release/oci2git dist/${{ matrix.asset_name }} - - name: Create tarball + - name: Prepare binary (Windows) + if: runner.os == 'Windows' + run: | + mkdir dist + cp target/${{ matrix.target }}/release/oci2git.exe dist/${{ matrix.asset_name }} + + - name: Create tarball (Unix) + if: runner.os != 'Windows' run: | cd dist tar czf ${{ matrix.asset_name }}.tar.gz ${{ matrix.asset_name }} sha256sum ${{ matrix.asset_name }}.tar.gz > ${{ matrix.asset_name }}.tar.gz.sha256 - - name: Upload Release Asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }}.tar.gz - asset_name: ${{ matrix.asset_name }}.tar.gz - asset_content_type: application/gzip - - - name: Upload Checksum - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }}.tar.gz.sha256 - asset_name: ${{ matrix.asset_name }}.tar.gz.sha256 - asset_content_type: text/plain + - name: Create tarball (Windows) + if: runner.os == 'Windows' + run: | + cd dist + tar czf ${{ matrix.asset_name }}.tar.gz ${{ matrix.asset_name }} + certutil -hashfile ${{ matrix.asset_name }}.tar.gz SHA256 | findstr /v ":" > ${{ matrix.asset_name }}.tar.gz.sha256 - - name: Save artifacts for packaging + - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: binary-${{ matrix.target }} @@ -132,13 +85,23 @@ jobs: build-deb: name: Build Debian Package - needs: [create-release, build-release] + needs: build-release runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 + - name: Get version from tag + id: get_version + run: | + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + VERSION="${GITHUB_REF#refs/tags/v}" + else + VERSION="$(grep '^version' Cargo.toml | head -1 | sed 's/version = "\(.*\)"/\1/')" + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Download Linux x86_64 binary uses: actions/download-artifact@v4 with: @@ -156,7 +119,7 @@ jobs: - name: Build Debian package (amd64) run: | - VERSION="${{ needs.create-release.outputs.version }}" + VERSION="${{ steps.get_version.outputs.version }}" ARCH="amd64" PKG_NAME="oci2git_${VERSION}_${ARCH}" @@ -190,7 +153,7 @@ jobs: - name: Build Debian package (arm64) run: | - VERSION="${{ needs.create-release.outputs.version }}" + VERSION="${{ steps.get_version.outputs.version }}" ARCH="arm64" PKG_NAME="oci2git_${VERSION}_${ARCH}" @@ -224,34 +187,44 @@ jobs: - name: Generate checksums run: | - sha256sum oci2git_*.deb > checksums.txt + sha256sum oci2git_*.deb > debian-checksums.txt - - name: Upload Debian package (amd64) - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Debian artifacts + uses: actions/upload-artifact@v4 with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ./oci2git_${{ needs.create-release.outputs.version }}_amd64.deb - asset_name: oci2git_${{ needs.create-release.outputs.version }}_amd64.deb - asset_content_type: application/vnd.debian.binary-package + name: debian-packages + path: | + oci2git_*.deb + debian-checksums.txt - - name: Upload Debian package (arm64) - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + create-release: + name: Create Release + needs: [build-release, build-deb] + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ./oci2git_${{ needs.create-release.outputs.version }}_arm64.deb - asset_name: oci2git_${{ needs.create-release.outputs.version }}_arm64.deb - asset_content_type: application/vnd.debian.binary-package + path: artifacts - - name: Upload checksums - uses: actions/upload-release-asset@v1 + - name: Prepare release assets + run: | + mkdir -p release-assets + find artifacts -type f \( -name "*.tar.gz" -o -name "*.sha256" -o -name "*.deb" -o -name "*checksums.txt" \) -exec cp {} release-assets/ \; + ls -lh release-assets/ + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + files: release-assets/* + generate_release_notes: true + draft: false + prerelease: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ./checksums.txt - asset_name: debian-checksums.txt - asset_content_type: text/plain diff --git a/src/git.rs b/src/git.rs index 95135cb..c2a7af5 100644 --- a/src/git.rs +++ b/src/git.rs @@ -340,7 +340,7 @@ impl GitRepo { } None => { // File doesn't exist in this commit - Err(anyhow::anyhow!("File '{}' not found in commit", file_path)) + Err(anyhow::anyhow!("File '{file_path}' not found in commit")) } } } diff --git a/src/main.rs b/src/main.rs index b7ea904..72244f3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -83,7 +83,7 @@ fn main() -> Result<()> { notifier.debug("Initializing Docker source"); let source = DockerSource::new() - .map_err(|e| anyhow!("Failed to initialize Docker source: {}", e))?; + .map_err(|e| anyhow!("Failed to initialize Docker source: {e}"))?; let processor = ImageProcessor::new(source, notifier); processor.convert(&cli.image, &cli.output)?; @@ -96,7 +96,7 @@ fn main() -> Result<()> { notifier.debug("Initializing nerdctl source"); let source = NerdctlSource::new() - .map_err(|e| anyhow!("Failed to initialize nerdctl source: {}", e))?; + .map_err(|e| anyhow!("Failed to initialize nerdctl source: {e}"))?; let processor = ImageProcessor::new(source, notifier); processor.convert(&cli.image, &cli.output)?; @@ -109,7 +109,7 @@ fn main() -> Result<()> { notifier.debug("Initializing tar source"); let source = - TarSource::new().map_err(|e| anyhow!("Failed to initialize tar source: {}", e))?; + TarSource::new().map_err(|e| anyhow!("Failed to initialize tar source: {e}"))?; let processor = ImageProcessor::new(source, notifier); processor.convert(&cli.image, &cli.output)?; diff --git a/src/sources/docker.rs b/src/sources/docker.rs index 682a7e2..037a270 100644 --- a/src/sources/docker.rs +++ b/src/sources/docker.rs @@ -32,7 +32,7 @@ impl DockerSource { if !output.status.success() { let error = String::from_utf8_lossy(&output.stderr); - return Err(anyhow!("Docker command failed: {}", error)); + return Err(anyhow!("Docker command failed: {error}")); } let stdout = String::from_utf8_lossy(&output.stdout).to_string(); @@ -49,7 +49,7 @@ impl DockerSource { if !output.status.success() { let error = String::from_utf8_lossy(&output.stderr); - return Err(anyhow!("Docker pull failed: {}", error)); + return Err(anyhow!("Docker pull failed: {error}")); } notifier.info(&format!( diff --git a/src/tar_extractor.rs b/src/tar_extractor.rs index 7547af2..e8bf543 100644 --- a/src/tar_extractor.rs +++ b/src/tar_extractor.rs @@ -100,8 +100,8 @@ pub fn extract_tar(tar_path: &Path, extract_dir: &Path) -> Result<()> { // First pass: extract all regular files, directories, and symlinks // Store hardlinks and failed symlinks for second pass - let mut pending_hardlinks = Vec::new(); - let mut pending_symlinks = Vec::new(); + let mut pending_hardlinks: Vec = Vec::new(); + let mut pending_symlinks: Vec = Vec::new(); for entry_result in archive.entries()? { let mut entry = entry_result.context("Failed to read tar entry")?; From f5781064960b65446c44703b0752753dc29c696e Mon Sep 17 00:00:00 2001 From: Dmitry Rubinstein Date: Mon, 22 Dec 2025 12:07:37 +0200 Subject: [PATCH 4/4] Bump version to 0.2.5 # Conflicts: # Cargo.lock # Cargo.toml --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 201b2d9..343635f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1057,7 +1057,7 @@ dependencies = [ [[package]] name = "oci2git" -version = "0.2.3" +version = "0.2.5" dependencies = [ "anyhow", "atty", diff --git a/Cargo.toml b/Cargo.toml index 2c0c2b4..f4cbb79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oci2git" -version = "0.2.3" +version = "0.2.5" edition = "2021" authors = ["Dmitry Rubinstein "] description = "A tool to convert OCI images to Git repositories"