From 8b574d4342a1ae71116ca86fc5742fc46432ffba Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Sat, 15 Jun 2019 10:28:34 -0700 Subject: [PATCH] Add initial filegrain-esque continuity manifest --- cmd/convert/main.go | 2 +- converter.go | 98 +++++++++++++++++++++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 24 +++++++++++ 4 files changed, 125 insertions(+), 2 deletions(-) diff --git a/cmd/convert/main.go b/cmd/convert/main.go index 83af558..c9c77d0 100644 --- a/cmd/convert/main.go +++ b/cmd/convert/main.go @@ -70,7 +70,7 @@ func Convert(ctx context.Context, ipfsCln iface.CoreAPI, ctrdCln *containerd.Cli return errors.Wrapf(err, "failed to create fetcher for %q", src) } - converter := ipcs.NewConverter(ipfsCln, contentutil.FromFetcher(fetcher)) + converter := ipcs.NewContinuityConverter(ipfsCln, contentutil.FromFetcher(fetcher)) mfstDesc, err := converter.Convert(ctx, srcDesc) if err != nil { return errors.Wrapf(err, "failed to convert %q to ipfs manifest", srcName) diff --git a/converter.go b/converter.go index b8666f6..341aec9 100644 --- a/converter.go +++ b/converter.go @@ -9,11 +9,13 @@ import ( "log" "os" + "github.com/AkihiroSuda/filegrain/continuityutil" "github.com/containerd/containerd/archive" "github.com/containerd/containerd/archive/compression" "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" + "github.com/containerd/continuity" "github.com/hinshun/ipcs/digestconv" files "github.com/ipfs/go-ipfs-files" iface "github.com/ipfs/interface-go-ipfs-core" @@ -116,6 +118,7 @@ func addFile(ctx context.Context, api iface.CoreAPI, n files.Node) (digest.Diges if err != nil { return "", errors.Wrapf(err, "failed to convert cid %q to digest", p.Cid()) } + log.Printf("Added as %q [%s]", p.Cid(), dgst) return dgst, nil } @@ -220,3 +223,98 @@ func RegularTypeFilter(header *tar.Header) (bool, error) { return false, nil } } + +type continuityConverter struct { + api iface.CoreAPI + provider content.Provider +} + +func NewContinuityConverter(api iface.CoreAPI, provider content.Provider) Converter { + return &continuityConverter{ + api: api, + provider: provider, + } +} + +func (c *continuityConverter) Convert(ctx context.Context, desc ocispec.Descriptor) (ocispec.Descriptor, error) { + mfst, err := images.Manifest(ctx, c.provider, desc, platforms.Default()) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to get manifest") + } + + rootfs, err := ioutil.TempDir("", "ipcs-rootfs") + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to create tmp root directory") + } + + for _, layer := range mfst.Layers { + ra, err := c.provider.ReaderAt(ctx, layer) + if err != nil { + return ocispec.Descriptor{}, err + } + + cr := content.NewReader(ra) + r, err := compression.DecompressStream(cr) + if err != nil { + return ocispec.Descriptor{}, err + } + + _, err = archive.Apply(ctx, rootfs, r, archive.WithFilter(func(hdr *tar.Header) (bool, error) { + return true, nil + })) + if err != nil { + r.Close() + return ocispec.Descriptor{}, err + } + r.Close() + } + + pctx, err := continuity.NewContext(rootfs) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to create continuity context") + } + + m, err := continuity.BuildManifest(pctx) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to build continuity manifest") + } + + p, err := continuity.Marshal(m) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal continuity manifest") + } + + continuityDigest, err := addFile(ctx, c.api, files.NewBytesFile(p)) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to upload manifest") + } + mfst.Layers = []ocispec.Descriptor{ + { + MediaType: continuityutil.MediaTypeManifestV0Protobuf, + Digest: continuityDigest, + Size: int64(len(p)), + }, + } + + mfst.Config.Digest, err = copyFile(ctx, c.api, c.provider, mfst.Config) + if err != nil { + return ocispec.Descriptor{}, errors.Wrapf(err, "failed to upload manifest config blob %q", mfst.Config.Digest) + } + + mfstJSON, err := json.MarshalIndent(mfst, "", " ") + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal manifest JSON") + } + + mfstDigest, err := addFile(ctx, c.api, files.NewBytesFile(mfstJSON)) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "failed to upload manifest") + } + log.Printf("Converted Manifest [%d] %s:\n%s", len(mfstJSON), mfstDigest, mfstJSON) + + return ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: mfstDigest, + Size: int64(len(mfstJSON)), + }, nil +} diff --git a/go.mod b/go.mod index d9ecd47..e5ce362 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/hinshun/ipcs go 1.12 require ( + github.com/AkihiroSuda/filegrain v0.0.0-20190607033141-66f306520d62 github.com/Microsoft/go-winio v0.4.12 // indirect github.com/Microsoft/hcsshim v0.8.6 // indirect github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect @@ -11,7 +12,7 @@ require ( github.com/containerd/cgroups v0.0.0-20190226200435-dbea6f2bd416 // indirect github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50 // indirect github.com/containerd/containerd v1.2.6 - github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 // indirect + github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect github.com/containerd/go-runc v0.0.0-20190226155025-7d11b49dc076 // indirect github.com/containerd/ttrpc v0.0.0-20190211042230-69144327078c // indirect diff --git a/go.sum b/go.sum index 37867f0..a85f5a1 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AkihiroSuda/filegrain v0.0.0-20190607033141-66f306520d62 h1:r4WMZt0dA/9DIszP7ZcmBtohmgFgkEqo6dfje9upP9E= +github.com/AkihiroSuda/filegrain v0.0.0-20190607033141-66f306520d62/go.mod h1:8sU3hhE0fWQ1+lWEVLOP9ExV/vGgMCA+Z0dsk3hD1LY= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -8,6 +10,8 @@ github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcy github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Sirupsen/logrus v0.11.5 h1:aIMrrsnipdTlAieMe7FC/iiuJ0+ELiXCT4YiVQiK9j8= +github.com/Sirupsen/logrus v0.11.5/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U= github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= @@ -32,6 +36,8 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/cheggaaa/pb v1.0.13 h1:vSXdStIZ5JH9knGF+zbK1gpOJZPQ+CvU24ETBe7LyS0= +github.com/cheggaaa/pb v1.0.13/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/containerd/aufs v0.0.0-20190114185352-f894a800659b h1:MRsS/11KfevkQGq+jwFmyNKaY//l6hvFPL5/I333MAg= github.com/containerd/aufs v0.0.0-20190114185352-f894a800659b/go.mod h1:KWTFEKnST/R6wiSG+n7rid/ZQJxi73594So04IsWRpc= @@ -45,6 +51,8 @@ github.com/containerd/containerd v1.2.6 h1:K38ZSAA9oKSrX3iFNY+4SddZ8hH1TCMCerc8N github.com/containerd/containerd v1.2.6/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M= github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 h1:PUD50EuOMkXVcpBIA/R95d56duJR9VxhwncsFbNnxW4= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/go-runc v0.0.0-20190226155025-7d11b49dc076 h1:biYGul6kwz5/Lh6UxkYQyRKPeYA/ZL5n2pdCAcYUpLA= @@ -75,6 +83,7 @@ github.com/docker/go-events v0.0.0-20170721190031-9461782956ad h1:VXIse57M5C6ezD github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 h1:X0fj836zx99zFu83v/M79DuBn84IL/Syx1SY6Y5ZEMA= github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= +github.com/docker/go-units v0.3.2/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -101,6 +110,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20170427213220-18c9bb326172/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk= @@ -125,6 +135,8 @@ github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824 h1:TF4mX7zXpeyz/xintezebSa7ZDxAGBnqDwcoobvaz2o= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= +github.com/hanwen/go-fuse v0.0.0-20170424203904-5404bf0e372d h1:06w1Xwq+3gQbm76tgdC8QL+2B6aBxSEMCRH73aSv+fw= +github.com/hanwen/go-fuse v0.0.0-20170424203904-5404bf0e372d/go.mod h1:4ZJ05v9yt5k/mcFkGvSPKJB5T8G/6nuumL63ZqlrPvI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -368,6 +380,8 @@ github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0X github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -419,8 +433,10 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0-rc0/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v0.0.0-20170501194034-c87455c1b399/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= @@ -467,6 +483,12 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cobra v0.0.0-20170501210834-69f86e6d5d7a h1:9OzjTlfCOEYy/+ivO76kJxif+t9vmOjlXZDL0cnxdvg= +github.com/spf13/cobra v0.0.0-20170501210834-69f86e6d5d7a/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v0.0.0-20170427125145-f1d95a35e132 h1:Z9FpVnwd+lo1kp/Mf5kL1XW57L+Vcu/40wC3/aFGCHY= +github.com/spf13/pflag v0.0.0-20170427125145-f1d95a35e132/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stevvooe/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:1GLiICsIP1hnDnXS8MFG17DyorbYXMTFipmAfAOrTKk= +github.com/stevvooe/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:AXZEju8Nky8hvQW5KS9REMxcmvWpBFJRqnhQ6W/7S6E= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -544,6 +566,8 @@ golang.org/x/sys v0.0.0-20190302025703-b6889370fb10 h1:xQJI9OEiErEQ++DoXOHqEpzsG golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444 h1:/d2cWp6PSamH4jDPFLyO150psQdqvtoNX8Zjg3AQ31g= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=