From 62945df1f3ba531d2afe2e0682582efc6c2dd1d8 Mon Sep 17 00:00:00 2001 From: Jonathan Boulle Date: Wed, 11 Oct 2017 16:45:38 +0200 Subject: [PATCH 1/3] *: add ability to use a shared CAS directory umoci currently always expects blobs to live within the image layout it's working on, in the aptly-named "blobs" subdirectory. However, the OCI specification [1][1] permits this directory to be empty and for implementations to look in other locations for referenced blobs. This is particularly useful when working with multiple image layouts and wanting to share blobs between them. This patch adds a `--shared-cas` flag to all umoci commands that work with image layouts. The flag expects a directory to be passed, and if set, this directory will be used for all blob operations. In the first implementation, we extend the `dirEngine` implementation of the CAS interface to accept another (optional) `sharedCasPath` field. Analogously to the CLI flag, if this field is supplied, it will be used internally by the dirEngine for all blob-related operations. [1]: https://github.com/opencontainers/image-spec/blob/7c889fafd04a893f5c5f50b7ab9963d5d64e5242/image-layout.md#blobs Signed-off-by: Jonathan Boulle --- cmd/umoci/config.go | 3 +- cmd/umoci/gc.go | 3 +- cmd/umoci/new.go | 3 +- cmd/umoci/raw-runtime-config.go | 3 +- cmd/umoci/repack.go | 3 +- cmd/umoci/stat.go | 3 +- cmd/umoci/tag.go | 9 ++-- cmd/umoci/unpack.go | 3 +- cmd/umoci/utils_ux.go | 16 +++++- mutate/mutate.go | 2 +- mutate/mutate_test.go | 8 +-- oci/cas/cas.go | 13 +++-- oci/cas/dir/cas_test.go | 22 ++++---- oci/cas/dir/dir.go | 90 ++++++++++++++++++++------------- oci/cas/dir/dir_test.go | 14 ++--- oci/casext/json_dir_test.go | 6 +-- oci/casext/refname_dir_test.go | 8 +-- oci/layer/unpack_test.go | 2 +- 18 files changed, 129 insertions(+), 82 deletions(-) diff --git a/cmd/umoci/config.go b/cmd/umoci/config.go index 32cdfa91f..66497e28a 100644 --- a/cmd/umoci/config.go +++ b/cmd/umoci/config.go @@ -123,6 +123,7 @@ func parseKV(input string) (string, string, error) { func config(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) fromName := ctx.App.Metadata["--image-tag"].(string) + sharedCasPath := ctx.String("shared-cas") // By default we clobber the old tag. tagName := fromName @@ -131,7 +132,7 @@ func config(ctx *cli.Context) error { } // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/gc.go b/cmd/umoci/gc.go index 45ad00905..586f57696 100644 --- a/cmd/umoci/gc.go +++ b/cmd/umoci/gc.go @@ -51,9 +51,10 @@ root set of references. All other blobs will be removed.`, func gc(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) + sharedCasPath := ctx.String("shared-cas") // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/new.go b/cmd/umoci/new.go index 8f6ee84b9..6bd3b6ecc 100644 --- a/cmd/umoci/new.go +++ b/cmd/umoci/new.go @@ -54,9 +54,10 @@ needing a base image to start from.`, func newImage(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) tagName := ctx.App.Metadata["--image-tag"].(string) + sharedCasPath := ctx.String("shared-cas") // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/raw-runtime-config.go b/cmd/umoci/raw-runtime-config.go index 60435ebb5..823c062c4 100644 --- a/cmd/umoci/raw-runtime-config.go +++ b/cmd/umoci/raw-runtime-config.go @@ -87,6 +87,7 @@ func rawConfig(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) fromName := ctx.App.Metadata["--image-tag"].(string) configPath := ctx.App.Metadata["config"].(string) + sharedCasPath := ctx.String("shared-cas") var meta UmociMeta meta.Version = UmociMetaVersion @@ -124,7 +125,7 @@ func rawConfig(ctx *cli.Context) error { }).Debugf("parsed mappings") // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/repack.go b/cmd/umoci/repack.go index c1b737811..950132089 100644 --- a/cmd/umoci/repack.go +++ b/cmd/umoci/repack.go @@ -99,6 +99,7 @@ func repack(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) tagName := ctx.App.Metadata["--image-tag"].(string) bundlePath := ctx.App.Metadata["bundle"].(string) + sharedCasPath := ctx.String("shared-cas") // Read the metadata first. meta, err := ReadBundleMeta(bundlePath) @@ -117,7 +118,7 @@ func repack(ctx *cli.Context) error { } // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/stat.go b/cmd/umoci/stat.go index 24c15af49..14991396a 100644 --- a/cmd/umoci/stat.go +++ b/cmd/umoci/stat.go @@ -58,9 +58,10 @@ humans to read, and might change in future versions.`, func stat(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) tagName := ctx.App.Metadata["--image-tag"].(string) + sharedCasPath := ctx.String("shared-cas") // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/tag.go b/cmd/umoci/tag.go index 484b8ecca..186472cef 100644 --- a/cmd/umoci/tag.go +++ b/cmd/umoci/tag.go @@ -60,9 +60,10 @@ func tagAdd(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) fromName := ctx.App.Metadata["--image-tag"].(string) tagName := ctx.App.Metadata["new-tag"].(string) + sharedCasPath := ctx.String("shared-cas") // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } @@ -111,9 +112,10 @@ tag to remove.`, func tagRemove(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) tagName := ctx.App.Metadata["--image-tag"].(string) + sharedCasPath := ctx.String("shared-cas") // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } @@ -148,9 +150,10 @@ line. See umoci-stat(1) to get more information about each tagged image.`, func tagList(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) + sharedCasPath := ctx.String("shared-cas") // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/unpack.go b/cmd/umoci/unpack.go index 6b34c1105..321caca27 100644 --- a/cmd/umoci/unpack.go +++ b/cmd/umoci/unpack.go @@ -83,6 +83,7 @@ func unpack(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) fromName := ctx.App.Metadata["--image-tag"].(string) bundlePath := ctx.App.Metadata["bundle"].(string) + sharedCasPath := ctx.String("shared-cas") var meta UmociMeta meta.Version = UmociMetaVersion @@ -120,7 +121,7 @@ func unpack(ctx *cli.Context) error { }).Debugf("parsed mappings") // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.Open(imagePath, sharedCasPath) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/utils_ux.go b/cmd/umoci/utils_ux.go index f46244737..1ece75ab8 100644 --- a/cmd/umoci/utils_ux.go +++ b/cmd/umoci/utils_ux.go @@ -130,12 +130,19 @@ func uxTag(cmd cli.Command) cli.Command { // tag) will be stored in ctx.Metadata["--image-path"] and // ctx.Metadata["--image-tag"] as strings (both will be nil if --image is not // specified). +// A --shared-cas flag is also added to the given cli.Command, which allows +// users to provide a different directory to be used for blob operations func uxImage(cmd cli.Command) cli.Command { cmd.Flags = append(cmd.Flags, cli.StringFlag{ Name: "image", Usage: "OCI image URI of the form 'path[:tag]'", }) + cmd.Flags = append(cmd.Flags, cli.StringFlag{ + Name: "shared-cas", + Usage: "shared directory to use for blobs (instead of internal image layout directory)", + }) + oldBefore := cmd.Before cmd.Before = func(ctx *cli.Context) error { // Verify and parse --image. @@ -181,15 +188,22 @@ func uxImage(cmd cli.Command) cli.Command { return cmd } -// uxLayout adds an --layout flag to the given cli.Command as well as adding +// uxLayout adds a --layout flag to the given cli.Command as well as adding // relevant validation logic to the .Before of the command. The value is stored // in ctx.App.Metadata["--image-path"] as a string (or nil --layout was not set). +// A --shared-cas flag is also added to the given cli.Command, which allows +// users to provide a different directory to be used for blob operations func uxLayout(cmd cli.Command) cli.Command { cmd.Flags = append(cmd.Flags, cli.StringFlag{ Name: "layout", Usage: "path to an OCI image layout", }) + cmd.Flags = append(cmd.Flags, cli.StringFlag{ + Name: "shared-cas", + Usage: "shared directory to use for blobs (instead of internal image layout directory)", + }) + oldBefore := cmd.Before cmd.Before = func(ctx *cli.Context) error { // Verify and parse --layout. diff --git a/mutate/mutate.go b/mutate/mutate.go index 8f9fa23b7..660e2f4de 100644 --- a/mutate/mutate.go +++ b/mutate/mutate.go @@ -223,7 +223,7 @@ func (m *Mutator) add(ctx context.Context, reader io.Reader) (digest.Digest, int return "", -1, errors.Wrap(err, "getting cache failed") } - diffidDigester := cas.BlobAlgorithm.Digester() + diffidDigester := cas.DefaultBlobAlgorithm.Digester() hashReader := io.TeeReader(reader, diffidDigester.Hash()) pipeReader, pipeWriter := io.Pipe() diff --git a/mutate/mutate_test.go b/mutate/mutate_test.go index 49f5b7424..cc8d541a4 100644 --- a/mutate/mutate_test.go +++ b/mutate/mutate_test.go @@ -46,12 +46,12 @@ const ( ) func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) { - dir = filepath.Join(dir, "image") - if err := casdir.Create(dir); err != nil { + image := filepath.Join(dir, "image") + if err := casdir.Create(image); err != nil { t.Fatal(err) } - engine, err := casdir.Open(dir) + engine, err := casdir.Open(image, "") if err != nil { t.Fatal(err) } @@ -71,7 +71,7 @@ func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) { tw.Close() // Push the base layer. - diffidDigester := cas.BlobAlgorithm.Digester() + diffidDigester := cas.DefaultBlobAlgorithm.Digester() hashReader := io.TeeReader(&buffer, diffidDigester.Hash()) layerDigest, layerSize, err := engine.PutBlob(context.Background(), hashReader) if err != nil { diff --git a/oci/cas/cas.go b/oci/cas/cas.go index 423dfee40..1e1c2bec2 100644 --- a/oci/cas/cas.go +++ b/oci/cas/cas.go @@ -32,13 +32,18 @@ import ( ) const ( - // BlobAlgorithm is the name of the only supported digest algorithm for blobs. - // FIXME: We can make this a list. - BlobAlgorithm = digest.SHA256 + // DefaultBlobAlgorithm is the default supported digest algorithm for blobs + DefaultBlobAlgorithm = digest.SHA256 ) -// Exposed errors. var ( + // BlobAlgorithms contains the supported digest algorithms for blobs + BlobAlgorithms = []digest.Algorithm{ + digest.SHA256, + } + + // Exposed errors + // ErrNotExist is effectively an implementation-neutral version of // os.ErrNotExist. ErrNotExist = fmt.Errorf("no such blob or index") diff --git a/oci/cas/dir/cas_test.go b/oci/cas/dir/cas_test.go index 806158c4e..defc934de 100644 --- a/oci/cas/dir/cas_test.go +++ b/oci/cas/dir/cas_test.go @@ -47,7 +47,7 @@ func TestCreateLayout(t *testing.T) { t.Fatalf("unexpected error creating image: %+v", err) } - engine, err := Open(image) + engine, err := Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } @@ -85,7 +85,7 @@ func TestEngineBlob(t *testing.T) { t.Fatalf("unexpected error creating image: %+v", err) } - engine, err := Open(image) + engine, err := Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } @@ -98,7 +98,7 @@ func TestEngineBlob(t *testing.T) { {[]byte("some blob")}, {[]byte("another blob")}, } { - digester := cas.BlobAlgorithm.Digester() + digester := cas.DefaultBlobAlgorithm.Digester() if _, err := io.Copy(digester.Hash(), bytes.NewReader(test.bytes)); err != nil { t.Fatalf("could not hash bytes: %+v", err) } @@ -172,7 +172,7 @@ func TestEngineValidate(t *testing.T) { if err != nil { t.Fatal(err) } - engine, err = Open(image) + engine, err = Open(image, "") if err == nil { t.Errorf("expected to get an error") engine.Close() @@ -186,7 +186,7 @@ func TestEngineValidate(t *testing.T) { if err := ioutil.WriteFile(filepath.Join(image, layoutFile), []byte("invalid JSON"), 0644); err != nil { t.Fatal(err) } - engine, err = Open(image) + engine, err = Open(image, "") if err == nil { t.Errorf("expected to get an error") engine.Close() @@ -200,7 +200,7 @@ func TestEngineValidate(t *testing.T) { if err := ioutil.WriteFile(filepath.Join(image, layoutFile), []byte("{}"), 0644); err != nil { t.Fatal(err) } - engine, err = Open(image) + engine, err = Open(image, "") if err == nil { t.Errorf("expected to get an error") engine.Close() @@ -220,7 +220,7 @@ func TestEngineValidate(t *testing.T) { if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil { t.Fatalf("unexpected error deleting blobdir: %+v", err) } - engine, err = Open(image) + engine, err = Open(image, "") if err == nil { t.Errorf("expected to get an error") engine.Close() @@ -243,7 +243,7 @@ func TestEngineValidate(t *testing.T) { if err := ioutil.WriteFile(filepath.Join(image, blobDirectory), []byte(""), 0755); err != nil { t.Fatal(err) } - engine, err = Open(image) + engine, err = Open(image, "") if err == nil { t.Errorf("expected to get an error") engine.Close() @@ -263,7 +263,7 @@ func TestEngineValidate(t *testing.T) { if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil { t.Fatalf("unexpected error deleting index: %+v", err) } - engine, err = Open(image) + engine, err = Open(image, "") if err == nil { t.Errorf("expected to get an error") engine.Close() @@ -286,7 +286,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Mkdir(filepath.Join(image, indexFile), 0755); err != nil { t.Fatal(err) } - engine, err = Open(image) + engine, err = Open(image, "") if err == nil { t.Errorf("expected to get an error") engine.Close() @@ -294,7 +294,7 @@ func TestEngineValidate(t *testing.T) { // No such directory. image = filepath.Join(root, "non-exist") - engine, err = Open(image) + engine, err = Open(image, "") if err == nil { t.Errorf("expected to get an error") engine.Close() diff --git a/oci/cas/dir/dir.go b/oci/cas/dir/dir.go index 8b83e4e57..c696dd89e 100644 --- a/oci/cas/dir/dir.go +++ b/oci/cas/dir/dir.go @@ -26,7 +26,7 @@ import ( "github.com/apex/log" "github.com/openSUSE/umoci/oci/cas" - "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" imeta "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -53,9 +53,15 @@ const ( layoutFile = "oci-layout" ) -// blobPath returns the path to a blob given its digest, relative to the root -// of the OCI image. The digest must be of the form algorithm:hex. -func blobPath(digest digest.Digest) (string, error) { +type dirEngine struct { + imagePath string + temp string + tempFile *os.File + sharedCasPath string +} + +// blobPath returns the full path to a blob given its digest. +func (e *dirEngine) blobPath(digest digest.Digest) (string, error) { if err := digest.Validate(); err != nil { return "", errors.Wrapf(err, "invalid digest: %q", digest) } @@ -63,22 +69,19 @@ func blobPath(digest digest.Digest) (string, error) { algo := digest.Algorithm() hash := digest.Hex() - if algo != cas.BlobAlgorithm { + if algo != cas.DefaultBlobAlgorithm { return "", errors.Errorf("unsupported algorithm: %q", algo) } - return filepath.Join(blobDirectory, algo.String(), hash), nil -} - -type dirEngine struct { - path string - temp string - tempFile *os.File + if e.sharedCasPath != "" { + return filepath.Join(e.sharedCasPath, algo.String(), hash), nil + } + return filepath.Join(e.imagePath, blobDirectory, algo.String(), hash), nil } func (e *dirEngine) ensureTempDir() error { if e.temp == "" { - tempDir, err := ioutil.TempDir(e.path, ".umoci-") + tempDir, err := ioutil.TempDir(e.imagePath, ".umoci-") if err != nil { return errors.Wrap(err, "create tempdir") } @@ -100,9 +103,9 @@ func (e *dirEngine) ensureTempDir() error { return nil } -// verify ensures that the image is valid. +// validate ensures that the image is valid. func (e *dirEngine) validate() error { - content, err := ioutil.ReadFile(filepath.Join(e.path, layoutFile)) + content, err := ioutil.ReadFile(filepath.Join(e.imagePath, layoutFile)) if err != nil { if os.IsNotExist(err) { err = cas.ErrInvalid @@ -122,10 +125,10 @@ func (e *dirEngine) validate() error { } // Check that "blobs" and "index.json" exist in the image. - // FIXME: We also should check that blobs *only* contains a cas.BlobAlgorithm + // FIXME: We also should check that blobs *only* contains a cas.DefaultBlobAlgorithm // directory (with no subdirectories) and that refs *only* contains // files (optionally also making sure they're all JSON descriptors). - if fi, err := os.Stat(filepath.Join(e.path, blobDirectory)); err != nil { + if fi, err := os.Stat(filepath.Join(e.imagePath, blobDirectory)); err != nil { if os.IsNotExist(err) { err = cas.ErrInvalid } @@ -134,7 +137,7 @@ func (e *dirEngine) validate() error { return errors.Wrap(cas.ErrInvalid, "blobdir is not a directory") } - if fi, err := os.Stat(filepath.Join(e.path, indexFile)); err != nil { + if fi, err := os.Stat(filepath.Join(e.imagePath, indexFile)); err != nil { if os.IsNotExist(err) { err = cas.ErrInvalid } @@ -154,7 +157,7 @@ func (e *dirEngine) PutBlob(ctx context.Context, reader io.Reader) (digest.Diges return "", -1, errors.Wrap(err, "ensure tempdir") } - digester := cas.BlobAlgorithm.Digester() + digester := cas.DefaultBlobAlgorithm.Digester() // We copy this into a temporary file because we need to get the blob hash, // but also to avoid half-writing an invalid blob. @@ -173,13 +176,12 @@ func (e *dirEngine) PutBlob(ctx context.Context, reader io.Reader) (digest.Diges fh.Close() // Get the digest. - path, err := blobPath(digester.Digest()) + path, err := e.blobPath(digester.Digest()) if err != nil { return "", -1, errors.Wrap(err, "compute blob name") } // Move the blob to its correct path. - path = filepath.Join(e.path, path) if err := os.Rename(tempPath, path); err != nil { return "", -1, errors.Wrap(err, "rename temporary blob") } @@ -190,11 +192,11 @@ func (e *dirEngine) PutBlob(ctx context.Context, reader io.Reader) (digest.Diges // GetBlob returns a reader for retrieving a blob from the image, which the // caller must Close(). Returns os.ErrNotExist if the digest is not found. func (e *dirEngine) GetBlob(ctx context.Context, digest digest.Digest) (io.ReadCloser, error) { - path, err := blobPath(digest) + path, err := e.blobPath(digest) if err != nil { return nil, errors.Wrap(err, "compute blob path") } - fh, err := os.Open(filepath.Join(e.path, path)) + fh, err := os.Open(path) return fh, errors.Wrap(err, "open blob") } @@ -223,7 +225,7 @@ func (e *dirEngine) PutIndex(ctx context.Context, index ispec.Index) error { fh.Close() // Move the blob to its correct path. - path := filepath.Join(e.path, indexFile) + path := filepath.Join(e.imagePath, indexFile) if err := os.Rename(tempPath, path); err != nil { return errors.Wrap(err, "rename temporary index") } @@ -240,7 +242,7 @@ func (e *dirEngine) PutIndex(ctx context.Context, index ispec.Index) error { // that implements various reference resolution functions that should work for // most users. func (e *dirEngine) GetIndex(ctx context.Context) (ispec.Index, error) { - content, err := ioutil.ReadFile(filepath.Join(e.path, indexFile)) + content, err := ioutil.ReadFile(filepath.Join(e.imagePath, indexFile)) if err != nil { if os.IsNotExist(err) { err = cas.ErrInvalid @@ -260,12 +262,12 @@ func (e *dirEngine) GetIndex(ctx context.Context) (ispec.Index, error) { // error means "the content is not in the store" without implying "because // of this DeleteBlob() call". func (e *dirEngine) DeleteBlob(ctx context.Context, digest digest.Digest) error { - path, err := blobPath(digest) + path, err := e.blobPath(digest) if err != nil { return errors.Wrap(err, "compute blob path") } - err = os.Remove(filepath.Join(e.path, path)) + err = os.Remove(path) if err != nil && !os.IsNotExist(err) { return errors.Wrap(err, "remove blob") } @@ -275,7 +277,7 @@ func (e *dirEngine) DeleteBlob(ctx context.Context, digest digest.Digest) error // ListBlobs returns the set of blob digests stored in the image. func (e *dirEngine) ListBlobs(ctx context.Context) ([]digest.Digest, error) { digests := []digest.Digest{} - blobDir := filepath.Join(e.path, blobDirectory, cas.BlobAlgorithm.String()) + blobDir := filepath.Join(e.imagePath, blobDirectory, cas.DefaultBlobAlgorithm.String()) if err := filepath.Walk(blobDir, func(path string, _ os.FileInfo, _ error) error { // Skip the actual directory. @@ -284,7 +286,7 @@ func (e *dirEngine) ListBlobs(ctx context.Context) ([]digest.Digest, error) { } // XXX: Do we need to handle multiple-directory-deep cases? - digest := digest.NewDigestFromHex(cas.BlobAlgorithm.String(), filepath.Base(path)) + digest := digest.NewDigestFromHex(cas.DefaultBlobAlgorithm.String(), filepath.Base(path)) digests = append(digests, digest) return nil }); err != nil { @@ -299,7 +301,7 @@ func (e *dirEngine) ListBlobs(ctx context.Context) ([]digest.Digest, error) { // interface). This MUST NOT remove any blobs or references in the store. func (e *dirEngine) Clean(ctx context.Context) error { // Remove every .umoci directory that isn't flocked. - matches, err := filepath.Glob(filepath.Join(e.path, ".umoci-*")) + matches, err := filepath.Glob(filepath.Join(e.imagePath, ".umoci-*")) if err != nil { return errors.Wrap(err, "glob .umoci-*") } @@ -357,10 +359,17 @@ func (e *dirEngine) Close() error { // Open opens a new reference to the directory-backed OCI image referenced by // the provided path. -func Open(path string) (cas.Engine, error) { +func Open(imagePath string, sharedCasPath string) (cas.Engine, error) { engine := &dirEngine{ - path: path, - temp: "", + imagePath: imagePath, + temp: "", + sharedCasPath: sharedCasPath, + } + + if sharedCasPath != "" { + if err := ensureAlgorithmDirs(sharedCasPath); err != nil { + return nil, err + } } if err := engine.validate(); err != nil { @@ -370,6 +379,15 @@ func Open(path string) (cas.Engine, error) { return engine, nil } +func ensureAlgorithmDirs(path string) error { + for _, algo := range cas.BlobAlgorithms { + if err := os.MkdirAll(filepath.Join(path, algo.String()), 0755); err != nil { + return errors.Wrap(err, "mkdir algorithm") + } + } + return nil +} + // Create creates a new OCI image layout at the given path. If the path already // exists, os.ErrExist is returned. However, all of the parent components of // the path will be created if necessary. @@ -387,11 +405,11 @@ func Create(path string) error { } // Create the necessary directories and "oci-layout" file. - if err := os.Mkdir(filepath.Join(path, blobDirectory), 0755); err != nil { + if err := os.MkdirAll(filepath.Join(path, blobDirectory), 0755); err != nil { return errors.Wrap(err, "mkdir blobdir") } - if err := os.Mkdir(filepath.Join(path, blobDirectory, cas.BlobAlgorithm.String()), 0755); err != nil { - return errors.Wrap(err, "mkdir algorithm") + if err := ensureAlgorithmDirs(filepath.Join(path, blobDirectory)); err != nil { + return err } indexFh, err := os.Create(filepath.Join(path, indexFile)) diff --git a/oci/cas/dir/dir_test.go b/oci/cas/dir/dir_test.go index 29b143137..b2f85a8da 100644 --- a/oci/cas/dir/dir_test.go +++ b/oci/cas/dir/dir_test.go @@ -85,7 +85,7 @@ func TestCreateLayoutReadonly(t *testing.T) { readonly(t, image) defer readwrite(t, image) - engine, err := Open(image) + engine, err := Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } @@ -125,12 +125,12 @@ func TestEngineBlobReadonly(t *testing.T) { {[]byte("some blob")}, {[]byte("another blob")}, } { - engine, err := Open(image) + engine, err := Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } - digester := cas.BlobAlgorithm.Digester() + digester := cas.DefaultBlobAlgorithm.Digester() if _, err := io.Copy(digester.Hash(), bytes.NewReader(test.bytes)); err != nil { t.Fatalf("could not hash bytes: %+v", err) } @@ -155,7 +155,7 @@ func TestEngineBlobReadonly(t *testing.T) { // make it readonly readonly(t, image) - newEngine, err := Open(image) + newEngine, err := Open(image, "") if err != nil { t.Errorf("unexpected error opening ro image: %+v", err) } @@ -209,12 +209,12 @@ func TestEngineGCLocking(t *testing.T) { content := []byte("here's some sample content") // Open a reference to the CAS, and make sure that it has a .temp set up. - engine, err := Open(image) + engine, err := Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } - digester := cas.BlobAlgorithm.Digester() + digester := cas.DefaultBlobAlgorithm.Digester() if _, err := io.Copy(digester.Hash(), bytes.NewReader(content)); err != nil { t.Fatalf("could not hash bytes: %+v", err) } @@ -248,7 +248,7 @@ func TestEngineGCLocking(t *testing.T) { } // Open a new reference and GC it. - gcEngine, err := Open(image) + gcEngine, err := Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } diff --git a/oci/casext/json_dir_test.go b/oci/casext/json_dir_test.go index 3ec2072fd..3c8c6cf73 100644 --- a/oci/casext/json_dir_test.go +++ b/oci/casext/json_dir_test.go @@ -44,7 +44,7 @@ func TestEngineBlobJSON(t *testing.T) { t.Fatalf("unexpected error creating image: %+v", err) } - engine, err := dir.Open(image) + engine, err := dir.Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } @@ -140,7 +140,7 @@ func TestEngineBlobJSONReadonly(t *testing.T) { {object{"a value", 100}}, {object{"another value", 200}}, } { - engine, err := dir.Open(image) + engine, err := dir.Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } @@ -158,7 +158,7 @@ func TestEngineBlobJSONReadonly(t *testing.T) { // make it readonly readonly(t, image) - newEngine, err := dir.Open(image) + newEngine, err := dir.Open(image, "") if err != nil { t.Errorf("unexpected error opening ro image: %+v", err) } diff --git a/oci/casext/refname_dir_test.go b/oci/casext/refname_dir_test.go index d1815e866..060348a81 100644 --- a/oci/casext/refname_dir_test.go +++ b/oci/casext/refname_dir_test.go @@ -250,7 +250,7 @@ func TestEngineReference(t *testing.T) { t.Fatalf("unexpected error creating image: %+v", err) } - engine, err := dir.Open(image) + engine, err := dir.Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } @@ -314,7 +314,7 @@ func TestEngineReferenceReadonly(t *testing.T) { t.Fatalf("unexpected error creating image: %+v", err) } - engine, err := dir.Open(image) + engine, err := dir.Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } @@ -332,7 +332,7 @@ func TestEngineReferenceReadonly(t *testing.T) { for idx, test := range descMap { name := fmt.Sprintf("new_tag_%d", idx) - engine, err := dir.Open(image) + engine, err := dir.Open(image, "") if err != nil { t.Fatalf("unexpected error opening image: %+v", err) } @@ -349,7 +349,7 @@ func TestEngineReferenceReadonly(t *testing.T) { // make it readonly readonly(t, image) - newEngine, err := dir.Open(image) + newEngine, err := dir.Open(image, "") if err != nil { t.Errorf("unexpected error opening ro image: %+v", err) } diff --git a/oci/layer/unpack_test.go b/oci/layer/unpack_test.go index 51148ca98..fed87aca6 100644 --- a/oci/layer/unpack_test.go +++ b/oci/layer/unpack_test.go @@ -91,7 +91,7 @@ yRAbACGEEEIIIYQQQgghhBBCCKEr+wTE0sQyACgAAA==`, if err := dir.Create(image); err != nil { t.Fatal(err) } - engine, err := dir.Open(image) + engine, err := dir.Open(image, "") if err != nil { t.Fatal(err) } From a9347c5c27198cab419d6767a4d71393cb707fde Mon Sep 17 00:00:00 2001 From: Jonathan Boulle Date: Tue, 24 Oct 2017 14:26:46 +0200 Subject: [PATCH 2/3] cmd/gc: error out if --shared-cas is supplied Signed-off-by: Jonathan Boulle --- cmd/umoci/gc.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/umoci/gc.go b/cmd/umoci/gc.go index 586f57696..9e704f245 100644 --- a/cmd/umoci/gc.go +++ b/cmd/umoci/gc.go @@ -51,10 +51,12 @@ root set of references. All other blobs will be removed.`, func gc(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) - sharedCasPath := ctx.String("shared-cas") + if ctx.String("shared-cas") != "" { + return errors.New("gc not supported with --shared-cas") + } // Get a reference to the CAS. - engine, err := dir.Open(imagePath, sharedCasPath) + engine, err := dir.Open(imagePath, "") if err != nil { return errors.Wrap(err, "open CAS") } From d5e1a58fe8c9b7cd2c68f85ed875ce112fc80fa4 Mon Sep 17 00:00:00 2001 From: Jonathan Boulle Date: Tue, 24 Oct 2017 19:34:08 +0200 Subject: [PATCH 3/3] test: add {re,un}pack test for --shared-cas Signed-off-by: Jonathan Boulle --- test/repack.bats | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/repack.bats b/test/repack.bats index f5edbe75b..4ad1cc2a5 100644 --- a/test/repack.bats +++ b/test/repack.bats @@ -529,6 +529,47 @@ function teardown() { image-verify "${IMAGE}" } +@test "umoci {un,re}pack [--shared-cas]" { + BUNDLE="$(setup_tmpdir)" + CASDIR="$(setup_tmpdir)" + cp -ar ${IMAGE}/blobs/* "${CASDIR}"/ + rm -fr ${IMAGE}/blobs/* + + # Sanity checks. + # No blobs left inside the image. + numBlobsImage=$(find ${IMAGE}/blobs/ -type f | wc -l) + [ "$numBlobsImage" -eq 0 ] + # At least three blobs (image manifest, image config, rootfs) in the shared CAS. + numBlobsCasdir=$(find ${CASDIR}/ -type f | wc -l) + [ "$numBlobsCasdir" -ge 3 ] + +# TODO(jonboulle): this requires https://github.com/openSUSE/umoci/pull/207 +# run image-verify "${IMAGE}" +# [ "$status" -ne 0 ] + + # Unpacking the image normally should fail. + umoci unpack --image "${IMAGE}:${TAG}" "$BUNDLE" + [ "$status" -ne 0 ] + ! [ -e $BUNDLE ] + + # Unpacking with the shared CAS should succeed. + umoci unpack --shared-cas "${CASDIR}" --image "${IMAGE}:${TAG}" "$BUNDLE" + [ "$status" -eq 0 ] + bundle-verify "$BUNDLE" + + # Make a new layer and repack + echo "a file" > "$BUNDLE/rootfs/newfile" + umoci repack --shared-cas "${CASDIR}" --image "${IMAGE}:${TAG}-new" "$BUNDLE" + + # Should still have no blobs inside the image + numBlobsImage=$(find ${IMAGE}/blobs/ -type f | wc -l) + [ "$numBlobsImage" -eq 0 ] + # At least two more blobs (image manifest, rootfs) in the shared CAS. + newNumBlobsCasdir=$(find ${CASDIR}/ -type f | wc -l) + [ "$newNumBlobsCasdir" -ge $((numBlobsCasDir+3)) ] +} + + @test "umoci repack [--config.volumes]" { BUNDLE_A="$(setup_tmpdir)" BUNDLE_B="$(setup_tmpdir)"