Skip to content

Commit 0282634

Browse files
dmcgowandmcgowan
committed
Add option to compress blobs on import
Change the default back to leave uncompressed and add option to do the compression. Signed-off-by: Derek McGowan <derek@mcgstyle.net>
1 parent 41e1723 commit 0282634

File tree

3 files changed

+66
-14
lines changed

3 files changed

+66
-14
lines changed

‎cmd/ctr/commands/images/import.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,10 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
7272
Name: "no-unpack",
7373
Usage: "skip unpacking the images, false by default",
7474
},
75+
cli.BoolFlag{
76+
Name: "compress-blobs",
77+
Usage: "compress uncompressed blobs when creating manifest (Docker format only)",
78+
},
7579
}, commands.SnapshotterFlags...),
7680

7781
Action: func(context *cli.Context) error {
@@ -97,6 +101,10 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
97101
opts = append(opts, containerd.WithIndexName(idxName))
98102
}
99103

104+
if context.Bool("compress-blobs") {
105+
opts = append(opts, containerd.WithImportCompression())
106+
}
107+
100108
opts = append(opts, containerd.WithAllPlatforms(context.Bool("all-platforms")))
101109

102110
client, ctx, cancel, err := commands.NewClient(context)

‎images/archive/importer.go‎

Lines changed: 42 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,30 @@ import (
3838
"github.com/pkg/errors"
3939
)
4040

41+
type importOpts struct {
42+
compress bool
43+
}
44+
45+
// ImportOpt is an option for importing an OCI index
46+
type ImportOpt func(*importOpts) error
47+
48+
// WithImportCompression compresses uncompressed layers on import.
49+
// This is used for import formats which do not include the manifest.
50+
func WithImportCompression() ImportOpt {
51+
return func(io *importOpts) error {
52+
io.compress = true
53+
return nil
54+
}
55+
}
56+
4157
// ImportIndex imports an index from a tar archive image bundle
4258
// - implements Docker v1.1, v1.2 and OCI v1.
4359
// - prefers OCI v1 when provided
4460
// - creates OCI index for Docker formats
4561
// - normalizes Docker references and adds as OCI ref name
4662
// e.g. alpine:latest -> docker.io/library/alpine:latest
4763
// - existing OCI reference names are untouched
48-
// - TODO: support option to compress layers on ingest
49-
func ImportIndex(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) {
64+
func ImportIndex(ctx context.Context, store content.Store, reader io.Reader, opts ...ImportOpt) (ocispec.Descriptor, error) {
5065
var (
5166
tr = tar.NewReader(reader)
5267

@@ -58,7 +73,15 @@ func ImportIndex(ctx context.Context, store content.Store, reader io.Reader) (oc
5873
}
5974
symlinks = make(map[string]string)
6075
blobs = make(map[string]ocispec.Descriptor)
76+
iopts importOpts
6177
)
78+
79+
for _, o := range opts {
80+
if err := o(&iopts); err != nil {
81+
return ocispec.Descriptor{}, err
82+
}
83+
}
84+
6285
for {
6386
hdr, err := tr.Next()
6487
if err == io.EOF {
@@ -141,7 +164,7 @@ func ImportIndex(ctx context.Context, store content.Store, reader io.Reader) (oc
141164
}
142165
config.MediaType = images.MediaTypeDockerSchema2Config
143166

144-
layers, err := resolveLayers(ctx, store, mfst.Layers, blobs)
167+
layers, err := resolveLayers(ctx, store, mfst.Layers, blobs, iopts.compress)
145168
if err != nil {
146169
return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers")
147170
}
@@ -217,7 +240,7 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size
217240
return dgstr.Digest(), nil
218241
}
219242

220-
func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor) ([]ocispec.Descriptor, error) {
243+
func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor, compress bool) ([]ocispec.Descriptor, error) {
221244
layers := make([]ocispec.Descriptor, len(layerFiles))
222245
descs := map[digest.Digest]*ocispec.Descriptor{}
223246
filters := []string{}
@@ -261,17 +284,23 @@ func resolveLayers(ctx context.Context, store content.Store, layerFiles []string
261284
return nil, errors.Wrapf(err, "failed to detect compression for %q", layerFiles[i])
262285
}
263286
if s.GetCompression() == compression.Uncompressed {
264-
ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
265-
labels := map[string]string{
266-
"containerd.io/uncompressed": desc.Digest.String(),
267-
}
268-
layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
269-
if err != nil {
270-
s.Close()
271-
return nil, err
287+
if compress {
288+
ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
289+
labels := map[string]string{
290+
"containerd.io/uncompressed": desc.Digest.String(),
291+
}
292+
layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
293+
if err != nil {
294+
s.Close()
295+
return nil, err
296+
}
297+
layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
298+
} else {
299+
layers[i].MediaType = images.MediaTypeDockerSchema2Layer
272300
}
301+
} else {
302+
layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
273303
}
274-
layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
275304
s.Close()
276305

277306
}

‎import.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ type importOpts struct {
3535
imageRefT func(string) string
3636
dgstRefT func(digest.Digest) string
3737
allPlatforms bool
38+
compress bool
3839
}
3940

4041
// ImportOpt allows the caller to specify import specific options
@@ -74,6 +75,15 @@ func WithAllPlatforms(allPlatforms bool) ImportOpt {
7475
}
7576
}
7677

78+
// WithImportCompression compresses uncompressed layers on import.
79+
// This is used for import formats which do not include the manifest.
80+
func WithImportCompression() ImportOpt {
81+
return func(c *importOpts) error {
82+
c.compress = true
83+
return nil
84+
}
85+
}
86+
7787
// Import imports an image from a Tar stream using reader.
7888
// Caller needs to specify importer. Future version may use oci.v1 as the default.
7989
// Note that unreferrenced blobs may be imported to the content store as well.
@@ -91,7 +101,12 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt
91101
}
92102
defer done(ctx)
93103

94-
index, err := archive.ImportIndex(ctx, c.ContentStore(), reader)
104+
var aio []archive.ImportOpt
105+
if iopts.compress {
106+
aio = append(aio, archive.WithImportCompression())
107+
}
108+
109+
index, err := archive.ImportIndex(ctx, c.ContentStore(), reader, aio...)
95110
if err != nil {
96111
return nil, err
97112
}

0 commit comments

Comments
 (0)