Skip to content

Commit a0b818e

Browse files
dmcgowandmcgowan
authored
Merge pull request containerd#2200 from jessvalarezo/multiarch-pulls
allow content to be pulled for specific platform(s), all platforms
2 parents b307df2 + c3cf3d7 commit a0b818e

File tree

9 files changed

+173
-38
lines changed

9 files changed

+173
-38
lines changed

‎client.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ import (
4343
"github.com/containerd/containerd/events"
4444
"github.com/containerd/containerd/images"
4545
"github.com/containerd/containerd/namespaces"
46-
"github.com/containerd/containerd/platforms"
4746
"github.com/containerd/containerd/plugin"
4847
"github.com/containerd/containerd/remotes"
4948
"github.com/containerd/containerd/remotes/docker"
@@ -236,6 +235,10 @@ type RemoteContext struct {
236235
// If no resolver is provided, defaults to Docker registry resolver.
237236
Resolver remotes.Resolver
238237

238+
// Platforms defines which platforms to handle when doing the image operation.
239+
// If this field is empty, content for all platforms will be pulled.
240+
Platforms []string
241+
239242
// Unpack is done after an image is pulled to extract into a snapshotter.
240243
// If an image is not unpacked on pull, it can be unpacked any time
241244
// afterwards. Unpacking is required to run an image.
@@ -287,6 +290,7 @@ func (c *Client) Pull(ctx context.Context, ref string, opts ...RemoteOpt) (Image
287290
if err != nil {
288291
return nil, errors.Wrapf(err, "failed to resolve reference %q", ref)
289292
}
293+
290294
fetcher, err := pullCtx.Resolver.Fetcher(ctx, name)
291295
if err != nil {
292296
return nil, errors.Wrapf(err, "failed to get fetcher for %q", name)
@@ -304,8 +308,8 @@ func (c *Client) Pull(ctx context.Context, ref string, opts ...RemoteOpt) (Image
304308
childrenHandler := images.ChildrenHandler(store)
305309
// Set any children labels for that content
306310
childrenHandler = images.SetChildrenLabels(store, childrenHandler)
307-
// Filter the childen by the platform
308-
childrenHandler = images.FilterPlatform(platforms.Default(), childrenHandler)
311+
// Filter childen by platforms
312+
childrenHandler = images.FilterPlatforms(childrenHandler, pullCtx.Platforms...)
309313

310314
handler = images.Handlers(append(pullCtx.BaseHandlers,
311315
remotes.FetchHandler(store, fetcher),
@@ -371,7 +375,7 @@ func (c *Client) Push(ctx context.Context, ref string, desc ocispec.Descriptor,
371375
return err
372376
}
373377

374-
return remotes.PushContent(ctx, pusher, desc, c.ContentStore(), pushCtx.BaseHandlers...)
378+
return remotes.PushContent(ctx, pusher, desc, c.ContentStore(), pushCtx.Platforms, pushCtx.BaseHandlers...)
375379
}
376380

377381
// GetImage returns an existing image

‎client_opts.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,21 @@ func WithServices(opts ...ServicesOpt) ClientOpt {
6464
// RemoteOpt allows the caller to set distribution options for a remote
6565
type RemoteOpt func(*Client, *RemoteContext) error
6666

67+
// WithPlatform allows the caller to specify a platform to retrieve
68+
// content for
69+
func WithPlatform(platform string) RemoteOpt {
70+
return func(_ *Client, c *RemoteContext) error {
71+
for _, p := range c.Platforms {
72+
if p == platform {
73+
return nil
74+
}
75+
}
76+
77+
c.Platforms = append(c.Platforms, platform)
78+
return nil
79+
}
80+
}
81+
6782
// WithPullUnpack is used to unpack an image after pull. This
6883
// uses the snapshotter, content store, and diff service
6984
// configured for the client.

‎client_test.go‎

Lines changed: 107 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,10 @@ import (
3030

3131
"google.golang.org/grpc/grpclog"
3232

33+
"github.com/containerd/containerd/images"
3334
"github.com/containerd/containerd/log"
3435
"github.com/containerd/containerd/namespaces"
36+
"github.com/containerd/containerd/platforms"
3537
"github.com/containerd/containerd/sys"
3638
"github.com/containerd/containerd/testutil"
3739
"github.com/sirupsen/logrus"
@@ -116,7 +118,7 @@ func TestMain(m *testing.M) {
116118
}).Info("running tests against containerd")
117119

118120
// pull a seed image
119-
if _, err = client.Pull(ctx, testImage, WithPullUnpack); err != nil {
121+
if _, err = client.Pull(ctx, testImage, WithPullUnpack, WithPlatform(platforms.Default())); err != nil {
120122
ctrd.Stop()
121123
ctrd.Wait()
122124
fmt.Fprintf(os.Stderr, "%s: %s\n", err, buf.String())
@@ -187,12 +189,115 @@ func TestImagePull(t *testing.T) {
187189

188190
ctx, cancel := testContext()
189191
defer cancel()
190-
_, err = client.Pull(ctx, testImage)
192+
_, err = client.Pull(ctx, testImage, WithPlatform(platforms.Default()))
191193
if err != nil {
192194
t.Fatal(err)
193195
}
194196
}
195197

198+
func TestImagePullAllPlatforms(t *testing.T) {
199+
client, err := newClient(t, address)
200+
if err != nil {
201+
t.Fatal(err)
202+
}
203+
defer client.Close()
204+
ctx, cancel := testContext()
205+
defer cancel()
206+
207+
cs := client.ContentStore()
208+
img, err := client.Pull(ctx, testImage)
209+
if err != nil {
210+
t.Fatal(err)
211+
}
212+
index := img.Target()
213+
manifests, err := images.Children(ctx, cs, index)
214+
if err != nil {
215+
t.Fatal(err)
216+
}
217+
for _, manifest := range manifests {
218+
children, err := images.Children(ctx, cs, manifest)
219+
if err != nil {
220+
t.Fatal("Th")
221+
}
222+
// check if childless data type has blob in content store
223+
for _, desc := range children {
224+
ra, err := cs.ReaderAt(ctx, desc.Digest)
225+
if err != nil {
226+
t.Fatal(err)
227+
}
228+
ra.Close()
229+
}
230+
}
231+
}
232+
233+
func TestImagePullSomePlatforms(t *testing.T) {
234+
client, err := newClient(t, address)
235+
if err != nil {
236+
t.Fatal(err)
237+
}
238+
defer client.Close()
239+
ctx, cancel := testContext()
240+
defer cancel()
241+
242+
cs := client.ContentStore()
243+
platformList := []string{"linux/arm64/v8", "linux/386"}
244+
m := make(map[string]platforms.Matcher)
245+
var opts []RemoteOpt
246+
247+
for _, platform := range platformList {
248+
p, err := platforms.Parse(platform)
249+
if err != nil {
250+
t.Fatal(err)
251+
}
252+
m[platform] = platforms.NewMatcher(p)
253+
opts = append(opts, WithPlatform(platform))
254+
}
255+
256+
img, err := client.Pull(ctx, "docker.io/library/busybox:latest", opts...)
257+
if err != nil {
258+
t.Fatal(err)
259+
}
260+
261+
index := img.Target()
262+
manifests, err := images.Children(ctx, cs, index)
263+
if err != nil {
264+
t.Fatal(err)
265+
}
266+
267+
count := 0
268+
for _, manifest := range manifests {
269+
children, err := images.Children(ctx, cs, manifest)
270+
found := false
271+
for _, matcher := range m {
272+
if matcher.Match(*manifest.Platform) {
273+
count++
274+
found = true
275+
}
276+
}
277+
278+
if found {
279+
if len(children) == 0 {
280+
t.Fatal("manifest should have pulled children content")
281+
}
282+
283+
// check if childless data type has blob in content store
284+
for _, desc := range children {
285+
ra, err := cs.ReaderAt(ctx, desc.Digest)
286+
if err != nil {
287+
t.Fatal(err)
288+
}
289+
ra.Close()
290+
}
291+
} else if !found && err == nil {
292+
t.Fatal("manifest should not have pulled children content")
293+
}
294+
}
295+
296+
if count != len(platformList) {
297+
t.Fatal("expected a different number of pulled manifests")
298+
}
299+
}
300+
196301
func TestClientReconnect(t *testing.T) {
197302
t.Parallel()
198303

‎cmd/ctr/commands/content/fetch.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,12 +101,20 @@ func Fetch(ref string, cliContext *cli.Context) (containerd.Image, error) {
101101

102102
log.G(pctx).WithField("image", ref).Debug("fetching")
103103
labels := commands.LabelArgs(cliContext.StringSlice("label"))
104-
img, err := client.Pull(pctx, ref,
104+
opts := []containerd.RemoteOpt{
105105
containerd.WithPullLabels(labels),
106106
containerd.WithResolver(resolver),
107107
containerd.WithImageHandler(h),
108108
containerd.WithSchema1Conversion,
109-
)
109+
}
110+
111+
if !cliContext.Bool("all-platforms") {
112+
for _, platform := range cliContext.StringSlice("platform") {
113+
opts = append(opts, containerd.WithPlatform(platform))
114+
}
115+
}
116+
117+
img, err := client.Pull(pctx, ref, opts...)
110118
stopProgress()
111119
if err != nil {
112120
return nil, err

‎cmd/ctr/commands/images/pull.go‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"github.com/containerd/containerd/cmd/ctr/commands"
2323
"github.com/containerd/containerd/cmd/ctr/commands/content"
2424
"github.com/containerd/containerd/log"
25+
"github.com/containerd/containerd/platforms"
2526
"github.com/urfave/cli"
2627
)
2728

@@ -38,7 +39,17 @@ command. As part of this process, we do the following:
3839
2. Prepare the snapshot filesystem with the pulled resources.
3940
3. Register metadata for the image.
4041
`,
41-
Flags: append(commands.RegistryFlags, append(commands.SnapshotterFlags, commands.LabelFlag)...),
42+
Flags: append(append(commands.RegistryFlags, append(commands.SnapshotterFlags, commands.LabelFlag)...),
43+
cli.StringSliceFlag{
44+
Name: "platform",
45+
Usage: "Pull content from a specific platform",
46+
Value: &cli.StringSlice{platforms.Default()},
47+
},
48+
cli.BoolFlag{
49+
Name: "all-platforms",
50+
Usage: "pull content from all platforms",
51+
},
52+
),
4253
Action: func(context *cli.Context) error {
4354
var (
4455
ref = context.Args().First()

‎images/handlers.go‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -182,42 +182,35 @@ func SetChildrenLabels(manager content.Manager, f HandlerFunc) HandlerFunc {
182182
}
183183
}
184184

185-
// FilterPlatform is a handler wrapper which limits the descriptors returned
186-
// by a handler to a single platform.
187-
func FilterPlatform(platform string, f HandlerFunc) HandlerFunc {
185+
// FilterPlatforms is a handler wrapper which limits the descriptors returned
186+
// by a handler to the specified platforms.
187+
func FilterPlatforms(f HandlerFunc, platformList ...string) HandlerFunc {
188188
return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
189189
children, err := f(ctx, desc)
190190
if err != nil {
191191
return children, err
192192
}
193193

194194
var descs []ocispec.Descriptor
195-
if platform != "" && isMultiPlatform(desc.MediaType) {
196-
p, err := platforms.Parse(platform)
197-
if err != nil {
198-
return nil, err
199-
}
200-
matcher := platforms.NewMatcher(p)
201195

202-
for _, d := range children {
203-
if d.Platform == nil || matcher.Match(*d.Platform) {
204-
descs = append(descs, d)
196+
if len(platformList) == 0 {
197+
descs = children
198+
} else {
199+
for _, platform := range platformList {
200+
p, err := platforms.Parse(platform)
201+
if err != nil {
202+
return nil, err
203+
}
204+
matcher := platforms.NewMatcher(p)
205+
206+
for _, d := range children {
207+
if d.Platform == nil || matcher.Match(*d.Platform) {
208+
descs = append(descs, d)
209+
}
205210
}
206211
}
207-
} else {
208-
descs = children
209212
}
210213

211214
return descs, nil
212215
}
213-
214-
}
215-
216-
func isMultiPlatform(mediaType string) bool {
217-
switch mediaType {
218-
case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
219-
return true
220-
default:
221-
return false
222-
}
223216
}

‎images/image.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ func (image *Image) Size(ctx context.Context, provider content.Provider, platfor
118118
}
119119
size += desc.Size
120120
return nil, nil
121-
}), FilterPlatform(platform, ChildrenHandler(provider))), image.Target)
121+
}), FilterPlatforms(ChildrenHandler(provider), platform)), image.Target)
122122
}
123123

124124
// Manifest resolves a manifest from the image for the given platform.

‎images/oci/exporter.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func (oe *V1Exporter) Export(ctx context.Context, store content.Provider, desc o
5858
}
5959

6060
handlers := images.Handlers(
61-
images.FilterPlatform(platforms.Default(), images.ChildrenHandler(store)),
61+
images.FilterPlatforms(images.ChildrenHandler(store), platforms.Default()),
6262
images.HandlerFunc(exportHandler),
6363
)
6464

‎remotes/handlers.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ import (
2929
"github.com/containerd/containerd/errdefs"
3030
"github.com/containerd/containerd/images"
3131
"github.com/containerd/containerd/log"
32-
"github.com/containerd/containerd/platforms"
3332
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
3433
"github.com/pkg/errors"
3534
"github.com/sirupsen/logrus"
@@ -182,7 +181,7 @@ func push(ctx context.Context, provider content.Provider, pusher Pusher, desc oc
182181
//
183182
// Base handlers can be provided which will be called before any push specific
184183
// handlers.
185-
func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, provider content.Provider, baseHandlers ...images.Handler) error {
184+
func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, provider content.Provider, platforms []string, baseHandlers ...images.Handler) error {
186185
var m sync.Mutex
187186
manifestStack := []ocispec.Descriptor{}
188187

@@ -202,7 +201,7 @@ func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, pr
202201
pushHandler := PushHandler(pusher, provider)
203202

204203
handlers := append(baseHandlers,
205-
images.FilterPlatform(platforms.Default(), images.ChildrenHandler(provider)),
204+
images.FilterPlatforms(images.ChildrenHandler(provider), platforms...),
206205
filterHandler,
207206
pushHandler,
208207
)

0 commit comments

Comments
 (0)