Skip to content

Commit ce8e529

Browse files
author
John Howard
committed
LCOW: Re-coalesce stores
Signed-off-by: John Howard <jhoward@microsoft.com> The re-coalesces the daemon stores which were split as part of the original LCOW implementation. This is part of the work discussed in moby#34617, in particular see the document linked to in that issue.
1 parent 6feae06 commit ce8e529

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+548
-634
lines changed

‎api/server/backend/build/backend.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import (
1717
// ImageComponent provides an interface for working with images
1818
type ImageComponent interface {
1919
SquashImage(from string, to string) (string, error)
20-
TagImageWithReference(image.ID, string, reference.Named) error
20+
TagImageWithReference(image.ID, reference.Named) error
2121
}
2222

2323
// Builder defines interface for running a build

‎api/server/backend/build/tag.go‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,9 @@ package build
33
import (
44
"fmt"
55
"io"
6-
"runtime"
76

87
"github.com/docker/distribution/reference"
98
"github.com/docker/docker/image"
10-
"github.com/docker/docker/pkg/system"
119
"github.com/pkg/errors"
1210
)
1311

@@ -35,12 +33,7 @@ func NewTagger(backend ImageComponent, stdout io.Writer, names []string) (*Tagge
3533
// TagImages creates image tags for the imageID
3634
func (bt *Tagger) TagImages(imageID image.ID) error {
3735
for _, rt := range bt.repoAndTags {
38-
// TODO @jhowardmsft LCOW support. Will need revisiting.
39-
platform := runtime.GOOS
40-
if system.LCOWSupported() {
41-
platform = "linux"
42-
}
43-
if err := bt.imageComponent.TagImageWithReference(imageID, platform, rt); err != nil {
36+
if err := bt.imageComponent.TagImageWithReference(imageID, rt); err != nil {
4437
return err
4538
}
4639
fmt.Fprintf(bt.stdout, "Successfully tagged %s\n", reference.FamiliarString(rt))

‎builder/builder.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ type Backend interface {
4444
// ContainerCreateWorkdir creates the workdir
4545
ContainerCreateWorkdir(containerID string) error
4646

47-
CreateImage(config []byte, parent string, platform string) (Image, error)
47+
CreateImage(config []byte, parent string) (Image, error)
4848

4949
ImageCacheBuilder
5050
}
@@ -79,7 +79,7 @@ type Result struct {
7979
// ImageCacheBuilder represents a generator for stateful image cache.
8080
type ImageCacheBuilder interface {
8181
// MakeImageCache creates a stateful image cache.
82-
MakeImageCache(cacheFrom []string, platform string) ImageCache
82+
MakeImageCache(cacheFrom []string) ImageCache
8383
}
8484

8585
// ImageCache abstracts an image cache.

‎builder/dockerfile/builder.go‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
123123
PathCache: bm.pathCache,
124124
IDMappings: bm.idMappings,
125125
}
126-
return newBuilder(ctx, builderOptions, os).build(source, dockerfile)
126+
return newBuilder(ctx, builderOptions).build(source, dockerfile)
127127
}
128128

129129
func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) (builder.Source, error) {
@@ -190,7 +190,7 @@ type Builder struct {
190190
}
191191

192192
// newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
193-
func newBuilder(clientCtx context.Context, options builderOptions, os string) *Builder {
193+
func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
194194
config := options.Options
195195
if config == nil {
196196
config = new(types.ImageBuildOptions)
@@ -207,7 +207,7 @@ func newBuilder(clientCtx context.Context, options builderOptions, os string) *B
207207
idMappings: options.IDMappings,
208208
imageSources: newImageSources(clientCtx, options),
209209
pathCache: options.PathCache,
210-
imageProber: newImageProber(options.Backend, config.CacheFrom, os, config.NoCache),
210+
imageProber: newImageProber(options.Backend, config.CacheFrom, config.NoCache),
211211
containerManager: newContainerManager(options.Backend),
212212
}
213213

@@ -367,14 +367,9 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
367367
return nil, errdefs.InvalidParameter(err)
368368
}
369369

370-
os := runtime.GOOS
371-
if dockerfile.OS != "" {
372-
os = dockerfile.OS
373-
}
374-
375370
b := newBuilder(context.Background(), builderOptions{
376371
Options: &types.ImageBuildOptions{NoCache: true},
377-
}, os)
372+
})
378373

379374
// ensure that the commands are valid
380375
for _, n := range dockerfile.AST.Children {

‎builder/dockerfile/dispatchers_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func newBuilderWithMockBackend() *Builder {
3131
Options: &types.ImageBuildOptions{Platform: runtime.GOOS},
3232
Backend: mockBackend,
3333
}),
34-
imageProber: newImageProber(mockBackend, nil, runtime.GOOS, false),
34+
imageProber: newImageProber(mockBackend, nil, false),
3535
containerManager: newContainerManager(mockBackend),
3636
}
3737
return b
@@ -427,10 +427,10 @@ func TestRunWithBuildArgs(t *testing.T) {
427427
}
428428

429429
mockBackend := b.docker.(*MockBackend)
430-
mockBackend.makeImageCacheFunc = func(_ []string, _ string) builder.ImageCache {
430+
mockBackend.makeImageCacheFunc = func(_ []string) builder.ImageCache {
431431
return imageCache
432432
}
433-
b.imageProber = newImageProber(mockBackend, nil, runtime.GOOS, false)
433+
b.imageProber = newImageProber(mockBackend, nil, false)
434434
mockBackend.getImageFunc = func(_ string) (builder.Image, builder.ReleaseableLayer, error) {
435435
return &mockImage{
436436
id: "abcdef",

‎builder/dockerfile/imageprobe.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@ type imageProber struct {
1919
cacheBusted bool
2020
}
2121

22-
func newImageProber(cacheBuilder builder.ImageCacheBuilder, cacheFrom []string, platform string, noCache bool) ImageProber {
22+
func newImageProber(cacheBuilder builder.ImageCacheBuilder, cacheFrom []string, noCache bool) ImageProber {
2323
if noCache {
2424
return &nopProber{}
2525
}
2626

2727
reset := func() builder.ImageCache {
28-
return cacheBuilder.MakeImageCache(cacheFrom, platform)
28+
return cacheBuilder.MakeImageCache(cacheFrom)
2929
}
3030
return &imageProber{cache: reset(), reset: reset}
3131
}

‎builder/dockerfile/internals.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runC
154154
return errors.Wrap(err, "failed to encode image config")
155155
}
156156

157-
exportedImage, err := b.docker.CreateImage(config, state.imageID, parentImage.OS)
157+
exportedImage, err := b.docker.CreateImage(config, state.imageID)
158158
if err != nil {
159159
return errors.Wrapf(err, "failed to export image")
160160
}

‎builder/dockerfile/mockbackend_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ type MockBackend struct {
2020
containerCreateFunc func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
2121
commitFunc func(string, *backend.ContainerCommitConfig) (string, error)
2222
getImageFunc func(string) (builder.Image, builder.ReleaseableLayer, error)
23-
makeImageCacheFunc func(cacheFrom []string, platform string) builder.ImageCache
23+
makeImageCacheFunc func(cacheFrom []string) builder.ImageCache
2424
}
2525

2626
func (m *MockBackend) ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error {
@@ -73,14 +73,14 @@ func (m *MockBackend) GetImageAndReleasableLayer(ctx context.Context, refOrID st
7373
return &mockImage{id: "theid"}, &mockLayer{}, nil
7474
}
7575

76-
func (m *MockBackend) MakeImageCache(cacheFrom []string, platform string) builder.ImageCache {
76+
func (m *MockBackend) MakeImageCache(cacheFrom []string) builder.ImageCache {
7777
if m.makeImageCacheFunc != nil {
78-
return m.makeImageCacheFunc(cacheFrom, platform)
78+
return m.makeImageCacheFunc(cacheFrom)
7979
}
8080
return nil
8181
}
8282

83-
func (m *MockBackend) CreateImage(config []byte, parent string, platform string) (builder.Image, error) {
83+
func (m *MockBackend) CreateImage(config []byte, parent string) (builder.Image, error) {
8484
return nil, nil
8585
}
8686

‎daemon/build.go‎

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package daemon
22

33
import (
44
"io"
5-
"runtime"
65

76
"github.com/docker/distribution/reference"
87
"github.com/docker/docker/api/types"
@@ -24,6 +23,7 @@ type releaseableLayer struct {
2423
layerStore layer.Store
2524
roLayer layer.Layer
2625
rwLayer layer.RWLayer
26+
os string
2727
}
2828

2929
func (rl *releaseableLayer) Mount() (containerfs.ContainerFS, error) {
@@ -35,7 +35,7 @@ func (rl *releaseableLayer) Mount() (containerfs.ContainerFS, error) {
3535
}
3636

3737
mountID := stringid.GenerateRandomID()
38-
rl.rwLayer, err = rl.layerStore.CreateRWLayer(mountID, chainID, nil)
38+
rl.rwLayer, err = rl.layerStore.CreateRWLayer(mountID, chainID, rl.os, nil)
3939
if err != nil {
4040
return nil, errors.Wrap(err, "failed to create rwlayer")
4141
}
@@ -67,12 +67,12 @@ func (rl *releaseableLayer) Commit(os string) (builder.ReleaseableLayer, error)
6767
}
6868
defer stream.Close()
6969

70-
newLayer, err := rl.layerStore.Register(stream, chainID, layer.OS(os))
70+
newLayer, err := rl.layerStore.Register(stream, chainID, os)
7171
if err != nil {
7272
return nil, err
7373
}
74-
// TODO: An optimization would be to handle empty layers before returning
75-
return &releaseableLayer{layerStore: rl.layerStore, roLayer: newLayer}, nil
74+
// TODO: An optimization woudld be to handle empty layers before returning
75+
return &releaseableLayer{layerStore: rl.layerStore, roLayer: newLayer, os: os}, nil
7676
}
7777

7878
func (rl *releaseableLayer) DiffID() layer.DiffID {
@@ -128,21 +128,21 @@ func (rl *releaseableLayer) releaseROLayer() error {
128128
return err
129129
}
130130

131-
func newReleasableLayerForImage(img *image.Image, layerStore layer.Store) (builder.ReleaseableLayer, error) {
131+
func newReleasableLayerForImage(img *image.Image, layerStore layer.Store, os string) (builder.ReleaseableLayer, error) {
132132
if img == nil || img.RootFS.ChainID() == "" {
133-
return &releaseableLayer{layerStore: layerStore}, nil
133+
return &releaseableLayer{layerStore: layerStore, os: os}, nil
134134
}
135135
// Hold a reference to the image layer so that it can't be removed before
136136
// it is released
137137
roLayer, err := layerStore.Get(img.RootFS.ChainID())
138138
if err != nil {
139139
return nil, errors.Wrapf(err, "failed to get layer for image %s", img.ImageID())
140140
}
141-
return &releaseableLayer{layerStore: layerStore, roLayer: roLayer}, nil
141+
return &releaseableLayer{layerStore: layerStore, roLayer: roLayer, os: os}, nil
142142
}
143143

144144
// TODO: could this use the regular daemon PullImage ?
145-
func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, platform string) (*image.Image, error) {
145+
func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) {
146146
ref, err := reference.ParseNormalizedNamed(name)
147147
if err != nil {
148148
return nil, err
@@ -161,7 +161,7 @@ func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfi
161161
pullRegistryAuth = &resolvedConfig
162162
}
163163

164-
if err := daemon.pullImageWithReference(ctx, ref, platform, nil, pullRegistryAuth, output); err != nil {
164+
if err := daemon.pullImageWithReference(ctx, ref, os, nil, pullRegistryAuth, output); err != nil {
165165
return nil, err
166166
}
167167
return daemon.GetImage(name)
@@ -172,7 +172,7 @@ func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfi
172172
// leaking of layers.
173173
func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ReleaseableLayer, error) {
174174
if refOrID == "" {
175-
layer, err := newReleasableLayerForImage(nil, daemon.stores[opts.OS].layerStore)
175+
layer, err := newReleasableLayerForImage(nil, daemon.layerStore, opts.OS)
176176
return nil, layer, err
177177
}
178178

@@ -183,7 +183,7 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st
183183
}
184184
// TODO: shouldn't we error out if error is different from "not found" ?
185185
if image != nil {
186-
layer, err := newReleasableLayerForImage(image, daemon.stores[opts.OS].layerStore)
186+
layer, err := newReleasableLayerForImage(image, daemon.layerStore, image.OperatingSystem())
187187
return image, layer, err
188188
}
189189
}
@@ -192,29 +192,26 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st
192192
if err != nil {
193193
return nil, nil, err
194194
}
195-
layer, err := newReleasableLayerForImage(image, daemon.stores[opts.OS].layerStore)
195+
layer, err := newReleasableLayerForImage(image, daemon.layerStore, image.OperatingSystem())
196196
return image, layer, err
197197
}
198198

199199
// CreateImage creates a new image by adding a config and ID to the image store.
200200
// This is similar to LoadImage() except that it receives JSON encoded bytes of
201201
// an image instead of a tar archive.
202-
func (daemon *Daemon) CreateImage(config []byte, parent string, platform string) (builder.Image, error) {
203-
if platform == "" {
204-
platform = runtime.GOOS
205-
}
206-
id, err := daemon.stores[platform].imageStore.Create(config)
202+
func (daemon *Daemon) CreateImage(config []byte, parent string) (builder.Image, error) {
203+
id, err := daemon.imageStore.Create(config)
207204
if err != nil {
208205
return nil, errors.Wrapf(err, "failed to create image")
209206
}
210207

211208
if parent != "" {
212-
if err := daemon.stores[platform].imageStore.SetParent(id, image.ID(parent)); err != nil {
209+
if err := daemon.imageStore.SetParent(id, image.ID(parent)); err != nil {
213210
return nil, errors.Wrapf(err, "failed to set parent %s", parent)
214211
}
215212
}
216213

217-
return daemon.stores[platform].imageStore.Get(id)
214+
return daemon.imageStore.Get(id)
218215
}
219216

220217
// IDMappings returns uid/gid mappings for the builder

‎daemon/cache.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ import (
77
)
88

99
// MakeImageCache creates a stateful image cache.
10-
func (daemon *Daemon) MakeImageCache(sourceRefs []string, platform string) builder.ImageCache {
10+
func (daemon *Daemon) MakeImageCache(sourceRefs []string) builder.ImageCache {
1111
if len(sourceRefs) == 0 {
12-
return cache.NewLocal(daemon.stores[platform].imageStore)
12+
return cache.NewLocal(daemon.imageStore)
1313
}
1414

15-
cache := cache.New(daemon.stores[platform].imageStore)
15+
cache := cache.New(daemon.imageStore)
1616

1717
for _, ref := range sourceRefs {
1818
img, err := daemon.GetImage(ref)

0 commit comments

Comments
 (0)