Skip to content

Commit d4fbff1

Browse files
jterry75jterry75
authored
Merge pull request containerd#4399 from TBBle/wcow_compare_layers_to_tar
Implement windowsDiff.Compare to allow outputting OCI images
2 parents 6bf77c5 + c750498 commit d4fbff1

File tree

12 files changed

+509
-151
lines changed

12 files changed

+509
-151
lines changed

‎archive/tar.go‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,13 +63,34 @@ func Diff(ctx context.Context, a, b string) io.ReadCloser {
6363
}
6464

6565
// WriteDiff writes a tar stream of the computed difference between the
66-
// provided directories.
66+
// provided paths.
6767
//
6868
// Produces a tar using OCI style file markers for deletions. Deleted
6969
// files will be prepended with the prefix ".wh.". This style is
7070
// based off AUFS whiteouts.
7171
// See https://github.com/opencontainers/image-spec/blob/master/layer.md
72-
func WriteDiff(ctx context.Context, w io.Writer, a, b string) error {
72+
func WriteDiff(ctx context.Context, w io.Writer, a, b string, opts ...WriteDiffOpt) error {
73+
var options WriteDiffOptions
74+
for _, opt := range opts {
75+
if err := opt(&options); err != nil {
76+
return errors.Wrap(err, "failed to apply option")
77+
}
78+
}
79+
if options.writeDiffFunc == nil {
80+
options.writeDiffFunc = writeDiffNaive
81+
}
82+
83+
return options.writeDiffFunc(ctx, w, a, b, options)
84+
}
85+
86+
// writeDiffNaive writes a tar stream of the computed difference between the
87+
// provided directories on disk.
88+
//
89+
// Produces a tar using OCI style file markers for deletions. Deleted
90+
// files will be prepended with the prefix ".wh.". This style is
91+
// based off AUFS whiteouts.
92+
// See https://github.com/opencontainers/image-spec/blob/master/layer.md
93+
func writeDiffNaive(ctx context.Context, w io.Writer, a, b string, _ WriteDiffOptions) error {
7394
cw := newChangeWriter(w, b)
7495
err := fs.Changes(ctx, a, b, cw.HandleChange)
7596
if err != nil {

‎archive/tar_linux_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func TestOverlayApplyNoParents(t *testing.T) {
7070
}
7171
fstest.FSSuite(t, overlayDiffApplier{
7272
tmp: base,
73-
diff: func(ctx context.Context, w io.Writer, a, b string) error {
73+
diff: func(ctx context.Context, w io.Writer, a, b string, _ ...WriteDiffOpt) error {
7474
cw := newChangeWriter(w, b)
7575
cw.addedDirs = nil
7676
err := fs.Changes(ctx, a, b, cw.HandleChange)
@@ -85,7 +85,7 @@ func TestOverlayApplyNoParents(t *testing.T) {
8585

8686
type overlayDiffApplier struct {
8787
tmp string
88-
diff func(context.Context, io.Writer, string, string) error
88+
diff func(context.Context, io.Writer, string, string, ...WriteDiffOpt) error
8989
t *testing.T
9090
}
9191

‎archive/tar_opts.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,13 @@ func WithParents(p []string) ApplyOpt {
7373
return nil
7474
}
7575
}
76+
77+
// WriteDiffOptions provides additional options for a WriteDiff operation
78+
type WriteDiffOptions struct {
79+
ParentLayers []string // Windows needs the full list of parent layers
80+
81+
writeDiffFunc func(context.Context, io.Writer, string, string, WriteDiffOptions) error
82+
}
83+
84+
// WriteDiffOpt allows setting mutable archive write properties on creation
85+
type WriteDiffOpt func(options *WriteDiffOptions) error

‎archive/tar_opts_windows.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,19 @@
1818

1919
package archive
2020

21+
import (
22+
"context"
23+
"io"
24+
25+
"github.com/Microsoft/hcsshim/pkg/ociwclayer"
26+
)
27+
28+
// applyWindowsLayer applies a tar stream of an OCI style diff tar of a Windows layer
29+
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
30+
func applyWindowsLayer(ctx context.Context, root string, r io.Reader, options ApplyOptions) (size int64, err error) {
31+
return ociwclayer.ImportLayerFromTar(ctx, r, root, options.Parents)
32+
}
33+
2134
// AsWindowsContainerLayer indicates that the tar stream to apply is that of
2235
// a Windows Container Layer. The caller must be holding SeBackupPrivilege and
2336
// SeRestorePrivilege.
@@ -27,3 +40,33 @@ func AsWindowsContainerLayer() ApplyOpt {
2740
return nil
2841
}
2942
}
43+
44+
// writeDiffWindowsLayers writes a tar stream of the computed difference between the
45+
// provided Windows layers
46+
//
47+
// Produces a tar using OCI style file markers for deletions. Deleted
48+
// files will be prepended with the prefix ".wh.". This style is
49+
// based off AUFS whiteouts.
50+
// See https://github.com/opencontainers/image-spec/blob/master/layer.md
51+
func writeDiffWindowsLayers(ctx context.Context, w io.Writer, _, layer string, options WriteDiffOptions) error {
52+
return ociwclayer.ExportLayerToTar(ctx, w, layer, options.ParentLayers)
53+
}
54+
55+
// AsWindowsContainerLayerPair indicates that the paths to diff are a pair of
56+
// Windows Container Layers. The caller must be holding SeBackupPrivilege.
57+
func AsWindowsContainerLayerPair() WriteDiffOpt {
58+
return func(options *WriteDiffOptions) error {
59+
options.writeDiffFunc = writeDiffWindowsLayers
60+
return nil
61+
}
62+
}
63+
64+
// WithParentLayers provides the Windows Container Layers that are the parents
65+
// of the target (right-hand, "upper") layer, if any. The source (left-hand, "lower")
66+
// layer passed to WriteDiff should be "" in this case.
67+
func WithParentLayers(p []string) WriteDiffOpt {
68+
return func(options *WriteDiffOptions) error {
69+
options.ParentLayers = p
70+
return nil
71+
}
72+
}

‎archive/tar_windows.go‎

Lines changed: 0 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -20,33 +20,14 @@ package archive
2020

2121
import (
2222
"archive/tar"
23-
"bufio"
24-
"context"
2523
"fmt"
26-
"io"
2724
"os"
28-
"path"
29-
"path/filepath"
3025
"strings"
3126

32-
"github.com/Microsoft/go-winio"
33-
"github.com/Microsoft/go-winio/backuptar"
34-
"github.com/Microsoft/hcsshim"
3527
"github.com/containerd/containerd/sys"
3628
"github.com/pkg/errors"
3729
)
3830

39-
var (
40-
// mutatedFiles is a list of files that are mutated by the import process
41-
// and must be backed up and restored.
42-
mutatedFiles = map[string]string{
43-
"UtilityVM/Files/EFI/Microsoft/Boot/BCD": "bcd.bak",
44-
"UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG": "bcd.log.bak",
45-
"UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG1": "bcd.log1.bak",
46-
"UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG2": "bcd.log2.bak",
47-
}
48-
)
49-
5031
// tarName returns platform-specific filepath
5132
// to canonical posix-style path for tar archival. p is relative
5233
// path.
@@ -141,121 +122,3 @@ func copyDirInfo(fi os.FileInfo, path string) error {
141122
func copyUpXAttrs(dst, src string) error {
142123
return nil
143124
}
144-
145-
// applyWindowsLayer applies a tar stream of an OCI style diff tar of a Windows
146-
// layer using the hcsshim layer writer and backup streams.
147-
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
148-
func applyWindowsLayer(ctx context.Context, root string, r io.Reader, options ApplyOptions) (size int64, err error) {
149-
home, id := filepath.Split(root)
150-
info := hcsshim.DriverInfo{
151-
HomeDir: home,
152-
}
153-
154-
w, err := hcsshim.NewLayerWriter(info, id, options.Parents)
155-
if err != nil {
156-
return 0, err
157-
}
158-
defer func() {
159-
if err2 := w.Close(); err2 != nil {
160-
// This error should not be discarded as a failure here
161-
// could result in an invalid layer on disk
162-
if err == nil {
163-
err = err2
164-
}
165-
}
166-
}()
167-
168-
tr := tar.NewReader(r)
169-
buf := bufio.NewWriter(nil)
170-
hdr, nextErr := tr.Next()
171-
// Iterate through the files in the archive.
172-
for {
173-
select {
174-
case <-ctx.Done():
175-
return 0, ctx.Err()
176-
default:
177-
}
178-
179-
if nextErr == io.EOF {
180-
// end of tar archive
181-
break
182-
}
183-
if nextErr != nil {
184-
return 0, nextErr
185-
}
186-
187-
// Note: path is used instead of filepath to prevent OS specific handling
188-
// of the tar path
189-
base := path.Base(hdr.Name)
190-
if strings.HasPrefix(base, whiteoutPrefix) {
191-
dir := path.Dir(hdr.Name)
192-
originalBase := base[len(whiteoutPrefix):]
193-
originalPath := path.Join(dir, originalBase)
194-
if err := w.Remove(filepath.FromSlash(originalPath)); err != nil {
195-
return 0, err
196-
}
197-
hdr, nextErr = tr.Next()
198-
} else if hdr.Typeflag == tar.TypeLink {
199-
err := w.AddLink(filepath.FromSlash(hdr.Name), filepath.FromSlash(hdr.Linkname))
200-
if err != nil {
201-
return 0, err
202-
}
203-
hdr, nextErr = tr.Next()
204-
} else {
205-
name, fileSize, fileInfo, err := backuptar.FileInfoFromHeader(hdr)
206-
if err != nil {
207-
return 0, err
208-
}
209-
if err := w.Add(filepath.FromSlash(name), fileInfo); err != nil {
210-
return 0, err
211-
}
212-
size += fileSize
213-
hdr, nextErr = tarToBackupStreamWithMutatedFiles(buf, w, tr, hdr, root)
214-
}
215-
}
216-
217-
return
218-
}
219-
220-
// tarToBackupStreamWithMutatedFiles reads data from a tar stream and
221-
// writes it to a backup stream, and also saves any files that will be mutated
222-
// by the import layer process to a backup location.
223-
func tarToBackupStreamWithMutatedFiles(buf *bufio.Writer, w io.Writer, t *tar.Reader, hdr *tar.Header, root string) (nextHdr *tar.Header, err error) {
224-
var (
225-
bcdBackup *os.File
226-
bcdBackupWriter *winio.BackupFileWriter
227-
)
228-
if backupPath, ok := mutatedFiles[hdr.Name]; ok {
229-
bcdBackup, err = os.Create(filepath.Join(root, backupPath))
230-
if err != nil {
231-
return nil, err
232-
}
233-
defer func() {
234-
cerr := bcdBackup.Close()
235-
if err == nil {
236-
err = cerr
237-
}
238-
}()
239-
240-
bcdBackupWriter = winio.NewBackupFileWriter(bcdBackup, false)
241-
defer func() {
242-
cerr := bcdBackupWriter.Close()
243-
if err == nil {
244-
err = cerr
245-
}
246-
}()
247-
248-
buf.Reset(io.MultiWriter(w, bcdBackupWriter))
249-
} else {
250-
buf.Reset(w)
251-
}
252-
253-
defer func() {
254-
ferr := buf.Flush()
255-
if err == nil {
256-
err = ferr
257-
}
258-
}()
259-
260-
return backuptar.WriteBackupStreamFromTarFile(buf, t, hdr)
261-
}

‎diff/walking/differ.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,8 @@ func (s *walkingDiff) Compare(ctx context.Context, lower, upper []mount.Mount, o
9999
if err != nil {
100100
cw.Close()
101101
if newReference {
102-
if err := s.store.Abort(ctx, config.Reference); err != nil {
103-
log.G(ctx).WithField("ref", config.Reference).Warnf("failed to delete diff upload")
102+
if abortErr := s.store.Abort(ctx, config.Reference); abortErr != nil {
103+
log.G(ctx).WithError(abortErr).WithField("ref", config.Reference).Warnf("failed to delete diff upload")
104104
}
105105
}
106106
}

0 commit comments

Comments
 (0)