Add push object

dmcgowan · dmcgowan · commit 735b0e515e41 · 2017-05-23T10:52:51.000-07:00
Split resolver to only return a name with separate methods
for getting a fetcher and pusher. Add implementation for
push.

Signed-off-by: Derek McGowan &lt;derek@mcgstyle.net&gt;
diff --git a/cmd/dist/fetch.go b/cmd/dist/fetch.go
@@ -72,7 +72,11 @@ Most of this is experimental and there are few leaps to make this work.`,
 		resolved := make(chan struct{})
 		eg.Go(func() error {
 			ongoing.add(ref)
-			name, desc, fetcher, err := resolver.Resolve(ctx, ref)
+			name, desc, err := resolver.Resolve(ctx, ref)
+			if err != nil {
+				return err
+			}
+			fetcher, err := resolver.Fetcher(ctx, name)
 			if err != nil {
 				return err
 			}
diff --git a/cmd/dist/fetchobject.go b/cmd/dist/fetchobject.go
@@ -33,7 +33,11 @@ var fetchObjectCommand = cli.Command{
 		ctx = log.WithLogger(ctx, log.G(ctx).WithField("ref", ref))
 
 		log.G(ctx).Infof("resolving")
-		_, desc, fetcher, err := resolver.Resolve(ctx, ref)
+		name, desc, err := resolver.Resolve(ctx, ref)
+		if err != nil {
+			return err
+		}
+		fetcher, err := resolver.Fetcher(ctx, name)
 		if err != nil {
 			return err
 		}
diff --git a/cmd/dist/main.go b/cmd/dist/main.go
@@ -77,6 +77,7 @@ distribution tool
 		fetchObjectCommand,
 		applyCommand,
 		rootfsCommand,
+		pushObjectCommand,
 	}
 	app.Before = func(context *cli.Context) error {
 		timeout = context.GlobalDuration("timeout")
diff --git a/cmd/dist/pull.go b/cmd/dist/pull.go
@@ -64,11 +64,16 @@ command. As part of this process, we do the following:
 		resolved := make(chan struct{})
 		eg.Go(func() error {
 			ongoing.add(ref)
-			name, desc, fetcher, err := resolver.Resolve(ctx, ref)
+			name, desc, err := resolver.Resolve(ctx, ref)
 			if err != nil {
 				log.G(ctx).WithError(err).Error("failed to resolve")
 				return err
 			}
+			fetcher, err := resolver.Fetcher(ctx, name)
+			if err != nil {
+				return err
+			}
+
 			log.G(ctx).WithField("image", name).Debug("fetching")
 			resolvedImageName = name
 			close(resolved)
diff --git a/cmd/dist/pushobject.go b/cmd/dist/pushobject.go
@@ -0,0 +1,75 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/containerd/containerd/log"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/urfave/cli"
+)
+
+var pushObjectCommand = cli.Command{
+	Name:        "push-object",
+	Usage:       "pushes an object to a remote",
+	ArgsUsage:   "[flags] <remote> <object> <type>",
+	Description: `Push objects by identifier to a remote.`,
+	Flags:       registryFlags,
+	Action: func(clicontext *cli.Context) error {
+		var (
+			ref    = clicontext.Args().Get(0)
+			object = clicontext.Args().Get(1)
+			media  = clicontext.Args().Get(2)
+		)
+		dgst, err := digest.Parse(object)
+		if err != nil {
+			return err
+		}
+
+		ctx, cancel := appContext()
+		defer cancel()
+
+		resolver, err := getResolver(ctx, clicontext)
+		if err != nil {
+			return err
+		}
+
+		ctx = log.WithLogger(ctx, log.G(ctx).WithField("ref", ref))
+
+		log.G(ctx).Infof("resolving")
+		pusher, err := resolver.Pusher(ctx, ref)
+		if err != nil {
+			return err
+		}
+
+		cs, err := resolveContentStore(clicontext)
+		if err != nil {
+			return err
+		}
+
+		info, err := cs.Info(ctx, dgst)
+		if err != nil {
+			return err
+		}
+		desc := ocispec.Descriptor{
+			MediaType: media,
+			Digest:    dgst,
+			Size:      info.Size,
+		}
+
+		rc, err := cs.Reader(ctx, dgst)
+		if err != nil {
+			return err
+		}
+		defer rc.Close()
+
+		// TODO: Progress reader
+		if err = pusher.Push(ctx, desc, rc); err != nil {
+			return err
+		}
+
+		fmt.Printf("Pushed %s %s\n", desc.Digest, desc.MediaType)
+
+		return nil
+	},
+}
diff --git a/reference/reference.go b/reference/reference.go
@@ -79,20 +79,20 @@ func Parse(s string) (Spec, error) {
 		return Spec{}, ErrHostnameRequired
 	}
 
-	parts := splitRe.Split(u.Path, 2)
-	if len(parts) < 2 {
-		return Spec{}, ErrObjectRequired
-	}
-
-	// This allows us to retain the @ to signify digests or shortend digests in
-	// the object.
-	object := u.Path[len(parts[0]):]
-	if object[:1] == ":" {
-		object = object[1:]
+	var object string
+
+	if idx := splitRe.FindStringIndex(u.Path); idx != nil {
+		// This allows us to retain the @ to signify digests or shortend digests in
+		// the object.
+		object = u.Path[idx[0]:]
+		if object[:1] == ":" {
+			object = object[1:]
+		}
+		u.Path = u.Path[:idx[0]]
 	}
 
 	return Spec{
-		Locator: path.Join(u.Host, parts[0]),
+		Locator: path.Join(u.Host, u.Path),
 		Object:  object,
 	}, nil
 }
@@ -119,6 +119,9 @@ func (r Spec) Digest() digest.Digest {
 
 // String returns the normalized string for the ref.
 func (r Spec) String() string {
+	if r.Object == "" {
+		return r.Locator
+	}
 	if r.Object[:1] == "@" {
 		return fmt.Sprintf("%v%v", r.Locator, r.Object)
 	}
diff --git a/reference/reference_test.go b/reference/reference_test.go
@@ -78,9 +78,14 @@ func TestReferenceParser(t *testing.T) {
 			Err:   ErrHostnameRequired,
 		},
 		{
-			Name:  "ErrObjectRequired",
-			Input: "docker.io/library/redis?fooo=asdf",
-			Err:   ErrObjectRequired,
+			Name:       "ErrObjectRequired",
+			Input:      "docker.io/library/redis?fooo=asdf",
+			Hostname:   "docker.io",
+			Normalized: "docker.io/library/redis",
+			Expected: Spec{
+				Locator: "docker.io/library/redis",
+				Object:  "",
+			},
 		},
 		{
 			Name:     "Subdomain",
diff --git a/remotes/docker/fetcher.go b/remotes/docker/fetcher.go
@@ -0,0 +1,78 @@
+package docker
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/log"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+type dockerFetcher struct {
+	*dockerBase
+}
+
+func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
+	ctx = log.WithLogger(ctx, log.G(ctx).WithFields(
+		logrus.Fields{
+			"base":   r.base.String(),
+			"digest": desc.Digest,
+		},
+	))
+
+	paths, err := getV2URLPaths(desc)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, path := range paths {
+		u := r.url(path)
+
+		req, err := http.NewRequest(http.MethodGet, u, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		req.Header.Set("Accept", strings.Join([]string{desc.MediaType, `*`}, ", "))
+		resp, err := r.doRequestWithRetries(ctx, req, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		if resp.StatusCode > 299 {
+			if resp.StatusCode == http.StatusNotFound {
+				continue // try one of the other urls.
+			}
+			resp.Body.Close()
+			return nil, errors.Errorf("unexpected status code %v: %v", u, resp.Status)
+		}
+
+		return resp.Body, nil
+	}
+
+	return nil, errors.New("not found")
+}
+
+// getV2URLPaths generates the candidate urls paths for the object based on the
+// set of hints and the provided object id. URLs are returned in the order of
+// most to least likely succeed.
+func getV2URLPaths(desc ocispec.Descriptor) ([]string, error) {
+	var urls []string
+
+	switch desc.MediaType {
+	case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList,
+		ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex:
+		urls = append(urls, path.Join("manifests", desc.Digest.String()))
+	}
+
+	// always fallback to attempting to get the object out of the blobs store.
+	urls = append(urls, path.Join("blobs", desc.Digest.String()))
+
+	return urls, nil
+}
diff --git a/remotes/docker/pusher.go b/remotes/docker/pusher.go
diff --git a/remotes/docker/resolver.go b/remotes/docker/resolver.go
diff --git a/remotes/docker/resolver_test.go b/remotes/docker/resolver_test.go
diff --git a/remotes/resolver.go b/remotes/resolver.go

Original file line number	Diff line number	Diff line change
`@@ -77,6 +77,7 @@ distribution tool`
`77`	`77`	`fetchObjectCommand,`
`78`	`78`	`applyCommand,`
`79`	`79`	`rootfsCommand,`
	`80`	`+ pushObjectCommand,`
`80`	`81`	`}`
`81`	`82`	`app.Before = func(context *cli.Context) error {`
`82`	`83`	`timeout = context.GlobalDuration("timeout")`