Skip to content

Commit 193abed

Browse files
stevvooestevvooe
committed
content: unify provider and ingester
The split between provider and ingester was a long standing division reflecting the client-side use cases. For the most part, we were differentiating these for the algorithms that operate them, but it made instantation and use of the types challenging. On the server-side, this distinction is generally less important. This change unifies these types and in the process we get a few benefits. The first is that we now completely access the content store over GRPC. This was the initial intent and we have now satisfied this goal completely. There are a few issues around listing content and getting status, but we resolve these with simple streaming and regexp filters. More can probably be done to polish this but the result is clean. Several other content-oriented methods were polished in the process of unification. We have now properly seperated out the `Abort` method to cancel ongoing or stalled ingest processes. We have also replaced the `Active` method with a single status method. The transition went extremely smoothly. Once the clients were updated to use the new methods, every thing worked as expected on the first compile. Signed-off-by: Stephen J Day <stephen.day@docker.com>
1 parent 31cf34b commit 193abed

File tree

34 files changed

+1792
-872
lines changed

34 files changed

+1792
-872
lines changed

‎api/services/content/content.pb.go‎

Lines changed: 1285 additions & 580 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/services/content/content.proto‎

Lines changed: 49 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@ service Content {
1414
// existence.
1515
rpc Info(InfoRequest) returns (InfoResponse);
1616

17+
// List streams the entire set of content as Info objects and closes the
18+
// stream.
19+
//
20+
// Typically, this will yield a large response, chunked into messages.
21+
// Clients should make provisions to ensure they can handle the entire data
22+
// set.
23+
rpc List(ListContentRequest) returns (stream ListContentResponse);
24+
1725
// Delete will delete the referenced object.
1826
rpc Delete(DeleteContentRequest) returns (google.protobuf.Empty);
1927

@@ -25,9 +33,10 @@ service Content {
2533
// Status returns the status of ongoing object ingestions, started via
2634
// Write.
2735
//
28-
// For active ingestions, the status will be streamed until the client
29-
// closes the connection or all matched ingestions are committed.
30-
rpc Status(StatusRequest) returns (stream StatusResponse);
36+
// Only those matching the regular expression will be provided in the
37+
// response. If the provided regular expression is empty, all ingestions
38+
// will be provided.
39+
rpc Status(StatusRequest) returns (StatusResponse);
3140

3241
// Write begins or resumes writes to a resource identified by a unique ref.
3342
// Only one active stream may exist at a time for each ref.
@@ -46,13 +55,13 @@ service Content {
4655
// When completed, the commit flag should be set to true. If expected size
4756
// or digest is set, the content will be validated against those values.
4857
rpc Write(stream WriteRequest) returns (stream WriteResponse);
49-
}
5058

51-
message InfoRequest {
52-
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
59+
// Abort cancels the ongoing write named in the request. Any resources
60+
// associated with the write will be collected.
61+
rpc Abort(AbortRequest) returns (google.protobuf.Empty);
5362
}
5463

55-
message InfoResponse {
64+
message Info {
5665
// Digest is the hash identity of the blob.
5766
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
5867

@@ -63,6 +72,20 @@ message InfoResponse {
6372
google.protobuf.Timestamp committed_at = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
6473
}
6574

75+
message InfoRequest {
76+
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
77+
}
78+
79+
message InfoResponse {
80+
Info info = 1 [(gogoproto.nullable) = false];
81+
}
82+
83+
message ListContentRequest {}
84+
85+
message ListContentResponse {
86+
repeated Info info = 1 [(gogoproto.nullable) = false];
87+
}
88+
6689
message DeleteContentRequest {
6790
// Digest specifies which content to delete.
6891
string digest = 1 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
@@ -90,6 +113,22 @@ message ReadResponse {
90113
bytes data = 2; // actual data
91114
}
92115

116+
message StatusRequest {
117+
string regexp = 1;
118+
}
119+
120+
message Status {
121+
google.protobuf.Timestamp started_at = 1 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
122+
google.protobuf.Timestamp updated_at = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
123+
string ref = 3;
124+
int64 offset = 4;
125+
int64 total = 5;
126+
string expected = 6 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
127+
}
128+
129+
message StatusResponse {
130+
repeated Status statuses = 1 [(gogoproto.nullable) = false];
131+
}
93132

94133
// WriteAction defines the behavior of a WriteRequest.
95134
enum WriteAction {
@@ -116,12 +155,6 @@ enum WriteAction {
116155
//
117156
// This action will always terminate the write.
118157
COMMIT = 2 [(gogoproto.enumvalue_customname) = "WriteActionCommit"];
119-
120-
// WriteActionAbort will release any resources associated with the write
121-
// and free up the ref for a completely new set of writes.
122-
//
123-
// This action will always terminate the write.
124-
ABORT = -1 [(gogoproto.enumvalue_customname) = "WriteActionAbort"];
125158
}
126159

127160
// WriteRequest writes data to the request ref at offset.
@@ -213,20 +246,10 @@ message WriteResponse {
213246

214247
// Digest, if present, includes the digest up to the currently committed
215248
// bytes. If action is commit, this field will be set. It is implementation
216-
// defined if this is set for other actions, except abort. On abort, this
217-
// will be empty.
249+
// defined if this is set for other actions.
218250
string digest = 6 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false];
219251
}
220252

221-
message StatusRequest {
222-
repeated string refs = 1;
223-
repeated string prefix = 2;
224-
}
225-
226-
message StatusResponse {
227-
google.protobuf.Timestamp started_at = 1 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
228-
google.protobuf.Timestamp updated_at = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
229-
string ref = 3;
230-
int64 offset = 4;
231-
int64 total = 5;
253+
message AbortRequest {
254+
string ref = 1;
232255
}

‎api/services/execution/execution.pb.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/services/images/images.pb.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/services/rootfs/rootfs.pb.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/services/shim/shim.pb.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/types/container/container.pb.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/types/descriptor/descriptor.pb.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/types/mount/mount.pb.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cmd/containerd/main.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ func serveDebugAPI() error {
242242
return nil
243243
}
244244

245-
func resolveContentStore() (*content.Store, error) {
245+
func resolveContentStore() (content.Store, error) {
246246
cp := filepath.Join(conf.Root, "content")
247247
return content.NewStore(cp)
248248
}
@@ -315,7 +315,7 @@ func loadMonitor() (plugin.ContainerMonitor, error) {
315315
return plugin.NewMultiContainerMonitor(monitors...), nil
316316
}
317317

318-
func loadSnapshotter(store *content.Store) (snapshot.Snapshotter, error) {
318+
func loadSnapshotter(store content.Store) (snapshot.Snapshotter, error) {
319319
for name, sr := range plugin.Registrations() {
320320
if sr.Type != plugin.SnapshotPlugin {
321321
continue
@@ -356,7 +356,7 @@ func newGRPCServer() *grpc.Server {
356356
return s
357357
}
358358

359-
func loadServices(runtimes map[string]containerd.Runtime, store *content.Store, sn snapshot.Snapshotter, meta *bolt.DB) ([]plugin.Service, error) {
359+
func loadServices(runtimes map[string]containerd.Runtime, store content.Store, sn snapshot.Snapshotter, meta *bolt.DB) ([]plugin.Service, error) {
360360
var o []plugin.Service
361361
for name, sr := range plugin.Registrations() {
362362
if sr.Type != plugin.GRPCPlugin {

0 commit comments

Comments
 (0)