Skip to content

Commit 669c067

Browse files
simonferquelsimonferquel
committed
Introduce a typed command system and 2 phase parse/dispatch build
This is a work base to introduce more features like build time dockerfile optimisations, dependency analysis and parallel build, as well as a first step to go from a dispatch-inline process to a frontend+backend process. Signed-off-by: Simon Ferquel <simon.ferquel@docker.com>
1 parent c5c0702 commit 669c067

23 files changed

+2060
-1344
lines changed

‎builder/dockerfile/buildargs.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,26 @@ func newBuildArgs(argsFromOptions map[string]*string) *buildArgs {
4242
}
4343
}
4444

45+
func (b *buildArgs) Clone() *buildArgs {
46+
result := newBuildArgs(b.argsFromOptions)
47+
for k, v := range b.allowedBuildArgs {
48+
result.allowedBuildArgs[k] = v
49+
}
50+
for k, v := range b.allowedMetaArgs {
51+
result.allowedMetaArgs[k] = v
52+
}
53+
for k := range b.referencedArgs {
54+
result.referencedArgs[k] = struct{}{}
55+
}
56+
return result
57+
}
58+
59+
func (b *buildArgs) MergeReferencedArgs(other *buildArgs) {
60+
for k := range other.referencedArgs {
61+
b.referencedArgs[k] = struct{}{}
62+
}
63+
}
64+
4565
// WarnOnUnusedBuildArgs checks if there are any leftover build-args that were
4666
// passed but not consumed during build. Print a warning, if there are any.
4767
func (b *buildArgs) WarnOnUnusedBuildArgs(out io.Writer) {

‎builder/dockerfile/builder.go‎

Lines changed: 115 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"github.com/docker/docker/api/types/backend"
1414
"github.com/docker/docker/api/types/container"
1515
"github.com/docker/docker/builder"
16-
"github.com/docker/docker/builder/dockerfile/command"
16+
"github.com/docker/docker/builder/dockerfile/instructions"
1717
"github.com/docker/docker/builder/dockerfile/parser"
1818
"github.com/docker/docker/builder/fscache"
1919
"github.com/docker/docker/builder/remotecontext"
@@ -41,6 +41,10 @@ var validCommitCommands = map[string]bool{
4141
"workdir": true,
4242
}
4343

44+
const (
45+
stepFormat = "Step %d/%d : %v"
46+
)
47+
4448
// SessionGetter is object used to get access to a session by uuid
4549
type SessionGetter interface {
4650
Get(ctx context.Context, uuid string) (session.Caller, error)
@@ -176,9 +180,7 @@ type Builder struct {
176180
clientCtx context.Context
177181

178182
idMappings *idtools.IDMappings
179-
buildStages *buildStages
180183
disableCommit bool
181-
buildArgs *buildArgs
182184
imageSources *imageSources
183185
pathCache pathCache
184186
containerManager *containerManager
@@ -218,8 +220,6 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
218220
Output: options.ProgressWriter.Output,
219221
docker: options.Backend,
220222
idMappings: options.IDMappings,
221-
buildArgs: newBuildArgs(config.BuildArgs),
222-
buildStages: newBuildStages(),
223223
imageSources: newImageSources(clientCtx, options),
224224
pathCache: options.PathCache,
225225
imageProber: newImageProber(options.Backend, config.CacheFrom, options.Platform, config.NoCache),
@@ -237,24 +237,27 @@ func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*buil
237237

238238
addNodesForLabelOption(dockerfile.AST, b.options.Labels)
239239

240-
if err := checkDispatchDockerfile(dockerfile.AST); err != nil {
241-
buildsFailed.WithValues(metricsDockerfileSyntaxError).Inc()
240+
stages, metaArgs, err := instructions.Parse(dockerfile.AST)
241+
if err != nil {
242+
if instructions.IsUnknownInstruction(err) {
243+
buildsFailed.WithValues(metricsUnknownInstructionError).Inc()
244+
}
242245
return nil, validationError{err}
243246
}
247+
if b.options.Target != "" {
248+
targetIx, found := instructions.HasStage(stages, b.options.Target)
249+
if !found {
250+
buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc()
251+
return nil, errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target)
252+
}
253+
stages = stages[:targetIx+1]
254+
}
244255

245-
dispatchState, err := b.dispatchDockerfileWithCancellation(dockerfile, source)
256+
dockerfile.PrintWarnings(b.Stderr)
257+
dispatchState, err := b.dispatchDockerfileWithCancellation(stages, metaArgs, dockerfile.EscapeToken, source)
246258
if err != nil {
247259
return nil, err
248260
}
249-
250-
if b.options.Target != "" && !dispatchState.isCurrentStage(b.options.Target) {
251-
buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc()
252-
return nil, errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target)
253-
}
254-
255-
dockerfile.PrintWarnings(b.Stderr)
256-
b.buildArgs.WarnOnUnusedBuildArgs(b.Stderr)
257-
258261
if dispatchState.imageID == "" {
259262
buildsFailed.WithValues(metricsDockerfileEmptyError).Inc()
260263
return nil, errors.New("No image was generated. Is your Dockerfile empty?")
@@ -269,61 +272,91 @@ func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error
269272
return aux.Emit(types.BuildResult{ID: state.imageID})
270273
}
271274

272-
func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result, source builder.Source) (*dispatchState, error) {
273-
shlex := NewShellLex(dockerfile.EscapeToken)
274-
state := newDispatchState()
275-
total := len(dockerfile.AST.Children)
276-
var err error
277-
for i, n := range dockerfile.AST.Children {
278-
select {
279-
case <-b.clientCtx.Done():
280-
logrus.Debug("Builder: build cancelled!")
281-
fmt.Fprint(b.Stdout, "Build cancelled")
282-
buildsFailed.WithValues(metricsBuildCanceled).Inc()
283-
return nil, errors.New("Build cancelled")
284-
default:
285-
// Not cancelled yet, keep going...
286-
}
275+
func processMetaArg(meta instructions.ArgCommand, shlex *ShellLex, args *buildArgs) error {
276+
// ShellLex currently only support the concatenated string format
277+
envs := convertMapToEnvList(args.GetAllAllowed())
278+
if err := meta.Expand(func(word string) (string, error) {
279+
return shlex.ProcessWord(word, envs)
280+
}); err != nil {
281+
return err
282+
}
283+
args.AddArg(meta.Key, meta.Value)
284+
args.AddMetaArg(meta.Key, meta.Value)
285+
return nil
286+
}
287287

288-
// If this is a FROM and we have a previous image then
289-
// emit an aux message for that image since it is the
290-
// end of the previous stage
291-
if n.Value == command.From {
292-
if err := emitImageID(b.Aux, state); err != nil {
293-
return nil, err
294-
}
288+
func printCommand(out io.Writer, currentCommandIndex int, totalCommands int, cmd interface{}) int {
289+
fmt.Fprintf(out, stepFormat, currentCommandIndex, totalCommands, cmd)
290+
fmt.Fprintln(out)
291+
return currentCommandIndex + 1
292+
}
293+
294+
func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions.Stage, metaArgs []instructions.ArgCommand, escapeToken rune, source builder.Source) (*dispatchState, error) {
295+
dispatchRequest := dispatchRequest{}
296+
buildArgs := newBuildArgs(b.options.BuildArgs)
297+
totalCommands := len(metaArgs) + len(parseResult)
298+
currentCommandIndex := 1
299+
for _, stage := range parseResult {
300+
totalCommands += len(stage.Commands)
301+
}
302+
shlex := NewShellLex(escapeToken)
303+
for _, meta := range metaArgs {
304+
currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &meta)
305+
306+
err := processMetaArg(meta, shlex, buildArgs)
307+
if err != nil {
308+
return nil, err
295309
}
310+
}
311+
312+
stagesResults := newStagesBuildResults()
296313

297-
if n.Value == command.From && state.isCurrentStage(b.options.Target) {
298-
break
314+
for _, stage := range parseResult {
315+
if err := stagesResults.checkStageNameAvailable(stage.Name); err != nil {
316+
return nil, err
299317
}
318+
dispatchRequest = newDispatchRequest(b, escapeToken, source, buildArgs, stagesResults)
300319

301-
opts := dispatchOptions{
302-
state: state,
303-
stepMsg: formatStep(i, total),
304-
node: n,
305-
shlex: shlex,
306-
source: source,
320+
currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, stage.SourceCode)
321+
if err := initializeStage(dispatchRequest, &stage); err != nil {
322+
return nil, err
307323
}
308-
if state, err = b.dispatch(opts); err != nil {
309-
if b.options.ForceRemove {
310-
b.containerManager.RemoveAll(b.Stdout)
324+
dispatchRequest.state.updateRunConfig()
325+
fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID))
326+
for _, cmd := range stage.Commands {
327+
select {
328+
case <-b.clientCtx.Done():
329+
logrus.Debug("Builder: build cancelled!")
330+
fmt.Fprint(b.Stdout, "Build cancelled\n")
331+
buildsFailed.WithValues(metricsBuildCanceled).Inc()
332+
return nil, errors.New("Build cancelled")
333+
default:
334+
// Not cancelled yet, keep going...
311335
}
336+
337+
currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, cmd)
338+
339+
if err := dispatch(dispatchRequest, cmd); err != nil {
340+
return nil, err
341+
}
342+
343+
dispatchRequest.state.updateRunConfig()
344+
fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID))
345+
346+
}
347+
if err := emitImageID(b.Aux, dispatchRequest.state); err != nil {
312348
return nil, err
313349
}
314-
315-
fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(state.imageID))
316-
if b.options.Remove {
317-
b.containerManager.RemoveAll(b.Stdout)
350+
buildArgs.MergeReferencedArgs(dispatchRequest.state.buildArgs)
351+
if err := commitStage(dispatchRequest.state, stagesResults); err != nil {
352+
return nil, err
318353
}
319354
}
320-
321-
// Emit a final aux message for the final image
322-
if err := emitImageID(b.Aux, state); err != nil {
323-
return nil, err
355+
if b.options.Remove {
356+
b.containerManager.RemoveAll(b.Stdout)
324357
}
325-
326-
return state, nil
358+
buildArgs.WarnOnUnusedBuildArgs(b.Stdout)
359+
return dispatchRequest.state, nil
327360
}
328361

329362
func addNodesForLabelOption(dockerfile *parser.Node, labels map[string]string) {
@@ -380,39 +413,33 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
380413
b.Stderr = ioutil.Discard
381414
b.disableCommit = true
382415

383-
if err := checkDispatchDockerfile(dockerfile.AST); err != nil {
384-
return nil, validationError{err}
416+
commands := []instructions.Command{}
417+
for _, n := range dockerfile.AST.Children {
418+
cmd, err := instructions.ParseCommand(n)
419+
if err != nil {
420+
return nil, validationError{err}
421+
}
422+
commands = append(commands, cmd)
385423
}
386-
dispatchState := newDispatchState()
387-
dispatchState.runConfig = config
388-
return dispatchFromDockerfile(b, dockerfile, dispatchState, nil)
389-
}
390424

391-
func checkDispatchDockerfile(dockerfile *parser.Node) error {
392-
for _, n := range dockerfile.Children {
393-
if err := checkDispatch(n); err != nil {
394-
return errors.Wrapf(err, "Dockerfile parse error line %d", n.StartLine)
425+
dispatchRequest := newDispatchRequest(b, dockerfile.EscapeToken, nil, newBuildArgs(b.options.BuildArgs), newStagesBuildResults())
426+
dispatchRequest.state.runConfig = config
427+
dispatchRequest.state.imageID = config.Image
428+
for _, cmd := range commands {
429+
err := dispatch(dispatchRequest, cmd)
430+
if err != nil {
431+
return nil, validationError{err}
395432
}
433+
dispatchRequest.state.updateRunConfig()
396434
}
397-
return nil
435+
436+
return dispatchRequest.state.runConfig, nil
398437
}
399438

400-
func dispatchFromDockerfile(b *Builder, result *parser.Result, dispatchState *dispatchState, source builder.Source) (*container.Config, error) {
401-
shlex := NewShellLex(result.EscapeToken)
402-
ast := result.AST
403-
total := len(ast.Children)
404-
405-
for i, n := range ast.Children {
406-
opts := dispatchOptions{
407-
state: dispatchState,
408-
stepMsg: formatStep(i, total),
409-
node: n,
410-
shlex: shlex,
411-
source: source,
412-
}
413-
if _, err := b.dispatch(opts); err != nil {
414-
return nil, err
415-
}
439+
func convertMapToEnvList(m map[string]string) []string {
440+
result := []string{}
441+
for k, v := range m {
442+
result = append(result, k+"="+v)
416443
}
417-
return dispatchState.runConfig, nil
444+
return result
418445
}

0 commit comments

Comments
 (0)