Merge pull request #8953 from ulyssessouza/test-multiargs

Add multiargs build e2e tests

cmd/compose/convert.go

@@ -44,9 +44,11 @@ type convertOptions struct {
 	quiet               bool
 	resolveImageDigests bool
 	noInterpolate       bool
+	noNormalize         bool
 	services            bool
 	volumes             bool
 	profiles            bool
+	images              bool
 	hash                string
 }
 
@@ -66,6 +68,9 @@ func convertCommand(p *projectOptions, backend api.Service) *cobra.Command {
 				}
 				os.Stdout = devnull
 			}
+			if p.Compatibility {
+				opts.noNormalize = true
+			}
 			return nil
 		}),
 		RunE: Adapt(func(ctx context.Context, args []string) error {
@@ -81,6 +86,9 @@ func convertCommand(p *projectOptions, backend api.Service) *cobra.Command {
 			if opts.profiles {
 				return runProfiles(opts, args)
 			}
+			if opts.images {
+				return runConfigImages(opts, args)
+			}
 
 			return runConvert(ctx, backend, opts, args)
 		}),
@@ -91,10 +99,12 @@ func convertCommand(p *projectOptions, backend api.Service) *cobra.Command {
 	flags.BoolVar(&opts.resolveImageDigests, "resolve-image-digests", false, "Pin image tags to digests.")
 	flags.BoolVarP(&opts.quiet, "quiet", "q", false, "Only validate the configuration, don't print anything.")
 	flags.BoolVar(&opts.noInterpolate, "no-interpolate", false, "Don't interpolate environment variables.")
+	flags.BoolVar(&opts.noNormalize, "no-normalize", false, "Don't normalize compose model.")
 
 	flags.BoolVar(&opts.services, "services", false, "Print the service names, one per line.")
 	flags.BoolVar(&opts.volumes, "volumes", false, "Print the volume names, one per line.")
 	flags.BoolVar(&opts.profiles, "profiles", false, "Print the profile names, one per line.")
+	flags.BoolVar(&opts.images, "images", false, "Print the image names, one per line.")
 	flags.StringVar(&opts.hash, "hash", "", "Print the service config hash, one per line.")
 	flags.StringVarP(&opts.Output, "output", "o", "", "Save to file (default to stdout)")
 
@@ -103,7 +113,10 @@ func convertCommand(p *projectOptions, backend api.Service) *cobra.Command {
 
 func runConvert(ctx context.Context, backend api.Service, opts convertOptions, services []string) error {
 	var json []byte
-	project, err := opts.toProject(services, cli.WithInterpolation(!opts.noInterpolate), cli.WithResolvedPaths(true))
+	project, err := opts.toProject(services,
+		cli.WithInterpolation(!opts.noInterpolate),
+		cli.WithResolvedPaths(true),
+		cli.WithNormalization(!opts.noNormalize))
 	if err != nil {
 		return err
 	}
@@ -207,3 +220,18 @@ func runProfiles(opts convertOptions, services []string) error {
 	}
 	return nil
 }
+
+func runConfigImages(opts convertOptions, services []string) error {
+	project, err := opts.toProject(services)
+	if err != nil {
+		return err
+	}
+	for _, s := range project.Services {
+		if s.Image != "" {
+			fmt.Println(s.Image)
+		} else {
+			fmt.Printf("%s_%s\n", project.Name, s.Name)
+		}
+	}
+	return nil
+}

cmd/compose/run.go

@@ -54,6 +54,7 @@ type runOptions struct {
 	servicePorts  bool
 	name          string
 	noDeps        bool
+	quietPull     bool
 }
 
 func (opts runOptions) apply(project *types.Project) error {
@@ -153,6 +154,7 @@ func runCommand(p *projectOptions, backend api.Service) *cobra.Command {
 	flags.StringArrayVarP(&opts.publish, "publish", "p", []string{}, "Publish a container's port(s) to the host.")
 	flags.BoolVar(&opts.useAliases, "use-aliases", false, "Use the service's network useAliases in the network(s) the container connects to.")
 	flags.BoolVar(&opts.servicePorts, "service-ports", false, "Run command with the service's ports enabled and mapped to the host.")
+	flags.BoolVar(&opts.quietPull, "quiet-pull", false, "Pull without printing progress information.")
 
 	flags.SetNormalizeFunc(normalizeRunFlags)
 	flags.SetInterspersed(false)
@@ -215,6 +217,7 @@ func runRun(ctx context.Context, backend api.Service, project *types.Project, op
 		UseNetworkAliases: opts.useAliases,
 		NoDeps:            opts.noDeps,
 		Index:             0,
+		QuietPull:         opts.quietPull,
 	}
 	exitCode, err := backend.RunOneOffContainer(ctx, project, runOpts)
 	if exitCode != 0 {

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.2.3
 	github.com/buger/goterm v1.0.0
 	github.com/cnabio/cnab-to-oci v0.3.1-beta1
-	github.com/compose-spec/compose-go v1.0.5
+	github.com/compose-spec/compose-go v1.0.8
 	github.com/containerd/console v1.0.2
 	github.com/containerd/containerd v1.5.8
 	github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e
@@ -47,7 +47,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/cnabio/cnab-go v0.10.0-beta1 // indirect
-	github.com/compose-spec/godotenv v1.1.0 // indirect
+	github.com/compose-spec/godotenv v1.1.1 // indirect
 	github.com/containerd/cgroups v1.0.1 // indirect
 	github.com/containerd/continuity v0.1.0 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect

go.sum

@@ -197,10 +197,10 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/compose-spec/compose-go v1.0.5 h1:WtfK7tJsk5C8h12iggum7p28kTxeXH7Xi5c/pLfnBwk=
-github.com/compose-spec/compose-go v1.0.5/go.mod h1:LQ/JAjSIyh8bTu4RV6nkyf0Ow/Yf3qpvzrdEigxduiw=
-github.com/compose-spec/godotenv v1.1.0 h1:wzShe5P6L/Aw3wsV357eWlZdMcPaOe2V2+3+qGwMEL4=
-github.com/compose-spec/godotenv v1.1.0/go.mod h1:zF/3BOa18Z24tts5qnO/E9YURQanJTBUf7nlcCTNsyc=
+github.com/compose-spec/compose-go v1.0.8 h1:fgT7mYYu5Sp37i2lUIAAvwJpkAHk6dP5ITHy/LlutUk=
+github.com/compose-spec/compose-go v1.0.8/go.mod h1:REnCbBugoIdHB7S1sfkN/aJ7AJpNApGNjNiVjA9L8x4=
+github.com/compose-spec/godotenv v1.1.1 h1:lp+WpAInnw06YN9sV/XLUOV/9z4C+6wjJdWlrdVac7o=
+github.com/compose-spec/godotenv v1.1.1/go.mod h1:zF/3BOa18Z24tts5qnO/E9YURQanJTBUf7nlcCTNsyc=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
 github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=

pkg/api/api.go

@@ -227,6 +227,8 @@ type RunOptions struct {
 	Privileged        bool
 	UseNetworkAliases bool
 	NoDeps            bool
+	// QuietPull makes the pulling process quiet
+	QuietPull bool
 	// used by exec
 	Index int
 }

pkg/compose/convergence.go

@@ -448,7 +448,7 @@ func (s *composeService) createMobyContainer(ctx context.Context, project *types
 			Networks: inspectedContainer.NetworkSettings.Networks,
 		},
 	}
-	links := append(service.Links, service.ExternalLinks...)
+	links := s.getLinks(ctx, project.Name, service, number)
 	for _, netName := range service.NetworksByPriority() {
 		netwrk := project.Networks[netName]
 		cfg := service.Networks[netName]
@@ -476,6 +476,64 @@ func (s *composeService) createMobyContainer(ctx context.Context, project *types
 	return created, err
 }
 
+// getLinks mimics V1 compose/service.py::Service::_get_links()
+func (s composeService) getLinks(ctx context.Context, projectName string, service types.ServiceConfig, number int) []string {
+	var links []string
+	format := func(k, v string) string {
+		return fmt.Sprintf("%s:%s", k, v)
+	}
+	getServiceContainers := func(serviceName string) (Containers, error) {
+		return s.getContainers(ctx, projectName, oneOffExclude, true, serviceName)
+	}
+
+	for _, rawLink := range service.Links {
+		linkSplit := strings.Split(rawLink, ":")
+		linkServiceName := linkSplit[0]
+		linkName := linkServiceName
+		if len(linkSplit) == 2 {
+			linkName = linkSplit[1] // linkName if informed like in: "serviceName:linkName"
+		}
+		cnts, err := getServiceContainers(linkServiceName)
+		if err != nil {
+			return nil
+		}
+		for _, c := range cnts {
+			containerName := getCanonicalContainerName(c)
+			links = append(links,
+				format(containerName, linkName),
+				format(containerName, strings.Join([]string{linkServiceName, strconv.Itoa(number)}, Separator)),
+				format(containerName, strings.Join([]string{projectName, linkServiceName, strconv.Itoa(number)}, Separator)),
+			)
+		}
+	}
+
+	if service.Labels[api.OneoffLabel] == "True" {
+		cnts, err := getServiceContainers(service.Name)
+		if err != nil {
+			return nil
+		}
+		for _, c := range cnts {
+			containerName := getCanonicalContainerName(c)
+			links = append(links,
+				format(containerName, service.Name),
+				format(containerName, strings.TrimPrefix(containerName, projectName+Separator)),
+				format(containerName, containerName),
+			)
+		}
+	}
+
+	for _, rawExtLink := range service.ExternalLinks {
+		extLinkSplit := strings.Split(rawExtLink, ":")
+		externalLink := extLinkSplit[0]
+		linkName := externalLink
+		if len(extLinkSplit) == 2 {
+			linkName = extLinkSplit[1]
+		}
+		links = append(links, format(externalLink, linkName))
+	}
+	return links
+}
+
 func shortIDAliasExists(containerID string, aliases ...string) bool {
 	for _, alias := range aliases {
 		if alias == containerID[:12] {
@@ -559,6 +617,10 @@ func (s *composeService) isServiceCompleted(ctx context.Context, project *types.
 }
 
 func (s *composeService) startService(ctx context.Context, project *types.Project, service types.ServiceConfig) error {
+	if service.Deploy != nil && service.Deploy.Replicas != nil && *service.Deploy.Replicas == 0 {
+		return nil
+	}
+
 	err := s.waitDependencies(ctx, project, service.DependsOn)
 	if err != nil {
 		return err
@@ -579,7 +641,7 @@ func (s *composeService) startService(ctx context.Context, project *types.Projec
 		if scale, err := getScale(service); err != nil && scale == 0 {
 			return nil
 		}
-		return fmt.Errorf("no containers to start")
+		return fmt.Errorf("service %q has no container to start", service.Name)
 	}
 
 	w := progress.ContextWriter(ctx)

pkg/compose/convergence_test.go

@@ -17,10 +17,16 @@
 package compose
 
 import (
+	"context"
 	"fmt"
 	"testing"
 
 	"github.com/compose-spec/compose-go/types"
+	"github.com/docker/compose/v2/pkg/api"
+	"github.com/docker/compose/v2/pkg/mocks"
+	moby "github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/golang/mock/gomock"
 	"gotest.tools/assert"
 )
 
@@ -46,3 +52,128 @@ func TestContainerName(t *testing.T) {
 	_, err = getScale(s)
 	assert.Error(t, err, fmt.Sprintf(doubledContainerNameWarning, s.Name, s.ContainerName))
 }
+
+func TestServiceLinks(t *testing.T) {
+	const dbContainerName = "/" + testProject + "-db-1"
+	const webContainerName = "/" + testProject + "-web-1"
+	s := types.ServiceConfig{
+		Name:  "web",
+		Scale: 1,
+	}
+
+	containerListOptions := moby.ContainerListOptions{
+		Filters: filters.NewArgs(
+			projectFilter(testProject),
+			serviceFilter("db"),
+			oneOffFilter(false),
+		),
+		All: true,
+	}
+
+	t.Run("service links default", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+		defer mockCtrl.Finish()
+		apiClient := mocks.NewMockAPIClient(mockCtrl)
+		tested.apiClient = apiClient
+
+		s.Links = []string{"db"}
+
+		c := testContainer("db", dbContainerName, false)
+		apiClient.EXPECT().ContainerList(gomock.Any(), containerListOptions).Return([]moby.Container{c}, nil)
+
+		links := tested.getLinks(context.Background(), testProject, s, 1)
+
+		assert.Equal(t, len(links), 3)
+		assert.Equal(t, links[0], "testProject-db-1:db")
+		assert.Equal(t, links[1], "testProject-db-1:db-1")
+		assert.Equal(t, links[2], "testProject-db-1:testProject-db-1")
+	})
+
+	t.Run("service links", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+		defer mockCtrl.Finish()
+		apiClient := mocks.NewMockAPIClient(mockCtrl)
+		tested.apiClient = apiClient
+
+		s.Links = []string{"db:db"}
+
+		c := testContainer("db", dbContainerName, false)
+
+		apiClient.EXPECT().ContainerList(gomock.Any(), containerListOptions).Return([]moby.Container{c}, nil)
+		links := tested.getLinks(context.Background(), testProject, s, 1)
+
+		assert.Equal(t, len(links), 3)
+		assert.Equal(t, links[0], "testProject-db-1:db")
+		assert.Equal(t, links[1], "testProject-db-1:db-1")
+		assert.Equal(t, links[2], "testProject-db-1:testProject-db-1")
+	})
+
+	t.Run("service links name", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+		defer mockCtrl.Finish()
+		apiClient := mocks.NewMockAPIClient(mockCtrl)
+		tested.apiClient = apiClient
+
+		s.Links = []string{"db:dbname"}
+
+		c := testContainer("db", dbContainerName, false)
+		apiClient.EXPECT().ContainerList(gomock.Any(), containerListOptions).Return([]moby.Container{c}, nil)
+
+		links := tested.getLinks(context.Background(), testProject, s, 1)
+
+		assert.Equal(t, len(links), 3)
+		assert.Equal(t, links[0], "testProject-db-1:dbname")
+		assert.Equal(t, links[1], "testProject-db-1:db-1")
+		assert.Equal(t, links[2], "testProject-db-1:testProject-db-1")
+	})
+
+	t.Run("service links external links", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+		defer mockCtrl.Finish()
+		apiClient := mocks.NewMockAPIClient(mockCtrl)
+		tested.apiClient = apiClient
+
+		s.Links = []string{"db:dbname"}
+		s.ExternalLinks = []string{"db1:db2"}
+
+		c := testContainer("db", dbContainerName, false)
+		apiClient.EXPECT().ContainerList(gomock.Any(), containerListOptions).Return([]moby.Container{c}, nil)
+
+		links := tested.getLinks(context.Background(), testProject, s, 1)
+		assert.Equal(t, len(links), 4)
+		assert.Equal(t, links[0], "testProject-db-1:dbname")
+		assert.Equal(t, links[1], "testProject-db-1:db-1")
+		assert.Equal(t, links[2], "testProject-db-1:testProject-db-1")
+
+		// ExternalLink
+		assert.Equal(t, links[3], "db1:db2")
+	})
+
+	t.Run("service links itself oneoff", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+		defer mockCtrl.Finish()
+		apiClient := mocks.NewMockAPIClient(mockCtrl)
+		tested.apiClient = apiClient
+
+		s.Links = []string{}
+		s.ExternalLinks = []string{}
+		s.Labels = s.Labels.Add(api.OneoffLabel, "True")
+
+		c := testContainer("web", webContainerName, true)
+		containerListOptionsOneOff := moby.ContainerListOptions{
+			Filters: filters.NewArgs(
+				projectFilter(testProject),
+				serviceFilter("web"),
+				oneOffFilter(false),
+			),
+			All: true,
+		}
+		apiClient.EXPECT().ContainerList(gomock.Any(), containerListOptionsOneOff).Return([]moby.Container{c}, nil)
+
+		links := tested.getLinks(context.Background(), testProject, s, 1)
+		assert.Equal(t, len(links), 3)
+		assert.Equal(t, links[0], "testProject-web-1:web")
+		assert.Equal(t, links[1], "testProject-web-1:web-1")
+		assert.Equal(t, links[2], "testProject-web-1:testProject-web-1")
+	})
+}

pkg/compose/create.go

@@ -1092,10 +1092,17 @@ func (s *composeService) ensureVolume(ctx context.Context, volume types.VolumeCo
 		if !errdefs.IsNotFound(err) {
 			return err
 		}
+		if volume.External.External {
+			return fmt.Errorf("external volume %q not found", volume.External.Name)
+		}
 		err := s.createVolume(ctx, volume)
 		return err
 	}
 
+	if volume.External.External {
+		return nil
+	}
+
 	// Volume exists with name, but let's double check this is the expected one
 	// (better safe than sorry when it comes to user's data)
 	p, ok := inspected.Labels[api.ProjectLabel]

pkg/compose/dependencies.go

@@ -93,7 +93,7 @@ func run(ctx context.Context, graph *Graph, eg *errgroup.Group, nodes []*Vertex,
 	for _, node := range nodes {
 		// Don't start this service yet if all of its children have
 		// not been started yet.
-		if len(traversalConfig.filterAdjacentByStatusFn(graph, node.Service, traversalConfig.adjacentServiceStatusToSkip)) != 0 {
+		if len(traversalConfig.filterAdjacentByStatusFn(graph, node.Key, traversalConfig.adjacentServiceStatusToSkip)) != 0 {
 			continue
 		}
 
@@ -104,7 +104,7 @@ func run(ctx context.Context, graph *Graph, eg *errgroup.Group, nodes []*Vertex,
 				return err
 			}
 
-			graph.UpdateStatus(node.Service, traversalConfig.targetServiceStatus)
+			graph.UpdateStatus(node.Key, traversalConfig.targetServiceStatus)
 
 			return run(ctx, graph, eg, traversalConfig.adjacentNodesFn(node), traversalConfig, fn)
 		})

pkg/compose/run.go

@@ -156,7 +156,7 @@ func (s *composeService) prepareRun(ctx context.Context, project *types.Project,
 	service.Labels = service.Labels.Add(api.SlugLabel, slug)
 	service.Labels = service.Labels.Add(api.OneoffLabel, "True")
 
-	if err := s.ensureImagesExists(ctx, project, false); err != nil { // all dependencies already checked, but might miss service img
+	if err := s.ensureImagesExists(ctx, project, opts.QuietPull); err != nil { // all dependencies already checked, but might miss service img
 		return "", err
 	}
 	if !opts.NoDeps {

pkg/e2e/compose_build_test.go

@@ -66,6 +66,19 @@ func TestLocalComposeBuild(t *testing.T) {
 		res.Assert(t, icmd.Expected{Out: `"FOO": "BAR"`})
 	})
 
+	t.Run("build with multiple build-args ", func(t *testing.T) {
+		// ensure local test run does not reuse previously build image
+		c.RunDockerOrExitError("rmi", "-f", "multi-args_multiargs")
+		cmd := c.NewDockerCmd("compose", "--project-directory", "fixtures/build-test/multi-args", "build")
+
+		icmd.RunCmd(cmd, func(cmd *icmd.Cmd) {
+			cmd.Env = append(cmd.Env, "DOCKER_BUILDKIT=0")
+		})
+
+		res := c.RunDockerCmd("image", "inspect", "multi-args_multiargs")
+		res.Assert(t, icmd.Expected{Out: `"RESULT": "SUCCESS"`})
+	})
+
 	t.Run("build as part of up", func(t *testing.T) {
 		c.RunDockerOrExitError("rmi", "build-test_nginx")
 		c.RunDockerOrExitError("rmi", "custom-nginx")

pkg/e2e/fixtures/build-test/multi-args/Dockerfile

@@ -0,0 +1,19 @@
+#   Copyright 2020 Docker Compose CLI authors
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+ARG IMAGE=666
+ARG TAG=666
+
+FROM ${IMAGE}:${TAG}
+RUN echo "SUCCESS"

pkg/e2e/fixtures/build-test/multi-args/compose.yaml

@@ -0,0 +1,9 @@
+services:
+  multiargs:
+    build:
+      context: .
+      args:
+        IMAGE: alpine
+        TAG: latest
+      labels:
+        - RESULT=SUCCESS

pkg/e2e/framework.go

@@ -104,9 +104,9 @@ func newE2eCLI(t *testing.T, binDir string) *E2eCLI {
 }
 
 func dirContents(dir string) []string {
-	res := []string{}
+	var res []string
 	_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		res = append(res, filepath.Join(dir, path))
+		res = append(res, path)
 		return nil
 	})
 	return res

pkg/e2e/metrics_test.go

@@ -36,7 +36,7 @@ func TestComposeMetrics(t *testing.T) {
 		res = c.RunDockerOrExitError("compose", "-f", "fixtures/wrong-composefile/compose.yaml", "up", "-d")
 		res.Assert(t, icmd.Expected{ExitCode: 15, Err: "services.simple Additional property wrongField is not allowed"})
 		res = c.RunDockerOrExitError("compose", "up")
-		res.Assert(t, icmd.Expected{ExitCode: 14, Err: "can't find a suitable configuration file in this directory or any parent: not found"})
+		res.Assert(t, icmd.Expected{ExitCode: 14, Err: "no configuration file provided: not found"})
 		res = c.RunDockerOrExitError("compose", "up", "-f", "fixtures/wrong-composefile/compose.yaml")
 		res.Assert(t, icmd.Expected{ExitCode: 16, Err: "unknown shorthand flag: 'f' in -f"})
 		res = c.RunDockerOrExitError("compose", "up", "--file", "fixtures/wrong-composefile/compose.yaml")