diff --git a/pkg/compose/publish.go b/pkg/compose/publish.go index 396ad0f58..b2a9127ae 100644 --- a/pkg/compose/publish.go +++ b/pkg/compose/publish.go @@ -39,6 +39,7 @@ import ( "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" ) func (s *composeService) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error { @@ -65,54 +66,33 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re return err } - named, err := reference.ParseDockerRef(repository) + layers, err := s.createLayers(ctx, project, options) if err != nil { return err } - config := s.dockerCli.ConfigFile() - - resolver := oci.NewResolver(config) - - var layers []v1.Descriptor - extFiles := map[string]string{} - for _, file := range project.ComposeFiles { - data, err := processFile(ctx, file, project, extFiles) - if err != nil { - return err - } - - layerDescriptor := oci.DescriptorForComposeFile(file, data) - layers = append(layers, layerDescriptor) - } - - extLayers, err := processExtends(ctx, project, extFiles) - if err != nil { - return err - } - layers = append(layers, extLayers...) - - if options.WithEnvironment { - layers = append(layers, envFileLayers(project)...) - } - - if options.ResolveImageDigests { - yaml, err := s.generateImageDigestsOverride(ctx, project) - if err != nil { - return err - } - - layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml) - layers = append(layers, layerDescriptor) - } - w := progress.ContextWriter(ctx) w.Event(progress.Event{ ID: repository, Text: "publishing", Status: progress.Working, }) + if logrus.IsLevelEnabled(logrus.DebugLevel) { + logrus.Debug("publishing layers") + for _, layer := range layers { + indent, _ := json.MarshalIndent(layer, "", " ") + fmt.Println(string(indent)) + } + } if !s.dryRun { + named, err := reference.ParseDockerRef(repository) + if err != nil { + return err + } + + config := s.dockerCli.ConfigFile() + resolver := oci.NewResolver(config) + descriptor, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion) if err != nil { w.Event(progress.Event{ @@ -175,11 +155,47 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re return nil } +func (s *composeService) createLayers(ctx context.Context, project *types.Project, options api.PublishOptions) ([]v1.Descriptor, error) { + var layers []v1.Descriptor + extFiles := map[string]string{} + envFiles := map[string]string{} + for _, file := range project.ComposeFiles { + data, err := processFile(ctx, file, project, extFiles, envFiles) + if err != nil { + return nil, err + } + + layerDescriptor := oci.DescriptorForComposeFile(file, data) + layers = append(layers, layerDescriptor) + } + + extLayers, err := processExtends(ctx, project, extFiles) + if err != nil { + return nil, err + } + layers = append(layers, extLayers...) + + if options.WithEnvironment { + layers = append(layers, envFileLayers(envFiles)...) + } + + if options.ResolveImageDigests { + yaml, err := s.generateImageDigestsOverride(ctx, project) + if err != nil { + return nil, err + } + + layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml) + layers = append(layers, layerDescriptor) + } + return layers, nil +} + func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]v1.Descriptor, error) { var layers []v1.Descriptor moreExtFiles := map[string]string{} for xf, hash := range extFiles { - data, err := processFile(ctx, xf, project, moreExtFiles) + data, err := processFile(ctx, xf, project, moreExtFiles, nil) if err != nil { return nil, err } @@ -204,7 +220,7 @@ func processExtends(ctx context.Context, project *types.Project, extFiles map[st return layers, nil } -func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string) ([]byte, error) { +func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string, envFiles map[string]string) ([]byte, error) { f, err := os.ReadFile(file) if err != nil { return nil, err @@ -230,6 +246,15 @@ func processFile(ctx context.Context, file string, project *types.Project, extFi return nil, err } for name, service := range base.Services { + for i, envFile := range service.EnvFiles { + hash := fmt.Sprintf("%x.env", sha256.Sum256([]byte(envFile.Path))) + envFiles[envFile.Path] = hash + f, err = transform.ReplaceEnvFile(f, name, i, hash) + if err != nil { + return nil, err + } + } + if service.Extends == nil { continue } @@ -376,18 +401,16 @@ func (s *composeService) checkEnvironmentVariables(project *types.Project, optio return envVarList, nil } -func envFileLayers(project *types.Project) []v1.Descriptor { +func envFileLayers(files map[string]string) []v1.Descriptor { var layers []v1.Descriptor - for _, service := range project.Services { - for _, envFile := range service.EnvFiles { - f, err := os.ReadFile(envFile.Path) - if err != nil { - // if we can't read the file, skip to the next one - continue - } - layerDescriptor := oci.DescriptorForEnvFile(envFile.Path, f) - layers = append(layers, layerDescriptor) + for file, hash := range files { + f, err := os.ReadFile(file) + if err != nil { + // if we can't read the file, skip to the next one + continue } + layerDescriptor := oci.DescriptorForEnvFile(hash, f) + layers = append(layers, layerDescriptor) } return layers } diff --git a/pkg/compose/publish_test.go b/pkg/compose/publish_test.go index 817684433..8d2c30c76 100644 --- a/pkg/compose/publish_test.go +++ b/pkg/compose/publish_test.go @@ -18,17 +18,19 @@ package compose import ( "context" - "os" + "slices" "testing" "github.com/compose-spec/compose-go/v2/loader" "github.com/compose-spec/compose-go/v2/types" + "github.com/docker/compose/v2/internal" "github.com/docker/compose/v2/pkg/api" + "github.com/google/go-cmp/cmp" v1 "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" ) -func Test_processExtends(t *testing.T) { +func Test_createLayers(t *testing.T) { project, err := loader.LoadWithContext(context.TODO(), types.ConfigDetails{ WorkingDir: "testdata/publish/", Environment: types.Mapping{}, @@ -39,35 +41,62 @@ func Test_processExtends(t *testing.T) { }, }) assert.NilError(t, err) - extFiles := map[string]string{} - file, err := processFile(context.TODO(), "testdata/publish/compose.yaml", project, extFiles) + project.ComposeFiles = []string{"testdata/publish/compose.yaml"} + + service := &composeService{} + layers, err := service.createLayers(context.TODO(), project, api.PublishOptions{ + WithEnvironment: true, + }) assert.NilError(t, err) - v := string(file) - assert.Equal(t, v, `name: test + published := string(layers[0].Data) + assert.Equal(t, published, `name: test services: test: extends: file: f8f9ede3d201ec37d5a5e3a77bbadab79af26035e53135e19571f50d541d390c.yaml service: foo + + string: + image: test + env_file: 5efca9cdbac9f5394c6c2e2094b1b42661f988f57fcab165a0bf72b205451af3.env + + list: + image: test + env_file: + - 5efca9cdbac9f5394c6c2e2094b1b42661f988f57fcab165a0bf72b205451af3.env + + mapping: + image: test + env_file: + - path: 5efca9cdbac9f5394c6c2e2094b1b42661f988f57fcab165a0bf72b205451af3.env `) - layers, err := processExtends(context.TODO(), project, extFiles) - assert.NilError(t, err) - - b, err := os.ReadFile("testdata/publish/common.yaml") - assert.NilError(t, err) - assert.DeepEqual(t, []v1.Descriptor{ + expectedLayers := []v1.Descriptor{ + { + MediaType: "application/vnd.docker.compose.file+yaml", + Annotations: map[string]string{ + "com.docker.compose.file": "compose.yaml", + "com.docker.compose.version": internal.Version}, + }, { MediaType: "application/vnd.docker.compose.file+yaml", - Digest: "sha256:d3ba84507b56ec783f4b6d24306b99a15285f0a23a835f0b668c2dbf9c59c241", - Size: 32, Annotations: map[string]string{ "com.docker.compose.extends": "true", - "com.docker.compose.file": "f8f9ede3d201ec37d5a5e3a77bbadab79af26035e53135e19571f50d541d390c.yaml", - "com.docker.compose.version": api.ComposeVersion, + "com.docker.compose.file": "f8f9ede3d201ec37d5a5e3a77bbadab79af26035e53135e19571f50d541d390c", + "com.docker.compose.version": internal.Version, }, - Data: b, }, - }, layers) + { + MediaType: "application/vnd.docker.compose.envfile", + Annotations: map[string]string{ + "com.docker.compose.envfile": "5efca9cdbac9f5394c6c2e2094b1b42661f988f57fcab165a0bf72b205451af3", + "com.docker.compose.version": internal.Version, + }, + }, + } + assert.DeepEqual(t, expectedLayers, layers, cmp.FilterPath(func(path cmp.Path) bool { + return !slices.Contains([]string{".Data", ".Digest", ".Size"}, path.String()) + }, cmp.Ignore())) + } diff --git a/pkg/compose/testdata/publish/compose.yaml b/pkg/compose/testdata/publish/compose.yaml index 413851cee..9c9f3659b 100644 --- a/pkg/compose/testdata/publish/compose.yaml +++ b/pkg/compose/testdata/publish/compose.yaml @@ -4,3 +4,17 @@ services: extends: file: common.yaml service: foo + + string: + image: test + env_file: test.env + + list: + image: test + env_file: + - test.env + + mapping: + image: test + env_file: + - path: test.env diff --git a/pkg/compose/testdata/publish/test.env b/pkg/compose/testdata/publish/test.env new file mode 100644 index 000000000..6e1f61b59 --- /dev/null +++ b/pkg/compose/testdata/publish/test.env @@ -0,0 +1 @@ +HELLO=WORLD \ No newline at end of file diff --git a/pkg/compose/transform/replace.go b/pkg/compose/transform/replace.go index 12ebfd353..a33959fd2 100644 --- a/pkg/compose/transform/replace.go +++ b/pkg/compose/transform/replace.go @@ -61,6 +61,52 @@ func ReplaceExtendsFile(in []byte, service string, value string) ([]byte, error) return replace(in, file.Line, file.Column, value), nil } +// ReplaceEnvFile changes value for service.extends.env_file in input yaml stream, preserving formatting +func ReplaceEnvFile(in []byte, service string, i int, value string) ([]byte, error) { + var doc yaml.Node + err := yaml.Unmarshal(in, &doc) + if err != nil { + return nil, err + } + if doc.Kind != yaml.DocumentNode { + return nil, fmt.Errorf("expected document kind %v, got %v", yaml.DocumentNode, doc.Kind) + } + root := doc.Content[0] + if root.Kind != yaml.MappingNode { + return nil, fmt.Errorf("expected document root to be a mapping, got %v", root.Kind) + } + + services, err := getMapping(root, "services") + if err != nil { + return nil, err + } + + target, err := getMapping(services, service) + if err != nil { + return nil, err + } + + envFile, err := getMapping(target, "env_file") + if err != nil { + return nil, err + } + + // env_file can be either a string, sequence of strings, or sequence of mappings with path attribute + if envFile.Kind == yaml.SequenceNode { + envFile = envFile.Content[i] + if envFile.Kind == yaml.MappingNode { + envFile, err = getMapping(envFile, "path") + if err != nil { + return nil, err + } + } + return replace(in, envFile.Line, envFile.Column, value), nil + } else { + return replace(in, envFile.Line, envFile.Column, value), nil + } + +} + func getMapping(root *yaml.Node, key string) (*yaml.Node, error) { var node *yaml.Node l := len(root.Content) diff --git a/pkg/remote/oci.go b/pkg/remote/oci.go index 33458e109..7e25543e1 100644 --- a/pkg/remote/oci.go +++ b/pkg/remote/oci.go @@ -217,8 +217,9 @@ func (g ociRemoteLoader) pullComposeFiles(ctx context.Context, local string, man func writeComposeFile(layer spec.Descriptor, i int, local string, content []byte) error { file := "compose.yaml" - if extends, ok := layer.Annotations["com.docker.compose.extends"]; ok { - if err := validatePathInBase(local, extends); err != nil { + if _, ok := layer.Annotations["com.docker.compose.extends"]; ok { + file = layer.Annotations["com.docker.compose.file"] + if err := validatePathInBase(local, file); err != nil { return err } }