Merge pull request #7155 from docker/bump-1.25.2-rc1

Bump 1.25.2-rc1

.circleci/config.yml

@@ -1,63 +0,0 @@
-version: 2
-jobs:
-  test:
-    macos:
-      xcode: "9.4.1"
-    steps:
-    - checkout
-    - run:
-        name: setup script
-        command: ./script/setup/osx
-    - run:
-        name: install tox
-        command: sudo pip install --upgrade tox==2.1.1 virtualenv==16.2.0
-    - run:
-        name: unit tests
-        command: tox -e py27,py37 -- tests/unit
-
-  build-osx-binary:
-    macos:
-      xcode: "9.4.1"
-    steps:
-      - checkout
-      - run:
-          name: upgrade python tools
-          command: sudo pip install --upgrade pip virtualenv==16.2.0
-      - run:
-         name: setup script
-         command: DEPLOYMENT_TARGET=10.11 ./script/setup/osx
-      - run:
-         name: build script
-         command: ./script/build/osx
-      - store_artifacts:
-          path: dist/docker-compose-Darwin-x86_64
-          destination: docker-compose-Darwin-x86_64
-      - deploy:
-          name: Deploy binary to bintray
-          command: |
-            OS_NAME=Darwin PKG_NAME=osx ./script/circle/bintray-deploy.sh
-
-  build-linux-binary:
-    machine:
-      enabled: true
-    steps:
-      - checkout
-      - run:
-          name: build Linux binary
-          command: ./script/build/linux
-      - store_artifacts:
-          path: dist/docker-compose-Linux-x86_64
-          destination: docker-compose-Linux-x86_64
-      - deploy:
-          name: Deploy binary to bintray
-          command: |
-            OS_NAME=Linux PKG_NAME=linux ./script/circle/bintray-deploy.sh
-
-
-workflows:
-  version: 2
-  all:
-    jobs:
-      - test
-      - build-linux-binary
-      - build-osx-binary

.dockerignore

@@ -11,3 +11,4 @@ docs/_site
 .tox
 **/__pycache__
 *.pyc
+Jenkinsfile

.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# GitHub code owners
+# See https://help.github.com/articles/about-codeowners/
+#
+# KEEP THIS FILE SORTED. Order is important. Last match takes precedence.
+
+*       @ndeloof @rumpl @ulyssessouza

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,9 @@
 ---
 name: Bug report
 about: Report a bug encountered while using docker-compose
+title: ''
+labels: kind/bug
+assignees: ''
 
 ---
 

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,9 @@
 ---
 name: Feature request
 about: Suggest an idea to improve Compose
+title: ''
+labels: kind/feature
+assignees: ''
 
 ---
 

.github/ISSUE_TEMPLATE/question-about-using-compose.md

@@ -1,6 +1,9 @@
 ---
 name: Question about using Compose
 about: This is not the appropriate channel
+title: ''
+labels: kind/question
+assignees: ''
 
 ---
 

.github/stale.yml

@@ -0,0 +1,59 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 180
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - kind/feature
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: true
+
+# Label to use when marking as stale
+staleLabel: stale
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+
+# Comment to post when removing the stale label.
+unmarkComment: >
+  This issue has been automatically marked as not stale anymore due to the recent activity.
+
+# Comment to post when closing a stale Issue or Pull Request.
+closeComment: >
+  This issue has been automatically closed because it had not recent activity during the stale period.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

CHANGELOG.md

@@ -1,18 +1,91 @@
 Change log
 ==========
 
-1.25.0-rc2 (2019-08-06)
+1.25.2 (2020-01-17)
 -------------------
 
 ### Features
 
+- Allow compatibility option with `COMPOSE_COMPATIBILITY` environment variable
+
+- Bump PyInstaller from 3.5 to 3.6
+
+- Bump pysocks from 1.6.7 to 1.7.1
+
+- Bump websocket-client from 0.32.0 to 0.57.0
+
+- Bump urllib3 from 1.24.2 to 1.25.7
+
+- Bump jsonschema from 3.0.1 to 3.2.0
+
+- Bump PyYAML from 4.2b1 to 5.3
+
+- Bump certifi from 2017.4.17 to 2019.11.28
+
+- Bump coverage from 4.5.4 to 5.0.3
+
+- Bump paramiko from 2.6.0 to 2.7.1
+
+- Bump cached-property from 1.3.0 to 1.5.1
+
+- Bump minor Linux and MacOSX dependencies
+
+### Bugfixes
+
+- Validate version format on formats 2+
+
+- Assume infinite terminal width when not running in a terminal
+
+1.25.1 (2020-01-06)
+-------------------
+
+### Features
+
+- Bump `pytest-cov` 2.8.1
+
+- Bump `flake8` 3.7.9
+
+- Bump `coverage` 4.5.4
+
+### Bugfixes
+
+- Decode APIError explanation to unicode before usage on start and create of a container
+
+- Reports when images that cannot be pulled and must be built
+
+- Discard label `com.docker.compose.filepaths` having None as value. Typically, when coming from stdin
+
+- Added OSX binary as a directory to solve slow start up time caused by MacOS Catalina binary scan
+
+- Passed in HOME env-var in container mode (running with `script/run/run.sh`)
+
+- Reverted behavior of "only pull images that we can't build" and replace by a warning informing the image we can't pull and must be built
+
+
+1.25.0 (2019-11-18)
+-------------------
+
+### Features
+
+- Set no-colors to true if CLICOLOR env variable is set to 0
+
+- Add working dir, config files and env file in service labels
+
+- Add dependencies for ARM build
+
+- Add BuildKit support, use `DOCKER_BUILDKIT=1` and `COMPOSE_DOCKER_CLI_BUILD=1`
+
+- Bump paramiko to 2.6.0
+
+- Add working dir, config files and env file in service labels
+
 - Add tag `docker-compose:latest`
 
 - Add `docker-compose:<version>-alpine` image/tag
 
 - Add `docker-compose:<version>-debian` image/tag
 
-- Bumped `docker-py` 4.0.1
+- Bumped `docker-py` 4.1.0
 
 - Supports `requests` up to 2.22.0 version
 
@@ -28,7 +101,7 @@ Change log
 
 - Added `--no-interpolate` to `docker-compose config`
 
-- Bump OpenSSL for macOS build (`1.1.0j` to `1.1.1a`)
+- Bump OpenSSL for macOS build (`1.1.0j` to `1.1.1c`)
 
 - Added `--no-rm` to `build` command
 
@@ -48,6 +121,20 @@ Change log
 
 ### Bugfixes
 
+- Make container service color deterministic, remove red from chosen colors
+
+- Fix non ascii chars error. Python2 only
+
+- Format image size as decimal to be align with Docker CLI
+
+- Use Python Posix support to get tty size
+
+- Fix same file 'extends' optimization
+
+- Use python POSIX support to get tty size
+
+- Format image size as decimal to be align with Docker CLI
+
 - Fixed stdin_open
 
 - Fixed `--remove-orphans` when used with `up --no-start`
@@ -64,7 +151,7 @@ Change log
 
 - Fixed race condition after pulling image
 
-- Fixed error on duplicate mount points.
+- Fixed error on duplicate mount points
 
 - Fixed merge on networks section
 
@@ -72,6 +159,13 @@ Change log
 
 - Fixed the presentation of failed services on 'docker-compose start' when containers are not available
 
+1.24.1 (2019-06-24)
+-------------------
+
+### Bugfixes
+
+- Fixed acceptance tests
+
 1.24.0 (2019-03-28)
 -------------------
 

Dockerfile

@@ -1,9 +1,9 @@
-ARG DOCKER_VERSION=18.09.7
-ARG PYTHON_VERSION=3.7.4
+ARG DOCKER_VERSION=19.03.5
+ARG PYTHON_VERSION=3.7.5
 ARG BUILD_ALPINE_VERSION=3.10
 ARG BUILD_DEBIAN_VERSION=slim-stretch
-ARG RUNTIME_ALPINE_VERSION=3.10.0
-ARG RUNTIME_DEBIAN_VERSION=stretch-20190708-slim
+ARG RUNTIME_ALPINE_VERSION=3.10.3
+ARG RUNTIME_DEBIAN_VERSION=stretch-20191118-slim
 
 ARG BUILD_PLATFORM=alpine
 
@@ -30,15 +30,18 @@ RUN apk add --no-cache \
 ENV BUILD_BOOTLOADER=1
 
 FROM python:${PYTHON_VERSION}-${BUILD_DEBIAN_VERSION} AS build-debian
-RUN apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install --no-install-recommends -y \
     curl \
     gcc \
     git \
     libc-dev \
+    libffi-dev \
     libgcc-6-dev \
+    libssl-dev \
     make \
     openssl \
-    python2.7-dev
+    python2.7-dev \
+    zlib1g-dev
 
 FROM build-${BUILD_PLATFORM} AS build
 COPY docker-compose-entrypoint.sh /usr/local/bin/

Dockerfile.s390x

@@ -1,4 +1,4 @@
-FROM s390x/alpine:3.6
+FROM s390x/alpine:3.10.1
 
 ARG COMPOSE_VERSION=1.16.1
 

Jenkinsfile

@@ -1,95 +1,112 @@
 #!groovy
 
-def buildImage = { String baseImage ->
-  def image
-  wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) {
-    stage("build image for \"${baseImage}\"") {
-      checkout(scm)
-      def imageName = "dockerbuildbot/compose:${baseImage}-${gitCommit()}"
-      image = docker.image(imageName)
-      try {
-        image.pull()
-      } catch (Exception exc) {
-        sh """GIT_COMMIT=\$(script/build/write-git-sha) && \\
-            docker build -t ${imageName} \\
-            --target build \\
-            --build-arg BUILD_PLATFORM="${baseImage}" \\
-            --build-arg GIT_COMMIT="${GIT_COMMIT}" \\
-            .\\
-        """
-        sh "docker push ${imageName}"
-        echo "${imageName}"
-        return imageName
-      }
-    }
-  }
-  echo "image.id: ${image.id}"
-  return image.id
-}
-
-def get_versions = { String imageId, int number ->
-  def docker_versions
-  wrappedNode(label: "ubuntu && !zfs") {
-    def result = sh(script: """docker run --rm \\
-        --entrypoint=/code/.tox/py27/bin/python \\
-        ${imageId} \\
-        /code/script/test/versions.py -n ${number} docker/docker-ce recent
-      """, returnStdout: true
-    )
-    docker_versions = result.split()
-  }
-  return docker_versions
-}
-
-def runTests = { Map settings ->
-  def dockerVersions = settings.get("dockerVersions", null)
-  def pythonVersions = settings.get("pythonVersions", null)
-  def baseImage = settings.get("baseImage", null)
-  def imageName = settings.get("image", null)
-
-  if (!pythonVersions) {
-    throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py37')`")
-  }
-  if (!dockerVersions) {
-    throw new Exception("Need Docker versions to test. e.g.: `runTests(dockerVersions: 'all')`")
-  }
-
-  { ->
-    wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) {
-      stage("test python=${pythonVersions} / docker=${dockerVersions} / baseImage=${baseImage}") {
-        checkout(scm)
-        def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim()
-        echo "Using local system's storage driver: ${storageDriver}"
-        sh """docker run \\
-          -t \\
-          --rm \\
-          --privileged \\
-          --volume="\$(pwd)/.git:/code/.git" \\
-          --volume="/var/run/docker.sock:/var/run/docker.sock" \\
-          -e "TAG=${imageName}" \\
-          -e "STORAGE_DRIVER=${storageDriver}" \\
-          -e "DOCKER_VERSIONS=${dockerVersions}" \\
-          -e "BUILD_NUMBER=\$BUILD_TAG" \\
-          -e "PY_TEST_VERSIONS=${pythonVersions}" \\
-          --entrypoint="script/test/ci" \\
-          ${imageName} \\
-          --verbose
-        """
-      }
-    }
-  }
-}
-
-def testMatrix = [failFast: true]
+def dockerVersions = ['19.03.5', '18.09.9']
 def baseImages = ['alpine', 'debian']
 def pythonVersions = ['py27', 'py37']
-baseImages.each { baseImage ->
-  def imageName = buildImage(baseImage)
-  get_versions(imageName, 2).each { dockerVersion ->
-    pythonVersions.each { pyVersion ->
-      testMatrix["${baseImage}_${dockerVersion}_${pyVersion}"] = runTests([baseImage: baseImage, image: imageName, dockerVersions: dockerVersion, pythonVersions: pyVersion])
+
+pipeline {
+    agent none
+
+    options {
+        skipDefaultCheckout(true)
+        buildDiscarder(logRotator(daysToKeepStr: '30'))
+        timeout(time: 2, unit: 'HOURS')
+        timestamps()
+    }
+
+    stages {
+        stage('Build test images') {
+            // TODO use declarative 1.5.0 `matrix` once available on CI
+            parallel {
+                stage('alpine') {
+                    agent {
+                        label 'ubuntu && amd64 && !zfs'
+                    }
+                    steps {
+                        buildImage('alpine')
+                    }
+                }
+                stage('debian') {
+                    agent {
+                        label 'ubuntu && amd64 && !zfs'
+                    }
+                    steps {
+                        buildImage('debian')
+                    }
+                }
+            }
+        }
+        stage('Test') {
+            steps {
+                // TODO use declarative 1.5.0 `matrix` once available on CI
+                script {
+                    def testMatrix = [:]
+                    baseImages.each { baseImage ->
+                      dockerVersions.each { dockerVersion ->
+                        pythonVersions.each { pythonVersion ->
+                          testMatrix["${baseImage}_${dockerVersion}_${pythonVersion}"] = runTests(dockerVersion, pythonVersion, baseImage)
+                        }
+                      }
+                    }
+
+                    parallel testMatrix
+                }
+            }
+        }
     }
-  }
 }
 
-parallel(testMatrix)
+
+def buildImage(baseImage) {
+    def scmvar = checkout(scm)
+    def imageName = "dockerbuildbot/compose:${baseImage}-${scmvar.GIT_COMMIT}"
+    image = docker.image(imageName)
+
+    withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
+        try {
+            image.pull()
+        } catch (Exception exc) {
+            ansiColor('xterm') {
+                sh """docker build -t ${imageName} \\
+                    --target build \\
+                    --build-arg BUILD_PLATFORM="${baseImage}" \\
+                    --build-arg GIT_COMMIT="${scmvar.GIT_COMMIT}" \\
+                    .\\
+                """
+                sh "docker push ${imageName}"
+            }
+            echo "${imageName}"
+            return imageName
+        }
+    }
+}
+
+def runTests(dockerVersion, pythonVersion, baseImage) {
+    return {
+        stage("python=${pythonVersion} docker=${dockerVersion} ${baseImage}") {
+            node("ubuntu && amd64 && !zfs") {
+                def scmvar = checkout(scm)
+                def imageName = "dockerbuildbot/compose:${baseImage}-${scmvar.GIT_COMMIT}"
+                def storageDriver = sh(script: "docker info -f \'{{.Driver}}\'", returnStdout: true).trim()
+                echo "Using local system's storage driver: ${storageDriver}"
+                withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
+                    sh """docker run \\
+                      -t \\
+                      --rm \\
+                      --privileged \\
+                      --volume="\$(pwd)/.git:/code/.git" \\
+                      --volume="/var/run/docker.sock:/var/run/docker.sock" \\
+                      -e "TAG=${imageName}" \\
+                      -e "STORAGE_DRIVER=${storageDriver}" \\
+                      -e "DOCKER_VERSIONS=${dockerVersion}" \\
+                      -e "BUILD_NUMBER=${env.BUILD_NUMBER}" \\
+                      -e "PY_TEST_VERSIONS=${pythonVersion}" \\
+                      --entrypoint="script/test/ci" \\
+                      ${imageName} \\
+                      --verbose
+                    """
+                }
+            }
+        }
+    }
+}

MAINTAINERS

@@ -11,6 +11,7 @@
 [Org]
 	[Org."Core maintainers"]
 		people = [
+			"ndeloof",
 			"rumpl",
 			"ulyssessouza",
 		]
@@ -77,6 +78,11 @@
 	Email = "mazz@houseofmnowster.com"
 	GitHub = "mnowster"
 
+	[people.ndeloof]
+	Name = "Nicolas De Loof"
+	Email = "nicolas.deloof@gmail.com"
+	GitHub = "ndeloof"
+
 	[people.rumpl]
 	Name = "Djordje Lukic"
 	Email = "djordje.lukic@docker.com"

README.md

@@ -2,15 +2,17 @@ Docker Compose
 ==============
 ![Docker Compose](logo.png?raw=true "Docker Compose Logo")
 
+## :exclamation: The docker-compose project announces that as Python 2 reaches it's EOL, versions 1.25.x will be the last to support it. For more information, please refer to this [issue](https://github.com/docker/compose/issues/6890).
+
 Compose is a tool for defining and running multi-container Docker applications.
 With Compose, you use a Compose file to configure your application's services.
 Then, using a single command, you create and start all the services
 from your configuration. To learn more about all the features of Compose
-see [the list of features](https://github.com/docker/docker.github.io/blob/master/compose/overview.md#features).
+see [the list of features](https://github.com/docker/docker.github.io/blob/master/compose/index.md#features).
 
 Compose is great for development, testing, and staging environments, as well as
 CI workflows. You can learn more about each case in
-[Common Use Cases](https://github.com/docker/docker.github.io/blob/master/compose/overview.md#common-use-cases).
+[Common Use Cases](https://github.com/docker/docker.github.io/blob/master/compose/index.md#common-use-cases).
 
 Using Compose is basically a three-step process.
 

Release.Jenkinsfile

@@ -0,0 +1,315 @@
+#!groovy
+
+def dockerVersions = ['19.03.5', '18.09.9']
+def baseImages = ['alpine', 'debian']
+def pythonVersions = ['py27', 'py37']
+
+pipeline {
+    agent none
+
+    options {
+        skipDefaultCheckout(true)
+        buildDiscarder(logRotator(daysToKeepStr: '30'))
+        timeout(time: 2, unit: 'HOURS')
+        timestamps()
+    }
+
+    stages {
+        stage('Build test images') {
+            // TODO use declarative 1.5.0 `matrix` once available on CI
+            parallel {
+                stage('alpine') {
+                    agent {
+                        label 'linux'
+                    }
+                    steps {
+                        buildImage('alpine')
+                    }
+                }
+                stage('debian') {
+                    agent {
+                        label 'linux'
+                    }
+                    steps {
+                        buildImage('debian')
+                    }
+                }
+            }
+        }
+        stage('Test') {
+            steps {
+                // TODO use declarative 1.5.0 `matrix` once available on CI
+                script {
+                    def testMatrix = [:]
+                    baseImages.each { baseImage ->
+                      dockerVersions.each { dockerVersion ->
+                        pythonVersions.each { pythonVersion ->
+                          testMatrix["${baseImage}_${dockerVersion}_${pythonVersion}"] = runTests(dockerVersion, pythonVersion, baseImage)
+                        }
+                      }
+                    }
+
+                    parallel testMatrix
+                }
+            }
+        }
+        stage('Generate Changelog') {
+            agent {
+                label 'linux'
+            }
+            steps {
+                checkout scm
+                withCredentials([string(credentialsId: 'github-compose-release-test-token', variable: 'GITHUB_TOKEN')]) {
+                    sh "./script/release/generate_changelog.sh"
+                }
+                archiveArtifacts artifacts: 'CHANGELOG.md'
+                stash( name: "changelog", includes: 'CHANGELOG.md' )
+            }
+        }
+        stage('Package') {
+            parallel {
+                stage('macosx binary') {
+                    agent {
+                        label 'mac-python'
+                    }
+                    steps {
+                        checkout scm
+                        sh './script/setup/osx'
+                        sh 'tox -e py27,py37 -- tests/unit'
+                        sh './script/build/osx'
+                        dir ('dist') {
+                          checksum('docker-compose-Darwin-x86_64')
+                          checksum('docker-compose-Darwin-x86_64.tgz')
+                        }
+                        archiveArtifacts artifacts: 'dist/*', fingerprint: true
+                        dir("dist") {
+                            stash name: "bin-darwin"
+                        }
+                    }
+                }
+                stage('linux binary') {
+                    agent {
+                        label 'linux'
+                    }
+                    steps {
+                        checkout scm
+                        sh ' ./script/build/linux'
+                        dir ('dist') {
+                          checksum('docker-compose-Linux-x86_64')
+                        }
+                        archiveArtifacts artifacts: 'dist/*', fingerprint: true
+                        dir("dist") {
+                            stash name: "bin-linux"
+                        }
+                    }
+                }
+                stage('windows binary') {
+                    agent {
+                        label 'windows-python'
+                    }
+                    environment {
+                        PATH = "$PATH;C:\\Python37;C:\\Python37\\Scripts"
+                    }
+                    steps {
+                        checkout scm
+                        bat 'tox.exe -e py27,py37 -- tests/unit'
+                        powershell '.\\script\\build\\windows.ps1'
+                        dir ('dist') {
+                            checksum('docker-compose-Windows-x86_64.exe')
+                        }
+                        archiveArtifacts artifacts: 'dist/*', fingerprint: true
+                        dir("dist") {
+                            stash name: "bin-win"
+                        }
+                    }
+                }
+                stage('alpine image') {
+                    agent {
+                        label 'linux'
+                    }
+                    steps {
+                        buildRuntimeImage('alpine')
+                    }
+                }
+                stage('debian image') {
+                    agent {
+                        label 'linux'
+                    }
+                    steps {
+                        buildRuntimeImage('debian')
+                    }
+                }
+            }
+        }
+        stage('Release') {
+            when {
+                buildingTag()
+            }
+            parallel {
+                stage('Pushing images') {
+                    agent {
+                        label 'linux'
+                    }
+                    steps {
+                        pushRuntimeImage('alpine')
+                        pushRuntimeImage('debian')
+                    }
+                }
+                stage('Creating Github Release') {
+                    agent {
+                        label 'linux'
+                    }
+                    steps {
+                        checkout scm
+                        sh 'mkdir -p dist'
+                        dir("dist") {
+                            unstash "bin-darwin"
+                            unstash "bin-linux"
+                            unstash "bin-win"
+                            unstash "changelog"
+                            githubRelease()
+                        }
+                    }
+                }
+                stage('Publishing Python packages') {
+                    agent {
+                        label 'linux'
+                    }
+                    steps {
+                        checkout scm
+                        withCredentials([[$class: "FileBinding", credentialsId: 'pypirc-docker-dsg-cibot', variable: 'PYPIRC']]) {
+                            sh """
+                                virtualenv venv-publish
+                                source venv-publish/bin/activate
+                                python setup.py sdist bdist_wheel
+                                pip install twine
+                                twine upload --config-file ${PYPIRC} ./dist/docker-compose-${env.TAG_NAME}.tar.gz ./dist/docker_compose-${env.TAG_NAME}-py2.py3-none-any.whl
+                            """
+                        }
+                    }
+                    post {
+                        sh 'deactivate; rm -rf venv-publish'
+                    }
+                }
+            }
+        }
+    }
+}
+
+
+def buildImage(baseImage) {
+    def scmvar = checkout(scm)
+    def imageName = "dockerbuildbot/compose:${baseImage}-${scmvar.GIT_COMMIT}"
+    image = docker.image(imageName)
+
+    withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
+        try {
+            image.pull()
+        } catch (Exception exc) {
+            ansiColor('xterm') {
+                sh """docker build -t ${imageName} \\
+                    --target build \\
+                    --build-arg BUILD_PLATFORM="${baseImage}" \\
+                    --build-arg GIT_COMMIT="${scmvar.GIT_COMMIT}" \\
+                    .\\
+                """
+                sh "docker push ${imageName}"
+            }
+            echo "${imageName}"
+            return imageName
+        }
+    }
+}
+
+def runTests(dockerVersion, pythonVersion, baseImage) {
+    return {
+        stage("python=${pythonVersion} docker=${dockerVersion} ${baseImage}") {
+            node("linux") {
+                def scmvar = checkout(scm)
+                def imageName = "dockerbuildbot/compose:${baseImage}-${scmvar.GIT_COMMIT}"
+                def storageDriver = sh(script: "docker info -f \'{{.Driver}}\'", returnStdout: true).trim()
+                echo "Using local system's storage driver: ${storageDriver}"
+                withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
+                    sh """docker run \\
+                      -t \\
+                      --rm \\
+                      --privileged \\
+                      --volume="\$(pwd)/.git:/code/.git" \\
+                      --volume="/var/run/docker.sock:/var/run/docker.sock" \\
+                      -e "TAG=${imageName}" \\
+                      -e "STORAGE_DRIVER=${storageDriver}" \\
+                      -e "DOCKER_VERSIONS=${dockerVersion}" \\
+                      -e "BUILD_NUMBER=${env.BUILD_NUMBER}" \\
+                      -e "PY_TEST_VERSIONS=${pythonVersion}" \\
+                      --entrypoint="script/test/ci" \\
+                      ${imageName} \\
+                      --verbose
+                    """
+                }
+            }
+        }
+    }
+}
+
+def buildRuntimeImage(baseImage) {
+    scmvar = checkout scm
+    def imageName = "docker/compose:${baseImage}-${env.BRANCH_NAME}"
+    ansiColor('xterm') {
+        sh """docker build -t ${imageName} \\
+            --build-arg BUILD_PLATFORM="${baseImage}" \\
+            --build-arg GIT_COMMIT="${scmvar.GIT_COMMIT.take(7)}" \\
+            .
+        """
+    }
+    sh "mkdir -p dist"
+    sh "docker save ${imageName} -o dist/docker-compose-${baseImage}.tar"
+    stash name: "compose-${baseImage}", includes: "dist/docker-compose-${baseImage}.tar"
+}
+
+def pushRuntimeImage(baseImage) {
+    unstash "compose-${baseImage}"
+    sh 'echo -n "${DOCKERHUB_CREDS_PSW}" | docker login --username "${DOCKERHUB_CREDS_USR}" --password-stdin'
+    sh "docker load -i dist/docker-compose-${baseImage}.tar"
+    withDockerRegistry(credentialsId: 'dockerbuildbot-hub.docker.com') {
+        sh "docker push docker/compose:${baseImage}-${env.TAG_NAME}"
+        if (baseImage == "alpine" && env.TAG_NAME != null) {
+            sh "docker tag docker/compose:alpine-${env.TAG_NAME} docker/compose:${env.TAG_NAME}"
+            sh "docker push docker/compose:${env.TAG_NAME}"
+        }
+    }
+}
+
+def githubRelease() {
+    withCredentials([string(credentialsId: 'github-compose-release-test-token', variable: 'GITHUB_TOKEN')]) {
+        def prerelease = !( env.TAG_NAME ==~ /v[0-9\.]+/ )
+        changelog = readFile "CHANGELOG.md"
+        def data = """{
+            \"tag_name\": \"${env.TAG_NAME}\",
+            \"name\": \"${env.TAG_NAME}\",
+            \"draft\": true,
+            \"prerelease\": ${prerelease},
+            \"body\" : \"${changelog}\"
+        """
+        echo $data
+
+        def url = "https://api.github.com/repos/docker/compose/releases"
+        def upload_url = sh(returnStdout: true, script: """
+            curl -sSf -H 'Authorization: token ${GITHUB_TOKEN}' -H 'Accept: application/json' -H 'Content-type: application/json' -X POST -d '$data' $url") \\
+            | jq '.upload_url | .[:rindex("{")]'
+        """)
+        sh("""
+            for f in * ; do
+                curl -sf -H 'Authorization: token ${GITHUB_TOKEN}' -H 'Accept: application/json' -H 'Content-type: application/octet-stream' \\
+                -X POST --data-binary @\$f ${upload_url}?name=\$f;
+            done
+        """)
+    }
+}
+
+def checksum(filepath) {
+    if (isUnix()) {
+        sh "openssl sha256 -r -out ${filepath}.sha256 ${filepath}"
+    } else {
+        powershell "(Get-FileHash -Path ${filepath} -Algorithm SHA256 | % hash) + ' *${filepath}' > ${filepath}.sha256"
+    }
+}

appveyor.yml

@@ -1,24 +0,0 @@
-
-version: '{branch}-{build}'
-
-install:
-  - "SET PATH=C:\\Python37-x64;C:\\Python37-x64\\Scripts;%PATH%"
-  - "python --version"
-  - "pip install tox==2.9.1 virtualenv==16.2.0"
-
-# Build the binary after tests
-build: false
-
-test_script:
-  - "tox -e py27,py37 -- tests/unit"
-  - ps: ".\\script\\build\\windows.ps1"
-
-artifacts:
-  - path: .\dist\docker-compose-Windows-x86_64.exe
-    name: "Compose Windows binary"
-
-deploy:
-  - provider: Environment
-    name: master-builds
-    on:
-      branch: master

compose/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-__version__ = '1.25.0-rc2'
+__version__ = '1.25.2-rc1'

compose/bundle.py

@@ -1,275 +0,0 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-import json
-import logging
-
-import six
-from docker.utils import split_command
-from docker.utils.ports import split_port
-
-from .cli.errors import UserError
-from .config.serialize import denormalize_config
-from .network import get_network_defs_for_service
-from .service import format_environment
-from .service import NoSuchImageError
-from .service import parse_repository_tag
-
-
-log = logging.getLogger(__name__)
-
-
-SERVICE_KEYS = {
-    'working_dir': 'WorkingDir',
-    'user': 'User',
-    'labels': 'Labels',
-}
-
-IGNORED_KEYS = {'build'}
-
-SUPPORTED_KEYS = {
-    'image',
-    'ports',
-    'expose',
-    'networks',
-    'command',
-    'environment',
-    'entrypoint',
-} | set(SERVICE_KEYS)
-
-VERSION = '0.1'
-
-
-class NeedsPush(Exception):
-    def __init__(self, image_name):
-        self.image_name = image_name
-
-
-class NeedsPull(Exception):
-    def __init__(self, image_name, service_name):
-        self.image_name = image_name
-        self.service_name = service_name
-
-
-class MissingDigests(Exception):
-    def __init__(self, needs_push, needs_pull):
-        self.needs_push = needs_push
-        self.needs_pull = needs_pull
-
-
-def serialize_bundle(config, image_digests):
-    return json.dumps(to_bundle(config, image_digests), indent=2, sort_keys=True)
-
-
-def get_image_digests(project, allow_push=False):
-    digests = {}
-    needs_push = set()
-    needs_pull = set()
-
-    for service in project.services:
-        try:
-            digests[service.name] = get_image_digest(
-                service,
-                allow_push=allow_push,
-            )
-        except NeedsPush as e:
-            needs_push.add(e.image_name)
-        except NeedsPull as e:
-            needs_pull.add(e.service_name)
-
-    if needs_push or needs_pull:
-        raise MissingDigests(needs_push, needs_pull)
-
-    return digests
-
-
-def get_image_digest(service, allow_push=False):
-    if 'image' not in service.options:
-        raise UserError(
-            "Service '{s.name}' doesn't define an image tag. An image name is "
-            "required to generate a proper image digest for the bundle. Specify "
-            "an image repo and tag with the 'image' option.".format(s=service))
-
-    _, _, separator = parse_repository_tag(service.options['image'])
-    # Compose file already uses a digest, no lookup required
-    if separator == '@':
-        return service.options['image']
-
-    digest = get_digest(service)
-
-    if digest:
-        return digest
-
-    if 'build' not in service.options:
-        raise NeedsPull(service.image_name, service.name)
-
-    if not allow_push:
-        raise NeedsPush(service.image_name)
-
-    return push_image(service)
-
-
-def get_digest(service):
-    digest = None
-    try:
-        image = service.image()
-        # TODO: pick a digest based on the image tag if there are multiple
-        # digests
-        if image['RepoDigests']:
-            digest = image['RepoDigests'][0]
-    except NoSuchImageError:
-        try:
-            # Fetch the image digest from the registry
-            distribution = service.get_image_registry_data()
-
-            if distribution['Descriptor']['digest']:
-                digest = '{image_name}@{digest}'.format(
-                    image_name=service.image_name,
-                    digest=distribution['Descriptor']['digest']
-                )
-        except NoSuchImageError:
-            raise UserError(
-                "Digest not found for service '{service}'. "
-                "Repository does not exist or may require 'docker login'"
-                .format(service=service.name))
-    return digest
-
-
-def push_image(service):
-    try:
-        digest = service.push()
-    except Exception:
-        log.error(
-            "Failed to push image for service '{s.name}'. Please use an "
-            "image tag that can be pushed to a Docker "
-            "registry.".format(s=service))
-        raise
-
-    if not digest:
-        raise ValueError("Failed to get digest for %s" % service.name)
-
-    repo, _, _ = parse_repository_tag(service.options['image'])
-    identifier = '{repo}@{digest}'.format(repo=repo, digest=digest)
-
-    # only do this if RepoDigests isn't already populated
-    image = service.image()
-    if not image['RepoDigests']:
-        # Pull by digest so that image['RepoDigests'] is populated for next time
-        # and we don't have to pull/push again
-        service.client.pull(identifier)
-        log.info("Stored digest for {}".format(service.image_name))
-
-    return identifier
-
-
-def to_bundle(config, image_digests):
-    if config.networks:
-        log.warning("Unsupported top level key 'networks' - ignoring")
-
-    if config.volumes:
-        log.warning("Unsupported top level key 'volumes' - ignoring")
-
-    config = denormalize_config(config)
-
-    return {
-        'Version': VERSION,
-        'Services': {
-            name: convert_service_to_bundle(
-                name,
-                service_dict,
-                image_digests[name],
-            )
-            for name, service_dict in config['services'].items()
-        },
-    }
-
-
-def convert_service_to_bundle(name, service_dict, image_digest):
-    container_config = {'Image': image_digest}
-
-    for key, value in service_dict.items():
-        if key in IGNORED_KEYS:
-            continue
-
-        if key not in SUPPORTED_KEYS:
-            log.warning("Unsupported key '{}' in services.{} - ignoring".format(key, name))
-            continue
-
-        if key == 'environment':
-            container_config['Env'] = format_environment({
-                envkey: envvalue for envkey, envvalue in value.items()
-                if envvalue
-            })
-            continue
-
-        if key in SERVICE_KEYS:
-            container_config[SERVICE_KEYS[key]] = value
-            continue
-
-    set_command_and_args(
-        container_config,
-        service_dict.get('entrypoint', []),
-        service_dict.get('command', []))
-    container_config['Networks'] = make_service_networks(name, service_dict)
-
-    ports = make_port_specs(service_dict)
-    if ports:
-        container_config['Ports'] = ports
-
-    return container_config
-
-
-# See https://github.com/docker/swarmkit/blob/agent/exec/container/container.go#L95
-def set_command_and_args(config, entrypoint, command):
-    if isinstance(entrypoint, six.string_types):
-        entrypoint = split_command(entrypoint)
-    if isinstance(command, six.string_types):
-        command = split_command(command)
-
-    if entrypoint:
-        config['Command'] = entrypoint + command
-        return
-
-    if command:
-        config['Args'] = command
-
-
-def make_service_networks(name, service_dict):
-    networks = []
-
-    for network_name, network_def in get_network_defs_for_service(service_dict).items():
-        for key in network_def.keys():
-            log.warning(
-                "Unsupported key '{}' in services.{}.networks.{} - ignoring"
-                .format(key, name, network_name))
-
-        networks.append(network_name)
-
-    return networks
-
-
-def make_port_specs(service_dict):
-    ports = []
-
-    internal_ports = [
-        internal_port
-        for port_def in service_dict.get('ports', [])
-        for internal_port in split_port(port_def)[0]
-    ]
-
-    internal_ports += service_dict.get('expose', [])
-
-    for internal_port in internal_ports:
-        spec = make_port_spec(internal_port)
-        if spec not in ports:
-            ports.append(spec)
-
-    return ports
-
-
-def make_port_spec(value):
-    components = six.text_type(value).partition('/')
-    return {
-        'Protocol': components[2] or 'tcp',
-        'Port': int(components[0]),
-    }

compose/cli/colors.py

@@ -41,9 +41,9 @@ for (name, code) in get_pairs():
 
 
 def rainbow():
-    cs = ['cyan', 'yellow', 'green', 'magenta', 'red', 'blue',
+    cs = ['cyan', 'yellow', 'green', 'magenta', 'blue',
           'intense_cyan', 'intense_yellow', 'intense_green',
-          'intense_magenta', 'intense_red', 'intense_blue']
+          'intense_magenta', 'intense_blue']
 
     for c in cs:
         yield globals()[c]

compose/cli/command.py

@@ -13,6 +13,9 @@ from .. import config
 from .. import parallel
 from ..config.environment import Environment
 from ..const import API_VERSIONS
+from ..const import LABEL_CONFIG_FILES
+from ..const import LABEL_ENVIRONMENT_FILE
+from ..const import LABEL_WORKING_DIR
 from ..project import Project
 from .docker_client import docker_client
 from .docker_client import get_tls_version
@@ -37,7 +40,8 @@ SILENT_COMMANDS = {
 }
 
 
-def project_from_options(project_dir, options, additional_options={}):
+def project_from_options(project_dir, options, additional_options=None):
+    additional_options = additional_options or {}
     override_dir = options.get('--project-directory')
     environment_file = options.get('--env-file')
     environment = Environment.from_env_file(override_dir or project_dir, environment_file)
@@ -56,8 +60,9 @@ def project_from_options(project_dir, options, additional_options={}):
         tls_config=tls_config_from_options(options, environment),
         environment=environment,
         override_dir=override_dir,
-        compatibility=options.get('--compatibility'),
-        interpolate=(not additional_options.get('--no-interpolate'))
+        compatibility=compatibility_from_options(project_dir, options, environment),
+        interpolate=(not additional_options.get('--no-interpolate')),
+        environment_file=environment_file
     )
 
 
@@ -77,7 +82,8 @@ def set_parallel_limit(environment):
         parallel.GlobalLimit.set_global_limit(parallel_limit)
 
 
-def get_config_from_options(base_dir, options, additional_options={}):
+def get_config_from_options(base_dir, options, additional_options=None):
+    additional_options = additional_options or {}
     override_dir = options.get('--project-directory')
     environment_file = options.get('--env-file')
     environment = Environment.from_env_file(override_dir or base_dir, environment_file)
@@ -86,7 +92,7 @@ def get_config_from_options(base_dir, options, additional_options={}):
     )
     return config.load(
         config.find(base_dir, config_path, environment, override_dir),
-        options.get('--compatibility'),
+        compatibility_from_options(config_path, options, environment),
         not additional_options.get('--no-interpolate')
     )
 
@@ -125,7 +131,7 @@ def get_client(environment, verbose=False, version=None, tls_config=None, host=N
 
 def get_project(project_dir, config_path=None, project_name=None, verbose=False,
                 host=None, tls_config=None, environment=None, override_dir=None,
-                compatibility=False, interpolate=True):
+                compatibility=False, interpolate=True, environment_file=None):
     if not environment:
         environment = Environment.from_env_file(project_dir)
     config_details = config.find(project_dir, config_path, environment, override_dir)
@@ -145,10 +151,40 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False,
 
     with errors.handle_connection_errors(client):
         return Project.from_config(
-            project_name, config_data, client, environment.get('DOCKER_DEFAULT_PLATFORM')
+            project_name,
+            config_data,
+            client,
+            environment.get('DOCKER_DEFAULT_PLATFORM'),
+            execution_context_labels(config_details, environment_file),
         )
 
 
+def execution_context_labels(config_details, environment_file):
+    extra_labels = [
+        '{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir))
+    ]
+
+    if not use_config_from_stdin(config_details):
+        extra_labels.append('{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details)))
+
+    if environment_file is not None:
+        extra_labels.append('{0}={1}'.format(LABEL_ENVIRONMENT_FILE,
+                                             os.path.normpath(environment_file)))
+    return extra_labels
+
+
+def use_config_from_stdin(config_details):
+    for c in config_details.config_files:
+        if not c.filename:
+            return True
+    return False
+
+
+def config_files_label(config_details):
+    return ",".join(
+        map(str, (os.path.normpath(c.filename) for c in config_details.config_files)))
+
+
 def get_project_name(working_dir, project_name=None, environment=None):
     def normalize_name(name):
         return re.sub(r'[^-_a-z0-9]', '', name.lower())
@@ -164,3 +200,13 @@ def get_project_name(working_dir, project_name=None, environment=None):
         return normalize_name(project)
 
     return 'default'
+
+
+def compatibility_from_options(working_dir, options=None, environment=None):
+    """Get compose v3 compatibility from --compatibility option
+       or from COMPOSE_COMPATIBILITY environment variable."""
+
+    compatibility_option = options.get('--compatibility')
+    compatibility_environment = environment.get_boolean('COMPOSE_COMPATIBILITY')
+
+    return compatibility_option or compatibility_environment

compose/cli/formatter.py

@@ -2,25 +2,37 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import logging
-import os
+import shutil
 
 import six
 import texttable
 
 from compose.cli import colors
 
+if hasattr(shutil, "get_terminal_size"):
+    from shutil import get_terminal_size
+else:
+    from backports.shutil_get_terminal_size import get_terminal_size
+
 
 def get_tty_width():
-    tty_size = os.popen('stty size 2> /dev/null', 'r').read().split()
-    if len(tty_size) != 2:
+    try:
+        # get_terminal_size can't determine the size if compose is piped
+        # to another command. But in such case it doesn't make sense to
+        # try format the output by terminal size as this output is consumed
+        # by another command. So let's pretend we have a huge terminal so
+        # output is single-lined
+        width, _ = get_terminal_size(fallback=(999, 0))
+        return int(width)
+    except OSError:
         return 0
-    _, width = tty_size
-    return int(width)
 
 
-class Formatter(object):
+class Formatter:
     """Format tabular data for printing."""
-    def table(self, headers, rows):
+
+    @staticmethod
+    def table(headers, rows):
         table = texttable.Texttable(max_width=get_tty_width())
         table.set_cols_dtype(['t' for h in headers])
         table.add_rows([headers] + rows)

compose/cli/log_printer.py

@@ -134,7 +134,10 @@ def build_thread(container, presenter, queue, log_args):
 def build_thread_map(initial_containers, presenters, thread_args):
     return {
         container.id: build_thread(container, next(presenters), *thread_args)
-        for container in initial_containers
+        # Container order is unspecified, so they are sorted by name in order to make
+        # container:presenter (log color) assignment deterministic when given a list of containers
+        # with the same names.
+        for container in sorted(initial_containers, key=lambda c: c.name)
     }
 
 
@@ -230,7 +233,13 @@ def watch_events(thread_map, event_stream, presenters, thread_args):
 
         # Container crashed so we should reattach to it
         if event['id'] in crashed_containers:
-            event['container'].attach_log_stream()
+            container = event['container']
+            if not container.is_restarting:
+                try:
+                    container.attach_log_stream()
+                except APIError:
+                    # Just ignore errors when reattaching to already crashed containers
+                    pass
             crashed_containers.remove(event['id'])
 
         thread_map[event['id']] = build_thread(

compose/cli/main.py

@@ -6,6 +6,7 @@ import contextlib
 import functools
 import json
 import logging
+import os
 import pipes
 import re
 import subprocess
@@ -14,14 +15,12 @@ from distutils.spawn import find_executable
 from inspect import getdoc
 from operator import attrgetter
 
-import docker
+import docker.errors
+import docker.utils
 
 from . import errors
 from . import signals
 from .. import __version__
-from ..bundle import get_image_digests
-from ..bundle import MissingDigests
-from ..bundle import serialize_bundle
 from ..config import ConfigurationError
 from ..config import parse_environment
 from ..config import parse_labels
@@ -33,6 +32,8 @@ from ..const import COMPOSEFILE_V2_2 as V2_2
 from ..const import IS_WINDOWS_PLATFORM
 from ..errors import StreamParseError
 from ..progress_stream import StreamOutputError
+from ..project import get_image_digests
+from ..project import MissingDigests
 from ..project import NoSuchService
 from ..project import OneOffFilter
 from ..project import ProjectError
@@ -102,9 +103,9 @@ def dispatch():
     options, handler, command_options = dispatcher.parse(sys.argv[1:])
     setup_console_handler(console_handler,
                           options.get('--verbose'),
-                          options.get('--no-ansi'),
+                          set_no_color_if_clicolor(options.get('--no-ansi')),
                           options.get("--log-level"))
-    setup_parallel_logger(options.get('--no-ansi'))
+    setup_parallel_logger(set_no_color_if_clicolor(options.get('--no-ansi')))
     if options.get('--no-ansi'):
         command_options['--no-color'] = True
     return functools.partial(perform_command, options, handler, command_options)
@@ -212,7 +213,6 @@ class TopLevelCommand(object):
 
     Commands:
       build              Build or rebuild services
-      bundle             Generate a Docker bundle from the Compose file
       config             Validate and view the Compose file
       create             Create services
       down               Stop and remove containers, networks, images, and volumes
@@ -263,14 +263,17 @@ class TopLevelCommand(object):
         Usage: build [options] [--build-arg key=val...] [SERVICE...]
 
         Options:
+            --build-arg key=val     Set build-time variables for services.
             --compress              Compress the build context using gzip.
             --force-rm              Always remove intermediate containers.
+            -m, --memory MEM        Set memory limit for the build container.
             --no-cache              Do not use cache when building the image.
             --no-rm                 Do not remove intermediate containers after a successful build.
-            --pull                  Always attempt to pull a newer version of the image.
-            -m, --memory MEM        Sets memory limit for the build container.
-            --build-arg key=val     Set build-time variables for services.
             --parallel              Build images in parallel.
+            --progress string       Set type of progress output (auto, plain, tty).
+                                    EXPERIMENTAL flag for native builder.
+                                    To enable, run with COMPOSE_DOCKER_CLI_BUILD=1)
+            --pull                  Always attempt to pull a newer version of the image.
             -q, --quiet             Don't print anything to STDOUT
         """
         service_names = options['SERVICE']
@@ -283,6 +286,8 @@ class TopLevelCommand(object):
                 )
             build_args = resolve_build_args(build_args, self.toplevel_environment)
 
+        native_builder = self.toplevel_environment.get_boolean('COMPOSE_DOCKER_CLI_BUILD')
+
         self.project.build(
             service_names=options['SERVICE'],
             no_cache=bool(options.get('--no-cache', False)),
@@ -293,41 +298,11 @@ class TopLevelCommand(object):
             build_args=build_args,
             gzip=options.get('--compress', False),
             parallel_build=options.get('--parallel', False),
-            silent=options.get('--quiet', False)
+            silent=options.get('--quiet', False),
+            cli=native_builder,
+            progress=options.get('--progress'),
         )
 
-    def bundle(self, options):
-        """
-        Generate a Distributed Application Bundle (DAB) from the Compose file.
-
-        Images must have digests stored, which requires interaction with a
-        Docker registry. If digests aren't stored for all images, you can fetch
-        them with `docker-compose pull` or `docker-compose push`. To push images
-        automatically when bundling, pass `--push-images`. Only services with
-        a `build` option specified will have their images pushed.
-
-        Usage: bundle [options]
-
-        Options:
-            --push-images              Automatically push images for any services
-                                       which have a `build` option specified.
-
-            -o, --output PATH          Path to write the bundle file to.
-                                       Defaults to "<project name>.dab".
-        """
-        compose_config = get_config_from_options('.', self.toplevel_options)
-
-        output = options["--output"]
-        if not output:
-            output = "{}.dab".format(self.project.name)
-
-        image_digests = image_digests_for_project(self.project, options['--push-images'])
-
-        with open(output, 'w') as f:
-            f.write(serialize_bundle(compose_config, image_digests))
-
-        log.info("Wrote bundle to {}".format(output))
-
     def config(self, options):
         """
         Validate and view the Compose file.
@@ -613,7 +588,7 @@ class TopLevelCommand(object):
                 image_id,
                 size
             ])
-        print(Formatter().table(headers, rows))
+        print(Formatter.table(headers, rows))
 
     def kill(self, options):
         """
@@ -659,7 +634,7 @@ class TopLevelCommand(object):
         log_printer_from_project(
             self.project,
             containers,
-            options['--no-color'],
+            set_no_color_if_clicolor(options['--no-color']),
             log_args,
             event_stream=self.project.events(service_names=options['SERVICE'])).run()
 
@@ -747,7 +722,7 @@ class TopLevelCommand(object):
                     container.human_readable_state,
                     container.human_readable_ports,
                 ])
-            print(Formatter().table(headers, rows))
+            print(Formatter.table(headers, rows))
 
     def pull(self, options):
         """
@@ -987,7 +962,7 @@ class TopLevelCommand(object):
                 rows.append(process)
 
             print(container.name)
-            print(Formatter().table(headers, rows))
+            print(Formatter.table(headers, rows))
 
     def unpause(self, options):
         """
@@ -1037,6 +1012,7 @@ class TopLevelCommand(object):
             --build                    Build images before starting containers.
             --abort-on-container-exit  Stops all containers if any container was
                                        stopped. Incompatible with -d.
+            --attach-dependencies      Attach to dependent containers
             -t, --timeout TIMEOUT      Use this timeout in seconds for container
                                        shutdown when attached or when containers are
                                        already running. (default: 10)
@@ -1058,19 +1034,23 @@ class TopLevelCommand(object):
         remove_orphans = options['--remove-orphans']
         detached = options.get('--detach')
         no_start = options.get('--no-start')
+        attach_dependencies = options.get('--attach-dependencies')
 
-        if detached and (cascade_stop or exit_value_from):
-            raise UserError("--abort-on-container-exit and -d cannot be combined.")
+        if detached and (cascade_stop or exit_value_from or attach_dependencies):
+            raise UserError(
+                "-d cannot be combined with --abort-on-container-exit or --attach-dependencies.")
 
         ignore_orphans = self.toplevel_environment.get_boolean('COMPOSE_IGNORE_ORPHANS')
 
         if ignore_orphans and remove_orphans:
             raise UserError("COMPOSE_IGNORE_ORPHANS and --remove-orphans cannot be combined.")
 
-        opts = ['--detach', '--abort-on-container-exit', '--exit-code-from']
+        opts = ['--detach', '--abort-on-container-exit', '--exit-code-from', '--attach-dependencies']
         for excluded in [x for x in opts if options.get(x) and no_start]:
             raise UserError('--no-start and {} cannot be combined.'.format(excluded))
 
+        native_builder = self.toplevel_environment.get_boolean('COMPOSE_DOCKER_CLI_BUILD')
+
         with up_shutdown_context(self.project, service_names, timeout, detached):
             warn_for_swarm_mode(self.project.client)
 
@@ -1090,6 +1070,7 @@ class TopLevelCommand(object):
                     reset_container_image=rebuild,
                     renew_anonymous_volumes=options.get('--renew-anon-volumes'),
                     silent=options.get('--quiet-pull'),
+                    cli=native_builder,
                 )
 
             try:
@@ -1109,12 +1090,15 @@ class TopLevelCommand(object):
             if detached or no_start:
                 return
 
-            attached_containers = filter_containers_to_service_names(to_attach, service_names)
+            attached_containers = filter_attached_containers(
+                to_attach,
+                service_names,
+                attach_dependencies)
 
             log_printer = log_printer_from_project(
                 self.project,
                 attached_containers,
-                options['--no-color'],
+                set_no_color_if_clicolor(options['--no-color']),
                 {'follow': True},
                 cascade_stop,
                 event_stream=self.project.events(service_names=service_names))
@@ -1205,12 +1189,10 @@ def timeout_from_opts(options):
     return None if timeout is None else int(timeout)
 
 
-def image_digests_for_project(project, allow_push=False):
+def image_digests_for_project(project):
     try:
-        return get_image_digests(
-            project,
-            allow_push=allow_push
-        )
+        return get_image_digests(project)
+
     except MissingDigests as e:
         def list_images(images):
             return "\n".join("    {}".format(name) for name in sorted(images))
@@ -1416,8 +1398,8 @@ def log_printer_from_project(
         log_args=log_args)
 
 
-def filter_containers_to_service_names(containers, service_names):
-    if not service_names:
+def filter_attached_containers(containers, service_names, attach_dependencies=False):
+    if attach_dependencies or not service_names:
         return containers
 
     return [
@@ -1592,3 +1574,7 @@ def warn_for_swarm_mode(client):
             "To deploy your application across the swarm, "
             "use `docker stack deploy`.\n"
         )
+
+
+def set_no_color_if_clicolor(no_color_flag):
+    return no_color_flag or os.environ.get('CLICOLOR') == "0"

compose/cli/utils.py

@@ -133,12 +133,12 @@ def generate_user_agent():
 
 def human_readable_file_size(size):
     suffixes = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', ]
-    order = int(math.log(size, 2) / 10) if size else 0
+    order = int(math.log(size, 1000)) if size else 0
     if order >= len(suffixes):
         order = len(suffixes) - 1
 
     return '{0:.4g} {1}'.format(
-        size / float(1 << (order * 10)),
+        size / pow(10, order * 3),
         suffixes[order]
     )
 

compose/config/config.py

@@ -5,6 +5,7 @@ import functools
 import io
 import logging
 import os
+import re
 import string
 import sys
 from collections import namedtuple
@@ -214,6 +215,12 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
                 .format(self.filename, VERSION_EXPLANATION)
             )
 
+        version_pattern = re.compile(r"^[2-9]+(\.\d+)?$")
+        if not version_pattern.match(version):
+            raise ConfigurationError(
+                'Version "{}" in "{}" is invalid.'
+                .format(version, self.filename))
+
         if version == '2':
             return const.COMPOSEFILE_V2_0
 
@@ -615,7 +622,7 @@ class ServiceExtendsResolver(object):
         config_path = self.get_extended_config_path(extends)
         service_name = extends['service']
 
-        if config_path == self.config_file.filename:
+        if config_path == os.path.abspath(self.config_file.filename):
             try:
                 service_config = self.config_file.get_service(service_name)
             except KeyError:

compose/const.py

@@ -11,6 +11,9 @@ IS_WINDOWS_PLATFORM = (sys.platform == "win32")
 LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number'
 LABEL_ONE_OFF = 'com.docker.compose.oneoff'
 LABEL_PROJECT = 'com.docker.compose.project'
+LABEL_WORKING_DIR = 'com.docker.compose.project.working_dir'
+LABEL_CONFIG_FILES = 'com.docker.compose.project.config_files'
+LABEL_ENVIRONMENT_FILE = 'com.docker.compose.project.environment_file'
 LABEL_SERVICE = 'com.docker.compose.service'
 LABEL_NETWORK = 'com.docker.compose.network'
 LABEL_VERSION = 'com.docker.compose.version'

compose/network.py

@@ -226,7 +226,7 @@ def check_remote_network_config(remote, local):
         raise NetworkConfigChangedError(local.true_name, 'enable_ipv6')
 
     local_labels = local.labels or {}
-    remote_labels = remote.get('Labels', {})
+    remote_labels = remote.get('Labels') or {}
     for k in set.union(set(remote_labels.keys()), set(local_labels.keys())):
         if k.startswith('com.docker.'):  # We are only interested in user-specified labels
             continue

compose/progress_stream.py

@@ -114,3 +114,13 @@ def get_digest_from_push(events):
         if digest:
             return digest
     return None
+
+
+def read_status(event):
+    status = event['status'].lower()
+    if 'progressDetail' in event:
+        detail = event['progressDetail']
+        if 'current' in detail and 'total' in detail:
+            percentage = float(detail['current']) / float(detail['total'])
+            status = '{} ({:.1%})'.format(status, percentage)
+    return status

compose/project.py

@@ -6,13 +6,17 @@ import logging
 import operator
 import re
 from functools import reduce
+from os import path
 
 import enum
 import six
 from docker.errors import APIError
+from docker.errors import ImageNotFound
+from docker.errors import NotFound
 from docker.utils import version_lt
 
 from . import parallel
+from .cli.errors import UserError
 from .config import ConfigurationError
 from .config.config import V1
 from .config.sort_services import get_container_name_from_network_mode
@@ -24,11 +28,13 @@ from .container import Container
 from .network import build_networks
 from .network import get_networks
 from .network import ProjectNetworks
+from .progress_stream import read_status
 from .service import BuildAction
 from .service import ContainerNetworkMode
 from .service import ContainerPidMode
 from .service import ConvergenceStrategy
 from .service import NetworkMode
+from .service import NoSuchImageError
 from .service import parse_repository_tag
 from .service import PidMode
 from .service import Service
@@ -38,7 +44,6 @@ from .utils import microseconds_from_time_nano
 from .utils import truncate_string
 from .volume import ProjectVolumes
 
-
 log = logging.getLogger(__name__)
 
 
@@ -82,10 +87,11 @@ class Project(object):
         return labels
 
     @classmethod
-    def from_config(cls, name, config_data, client, default_platform=None):
+    def from_config(cls, name, config_data, client, default_platform=None, extra_labels=None):
         """
         Construct a Project from a config.Config object.
         """
+        extra_labels = extra_labels or []
         use_networking = (config_data.version and config_data.version != V1)
         networks = build_networks(name, config_data, client)
         project_networks = ProjectNetworks.from_services(
@@ -135,6 +141,7 @@ class Project(object):
                     pid_mode=pid_mode,
                     platform=service_dict.pop('platform', None),
                     default_platform=default_platform,
+                    extra_labels=extra_labels,
                     **service_dict)
             )
 
@@ -355,7 +362,8 @@ class Project(object):
         return containers
 
     def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None,
-              build_args=None, gzip=False, parallel_build=False, rm=True, silent=False):
+              build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False,
+              progress=None):
 
         services = []
         for service in self.get_services(service_names):
@@ -364,8 +372,18 @@ class Project(object):
             elif not silent:
                 log.info('%s uses an image, skipping' % service.name)
 
+        if cli:
+            log.warning("Native build is an experimental feature and could change at any time")
+            if parallel_build:
+                log.warning("Flag '--parallel' is ignored when building with "
+                            "COMPOSE_DOCKER_CLI_BUILD=1")
+            if gzip:
+                log.warning("Flag '--compress' is ignored when building with "
+                            "COMPOSE_DOCKER_CLI_BUILD=1")
+
         def build_service(service):
-            service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent)
+            service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress)
+
         if parallel_build:
             _, errors = parallel.parallel_execute(
                 services,
@@ -509,8 +527,12 @@ class Project(object):
            reset_container_image=False,
            renew_anonymous_volumes=False,
            silent=False,
+           cli=False,
            ):
 
+        if cli:
+            log.warning("Native build is an experimental feature and could change at any time")
+
         self.initialize()
         if not ignore_orphans:
             self.find_orphan_containers(remove_orphans)
@@ -523,7 +545,7 @@ class Project(object):
             include_deps=start_deps)
 
         for svc in services:
-            svc.ensure_image_exists(do_build=do_build, silent=silent)
+            svc.ensure_image_exists(do_build=do_build, silent=silent, cli=cli)
         plans = self._get_convergence_plans(
             services, strategy, always_recreate_deps=always_recreate_deps)
 
@@ -603,49 +625,68 @@ class Project(object):
     def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False,
              include_deps=False):
         services = self.get_services(service_names, include_deps)
-        images_to_build = {service.image_name for service in services if service.can_be_built()}
-        services_to_pull = [service for service in services if service.image_name not in images_to_build]
-
-        msg = not silent and 'Pulling' or None
 
         if parallel_pull:
-            def pull_service(service):
-                strm = service.pull(ignore_pull_failures, True, stream=True)
-                if strm is None:  # Attempting to pull service with no `image` key is a no-op
-                    return
+            self.parallel_pull(services, silent=silent)
 
+        else:
+            must_build = []
+            for service in services:
+                try:
+                    service.pull(ignore_pull_failures, silent=silent)
+                except (ImageNotFound, NotFound):
+                    if service.can_be_built():
+                        must_build.append(service.name)
+                    else:
+                        raise
+
+            if len(must_build):
+                log.warning('Some service image(s) must be built from source by running:\n'
+                            '    docker-compose build {}'
+                            .format(' '.join(must_build)))
+
+    def parallel_pull(self, services, ignore_pull_failures=False, silent=False):
+        msg = 'Pulling' if not silent else None
+        must_build = []
+
+        def pull_service(service):
+            strm = service.pull(ignore_pull_failures, True, stream=True)
+
+            if strm is None:  # Attempting to pull service with no `image` key is a no-op
+                return
+
+            try:
                 writer = parallel.get_stream_writer()
-
                 for event in strm:
                     if 'status' not in event:
                         continue
-                    status = event['status'].lower()
-                    if 'progressDetail' in event:
-                        detail = event['progressDetail']
-                        if 'current' in detail and 'total' in detail:
-                            percentage = float(detail['current']) / float(detail['total'])
-                            status = '{} ({:.1%})'.format(status, percentage)
-
+                    status = read_status(event)
                     writer.write(
                         msg, service.name, truncate_string(status), lambda s: s
                     )
+            except (ImageNotFound, NotFound):
+                if service.can_be_built():
+                    must_build.append(service.name)
+                else:
+                    raise
 
-            _, errors = parallel.parallel_execute(
-                services_to_pull,
-                pull_service,
-                operator.attrgetter('name'),
-                msg,
-                limit=5,
-            )
-            if len(errors):
-                combined_errors = '\n'.join([
-                    e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
-                ])
-                raise ProjectError(combined_errors)
+        _, errors = parallel.parallel_execute(
+            services,
+            pull_service,
+            operator.attrgetter('name'),
+            msg,
+            limit=5,
+        )
 
-        else:
-            for service in services_to_pull:
-                service.pull(ignore_pull_failures, silent=silent)
+        if len(must_build):
+            log.warning('Some service image(s) must be built from source by running:\n'
+                        '    docker-compose build {}'
+                        .format(' '.join(must_build)))
+        if len(errors):
+            combined_errors = '\n'.join([
+                e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
+            ])
+            raise ProjectError(combined_errors)
 
     def push(self, service_names=None, ignore_push_failures=False):
         unique_images = set()
@@ -793,11 +834,104 @@ def get_secrets(service, service_secrets, secret_defs):
                 )
             )
 
-        secrets.append({'secret': secret, 'file': secret_def.get('file')})
+        secret_file = secret_def.get('file')
+        if not path.isfile(str(secret_file)):
+            log.warning(
+                "Service \"{service}\" uses an undefined secret file \"{secret_file}\", "
+                "the following file should be created \"{secret_file}\"".format(
+                    service=service, secret_file=secret_file
+                )
+            )
+        secrets.append({'secret': secret, 'file': secret_file})
 
     return secrets
 
 
+def get_image_digests(project):
+    digests = {}
+    needs_push = set()
+    needs_pull = set()
+
+    for service in project.services:
+        try:
+            digests[service.name] = get_image_digest(service)
+        except NeedsPush as e:
+            needs_push.add(e.image_name)
+        except NeedsPull as e:
+            needs_pull.add(e.service_name)
+
+    if needs_push or needs_pull:
+        raise MissingDigests(needs_push, needs_pull)
+
+    return digests
+
+
+def get_image_digest(service):
+    if 'image' not in service.options:
+        raise UserError(
+            "Service '{s.name}' doesn't define an image tag. An image name is "
+            "required to generate a proper image digest. Specify an image repo "
+            "and tag with the 'image' option.".format(s=service))
+
+    _, _, separator = parse_repository_tag(service.options['image'])
+    # Compose file already uses a digest, no lookup required
+    if separator == '@':
+        return service.options['image']
+
+    digest = get_digest(service)
+
+    if digest:
+        return digest
+
+    if 'build' not in service.options:
+        raise NeedsPull(service.image_name, service.name)
+
+    raise NeedsPush(service.image_name)
+
+
+def get_digest(service):
+    digest = None
+    try:
+        image = service.image()
+        # TODO: pick a digest based on the image tag if there are multiple
+        # digests
+        if image['RepoDigests']:
+            digest = image['RepoDigests'][0]
+    except NoSuchImageError:
+        try:
+            # Fetch the image digest from the registry
+            distribution = service.get_image_registry_data()
+
+            if distribution['Descriptor']['digest']:
+                digest = '{image_name}@{digest}'.format(
+                    image_name=service.image_name,
+                    digest=distribution['Descriptor']['digest']
+                )
+        except NoSuchImageError:
+            raise UserError(
+                "Digest not found for service '{service}'. "
+                "Repository does not exist or may require 'docker login'"
+                .format(service=service.name))
+    return digest
+
+
+class MissingDigests(Exception):
+    def __init__(self, needs_push, needs_pull):
+        self.needs_push = needs_push
+        self.needs_pull = needs_pull
+
+
+class NeedsPush(Exception):
+    def __init__(self, image_name):
+        self.image_name = image_name
+
+
+class NeedsPull(Exception):
+    def __init__(self, image_name, service_name):
+        self.image_name = image_name
+        self.service_name = service_name
+
+
 class NoSuchService(Exception):
     def __init__(self, name):
         if isinstance(name, six.binary_type):

compose/service.py

@@ -2,10 +2,12 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import itertools
+import json
 import logging
 import os
 import re
 import sys
+import tempfile
 from collections import namedtuple
 from collections import OrderedDict
 from operator import attrgetter
@@ -58,10 +60,15 @@ from .utils import parse_bytes
 from .utils import parse_seconds_float
 from .utils import truncate_id
 from .utils import unique_everseen
+from compose.cli.utils import binarystr_to_unicode
+
+if six.PY2:
+    import subprocess32 as subprocess
+else:
+    import subprocess
 
 log = logging.getLogger(__name__)
 
-
 HOST_CONFIG_KEYS = [
     'cap_add',
     'cap_drop',
@@ -130,7 +137,6 @@ class NoSuchImageError(Exception):
 
 ServiceName = namedtuple('ServiceName', 'project service number')
 
-
 ConvergencePlan = namedtuple('ConvergencePlan', 'action containers')
 
 
@@ -166,20 +172,21 @@ class BuildAction(enum.Enum):
 
 class Service(object):
     def __init__(
-        self,
-        name,
-        client=None,
-        project='default',
-        use_networking=False,
-        links=None,
-        volumes_from=None,
-        network_mode=None,
-        networks=None,
-        secrets=None,
-        scale=1,
-        pid_mode=None,
-        default_platform=None,
-        **options
+            self,
+            name,
+            client=None,
+            project='default',
+            use_networking=False,
+            links=None,
+            volumes_from=None,
+            network_mode=None,
+            networks=None,
+            secrets=None,
+            scale=1,
+            pid_mode=None,
+            default_platform=None,
+            extra_labels=None,
+            **options
     ):
         self.name = name
         self.client = client
@@ -194,6 +201,7 @@ class Service(object):
         self.scale_num = scale
         self.default_platform = default_platform
         self.options = options
+        self.extra_labels = extra_labels or []
 
     def __repr__(self):
         return '<Service: {}>'.format(self.name)
@@ -208,7 +216,7 @@ class Service(object):
             for container in self.client.containers(
                 all=stopped,
                 filters=filters)])
-        )
+                      )
         if result:
             return result
 
@@ -336,11 +344,11 @@ class Service(object):
             return Container.create(self.client, **container_options)
         except APIError as ex:
             raise OperationFailedError("Cannot create container for service %s: %s" %
-                                       (self.name, ex.explanation))
+                                       (self.name, binarystr_to_unicode(ex.explanation)))
 
-    def ensure_image_exists(self, do_build=BuildAction.none, silent=False):
+    def ensure_image_exists(self, do_build=BuildAction.none, silent=False, cli=False):
         if self.can_be_built() and do_build == BuildAction.force:
-            self.build()
+            self.build(cli=cli)
             return
 
         try:
@@ -356,7 +364,7 @@ class Service(object):
         if do_build == BuildAction.skip:
             raise NeedsBuildError(self)
 
-        self.build()
+        self.build(cli=cli)
         log.warning(
             "Image for service {} was built because it did not already exist. To "
             "rebuild this image you must use `docker-compose build` or "
@@ -397,8 +405,8 @@ class Service(object):
             return ConvergencePlan('start', containers)
 
         if (
-            strategy is ConvergenceStrategy.always or
-            self._containers_have_diverged(containers)
+                strategy is ConvergenceStrategy.always or
+                self._containers_have_diverged(containers)
         ):
             return ConvergencePlan('recreate', containers)
 
@@ -475,6 +483,7 @@ class Service(object):
                 container, timeout=timeout, attach_logs=not detached,
                 start_new_container=start, renew_anonymous_volumes=renew_anonymous_volumes
             )
+
         containers, errors = parallel_execute(
             containers,
             recreate,
@@ -616,7 +625,10 @@ class Service(object):
         try:
             container.start()
         except APIError as ex:
-            raise OperationFailedError("Cannot start service %s: %s" % (self.name, ex.explanation))
+            expl = binarystr_to_unicode(ex.explanation)
+            if "driver failed programming external connectivity" in expl:
+                log.warn("Host is already in use by another container")
+            raise OperationFailedError("Cannot start service %s: %s" % (self.name, expl))
         return container
 
     @property
@@ -696,11 +708,11 @@ class Service(object):
         net_name = self.network_mode.service_name
         pid_namespace = self.pid_mode.service_name
         return (
-            self.get_linked_service_names() +
-            self.get_volumes_from_names() +
-            ([net_name] if net_name else []) +
-            ([pid_namespace] if pid_namespace else []) +
-            list(self.options.get('depends_on', {}).keys())
+                self.get_linked_service_names() +
+                self.get_volumes_from_names() +
+                ([net_name] if net_name else []) +
+                ([pid_namespace] if pid_namespace else []) +
+                list(self.options.get('depends_on', {}).keys())
         )
 
     def get_dependency_configs(self):
@@ -890,7 +902,7 @@ class Service(object):
 
         container_options['labels'] = build_container_labels(
             container_options.get('labels', {}),
-            self.labels(one_off=one_off),
+            self.labels(one_off=one_off) + self.extra_labels,
             number,
             self.config_hash if add_config_hash else None,
             slug
@@ -1049,7 +1061,7 @@ class Service(object):
         return [build_spec(secret) for secret in self.secrets]
 
     def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None,
-              gzip=False, rm=True, silent=False):
+              gzip=False, rm=True, silent=False, cli=False, progress=None):
         output_stream = open(os.devnull, 'w')
         if not silent:
             output_stream = sys.stdout
@@ -1070,7 +1082,8 @@ class Service(object):
                 'Impossible to perform platform-targeted builds for API version < 1.35'
             )
 
-        build_output = self.client.build(
+        builder = self.client if not cli else _CLIBuilder(progress)
+        build_output = builder.build(
             path=path,
             tag=self.image_name,
             rm=rm,
@@ -1542,9 +1555,9 @@ def warn_on_masked_volume(volumes_option, container_volumes, service):
 
     for volume in volumes_option:
         if (
-            volume.external and
-            volume.internal in container_volumes and
-            container_volumes.get(volume.internal) != volume.external
+                volume.external and
+                volume.internal in container_volumes and
+                container_volumes.get(volume.internal) != volume.external
         ):
             log.warning((
                 "Service \"{service}\" is using volume \"{volume}\" from the "
@@ -1591,6 +1604,7 @@ def build_mount(mount_spec):
         read_only=mount_spec.read_only, consistency=mount_spec.consistency, **kwargs
     )
 
+
 # Labels
 
 
@@ -1645,6 +1659,7 @@ def format_environment(environment):
         if isinstance(value, six.binary_type):
             value = value.decode('utf-8')
         return '{key}={value}'.format(key=key, value=value)
+
     return [format_env(*item) for item in environment.items()]
 
 
@@ -1701,3 +1716,139 @@ def rewrite_build_path(path):
         path = WINDOWS_LONGPATH_PREFIX + os.path.normpath(path)
 
     return path
+
+
+class _CLIBuilder(object):
+    def __init__(self, progress):
+        self._progress = progress
+
+    def build(self, path, tag=None, quiet=False, fileobj=None,
+              nocache=False, rm=False, timeout=None,
+              custom_context=False, encoding=None, pull=False,
+              forcerm=False, dockerfile=None, container_limits=None,
+              decode=False, buildargs=None, gzip=False, shmsize=None,
+              labels=None, cache_from=None, target=None, network_mode=None,
+              squash=None, extra_hosts=None, platform=None, isolation=None,
+              use_config_proxy=True):
+        """
+        Args:
+            path (str): Path to the directory containing the Dockerfile
+            buildargs (dict): A dictionary of build arguments
+            cache_from (:py:class:`list`): A list of images used for build
+                cache resolution
+            container_limits (dict): A dictionary of limits applied to each
+                container created by the build process. Valid keys:
+                - memory (int): set memory limit for build
+                - memswap (int): Total memory (memory + swap), -1 to disable
+                    swap
+                - cpushares (int): CPU shares (relative weight)
+                - cpusetcpus (str): CPUs in which to allow execution, e.g.,
+                    ``"0-3"``, ``"0,1"``
+            custom_context (bool): Optional if using ``fileobj``
+            decode (bool): If set to ``True``, the returned stream will be
+                decoded into dicts on the fly. Default ``False``
+            dockerfile (str): path within the build context to the Dockerfile
+            encoding (str): The encoding for a stream. Set to ``gzip`` for
+                compressing
+            extra_hosts (dict): Extra hosts to add to /etc/hosts in building
+                containers, as a mapping of hostname to IP address.
+            fileobj: A file object to use as the Dockerfile. (Or a file-like
+                object)
+            forcerm (bool): Always remove intermediate containers, even after
+                unsuccessful builds
+            isolation (str): Isolation technology used during build.
+                Default: `None`.
+            labels (dict): A dictionary of labels to set on the image
+            network_mode (str): networking mode for the run commands during
+                build
+            nocache (bool): Don't use the cache when set to ``True``
+            platform (str): Platform in the format ``os[/arch[/variant]]``
+            pull (bool): Downloads any updates to the FROM image in Dockerfiles
+            quiet (bool): Whether to return the status
+            rm (bool): Remove intermediate containers. The ``docker build``
+                command now defaults to ``--rm=true``, but we have kept the old
+                default of `False` to preserve backward compatibility
+            shmsize (int): Size of `/dev/shm` in bytes. The size must be
+                greater than 0. If omitted the system uses 64MB
+            squash (bool): Squash the resulting images layers into a
+                single layer.
+            tag (str): A tag to add to the final image
+            target (str): Name of the build-stage to build in a multi-stage
+                Dockerfile
+            timeout (int): HTTP timeout
+            use_config_proxy (bool): If ``True``, and if the docker client
+                configuration file (``~/.docker/config.json`` by default)
+                contains a proxy configuration, the corresponding environment
+                variables will be set in the container being built.
+        Returns:
+            A generator for the build output.
+        """
+        if dockerfile:
+            dockerfile = os.path.join(path, dockerfile)
+        iidfile = tempfile.mktemp()
+
+        command_builder = _CommandBuilder()
+        command_builder.add_params("--build-arg", buildargs)
+        command_builder.add_list("--cache-from", cache_from)
+        command_builder.add_arg("--file", dockerfile)
+        command_builder.add_flag("--force-rm", forcerm)
+        command_builder.add_arg("--memory", container_limits.get("memory"))
+        command_builder.add_flag("--no-cache", nocache)
+        command_builder.add_arg("--progress", self._progress)
+        command_builder.add_flag("--pull", pull)
+        command_builder.add_arg("--tag", tag)
+        command_builder.add_arg("--target", target)
+        command_builder.add_arg("--iidfile", iidfile)
+        args = command_builder.build([path])
+
+        magic_word = "Successfully built "
+        appear = False
+        with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p:
+            while True:
+                line = p.stdout.readline()
+                if not line:
+                    break
+                # Fix non ascii chars on Python2. To remove when #6890 is complete.
+                if six.PY2:
+                    magic_word = str(magic_word)
+                if line.startswith(magic_word):
+                    appear = True
+                yield json.dumps({"stream": line})
+
+        with open(iidfile) as f:
+            line = f.readline()
+            image_id = line.split(":")[1].strip()
+        os.remove(iidfile)
+
+        # In case of `DOCKER_BUILDKIT=1`
+        # there is no success message already present in the output.
+        # Since that's the way `Service::build` gets the `image_id`
+        # it has to be added `manually`
+        if not appear:
+            yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)})
+
+
+class _CommandBuilder(object):
+    def __init__(self):
+        self._args = ["docker", "build"]
+
+    def add_arg(self, name, value):
+        if value:
+            self._args.extend([name, str(value)])
+
+    def add_flag(self, name, flag):
+        if flag:
+            self._args.extend([name])
+
+    def add_params(self, name, params):
+        if params:
+            for key, val in params.items():
+                self._args.extend([name, "{}={}".format(key, val)])
+
+    def add_list(self, name, values):
+        if values:
+            for val in values:
+                self._args.extend([name, val])
+
+    def build(self, args):
+        return self._args + args

contrib/completion/bash/docker-compose

@@ -126,18 +126,6 @@ _docker_compose_build() {
 }
 
 
-_docker_compose_bundle() {
-	case "$prev" in
-		--output|-o)
-			_filedir
-			return
-			;;
-	esac
-
-	COMPREPLY=( $( compgen -W "--push-images --help --output -o" -- "$cur" ) )
-}
-
-
 _docker_compose_config() {
 	case "$prev" in
 		--hash)
@@ -557,7 +545,7 @@ _docker_compose_up() {
 
 	case "$cur" in
 		-*)
-			COMPREPLY=( $( compgen -W "--abort-on-container-exit --always-recreate-deps --build -d --detach --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --no-start --renew-anon-volumes -V --remove-orphans --scale --timeout -t" -- "$cur" ) )
+			COMPREPLY=( $( compgen -W "--abort-on-container-exit --always-recreate-deps --attach-dependencies --build -d --detach --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --no-start --renew-anon-volumes -V --remove-orphans --scale --timeout -t" -- "$cur" ) )
 			;;
 		*)
 			__docker_compose_complete_services
@@ -581,7 +569,6 @@ _docker_compose() {
 
 	local commands=(
 		build
-		bundle
 		config
 		create
 		down

contrib/completion/zsh/_docker-compose

@@ -121,12 +121,6 @@ __docker-compose_subcommand() {
                 '--parallel[Build images in parallel.]' \
                 '*:services:__docker-compose_services_from_build' && ret=0
             ;;
-        (bundle)
-            _arguments \
-                $opts_help \
-                '--push-images[Automatically push images for any services which have a `build` option specified.]' \
-                '(--output -o)'{--output,-o}'[Path to write the bundle file to. Defaults to "<project name>.dab".]:file:_files' && ret=0
-            ;;
         (config)
             _arguments \
                 $opts_help \
@@ -290,7 +284,7 @@ __docker-compose_subcommand() {
         (up)
             _arguments \
                 $opts_help \
-                '(--abort-on-container-exit)-d[Detached mode: Run containers in the background, print new container names. Incompatible with --abort-on-container-exit.]' \
+                '(--abort-on-container-exit)-d[Detached mode: Run containers in the background, print new container names. Incompatible with --abort-on-container-exit and --attach-dependencies.]' \
                 $opts_no_color \
                 $opts_no_deps \
                 $opts_force_recreate \
@@ -298,6 +292,7 @@ __docker-compose_subcommand() {
                 $opts_no_build \
                 "(--no-build)--build[Build images before starting containers.]" \
                 "(-d)--abort-on-container-exit[Stops all containers if any container was stopped. Incompatible with -d.]" \
+                "(-d)--attach-dependencies[Attach to dependent containers. Incompatible with -d.]" \
                 '(-t --timeout)'{-t,--timeout}"[Use this timeout in seconds for container shutdown when attached or when containers are already running. (default: 10)]:seconds: " \
                 '--scale[SERVICE=NUM Scale SERVICE to NUM instances. Overrides the `scale` setting in the Compose file if present.]:service scale SERVICE=NUM: ' \
                 '--exit-code-from=[Return the exit code of the selected service container. Implies --abort-on-container-exit]:service:__docker-compose_services' \

docker-compose_darwin.spec

@@ -0,0 +1,108 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+a = Analysis(['bin/docker-compose'],
+             pathex=['.'],
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             cipher=block_cipher)
+
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+
+exe = EXE(pyz,
+          a.scripts,
+          exclude_binaries=True,
+          name='docker-compose',
+          debug=False,
+          strip=False,
+          upx=True,
+          console=True,
+          bootloader_ignore_signals=True)
+coll = COLLECT(exe,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          [
+            (
+                'compose/config/config_schema_v1.json',
+                'compose/config/config_schema_v1.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.0.json',
+                'compose/config/config_schema_v2.0.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.1.json',
+                'compose/config/config_schema_v2.1.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.2.json',
+                'compose/config/config_schema_v2.2.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.3.json',
+                'compose/config/config_schema_v2.3.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.4.json',
+                'compose/config/config_schema_v2.4.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.0.json',
+                'compose/config/config_schema_v3.0.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.1.json',
+                'compose/config/config_schema_v3.1.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.2.json',
+                'compose/config/config_schema_v3.2.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.3.json',
+                'compose/config/config_schema_v3.3.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.4.json',
+                'compose/config/config_schema_v3.4.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.5.json',
+                'compose/config/config_schema_v3.5.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.6.json',
+                'compose/config/config_schema_v3.6.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.7.json',
+                'compose/config/config_schema_v3.7.json',
+                'DATA'
+            ),
+            (
+                'compose/GITSHA',
+                'compose/GITSHA',
+                'DATA'
+            )
+          ],
+          strip=False,
+          upx=True,
+          upx_exclude=[],
+          name='docker-compose-Darwin-x86_64')

requirements-build.txt

@@ -1 +1 @@
-pyinstaller==3.4
+pyinstaller==3.6

requirements-dev.txt

@@ -1,6 +1,7 @@
-coverage==4.4.2
-ddt==1.2.0
-flake8==3.5.0
+coverage==5.0.3
+ddt==1.2.2
+flake8==3.7.9
 mock==3.0.5
-pytest==3.6.3
-pytest-cov==2.5.1
+pytest==5.3.2; python_version >= '3.5'
+pytest==4.6.5; python_version < '3.5'
+pytest-cov==2.8.1

requirements.txt

@@ -1,24 +1,26 @@
+backports.shutil_get_terminal_size==1.0.0
 backports.ssl-match-hostname==3.5.0.1; python_version < '3'
-cached-property==1.3.0
-certifi==2017.4.17
+cached-property==1.5.1
+certifi==2019.11.28
 chardet==3.0.4
-colorama==0.4.0; sys_platform == 'win32'
-docker==4.0.1
+colorama==0.4.3; sys_platform == 'win32'
+docker==4.1.0
 docker-pycreds==0.4.0
 dockerpty==0.4.1
 docopt==0.6.2
 enum34==1.1.6; python_version < '3.4'
 functools32==3.2.3.post2; python_version < '3.2'
-idna==2.5
-ipaddress==1.0.18
-jsonschema==2.6.0
-paramiko==2.4.2
+idna==2.8
+ipaddress==1.0.23
+jsonschema==3.2.0
+paramiko==2.7.1
 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6'
 pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6'
-PySocks==1.6.7
-PyYAML==4.2b1
+PySocks==1.7.1
+PyYAML==5.3
 requests==2.22.0
-six==1.10.0
+six==1.12.0
+subprocess32==3.5.4; python_version < '3.2'
 texttable==1.6.2
-urllib3==1.24.2; python_version == '3.3'
-websocket-client==0.32.0
+urllib3==1.25.7; python_version == '3.3'
+websocket-client==0.57.0

script/build/linux

@@ -12,6 +12,7 @@ docker build -t "${TAG}" . \
        --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}"
 TMP_CONTAINER=$(docker create "${TAG}")
 mkdir -p dist
-docker cp "${TMP_CONTAINER}":/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64
+ARCH=$(uname -m)
+docker cp "${TMP_CONTAINER}":/usr/local/bin/docker-compose "dist/docker-compose-Linux-${ARCH}"
 docker container rm -f "${TMP_CONTAINER}"
 docker image rm -f "${TAG}"

script/build/linux-entrypoint

@@ -20,10 +20,11 @@ echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
 export PATH="${CODE_PATH}/pyinstaller:${PATH}"
 
 if [ ! -z "${BUILD_BOOTLOADER}" ]; then
-    # Build bootloader for alpine
-    git clone --single-branch --branch master https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller
+    # Build bootloader for alpine; develop is the main branch
+    git clone --single-branch --branch develop https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller
     cd /tmp/pyinstaller/bootloader
-    git checkout v3.4
+    # Checkout commit corresponding to version in requirements-build
+    git checkout v3.6
     "${VENV}"/bin/python3 ./waf configure --no-lsb all
     "${VENV}"/bin/pip3 install ..
     cd "${CODE_PATH}"

script/build/osx

@@ -11,6 +11,14 @@ venv/bin/pip install -r requirements-build.txt
 venv/bin/pip install --no-deps .
 DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
 echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
+
+# Build as a folder for macOS Catalina.
+venv/bin/pyinstaller docker-compose_darwin.spec
+dist/docker-compose-Darwin-x86_64/docker-compose version
+(cd dist/docker-compose-Darwin-x86_64/ && tar zcvf ../docker-compose-Darwin-x86_64.tgz .)
+rm -rf dist/docker-compose-Darwin-x86_64
+
+# Build static binary for legacy.
 venv/bin/pyinstaller docker-compose.spec
 mv dist/docker-compose dist/docker-compose-Darwin-x86_64
 dist/docker-compose-Darwin-x86_64 version

script/circle/bintray-deploy.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -x
-
 curl -f -u$BINTRAY_USERNAME:$BINTRAY_API_KEY -X GET \
   https://api.bintray.com/repos/docker-compose/${CIRCLE_BRANCH}
 
@@ -27,3 +25,11 @@ curl -f -T dist/docker-compose-${OS_NAME}-x86_64 -u$BINTRAY_USERNAME:$BINTRAY_AP
   -H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \
   -H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \
   https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64 || exit 1
+
+# Upload folder format of docker-compose for macOS in addition to binary.
+if [ "${OS_NAME}" == "Darwin" ]; then
+  curl -f -T dist/docker-compose-${OS_NAME}-x86_64.tgz -u$BINTRAY_USERNAME:$BINTRAY_API_KEY \
+    -H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \
+    -H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \
+    https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64.tgz || exit 1
+fi

script/release/generate_changelog.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+set -x
+
+## Usage :
+## changelog PREVIOUS_TAG..HEAD
+
+# configure refs so we get pull-requests metadata
+git config --add remote.origin.fetch +refs/pull/*/head:refs/remotes/origin/pull/*
+git fetch origin
+
+RANGE=${1:-"$(git describe --tags  --abbrev=0)..HEAD"}
+echo "Generate changelog for range ${RANGE}"
+echo
+
+pullrequests() {
+    for commit in $(git log ${RANGE} --format='format:%H'); do
+        # Get the oldest remotes/origin/pull/* branch to include this commit, i.e. the one to introduce it
+        git branch -a --sort=committerdate  --contains $commit --list 'origin/pull/*' | head -1 | cut -d'/' -f4
+    done
+}
+
+changes=$(pullrequests | uniq)
+
+echo "pull requests merged within range:"
+echo $changes
+
+echo '#Features' > CHANGELOG.md
+for pr in $changes; do
+    curl -fs -H "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/docker/compose/pulls/${pr} \
+    | jq -r ' select( .labels[].name | contains("kind/feature") ) | "* "+.title' >> CHANGELOG.md
+done
+
+echo '#Bugs' >> CHANGELOG.md
+for pr in $changes; do
+    curl -fs -H "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/docker/compose/pulls/${pr} \
+    | jq -r ' select( .labels[].name | contains("kind/bug") ) | "* "+.title' >> CHANGELOG.md
+done

script/release/release.py

@@ -204,7 +204,8 @@ def resume(args):
             gh_release = create_release_draft(repository, args.release, pr_data, files)
         delete_assets(gh_release)
         upload_assets(gh_release, files)
-        img_manager = ImageManager(args.release)
+        tag_as_latest = is_tag_latest(args.release)
+        img_manager = ImageManager(args.release, tag_as_latest)
         img_manager.build_images(repository)
     except ScriptError as e:
         print(e)
@@ -244,7 +245,8 @@ def start(args):
         files = downloader.download_all(args.release)
         gh_release = create_release_draft(repository, args.release, pr_data, files)
         upload_assets(gh_release, files)
-        img_manager = ImageManager(args.release)
+        tag_as_latest = is_tag_latest(args.release)
+        img_manager = ImageManager(args.release, tag_as_latest)
         img_manager.build_images(repository)
     except ScriptError as e:
         print(e)

script/release/release/downloader.py

@@ -55,6 +55,7 @@ class BinaryDownloader(requests.Session):
 
     def download_all(self, version):
         files = {
+            'docker-compose-Darwin-x86_64.tgz': None,
             'docker-compose-Darwin-x86_64': None,
             'docker-compose-Linux-x86_64': None,
             'docker-compose-Windows-x86_64.exe': None,

script/run/run.sh

@@ -15,7 +15,7 @@
 
 set -e
 
-VERSION="1.25.0-rc2"
+VERSION="1.25.2-rc1"
 IMAGE="docker/compose:$VERSION"
 
 
@@ -36,18 +36,18 @@ if [ "$(pwd)" != '/' ]; then
 fi
 if [ -n "$COMPOSE_FILE" ]; then
     COMPOSE_OPTIONS="$COMPOSE_OPTIONS -e COMPOSE_FILE=$COMPOSE_FILE"
-    compose_dir=$(realpath $(dirname $COMPOSE_FILE))
+    compose_dir=$(realpath "$(dirname "$COMPOSE_FILE")")
 fi
 # TODO: also check --file argument
 if [ -n "$compose_dir" ]; then
     VOLUMES="$VOLUMES -v $compose_dir:$compose_dir"
 fi
 if [ -n "$HOME" ]; then
-    VOLUMES="$VOLUMES -v $HOME:$HOME -v $HOME:/root" # mount $HOME in /root to share docker.config
+    VOLUMES="$VOLUMES -v $HOME:$HOME -e HOME" # Pass in HOME to share docker.config and allow ~/-relative paths to work.
 fi
 
 # Only allocate tty if we detect one
-if [ -t 0 -a -t 1 ]; then
+if [ -t 0 ] && [ -t 1 ]; then
     DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t"
 fi
 
@@ -56,8 +56,9 @@ DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i"
 
 
 # Handle userns security
-if [ ! -z "$(docker info 2>/dev/null | grep userns)" ]; then
+if docker info --format '{{json .SecurityOptions}}' 2>/dev/null | grep -q 'name=userns'; then
     DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS --userns=host"
 fi
 
+# shellcheck disable=SC2086
 exec docker run --rm $DOCKER_RUN_OPTIONS $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w "$(pwd)" $IMAGE "$@"

script/setup/osx

@@ -13,13 +13,13 @@ if ! [ ${DEPLOYMENT_TARGET} == "$(macos_version)" ]; then
   SDK_SHA1=dd228a335194e3392f1904ce49aff1b1da26ca62
 fi
 
-OPENSSL_VERSION=1.1.1c
+OPENSSL_VERSION=1.1.1d
 OPENSSL_URL=https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-OPENSSL_SHA1=71b830a077276cbeccc994369538617a21bee808
+OPENSSL_SHA1=056057782325134b76d1931c48f2c7e6595d7ef4
 
-PYTHON_VERSION=3.7.4
+PYTHON_VERSION=3.7.5
 PYTHON_URL=https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
-PYTHON_SHA1=fb1d764be8a9dcd40f2f152a610a0ab04e0d0ed3
+PYTHON_SHA1=8b0311d4cca19f0ea9181731189fa33c9f5aedf9
 
 #
 # Install prerequisites.
@@ -36,7 +36,7 @@ if ! [ -x "$(command -v python3)" ]; then
   brew install python3
 fi
 if ! [ -x "$(command -v virtualenv)" ]; then
-  pip install virtualenv==16.2.0
+  pip3 install virtualenv==16.2.0
 fi
 
 #

setup.py

@@ -32,14 +32,14 @@ def find_version(*file_paths):
 install_requires = [
     'cached-property >= 1.2.0, < 2',
     'docopt >= 0.6.1, < 1',
-    'PyYAML >= 3.10, < 5',
+    'PyYAML >= 3.10, < 6',
     'requests >= 2.20.0, < 3',
     'texttable >= 0.9.0, < 2',
     'websocket-client >= 0.32.0, < 1',
     'docker[ssh] >= 3.7.0, < 5',
     'dockerpty >= 0.4.1, < 1',
     'six >= 1.3.0, < 2',
-    'jsonschema >= 2.5.1, < 3',
+    'jsonschema >= 2.5.1, < 4',
 ]
 
 
@@ -52,9 +52,11 @@ if sys.version_info[:2] < (3, 4):
     tests_require.append('mock >= 1.0.1, < 4')
 
 extras_require = {
+    ':python_version < "3.2"': ['subprocess32 >= 3.5.4, < 4'],
     ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'],
     ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5, < 4'],
-    ':python_version < "3.3"': ['ipaddress >= 1.0.16, < 2'],
+    ':python_version < "3.3"': ['backports.shutil_get_terminal_size == 1.0.0',
+                                'ipaddress >= 1.0.16, < 2'],
     ':sys_platform == "win32"': ['colorama >= 0.4, < 1'],
     'socks': ['PySocks >= 1.5.6, != 1.5.7, < 2'],
 }

tests/acceptance/cli_test.py

@@ -43,11 +43,30 @@ ProcessResult = namedtuple('ProcessResult', 'stdout stderr')
 
 BUILD_CACHE_TEXT = 'Using cache'
 BUILD_PULL_TEXT = 'Status: Image is up to date for busybox:1.27.2'
+COMPOSE_COMPATIBILITY_DICT = {
+    'version': '2.3',
+    'volumes': {'foo': {'driver': 'default'}},
+    'networks': {'bar': {}},
+    'services': {
+        'foo': {
+            'command': '/bin/true',
+            'image': 'alpine:3.10.1',
+            'scale': 3,
+            'restart': 'always:7',
+            'mem_limit': '300M',
+            'mem_reservation': '100M',
+            'cpus': 0.7,
+            'volumes': ['foo:/bar:rw'],
+            'networks': {'bar': None},
+        }
+    },
+}
 
 
 def start_process(base_dir, options):
     proc = subprocess.Popen(
         ['docker-compose'] + options,
+        stdin=subprocess.PIPE,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         cwd=base_dir)
@@ -55,8 +74,8 @@ def start_process(base_dir, options):
     return proc
 
 
-def wait_on_process(proc, returncode=0):
-    stdout, stderr = proc.communicate()
+def wait_on_process(proc, returncode=0, stdin=None):
+    stdout, stderr = proc.communicate(input=stdin)
     if proc.returncode != returncode:
         print("Stderr: {}".format(stderr))
         print("Stdout: {}".format(stdout))
@@ -64,10 +83,10 @@ def wait_on_process(proc, returncode=0):
     return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8'))
 
 
-def dispatch(base_dir, options, project_options=None, returncode=0):
+def dispatch(base_dir, options, project_options=None, returncode=0, stdin=None):
     project_options = project_options or []
     proc = start_process(base_dir, project_options + options)
-    return wait_on_process(proc, returncode=returncode)
+    return wait_on_process(proc, returncode=returncode, stdin=stdin)
 
 
 def wait_on_condition(condition, delay=0.1, timeout=40):
@@ -156,8 +175,8 @@ class CLITestCase(DockerClientTestCase):
             self._project = get_project(self.base_dir, override_dir=self.override_dir)
         return self._project
 
-    def dispatch(self, options, project_options=None, returncode=0):
-        return dispatch(self.base_dir, options, project_options, returncode)
+    def dispatch(self, options, project_options=None, returncode=0, stdin=None):
+        return dispatch(self.base_dir, options, project_options, returncode, stdin)
 
     def execute(self, container, cmd):
         # Remove once Hijack and CloseNotifier sign a peace treaty
@@ -241,6 +260,17 @@ class CLITestCase(DockerClientTestCase):
         self.base_dir = 'tests/fixtures/v2-full'
         assert self.dispatch(['config', '--quiet']).stdout == ''
 
+    def test_config_stdin(self):
+        config = b"""version: "3.7"
+services:
+  web:
+    image: nginx
+  other:
+    image: alpine
+"""
+        result = self.dispatch(['-f', '-', 'config', '--services'], stdin=config)
+        assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'}
+
     def test_config_with_hash_option(self):
         self.base_dir = 'tests/fixtures/v2-full'
         result = self.dispatch(['config', '--hash=*'])
@@ -257,7 +287,7 @@ class CLITestCase(DockerClientTestCase):
         # assert there are no python objects encoded in the output
         assert '!!' not in result.stdout
 
-        output = yaml.load(result.stdout)
+        output = yaml.safe_load(result.stdout)
         expected = {
             'version': '2.0',
             'volumes': {'data': {'driver': 'local'}},
@@ -282,7 +312,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_restart(self):
         self.base_dir = 'tests/fixtures/restart'
         result = self.dispatch(['config'])
-        assert yaml.load(result.stdout) == {
+        assert yaml.safe_load(result.stdout) == {
             'version': '2.0',
             'services': {
                 'never': {
@@ -311,7 +341,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_external_network(self):
         self.base_dir = 'tests/fixtures/networks'
         result = self.dispatch(['-f', 'external-networks.yml', 'config'])
-        json_result = yaml.load(result.stdout)
+        json_result = yaml.safe_load(result.stdout)
         assert 'networks' in json_result
         assert json_result['networks'] == {
             'networks_foo': {
@@ -325,7 +355,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_with_dot_env(self):
         self.base_dir = 'tests/fixtures/default-env-file'
         result = self.dispatch(['config'])
-        json_result = yaml.load(result.stdout)
+        json_result = yaml.safe_load(result.stdout)
         assert json_result == {
             'services': {
                 'web': {
@@ -340,7 +370,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_with_env_file(self):
         self.base_dir = 'tests/fixtures/default-env-file'
         result = self.dispatch(['--env-file', '.env2', 'config'])
-        json_result = yaml.load(result.stdout)
+        json_result = yaml.safe_load(result.stdout)
         assert json_result == {
             'services': {
                 'web': {
@@ -355,12 +385,12 @@ class CLITestCase(DockerClientTestCase):
     def test_config_with_dot_env_and_override_dir(self):
         self.base_dir = 'tests/fixtures/default-env-file'
         result = self.dispatch(['--project-directory', 'alt/', 'config'])
-        json_result = yaml.load(result.stdout)
+        json_result = yaml.safe_load(result.stdout)
         assert json_result == {
             'services': {
                 'web': {
                     'command': 'echo uwu',
-                    'image': 'alpine:3.4',
+                    'image': 'alpine:3.10.1',
                     'ports': ['3341/tcp', '4449/tcp']
                 }
             },
@@ -370,7 +400,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_external_volume_v2(self):
         self.base_dir = 'tests/fixtures/volumes'
         result = self.dispatch(['-f', 'external-volumes-v2.yml', 'config'])
-        json_result = yaml.load(result.stdout)
+        json_result = yaml.safe_load(result.stdout)
         assert 'volumes' in json_result
         assert json_result['volumes'] == {
             'foo': {
@@ -386,7 +416,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_external_volume_v2_x(self):
         self.base_dir = 'tests/fixtures/volumes'
         result = self.dispatch(['-f', 'external-volumes-v2-x.yml', 'config'])
-        json_result = yaml.load(result.stdout)
+        json_result = yaml.safe_load(result.stdout)
         assert 'volumes' in json_result
         assert json_result['volumes'] == {
             'foo': {
@@ -402,7 +432,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_external_volume_v3_x(self):
         self.base_dir = 'tests/fixtures/volumes'
         result = self.dispatch(['-f', 'external-volumes-v3-x.yml', 'config'])
-        json_result = yaml.load(result.stdout)
+        json_result = yaml.safe_load(result.stdout)
         assert 'volumes' in json_result
         assert json_result['volumes'] == {
             'foo': {
@@ -418,7 +448,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_external_volume_v3_4(self):
         self.base_dir = 'tests/fixtures/volumes'
         result = self.dispatch(['-f', 'external-volumes-v3-4.yml', 'config'])
-        json_result = yaml.load(result.stdout)
+        json_result = yaml.safe_load(result.stdout)
         assert 'volumes' in json_result
         assert json_result['volumes'] == {
             'foo': {
@@ -434,7 +464,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_external_network_v3_5(self):
         self.base_dir = 'tests/fixtures/networks'
         result = self.dispatch(['-f', 'external-networks-v3-5.yml', 'config'])
-        json_result = yaml.load(result.stdout)
+        json_result = yaml.safe_load(result.stdout)
         assert 'networks' in json_result
         assert json_result['networks'] == {
             'foo': {
@@ -450,7 +480,7 @@ class CLITestCase(DockerClientTestCase):
     def test_config_v1(self):
         self.base_dir = 'tests/fixtures/v1-config'
         result = self.dispatch(['config'])
-        assert yaml.load(result.stdout) == {
+        assert yaml.safe_load(result.stdout) == {
             'version': '2.1',
             'services': {
                 'net': {
@@ -475,7 +505,7 @@ class CLITestCase(DockerClientTestCase):
         self.base_dir = 'tests/fixtures/v3-full'
         result = self.dispatch(['config'])
 
-        assert yaml.load(result.stdout) == {
+        assert yaml.safe_load(result.stdout) == {
             'version': '3.5',
             'volumes': {
                 'foobar': {
@@ -552,24 +582,23 @@ class CLITestCase(DockerClientTestCase):
         self.base_dir = 'tests/fixtures/compatibility-mode'
         result = self.dispatch(['--compatibility', 'config'])
 
-        assert yaml.load(result.stdout) == {
-            'version': '2.3',
-            'volumes': {'foo': {'driver': 'default'}},
-            'networks': {'bar': {}},
-            'services': {
-                'foo': {
-                    'command': '/bin/true',
-                    'image': 'alpine:3.7',
-                    'scale': 3,
-                    'restart': 'always:7',
-                    'mem_limit': '300M',
-                    'mem_reservation': '100M',
-                    'cpus': 0.7,
-                    'volumes': ['foo:/bar:rw'],
-                    'networks': {'bar': None},
-                }
-            },
-        }
+        assert yaml.load(result.stdout) == COMPOSE_COMPATIBILITY_DICT
+
+    @mock.patch.dict(os.environ)
+    def test_config_compatibility_mode_from_env(self):
+        self.base_dir = 'tests/fixtures/compatibility-mode'
+        os.environ['COMPOSE_COMPATIBILITY'] = 'true'
+        result = self.dispatch(['config'])
+
+        assert yaml.load(result.stdout) == COMPOSE_COMPATIBILITY_DICT
+
+    @mock.patch.dict(os.environ)
+    def test_config_compatibility_mode_from_env_and_option_precedence(self):
+        self.base_dir = 'tests/fixtures/compatibility-mode'
+        os.environ['COMPOSE_COMPATIBILITY'] = 'false'
+        result = self.dispatch(['--compatibility', 'config'])
+
+        assert yaml.load(result.stdout) == COMPOSE_COMPATIBILITY_DICT
 
     def test_ps(self):
         self.project.get_service('simple').create_container()
@@ -661,13 +690,6 @@ class CLITestCase(DockerClientTestCase):
                 'image library/nonexisting-image:latest not found' in result.stderr or
                 'pull access denied for nonexisting-image' in result.stderr)
 
-    def test_pull_with_build(self):
-        result = self.dispatch(['-f', 'pull-with-build.yml', 'pull'])
-
-        assert 'Pulling simple' not in result.stderr
-        assert 'Pulling from_simple' not in result.stderr
-        assert 'Pulling another ...' in result.stderr
-
     def test_pull_with_quiet(self):
         assert self.dispatch(['pull', '--quiet']).stderr == ''
         assert self.dispatch(['pull', '--quiet']).stdout == ''
@@ -689,6 +711,14 @@ class CLITestCase(DockerClientTestCase):
             result.stderr
         )
 
+    def test_pull_can_build(self):
+        result = self.dispatch([
+            '-f', 'can-build-pull-failures.yml', 'pull'],
+            returncode=0
+        )
+        assert 'Some service image(s) must be built from source' in result.stderr
+        assert 'docker-compose build can_build' in result.stderr
+
     def test_pull_with_no_deps(self):
         self.base_dir = 'tests/fixtures/links-composefile'
         result = self.dispatch(['pull', '--no-parallel', 'web'])
@@ -842,32 +872,6 @@ class CLITestCase(DockerClientTestCase):
         )
         assert 'Favorite Touhou Character: hong.meiling' in result.stdout
 
-    def test_bundle_with_digests(self):
-        self.base_dir = 'tests/fixtures/bundle-with-digests/'
-        tmpdir = pytest.ensuretemp('cli_test_bundle')
-        self.addCleanup(tmpdir.remove)
-        filename = str(tmpdir.join('example.dab'))
-
-        self.dispatch(['bundle', '--output', filename])
-        with open(filename, 'r') as fh:
-            bundle = json.load(fh)
-
-        assert bundle == {
-            'Version': '0.1',
-            'Services': {
-                'web': {
-                    'Image': ('dockercloud/hello-world@sha256:fe79a2cfbd17eefc3'
-                              '44fb8419420808df95a1e22d93b7f621a7399fd1e9dca1d'),
-                    'Networks': ['default'],
-                },
-                'redis': {
-                    'Image': ('redis@sha256:a84cb8f53a70e19f61ff2e1d5e73fb7ae62d'
-                              '374b2b7392de1e7d77be26ef8f7b'),
-                    'Networks': ['default'],
-                }
-            },
-        }
-
     def test_build_override_dir(self):
         self.base_dir = 'tests/fixtures/build-path-override-dir'
         self.override_dir = os.path.abspath('tests/fixtures')
@@ -1567,6 +1571,26 @@ class CLITestCase(DockerClientTestCase):
         assert len(db.containers()) == 0
         assert len(console.containers()) == 0
 
+    def test_up_with_attach_dependencies(self):
+        self.base_dir = 'tests/fixtures/echo-services-dependencies'
+        result = self.dispatch(['up', '--attach-dependencies', '--no-color', 'simple'], None)
+        simple_name = self.project.get_service('simple').containers(stopped=True)[0].name_without_project
+        another_name = self.project.get_service('another').containers(
+            stopped=True
+        )[0].name_without_project
+
+        assert '{}   | simple'.format(simple_name) in result.stdout
+        assert '{}  | another'.format(another_name) in result.stdout
+
+    def test_up_handles_aborted_dependencies(self):
+        self.base_dir = 'tests/fixtures/abort-on-container-exit-dependencies'
+        proc = start_process(
+            self.base_dir,
+            ['up', 'simple', '--attach-dependencies', '--abort-on-container-exit'])
+        wait_on_condition(ContainerCountCondition(self.project, 0))
+        proc.wait()
+        assert proc.returncode == 1
+
     def test_up_with_force_recreate(self):
         self.dispatch(['up', '-d'], None)
         service = self.project.get_service('simple')
@@ -2816,8 +2840,8 @@ class CLITestCase(DockerClientTestCase):
         result = self.dispatch(['images'])
 
         assert 'busybox' in result.stdout
-        assert 'multiple-composefiles_another_1' in result.stdout
-        assert 'multiple-composefiles_simple_1' in result.stdout
+        assert '_another_1' in result.stdout
+        assert '_simple_1' in result.stdout
 
     @mock.patch.dict(os.environ)
     def test_images_tagless_image(self):
@@ -2865,4 +2889,4 @@ class CLITestCase(DockerClientTestCase):
 
         assert re.search(r'foo1.+test[ \t]+dev', result.stdout) is not None
         assert re.search(r'foo2.+test[ \t]+prod', result.stdout) is not None
-        assert re.search(r'foo3.+_foo3[ \t]+latest', result.stdout) is not None
+        assert re.search(r'foo3.+test[ \t]+latest', result.stdout) is not None

tests/fixtures/abort-on-container-exit-dependencies/docker-compose.yml

@@ -0,0 +1,10 @@
+version: "2.0"
+services:
+  simple:
+    image: busybox:1.31.0-uclibc
+    command: top
+    depends_on:
+      - another
+  another:
+    image: busybox:1.31.0-uclibc
+    command: ls /thecakeisalie

tests/fixtures/bundle-with-digests/docker-compose.yml

@@ -1,9 +0,0 @@
-
-version: '2.0'
-
-services:
-  web:
-    image: dockercloud/hello-world@sha256:fe79a2cfbd17eefc344fb8419420808df95a1e22d93b7f621a7399fd1e9dca1d
-
-  redis:
-    image: redis@sha256:a84cb8f53a70e19f61ff2e1d5e73fb7ae62d374b2b7392de1e7d77be26ef8f7b

tests/fixtures/compatibility-mode/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.5'
 services:
   foo:
-    image: alpine:3.7
+    image: alpine:3.10.1
     command: /bin/true
     deploy:
       replicas: 3

tests/fixtures/default-env-file/alt/.env

@@ -1,4 +1,4 @@
-IMAGE=alpine:3.4
+IMAGE=alpine:3.10.1
 COMMAND=echo uwu
 PORT1=3341
 PORT2=4449

tests/fixtures/echo-services-dependencies/docker-compose.yml

@@ -0,0 +1,10 @@
+version: "2.0"
+services:
+  simple:
+    image: busybox:1.31.0-uclibc
+    command: echo simple
+    depends_on:
+      - another
+  another:
+    image: busybox:1.31.0-uclibc
+    command: echo another

tests/fixtures/images-service-tag/docker-compose.yml

@@ -8,3 +8,4 @@ services:
     image: test:prod
   foo3:
     build: .
+    image: test:latest

tests/fixtures/networks/docker-compose.yml

@@ -2,17 +2,17 @@ version: "2"
 
 services:
   web:
-    image: alpine:3.7
+    image: alpine:3.10.1
     command: top
     networks: ["front"]
   app:
-    image: alpine:3.7
+    image: alpine:3.10.1
     command: top
     networks: ["front", "back"]
     links:
       - "db:database"
   db:
-    image: alpine:3.7
+    image: alpine:3.10.1
     command: top
     networks: ["back"]
 

tests/fixtures/simple-composefile/can-build-pull-failures.yml

@@ -0,0 +1,6 @@
+version: '3'
+services:
+  can_build:
+    image: nonexisting-image-but-can-build:latest
+    build: .
+    command: top

tests/helpers.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import contextlib
 import os
 
 from compose.config.config import ConfigDetails
@@ -55,3 +56,17 @@ def create_host_file(client, filename):
         content = fh.read()
 
     return create_custom_host_file(client, filename, content)
+
+
+@contextlib.contextmanager
+def cd(path):
+    """
+    A context manager which changes the working directory to the given
+    path, and then changes it back to its previous value on exit.
+    """
+    prev_cwd = os.getcwd()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(prev_cwd)

tests/integration/project_test.py

@@ -8,7 +8,6 @@ import random
 import shutil
 import tempfile
 
-import py
 import pytest
 from docker.errors import APIError
 from docker.errors import NotFound
@@ -16,6 +15,7 @@ from docker.errors import NotFound
 from .. import mock
 from ..helpers import build_config as load_config
 from ..helpers import BUSYBOX_IMAGE_WITH_TAG
+from ..helpers import cd
 from ..helpers import create_host_file
 from .testcases import DockerClientTestCase
 from .testcases import SWARM_SKIP_CONTAINERS_ALL
@@ -1329,9 +1329,9 @@ class ProjectTest(DockerClientTestCase):
             })
         details = config.ConfigDetails('.', [base_file, override_file])
 
-        tmpdir = py.test.ensuretemp('logging_test')
-        self.addCleanup(tmpdir.remove)
-        with tmpdir.as_cwd():
+        tmpdir = tempfile.mkdtemp('logging_test')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        with cd(tmpdir):
             config_data = config.load(details)
         project = Project.from_config(
             name='composetest', config_data=config_data, client=self.client

tests/integration/service_test.py

@@ -38,6 +38,8 @@ from compose.container import Container
 from compose.errors import OperationFailedError
 from compose.parallel import ParallelStreamWriter
 from compose.project import OneOffFilter
+from compose.project import Project
+from compose.service import BuildAction
 from compose.service import ConvergencePlan
 from compose.service import ConvergenceStrategy
 from compose.service import NetworkMode
@@ -966,6 +968,43 @@ class ServiceTest(DockerClientTestCase):
 
         assert self.client.inspect_image('composetest_web')
 
+    def test_build_cli(self):
+        base_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base_dir)
+
+        with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
+            f.write("FROM busybox\n")
+
+        service = self.create_service('web',
+                                      build={'context': base_dir},
+                                      environment={
+                                          'COMPOSE_DOCKER_CLI_BUILD': '1',
+                                          'DOCKER_BUILDKIT': '1',
+                                      })
+        service.build(cli=True)
+        self.addCleanup(self.client.remove_image, service.image_name)
+        assert self.client.inspect_image('composetest_web')
+
+    def test_up_build_cli(self):
+        base_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base_dir)
+
+        with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
+            f.write("FROM busybox\n")
+
+        web = self.create_service('web',
+                                  build={'context': base_dir},
+                                  environment={
+                                      'COMPOSE_DOCKER_CLI_BUILD': '1',
+                                      'DOCKER_BUILDKIT': '1',
+                                  })
+        project = Project('composetest', [web], self.client)
+        project.up(do_build=BuildAction.force)
+
+        containers = project.containers(['web'])
+        assert len(containers) == 1
+        assert containers[0].name.startswith('composetest_web_')
+
     def test_build_non_ascii_filename(self):
         base_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, base_dir)

tests/integration/state_test.py

@@ -6,8 +6,10 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import copy
+import os
+import shutil
+import tempfile
 
-import py
 from docker.errors import ImageNotFound
 
 from ..helpers import BUSYBOX_IMAGE_WITH_TAG
@@ -426,29 +428,32 @@ class ServiceStateTest(DockerClientTestCase):
 
     @no_cluster('Can not guarantee the build will be run on the same node the service is deployed')
     def test_trigger_recreate_with_build(self):
-        context = py.test.ensuretemp('test_trigger_recreate_with_build')
-        self.addCleanup(context.remove)
+        context = tempfile.mkdtemp('test_trigger_recreate_with_build')
+        self.addCleanup(shutil.rmtree, context)
 
         base_image = "FROM busybox\nLABEL com.docker.compose.test_image=true\n"
-        dockerfile = context.join('Dockerfile')
-        dockerfile.write(base_image)
+        dockerfile = os.path.join(context, 'Dockerfile')
+        with open(dockerfile, mode="w") as dockerfile_fh:
+            dockerfile_fh.write(base_image)
 
         web = self.create_service('web', build={'context': str(context)})
         container = web.create_container()
 
-        dockerfile.write(base_image + 'CMD echo hello world\n')
+        with open(dockerfile, mode="w") as dockerfile_fh:
+            dockerfile_fh.write(base_image + 'CMD echo hello world\n')
         web.build()
 
         web = self.create_service('web', build={'context': str(context)})
         assert ('recreate', [container]) == web.convergence_plan()
 
     def test_image_changed_to_build(self):
-        context = py.test.ensuretemp('test_image_changed_to_build')
-        self.addCleanup(context.remove)
-        context.join('Dockerfile').write("""
-            FROM busybox
-            LABEL com.docker.compose.test_image=true
-        """)
+        context = tempfile.mkdtemp('test_image_changed_to_build')
+        self.addCleanup(shutil.rmtree, context)
+        with open(os.path.join(context, 'Dockerfile'), mode="w") as dockerfile:
+            dockerfile.write("""
+                FROM busybox
+                LABEL com.docker.compose.test_image=true
+            """)
 
         web = self.create_service('web', image='busybox')
         container = web.create_container()

tests/unit/bundle_test.py

@@ -1,233 +0,0 @@
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-import docker
-import pytest
-
-from .. import mock
-from compose import bundle
-from compose import service
-from compose.cli.errors import UserError
-from compose.config.config import Config
-from compose.const import COMPOSEFILE_V2_0 as V2_0
-from compose.service import NoSuchImageError
-
-
-@pytest.fixture
-def mock_service():
-    return mock.create_autospec(
-        service.Service,
-        client=mock.create_autospec(docker.APIClient),
-        options={})
-
-
-def test_get_image_digest_exists(mock_service):
-    mock_service.options['image'] = 'abcd'
-    mock_service.image.return_value = {'RepoDigests': ['digest1']}
-    digest = bundle.get_image_digest(mock_service)
-    assert digest == 'digest1'
-
-
-def test_get_image_digest_image_uses_digest(mock_service):
-    mock_service.options['image'] = image_id = 'redis@sha256:digest'
-
-    digest = bundle.get_image_digest(mock_service)
-    assert digest == image_id
-    assert not mock_service.image.called
-
-
-def test_get_image_digest_from_repository(mock_service):
-    mock_service.options['image'] = 'abcd'
-    mock_service.image_name = 'abcd'
-    mock_service.image.side_effect = NoSuchImageError(None)
-    mock_service.get_image_registry_data.return_value = {'Descriptor': {'digest': 'digest'}}
-
-    digest = bundle.get_image_digest(mock_service)
-    assert digest == 'abcd@digest'
-
-
-def test_get_image_digest_no_image(mock_service):
-    with pytest.raises(UserError) as exc:
-        bundle.get_image_digest(service.Service(name='theservice'))
-
-    assert "doesn't define an image tag" in exc.exconly()
-
-
-def test_push_image_with_saved_digest(mock_service):
-    mock_service.options['build'] = '.'
-    mock_service.options['image'] = image_id = 'abcd'
-    mock_service.push.return_value = expected = 'sha256:thedigest'
-    mock_service.image.return_value = {'RepoDigests': ['digest1']}
-
-    digest = bundle.push_image(mock_service)
-    assert digest == image_id + '@' + expected
-
-    mock_service.push.assert_called_once_with()
-    assert not mock_service.client.push.called
-
-
-def test_push_image(mock_service):
-    mock_service.options['build'] = '.'
-    mock_service.options['image'] = image_id = 'abcd'
-    mock_service.push.return_value = expected = 'sha256:thedigest'
-    mock_service.image.return_value = {'RepoDigests': []}
-
-    digest = bundle.push_image(mock_service)
-    assert digest == image_id + '@' + expected
-
-    mock_service.push.assert_called_once_with()
-    mock_service.client.pull.assert_called_once_with(digest)
-
-
-def test_to_bundle():
-    image_digests = {'a': 'aaaa', 'b': 'bbbb'}
-    services = [
-        {'name': 'a', 'build': '.', },
-        {'name': 'b', 'build': './b'},
-    ]
-    config = Config(
-        version=V2_0,
-        services=services,
-        volumes={'special': {}},
-        networks={'extra': {}},
-        secrets={},
-        configs={}
-    )
-
-    with mock.patch('compose.bundle.log.warning', autospec=True) as mock_log:
-        output = bundle.to_bundle(config, image_digests)
-
-    assert mock_log.mock_calls == [
-        mock.call("Unsupported top level key 'networks' - ignoring"),
-        mock.call("Unsupported top level key 'volumes' - ignoring"),
-    ]
-
-    assert output == {
-        'Version': '0.1',
-        'Services': {
-            'a': {'Image': 'aaaa', 'Networks': ['default']},
-            'b': {'Image': 'bbbb', 'Networks': ['default']},
-        }
-    }
-
-
-def test_convert_service_to_bundle():
-    name = 'theservice'
-    image_digest = 'thedigest'
-    service_dict = {
-        'ports': ['80'],
-        'expose': ['1234'],
-        'networks': {'extra': {}},
-        'command': 'foo',
-        'entrypoint': 'entry',
-        'environment': {'BAZ': 'ENV'},
-        'build': '.',
-        'working_dir': '/tmp',
-        'user': 'root',
-        'labels': {'FOO': 'LABEL'},
-        'privileged': True,
-    }
-
-    with mock.patch('compose.bundle.log.warning', autospec=True) as mock_log:
-        config = bundle.convert_service_to_bundle(name, service_dict, image_digest)
-
-    mock_log.assert_called_once_with(
-        "Unsupported key 'privileged' in services.theservice - ignoring")
-
-    assert config == {
-        'Image': image_digest,
-        'Ports': [
-            {'Protocol': 'tcp', 'Port': 80},
-            {'Protocol': 'tcp', 'Port': 1234},
-        ],
-        'Networks': ['extra'],
-        'Command': ['entry', 'foo'],
-        'Env': ['BAZ=ENV'],
-        'WorkingDir': '/tmp',
-        'User': 'root',
-        'Labels': {'FOO': 'LABEL'},
-    }
-
-
-def test_set_command_and_args_none():
-    config = {}
-    bundle.set_command_and_args(config, [], [])
-    assert config == {}
-
-
-def test_set_command_and_args_from_command():
-    config = {}
-    bundle.set_command_and_args(config, [], "echo ok")
-    assert config == {'Args': ['echo', 'ok']}
-
-
-def test_set_command_and_args_from_entrypoint():
-    config = {}
-    bundle.set_command_and_args(config, "echo entry", [])
-    assert config == {'Command': ['echo', 'entry']}
-
-
-def test_set_command_and_args_from_both():
-    config = {}
-    bundle.set_command_and_args(config, "echo entry", ["extra", "arg"])
-    assert config == {'Command': ['echo', 'entry', "extra", "arg"]}
-
-
-def test_make_service_networks_default():
-    name = 'theservice'
-    service_dict = {}
-
-    with mock.patch('compose.bundle.log.warning', autospec=True) as mock_log:
-        networks = bundle.make_service_networks(name, service_dict)
-
-    assert not mock_log.called
-    assert networks == ['default']
-
-
-def test_make_service_networks():
-    name = 'theservice'
-    service_dict = {
-        'networks': {
-            'foo': {
-                'aliases': ['one', 'two'],
-            },
-            'bar': {}
-        },
-    }
-
-    with mock.patch('compose.bundle.log.warning', autospec=True) as mock_log:
-        networks = bundle.make_service_networks(name, service_dict)
-
-    mock_log.assert_called_once_with(
-        "Unsupported key 'aliases' in services.theservice.networks.foo - ignoring")
-    assert sorted(networks) == sorted(service_dict['networks'])
-
-
-def test_make_port_specs():
-    service_dict = {
-        'expose': ['80', '500/udp'],
-        'ports': [
-            '400:80',
-            '222',
-            '127.0.0.1:8001:8001',
-            '127.0.0.1:5000-5001:3000-3001'],
-    }
-    port_specs = bundle.make_port_specs(service_dict)
-    assert port_specs == [
-        {'Protocol': 'tcp', 'Port': 80},
-        {'Protocol': 'tcp', 'Port': 222},
-        {'Protocol': 'tcp', 'Port': 8001},
-        {'Protocol': 'tcp', 'Port': 3000},
-        {'Protocol': 'tcp', 'Port': 3001},
-        {'Protocol': 'udp', 'Port': 500},
-    ]
-
-
-def test_make_port_spec_with_protocol():
-    port_spec = bundle.make_port_spec("5000/udp")
-    assert port_spec == {'Protocol': 'udp', 'Port': 5000}
-
-
-def test_make_port_spec_default_protocol():
-    port_spec = bundle.make_port_spec("50000")
-    assert port_spec == {'Protocol': 'tcp', 'Port': 50000}

tests/unit/cli/log_printer_test.py

@@ -152,6 +152,17 @@ class TestWatchEvents(object):
                 *thread_args)
         assert container_id in thread_map
 
+    def test_container_attach_event(self, thread_map, mock_presenters):
+        container_id = 'abcd'
+        mock_container = mock.Mock(is_restarting=False)
+        mock_container.attach_log_stream.side_effect = APIError("race condition")
+        event_die = {'action': 'die', 'id': container_id}
+        event_start = {'action': 'start', 'id': container_id, 'container': mock_container}
+        event_stream = [event_die, event_start]
+        thread_args = 'foo', 'bar'
+        watch_events(thread_map, event_stream, mock_presenters, thread_args)
+        assert mock_container.attach_log_stream.called
+
     def test_other_event(self, thread_map, mock_presenters):
         container_id = 'abcd'
         event_stream = [{'action': 'create', 'id': container_id}]

tests/unit/cli/main_test.py

@@ -12,7 +12,7 @@ from compose.cli.formatter import ConsoleWarningFormatter
 from compose.cli.main import build_one_off_container_options
 from compose.cli.main import call_docker
 from compose.cli.main import convergence_strategy_from_opts
-from compose.cli.main import filter_containers_to_service_names
+from compose.cli.main import filter_attached_containers
 from compose.cli.main import get_docker_start_call
 from compose.cli.main import setup_console_handler
 from compose.cli.main import warn_for_swarm_mode
@@ -37,7 +37,7 @@ def logging_handler():
 
 class TestCLIMainTestCase(object):
 
-    def test_filter_containers_to_service_names(self):
+    def test_filter_attached_containers(self):
         containers = [
             mock_container('web', 1),
             mock_container('web', 2),
@@ -46,17 +46,29 @@ class TestCLIMainTestCase(object):
             mock_container('another', 1),
         ]
         service_names = ['web', 'db']
-        actual = filter_containers_to_service_names(containers, service_names)
+        actual = filter_attached_containers(containers, service_names)
         assert actual == containers[:3]
 
-    def test_filter_containers_to_service_names_all(self):
+    def test_filter_attached_containers_with_dependencies(self):
+        containers = [
+            mock_container('web', 1),
+            mock_container('web', 2),
+            mock_container('db', 1),
+            mock_container('other', 1),
+            mock_container('another', 1),
+        ]
+        service_names = ['web', 'db']
+        actual = filter_attached_containers(containers, service_names, attach_dependencies=True)
+        assert actual == containers
+
+    def test_filter_attached_containers_all(self):
         containers = [
             mock_container('web', 1),
             mock_container('db', 1),
             mock_container('other', 1),
         ]
         service_names = []
-        actual = filter_containers_to_service_names(containers, service_names)
+        actual = filter_attached_containers(containers, service_names)
         assert actual == containers
 
     def test_warning_in_swarm_mode(self):

tests/unit/cli/utils_test.py

@@ -29,16 +29,20 @@ class HumanReadableFileSizeTest(unittest.TestCase):
         assert human_readable_file_size(100) == '100 B'
 
     def test_1kb(self):
-        assert human_readable_file_size(1024) == '1 kB'
+        assert human_readable_file_size(1000) == '1 kB'
+        assert human_readable_file_size(1024) == '1.024 kB'
 
     def test_1023b(self):
-        assert human_readable_file_size(1023) == '1023 B'
+        assert human_readable_file_size(1023) == '1.023 kB'
+
+    def test_999b(self):
+        assert human_readable_file_size(999) == '999 B'
 
     def test_units(self):
-        assert human_readable_file_size((2 ** 10) ** 0) == '1 B'
-        assert human_readable_file_size((2 ** 10) ** 1) == '1 kB'
-        assert human_readable_file_size((2 ** 10) ** 2) == '1 MB'
-        assert human_readable_file_size((2 ** 10) ** 3) == '1 GB'
-        assert human_readable_file_size((2 ** 10) ** 4) == '1 TB'
-        assert human_readable_file_size((2 ** 10) ** 5) == '1 PB'
-        assert human_readable_file_size((2 ** 10) ** 6) == '1 EB'
+        assert human_readable_file_size((10 ** 3) ** 0) == '1 B'
+        assert human_readable_file_size((10 ** 3) ** 1) == '1 kB'
+        assert human_readable_file_size((10 ** 3) ** 2) == '1 MB'
+        assert human_readable_file_size((10 ** 3) ** 3) == '1 GB'
+        assert human_readable_file_size((10 ** 3) ** 4) == '1 TB'
+        assert human_readable_file_size((10 ** 3) ** 5) == '1 PB'
+        assert human_readable_file_size((10 ** 3) ** 6) == '1 EB'

tests/unit/config/config_test.py

@@ -10,14 +10,17 @@ import tempfile
 from operator import itemgetter
 from random import shuffle
 
-import py
 import pytest
 import yaml
+from ddt import data
+from ddt import ddt
 
 from ...helpers import build_config_details
 from ...helpers import BUSYBOX_IMAGE_WITH_TAG
+from ...helpers import cd
 from compose.config import config
 from compose.config import types
+from compose.config.config import ConfigFile
 from compose.config.config import resolve_build_args
 from compose.config.config import resolve_environment
 from compose.config.environment import Environment
@@ -67,6 +70,7 @@ def secret_sort(secrets):
     return sorted(secrets, key=itemgetter('source'))
 
 
+@ddt
 class ConfigTest(unittest.TestCase):
 
     def test_load(self):
@@ -776,13 +780,14 @@ class ConfigTest(unittest.TestCase):
             })
         details = config.ConfigDetails('.', [base_file, override_file])
 
-        tmpdir = py.test.ensuretemp('config_test')
-        self.addCleanup(tmpdir.remove)
-        tmpdir.join('common.yml').write("""
-            base:
-              labels: ['label=one']
-        """)
-        with tmpdir.as_cwd():
+        tmpdir = tempfile.mkdtemp('config_test')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        with open(os.path.join(tmpdir, 'common.yml'), mode="w") as common_fh:
+            common_fh.write("""
+                base:
+                  labels: ['label=one']
+            """)
+        with cd(tmpdir):
             service_dicts = config.load(details).services
 
         expected = [
@@ -811,19 +816,20 @@ class ConfigTest(unittest.TestCase):
             }
         )
 
-        tmpdir = pytest.ensuretemp('config_test')
-        self.addCleanup(tmpdir.remove)
-        tmpdir.join('base.yml').write("""
-            version: '2.2'
-            services:
-              base:
-                image: base
-              web:
-                extends: base
-        """)
+        tmpdir = tempfile.mkdtemp('config_test')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        with open(os.path.join(tmpdir, 'base.yml'), mode="w") as base_fh:
+            base_fh.write("""
+                version: '2.2'
+                services:
+                  base:
+                    image: base
+                  web:
+                    extends: base
+            """)
 
         details = config.ConfigDetails('.', [main_file])
-        with tmpdir.as_cwd():
+        with cd(tmpdir):
             service_dicts = config.load(details).services
             assert service_dicts[0] == {
                 'name': 'prodweb',
@@ -1761,22 +1767,23 @@ class ConfigTest(unittest.TestCase):
         assert services[0]['environment']['SPRING_JPA_HIBERNATE_DDL-AUTO'] == 'none'
 
     def test_load_yaml_with_yaml_error(self):
-        tmpdir = py.test.ensuretemp('invalid_yaml_test')
-        self.addCleanup(tmpdir.remove)
-        invalid_yaml_file = tmpdir.join('docker-compose.yml')
-        invalid_yaml_file.write("""
-            web:
-              this is bogus: ok: what
-        """)
+        tmpdir = tempfile.mkdtemp('invalid_yaml_test')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        invalid_yaml_file = os.path.join(tmpdir, 'docker-compose.yml')
+        with open(invalid_yaml_file, mode="w") as invalid_yaml_file_fh:
+            invalid_yaml_file_fh.write("""
+web:
+    this is bogus: ok: what
+            """)
         with pytest.raises(ConfigurationError) as exc:
             config.load_yaml(str(invalid_yaml_file))
 
-        assert 'line 3, column 32' in exc.exconly()
+        assert 'line 3, column 22' in exc.exconly()
 
     def test_load_yaml_with_bom(self):
-        tmpdir = py.test.ensuretemp('bom_yaml')
-        self.addCleanup(tmpdir.remove)
-        bom_yaml = tmpdir.join('docker-compose.yml')
+        tmpdir = tempfile.mkdtemp('bom_yaml')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        bom_yaml = os.path.join(tmpdir, 'docker-compose.yml')
         with codecs.open(str(bom_yaml), 'w', encoding='utf-8') as f:
             f.write('''\ufeff
                 version: '2.3'
@@ -1884,6 +1891,26 @@ class ConfigTest(unittest.TestCase):
             }
         ]
 
+    @data(
+        '2 ',
+        '3.',
+        '3.0.0',
+        '3.0.a',
+        '3.a',
+        '3a')
+    def test_invalid_version_formats(self, version):
+        content = {
+            'version': version,
+            'services': {
+                'web': {
+                    'image': 'alpine',
+                }
+            }
+        }
+        with pytest.raises(ConfigurationError) as exc:
+            config.load(build_config_details(content))
+        assert 'Version "{}" in "filename.yml" is invalid.'.format(version) in exc.exconly()
+
     def test_group_add_option(self):
         actual = config.load(build_config_details({
             'version': '2',
@@ -3620,7 +3647,7 @@ class InterpolationTest(unittest.TestCase):
             'version': '3.5',
             'services': {
                 'foo': {
-                    'image': 'alpine:3.7',
+                    'image': 'alpine:3.10.1',
                     'deploy': {
                         'replicas': 3,
                         'restart_policy': {
@@ -3646,7 +3673,7 @@ class InterpolationTest(unittest.TestCase):
 
         service_dict = cfg.services[0]
         assert service_dict == {
-            'image': 'alpine:3.7',
+            'image': 'alpine:3.10.1',
             'scale': 3,
             'restart': {'MaximumRetryCount': 7, 'Name': 'always'},
             'mem_limit': '300M',
@@ -4700,43 +4727,48 @@ class ExtendsTest(unittest.TestCase):
 
     @mock.patch.dict(os.environ)
     def test_extends_with_environment_and_env_files(self):
-        tmpdir = py.test.ensuretemp('test_extends_with_environment')
-        self.addCleanup(tmpdir.remove)
-        commondir = tmpdir.mkdir('common')
-        commondir.join('base.yml').write("""
-            app:
-                image: 'example/app'
-                env_file:
-                    - 'envs'
-                environment:
-                    - SECRET
-                    - TEST_ONE=common
-                    - TEST_TWO=common
-        """)
-        tmpdir.join('docker-compose.yml').write("""
-            ext:
-                extends:
-                    file: common/base.yml
-                    service: app
-                env_file:
-                    - 'envs'
-                environment:
-                    - THING
-                    - TEST_ONE=top
-        """)
-        commondir.join('envs').write("""
-            COMMON_ENV_FILE
-            TEST_ONE=common-env-file
-            TEST_TWO=common-env-file
-            TEST_THREE=common-env-file
-            TEST_FOUR=common-env-file
-        """)
-        tmpdir.join('envs').write("""
-            TOP_ENV_FILE
-            TEST_ONE=top-env-file
-            TEST_TWO=top-env-file
-            TEST_THREE=top-env-file
-        """)
+        tmpdir = tempfile.mkdtemp('test_extends_with_environment')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        commondir = os.path.join(tmpdir, 'common')
+        os.mkdir(commondir)
+        with open(os.path.join(commondir, 'base.yml'), mode="w") as base_fh:
+            base_fh.write("""
+                app:
+                    image: 'example/app'
+                    env_file:
+                        - 'envs'
+                    environment:
+                        - SECRET
+                        - TEST_ONE=common
+                        - TEST_TWO=common
+            """)
+        with open(os.path.join(tmpdir, 'docker-compose.yml'), mode="w") as docker_compose_fh:
+            docker_compose_fh.write("""
+                ext:
+                    extends:
+                        file: common/base.yml
+                        service: app
+                    env_file:
+                        - 'envs'
+                    environment:
+                        - THING
+                        - TEST_ONE=top
+            """)
+        with open(os.path.join(commondir, 'envs'), mode="w") as envs_fh:
+            envs_fh.write("""
+                COMMON_ENV_FILE
+                TEST_ONE=common-env-file
+                TEST_TWO=common-env-file
+                TEST_THREE=common-env-file
+                TEST_FOUR=common-env-file
+            """)
+        with open(os.path.join(tmpdir, 'envs'), mode="w") as envs_fh:
+            envs_fh.write("""
+                TOP_ENV_FILE
+                TEST_ONE=top-env-file
+                TEST_TWO=top-env-file
+                TEST_THREE=top-env-file
+            """)
 
         expected = [
             {
@@ -4759,72 +4791,77 @@ class ExtendsTest(unittest.TestCase):
         os.environ['THING'] = 'thing'
         os.environ['COMMON_ENV_FILE'] = 'secret'
         os.environ['TOP_ENV_FILE'] = 'secret'
-        config = load_from_filename(str(tmpdir.join('docker-compose.yml')))
+        config = load_from_filename(str(os.path.join(tmpdir, 'docker-compose.yml')))
 
         assert config == expected
 
     def test_extends_with_mixed_versions_is_error(self):
-        tmpdir = py.test.ensuretemp('test_extends_with_mixed_version')
-        self.addCleanup(tmpdir.remove)
-        tmpdir.join('docker-compose.yml').write("""
-            version: "2"
-            services:
-              web:
-                extends:
-                  file: base.yml
-                  service: base
-                image: busybox
-        """)
-        tmpdir.join('base.yml').write("""
-            base:
-              volumes: ['/foo']
-              ports: ['3000:3000']
-        """)
-
-        with pytest.raises(ConfigurationError) as exc:
-            load_from_filename(str(tmpdir.join('docker-compose.yml')))
-        assert 'Version mismatch' in exc.exconly()
-
-    def test_extends_with_defined_version_passes(self):
-        tmpdir = py.test.ensuretemp('test_extends_with_defined_version')
-        self.addCleanup(tmpdir.remove)
-        tmpdir.join('docker-compose.yml').write("""
-            version: "2"
-            services:
-              web:
-                extends:
-                  file: base.yml
-                  service: base
-                image: busybox
-        """)
-        tmpdir.join('base.yml').write("""
-            version: "2"
-            services:
+        tmpdir = tempfile.mkdtemp('test_extends_with_mixed_version')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        with open(os.path.join(tmpdir, 'docker-compose.yml'), mode="w") as docker_compose_fh:
+            docker_compose_fh.write("""
+                version: "2"
+                services:
+                  web:
+                    extends:
+                      file: base.yml
+                      service: base
+                    image: busybox
+            """)
+        with open(os.path.join(tmpdir, 'base.yml'), mode="w") as base_fh:
+            base_fh.write("""
                 base:
                   volumes: ['/foo']
                   ports: ['3000:3000']
-                  command: top
-        """)
+            """)
 
-        service = load_from_filename(str(tmpdir.join('docker-compose.yml')))
+        with pytest.raises(ConfigurationError) as exc:
+            load_from_filename(str(os.path.join(tmpdir, 'docker-compose.yml')))
+        assert 'Version mismatch' in exc.exconly()
+
+    def test_extends_with_defined_version_passes(self):
+        tmpdir = tempfile.mkdtemp('test_extends_with_defined_version')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        with open(os.path.join(tmpdir, 'docker-compose.yml'), mode="w") as docker_compose_fh:
+            docker_compose_fh.write("""
+                version: "2"
+                services:
+                  web:
+                    extends:
+                      file: base.yml
+                      service: base
+                    image: busybox
+            """)
+        with open(os.path.join(tmpdir, 'base.yml'), mode="w") as base_fh:
+            base_fh.write("""
+                version: "2"
+                services:
+                  base:
+                    volumes: ['/foo']
+                    ports: ['3000:3000']
+                    command: top
+            """)
+
+        service = load_from_filename(str(os.path.join(tmpdir, 'docker-compose.yml')))
         assert service[0]['command'] == "top"
 
     def test_extends_with_depends_on(self):
-        tmpdir = py.test.ensuretemp('test_extends_with_depends_on')
-        self.addCleanup(tmpdir.remove)
-        tmpdir.join('docker-compose.yml').write("""
-            version: "2"
-            services:
-              base:
-                image: example
-              web:
-                extends: base
-                image: busybox
-                depends_on: ['other']
-              other:
-                image: example
-        """)
-        services = load_from_filename(str(tmpdir.join('docker-compose.yml')))
+        tmpdir = tempfile.mkdtemp('test_extends_with_depends_on')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        with open(os.path.join(tmpdir, 'docker-compose.yml'), mode="w") as docker_compose_fh:
+            docker_compose_fh.write("""
+                version: "2"
+                services:
+                  base:
+                    image: example
+                  web:
+                    extends: base
+                    image: busybox
+                    depends_on: ['other']
+                  other:
+                    image: example
+            """)
+        services = load_from_filename(str(os.path.join(tmpdir, 'docker-compose.yml')))
         assert service_sort(services)[2]['depends_on'] == {
             'other': {'condition': 'service_started'}
         }
@@ -4843,50 +4880,57 @@ class ExtendsTest(unittest.TestCase):
         }]
 
     def test_extends_with_ports(self):
-        tmpdir = py.test.ensuretemp('test_extends_with_ports')
-        self.addCleanup(tmpdir.remove)
-        tmpdir.join('docker-compose.yml').write("""
-            version: '2'
+        tmpdir = tempfile.mkdtemp('test_extends_with_ports')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        with open(os.path.join(tmpdir, 'docker-compose.yml'), mode="w") as docker_compose_fh:
+            docker_compose_fh.write("""
+                version: '2'
 
-            services:
-              a:
-                image: nginx
-                ports:
-                  - 80
+                services:
+                  a:
+                    image: nginx
+                    ports:
+                      - 80
 
-              b:
-                extends:
-                  service: a
-        """)
-        services = load_from_filename(str(tmpdir.join('docker-compose.yml')))
+                  b:
+                    extends:
+                      service: a
+            """)
+        services = load_from_filename(str(os.path.join(tmpdir, 'docker-compose.yml')))
 
         assert len(services) == 2
         for svc in services:
             assert svc['ports'] == [types.ServicePort('80', None, None, None, None)]
 
     def test_extends_with_security_opt(self):
-        tmpdir = py.test.ensuretemp('test_extends_with_ports')
-        self.addCleanup(tmpdir.remove)
-        tmpdir.join('docker-compose.yml').write("""
-            version: '2'
+        tmpdir = tempfile.mkdtemp('test_extends_with_ports')
+        self.addCleanup(shutil.rmtree, tmpdir)
+        with open(os.path.join(tmpdir, 'docker-compose.yml'), mode="w") as docker_compose_fh:
+            docker_compose_fh.write("""
+                version: '2'
 
-            services:
-              a:
-                image: nginx
-                security_opt:
-                  - apparmor:unconfined
-                  - seccomp:unconfined
+                services:
+                  a:
+                    image: nginx
+                    security_opt:
+                    - apparmor:unconfined
+                    - seccomp:unconfined
 
-              b:
-                extends:
-                  service: a
-        """)
-        services = load_from_filename(str(tmpdir.join('docker-compose.yml')))
+                  b:
+                    extends:
+                      service: a
+            """)
+        services = load_from_filename(str(os.path.join(tmpdir, 'docker-compose.yml')))
         assert len(services) == 2
         for svc in services:
             assert types.SecurityOpt.parse('apparmor:unconfined') in svc['security_opt']
             assert types.SecurityOpt.parse('seccomp:unconfined') in svc['security_opt']
 
+    @mock.patch.object(ConfigFile, 'from_filename', wraps=ConfigFile.from_filename)
+    def test_extends_same_file_optimization(self, from_filename_mock):
+        load_from_filename('tests/fixtures/extends/no-file-specified.yml')
+        from_filename_mock.assert_called_once()
+
 
 @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash')
 class ExpandPathTest(unittest.TestCase):
@@ -5031,7 +5075,7 @@ class HealthcheckTest(unittest.TestCase):
             })
         )
 
-        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_config = yaml.safe_load(serialize_config(config_dict))
         serialized_service = serialized_config['services']['test']
 
         assert serialized_service['healthcheck'] == {
@@ -5058,7 +5102,7 @@ class HealthcheckTest(unittest.TestCase):
             })
         )
 
-        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_config = yaml.safe_load(serialize_config(config_dict))
         serialized_service = serialized_config['services']['test']
 
         assert serialized_service['healthcheck'] == {
@@ -5265,7 +5309,7 @@ class SerializeTest(unittest.TestCase):
             'secrets': secrets_dict
         }))
 
-        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_config = yaml.safe_load(serialize_config(config_dict))
         serialized_service = serialized_config['services']['web']
         assert secret_sort(serialized_service['secrets']) == secret_sort(service_dict['secrets'])
         assert 'secrets' in serialized_config
@@ -5280,7 +5324,7 @@ class SerializeTest(unittest.TestCase):
             }
         ], volumes={}, networks={}, secrets={}, configs={})
 
-        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_config = yaml.safe_load(serialize_config(config_dict))
         assert '8080:80/tcp' in serialized_config['services']['web']['ports']
 
     def test_serialize_ports_with_ext_ip(self):
@@ -5292,7 +5336,7 @@ class SerializeTest(unittest.TestCase):
             }
         ], volumes={}, networks={}, secrets={}, configs={})
 
-        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_config = yaml.safe_load(serialize_config(config_dict))
         assert '127.0.0.1:8080:80/tcp' in serialized_config['services']['web']['ports']
 
     def test_serialize_configs(self):
@@ -5320,7 +5364,7 @@ class SerializeTest(unittest.TestCase):
             'configs': configs_dict
         }))
 
-        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_config = yaml.safe_load(serialize_config(config_dict))
         serialized_service = serialized_config['services']['web']
         assert secret_sort(serialized_service['configs']) == secret_sort(service_dict['configs'])
         assert 'configs' in serialized_config
@@ -5360,7 +5404,7 @@ class SerializeTest(unittest.TestCase):
         }
         config_dict = config.load(build_config_details(cfg))
 
-        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_config = yaml.safe_load(serialize_config(config_dict))
         serialized_service = serialized_config['services']['web']
         assert serialized_service['environment']['CURRENCY'] == '$$'
         assert serialized_service['command'] == 'echo $$FOO'
@@ -5382,7 +5426,7 @@ class SerializeTest(unittest.TestCase):
         }
         config_dict = config.load(build_config_details(cfg), interpolate=False)
 
-        serialized_config = yaml.load(serialize_config(config_dict, escape_dollar=False))
+        serialized_config = yaml.safe_load(serialize_config(config_dict, escape_dollar=False))
         serialized_service = serialized_config['services']['web']
         assert serialized_service['environment']['CURRENCY'] == '$'
         assert serialized_service['command'] == 'echo $FOO'
@@ -5401,7 +5445,7 @@ class SerializeTest(unittest.TestCase):
 
         config_dict = config.load(build_config_details(cfg))
 
-        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_config = yaml.safe_load(serialize_config(config_dict))
         serialized_service = serialized_config['services']['web']
         assert serialized_service['command'] == 'echo 十六夜　咲夜'
 
@@ -5417,6 +5461,6 @@ class SerializeTest(unittest.TestCase):
         }
 
         config_dict = config.load(build_config_details(cfg))
-        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_config = yaml.safe_load(serialize_config(config_dict))
         serialized_volume = serialized_config['volumes']['test']
         assert serialized_volume['external'] is False

tests/unit/config/environment_test.py

@@ -4,6 +4,9 @@ from __future__ import print_function
 from __future__ import unicode_literals
 
 import codecs
+import os
+import shutil
+import tempfile
 
 import pytest
 
@@ -46,19 +49,19 @@ class EnvironmentTest(unittest.TestCase):
         assert env.get_boolean('UNDEFINED') is False
 
     def test_env_vars_from_file_bom(self):
-        tmpdir = pytest.ensuretemp('env_file')
-        self.addCleanup(tmpdir.remove)
+        tmpdir = tempfile.mkdtemp('env_file')
+        self.addCleanup(shutil.rmtree, tmpdir)
         with codecs.open('{}/bom.env'.format(str(tmpdir)), 'w', encoding='utf-8') as f:
             f.write('\ufeffPARK_BOM=박봄\n')
-        assert env_vars_from_file(str(tmpdir.join('bom.env'))) == {
+        assert env_vars_from_file(str(os.path.join(tmpdir, 'bom.env'))) == {
             'PARK_BOM': '박봄'
         }
 
     def test_env_vars_from_file_whitespace(self):
-        tmpdir = pytest.ensuretemp('env_file')
-        self.addCleanup(tmpdir.remove)
+        tmpdir = tempfile.mkdtemp('env_file')
+        self.addCleanup(shutil.rmtree, tmpdir)
         with codecs.open('{}/whitespace.env'.format(str(tmpdir)), 'w', encoding='utf-8') as f:
             f.write('WHITESPACE =yes\n')
         with pytest.raises(ConfigurationError) as exc:
-            env_vars_from_file(str(tmpdir.join('whitespace.env')))
+            env_vars_from_file(str(os.path.join(tmpdir, 'whitespace.env')))
         assert 'environment variable' in exc.exconly()

tests/unit/network_test.py

@@ -168,3 +168,8 @@ class NetworkTest(unittest.TestCase):
         mock_log.warning.assert_called_once_with(mock.ANY)
         _, args, kwargs = mock_log.warning.mock_calls[0]
         assert 'label "com.project.touhou.character" has changed' in args[0]
+
+    def test_remote_config_labels_none(self):
+        remote = {'Labels': None}
+        local = Network(None, 'test_project', 'test_network')
+        check_remote_network_config(remote, local)

tests/unit/project_test.py

@@ -3,6 +3,8 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import datetime
+import os
+import tempfile
 
 import docker
 import pytest
@@ -11,6 +13,7 @@ from docker.errors import NotFound
 from .. import mock
 from .. import unittest
 from ..helpers import BUSYBOX_IMAGE_WITH_TAG
+from compose.config import ConfigurationError
 from compose.config.config import Config
 from compose.config.types import VolumeFromSpec
 from compose.const import COMPOSEFILE_V1 as V1
@@ -21,6 +24,7 @@ from compose.const import DEFAULT_TIMEOUT
 from compose.const import LABEL_SERVICE
 from compose.container import Container
 from compose.errors import OperationFailedError
+from compose.project import get_secrets
 from compose.project import NoSuchService
 from compose.project import Project
 from compose.project import ProjectError
@@ -841,3 +845,84 @@ class ProjectTest(unittest.TestCase):
         with mock.patch('compose.service.Service.push') as fake_push:
             project.push()
             assert fake_push.call_count == 2
+
+    def test_get_secrets_no_secret_def(self):
+        service = 'foo'
+        secret_source = 'bar'
+
+        secret_defs = mock.Mock()
+        secret_defs.get.return_value = None
+        secret = mock.Mock(source=secret_source)
+
+        with self.assertRaises(ConfigurationError):
+            get_secrets(service, [secret], secret_defs)
+
+    def test_get_secrets_external_warning(self):
+        service = 'foo'
+        secret_source = 'bar'
+
+        secret_def = mock.Mock()
+        secret_def.get.return_value = True
+
+        secret_defs = mock.Mock()
+        secret_defs.get.side_effect = secret_def
+        secret = mock.Mock(source=secret_source)
+
+        with mock.patch('compose.project.log') as mock_log:
+            get_secrets(service, [secret], secret_defs)
+
+        mock_log.warning.assert_called_with("Service \"{service}\" uses secret \"{secret}\" "
+                                            "which is external. External secrets are not available"
+                                            " to containers created by docker-compose."
+                                            .format(service=service, secret=secret_source))
+
+    def test_get_secrets_uid_gid_mode_warning(self):
+        service = 'foo'
+        secret_source = 'bar'
+
+        fd, filename_path = tempfile.mkstemp()
+        os.close(fd)
+        self.addCleanup(os.remove, filename_path)
+
+        def mock_get(key):
+            return {'external': False, 'file': filename_path}[key]
+
+        secret_def = mock.MagicMock()
+        secret_def.get = mock.MagicMock(side_effect=mock_get)
+
+        secret_defs = mock.Mock()
+        secret_defs.get.return_value = secret_def
+
+        secret = mock.Mock(uid=True, gid=True, mode=True, source=secret_source)
+
+        with mock.patch('compose.project.log') as mock_log:
+            get_secrets(service, [secret], secret_defs)
+
+        mock_log.warning.assert_called_with("Service \"{service}\" uses secret \"{secret}\" with uid, "
+                                            "gid, or mode. These fields are not supported by this "
+                                            "implementation of the Compose file"
+                                            .format(service=service, secret=secret_source))
+
+    def test_get_secrets_secret_file_warning(self):
+        service = 'foo'
+        secret_source = 'bar'
+        not_a_path = 'NOT_A_PATH'
+
+        def mock_get(key):
+            return {'external': False, 'file': not_a_path}[key]
+
+        secret_def = mock.MagicMock()
+        secret_def.get = mock.MagicMock(side_effect=mock_get)
+
+        secret_defs = mock.Mock()
+        secret_defs.get.return_value = secret_def
+
+        secret = mock.Mock(uid=False, gid=False, mode=False, source=secret_source)
+
+        with mock.patch('compose.project.log') as mock_log:
+            get_secrets(service, [secret], secret_defs)
+
+        mock_log.warning.assert_called_with("Service \"{service}\" uses an undefined secret file "
+                                            "\"{secret_file}\", the following file should be created "
+                                            "\"{secret_file}\""
+                                            .format(service=service, secret_file=not_a_path))

tests/unit/service_test.py

@@ -521,7 +521,37 @@ class ServiceTest(unittest.TestCase):
             assert 'was built because it did not already exist' in args[0]
 
         assert self.mock_client.build.call_count == 1
-        self.mock_client.build.call_args[1]['tag'] == 'default_foo'
+        assert self.mock_client.build.call_args[1]['tag'] == 'default_foo'
+
+    def test_create_container_binary_string_error(self):
+        service = Service('foo', client=self.mock_client, build={'context': '.'})
+        service.image = lambda: {'Id': 'abc123'}
+
+        self.mock_client.create_container.side_effect = APIError(None,
+                                                                 None,
+                                                                 b"Test binary string explanation")
+        with pytest.raises(OperationFailedError) as ex:
+            service.create_container()
+
+        assert ex.value.msg == "Cannot create container for service foo: Test binary string explanation"
+
+    def test_start_binary_string_error(self):
+        service = Service('foo', client=self.mock_client)
+        container = Container(self.mock_client, {'Id': 'abc123'})
+
+        self.mock_client.start.side_effect = APIError(None,
+                                                      None,
+                                                      b"Test binary string explanation with "
+                                                      b"driver failed programming external "
+                                                      b"connectivity")
+        with mock.patch('compose.service.log', autospec=True) as mock_log:
+            with pytest.raises(OperationFailedError) as ex:
+                service.start_container(container)
+
+        assert ex.value.msg == "Cannot start service foo: " \
+                               "Test binary string explanation " \
+                               "with driver failed programming external connectivity"
+        mock_log.warn.assert_called_once_with("Host is already in use by another container")
 
     def test_ensure_image_exists_no_build(self):
         service = Service('foo', client=self.mock_client, build={'context': '.'})