Compare commits

..

1 Commits

Author SHA1 Message Date
Nicolas De Loof
9ad10575d1 Prepare drop of python 2.x support
see https://github.com/docker/compose/issues/6890

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
2019-11-20 16:00:53 +01:00
32 changed files with 273 additions and 925 deletions

63
.circleci/config.yml Normal file
View File

@@ -0,0 +1,63 @@
version: 2
jobs:
test:
macos:
xcode: "9.4.1"
steps:
- checkout
- run:
name: setup script
command: ./script/setup/osx
- run:
name: install tox
command: sudo pip install --upgrade tox==2.1.1 virtualenv==16.2.0
- run:
name: unit tests
command: tox -e py27,py37 -- tests/unit
build-osx-binary:
macos:
xcode: "9.4.1"
steps:
- checkout
- run:
name: upgrade python tools
command: sudo pip install --upgrade pip virtualenv==16.2.0
- run:
name: setup script
command: DEPLOYMENT_TARGET=10.11 ./script/setup/osx
- run:
name: build script
command: ./script/build/osx
- store_artifacts:
path: dist/docker-compose-Darwin-x86_64
destination: docker-compose-Darwin-x86_64
- deploy:
name: Deploy binary to bintray
command: |
OS_NAME=Darwin PKG_NAME=osx ./script/circle/bintray-deploy.sh
build-linux-binary:
machine:
enabled: true
steps:
- checkout
- run:
name: build Linux binary
command: ./script/build/linux
- store_artifacts:
path: dist/docker-compose-Linux-x86_64
destination: docker-compose-Linux-x86_64
- deploy:
name: Deploy binary to bintray
command: |
OS_NAME=Linux PKG_NAME=linux ./script/circle/bintray-deploy.sh
workflows:
version: 2
all:
jobs:
- test
- build-linux-binary
- build-osx-binary

View File

@@ -11,4 +11,3 @@ docs/_site
.tox
**/__pycache__
*.pyc
Jenkinsfile

6
.github/CODEOWNERS vendored
View File

@@ -1,6 +0,0 @@
# GitHub code owners
# See https://help.github.com/articles/about-codeowners/
#
# KEEP THIS FILE SORTED. Order is important. Last match takes precedence.
* @ndeloof @rumpl @ulyssessouza

View File

@@ -1,136 +1,6 @@
Change log
==========
1.25.1 (2020-01-06)
-------------------
### Features
- Bump `pytest-cov` 2.8.1
- Bump `flake8` 3.7.9
- Bump `coverage` 4.5.4
### Bugfixes
- Decode APIError explanation to unicode before usage on start and create of a container
- Reports when images that cannot be pulled and must be built
- Discard label `com.docker.compose.filepaths` having None as value. Typically, when coming from stdin
- Added OSX binary as a directory to solve slow start up time caused by MacOS Catalina binary scan
- Passed in HOME env-var in container mode (running with `script/run/run.sh`)
- Reverted behavior of "only pull images that we can't build" and replace by a warning informing the image we can't pull and must be built
1.25.0 (2019-11-18)
-------------------
### Features
- Set no-colors to true if CLICOLOR env variable is set to 0
- Add working dir, config files and env file in service labels
- Add dependencies for ARM build
- Add BuildKit support, use `DOCKER_BUILDKIT=1` and `COMPOSE_DOCKER_CLI_BUILD=1`
- Bump paramiko to 2.6.0
- Add working dir, config files and env file in service labels
- Add tag `docker-compose:latest`
- Add `docker-compose:<version>-alpine` image/tag
- Add `docker-compose:<version>-debian` image/tag
- Bumped `docker-py` 4.1.0
- Supports `requests` up to 2.22.0 version
- Drops empty tag on `build:cache_from`
- `Dockerfile` now generates `libmusl` binaries for alpine
- Only pull images that can't be built
- Attribute `scale` can now accept `0` as a value
- Added `--quiet` build flag
- Added `--no-interpolate` to `docker-compose config`
- Bump OpenSSL for macOS build (`1.1.0j` to `1.1.1c`)
- Added `--no-rm` to `build` command
- Added support for `credential_spec`
- Resolve digests without pulling image
- Upgrade `pyyaml` to `4.2b1`
- Lowered severity to `warning` if `down` tries to remove nonexisting image
- Use improved API fields for project events when possible
- Update `setup.py` for modern `pypi/setuptools` and remove `pandoc` dependencies
- Removed `Dockerfile.armhf` which is no longer needed
### Bugfixes
- Make container service color deterministic, remove red from chosen colors
- Fix non ascii chars error. Python2 only
- Format image size as decimal to be align with Docker CLI
- Use Python Posix support to get tty size
- Fix same file 'extends' optimization
- Use python POSIX support to get tty size
- Format image size as decimal to be align with Docker CLI
- Fixed stdin_open
- Fixed `--remove-orphans` when used with `up --no-start`
- Fixed `docker-compose ps --all`
- Fixed `depends_on` dependency recreation behavior
- Fixed bash completion for `build --memory`
- Fixed misleading warning concerning env vars when performing an `exec` command
- Fixed failure check in parallel_execute_watch
- Fixed race condition after pulling image
- Fixed error on duplicate mount points
- Fixed merge on networks section
- Always connect Compose container to `stdin`
- Fixed the presentation of failed services on 'docker-compose start' when containers are not available
1.24.1 (2019-06-24)
-------------------
### Bugfixes
- Fixed acceptance tests
1.24.0 (2019-03-28)
-------------------

194
Jenkinsfile vendored
View File

@@ -1,112 +1,92 @@
#!groovy
def dockerVersions = ['19.03.5', '18.09.9']
def buildImage = { String baseImage ->
def image
wrappedNode(label: "ubuntu && amd64 && !zfs", cleanWorkspace: true) {
stage("build image for \"${baseImage}\"") {
checkout(scm)
def imageName = "dockerbuildbot/compose:${baseImage}-${gitCommit()}"
image = docker.image(imageName)
try {
image.pull()
} catch (Exception exc) {
sh """GIT_COMMIT=\$(script/build/write-git-sha) && \\
docker build -t ${imageName} \\
--target build \\
--build-arg BUILD_PLATFORM="${baseImage}" \\
--build-arg GIT_COMMIT="${GIT_COMMIT}" \\
.\\
"""
sh "docker push ${imageName}"
echo "${imageName}"
return imageName
}
}
}
echo "image.id: ${image.id}"
return image.id
}
def get_versions = { String imageId, int number ->
def docker_versions
wrappedNode(label: "ubuntu && amd64 && !zfs") {
def result = sh(script: """docker run --rm \\
--entrypoint=/code/.tox/py37/bin/python \\
${imageId} \\
/code/script/test/versions.py -n ${number} docker/docker-ce recent
""", returnStdout: true
)
docker_versions = result.split()
}
return docker_versions
}
def runTests = { Map settings ->
def dockerVersions = settings.get("dockerVersions", null)
def pythonVersions = settings.get("pythonVersions", null)
def baseImage = settings.get("baseImage", null)
def imageName = settings.get("image", null)
if (!pythonVersions) {
throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py37')`")
}
if (!dockerVersions) {
throw new Exception("Need Docker versions to test. e.g.: `runTests(dockerVersions: 'all')`")
}
{ ->
wrappedNode(label: "ubuntu && amd64 && !zfs", cleanWorkspace: true) {
stage("test python=${pythonVersions} / docker=${dockerVersions} / baseImage=${baseImage}") {
checkout(scm)
def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim()
echo "Using local system's storage driver: ${storageDriver}"
sh """docker run \\
-t \\
--rm \\
--privileged \\
--volume="\$(pwd)/.git:/code/.git" \\
--volume="/var/run/docker.sock:/var/run/docker.sock" \\
-e "TAG=${imageName}" \\
-e "STORAGE_DRIVER=${storageDriver}" \\
-e "DOCKER_VERSIONS=${dockerVersions}" \\
-e "BUILD_NUMBER=\$BUILD_TAG" \\
-e "PY_TEST_VERSIONS=${pythonVersions}" \\
--entrypoint="script/test/ci" \\
${imageName} \\
--verbose
"""
}
}
}
}
def testMatrix = [failFast: true]
def baseImages = ['alpine', 'debian']
def pythonVersions = ['py27', 'py37']
pipeline {
agent none
options {
skipDefaultCheckout(true)
buildDiscarder(logRotator(daysToKeepStr: '30'))
timeout(time: 2, unit: 'HOURS')
timestamps()
}
stages {
stage('Build test images') {
// TODO use declarative 1.5.0 `matrix` once available on CI
parallel {
stage('alpine') {
agent {
label 'ubuntu && amd64 && !zfs'
}
steps {
buildImage('alpine')
}
}
stage('debian') {
agent {
label 'ubuntu && amd64 && !zfs'
}
steps {
buildImage('debian')
}
}
}
}
stage('Test') {
steps {
// TODO use declarative 1.5.0 `matrix` once available on CI
script {
def testMatrix = [:]
baseImages.each { baseImage ->
dockerVersions.each { dockerVersion ->
pythonVersions.each { pythonVersion ->
testMatrix["${baseImage}_${dockerVersion}_${pythonVersion}"] = runTests(dockerVersion, pythonVersion, baseImage)
}
}
}
parallel testMatrix
}
}
}
}
baseImages.each { baseImage ->
def imageName = buildImage(baseImage)
get_versions(imageName, 2).each { dockerVersion ->
testMatrix["${baseImage}_${dockerVersion}"] = runTests([baseImage: baseImage, image: imageName, dockerVersions: dockerVersion, pythonVersions: 'py37'])
}
}
def buildImage(baseImage) {
def scmvar = checkout(scm)
def imageName = "dockerbuildbot/compose:${baseImage}-${scmvar.GIT_COMMIT}"
image = docker.image(imageName)
withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
try {
image.pull()
} catch (Exception exc) {
ansiColor('xterm') {
sh """docker build -t ${imageName} \\
--target build \\
--build-arg BUILD_PLATFORM="${baseImage}" \\
--build-arg GIT_COMMIT="${scmvar.GIT_COMMIT}" \\
.\\
"""
sh "docker push ${imageName}"
}
echo "${imageName}"
return imageName
}
}
}
def runTests(dockerVersion, pythonVersion, baseImage) {
return {
stage("python=${pythonVersion} docker=${dockerVersion} ${baseImage}") {
node("ubuntu && amd64 && !zfs") {
def scmvar = checkout(scm)
def imageName = "dockerbuildbot/compose:${baseImage}-${scmvar.GIT_COMMIT}"
def storageDriver = sh(script: "docker info -f \'{{.Driver}}\'", returnStdout: true).trim()
echo "Using local system's storage driver: ${storageDriver}"
withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
sh """docker run \\
-t \\
--rm \\
--privileged \\
--volume="\$(pwd)/.git:/code/.git" \\
--volume="/var/run/docker.sock:/var/run/docker.sock" \\
-e "TAG=${imageName}" \\
-e "STORAGE_DRIVER=${storageDriver}" \\
-e "DOCKER_VERSIONS=${dockerVersion}" \\
-e "BUILD_NUMBER=${env.BUILD_NUMBER}" \\
-e "PY_TEST_VERSIONS=${pythonVersion}" \\
--entrypoint="script/test/ci" \\
${imageName} \\
--verbose
"""
}
}
}
}
}
parallel(testMatrix)

View File

@@ -11,7 +11,6 @@
[Org]
[Org."Core maintainers"]
people = [
"ndeloof",
"rumpl",
"ulyssessouza",
]
@@ -78,11 +77,6 @@
Email = "mazz@houseofmnowster.com"
GitHub = "mnowster"
[people.ndeloof]
Name = "Nicolas De Loof"
Email = "nicolas.deloof@gmail.com"
GitHub = "ndeloof"
[people.rumpl]
Name = "Djordje Lukic"
Email = "djordje.lukic@docker.com"

View File

@@ -2,8 +2,6 @@ Docker Compose
==============
![Docker Compose](logo.png?raw=true "Docker Compose Logo")
## :exclamation: The docker-compose project announces that as Python 2 reaches it's EOL, versions 1.25.x will be the last to support it. For more information, please refer to this [issue](https://github.com/docker/compose/issues/6890).
Compose is a tool for defining and running multi-container Docker applications.
With Compose, you use a Compose file to configure your application's services.
Then, using a single command, you create and start all the services

View File

@@ -1,314 +0,0 @@
#!groovy
def dockerVersions = ['19.03.5', '18.09.9']
def baseImages = ['alpine', 'debian']
def pythonVersions = ['py37']
pipeline {
agent none
options {
skipDefaultCheckout(true)
buildDiscarder(logRotator(daysToKeepStr: '30'))
timeout(time: 2, unit: 'HOURS')
timestamps()
}
stages {
stage('Build test images') {
// TODO use declarative 1.5.0 `matrix` once available on CI
parallel {
stage('alpine') {
agent {
label 'linux'
}
steps {
buildImage('alpine')
}
}
stage('debian') {
agent {
label 'linux'
}
steps {
buildImage('debian')
}
}
}
}
stage('Test') {
steps {
// TODO use declarative 1.5.0 `matrix` once available on CI
script {
def testMatrix = [:]
baseImages.each { baseImage ->
dockerVersions.each { dockerVersion ->
pythonVersions.each { pythonVersion ->
testMatrix["${baseImage}_${dockerVersion}_${pythonVersion}"] = runTests(dockerVersion, pythonVersion, baseImage)
}
}
}
parallel testMatrix
}
}
}
stage('Generate Changelog') {
agent {
label 'linux'
}
steps {
checkout scm
withCredentials([string(credentialsId: 'github-compose-release-test-token', variable: 'GITHUB_TOKEN')]) {
sh "./script/release/generate_changelog.sh"
}
archiveArtifacts artifacts: 'CHANGELOG.md'
stash( name: "changelog", includes: 'CHANGELOG.md' )
}
}
stage('Package') {
parallel {
stage('macosx binary') {
agent {
label 'mac-python'
}
steps {
checkout scm
sh './script/setup/osx'
sh 'tox -e py37 -- tests/unit'
sh './script/build/osx'
dir ('dist') {
checksum('docker-compose-Darwin-x86_64')
checksum('docker-compose-Darwin-x86_64.tgz')
}
archiveArtifacts artifacts: 'dist/*', fingerprint: true
dir("dist") {
stash name: "bin-darwin"
}
}
}
stage('linux binary') {
agent {
label 'linux'
}
steps {
checkout scm
sh ' ./script/build/linux'
dir ('dist') {
checksum('docker-compose-Linux-x86_64')
}
archiveArtifacts artifacts: 'dist/*', fingerprint: true
dir("dist") {
stash name: "bin-linux"
}
}
}
stage('windows binary') {
agent {
label 'windows-python'
}
environment {
PATH = "$PATH;C:\\Python37;C:\\Python37\\Scripts"
}
steps {
checkout scm
bat 'tox.exe -e py37 -- tests/unit'
powershell '.\\script\\build\\windows.ps1'
dir ('dist') {
checksum('docker-compose-Windows-x86_64.exe')
}
archiveArtifacts artifacts: 'dist/*', fingerprint: true
dir("dist") {
stash name: "bin-win"
}
}
}
stage('alpine image') {
agent {
label 'linux'
}
steps {
buildRuntimeImage('alpine')
}
}
stage('debian image') {
agent {
label 'linux'
}
steps {
buildRuntimeImage('debian')
}
}
}
}
stage('Release') {
when {
buildingTag()
}
parallel {
stage('Pushing images') {
agent {
label 'linux'
}
steps {
pushRuntimeImage('alpine')
pushRuntimeImage('debian')
}
}
stage('Creating Github Release') {
agent {
label 'linux'
}
steps {
checkout scm
sh 'mkdir -p dist'
dir("dist") {
unstash "bin-darwin"
unstash "bin-linux"
unstash "bin-win"
unstash "changelog"
githubRelease()
}
}
}
stage('Publishing Python packages') {
agent {
label 'linux'
}
steps {
checkout scm
withCredentials([file(credentialsId: 'pypirc-docker-dsg-cibot', variable: 'PYPIRC')]) {
sh """
pip install --user twine wheel
python setup.py sdist bdist_wheel
twine upload --config-file ${PYPIRC} ./dist/docker-compose-${env.TAG_NAME}.tar.gz ./dist/docker_compose-${env.TAG_NAME}-py2.py3-none-any.whl
"""
}
}
}
}
}
}
}
def buildImage(baseImage) {
def scmvar = checkout(scm)
def imageName = "dockerbuildbot/compose:${baseImage}-${scmvar.GIT_COMMIT}"
image = docker.image(imageName)
withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
try {
image.pull()
} catch (Exception exc) {
ansiColor('xterm') {
sh """docker build -t ${imageName} \\
--target build \\
--build-arg BUILD_PLATFORM="${baseImage}" \\
--build-arg GIT_COMMIT="${scmvar.GIT_COMMIT}" \\
.\\
"""
sh "docker push ${imageName}"
}
echo "${imageName}"
return imageName
}
}
}
def runTests(dockerVersion, pythonVersion, baseImage) {
return {
stage("python=${pythonVersion} docker=${dockerVersion} ${baseImage}") {
node("linux") {
def scmvar = checkout(scm)
def imageName = "dockerbuildbot/compose:${baseImage}-${scmvar.GIT_COMMIT}"
def storageDriver = sh(script: "docker info -f \'{{.Driver}}\'", returnStdout: true).trim()
echo "Using local system's storage driver: ${storageDriver}"
withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
sh """docker run \\
-t \\
--rm \\
--privileged \\
--volume="\$(pwd)/.git:/code/.git" \\
--volume="/var/run/docker.sock:/var/run/docker.sock" \\
-e "TAG=${imageName}" \\
-e "STORAGE_DRIVER=${storageDriver}" \\
-e "DOCKER_VERSIONS=${dockerVersion}" \\
-e "BUILD_NUMBER=${env.BUILD_NUMBER}" \\
-e "PY_TEST_VERSIONS=${pythonVersion}" \\
--entrypoint="script/test/ci" \\
${imageName} \\
--verbose
"""
}
}
}
}
}
def buildRuntimeImage(baseImage) {
scmvar = checkout scm
def imageName = "docker/compose:${baseImage}-${env.BRANCH_NAME}"
ansiColor('xterm') {
sh """docker build -t ${imageName} \\
--build-arg BUILD_PLATFORM="${baseImage}" \\
--build-arg GIT_COMMIT="${scmvar.GIT_COMMIT.take(7)}" \\
.
"""
}
sh "mkdir -p dist"
sh "docker save ${imageName} -o dist/docker-compose-${baseImage}.tar"
stash name: "compose-${baseImage}", includes: "dist/docker-compose-${baseImage}.tar"
}
def pushRuntimeImage(baseImage) {
unstash "compose-${baseImage}"
sh "docker load -i dist/docker-compose-${baseImage}.tar"
withDockerRegistry(credentialsId: 'dockerbuildbot-hub.docker.com') {
sh "docker push docker/compose:${baseImage}-${env.TAG_NAME}"
if (baseImage == "alpine" && env.TAG_NAME != null) {
sh "docker tag docker/compose:alpine-${env.TAG_NAME} docker/compose:${env.TAG_NAME}"
sh "docker push docker/compose:${env.TAG_NAME}"
}
}
}
def githubRelease() {
withCredentials([string(credentialsId: 'github-compose-release-test-token', variable: 'GITHUB_TOKEN')]) {
def prerelease = !( env.TAG_NAME ==~ /v[0-9\.]+/ )
changelog = readFile "CHANGELOG.md"
def info = [:]
info.tag_name = env.TAG_NAME
info.name = env.TAG_NAME
info.draft = true
info.prerelease = prerelease
info.body = changelog
writeFile file: 'release.json', text: groovy.json.JsonOutput.toJson(info)
// debug
sh("cat release.json")
def url = "https://api.github.com/repos/docker/compose/releases"
def upload_url = sh(returnStdout: true, script: """
curl -sSf -H 'Authorization: token ${GITHUB_TOKEN}' -H 'Accept: application/json' -H 'Content-type: application/json' -X POST --data-binary '@release.json' $url) \\
| jq '.upload_url | .[:rindex("{")]'
""")
sh("""
for f in * ; do
curl -sf -H 'Authorization: token ${GITHUB_TOKEN}' -H 'Accept: application/json' -H 'Content-type: application/octet-stream' \\
-X POST --data-binary @\$f ${upload_url}?name=\$f;
done
""")
}
}
def checksum(filepath) {
if (isUnix()) {
sh "openssl sha256 -r -out ${filepath}.sha256 ${filepath}"
} else {
powershell "(Get-FileHash -Path ${filepath} -Algorithm SHA256 | % hash) + ' *${filepath}' > ${filepath}.sha256"
}
}

24
appveyor.yml Normal file
View File

@@ -0,0 +1,24 @@
version: '{branch}-{build}'
install:
- "SET PATH=C:\\Python37-x64;C:\\Python37-x64\\Scripts;%PATH%"
- "python --version"
- "pip install tox==2.9.1 virtualenv==16.2.0"
# Build the binary after tests
build: false
test_script:
- "tox -e py27,py37 -- tests/unit"
- ps: ".\\script\\build\\windows.ps1"
artifacts:
- path: .\dist\docker-compose-Windows-x86_64.exe
name: "Compose Windows binary"
deploy:
- provider: Environment
name: master-builds
on:
branch: master

View File

@@ -1,4 +1,4 @@
from __future__ import absolute_import
from __future__ import unicode_literals
__version__ = '1.26.0dev'
__version__ = '1.25.0dev'

View File

@@ -159,25 +159,15 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False,
def execution_context_labels(config_details, environment_file):
extra_labels = [
'{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir))
'{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir)),
'{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details)),
]
if not use_config_from_stdin(config_details):
extra_labels.append('{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details)))
if environment_file is not None:
extra_labels.append('{0}={1}'.format(LABEL_ENVIRONMENT_FILE,
os.path.normpath(environment_file)))
return extra_labels
def use_config_from_stdin(config_details):
for c in config_details.config_files:
if not c.filename:
return True
return False
def config_files_label(config_details):
return ",".join(
map(str, (os.path.normpath(c.filename) for c in config_details.config_files)))

View File

@@ -17,12 +17,7 @@ else:
def get_tty_width():
try:
# get_terminal_size can't determine the size if compose is piped
# to another command. But in such case it doesn't make sense to
# try format the output by terminal size as this output is consumed
# by another command. So let's pretend we have a huge terminal so
# output is single-lined
width, _ = get_terminal_size(fallback=(999, 0))
width, _ = get_terminal_size()
return int(width)
except OSError:
return 0

View File

@@ -5,7 +5,6 @@ import functools
import io
import logging
import os
import re
import string
import sys
from collections import namedtuple
@@ -215,12 +214,6 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
.format(self.filename, VERSION_EXPLANATION)
)
version_pattern = re.compile(r"^[2-9]+(\.\d+)?$")
if not version_pattern.match(version):
raise ConfigurationError(
'Version "{}" in "{}" is invalid.'
.format(version, self.filename))
if version == '2':
return const.COMPOSEFILE_V2_0

View File

@@ -114,13 +114,3 @@ def get_digest_from_push(events):
if digest:
return digest
return None
def read_status(event):
status = event['status'].lower()
if 'progressDetail' in event:
detail = event['progressDetail']
if 'current' in detail and 'total' in detail:
percentage = float(detail['current']) / float(detail['total'])
status = '{} ({:.1%})'.format(status, percentage)
return status

View File

@@ -11,8 +11,6 @@ from os import path
import enum
import six
from docker.errors import APIError
from docker.errors import ImageNotFound
from docker.errors import NotFound
from docker.utils import version_lt
from . import parallel
@@ -27,7 +25,6 @@ from .container import Container
from .network import build_networks
from .network import get_networks
from .network import ProjectNetworks
from .progress_stream import read_status
from .service import BuildAction
from .service import ContainerNetworkMode
from .service import ContainerPidMode
@@ -622,68 +619,49 @@ class Project(object):
def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False,
include_deps=False):
services = self.get_services(service_names, include_deps)
images_to_build = {service.image_name for service in services if service.can_be_built()}
services_to_pull = [service for service in services if service.image_name not in images_to_build]
msg = not silent and 'Pulling' or None
if parallel_pull:
self.parallel_pull(services, silent=silent)
def pull_service(service):
strm = service.pull(ignore_pull_failures, True, stream=True)
if strm is None: # Attempting to pull service with no `image` key is a no-op
return
else:
must_build = []
for service in services:
try:
service.pull(ignore_pull_failures, silent=silent)
except (ImageNotFound, NotFound):
if service.can_be_built():
must_build.append(service.name)
else:
raise
if len(must_build):
log.warning('Some service image(s) must be built from source by running:\n'
' docker-compose build {}'
.format(' '.join(must_build)))
def parallel_pull(self, services, ignore_pull_failures=False, silent=False):
msg = 'Pulling' if not silent else None
must_build = []
def pull_service(service):
strm = service.pull(ignore_pull_failures, True, stream=True)
if strm is None: # Attempting to pull service with no `image` key is a no-op
return
try:
writer = parallel.get_stream_writer()
for event in strm:
if 'status' not in event:
continue
status = read_status(event)
status = event['status'].lower()
if 'progressDetail' in event:
detail = event['progressDetail']
if 'current' in detail and 'total' in detail:
percentage = float(detail['current']) / float(detail['total'])
status = '{} ({:.1%})'.format(status, percentage)
writer.write(
msg, service.name, truncate_string(status), lambda s: s
)
except (ImageNotFound, NotFound):
if service.can_be_built():
must_build.append(service.name)
else:
raise
_, errors = parallel.parallel_execute(
services,
pull_service,
operator.attrgetter('name'),
msg,
limit=5,
)
_, errors = parallel.parallel_execute(
services_to_pull,
pull_service,
operator.attrgetter('name'),
msg,
limit=5,
)
if len(errors):
combined_errors = '\n'.join([
e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
])
raise ProjectError(combined_errors)
if len(must_build):
log.warning('Some service image(s) must be built from source by running:\n'
' docker-compose build {}'
.format(' '.join(must_build)))
if len(errors):
combined_errors = '\n'.join([
e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
])
raise ProjectError(combined_errors)
else:
for service in services_to_pull:
service.pull(ignore_pull_failures, silent=silent)
def push(self, service_names=None, ignore_push_failures=False):
unique_images = set()

View File

@@ -60,7 +60,6 @@ from .utils import parse_bytes
from .utils import parse_seconds_float
from .utils import truncate_id
from .utils import unique_everseen
from compose.cli.utils import binarystr_to_unicode
if six.PY2:
import subprocess32 as subprocess
@@ -344,7 +343,7 @@ class Service(object):
return Container.create(self.client, **container_options)
except APIError as ex:
raise OperationFailedError("Cannot create container for service %s: %s" %
(self.name, binarystr_to_unicode(ex.explanation)))
(self.name, ex.explanation))
def ensure_image_exists(self, do_build=BuildAction.none, silent=False, cli=False):
if self.can_be_built() and do_build == BuildAction.force:
@@ -625,10 +624,9 @@ class Service(object):
try:
container.start()
except APIError as ex:
expl = binarystr_to_unicode(ex.explanation)
if "driver failed programming external connectivity" in expl:
if "driver failed programming external connectivity" in ex.explanation:
log.warn("Host is already in use by another container")
raise OperationFailedError("Cannot start service %s: %s" % (self.name, expl))
raise OperationFailedError("Cannot start service %s: %s" % (self.name, ex.explanation))
return container
@property

View File

@@ -1,108 +0,0 @@
# -*- mode: python ; coding: utf-8 -*-
block_cipher = None
a = Analysis(['bin/docker-compose'],
pathex=['.'],
hiddenimports=[],
hookspath=[],
runtime_hooks=[],
cipher=block_cipher)
pyz = PYZ(a.pure, a.zipped_data,
cipher=block_cipher)
exe = EXE(pyz,
a.scripts,
exclude_binaries=True,
name='docker-compose',
debug=False,
strip=False,
upx=True,
console=True,
bootloader_ignore_signals=True)
coll = COLLECT(exe,
a.binaries,
a.zipfiles,
a.datas,
[
(
'compose/config/config_schema_v1.json',
'compose/config/config_schema_v1.json',
'DATA'
),
(
'compose/config/config_schema_v2.0.json',
'compose/config/config_schema_v2.0.json',
'DATA'
),
(
'compose/config/config_schema_v2.1.json',
'compose/config/config_schema_v2.1.json',
'DATA'
),
(
'compose/config/config_schema_v2.2.json',
'compose/config/config_schema_v2.2.json',
'DATA'
),
(
'compose/config/config_schema_v2.3.json',
'compose/config/config_schema_v2.3.json',
'DATA'
),
(
'compose/config/config_schema_v2.4.json',
'compose/config/config_schema_v2.4.json',
'DATA'
),
(
'compose/config/config_schema_v3.0.json',
'compose/config/config_schema_v3.0.json',
'DATA'
),
(
'compose/config/config_schema_v3.1.json',
'compose/config/config_schema_v3.1.json',
'DATA'
),
(
'compose/config/config_schema_v3.2.json',
'compose/config/config_schema_v3.2.json',
'DATA'
),
(
'compose/config/config_schema_v3.3.json',
'compose/config/config_schema_v3.3.json',
'DATA'
),
(
'compose/config/config_schema_v3.4.json',
'compose/config/config_schema_v3.4.json',
'DATA'
),
(
'compose/config/config_schema_v3.5.json',
'compose/config/config_schema_v3.5.json',
'DATA'
),
(
'compose/config/config_schema_v3.6.json',
'compose/config/config_schema_v3.6.json',
'DATA'
),
(
'compose/config/config_schema_v3.7.json',
'compose/config/config_schema_v3.7.json',
'DATA'
),
(
'compose/GITSHA',
'compose/GITSHA',
'DATA'
)
],
strip=False,
upx=True,
upx_exclude=[],
name='docker-compose-Darwin-x86_64')

View File

@@ -1,6 +1,6 @@
coverage==4.5.4
coverage==4.4.2
ddt==1.2.0
flake8==3.7.9
flake8==3.5.0
mock==3.0.5
pytest==3.6.3
pytest-cov==2.8.1
pytest-cov==2.5.1

View File

@@ -20,7 +20,6 @@ PySocks==1.6.7
PyYAML==4.2b1
requests==2.22.0
six==1.12.0
subprocess32==3.5.4; python_version < '3.2'
texttable==1.6.2
urllib3==1.24.2; python_version == '3.3'
websocket-client==0.32.0

20
script/Jenkinsfile.fossa Normal file
View File

@@ -0,0 +1,20 @@
pipeline {
agent any
stages {
stage("License Scan") {
agent {
label 'ubuntu-1604-aufs-edge'
}
steps {
withCredentials([
string(credentialsId: 'fossa-api-key', variable: 'FOSSA_API_KEY')
]) {
checkout scm
sh "FOSSA_API_KEY='${FOSSA_API_KEY}' BRANCH_NAME='${env.BRANCH_NAME}' make -f script/fossa.mk fossa-analyze"
sh "FOSSA_API_KEY='${FOSSA_API_KEY}' make -f script/fossa.mk fossa-test"
}
}
}
}
}

View File

@@ -11,14 +11,6 @@ venv/bin/pip install -r requirements-build.txt
venv/bin/pip install --no-deps .
DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
# Build as a folder for macOS Catalina.
venv/bin/pyinstaller docker-compose_darwin.spec
dist/docker-compose-Darwin-x86_64/docker-compose version
(cd dist/docker-compose-Darwin-x86_64/ && tar zcvf ../docker-compose-Darwin-x86_64.tgz .)
rm -rf dist/docker-compose-Darwin-x86_64
# Build static binary for legacy.
venv/bin/pyinstaller docker-compose.spec
mv dist/docker-compose dist/docker-compose-Darwin-x86_64
dist/docker-compose-Darwin-x86_64 version

View File

@@ -25,11 +25,3 @@ curl -f -T dist/docker-compose-${OS_NAME}-x86_64 -u$BINTRAY_USERNAME:$BINTRAY_AP
-H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \
-H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \
https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64 || exit 1
# Upload folder format of docker-compose for macOS in addition to binary.
if [ "${OS_NAME}" == "Darwin" ]; then
curl -f -T dist/docker-compose-${OS_NAME}-x86_64.tgz -u$BINTRAY_USERNAME:$BINTRAY_API_KEY \
-H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \
-H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \
https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64.tgz || exit 1
fi

16
script/fossa.mk Normal file
View File

@@ -0,0 +1,16 @@
# Variables for Fossa
BUILD_ANALYZER?=docker/fossa-analyzer
FOSSA_OPTS?=--option all-tags:true --option allow-unresolved:true
fossa-analyze:
docker run --rm -e FOSSA_API_KEY=$(FOSSA_API_KEY) \
-v $(CURDIR)/$*:/go/src/github.com/docker/compose \
-w /go/src/github.com/docker/compose \
$(BUILD_ANALYZER) analyze ${FOSSA_OPTS} --branch ${BRANCH_NAME}
# This command is used to run the fossa test command
fossa-test:
docker run -i -e FOSSA_API_KEY=$(FOSSA_API_KEY) \
-v $(CURDIR)/$*:/go/src/github.com/docker/compose \
-w /go/src/github.com/docker/compose \
$(BUILD_ANALYZER) test

View File

@@ -1,39 +0,0 @@
#!/bin/bash
set -e
set -x
## Usage :
## changelog PREVIOUS_TAG..HEAD
# configure refs so we get pull-requests metadata
git config --add remote.origin.fetch +refs/pull/*/head:refs/remotes/origin/pull/*
git fetch origin
RANGE=${1:-"$(git describe --tags --abbrev=0)..HEAD"}
echo "Generate changelog for range ${RANGE}"
echo
pullrequests() {
for commit in $(git log ${RANGE} --format='format:%H'); do
# Get the oldest remotes/origin/pull/* branch to include this commit, i.e. the one to introduce it
git branch -a --sort=committerdate --contains $commit --list 'origin/pull/*' | head -1 | cut -d'/' -f4
done
}
changes=$(pullrequests | uniq)
echo "pull requests merged within range:"
echo $changes
echo '#Features' > CHANGELOG.md
for pr in $changes; do
curl -fs -H "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/docker/compose/pulls/${pr} \
| jq -r ' select( .labels[].name | contains("kind/feature") ) | "* "+.title' >> CHANGELOG.md
done
echo '#Bugs' >> CHANGELOG.md
for pr in $changes; do
curl -fs -H "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/docker/compose/pulls/${pr} \
| jq -r ' select( .labels[].name | contains("kind/bug") ) | "* "+.title' >> CHANGELOG.md
done

View File

@@ -204,8 +204,7 @@ def resume(args):
gh_release = create_release_draft(repository, args.release, pr_data, files)
delete_assets(gh_release)
upload_assets(gh_release, files)
tag_as_latest = is_tag_latest(args.release)
img_manager = ImageManager(args.release, tag_as_latest)
img_manager = ImageManager(args.release)
img_manager.build_images(repository)
except ScriptError as e:
print(e)
@@ -245,8 +244,7 @@ def start(args):
files = downloader.download_all(args.release)
gh_release = create_release_draft(repository, args.release, pr_data, files)
upload_assets(gh_release, files)
tag_as_latest = is_tag_latest(args.release)
img_manager = ImageManager(args.release, tag_as_latest)
img_manager = ImageManager(args.release)
img_manager.build_images(repository)
except ScriptError as e:
print(e)

View File

@@ -55,7 +55,6 @@ class BinaryDownloader(requests.Session):
def download_all(self, version):
files = {
'docker-compose-Darwin-x86_64.tgz': None,
'docker-compose-Darwin-x86_64': None,
'docker-compose-Linux-x86_64': None,
'docker-compose-Windows-x86_64.exe': None,

View File

@@ -15,7 +15,7 @@
set -e
VERSION="1.25.1"
VERSION="1.24.0"
IMAGE="docker/compose:$VERSION"
@@ -36,18 +36,18 @@ if [ "$(pwd)" != '/' ]; then
fi
if [ -n "$COMPOSE_FILE" ]; then
COMPOSE_OPTIONS="$COMPOSE_OPTIONS -e COMPOSE_FILE=$COMPOSE_FILE"
compose_dir=$(realpath "$(dirname "$COMPOSE_FILE")")
compose_dir=$(realpath $(dirname $COMPOSE_FILE))
fi
# TODO: also check --file argument
if [ -n "$compose_dir" ]; then
VOLUMES="$VOLUMES -v $compose_dir:$compose_dir"
fi
if [ -n "$HOME" ]; then
VOLUMES="$VOLUMES -v $HOME:$HOME -e HOME" # Pass in HOME to share docker.config and allow ~/-relative paths to work.
VOLUMES="$VOLUMES -v $HOME:$HOME -v $HOME:/root" # mount $HOME in /root to share docker.config
fi
# Only allocate tty if we detect one
if [ -t 0 ] && [ -t 1 ]; then
if [ -t 0 -a -t 1 ]; then
DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t"
fi
@@ -56,9 +56,8 @@ DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i"
# Handle userns security
if docker info --format '{{json .SecurityOptions}}' 2>/dev/null | grep -q 'name=userns'; then
if [ ! -z "$(docker info 2>/dev/null | grep userns)" ]; then
DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS --userns=host"
fi
# shellcheck disable=SC2086
exec docker run --rm $DOCKER_RUN_OPTIONS $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w "$(pwd)" $IMAGE "$@"

View File

@@ -36,7 +36,7 @@ if ! [ -x "$(command -v python3)" ]; then
brew install python3
fi
if ! [ -x "$(command -v virtualenv)" ]; then
pip3 install virtualenv==16.2.0
pip install virtualenv==16.2.0
fi
#

View File

@@ -48,7 +48,6 @@ BUILD_PULL_TEXT = 'Status: Image is up to date for busybox:1.27.2'
def start_process(base_dir, options):
proc = subprocess.Popen(
['docker-compose'] + options,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=base_dir)
@@ -56,8 +55,8 @@ def start_process(base_dir, options):
return proc
def wait_on_process(proc, returncode=0, stdin=None):
stdout, stderr = proc.communicate(input=stdin)
def wait_on_process(proc, returncode=0):
stdout, stderr = proc.communicate()
if proc.returncode != returncode:
print("Stderr: {}".format(stderr))
print("Stdout: {}".format(stdout))
@@ -65,10 +64,10 @@ def wait_on_process(proc, returncode=0, stdin=None):
return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8'))
def dispatch(base_dir, options, project_options=None, returncode=0, stdin=None):
def dispatch(base_dir, options, project_options=None, returncode=0):
project_options = project_options or []
proc = start_process(base_dir, project_options + options)
return wait_on_process(proc, returncode=returncode, stdin=stdin)
return wait_on_process(proc, returncode=returncode)
def wait_on_condition(condition, delay=0.1, timeout=40):
@@ -157,8 +156,8 @@ class CLITestCase(DockerClientTestCase):
self._project = get_project(self.base_dir, override_dir=self.override_dir)
return self._project
def dispatch(self, options, project_options=None, returncode=0, stdin=None):
return dispatch(self.base_dir, options, project_options, returncode, stdin)
def dispatch(self, options, project_options=None, returncode=0):
return dispatch(self.base_dir, options, project_options, returncode)
def execute(self, container, cmd):
# Remove once Hijack and CloseNotifier sign a peace treaty
@@ -242,17 +241,6 @@ class CLITestCase(DockerClientTestCase):
self.base_dir = 'tests/fixtures/v2-full'
assert self.dispatch(['config', '--quiet']).stdout == ''
def test_config_stdin(self):
config = b"""version: "3.7"
services:
web:
image: nginx
other:
image: alpine
"""
result = self.dispatch(['-f', '-', 'config', '--services'], stdin=config)
assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'}
def test_config_with_hash_option(self):
self.base_dir = 'tests/fixtures/v2-full'
result = self.dispatch(['config', '--hash=*'])
@@ -673,6 +661,13 @@ services:
'image library/nonexisting-image:latest not found' in result.stderr or
'pull access denied for nonexisting-image' in result.stderr)
def test_pull_with_build(self):
result = self.dispatch(['-f', 'pull-with-build.yml', 'pull'])
assert 'Pulling simple' not in result.stderr
assert 'Pulling from_simple' not in result.stderr
assert 'Pulling another ...' in result.stderr
def test_pull_with_quiet(self):
assert self.dispatch(['pull', '--quiet']).stderr == ''
assert self.dispatch(['pull', '--quiet']).stdout == ''
@@ -694,14 +689,6 @@ services:
result.stderr
)
def test_pull_can_build(self):
result = self.dispatch([
'-f', 'can-build-pull-failures.yml', 'pull'],
returncode=0
)
assert 'Some service image(s) must be built from source' in result.stderr
assert 'docker-compose build can_build' in result.stderr
def test_pull_with_no_deps(self):
self.base_dir = 'tests/fixtures/links-composefile'
result = self.dispatch(['pull', '--no-parallel', 'web'])

View File

@@ -1,6 +0,0 @@
version: '3'
services:
can_build:
image: nonexisting-image-but-can-build:latest
build: .
command: top

View File

@@ -13,8 +13,6 @@ from random import shuffle
import py
import pytest
import yaml
from ddt import data
from ddt import ddt
from ...helpers import build_config_details
from ...helpers import BUSYBOX_IMAGE_WITH_TAG
@@ -70,7 +68,6 @@ def secret_sort(secrets):
return sorted(secrets, key=itemgetter('source'))
@ddt
class ConfigTest(unittest.TestCase):
def test_load(self):
@@ -1888,26 +1885,6 @@ class ConfigTest(unittest.TestCase):
}
]
@data(
'2 ',
'3.',
'3.0.0',
'3.0.a',
'3.a',
'3a')
def test_invalid_version_formats(self, version):
content = {
'version': version,
'services': {
'web': {
'image': 'alpine',
}
}
}
with pytest.raises(ConfigurationError) as exc:
config.load(build_config_details(content))
assert 'Version "{}" in "filename.yml" is invalid.'.format(version) in exc.exconly()
def test_group_add_option(self):
actual = config.load(build_config_details({
'version': '2',

View File

@@ -521,37 +521,7 @@ class ServiceTest(unittest.TestCase):
assert 'was built because it did not already exist' in args[0]
assert self.mock_client.build.call_count == 1
assert self.mock_client.build.call_args[1]['tag'] == 'default_foo'
def test_create_container_binary_string_error(self):
service = Service('foo', client=self.mock_client, build={'context': '.'})
service.image = lambda: {'Id': 'abc123'}
self.mock_client.create_container.side_effect = APIError(None,
None,
b"Test binary string explanation")
with pytest.raises(OperationFailedError) as ex:
service.create_container()
assert ex.value.msg == "Cannot create container for service foo: Test binary string explanation"
def test_start_binary_string_error(self):
service = Service('foo', client=self.mock_client)
container = Container(self.mock_client, {'Id': 'abc123'})
self.mock_client.start.side_effect = APIError(None,
None,
b"Test binary string explanation with "
b"driver failed programming external "
b"connectivity")
with mock.patch('compose.service.log', autospec=True) as mock_log:
with pytest.raises(OperationFailedError) as ex:
service.start_container(container)
assert ex.value.msg == "Cannot start service foo: " \
"Test binary string explanation " \
"with driver failed programming external connectivity"
mock_log.warn.assert_called_once_with("Host is already in use by another container")
self.mock_client.build.call_args[1]['tag'] == 'default_foo'
def test_ensure_image_exists_no_build(self):
service = Service('foo', client=self.mock_client, build={'context': '.'})