Bump 1.14.0-rc2

Signed-off-by: Joffrey F <joffrey@docker.com>

CHANGELOG.md

@@ -10,8 +10,8 @@ Change log
 
 - Introduced version 3.3 of the `docker-compose.yml` specification.
   This version requires to be used with Docker Engine 17.06.0 or above.
-  Note: the `credential_spec` key only applies to Swarm services and will
-  be ignored by Compose
+  Note: the `credential_spec` and `configs` keys only apply to Swarm services
+  and will be ignored by Compose
 
 #### Compose file version 2.2
 
@@ -50,6 +50,9 @@ Change log
 - Fixed a bug where services declaring ports would cause crashes on some
   versions of Python 3
 
+- Fixed a bug where the output of `docker-compose config` would sometimes
+  contain invalid port definitions
+
 1.13.0 (2017-05-02)
 -------------------
 

compose/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-__version__ = '1.14.0-rc1'
+__version__ = '1.14.0-rc2'

compose/config/config.py

@@ -211,8 +211,11 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
     def get_secrets(self):
         return {} if self.version < const.COMPOSEFILE_V3_1 else self.config.get('secrets', {})
 
+    def get_configs(self):
+        return {} if self.version < const.COMPOSEFILE_V3_3 else self.config.get('configs', {})
 
-class Config(namedtuple('_Config', 'version services volumes networks secrets')):
+
+class Config(namedtuple('_Config', 'version services volumes networks secrets configs')):
     """
     :param version: configuration version
     :type  version: int
@@ -224,6 +227,8 @@ class Config(namedtuple('_Config', 'version services volumes networks secrets'))
     :type  networks: :class:`dict`
     :param secrets: Dictionary mapping secret names to description dictionaries
     :type secrets: :class:`dict`
+    :param configs: Dictionary mapping config names to description dictionaries
+    :type configs: :class:`dict`
     """
 
 
@@ -340,6 +345,7 @@ def check_swarm_only_config(service_dicts):
 
     check_swarm_only_key(service_dicts, 'deploy')
     check_swarm_only_key(service_dicts, 'credential_spec')
+    check_swarm_only_key(service_dicts, 'configs')
 
 
 def load(config_details):
@@ -364,7 +370,12 @@ def load(config_details):
     networks = load_mapping(
         config_details.config_files, 'get_networks', 'Network'
     )
-    secrets = load_secrets(config_details.config_files, config_details.working_dir)
+    secrets = load_mapping(
+        config_details.config_files, 'get_secrets', 'Secret', config_details.working_dir
+    )
+    configs = load_mapping(
+        config_details.config_files, 'get_configs', 'Config', config_details.working_dir
+    )
     service_dicts = load_services(config_details, main_file)
 
     if main_file.version != V1:
@@ -373,10 +384,10 @@ def load(config_details):
 
     check_swarm_only_config(service_dicts)
 
-    return Config(main_file.version, service_dicts, volumes, networks, secrets)
+    return Config(main_file.version, service_dicts, volumes, networks, secrets, configs)
 
 
-def load_mapping(config_files, get_func, entity_type):
+def load_mapping(config_files, get_func, entity_type, working_dir=None):
     mapping = {}
 
     for config_file in config_files:
@@ -401,6 +412,9 @@ def load_mapping(config_files, get_func, entity_type):
             if 'labels' in config:
                 config['labels'] = parse_labels(config['labels'])
 
+            if 'file' in config:
+                config['file'] = expand_path(working_dir, config['file'])
+
     return mapping
 
 
@@ -414,29 +428,6 @@ def validate_external(entity_type, name, config):
             entity_type, name, ', '.join(k for k in config if k != 'external')))
 
 
-def load_secrets(config_files, working_dir):
-    mapping = {}
-
-    for config_file in config_files:
-        for name, config in config_file.get_secrets().items():
-            mapping[name] = config or {}
-            if not config:
-                continue
-
-            external = config.get('external')
-            if external:
-                validate_external('Secret', name, config)
-                if isinstance(external, dict):
-                    config['external_name'] = external.get('name')
-                else:
-                    config['external_name'] = name
-
-            if 'file' in config:
-                config['file'] = expand_path(working_dir, config['file'])
-
-    return mapping
-
-
 def load_services(config_details, config_file):
     def build_service(service_name, service_dict, service_names):
         service_config = ServiceConfig.with_abs_paths(
@@ -516,12 +507,20 @@ def process_config_file(config_file, environment, service_name=None):
             config_file.get_networks(),
             'network',
             environment)
-        if config_file.version in (const.COMPOSEFILE_V3_1, const.COMPOSEFILE_V3_2):
+        if config_file.version in (const.COMPOSEFILE_V3_1, const.COMPOSEFILE_V3_2,
+                                   const.COMPOSEFILE_V3_3):
             processed_config['secrets'] = interpolate_config_section(
                 config_file,
                 config_file.get_secrets(),
                 'secrets',
                 environment)
+        if config_file.version in (const.COMPOSEFILE_V3_3):
+            processed_config['configs'] = interpolate_config_section(
+                config_file,
+                config_file.get_configs(),
+                'configs',
+                environment
+            )
     else:
         processed_config = services
 
@@ -815,6 +814,11 @@ def finalize_service(service_config, service_names, version, environment):
             types.ServiceSecret.parse(s) for s in service_dict['secrets']
         ]
 
+    if 'configs' in service_dict:
+        service_dict['configs'] = [
+            types.ServiceConfig.parse(c) for c in service_dict['configs']
+        ]
+
     normalize_build(service_dict, service_config.working_dir, environment)
 
     service_dict['name'] = service_config.name
@@ -906,6 +910,7 @@ def merge_service_dicts(base, override, version):
     md.merge_mapping('depends_on', parse_depends_on)
     md.merge_sequence('links', ServiceLink.parse)
     md.merge_sequence('secrets', types.ServiceSecret.parse)
+    md.merge_sequence('configs', types.ServiceConfig.parse)
     md.merge_mapping('deploy', parse_deploy)
 
     for field in ['volumes', 'devices']:

compose/config/serialize.py

@@ -8,7 +8,6 @@ from compose.config import types
 from compose.const import COMPOSEFILE_V1 as V1
 from compose.const import COMPOSEFILE_V2_1 as V2_1
 from compose.const import COMPOSEFILE_V2_2 as V2_2
-from compose.const import COMPOSEFILE_V3_1 as V3_1
 from compose.const import COMPOSEFILE_V3_2 as V3_2
 from compose.const import COMPOSEFILE_V3_3 as V3_3
 
@@ -25,6 +24,7 @@ def serialize_dict_type(dumper, data):
 yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type)
 yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type)
 yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type)
+yaml.SafeDumper.add_representer(types.ServiceConfig, serialize_dict_type)
 yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type)
 
 
@@ -41,21 +41,15 @@ def denormalize_config(config, image_digests=None):
         service_dict.pop('name'): service_dict
         for service_dict in denormalized_services
     }
-    result['networks'] = config.networks.copy()
-    for net_name, net_conf in result['networks'].items():
-        if 'external_name' in net_conf:
-            del net_conf['external_name']
+    for key in ('networks', 'volumes', 'secrets', 'configs'):
+        config_dict = getattr(config, key)
+        if not config_dict:
+            continue
+        result[key] = config_dict.copy()
+        for name, conf in result[key].items():
+            if 'external_name' in conf:
+                del conf['external_name']
 
-    result['volumes'] = config.volumes.copy()
-    for vol_name, vol_conf in result['volumes'].items():
-        if 'external_name' in vol_conf:
-            del vol_conf['external_name']
-
-    if config.version in (V3_1, V3_2, V3_3):
-        result['secrets'] = config.secrets.copy()
-        for secret_name, secret_conf in result['secrets'].items():
-            if 'external_name' in secret_conf:
-                del secret_conf['external_name']
     return result
 
 

compose/config/types.py

@@ -238,8 +238,7 @@ class ServiceLink(namedtuple('_ServiceLink', 'target alias')):
         return self.alias
 
 
-class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')):
-
+class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode')):
     @classmethod
     def parse(cls, spec):
         if isinstance(spec, six.string_types):
@@ -262,7 +261,31 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')):
         )
 
 
+class ServiceSecret(ServiceConfigBase):
+    pass
+
+
+class ServiceConfig(ServiceConfigBase):
+    pass
+
+
 class ServicePort(namedtuple('_ServicePort', 'target published protocol mode external_ip')):
+    def __new__(cls, target, published, *args, **kwargs):
+        try:
+            if target:
+                target = int(target)
+        except ValueError:
+            raise ConfigurationError('Invalid target port: {}'.format(target))
+
+        try:
+            if published:
+                published = int(published)
+        except ValueError:
+            raise ConfigurationError('Invalid published port: {}'.format(published))
+
+        return super(ServicePort, cls).__new__(
+            cls, target, published, *args, **kwargs
+        )
 
     @classmethod
     def parse(cls, spec):

requirements.txt

@@ -2,7 +2,7 @@ PyYAML==3.11
 backports.ssl-match-hostname==3.5.0.1; python_version < '3'
 cached-property==1.2.0
 colorama==0.3.7
-docker==2.2.1
+docker==2.3.0
 dockerpty==0.4.1
 docopt==0.6.1
 enum34==1.0.4; python_version < '3.4'

script/run/run.sh

@@ -15,7 +15,7 @@
 
 set -e
 
-VERSION="1.14.0-rc1"
+VERSION="1.14.0-rc2"
 IMAGE="docker/compose:$VERSION"
 
 

tests/acceptance/cli_test.py

@@ -258,8 +258,6 @@ class CLITestCase(DockerClientTestCase):
                     'restart': ''
                 },
             },
-            'networks': {},
-            'volumes': {},
         }
 
     def test_config_external_network(self):
@@ -311,8 +309,6 @@ class CLITestCase(DockerClientTestCase):
                     'network_mode': 'service:net',
                 },
             },
-            'networks': {},
-            'volumes': {},
         }
 
     @v3_only()
@@ -322,8 +318,6 @@ class CLITestCase(DockerClientTestCase):
 
         assert yaml.load(result.stdout) == {
             'version': '3.2',
-            'networks': {},
-            'secrets': {},
             'volumes': {
                 'foobar': {
                     'labels': {

tests/integration/project_test.py

@@ -40,7 +40,9 @@ def build_config(**kwargs):
         services=kwargs.get('services'),
         volumes=kwargs.get('volumes'),
         networks=kwargs.get('networks'),
-        secrets=kwargs.get('secrets'))
+        secrets=kwargs.get('secrets'),
+        configs=kwargs.get('configs'),
+    )
 
 
 class ProjectTest(DockerClientTestCase):

tests/unit/bundle_test.py

@@ -78,7 +78,9 @@ def test_to_bundle():
         services=services,
         volumes={'special': {}},
         networks={'extra': {}},
-        secrets={})
+        secrets={},
+        configs={}
+    )
 
     with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log:
         output = bundle.to_bundle(config, image_digests)

tests/unit/config/config_test.py

@@ -1982,6 +1982,38 @@ class ConfigTest(unittest.TestCase):
         actual = config.merge_service_dicts(base, override, V3_1)
         assert actual['secrets'] == override['secrets']
 
+    def test_merge_different_configs(self):
+        base = {
+            'image': 'busybox',
+            'configs': [
+                {'source': 'src.txt'}
+            ]
+        }
+        override = {'configs': ['other-src.txt']}
+
+        actual = config.merge_service_dicts(base, override, V3_3)
+        assert secret_sort(actual['configs']) == secret_sort([
+            {'source': 'src.txt'},
+            {'source': 'other-src.txt'}
+        ])
+
+    def test_merge_configs_override(self):
+        base = {
+            'image': 'busybox',
+            'configs': ['src.txt'],
+        }
+        override = {
+            'configs': [
+                {
+                    'source': 'src.txt',
+                    'target': 'data.txt',
+                    'mode': 0o400
+                }
+            ]
+        }
+        actual = config.merge_service_dicts(base, override, V3_3)
+        assert actual['configs'] == override['configs']
+
     def test_merge_deploy(self):
         base = {
             'image': 'busybox',
@@ -2214,6 +2246,91 @@ class ConfigTest(unittest.TestCase):
         ]
         assert service_sort(service_dicts) == service_sort(expected)
 
+    def test_load_configs(self):
+        base_file = config.ConfigFile(
+            'base.yaml',
+            {
+                'version': '3.3',
+                'services': {
+                    'web': {
+                        'image': 'example/web',
+                        'configs': [
+                            'one',
+                            {
+                                'source': 'source',
+                                'target': 'target',
+                                'uid': '100',
+                                'gid': '200',
+                                'mode': 0o777,
+                            },
+                        ],
+                    },
+                },
+                'configs': {
+                    'one': {'file': 'secret.txt'},
+                },
+            })
+        details = config.ConfigDetails('.', [base_file])
+        service_dicts = config.load(details).services
+        expected = [
+            {
+                'name': 'web',
+                'image': 'example/web',
+                'configs': [
+                    types.ServiceConfig('one', None, None, None, None),
+                    types.ServiceConfig('source', 'target', '100', '200', 0o777),
+                ],
+            },
+        ]
+        assert service_sort(service_dicts) == service_sort(expected)
+
+    def test_load_configs_multi_file(self):
+        base_file = config.ConfigFile(
+            'base.yaml',
+            {
+                'version': '3.3',
+                'services': {
+                    'web': {
+                        'image': 'example/web',
+                        'configs': ['one'],
+                    },
+                },
+                'configs': {
+                    'one': {'file': 'secret.txt'},
+                },
+            })
+        override_file = config.ConfigFile(
+            'base.yaml',
+            {
+                'version': '3.3',
+                'services': {
+                    'web': {
+                        'configs': [
+                            {
+                                'source': 'source',
+                                'target': 'target',
+                                'uid': '100',
+                                'gid': '200',
+                                'mode': 0o777,
+                            },
+                        ],
+                    },
+                },
+            })
+        details = config.ConfigDetails('.', [base_file, override_file])
+        service_dicts = config.load(details).services
+        expected = [
+            {
+                'name': 'web',
+                'image': 'example/web',
+                'configs': [
+                    types.ServiceConfig('one', None, None, None, None),
+                    types.ServiceConfig('source', 'target', '100', '200', 0o777),
+                ],
+            },
+        ]
+        assert service_sort(service_dicts) == service_sort(expected)
+
 
 class NetworkModeTest(unittest.TestCase):
 
@@ -2533,6 +2650,24 @@ class InterpolationTest(unittest.TestCase):
             }
         }
 
+    @mock.patch.dict(os.environ)
+    def test_interpolation_configs_section(self):
+        os.environ['FOO'] = 'baz.bar'
+        config_dict = config.load(build_config_details({
+            'version': '3.3',
+            'configs': {
+                'configdata': {
+                    'external': {'name': '$FOO'}
+                }
+            }
+        }))
+        assert config_dict.configs == {
+            'configdata': {
+                'external': {'name': 'baz.bar'},
+                'external_name': 'baz.bar'
+            }
+        }
+
 
 class VolumeConfigTest(unittest.TestCase):
 
@@ -3964,7 +4099,38 @@ class SerializeTest(unittest.TestCase):
                 'image': 'alpine',
                 'name': 'web'
             }
-        ], volumes={}, networks={}, secrets={})
+        ], volumes={}, networks={}, secrets={}, configs={})
 
         serialized_config = yaml.load(serialize_config(config_dict))
         assert '8080:80/tcp' in serialized_config['services']['web']['ports']
+
+    def test_serialize_configs(self):
+        service_dict = {
+            'image': 'example/web',
+            'configs': [
+                {'source': 'one'},
+                {
+                    'source': 'source',
+                    'target': 'target',
+                    'uid': '100',
+                    'gid': '200',
+                    'mode': 0o777,
+                }
+            ]
+        }
+        configs_dict = {
+            'one': {'file': '/one.txt'},
+            'source': {'file': '/source.pem'},
+            'two': {'external': True},
+        }
+        config_dict = config.load(build_config_details({
+            'version': '3.3',
+            'services': {'web': service_dict},
+            'configs': configs_dict
+        }))
+
+        serialized_config = yaml.load(serialize_config(config_dict))
+        serialized_service = serialized_config['services']['web']
+        assert secret_sort(serialized_service['configs']) == secret_sort(service_dict['configs'])
+        assert 'configs' in serialized_config
+        assert serialized_config['configs']['two'] == configs_dict['two']

tests/unit/config/types_test.py

@@ -57,15 +57,15 @@ class TestServicePort(object):
     def test_parse_simple_target_port(self):
         ports = ServicePort.parse(8000)
         assert len(ports) == 1
-        assert ports[0].target == '8000'
+        assert ports[0].target == 8000
 
     def test_parse_complete_port_definition(self):
         port_def = '1.1.1.1:3000:3000/udp'
         ports = ServicePort.parse(port_def)
         assert len(ports) == 1
         assert ports[0].repr() == {
-            'target': '3000',
-            'published': '3000',
+            'target': 3000,
+            'published': 3000,
             'external_ip': '1.1.1.1',
             'protocol': 'udp',
         }
@@ -77,7 +77,7 @@ class TestServicePort(object):
         assert len(ports) == 1
         assert ports[0].legacy_repr() == port_def + '/tcp'
         assert ports[0].repr() == {
-            'target': '3000',
+            'target': 3000,
             'external_ip': '1.1.1.1',
         }
 
@@ -86,14 +86,19 @@ class TestServicePort(object):
         assert len(ports) == 2
         reprs = [p.repr() for p in ports]
         assert {
-            'target': '4000',
-            'published': '25000'
+            'target': 4000,
+            'published': 25000
         } in reprs
         assert {
-            'target': '4001',
-            'published': '25001'
+            'target': 4001,
+            'published': 25001
         } in reprs
 
+    def test_parse_invalid_port(self):
+        port_def = '4000p'
+        with pytest.raises(ConfigurationError):
+            ServicePort.parse(port_def)
+
 
 class TestVolumeSpec(object):
 

tests/unit/project_test.py

@@ -37,6 +37,7 @@ class ProjectTest(unittest.TestCase):
             networks=None,
             volumes=None,
             secrets=None,
+            configs=None,
         )
         project = Project.from_config(
             name='composetest',
@@ -66,6 +67,7 @@ class ProjectTest(unittest.TestCase):
             networks=None,
             volumes=None,
             secrets=None,
+            configs=None,
         )
         project = Project.from_config('composetest', config, None)
         self.assertEqual(len(project.services), 2)
@@ -173,6 +175,7 @@ class ProjectTest(unittest.TestCase):
                 networks=None,
                 volumes=None,
                 secrets=None,
+                configs=None,
             ),
         )
         assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"]
@@ -206,6 +209,7 @@ class ProjectTest(unittest.TestCase):
                 networks=None,
                 volumes=None,
                 secrets=None,
+                configs=None,
             ),
         )
         assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"]
@@ -232,6 +236,7 @@ class ProjectTest(unittest.TestCase):
                 networks=None,
                 volumes=None,
                 secrets=None,
+                configs=None,
             ),
         )
         with mock.patch.object(Service, 'containers') as mock_return:
@@ -366,6 +371,7 @@ class ProjectTest(unittest.TestCase):
                 networks=None,
                 volumes=None,
                 secrets=None,
+                configs=None,
             ),
         )
         service = project.get_service('test')
@@ -391,6 +397,7 @@ class ProjectTest(unittest.TestCase):
                 networks=None,
                 volumes=None,
                 secrets=None,
+                configs=None,
             ),
         )
         service = project.get_service('test')
@@ -425,6 +432,7 @@ class ProjectTest(unittest.TestCase):
                 networks=None,
                 volumes=None,
                 secrets=None,
+                configs=None,
             ),
         )
 
@@ -446,6 +454,7 @@ class ProjectTest(unittest.TestCase):
                 networks=None,
                 volumes=None,
                 secrets=None,
+                configs=None,
             ),
         )
 
@@ -467,6 +476,7 @@ class ProjectTest(unittest.TestCase):
                 networks={'custom': {}},
                 volumes=None,
                 secrets=None,
+                configs=None,
             ),
         )
 
@@ -498,6 +508,7 @@ class ProjectTest(unittest.TestCase):
                 networks=None,
                 volumes=None,
                 secrets=None,
+                configs=None,
             ),
         )
         self.assertEqual([c.id for c in project.containers()], ['1'])
@@ -515,6 +526,7 @@ class ProjectTest(unittest.TestCase):
                 networks={'default': {}},
                 volumes={'data': {}},
                 secrets=None,
+                configs=None,
             ),
         )
         self.mock_client.remove_network.side_effect = NotFound(None, None, 'oops')