Merge pull request #50 from orchardup/ship-0.1.3

Bump version to 0.1.3

CHANGES.md

@@ -1,6 +1,19 @@
 Change log
 ==========
 
+0.1.3 (2014-01-23)
+------------------
+
+ - Fix ports sometimes being configured incorrectly. (#46)
+ - Fix log output sometimes not displaying. (#47)
+
+0.1.2 (2014-01-22)
+------------------
+
+ - Add `-T` option to `fig run` to disable pseudo-TTY. (#34)
+ - Fix `fig up` requiring the ubuntu image to be pulled to recreate containers. (#33) Thanks @cameronmaske!
+ - Improve reliability, fix arrow keys and fix a race condition in `fig run`. (#34, #39, #40)
+
 0.1.1 (2014-01-17)
 ------------------
 

README.md

@@ -2,6 +2,7 @@ Fig
 ===
 
 [![Build Status](https://travis-ci.org/orchardup/fig.png?branch=master)](https://travis-ci.org/orchardup/fig)
+[![PyPI version](https://badge.fury.io/py/fig.png)](http://badge.fury.io/py/fig)
 
 Punctual, lightweight development environments using Docker.
 

fig/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import unicode_literals
 from .service import Service
 
-__version__ = '0.1.1'
+__version__ = '0.1.3'

fig/cli/log_printer.py

@@ -6,6 +6,7 @@ from itertools import cycle
 
 from .multiplexer import Multiplexer
 from . import colors
+from .utils import split_buffer
 
 
 class LogPrinter(object):
@@ -31,8 +32,9 @@ class LogPrinter(object):
 
     def _make_log_generator(self, container, color_fn):
         prefix = color_fn(container.name + " | ")
-        for line in split_buffer(self._attach(container), '\n'):
-            yield prefix + line
+        # Attach to container before log printer starts running
+        line_generator = split_buffer(self._attach(container), '\n')
+        return (prefix + line.decode('utf-8') for line in line_generator)
 
     def _attach(self, container):
         params = {
@@ -43,21 +45,3 @@ class LogPrinter(object):
         params.update(self.attach_params)
         params = dict((name, 1 if value else 0) for (name, value) in list(params.items()))
         return container.attach(**params)
-
-def split_buffer(reader, separator):
-    """
-    Given a generator which yields strings and a separator string,
-    joins all input, splits on the separator and yields each chunk.
-    Requires that each input string is decodable as UTF-8.
-    """
-    buffered = ''
-
-    for data in reader:
-        lines = (buffered + data.decode('utf-8')).split(separator)
-        for line in lines[:-1]:
-            yield line + separator
-        if len(lines) > 1:
-            buffered = lines[-1]
-
-    if len(buffered) > 0:
-        yield buffered

fig/cli/main.py

@@ -4,7 +4,6 @@ import logging
 import sys
 import re
 import signal
-import sys
 
 from inspect import getdoc
 
@@ -200,12 +199,20 @@ class TopLevelCommand(Command):
         Usage: run [options] SERVICE COMMAND [ARGS...]
 
         Options:
-            -d    Detached mode: Run container in the background, print new container name
+            -d    Detached mode: Run container in the background, print new
+                  container name
+            -T    Disable pseudo-tty allocation. By default `fig run`
+                  allocates a TTY.
         """
         service = self.project.get_service(options['SERVICE'])
+
+        tty = True
+        if options['-d'] or options['-T'] or not sys.stdin.isatty():
+            tty = False
+
         container_options = {
             'command': [options['COMMAND']] + options['ARGS'],
-            'tty': not options['-d'],
+            'tty': tty,
             'stdin_open': not options['-d'],
         }
         container = service.create_container(one_off=True, **container_options)
@@ -213,12 +220,7 @@ class TopLevelCommand(Command):
             service.start_container(container, ports=None)
             print(container.name)
         else:
-            with self._attach_to_container(
-                container.id,
-                interactive=True,
-                logs=True,
-                raw=True
-            ) as c:
+            with self._attach_to_container(container.id, raw=tty) as c:
                 service.start_container(container, ports=None)
                 c.run()
 
@@ -310,35 +312,15 @@ class TopLevelCommand(Command):
                 print("Gracefully stopping... (press Ctrl+C again to force)")
                 self.project.stop(service_names=options['SERVICE'])
 
-    def _attach_to_container(self, container_id, interactive, logs=False, stream=True, raw=False):
-        stdio = self.client.attach_socket(
-            container_id,
-            params={
-                'stdin': 1 if interactive else 0,
-                'stdout': 1,
-                'stderr': 0,
-                'logs': 1 if logs else 0,
-                'stream': 1 if stream else 0
-            },
-            ws=True,
-        )
-
-        stderr = self.client.attach_socket(
-            container_id,
-            params={
-                'stdin': 0,
-                'stdout': 0,
-                'stderr': 1,
-                'logs': 1 if logs else 0,
-                'stream': 1 if stream else 0
-            },
-            ws=True,
-        )
+    def _attach_to_container(self, container_id, raw=False):
+        socket_in = self.client.attach_socket(container_id, params={'stdin': 1, 'stream': 1})
+        socket_out = self.client.attach_socket(container_id, params={'stdout': 1, 'logs': 1, 'stream': 1})
+        socket_err = self.client.attach_socket(container_id, params={'stderr': 1, 'logs': 1, 'stream': 1})
 
         return SocketClient(
-            socket_in=stdio,
-            socket_out=stdio,
-            socket_err=stderr,
+            socket_in=socket_in,
+            socket_out=socket_out,
+            socket_err=socket_err,
             raw=raw,
         )
 

fig/cli/socketclient.py

@@ -1,7 +1,6 @@
 from __future__ import print_function
 # Adapted from https://github.com/benthor/remotty/blob/master/socketclient.py
 
-from select import select
 import sys
 import tty
 import fcntl
@@ -57,15 +56,15 @@ class SocketClient:
 
     def run(self):
         if self.socket_in is not None:
-            self.start_background_thread(target=self.send_ws, args=(self.socket_in, sys.stdin))
+            self.start_background_thread(target=self.send, args=(self.socket_in, sys.stdin))
 
         recv_threads = []
 
         if self.socket_out is not None:
-            recv_threads.append(self.start_background_thread(target=self.recv_ws, args=(self.socket_out, sys.stdout)))
+            recv_threads.append(self.start_background_thread(target=self.recv, args=(self.socket_out, sys.stdout)))
 
         if self.socket_err is not None:
-            recv_threads.append(self.start_background_thread(target=self.recv_ws, args=(self.socket_err, sys.stderr)))
+            recv_threads.append(self.start_background_thread(target=self.recv, args=(self.socket_err, sys.stderr)))
 
         for t in recv_threads:
             t.join()
@@ -76,10 +75,10 @@ class SocketClient:
         thread.start()
         return thread
 
-    def recv_ws(self, socket, stream):
+    def recv(self, socket, stream):
         try:
             while True:
-                chunk = socket.recv()
+                chunk = socket.recv(4096)
 
                 if chunk:
                     stream.write(chunk)
@@ -89,24 +88,21 @@ class SocketClient:
         except Exception as e:
             log.debug(e)
 
-    def send_ws(self, socket, stream):
+    def send(self, socket, stream):
         while True:
-            r, w, e = select([stream.fileno()], [], [])
+            chunk = stream.read(1)
 
-            if r:
-                chunk = stream.read(1)
-
-                if chunk == '':
-                    socket.send_close()
-                    break
-                else:
-                    try:
-                        socket.send(chunk)
-                    except Exception as e:
-                        if hasattr(e, 'errno') and e.errno == errno.EPIPE:
-                            break
-                        else:
-                            raise e
+            if chunk == '':
+                socket.close()
+                break
+            else:
+                try:
+                    socket.send(chunk)
+                except Exception as e:
+                    if hasattr(e, 'errno') and e.errno == errno.EPIPE:
+                        break
+                    else:
+                        raise e
 
     def destroy(self):
         if self.settings is not None:

fig/cli/utils.py

@@ -83,3 +83,28 @@ def mkdir(path, permissions=0o700):
 
 def docker_url():
     return os.environ.get('DOCKER_HOST')
+
+
+def split_buffer(reader, separator):
+    """
+    Given a generator which yields strings and a separator string,
+    joins all input, splits on the separator and yields each chunk.
+
+    Unlike string.split(), each chunk includes the trailing
+    separator, except for the last one if none was found on the end
+    of the input.
+    """
+    buffered = str('')
+    separator = str(separator)
+
+    for data in reader:
+        buffered += data
+        while True:
+            index = buffered.find(separator)
+            if index == -1:
+                break
+            yield buffered[:index+1]
+            buffered = buffered[index+1:]
+
+    if len(buffered) > 0:
+        yield buffered

fig/container.py

@@ -3,7 +3,7 @@ from __future__ import absolute_import
 
 class Container(object):
     """
-    Represents a Docker container, constructed from the output of 
+    Represents a Docker container, constructed from the output of
     GET /containers/:id:/json.
     """
     def __init__(self, client, dictionary, has_been_inspected=False):
@@ -38,6 +38,10 @@ class Container(object):
     def id(self):
         return self.dictionary['ID']
 
+    @property
+    def image(self):
+        return self.dictionary['Image']
+
     @property
     def short_id(self):
         return self.id[:10]

fig/packages/docker/client.py

@@ -122,7 +122,8 @@ class Client(requests.Session):
                           detach=False, stdin_open=False, tty=False,
                           mem_limit=0, ports=None, environment=None, dns=None,
                           volumes=None, volumes_from=None,
-                          network_disabled=False):
+                          network_disabled=False, entrypoint=None,
+                          cpu_shares=None, working_dir=None):
         if isinstance(command, six.string_types):
             command = shlex.split(str(command))
         if isinstance(environment, dict):
@@ -134,15 +135,12 @@ class Client(requests.Session):
             exposed_ports = {}
             for port_definition in ports:
                 port = port_definition
-                proto = None
+                proto = 'tcp'
                 if isinstance(port_definition, tuple):
                     if len(port_definition) == 2:
                         proto = port_definition[1]
                     port = port_definition[0]
-                exposed_ports['{0}{1}'.format(
-                    port,
-                    '/' + proto if proto else ''
-                )] = {}
+                exposed_ports['{0}/{1}'.format(port, proto)] = {}
             ports = exposed_ports
 
         if volumes and isinstance(volumes, list):
@@ -154,6 +152,7 @@ class Client(requests.Session):
         attach_stdin = False
         attach_stdout = False
         attach_stderr = False
+        stdin_once = False
 
         if not detach:
             attach_stdout = True
@@ -161,6 +160,7 @@ class Client(requests.Session):
 
             if stdin_open:
                 attach_stdin = True
+                stdin_once = True
 
         return {
             'Hostname':     hostname,
@@ -168,6 +168,7 @@ class Client(requests.Session):
             'User':         user,
             'Tty':          tty,
             'OpenStdin':    stdin_open,
+            'StdinOnce':    stdin_once,
             'Memory':       mem_limit,
             'AttachStdin':  attach_stdin,
             'AttachStdout': attach_stdout,
@@ -178,7 +179,10 @@ class Client(requests.Session):
             'Image':        image,
             'Volumes':      volumes,
             'VolumesFrom':  volumes_from,
-            'NetworkDisabled': network_disabled
+            'NetworkDisabled': network_disabled,
+            'Entrypoint':   entrypoint,
+            'CpuShares':    cpu_shares,
+            'WorkingDir':    working_dir
         }
 
     def _post_json(self, url, data, **kwargs):
@@ -409,11 +413,13 @@ class Client(requests.Session):
                          detach=False, stdin_open=False, tty=False,
                          mem_limit=0, ports=None, environment=None, dns=None,
                          volumes=None, volumes_from=None,
-                         network_disabled=False, name=None):
+                         network_disabled=False, name=None, entrypoint=None,
+                         cpu_shares=None, working_dir=None):
 
         config = self._container_config(
             image, command, hostname, user, detach, stdin_open, tty, mem_limit,
-            ports, environment, dns, volumes, volumes_from, network_disabled
+            ports, environment, dns, volumes, volumes_from, network_disabled,
+            entrypoint, cpu_shares, working_dir
         )
         return self.create_container_from_config(config, name)
 
@@ -475,27 +481,34 @@ class Client(requests.Session):
             return [x['Id'] for x in res]
         return res
 
-    def import_image(self, src, data=None, repository=None, tag=None):
+    def import_image(self, src=None, repository=None, tag=None, image=None):
         u = self._url("/images/create")
         params = {
             'repo': repository,
             'tag': tag
         }
-        try:
-            # XXX: this is ways not optimal but the only way
-            # for now to import tarballs through the API
-            fic = open(src)
-            data = fic.read()
-            fic.close()
-            src = "-"
-        except IOError:
-            # file does not exists or not a file (URL)
-            data = None
-        if isinstance(src, six.string_types):
-            params['fromSrc'] = src
-            return self._result(self._post(u, data=data, params=params))
 
-        return self._result(self._post(u, data=src, params=params))
+        if src:
+            try:
+                # XXX: this is ways not optimal but the only way
+                # for now to import tarballs through the API
+                fic = open(src)
+                data = fic.read()
+                fic.close()
+                src = "-"
+            except IOError:
+                # file does not exists or not a file (URL)
+                data = None
+            if isinstance(src, six.string_types):
+                params['fromSrc'] = src
+                return self._result(self._post(u, data=data, params=params))
+            return self._result(self._post(u, data=src, params=params))
+
+        if image:
+            params['fromImage'] = image
+            return self._result(self._post(u, data=None, params=params))
+
+        raise Exception("Must specify a src or image")
 
     def info(self):
         return self._result(self._get(self._url("/info")),
@@ -577,13 +590,13 @@ class Client(requests.Session):
         self._raise_for_status(res)
         json_ = res.json()
         s_port = str(private_port)
-        f_port = None
-        if s_port in json_['NetworkSettings']['PortMapping']['Udp']:
-            f_port = json_['NetworkSettings']['PortMapping']['Udp'][s_port]
-        elif s_port in json_['NetworkSettings']['PortMapping']['Tcp']:
-            f_port = json_['NetworkSettings']['PortMapping']['Tcp'][s_port]
+        h_ports = None
 
-        return f_port
+        h_ports = json_['NetworkSettings']['Ports'].get(s_port + '/udp')
+        if h_ports is None:
+            h_ports = json_['NetworkSettings']['Ports'].get(s_port + '/tcp')
+
+        return h_ports
 
     def pull(self, repository, tag=None, stream=False):
         registry, repo_name = auth.resolve_repository_name(repository)

fig/service.py

@@ -143,9 +143,10 @@ class Service(object):
 
         intermediate_container = Container.create(
             self.client,
-            image='ubuntu',
+            image=container.image,
             command='echo',
             volumes_from=container.id,
+            entrypoint=None
         )
         intermediate_container.start()
         intermediate_container.wait()
@@ -171,9 +172,10 @@ class Service(object):
                 port = str(port)
                 if ':' in port:
                     external_port, internal_port = port.split(':', 1)
-                    port_bindings[int(internal_port)] = int(external_port)
                 else:
-                    port_bindings[int(port)] = None
+                    external_port, internal_port = (None, port)
+
+                port_bindings[internal_port] = external_port
 
         volume_bindings = {}
 
@@ -212,7 +214,7 @@ class Service(object):
         return links
 
     def _get_container_options(self, override_options, one_off=False):
-        keys = ['image', 'command', 'hostname', 'user', 'detach', 'stdin_open', 'tty', 'mem_limit', 'ports', 'environment', 'dns', 'volumes', 'volumes_from']
+        keys = ['image', 'command', 'hostname', 'user', 'detach', 'stdin_open', 'tty', 'mem_limit', 'ports', 'environment', 'dns', 'volumes', 'volumes_from', 'entrypoint']
         container_options = dict((k, self.options[k]) for k in keys if k in self.options)
         container_options.update(override_options)
 
@@ -224,8 +226,8 @@ class Service(object):
                 port = str(port)
                 if ':' in port:
                     port = port.split(':')[-1]
-                if '/' not in port:
-                    port = "%s/tcp" % port
+                if '/' in port:
+                    port = tuple(port.split('/'))
                 ports.append(port)
             container_options['ports'] = ports
 

requirements.txt

@@ -5,3 +5,4 @@ PyYAML==3.10
 texttable==0.8.1
 # docker requires six==1.3.0
 six==1.3.0
+mock==1.0.1

script/travis

@@ -3,18 +3,21 @@
 # Exit on first error
 set -ex
 
+# Put Python eggs in a writeable directory
 export PYTHON_EGG_CACHE="/tmp/.python-eggs"
 
+# Activate correct virtualenv
 TRAVIS_PYTHON_VERSION=$1
 source /home/travis/virtualenv/python${TRAVIS_PYTHON_VERSION}/bin/activate
+
 env
 
 # Kill background processes on exit
-trap 'kill $(jobs -p)' SIGINT SIGTERM EXIT
+trap 'kill -9 $(jobs -p)' SIGINT SIGTERM EXIT
 
 # Start docker daemon
 docker -d -H unix:///var/run/docker.sock 2>> /dev/null >> /dev/null &
 sleep 2
 
 # $init is set by sekexe
-cd $(dirname $init)/.. && nosetests
+cd $(dirname $init)/.. && nosetests -v

setup.py

@@ -35,6 +35,7 @@ setup(
     url='https://github.com/orchardup/fig',
     author='Orchard Laboratories Ltd.',
     author_email='hello@orchardup.com',
+    license='BSD',
     packages=find_packages(),
     include_package_data=True,
     test_suite='nose.collector',

tests/cli_test.py

@@ -1,6 +1,8 @@
 from __future__ import unicode_literals
 from __future__ import absolute_import
 from . import unittest
+from mock import patch
+from six import StringIO
 from fig.cli.main import TopLevelCommand
 
 class CLITestCase(unittest.TestCase):
@@ -8,11 +10,18 @@ class CLITestCase(unittest.TestCase):
         self.command = TopLevelCommand()
         self.command.base_dir = 'tests/fixtures/simple-figfile'
 
+    def tearDown(self):
+        self.command.project.kill()
+        self.command.project.remove_stopped()
+
     def test_help(self):
         self.assertRaises(SystemExit, lambda: self.command.dispatch(['-h'], None))
 
-    def test_ps(self):
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_ps(self, mock_stdout):
+        self.command.project.get_service('simple').create_container()
         self.command.dispatch(['ps'], None)
+        self.assertIn('fig_simple_1', mock_stdout.getvalue())
 
     def test_scale(self):
         project = self.command.project

tests/service_test.py

@@ -110,8 +110,9 @@ class ServiceTest(DockerClientTestCase):
         self.assertIn('/var/db', container.inspect()['Volumes'])
 
     def test_recreate_containers(self):
-        service = self.create_service('db', environment={'FOO': '1'}, volumes=['/var/db'])
+        service = self.create_service('db', environment={'FOO': '1'}, volumes=['/var/db'], entrypoint=['ps'])
         old_container = service.create_container()
+        self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['ps'])
         self.assertEqual(old_container.dictionary['Config']['Env'], ['FOO=1'])
         self.assertEqual(old_container.name, 'figtest_db_1')
         service.start_container(old_container)
@@ -120,11 +121,15 @@ class ServiceTest(DockerClientTestCase):
         num_containers_before = len(self.client.containers(all=True))
 
         service.options['environment']['FOO'] = '2'
-        (old, new) = service.recreate_containers()
-        self.assertEqual(len(old), 1)
+        (intermediate, new) = service.recreate_containers()
+        self.assertEqual(len(intermediate), 1)
         self.assertEqual(len(new), 1)
 
         new_container = new[0]
+        intermediate_container = intermediate[0]
+        self.assertEqual(intermediate_container.dictionary['Config']['Entrypoint'], None)
+
+        self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['ps'])
         self.assertEqual(new_container.dictionary['Config']['Env'], ['FOO=2'])
         self.assertEqual(new_container.name, 'figtest_db_1')
         service.start_container(new_container)
@@ -179,9 +184,14 @@ class ServiceTest(DockerClientTestCase):
     def test_start_container_creates_ports(self):
         service = self.create_service('web', ports=[8000])
         container = service.start_container().inspect()
-        self.assertIn('8000/tcp', container['HostConfig']['PortBindings'])
+        self.assertEqual(container['HostConfig']['PortBindings'].keys(), ['8000/tcp'])
         self.assertNotEqual(container['HostConfig']['PortBindings']['8000/tcp'][0]['HostPort'], '8000')
 
+    def test_start_container_creates_port_with_explicit_protocol(self):
+        service = self.create_service('web', ports=['8000/udp'])
+        container = service.start_container().inspect()
+        self.assertEqual(container['HostConfig']['PortBindings'].keys(), ['8000/udp'])
+
     def test_start_container_creates_fixed_external_ports(self):
         service = self.create_service('web', ports=['8000:8000'])
         container = service.start_container().inspect()

tests/split_buffer_test.py

@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from fig.cli.utils import split_buffer
+from . import unittest
+
+class SplitBufferTest(unittest.TestCase):
+    def test_single_line_chunks(self):
+        def reader():
+            yield "abc\n"
+            yield "def\n"
+            yield "ghi\n"
+
+        self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi\n"])
+
+    def test_no_end_separator(self):
+        def reader():
+            yield "abc\n"
+            yield "def\n"
+            yield "ghi"
+
+        self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"])
+
+    def test_multiple_line_chunk(self):
+        def reader():
+            yield "abc\ndef\nghi"
+
+        self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"])
+
+    def test_chunked_line(self):
+        def reader():
+            yield "a"
+            yield "b"
+            yield "c"
+            yield "\n"
+            yield "d"
+
+        self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "d"])