extend docker setup to multiple version of ubuntu and swift (#357)

Motivation: easier testing across different permutations of os and language versions

Modifications:

* update dockerfile to take both swift_version adn ubuntu_version
* update default linux version to 16.04
* create multiple docker-compose files that use arguments to define os and swift versions
* make docker-compose file DRYer with yaml anchors

Result: users/ci can easily run tests across various versions of ubuntu and swift

README.md

@@ -169,7 +169,7 @@ Alternatively, you may want to develop or test with `docker-compose`.
 
 First make sure you have [Docker](https://www.docker.com/community-edition) installed, next run the following commands:
 
-- `docker-compose -f docker/docker-compose.yaml up test`
+- `docker-compose -f docker/docker-compose.yaml run test`
 
   Will create a base image with Swift runtime and other build and test dependencies, compile SwiftNIO and run the unit and integration tests
 

docker/Dockerfile

@@ -1,44 +1,56 @@
-FROM ubuntu:14.04
+ARG ubuntu_version=16.04
-MAINTAINER tomerd@apple.com
+FROM ubuntu:$ubuntu_version
+# needed to do again after FROM due to docker limitation
+ARG ubuntu_version
 
 ARG DEBIAN_FRONTEND=noninteractive
-
 # do not start services during installation as this will fail and log a warning / error.
-RUN echo "#!/bin/sh\nexit 0" > /usr/sbin/policy-rc.d 
+RUN echo "#!/bin/sh\nexit 0" > /usr/sbin/policy-rc.d
+
+# basic dependencies
+RUN apt-get update
+RUN apt-get install -y wget git build-essential software-properties-common pkg-config locales
+RUN apt-get install -y libicu-dev libblocksruntime0
+RUN apt-get install -y lsof dnsutils netcat-openbsd # used by integration tests
 
 # local
+RUN locale-gen en_US.UTF-8
 RUN locale-gen en_US en_US.UTF-8
 RUN dpkg-reconfigure locales
 RUN echo 'export LANG=en_US.UTF-8' >> $HOME/.profile
 RUN echo 'export LANGUAGE=en_US:en' >> $HOME/.profile
 RUN echo 'export LC_ALL=en_US.UTF-8' >> $HOME/.profile
 
-# basic dependencies
+# known_hosts
-RUN apt-get update
+RUN mkdir -p $HOME/.ssh
-RUN apt-get install -y wget git software-properties-common pkg-config
+RUN touch $HOME/.ssh/known_hosts
-RUN apt-get install -y libicu-dev libblocksruntime0
+RUN ssh-keyscan github.com 2> /dev/null >> $HOME/.ssh/known_hosts
-RUN apt-get install -y lsof dnsutils # used by integration tests
 
 # clang
-RUN wget -q -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+RUN apt-get install -y clang-3.9
-RUN apt-add-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-5.0 main"
+RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-3.9 100
-RUN apt-get update
+RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-3.9 100
-RUN apt-get install -y clang-5.0 lldb-5.0
-RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-5.0 100
-RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-5.0 100
 
-# modern curl
+# modern curl, if needed
-RUN apt-get install -y build-essential libssl-dev
+ARG install_curl_from_source
-RUN mkdir $HOME/.curl
+RUN [ ! -z $install_curl_from_source ] || apt-get install -y curl libcurl3 libz-dev
-RUN wget -q https://curl.haxx.se/download/curl-7.50.3.tar.gz -O $HOME/curl.tar.gz
+RUN [ -z $install_curl_from_source ] || apt-get install -y libssl-dev
-RUN tar xzf $HOME/curl.tar.gz --directory $HOME/.curl --strip-components=1
+RUN [ -z $install_curl_from_source ] || mkdir $HOME/.curl
-RUN cd $HOME/.curl && ./configure --with-ssl && make && make install && cd -
+RUN [ -z $install_curl_from_source ] || wget -q https://curl.haxx.se/download/curl-7.50.3.tar.gz -O $HOME/curl.tar.gz
-RUN ldconfig
+RUN [ -z $install_curl_from_source ] || tar xzf $HOME/curl.tar.gz --directory $HOME/.curl --strip-components=1
+RUN [ -z $install_curl_from_source ] || ( cd $HOME/.curl && ./configure --with-ssl && make && make install && cd - )
+RUN [ -z $install_curl_from_source ] || ldconfig
+
+# ruby and jazzy for docs generation
+RUN apt-add-repository -y ppa:brightbox/ruby-ng
+RUN apt-get update
+RUN apt-get install -y ruby2.4 ruby2.4-dev libsqlite3-dev
+RUN gem install jazzy --no-ri --no-rdoc
 
 # swift
-ARG version=4.0.3
+ARG swift_version=4.0.3
 RUN mkdir $HOME/.swift
-RUN wget -q https://swift.org/builds/swift-${version}-release/ubuntu1404/swift-${version}-RELEASE/swift-${version}-RELEASE-ubuntu14.04.tar.gz -O $HOME/swift.tar.gz
+RUN wget -q https://swift.org/builds/swift-${swift_version}-release/ubuntu$(echo $ubuntu_version | sed 's/\.//g')/swift-${swift_version}-RELEASE/swift-${swift_version}-RELEASE-ubuntu${ubuntu_version}.tar.gz -O $HOME/swift.tar.gz
 RUN tar xzf $HOME/swift.tar.gz --directory $HOME/.swift --strip-components=1
 RUN echo 'export PATH="$HOME/.swift/usr/bin:$PATH"' >> $HOME/.profile
 RUN echo 'export LINUX_SOURCEKIT_LIB_PATH="$HOME/.swift/usr/lib"' >> $HOME/.profile
@@ -48,13 +60,3 @@ RUN mkdir -p $HOME/.scripts
 RUN wget -q https://raw.githubusercontent.com/apple/swift/master/utils/symbolicate-linux-fatal -O $HOME/.scripts/symbolicate-linux-fatal
 RUN chmod 755 $HOME/.scripts/symbolicate-linux-fatal
 RUN echo 'export PATH="$HOME/.scripts:$PATH"' >> $HOME/.profile
-
-# ruby
-RUN apt-add-repository -y ppa:brightbox/ruby-ng
-RUN apt-get update
-RUN apt-get install -y ruby2.4 ruby2.4-dev libsqlite3-dev
-
-# known_hosts
-RUN mkdir -p $HOME/.ssh
-RUN touch $HOME/.ssh/known_hosts
-RUN ssh-keyscan github.com 2> /dev/null >> $HOME/.ssh/known_hosts

docker/docker-compose.1404.403.yaml

@@ -0,0 +1,26 @@
+version: "3"
+
+services:
+
+  runtime-setup:
+    image: swift-nio:14.04-4.0.3
+    build:
+      args:
+        ubuntu_version : "14.04"
+        swift_version : "4.0.3"
+        install_curl_from_source: "true"
+
+  unit-tests:
+    image: swift-nio:14.04-4.0.3
+
+  integration-tests:
+    image: swift-nio:14.04-4.0.3
+
+  test:
+    image: swift-nio:14.04-4.0.3
+
+  echo:
+    image: swift-nio:14.04-4.0.3
+
+  http:
+    image: swift-nio:14.04-4.0.3

docker/docker-compose.1404.41.yaml

@@ -0,0 +1,36 @@
+version: "3"
+
+services:
+
+  runtime-setup:
+    image: swift-nio:14.04-4.1
+    build:
+      args:
+        ubuntu_version : "14.04"
+        swift_version : "4.1"
+        install_curl_from_source: "true"
+
+  unit-tests:
+    image: swift-nio:14.04-4.1
+
+  integration-tests:
+    image: swift-nio:14.04-4.1
+    environment:
+      - MAX_ALLOCS_ALLOWED_1000_reqs_1_conn=52000
+      - MAX_ALLOCS_ALLOWED_1_reqs_1000_conn=867500
+      - MAX_ALLOCS_ALLOWED_ping_pong_1000_reqs_1_conn=5000
+      - MAX_ALLOCS_ALLOWED_bytebuffer_lots_of_rw=3500
+
+  test:
+    image: swift-nio:14.04-4.1
+    environment:
+      - MAX_ALLOCS_ALLOWED_1000_reqs_1_conn=52000
+      - MAX_ALLOCS_ALLOWED_1_reqs_1000_conn=867500
+      - MAX_ALLOCS_ALLOWED_ping_pong_1000_reqs_1_conn=5000
+      - MAX_ALLOCS_ALLOWED_bytebuffer_lots_of_rw=3500
+
+  echo:
+    image: swift-nio:14.04-4.1
+
+  http:
+    image: swift-nio:14.04-4.1

docker/docker-compose.1604.403.yaml

@@ -0,0 +1,25 @@
+version: "3"
+
+services:
+
+  runtime-setup:
+    image: swift-nio:16.04-4.0.3
+    build:
+      args:
+        ubuntu_version : "16.04"
+        swift_version : "4.0.3"
+
+  unit-tests:
+    image: swift-nio:16.04-4.0.3
+
+  integration-tests:
+    image: swift-nio:16.04-4.0.3
+
+  test:
+    image: swift-nio:16.04-4.0.3
+
+  echo:
+    image: swift-nio:16.04-4.0.3
+
+  http:
+    image: swift-nio:16.04-4.0.3

docker/docker-compose.1604.41.yaml

@@ -0,0 +1,35 @@
+version: "3"
+
+services:
+
+  runtime-setup:
+    image: swift-nio:16.04-4.1
+    build:
+      args:
+        ubuntu_version : "16.04"
+        swift_version : "4.1"
+
+  unit-tests:
+    image: swift-nio:16.04-4.1
+
+  integration-tests:
+    image: swift-nio:16.04-4.1
+    environment:
+      - MAX_ALLOCS_ALLOWED_1000_reqs_1_conn=52000
+      - MAX_ALLOCS_ALLOWED_1_reqs_1000_conn=868500
+      - MAX_ALLOCS_ALLOWED_ping_pong_1000_reqs_1_conn=5000
+      - MAX_ALLOCS_ALLOWED_bytebuffer_lots_of_rw=3500
+
+  test:
+    image: swift-nio:16.04-4.1
+    environment:
+      - MAX_ALLOCS_ALLOWED_1000_reqs_1_conn=52000
+      - MAX_ALLOCS_ALLOWED_1_reqs_1000_conn=868500
+      - MAX_ALLOCS_ALLOWED_ping_pong_1000_reqs_1_conn=5000
+      - MAX_ALLOCS_ALLOWED_bytebuffer_lots_of_rw=3500
+
+  echo:
+    image: swift-nio:16.04-4.1
+
+  http:
+    image: swift-nio:16.04-4.1

docker/docker-compose.yaml

@@ -1,58 +1,44 @@
+# this file is not designed to be run directly
+# instead, use the docker-compose.<os>.<swift> files
+# eg docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.1604.41.yaml run test
 version: "3"
 
 services:
 
-    swift-nio:
+  runtime-setup:
-        image: swift-nio:latest
+    image: swift-nio:default
-        build:
+    build:
-            context: .
+      context: .
-            dockerfile: Dockerfile
+      dockerfile: Dockerfile
 
-    unit-tests:
+  common: &common
-        depends_on: [swift-nio]
+    image: swift-nio:default
-        image: swift-nio:latest
+    depends_on: [runtime-setup]
-        command: /bin/bash -cl "swift test"
+    volumes:
-        volumes:
+      - ~/.ssh:/root/.ssh
-            - ~/.ssh:/root/.ssh
+      - ..:/code
-            - ..:/code
+    working_dir: /code
-        working_dir: /code
 
-    integration-tests:
+  unit-tests:
-        depends_on: [swift-nio]
+    <<: *common
-        image: swift-nio:latest
+    command: /bin/bash -cl "swift test"
-        command: /bin/bash -cl "./scripts/integration_tests.sh"
-        volumes:
-            - ~/.ssh:/root/.ssh
-            - ..:/code
-        working_dir: /code
 
-    test:
+  integration-tests:
-        depends_on: [swift-nio]
+    <<: *common
-        image: swift-nio:latest
+    command: /bin/bash -cl "./scripts/integration_tests.sh"
-        command: /bin/bash -cl "swift test && ./scripts/integration_tests.sh"
-        volumes:
-            - ~/.ssh:/root/.ssh
-            - ..:/code
-        working_dir: /code
 
-    echo:
+  test:
-        depends_on: [swift-nio]
+    <<: *common
-        image: swift-nio:latest
+    command: /bin/bash -cl "swift test && ./scripts/integration_tests.sh"
-        ports:
-            - "9999:9999"
-        command: /bin/bash -cl "swift run NIOEchoServer 0.0.0.0 9999"
-        volumes:
-            - ~/.ssh:/root/.ssh
-            - ..:/code
-        working_dir: /code
 
-    http:
+  echo:
-        depends_on: [swift-nio]
+    <<: *common
-        image: swift-nio:latest
+    ports:
-        ports:
+      - "9999:9999"
-            - "8888:8888"
+    command: /bin/bash -cl "swift run NIOEchoServer 0.0.0.0 9999"
-        command: /bin/bash -cl "swift run NIOHTTP1Server 0.0.0.0 8888"
+
-        volumes:
+  http:
-            - ~/.ssh:/root/.ssh
+    <<: *common
-            - ..:/code
+    ports:
-        working_dir: /code
+      - "8888:8888"
+    command: /bin/bash -cl "swift run NIOHTTP1Server 0.0.0.0 8888"