From 8914a5536e0e533469120d0ed60baf44dd96528a Mon Sep 17 00:00:00 2001 From: tomer doron Date: Tue, 1 May 2018 00:56:39 -0700 Subject: [PATCH] extend docker setup to multiple version of ubuntu and swift (#357) Motivation: easier testing across different permutations of os and language versions Modifications: * update dockerfile to take both swift_version adn ubuntu_version * update default linux version to 16.04 * create multiple docker-compose files that use arguments to define os and swift versions * make docker-compose file DRYer with yaml anchors Result: users/ci can easily run tests across various versions of ubuntu and swift --- README.md | 2 +- docker/Dockerfile | 70 ++++++++++++------------ docker/docker-compose.1404.403.yaml | 26 +++++++++ docker/docker-compose.1404.41.yaml | 36 +++++++++++++ docker/docker-compose.1604.403.yaml | 25 +++++++++ docker/docker-compose.1604.41.yaml | 35 ++++++++++++ docker/docker-compose.yaml | 84 ++++++++++++----------------- 7 files changed, 194 insertions(+), 84 deletions(-) create mode 100644 docker/docker-compose.1404.403.yaml create mode 100644 docker/docker-compose.1404.41.yaml create mode 100644 docker/docker-compose.1604.403.yaml create mode 100644 docker/docker-compose.1604.41.yaml diff --git a/README.md b/README.md index 211aed00..5370a8dd 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ Alternatively, you may want to develop or test with `docker-compose`. First make sure you have [Docker](https://www.docker.com/community-edition) installed, next run the following commands: -- `docker-compose -f docker/docker-compose.yaml up test` +- `docker-compose -f docker/docker-compose.yaml run test` Will create a base image with Swift runtime and other build and test dependencies, compile SwiftNIO and run the unit and integration tests diff --git a/docker/Dockerfile b/docker/Dockerfile index 18ce549a..0ef431bf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,44 +1,56 @@ -FROM ubuntu:14.04 -MAINTAINER tomerd@apple.com +ARG ubuntu_version=16.04 +FROM ubuntu:$ubuntu_version +# needed to do again after FROM due to docker limitation +ARG ubuntu_version ARG DEBIAN_FRONTEND=noninteractive - # do not start services during installation as this will fail and log a warning / error. -RUN echo "#!/bin/sh\nexit 0" > /usr/sbin/policy-rc.d +RUN echo "#!/bin/sh\nexit 0" > /usr/sbin/policy-rc.d + +# basic dependencies +RUN apt-get update +RUN apt-get install -y wget git build-essential software-properties-common pkg-config locales +RUN apt-get install -y libicu-dev libblocksruntime0 +RUN apt-get install -y lsof dnsutils netcat-openbsd # used by integration tests # local +RUN locale-gen en_US.UTF-8 RUN locale-gen en_US en_US.UTF-8 RUN dpkg-reconfigure locales RUN echo 'export LANG=en_US.UTF-8' >> $HOME/.profile RUN echo 'export LANGUAGE=en_US:en' >> $HOME/.profile RUN echo 'export LC_ALL=en_US.UTF-8' >> $HOME/.profile -# basic dependencies -RUN apt-get update -RUN apt-get install -y wget git software-properties-common pkg-config -RUN apt-get install -y libicu-dev libblocksruntime0 -RUN apt-get install -y lsof dnsutils # used by integration tests +# known_hosts +RUN mkdir -p $HOME/.ssh +RUN touch $HOME/.ssh/known_hosts +RUN ssh-keyscan github.com 2> /dev/null >> $HOME/.ssh/known_hosts # clang -RUN wget -q -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN apt-add-repository "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-5.0 main" -RUN apt-get update -RUN apt-get install -y clang-5.0 lldb-5.0 -RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-5.0 100 -RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-5.0 100 +RUN apt-get install -y clang-3.9 +RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-3.9 100 +RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-3.9 100 -# modern curl -RUN apt-get install -y build-essential libssl-dev -RUN mkdir $HOME/.curl -RUN wget -q https://curl.haxx.se/download/curl-7.50.3.tar.gz -O $HOME/curl.tar.gz -RUN tar xzf $HOME/curl.tar.gz --directory $HOME/.curl --strip-components=1 -RUN cd $HOME/.curl && ./configure --with-ssl && make && make install && cd - -RUN ldconfig +# modern curl, if needed +ARG install_curl_from_source +RUN [ ! -z $install_curl_from_source ] || apt-get install -y curl libcurl3 libz-dev +RUN [ -z $install_curl_from_source ] || apt-get install -y libssl-dev +RUN [ -z $install_curl_from_source ] || mkdir $HOME/.curl +RUN [ -z $install_curl_from_source ] || wget -q https://curl.haxx.se/download/curl-7.50.3.tar.gz -O $HOME/curl.tar.gz +RUN [ -z $install_curl_from_source ] || tar xzf $HOME/curl.tar.gz --directory $HOME/.curl --strip-components=1 +RUN [ -z $install_curl_from_source ] || ( cd $HOME/.curl && ./configure --with-ssl && make && make install && cd - ) +RUN [ -z $install_curl_from_source ] || ldconfig + +# ruby and jazzy for docs generation +RUN apt-add-repository -y ppa:brightbox/ruby-ng +RUN apt-get update +RUN apt-get install -y ruby2.4 ruby2.4-dev libsqlite3-dev +RUN gem install jazzy --no-ri --no-rdoc # swift -ARG version=4.0.3 +ARG swift_version=4.0.3 RUN mkdir $HOME/.swift -RUN wget -q https://swift.org/builds/swift-${version}-release/ubuntu1404/swift-${version}-RELEASE/swift-${version}-RELEASE-ubuntu14.04.tar.gz -O $HOME/swift.tar.gz +RUN wget -q https://swift.org/builds/swift-${swift_version}-release/ubuntu$(echo $ubuntu_version | sed 's/\.//g')/swift-${swift_version}-RELEASE/swift-${swift_version}-RELEASE-ubuntu${ubuntu_version}.tar.gz -O $HOME/swift.tar.gz RUN tar xzf $HOME/swift.tar.gz --directory $HOME/.swift --strip-components=1 RUN echo 'export PATH="$HOME/.swift/usr/bin:$PATH"' >> $HOME/.profile RUN echo 'export LINUX_SOURCEKIT_LIB_PATH="$HOME/.swift/usr/lib"' >> $HOME/.profile @@ -48,13 +60,3 @@ RUN mkdir -p $HOME/.scripts RUN wget -q https://raw.githubusercontent.com/apple/swift/master/utils/symbolicate-linux-fatal -O $HOME/.scripts/symbolicate-linux-fatal RUN chmod 755 $HOME/.scripts/symbolicate-linux-fatal RUN echo 'export PATH="$HOME/.scripts:$PATH"' >> $HOME/.profile - -# ruby -RUN apt-add-repository -y ppa:brightbox/ruby-ng -RUN apt-get update -RUN apt-get install -y ruby2.4 ruby2.4-dev libsqlite3-dev - -# known_hosts -RUN mkdir -p $HOME/.ssh -RUN touch $HOME/.ssh/known_hosts -RUN ssh-keyscan github.com 2> /dev/null >> $HOME/.ssh/known_hosts diff --git a/docker/docker-compose.1404.403.yaml b/docker/docker-compose.1404.403.yaml new file mode 100644 index 00000000..b571a7c9 --- /dev/null +++ b/docker/docker-compose.1404.403.yaml @@ -0,0 +1,26 @@ +version: "3" + +services: + + runtime-setup: + image: swift-nio:14.04-4.0.3 + build: + args: + ubuntu_version : "14.04" + swift_version : "4.0.3" + install_curl_from_source: "true" + + unit-tests: + image: swift-nio:14.04-4.0.3 + + integration-tests: + image: swift-nio:14.04-4.0.3 + + test: + image: swift-nio:14.04-4.0.3 + + echo: + image: swift-nio:14.04-4.0.3 + + http: + image: swift-nio:14.04-4.0.3 diff --git a/docker/docker-compose.1404.41.yaml b/docker/docker-compose.1404.41.yaml new file mode 100644 index 00000000..9829ab90 --- /dev/null +++ b/docker/docker-compose.1404.41.yaml @@ -0,0 +1,36 @@ +version: "3" + +services: + + runtime-setup: + image: swift-nio:14.04-4.1 + build: + args: + ubuntu_version : "14.04" + swift_version : "4.1" + install_curl_from_source: "true" + + unit-tests: + image: swift-nio:14.04-4.1 + + integration-tests: + image: swift-nio:14.04-4.1 + environment: + - MAX_ALLOCS_ALLOWED_1000_reqs_1_conn=52000 + - MAX_ALLOCS_ALLOWED_1_reqs_1000_conn=867500 + - MAX_ALLOCS_ALLOWED_ping_pong_1000_reqs_1_conn=5000 + - MAX_ALLOCS_ALLOWED_bytebuffer_lots_of_rw=3500 + + test: + image: swift-nio:14.04-4.1 + environment: + - MAX_ALLOCS_ALLOWED_1000_reqs_1_conn=52000 + - MAX_ALLOCS_ALLOWED_1_reqs_1000_conn=867500 + - MAX_ALLOCS_ALLOWED_ping_pong_1000_reqs_1_conn=5000 + - MAX_ALLOCS_ALLOWED_bytebuffer_lots_of_rw=3500 + + echo: + image: swift-nio:14.04-4.1 + + http: + image: swift-nio:14.04-4.1 diff --git a/docker/docker-compose.1604.403.yaml b/docker/docker-compose.1604.403.yaml new file mode 100644 index 00000000..3d0895de --- /dev/null +++ b/docker/docker-compose.1604.403.yaml @@ -0,0 +1,25 @@ +version: "3" + +services: + + runtime-setup: + image: swift-nio:16.04-4.0.3 + build: + args: + ubuntu_version : "16.04" + swift_version : "4.0.3" + + unit-tests: + image: swift-nio:16.04-4.0.3 + + integration-tests: + image: swift-nio:16.04-4.0.3 + + test: + image: swift-nio:16.04-4.0.3 + + echo: + image: swift-nio:16.04-4.0.3 + + http: + image: swift-nio:16.04-4.0.3 diff --git a/docker/docker-compose.1604.41.yaml b/docker/docker-compose.1604.41.yaml new file mode 100644 index 00000000..39f414a3 --- /dev/null +++ b/docker/docker-compose.1604.41.yaml @@ -0,0 +1,35 @@ +version: "3" + +services: + + runtime-setup: + image: swift-nio:16.04-4.1 + build: + args: + ubuntu_version : "16.04" + swift_version : "4.1" + + unit-tests: + image: swift-nio:16.04-4.1 + + integration-tests: + image: swift-nio:16.04-4.1 + environment: + - MAX_ALLOCS_ALLOWED_1000_reqs_1_conn=52000 + - MAX_ALLOCS_ALLOWED_1_reqs_1000_conn=868500 + - MAX_ALLOCS_ALLOWED_ping_pong_1000_reqs_1_conn=5000 + - MAX_ALLOCS_ALLOWED_bytebuffer_lots_of_rw=3500 + + test: + image: swift-nio:16.04-4.1 + environment: + - MAX_ALLOCS_ALLOWED_1000_reqs_1_conn=52000 + - MAX_ALLOCS_ALLOWED_1_reqs_1000_conn=868500 + - MAX_ALLOCS_ALLOWED_ping_pong_1000_reqs_1_conn=5000 + - MAX_ALLOCS_ALLOWED_bytebuffer_lots_of_rw=3500 + + echo: + image: swift-nio:16.04-4.1 + + http: + image: swift-nio:16.04-4.1 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 424e69c2..0f729b0b 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,58 +1,44 @@ +# this file is not designed to be run directly +# instead, use the docker-compose.. files +# eg docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.1604.41.yaml run test version: "3" services: - swift-nio: - image: swift-nio:latest - build: - context: . - dockerfile: Dockerfile + runtime-setup: + image: swift-nio:default + build: + context: . + dockerfile: Dockerfile - unit-tests: - depends_on: [swift-nio] - image: swift-nio:latest - command: /bin/bash -cl "swift test" - volumes: - - ~/.ssh:/root/.ssh - - ..:/code - working_dir: /code + common: &common + image: swift-nio:default + depends_on: [runtime-setup] + volumes: + - ~/.ssh:/root/.ssh + - ..:/code + working_dir: /code - integration-tests: - depends_on: [swift-nio] - image: swift-nio:latest - command: /bin/bash -cl "./scripts/integration_tests.sh" - volumes: - - ~/.ssh:/root/.ssh - - ..:/code - working_dir: /code + unit-tests: + <<: *common + command: /bin/bash -cl "swift test" - test: - depends_on: [swift-nio] - image: swift-nio:latest - command: /bin/bash -cl "swift test && ./scripts/integration_tests.sh" - volumes: - - ~/.ssh:/root/.ssh - - ..:/code - working_dir: /code + integration-tests: + <<: *common + command: /bin/bash -cl "./scripts/integration_tests.sh" - echo: - depends_on: [swift-nio] - image: swift-nio:latest - ports: - - "9999:9999" - command: /bin/bash -cl "swift run NIOEchoServer 0.0.0.0 9999" - volumes: - - ~/.ssh:/root/.ssh - - ..:/code - working_dir: /code + test: + <<: *common + command: /bin/bash -cl "swift test && ./scripts/integration_tests.sh" - http: - depends_on: [swift-nio] - image: swift-nio:latest - ports: - - "8888:8888" - command: /bin/bash -cl "swift run NIOHTTP1Server 0.0.0.0 8888" - volumes: - - ~/.ssh:/root/.ssh - - ..:/code - working_dir: /code + echo: + <<: *common + ports: + - "9999:9999" + command: /bin/bash -cl "swift run NIOEchoServer 0.0.0.0 9999" + + http: + <<: *common + ports: + - "8888:8888" + command: /bin/bash -cl "swift run NIOHTTP1Server 0.0.0.0 8888"