From 491d29cc87ea3d1bb1fbab744276c600e45c6169 Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 5 Aug 2023 17:23:57 +0800 Subject: [PATCH] feat: optimize multi platform image build (#754) --- .github/workflows/build-api-image.yml | 4 +- .github/workflows/build-web-image.yml | 4 +- api/Dockerfile | 5 +- api/requirements.txt | 2 +- docker/Dockerfile.base | 256 -------------------------- web/Dockerfile | 7 +- 6 files changed, 16 insertions(+), 262 deletions(-) delete mode 100644 docker/Dockerfile.base diff --git a/.github/workflows/build-api-image.yml b/.github/workflows/build-api-image.yml index 678eb2feed..ceffcc051f 100644 --- a/.github/workflows/build-api-image.yml +++ b/.github/workflows/build-api-image.yml @@ -42,12 +42,14 @@ jobs: uses: docker/build-push-action@v4 with: context: "{{defaultContext}}:api" - platforms: linux/amd64,linux/arm64 + platforms: ${{ github.ref == 'refs/heads/deploy/dev' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} build-args: | COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max - name: Deploy to server if: github.ref == 'refs/heads/deploy/dev' diff --git a/.github/workflows/build-web-image.yml b/.github/workflows/build-web-image.yml index 69a81b3db1..730a5dbecb 100644 --- a/.github/workflows/build-web-image.yml +++ b/.github/workflows/build-web-image.yml @@ -42,12 +42,14 @@ jobs: uses: docker/build-push-action@v4 with: context: "{{defaultContext}}:web" - platforms: linux/amd64,linux/arm64 + platforms: ${{ github.ref == 'refs/heads/deploy/dev' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} build-args: | COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max - name: Deploy to server if: github.ref == 'refs/heads/deploy/dev' diff --git a/api/Dockerfile b/api/Dockerfile index 1970ad3e9b..1170ecea0d 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,4 +1,4 @@ -FROM langgenius/base:1.0.1-bullseye-slim as langgenius-api +FROM python:3.10-slim LABEL maintainer="takatost@gmail.com" @@ -15,6 +15,9 @@ EXPOSE 5001 WORKDIR /app/api +RUN apt-get update && \ + apt-get install -y bash curl wget vim gcc g++ python3-dev libc-dev libffi-dev + COPY requirements.txt /app/api/requirements.txt RUN pip install -r requirements.txt diff --git a/api/requirements.txt b/api/requirements.txt index 30b9d23f09..ccbb0e18cf 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -8,7 +8,7 @@ flask-migrate~=4.0.4 flask-restful==0.3.9 flask-session2==1.3.1 flask-cors==3.0.10 -gunicorn~=20.1.0 +gunicorn~=21.2.0 gevent~=22.10.2 langchain==0.0.239 openai~=0.27.8 diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base deleted file mode 100644 index ec0e930f17..0000000000 --- a/docker/Dockerfile.base +++ /dev/null @@ -1,256 +0,0 @@ -FROM nginx:1.22 - -# ensure local python is preferred over distribution python -ENV PATH /usr/local/bin:$PATH - -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 - -# runtime dependencies -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - netbase \ - tzdata \ - ; \ - rm -rf /var/lib/apt/lists/* - -ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D -ENV PYTHON_VERSION 3.10.10 - -RUN set -eux; \ - \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - dpkg-dev \ - gcc \ - gnupg dirmngr \ - libbluetooth-dev \ - libbz2-dev \ - libc6-dev \ - libdb-dev \ - libexpat1-dev \ - libffi-dev \ - libgdbm-dev \ - liblzma-dev \ - libncursesw5-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - make \ - tk-dev \ - uuid-dev \ - wget \ - xz-utils \ - zlib1g-dev \ - ; \ - \ - wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ - wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ - gpg --batch --verify python.tar.xz.asc python.tar.xz; \ - command -v gpgconf > /dev/null && gpgconf --kill all || :; \ - rm -rf "$GNUPGHOME" python.tar.xz.asc; \ - mkdir -p /usr/src/python; \ - tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ - rm python.tar.xz; \ - \ - cd /usr/src/python; \ - gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ - ./configure \ - --build="$gnuArch" \ - --enable-loadable-sqlite-extensions \ - --enable-optimizations \ - --enable-option-checking=fatal \ - --enable-shared \ - --with-lto \ - --with-system-expat \ - --without-ensurepip \ - ; \ - nproc="$(nproc)"; \ - EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; \ - LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; \ - LDFLAGS="${LDFLAGS:--Wl},--strip-all"; \ - make -j "$nproc" \ - "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ - "LDFLAGS=${LDFLAGS:-}" \ - "PROFILE_TASK=${PROFILE_TASK:-}" \ - ; \ -# https://github.com/docker-library/python/issues/784 -# prevent accidental usage of a system installed libpython of the same version - rm python; \ - make -j "$nproc" \ - "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ - "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \ - "PROFILE_TASK=${PROFILE_TASK:-}" \ - python \ - ; \ - make install; \ - \ - cd /; \ - rm -rf /usr/src/python; \ - \ - find /usr/local -depth \ - \( \ - \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ - -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ - \) -exec rm -rf '{}' + \ - ; \ - \ - ldconfig; \ - \ - apt-mark auto '.*' > /dev/null; \ - apt-mark manual $savedAptMark; \ - find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - ; \ - rm -rf /var/lib/apt/lists/*; \ - \ - python3 --version - -# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) -RUN set -eux; \ - for src in idle3 pydoc3 python3 python3-config; do \ - dst="$(echo "$src" | tr -d 3)"; \ - [ -s "/usr/local/bin/$src" ]; \ - [ ! -e "/usr/local/bin/$dst" ]; \ - ln -svT "$src" "/usr/local/bin/$dst"; \ - done - -# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 22.3.1 -# https://github.com/docker-library/python/issues/365 -ENV PYTHON_SETUPTOOLS_VERSION 65.5.1 -# https://github.com/pypa/get-pip -ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/d5cb0afaf23b8520f1bbcfed521017b4a95f5c01/public/get-pip.py -ENV PYTHON_GET_PIP_SHA256 394be00f13fa1b9aaa47e911bdb59a09c3b2986472130f30aa0bfaf7f3980637 - -RUN set -eux; \ - \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends wget; \ - \ - wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ - echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -; \ - \ - apt-mark auto '.*' > /dev/null; \ - [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ - rm -rf /var/lib/apt/lists/*; \ - \ - export PYTHONDONTWRITEBYTECODE=1; \ - \ - python get-pip.py \ - --disable-pip-version-check \ - --no-cache-dir \ - --no-compile \ - "pip==$PYTHON_PIP_VERSION" \ - "setuptools==$PYTHON_SETUPTOOLS_VERSION" \ - ; \ - rm -f get-pip.py; \ - \ - pip --version - -RUN groupadd --gid 1000 node \ - && useradd --uid 1000 --gid node --shell /bin/bash --create-home node - -ENV NODE_VERSION 18.15.0 - -RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - ppc64el) ARCH='ppc64le';; \ - s390x) ARCH='s390x';; \ - arm64) ARCH='arm64';; \ - armhf) ARCH='armv7l';; \ - i386) ARCH='x86';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - && set -ex \ - # libatomic1 for arm - && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* \ - && for key in \ - 4ED778F539E3634C779C87C6D7062848A1AB005C \ - 141F07595B7B3FFE74309A937405533BE57C7D57 \ - 74F12602B6F1C4E913FAA37AD3A89613643B6201 \ - DD792F5973C6DE52C432CBDAC77ABFA00DDBF2B7 \ - 61FC681DFB92A079F1685E77973F295594EC4689 \ - 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ - C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ - 890C08DB8579162FEE0DF9DB8BEAB4DFCF555EF4 \ - C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ - 108F52B48DB57BB0CC439B2997B01419BD92F80A \ - ; do \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \ - gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ - && apt-mark auto '.*' > /dev/null \ - && find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ - # smoke tests - && node --version \ - && npm --version - -ENV YARN_VERSION 1.22.19 - -RUN set -ex \ - && savedAptMark="$(apt-mark showmanual)" \ - && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* \ - && for key in \ - 6A010C5166006599AA17F08146C2130DFD2497F5 \ - ; do \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \ - gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \ - && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \ - && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ - && mkdir -p /opt \ - && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \ - && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \ - && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \ - && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ - && apt-mark auto '.*' > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; } \ - && find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - # smoke test - && yarn --version - - - -RUN apt-get update && \ - apt-get install -y bash curl wget vim gcc g++ python3-dev libc-dev libffi-dev - -RUN pip3 install gunicorn -RUN npm install pm2 -g - -ENTRYPOINT ["/usr/local/bin/pm2-runtime", "start"] \ No newline at end of file diff --git a/web/Dockerfile b/web/Dockerfile index fe0868d414..4fb8a7626e 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,7 +1,8 @@ -FROM langgenius/base:1.0.1-bullseye-slim +FROM node:18.17.0-alpine LABEL maintainer="takatost@gmail.com" +ENV NODE_ENV=production ENV EDITION SELF_HOSTED ENV DEPLOY_ENV PRODUCTION ENV CONSOLE_API_URL http://127.0.0.1:5001 @@ -13,6 +14,8 @@ WORKDIR /app/web COPY package.json /app/web/package.json +RUN npm install pm2 -g + RUN npm install --only=prod COPY . /app/web/ @@ -26,4 +29,4 @@ RUN chmod +x /entrypoint.sh ARG COMMIT_SHA ENV COMMIT_SHA ${COMMIT_SHA} -ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] +ENTRYPOINT ["/bin/sh", "/entrypoint.sh"]