From c5831ac98f8adb6c5c3e8438df05803a660a9814 Mon Sep 17 00:00:00 2001 From: Fernando Mez Date: Mon, 9 Mar 2020 15:13:57 -0300 Subject: [PATCH] Fix CI config and coverage report --- .github/workflows/main.yml | 191 ++++++++++++++++++++++++++++++++ .travis.yml | 162 +++++++-------------------- scripts/append_codecov_token.py | 35 ++++++ scripts/report-coverage.sh | 18 +++ tox.ini | 1 + 5 files changed, 282 insertions(+), 125 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 scripts/append_codecov_token.py create mode 100755 scripts/report-coverage.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..a006fb092 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,191 @@ +name: main + +on: + push: + branches: + - 4.6.x + tags: + - "*" + + pull_request: + branches: + - 4.6.x + +jobs: + build: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + name: [ + "windows-py27", + "windows-py35", + "windows-py36", + "windows-py37", + "windows-py37-pluggy", + "windows-py38", + + "ubuntu-py27-pluggy", + "ubuntu-py27-nobyte", + "ubuntu-py37", + "ubuntu-py37-pluggy", + "ubuntu-py37-pexpect-py37-twisted", + "ubuntu-py37-freeze", + "ubuntu-pypy", + "ubuntu-pypy3", + + "macos-py27", + "macos-py38", + + ] + + include: + # Windows jobs + - name: "windows-py27" + python: "2.7" + os: windows-latest + tox_env: "py27-xdist" + use_coverage: true + - name: "windows-py35" + python: "3.5" + os: windows-latest + tox_env: "py35-xdist" + use_coverage: true + - name: "windows-py36" + python: "3.6" + os: windows-latest + tox_env: "py36-xdist" + use_coverage: true + - name: "windows-py37" + python: "3.7" + os: windows-latest + tox_env: "py37-twisted-numpy" + use_coverage: true + - name: "windows-py37-pluggy" + python: "3.7" + os: windows-latest + tox_env: "py37-pluggymaster-xdist" + use_coverage: true + - name: "windows-py38" + python: "3.8" + os: windows-latest + tox_env: "py38-xdist" + use_coverage: true + + # Ubuntu jobs – find the rest of them in .travis.yml + - name: "ubuntu-py27-pluggy" + python: "2.7" + os: ubuntu-latest + tox_env: "py27-pluggymaster-xdist" + use_coverage: true + - name: "ubuntu-py27-nobyte" + python: "2.7" + os: ubuntu-latest + tox_env: "py27-nobyte-numpy-xdist" + use_coverage: true + - name: "ubuntu-py37" + python: "3.7" + os: ubuntu-latest + tox_env: "py37-lsof-numpy-xdist" + use_coverage: true + - name: "ubuntu-py37-pluggy" + python: "3.7" + os: ubuntu-latest + tox_env: "py37-pluggymaster-xdist" + use_coverage: true + - name: "ubuntu-py37-pexpect-py37-twisted" + python: "3.7" + os: ubuntu-latest + tox_env: "py37-pexpect,py37-twisted" + use_coverage: true + - name: "ubuntu-py37-freeze" + python: "3.7" + os: ubuntu-latest + tox_env: "py37-freeze" + - name: "ubuntu-pypy" + python: "pypy2" + os: ubuntu-latest + tox_env: "pypy-xdist" + use_coverage: true + - name: "ubuntu-pypy3" + python: "pypy3" + os: ubuntu-latest + tox_env: "pypy3-xdist" + use_coverage: true + + # MacOS jobs + - name: "macos-py27" + python: "2.7" + os: macos-latest + tox_env: "py27-xdist" + use_coverage: true + - name: "macos-py38" + python: "3.8" + os: macos-latest + tox_env: "py38-xdist" + use_coverage: true + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python }} on ${{ matrix.os }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox coverage + - name: Test without coverage + if: "! matrix.use_coverage" + run: "tox -e ${{ matrix.tox_env }}" + + - name: Test with coverage + if: "matrix.use_coverage" + env: + _PYTEST_TOX_COVERAGE_RUN: "coverage run -m" + COVERAGE_PROCESS_START: ".coveragerc" + _PYTEST_TOX_EXTRA_DEP: "coverage-enable-subprocess" + run: "tox -vv -e ${{ matrix.tox_env }}" + + - name: Prepare coverage token + if: (matrix.use_coverage && ( github.repository == 'pytest-dev/pytest' || github.event_name == 'pull_request' )) + run: | + python scripts/append_codecov_token.py + - name: Report coverage + if: (matrix.use_coverage) + env: + CODECOV_NAME: ${{ matrix.name }} + run: bash scripts/report-coverage.sh -F GHA,${{ runner.os }} + + deploy: + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && github.repository == 'pytest-dev/pytest' + + runs-on: ubuntu-latest + + needs: [build] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: "3.7" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade wheel setuptools tox + - name: Build package + run: | + python setup.py sdist bdist_wheel + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_token }} + - name: Publish GitHub release notes + env: + GH_RELEASE_NOTES_TOKEN: ${{ secrets.release_notes }} + run: | + sudo apt-get install pandoc + tox -e publish-gh-release-notes diff --git a/.travis.yml b/.travis.yml index 2d6883f48..d4f2d8048 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,12 @@ language: python dist: xenial -stages: -- baseline -- name: test - if: repo = pytest-dev/pytest AND tag IS NOT present -- name: deploy - if: repo = pytest-dev/pytest AND tag IS present -python: '3.7' +python: '3.7.4' cache: false env: global: - - PYTEST_ADDOPTS=-vv + - PYTEST_ADDOPTS="-vv --showlocals --durations=100 --exitfirst" + - PYTEST_COVERAGE=1 # setuptools-scm needs all tags in order to obtain a proper version git: @@ -22,113 +17,44 @@ install: jobs: include: - # OSX tests - first (in test stage), since they are the slower ones. - - &test-macos - os: osx - osx_image: xcode10.1 - language: generic - # Coverage for: - # - py2 with symlink in test_cmdline_python_package_symlink. - env: TOXENV=py27-xdist PYTEST_COVERAGE=1 - before_install: - - python -V - - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27 - - <<: *test-macos - env: TOXENV=py37-pexpect,py37-xdist PYTEST_COVERAGE=1 - before_install: - - which python3 - - python3 -V - - ln -sfn "$(which python3)" /usr/local/bin/python - - python -V - - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37 - - # Full run of latest (major) supported versions, without xdist. - - env: TOXENV=py27 - python: '2.7' - - env: TOXENV=py37 - python: '3.7' - - # Coverage tracking is slow with pypy, skip it. - - env: TOXENV=pypy-xdist - python: 'pypy' - - env: TOXENV=pypy3-xdist - python: 'pypy3' - - - env: TOXENV=py34-xdist - python: '3.4' - - env: TOXENV=py35-xdist - python: '3.5' - # Coverage for: - # - pytester's LsofFdLeakChecker # - TestArgComplete (linux only) # - numpy - # Empty PYTEST_ADDOPTS to run this non-verbose. - - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1 PYTEST_ADDOPTS= - - # Specialized factors for py27. - - env: TOXENV=py27-nobyte-numpy-xdist - python: '2.7' - - env: TOXENV=py27-pluggymaster-xdist - python: '2.7' - - # Specialized factors for py37. - # Coverage for: - # - test_sys_breakpoint_interception (via pexpect). - - env: TOXENV=py37-pexpect,py37-twisted PYTEST_COVERAGE=1 - - env: TOXENV=py37-pluggymaster-xdist - - env: TOXENV=py37-freeze - - # Jobs only run via Travis cron jobs (currently daily). - - env: TOXENV=py38-xdist - python: '3.8-dev' - if: type = cron - + # - verbose=0 - stage: baseline - # Coverage for: - # - _pytest.unittest._handle_skip (via pexpect). - env: TOXENV=py27-pexpect,py27-twisted PYTEST_COVERAGE=1 + env: TOXENV=py27-xdist python: '2.7' - # Use py36 here for faster baseline. - - env: TOXENV=py36-xdist - python: '3.6' - - env: TOXENV=linting,docs,doctesting PYTEST_COVERAGE=1 + + - env: TOXENV=py38-xdist + python: '3.8' + + - stage: tests + # - _pytest.unittest._handle_skip (via pexpect). + env: TOXENV=py27-pexpect,py27-twisted + python: '2.7' + + - env: TOXENV=py35-xdist + python: '3.5.9' + + - env: TOXENV=py36-xdist PYTEST_REORDER_TESTS=0 + python: '3.6.9' + + - env: TOXENV=py37-numpy-pexpect-twisted + python: '3.7.4' + + # - test_sys_breakpoint_interception (via pexpect). + - env: TOXENV=py37-pexpect,py37-twisted + python: '3.7.4' + + # Run also non-verbosely, to gain coverage + - env: TOXENV=py38-xdist PYTEST_ADDOPTS="" + python: '3.8' + + - env: TOXENV=linting,docs,doctesting cache: directories: - $HOME/.cache/pre-commit - - stage: deploy - python: '3.6' - install: pip install -U setuptools setuptools_scm tox - script: skip - # token to upload github release notes: GH_RELEASE_NOTES_TOKEN - env: - - secure: "OjOeL7/0JUDkV00SsTs732e8vQjHynpbG9FKTNtZZJ+1Zn4Cib+hAlwmlBnvVukML0X60YpcfjnC4quDOIGLPsh5zeXnvJmYtAIIUNQXjWz8NhcGYrhyzuP1rqV22U68RTCdmOq3lMYU/W2acwHP7T49PwJtOiUM5kF120UAQ0Zi5EmkqkIvH8oM5mO9Dlver+/U7Htpz9rhKrHBXQNCMZI6yj2aUyukqB2PN2fjAlDbCF//+FmvYw9NjT4GeFOSkTCf4ER9yfqs7yglRfwiLtOCZ2qKQhWZNsSJDB89rxIRXWavJUjJKeY2EW2/NkomYJDpqJLIF4JeFRw/HhA47CYPeo6BJqyyNV+0CovL1frpWfi9UQw2cMbgFUkUIUk3F6DD59PHNIOX2R/HX56dQsw7WKl3QuHlCOkICXYg8F7Ta684IoKjeTX03/6QNOkURfDBwfGszY0FpbxrjCSWKom6RyZdyidnESaxv9RzjcIRZVh1rp8KMrwS1OrwRSdG0zjlsPr49hWMenN/8fKgcHTV4/r1Tj6mip0dorSRCrgUNIeRBKgmui6FS8642ab5JNKOxMteVPVR2sFuhjOQ0Jy+PmvceYY9ZMWc3+/B/KVh0dZ3hwvLGZep/vxDS2PwCA5/xw31714vT5LxidKo8yECjBynMU/wUTTS695D3NY=" - addons: - apt: - packages: - # required by publish_gh_release_notes - - pandoc - after_deploy: tox -e publish_gh_release_notes - deploy: - provider: pypi - user: nicoddemus - distributions: sdist bdist_wheel - skip_upload_docs: true - password: - secure: xanTgTUu6XDQVqB/0bwJQXoDMnU5tkwZc5koz6mBkkqZhKdNOi2CLoC1XhiSZ+ah24l4V1E0GAqY5kBBcy9d7NVe4WNg4tD095LsHw+CRU6/HCVIFfyk2IZ+FPAlguesCcUiJSXOrlBF+Wj68wEvLoK7EoRFbJeiZ/f91Ww1sbtDlqXABWGHrmhPJL5Wva7o7+wG7JwJowqdZg1pbQExsCc7b53w4v2RBu3D6TJaTAzHiVsW+nUSI67vKI/uf+cR/OixsTfy37wlHgSwihYmrYLFls3V0bSpahCim3bCgMaFZx8S8xrdgJ++PzBCof2HeflFKvW+VCkoYzGEG4NrTWJoNz6ni4red9GdvfjGH3YCjAKS56h9x58zp2E5rpsb/kVq5/45xzV+dq6JRuhQ1nJWjBC6fSKAc/bfwnuFK3EBxNLkvBssLHvsNjj5XG++cB8DdS9wVGUqjpoK4puaXUWFqy4q3S9F86HEsKNgExtieA9qNx+pCIZVs6JCXZNjr0I5eVNzqJIyggNgJG6RyravsU35t9Zd9doL5g4Y7UKmAGTn1Sz24HQ4sMQgXdm2SyD8gEK5je4tlhUvfGtDvMSlstq71kIn9nRpFnqB6MFlbYSEAZmo8dGbCquoUc++6Rum208wcVbrzzVtGlXB/Ow9AbFMYeAGA0+N/K1e59c= - on: - tags: true - repo: pytest-dev/pytest - -matrix: - allow_failures: - - python: '3.8-dev' - env: TOXENV=py38-xdist - # Temporary (https://github.com/pytest-dev/pytest/pull/5334). - - env: TOXENV=pypy3-xdist - python: 'pypy3' - before_script: - | # Do not (re-)upload coverage with cron runs. @@ -142,27 +68,13 @@ before_script: export _PYTEST_TOX_COVERAGE_RUN="coverage run -m" export _PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess fi - -script: tox +script: env COLUMNS=120 python -m tox after_success: - | if [[ "$PYTEST_COVERAGE" = 1 ]]; then - set -e - # Add last TOXENV to $PATH. - PATH="$PWD/.tox/${TOXENV##*,}/bin:$PATH" - coverage combine - coverage xml - coverage report -m - bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV-$TRAVIS_OS_NAME + env CODECOV_NAME="$TOXENV-$TRAVIS_OS_NAME" scripts/report-coverage.sh fi - -notifications: - irc: - channels: - - "chat.freenode.net#pytest" - on_success: change - on_failure: change - skip_join: true - email: - - pytest-commit@python.org +branches: + only: + - 4.6.x diff --git a/scripts/append_codecov_token.py b/scripts/append_codecov_token.py new file mode 100644 index 000000000..b3a82075f --- /dev/null +++ b/scripts/append_codecov_token.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +Appends the codecov token to the 'codecov.yml' file at the root of the repository. +This is done by CI during PRs and builds on the pytest-dev repository so we can upload coverage, at least +until codecov grows some native integration like it has with Travis and AppVeyor. +See discussion in https://github.com/pytest-dev/pytest/pull/6441 for more information. +""" +import os.path +from textwrap import dedent + + +def main(): + this_dir = os.path.dirname(__file__) + cov_file = os.path.join(this_dir, "..", "codecov.yml") + + assert os.path.isfile(cov_file), "{cov_file} does not exist".format( + cov_file=cov_file + ) + + with open(cov_file, "a") as f: + # token from: https://codecov.io/gh/pytest-dev/pytest/settings + # use same URL to regenerate it if needed + text = dedent( + """ + codecov: + token: "1eca3b1f-31a2-4fb8-a8c3-138b441b50a7" + """ + ) + f.write(text) + + print("Token updated:", cov_file) + + +if __name__ == "__main__": + main() diff --git a/scripts/report-coverage.sh b/scripts/report-coverage.sh new file mode 100755 index 000000000..fbcf20ca9 --- /dev/null +++ b/scripts/report-coverage.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -e +set -x + +if [ -z "$TOXENV" ]; then + python -m pip install coverage +else + # Add last TOXENV to $PATH. + PATH="$PWD/.tox/${TOXENV##*,}/bin:$PATH" +fi + +python -m coverage combine +python -m coverage xml +python -m coverage report -m +# Set --connect-timeout to work around https://github.com/curl/curl/issues/4461 +curl -S -L --connect-timeout 5 --retry 6 -s https://codecov.io/bash -o codecov-upload.sh +bash codecov-upload.sh -Z -X fix -f coverage.xml "$@" diff --git a/tox.ini b/tox.ini index b788020f1..61cb4bef0 100644 --- a/tox.ini +++ b/tox.ini @@ -119,6 +119,7 @@ changedir = testing/freeze # Disable PEP 517 with pip, which does not work with PyInstaller currently. deps = pyinstaller + setuptools < 45.0.0 commands = {envpython} create_executable.py {envpython} tox_run.py