Fix CI config and coverage report

.github/workflows/main.yml

@@ -0,0 +1,191 @@
+name: main
+
+on:
+  push:
+    branches:
+      - 4.6.x
+    tags:
+      - "*"
+
+  pull_request:
+    branches:
+      - 4.6.x
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        name: [
+          "windows-py27",
+          "windows-py35",
+          "windows-py36",
+          "windows-py37",
+          "windows-py37-pluggy",
+          "windows-py38",
+
+          "ubuntu-py27-pluggy",
+          "ubuntu-py27-nobyte",
+          "ubuntu-py37",
+          "ubuntu-py37-pluggy",
+          "ubuntu-py37-pexpect-py37-twisted",
+          "ubuntu-py37-freeze",
+          "ubuntu-pypy",
+          "ubuntu-pypy3",
+
+          "macos-py27",
+          "macos-py38",
+
+        ]
+
+        include:
+          # Windows jobs
+          - name: "windows-py27"
+            python: "2.7"
+            os: windows-latest
+            tox_env: "py27-xdist"
+            use_coverage: true
+          - name: "windows-py35"
+            python: "3.5"
+            os: windows-latest
+            tox_env: "py35-xdist"
+            use_coverage: true
+          - name: "windows-py36"
+            python: "3.6"
+            os: windows-latest
+            tox_env: "py36-xdist"
+            use_coverage: true
+          - name: "windows-py37"
+            python: "3.7"
+            os: windows-latest
+            tox_env: "py37-twisted-numpy"
+            use_coverage: true
+          - name: "windows-py37-pluggy"
+            python: "3.7"
+            os: windows-latest
+            tox_env: "py37-pluggymaster-xdist"
+            use_coverage: true
+          - name: "windows-py38"
+            python: "3.8"
+            os: windows-latest
+            tox_env: "py38-xdist"
+            use_coverage: true
+
+          # Ubuntu jobs – find the rest of them in .travis.yml
+          - name: "ubuntu-py27-pluggy"
+            python: "2.7"
+            os: ubuntu-latest
+            tox_env: "py27-pluggymaster-xdist"
+            use_coverage: true
+          - name: "ubuntu-py27-nobyte"
+            python: "2.7"
+            os: ubuntu-latest
+            tox_env: "py27-nobyte-numpy-xdist"
+            use_coverage: true
+          - name: "ubuntu-py37"
+            python: "3.7"
+            os: ubuntu-latest
+            tox_env: "py37-lsof-numpy-xdist"
+            use_coverage: true
+          - name: "ubuntu-py37-pluggy"
+            python: "3.7"
+            os: ubuntu-latest
+            tox_env: "py37-pluggymaster-xdist"
+            use_coverage: true
+          - name: "ubuntu-py37-pexpect-py37-twisted"
+            python: "3.7"
+            os: ubuntu-latest
+            tox_env: "py37-pexpect,py37-twisted"
+            use_coverage: true
+          - name: "ubuntu-py37-freeze"
+            python: "3.7"
+            os: ubuntu-latest
+            tox_env: "py37-freeze"
+          - name: "ubuntu-pypy"
+            python: "pypy2"
+            os: ubuntu-latest
+            tox_env: "pypy-xdist"
+            use_coverage: true
+          - name: "ubuntu-pypy3"
+            python: "pypy3"
+            os: ubuntu-latest
+            tox_env: "pypy3-xdist"
+            use_coverage: true
+
+          # MacOS jobs
+          - name: "macos-py27"
+            python: "2.7"
+            os: macos-latest
+            tox_env: "py27-xdist"
+            use_coverage: true
+          - name: "macos-py38"
+            python: "3.8"
+            os: macos-latest
+            tox_env: "py38-xdist"
+            use_coverage: true
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python }} on ${{ matrix.os }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox coverage
+    - name: Test without coverage
+      if: "! matrix.use_coverage"
+      run: "tox -e ${{ matrix.tox_env }}"
+
+    - name: Test with coverage
+      if: "matrix.use_coverage"
+      env:
+        _PYTEST_TOX_COVERAGE_RUN: "coverage run -m"
+        COVERAGE_PROCESS_START: ".coveragerc"
+        _PYTEST_TOX_EXTRA_DEP: "coverage-enable-subprocess"
+      run: "tox -vv -e ${{ matrix.tox_env }}"
+
+    - name: Prepare coverage token
+      if: (matrix.use_coverage && ( github.repository == 'pytest-dev/pytest' || github.event_name == 'pull_request' ))
+      run: |
+        python scripts/append_codecov_token.py
+    - name: Report coverage
+      if: (matrix.use_coverage)
+      env:
+        CODECOV_NAME: ${{ matrix.name }}
+      run: bash scripts/report-coverage.sh -F GHA,${{ runner.os }}
+
+  deploy:
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && github.repository == 'pytest-dev/pytest'
+
+    runs-on: ubuntu-latest
+
+    needs: [build]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: "3.7"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install --upgrade wheel setuptools tox
+    - name: Build package
+      run: |
+        python setup.py sdist bdist_wheel
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_token }}
+    - name: Publish GitHub release notes
+      env:
+        GH_RELEASE_NOTES_TOKEN: ${{ secrets.release_notes }}
+      run: |
+        sudo apt-get install pandoc
+        tox -e publish-gh-release-notes

.travis.yml

@@ -1,17 +1,12 @@
 language: python
 dist: xenial
-stages:
+python: '3.7.4'
-- baseline
-- name: test
-  if: repo = pytest-dev/pytest AND tag IS NOT present
-- name: deploy
-  if: repo = pytest-dev/pytest AND tag IS present
-python: '3.7'
 cache: false
 
 env:
   global:
-    - PYTEST_ADDOPTS=-vv
+    - PYTEST_ADDOPTS="-vv --showlocals --durations=100 --exitfirst"
+    - PYTEST_COVERAGE=1
 
 # setuptools-scm needs all tags in order to obtain a proper version
 git:
@@ -22,113 +17,44 @@ install:
 
 jobs:
   include:
-    # OSX tests - first (in test stage), since they are the slower ones.
-    - &test-macos
-      os: osx
-      osx_image: xcode10.1
-      language: generic
-      # Coverage for:
-      # - py2 with symlink in test_cmdline_python_package_symlink.
-      env: TOXENV=py27-xdist PYTEST_COVERAGE=1
-      before_install:
-        - python -V
-        - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27
-    - <<: *test-macos
-      env: TOXENV=py37-pexpect,py37-xdist PYTEST_COVERAGE=1
-      before_install:
-        - which python3
-        - python3 -V
-        - ln -sfn "$(which python3)" /usr/local/bin/python
-        - python -V
-        - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37
-
-    # Full run of latest (major) supported versions, without xdist.
-    - env: TOXENV=py27
-      python: '2.7'
-    - env: TOXENV=py37
-      python: '3.7'
-
-    # Coverage tracking is slow with pypy, skip it.
-    - env: TOXENV=pypy-xdist
-      python: 'pypy'
-    - env: TOXENV=pypy3-xdist
-      python: 'pypy3'
-
-    - env: TOXENV=py34-xdist
-      python: '3.4'
-    - env: TOXENV=py35-xdist
-      python: '3.5'
-
     # Coverage for:
-    # - pytester's LsofFdLeakChecker
     # - TestArgComplete (linux only)
     # - numpy
-    # Empty PYTEST_ADDOPTS to run this non-verbose.
+    # - verbose=0
-    - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1 PYTEST_ADDOPTS=
-
-    # Specialized factors for py27.
-    - env: TOXENV=py27-nobyte-numpy-xdist
-      python: '2.7'
-    - env: TOXENV=py27-pluggymaster-xdist
-      python: '2.7'
-
-    # Specialized factors for py37.
-    # Coverage for:
-    # - test_sys_breakpoint_interception (via pexpect).
-    - env: TOXENV=py37-pexpect,py37-twisted PYTEST_COVERAGE=1
-    - env: TOXENV=py37-pluggymaster-xdist
-    - env: TOXENV=py37-freeze
-
-    # Jobs only run via Travis cron jobs (currently daily).
-    - env: TOXENV=py38-xdist
-      python: '3.8-dev'
-      if: type = cron
-
     - stage: baseline
-    # Coverage for:
+      env: TOXENV=py27-xdist
-    # - _pytest.unittest._handle_skip (via pexpect).
-      env: TOXENV=py27-pexpect,py27-twisted PYTEST_COVERAGE=1
       python: '2.7'
-    # Use py36 here for faster baseline.
+
-    - env: TOXENV=py36-xdist
+    - env: TOXENV=py38-xdist
-      python: '3.6'
+      python: '3.8'
-    - env: TOXENV=linting,docs,doctesting PYTEST_COVERAGE=1
+
+    - stage: tests
+    # - _pytest.unittest._handle_skip (via pexpect).
+      env: TOXENV=py27-pexpect,py27-twisted
+      python: '2.7'
+
+    - env: TOXENV=py35-xdist
+      python: '3.5.9'
+
+    - env: TOXENV=py36-xdist PYTEST_REORDER_TESTS=0
+      python: '3.6.9'
+
+    - env: TOXENV=py37-numpy-pexpect-twisted
+      python: '3.7.4'
+
+    # - test_sys_breakpoint_interception (via pexpect).
+    - env: TOXENV=py37-pexpect,py37-twisted
+      python: '3.7.4'
+
+    # Run also non-verbosely, to gain coverage
+    - env: TOXENV=py38-xdist PYTEST_ADDOPTS=""
+      python: '3.8'
+
+    - env: TOXENV=linting,docs,doctesting
       cache:
         directories:
           - $HOME/.cache/pre-commit
 
-    - stage: deploy
-      python: '3.6'
-      install: pip install -U setuptools setuptools_scm tox
-      script: skip
-      # token to upload github release notes: GH_RELEASE_NOTES_TOKEN
-      env:
-        - secure: "OjOeL7/0JUDkV00SsTs732e8vQjHynpbG9FKTNtZZJ+1Zn4Cib+hAlwmlBnvVukML0X60YpcfjnC4quDOIGLPsh5zeXnvJmYtAIIUNQXjWz8NhcGYrhyzuP1rqV22U68RTCdmOq3lMYU/W2acwHP7T49PwJtOiUM5kF120UAQ0Zi5EmkqkIvH8oM5mO9Dlver+/U7Htpz9rhKrHBXQNCMZI6yj2aUyukqB2PN2fjAlDbCF//+FmvYw9NjT4GeFOSkTCf4ER9yfqs7yglRfwiLtOCZ2qKQhWZNsSJDB89rxIRXWavJUjJKeY2EW2/NkomYJDpqJLIF4JeFRw/HhA47CYPeo6BJqyyNV+0CovL1frpWfi9UQw2cMbgFUkUIUk3F6DD59PHNIOX2R/HX56dQsw7WKl3QuHlCOkICXYg8F7Ta684IoKjeTX03/6QNOkURfDBwfGszY0FpbxrjCSWKom6RyZdyidnESaxv9RzjcIRZVh1rp8KMrwS1OrwRSdG0zjlsPr49hWMenN/8fKgcHTV4/r1Tj6mip0dorSRCrgUNIeRBKgmui6FS8642ab5JNKOxMteVPVR2sFuhjOQ0Jy+PmvceYY9ZMWc3+/B/KVh0dZ3hwvLGZep/vxDS2PwCA5/xw31714vT5LxidKo8yECjBynMU/wUTTS695D3NY="
-      addons:
-        apt:
-          packages:
-            # required by publish_gh_release_notes
-            - pandoc
-      after_deploy: tox -e publish_gh_release_notes
-      deploy:
-        provider: pypi
-        user: nicoddemus
-        distributions: sdist bdist_wheel
-        skip_upload_docs: true
-        password:
-          secure: xanTgTUu6XDQVqB/0bwJQXoDMnU5tkwZc5koz6mBkkqZhKdNOi2CLoC1XhiSZ+ah24l4V1E0GAqY5kBBcy9d7NVe4WNg4tD095LsHw+CRU6/HCVIFfyk2IZ+FPAlguesCcUiJSXOrlBF+Wj68wEvLoK7EoRFbJeiZ/f91Ww1sbtDlqXABWGHrmhPJL5Wva7o7+wG7JwJowqdZg1pbQExsCc7b53w4v2RBu3D6TJaTAzHiVsW+nUSI67vKI/uf+cR/OixsTfy37wlHgSwihYmrYLFls3V0bSpahCim3bCgMaFZx8S8xrdgJ++PzBCof2HeflFKvW+VCkoYzGEG4NrTWJoNz6ni4red9GdvfjGH3YCjAKS56h9x58zp2E5rpsb/kVq5/45xzV+dq6JRuhQ1nJWjBC6fSKAc/bfwnuFK3EBxNLkvBssLHvsNjj5XG++cB8DdS9wVGUqjpoK4puaXUWFqy4q3S9F86HEsKNgExtieA9qNx+pCIZVs6JCXZNjr0I5eVNzqJIyggNgJG6RyravsU35t9Zd9doL5g4Y7UKmAGTn1Sz24HQ4sMQgXdm2SyD8gEK5je4tlhUvfGtDvMSlstq71kIn9nRpFnqB6MFlbYSEAZmo8dGbCquoUc++6Rum208wcVbrzzVtGlXB/Ow9AbFMYeAGA0+N/K1e59c=
-        on:
-          tags: true
-          repo: pytest-dev/pytest
-
-matrix:
-  allow_failures:
-    - python: '3.8-dev'
-      env: TOXENV=py38-xdist
-    # Temporary (https://github.com/pytest-dev/pytest/pull/5334).
-    - env: TOXENV=pypy3-xdist
-      python: 'pypy3'
-
 before_script:
   - |
     # Do not (re-)upload coverage with cron runs.
@@ -142,27 +68,13 @@ before_script:
       export _PYTEST_TOX_COVERAGE_RUN="coverage run -m"
       export _PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess
     fi
-
+script: env COLUMNS=120 python -m tox
-script: tox
 
 after_success:
   - |
     if [[ "$PYTEST_COVERAGE" = 1 ]]; then
-      set -e
+      env CODECOV_NAME="$TOXENV-$TRAVIS_OS_NAME" scripts/report-coverage.sh
-      # Add last TOXENV to $PATH.
-      PATH="$PWD/.tox/${TOXENV##*,}/bin:$PATH"
-      coverage combine
-      coverage xml
-      coverage report -m
-      bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV-$TRAVIS_OS_NAME
     fi
-
+branches:
-notifications:
+  only:
-  irc:
+    - 4.6.x
-    channels:
-      - "chat.freenode.net#pytest"
-    on_success: change
-    on_failure: change
-    skip_join: true
-  email:
-    - pytest-commit@python.org

scripts/append_codecov_token.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+Appends the codecov token to the 'codecov.yml' file at the root of the repository.
+This is done by CI during PRs and builds on the pytest-dev repository so we can upload coverage, at least
+until codecov grows some native integration like it has with Travis and AppVeyor.
+See discussion in https://github.com/pytest-dev/pytest/pull/6441 for more information.
+"""
+import os.path
+from textwrap import dedent
+
+
+def main():
+    this_dir = os.path.dirname(__file__)
+    cov_file = os.path.join(this_dir, "..", "codecov.yml")
+
+    assert os.path.isfile(cov_file), "{cov_file} does not exist".format(
+        cov_file=cov_file
+    )
+
+    with open(cov_file, "a") as f:
+        # token from: https://codecov.io/gh/pytest-dev/pytest/settings
+        # use same URL to regenerate it if needed
+        text = dedent(
+            """
+            codecov:
+              token: "1eca3b1f-31a2-4fb8-a8c3-138b441b50a7"
+        """
+        )
+        f.write(text)
+
+    print("Token updated:", cov_file)
+
+
+if __name__ == "__main__":
+    main()

scripts/report-coverage.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+if [ -z "$TOXENV" ]; then
+  python -m pip install coverage
+else
+  # Add last TOXENV to $PATH.
+  PATH="$PWD/.tox/${TOXENV##*,}/bin:$PATH"
+fi
+
+python -m coverage combine
+python -m coverage xml
+python -m coverage report -m
+# Set --connect-timeout to work around https://github.com/curl/curl/issues/4461
+curl -S -L --connect-timeout 5 --retry 6 -s https://codecov.io/bash -o codecov-upload.sh
+bash codecov-upload.sh -Z -X fix -f coverage.xml "$@"

tox.ini

@@ -119,6 +119,7 @@ changedir = testing/freeze
 # Disable PEP 517 with pip, which does not work with PyInstaller currently.
 deps =
     pyinstaller
+    setuptools < 45.0.0
 commands =
     {envpython} create_executable.py
     {envpython} tox_run.py