diff --git a/.github/actions/upload-blob-report/action.yml b/.github/actions/upload-blob-report/action.yml index 72a084142a..e407e67573 100644 --- a/.github/actions/upload-blob-report/action.yml +++ b/.github/actions/upload-blob-report/action.yml @@ -22,13 +22,3 @@ runs: name: blob-report-${{ inputs.job_name }} path: ${{ inputs.report_dir }}/** retention-days: 7 - - name: Write triggering pull request number in a file - if: ${{ !cancelled() && github.event_name == 'pull_request' }} - shell: bash - run: echo '${{ github.event.number }}' > pull_request_number.txt; - - name: Upload artifact with the pull request number - if: ${{ !cancelled() && github.event_name == 'pull_request' }} - uses: actions/upload-artifact@v4 - with: - name: pull-request-${{ inputs.job_name }} - path: pull_request_number.txt \ No newline at end of file diff --git a/.github/workflows/create_test_report.yml b/.github/workflows/create_test_report.yml index dfc6961c2c..8393ef446a 100644 --- a/.github/workflows/create_test_report.yml +++ b/.github/workflows/create_test_report.yml @@ -35,7 +35,9 @@ jobs: run: | npx playwright merge-reports --config .github/workflows/merge.config.ts ./all-blob-reports env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NODE_OPTIONS: --max-old-space-size=8192 + HTML_REPORT_URL: 'https://mspwblobreport.z1.web.core.windows.net/run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}/index.html' - name: Azure Login uses: azure/login@v2 @@ -51,73 +53,3 @@ jobs: echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/index.html" env: AZCOPY_AUTO_LOGIN_TYPE: AZCLI - - - name: Read pull request number - uses: ./.github/actions/download-artifact - with: - namePrefix: 'pull-request' - path: '.' - - - name: Comment on PR - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const fs = require('fs'); - let prNumber; - if (context.payload.workflow_run.event === 'pull_request') { - const prs = context.payload.workflow_run.pull_requests; - if (prs.length) { - prNumber = prs[0].number; - } else { - prNumber = parseInt(fs.readFileSync('pull_request_number.txt').toString()); - console.log('Read pull request number from file: ' + prNumber); - } - } else { - core.error('Unsupported workflow trigger event: ' + context.payload.workflow_run.event); - return; - } - if (!prNumber) { - core.error('No pull request found for commit ' + context.sha + ' and workflow triggered by: ' + context.payload.workflow_run.event); - return; - } - { - // Mark previous comments as outdated by minimizing them. - const { data: comments } = await github.rest.issues.listComments({ - ...context.repo, - issue_number: prNumber, - }); - for (const comment of comments) { - if (comment.user.login === 'github-actions[bot]' && /\[Test results\]\(https:\/\/.+?\) for "${{ github.event.workflow_run.name }}"/.test(comment.body)) { - await github.graphql(` - mutation { - minimizeComment(input: {subjectId: "${comment.node_id}", classifier: OUTDATED}) { - clientMutationId - } - } - `); - } - } - } - const reportDir = 'run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}'; - const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/${reportDir}/index.html#?q=s%3Afailed%20s%3Aflaky`; - core.notice('Report url: ' + reportUrl); - const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const reportMd = await fs.promises.readFile('report.md', 'utf8'); - function formatComment(lines) { - let body = lines.join('\n'); - if (body.length > 65535) - body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`; - return body; - } - const { data: response } = await github.rest.issues.createComment({ - ...context.repo, - issue_number: prNumber, - body: formatComment([ - `### [Test results](${reportUrl}) for "${{ github.event.workflow_run.name }}"`, - reportMd, - '', - `Merge [workflow run](${mergeWorkflowUrl}).` - ]), - }); - core.info('Posted comment: ' + response.html_url); diff --git a/.github/workflows/merge.config.ts b/.github/workflows/merge.config.ts index e8582ed521..0ef418b3b2 100644 --- a/.github/workflows/merge.config.ts +++ b/.github/workflows/merge.config.ts @@ -1,4 +1,4 @@ export default { testDir: '../../tests', - reporter: [[require.resolve('../../packages/playwright/lib/reporters/markdown')], ['html']] + reporter: [[require.resolve('../../tests/config/ghaMarkdownReporter.ts')], ['html']] }; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index d7b06db081..f17a65797e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ ], "devDependencies": { "@actions/core": "^1.10.0", + "@actions/github": "^6.0.0", "@babel/cli": "^7.26.4", "@babel/code-frame": "^7.26.2", "@babel/plugin-transform-class-properties": "^7.25.9", @@ -96,6 +97,19 @@ "@actions/io": "^1.0.1" } }, + "node_modules/@actions/github": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz", + "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@actions/http-client": "^2.2.0", + "@octokit/core": "^5.0.1", + "@octokit/plugin-paginate-rest": "^9.0.0", + "@octokit/plugin-rest-endpoint-methods": "^10.0.0" + } + }, "node_modules/@actions/http-client": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz", @@ -1523,6 +1537,178 @@ "node": ">= 8" } }, + "node_modules/@octokit/auth-token": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz", + "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.1.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/endpoint": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz", + "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz", + "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/request": "^8.4.1", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "24.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", + "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-rest": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz", + "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^12.6.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": "5" + } + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz", + "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^12.6.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": "5" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" + } + }, + "node_modules/@octokit/request": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz", + "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/endpoint": "^9.0.6", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/request-error": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz", + "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/types": { + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", + "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^24.2.0" + } + }, "node_modules/@playwright/browser-chromium": { "resolved": "packages/playwright-browser-chromium", "link": true @@ -2804,6 +2990,13 @@ "dev": true, "license": "MIT" }, + "node_modules/before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -3434,6 +3627,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==", + "dev": true, + "license": "ISC" + }, "node_modules/detect-node": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", @@ -7662,6 +7862,13 @@ "devOptional": true, "license": "MIT" }, + "node_modules/universal-user-agent": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", + "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==", + "dev": true, + "license": "ISC" + }, "node_modules/universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", diff --git a/package.json b/package.json index e97dd26745..d20a27e554 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ ], "devDependencies": { "@actions/core": "^1.10.0", + "@actions/github": "^6.0.0", "@babel/cli": "^7.26.4", "@babel/code-frame": "^7.26.2", "@babel/plugin-transform-class-properties": "^7.25.9", diff --git a/packages/playwright/src/internalsForTest.ts b/packages/playwright/src/internalsForTest.ts index 129f9e858d..1cdd2cedb4 100644 --- a/packages/playwright/src/internalsForTest.ts +++ b/packages/playwright/src/internalsForTest.ts @@ -17,6 +17,7 @@ import path from 'path'; import { fileDependenciesForTest } from './transform/compilationCache'; +export { default as MarkdownReporter } from './reporters/markdown'; export function fileDependencies() { return Object.fromEntries([...fileDependenciesForTest().entries()].map(entry => ( diff --git a/packages/playwright/src/reporters/markdown.ts b/packages/playwright/src/reporters/markdown.ts index 8b2ea93dac..5a2d1d4203 100644 --- a/packages/playwright/src/reporters/markdown.ts +++ b/packages/playwright/src/reporters/markdown.ts @@ -69,9 +69,13 @@ class MarkdownReporter extends TerminalReporter { lines.push(`:heavy_check_mark::heavy_check_mark::heavy_check_mark:`); lines.push(``); + await this.publishReport(lines.join('\n')); + } + + protected async publishReport(report: string): Promise { const reportFile = resolveReporterOutputPath('report.md', this._options.configDir, this._options.outputFile); await fs.promises.mkdir(path.dirname(reportFile), { recursive: true }); - await fs.promises.writeFile(reportFile, lines.join('\n')); + await fs.promises.writeFile(reportFile, report); } private _printTestList(prefix: string, tests: TestCase[], lines: string[], suffix?: string) { diff --git a/tests/config/ghaMarkdownReporter.ts b/tests/config/ghaMarkdownReporter.ts new file mode 100644 index 0000000000..37ddcccd63 --- /dev/null +++ b/tests/config/ghaMarkdownReporter.ts @@ -0,0 +1,103 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MarkdownReporter } from 'playwright/lib/internalsForTest'; +import { context, getOctokit } from '@actions/github'; +import * as core from '@actions/core'; + +import type { MetadataWithCommitInfo } from 'playwright/src/isomorphic/types'; + +function getGithubToken() { + const token = process.env.GITHUB_TOKEN || core.getInput('github-token'); + if (!token) { + core.setFailed('Missing "github-token" input'); + throw new Error('Missing "github-token" input'); + } + return token; +} + +const octokit = getOctokit(getGithubToken()); + +const magicComment = ''; + +class GHAMarkdownReporter extends MarkdownReporter { + // declare config: FullConfig; + + async publishReport(report: string) { + core.info('Publishing report to PR.'); + const { prNumber, prHref } = this.pullRequestFromMetadata(); + if (!prNumber) { + core.info(`No PR number found, skipping GHA comment. PR href: ${prHref}`); + return; + } + core.info(`Posting comment to PR ${prHref}`); + + await this.collapsePreviousComments(prNumber); + await this.addNewReportComment(prNumber, report); + } + + private async collapsePreviousComments(prNumber: number) { + const { data: comments } = await octokit.rest.issues.listComments({ + ...context.repo, + issue_number: prNumber, + }); + for (const comment of comments) { + if (comment.user.login === 'github-actions[bot]' && comment.body.includes(magicComment)) { + core.info(`Minimizing comment: ${comment.html_url}`); + await octokit.graphql(` + mutation { + minimizeComment(input: {subjectId: "${comment.node_id}", classifier: OUTDATED}) { + clientMutationId + } + } + `); + } + } + } + + private async addNewReportComment(prNumber: number, report: string) { + const reportUrl = process.env.HTML_REPORT_URL; + const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + + const { data: response } = await octokit.rest.issues.createComment({ + ...context.repo, + issue_number: prNumber, + body: formatComment([ + magicComment, + `### [Test results](${reportUrl}) for "${context.payload.workflow_run?.name}"`, + report, + '', + `Merge [workflow run](${mergeWorkflowUrl}).` + ]), + }); + core.info(`Posted comment: ${response.html_url}`); + } + + private pullRequestFromMetadata() { + const metadata = this.config.metadata as MetadataWithCommitInfo; + const prHref = metadata.ci?.prHref; + return { prNumber: parseInt(prHref?.split('/').pop(), 10), prHref }; + } +} + +function formatComment(lines) { + let body = lines.join('\n'); + if (body.length > 65535) + body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`; + return body; +} + +export default GHAMarkdownReporter;