devops: add GitHub actions markdown reporter (#35406)

.github/actions/upload-blob-report/action.yml

@@ -22,13 +22,3 @@ runs:
         name: blob-report-${{ inputs.job_name }}
         path: ${{ inputs.report_dir }}/**
         retention-days: 7
-    - name: Write triggering pull request number in a file
-      if: ${{ !cancelled() && github.event_name == 'pull_request' }}
-      shell: bash
-      run: echo '${{ github.event.number }}' > pull_request_number.txt;
-    - name: Upload artifact with the pull request number
-      if: ${{ !cancelled() && github.event_name == 'pull_request' }}
-      uses: actions/upload-artifact@v4
-      with:
-        name: pull-request-${{ inputs.job_name }}
-        path: pull_request_number.txt

.github/workflows/create_test_report.yml

@@ -35,7 +35,9 @@ jobs:
       run: |
         npx playwright merge-reports --config .github/workflows/merge.config.ts ./all-blob-reports
       env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         NODE_OPTIONS: --max-old-space-size=8192
+        HTML_REPORT_URL: 'https://mspwblobreport.z1.web.core.windows.net/run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}/index.html'
 
     - name: Azure Login
       uses: azure/login@v2
@@ -51,73 +53,3 @@ jobs:
         echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/index.html"
       env:
         AZCOPY_AUTO_LOGIN_TYPE: AZCLI
-
-    - name: Read pull request number
-      uses: ./.github/actions/download-artifact
-      with:
-        namePrefix: 'pull-request'
-        path: '.'
-
-    - name: Comment on PR
-      uses: actions/github-script@v7
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-        script: |
-          const fs = require('fs');
-          let prNumber;
-          if (context.payload.workflow_run.event === 'pull_request') {
-            const prs = context.payload.workflow_run.pull_requests;
-            if (prs.length) {
-              prNumber = prs[0].number;
-            } else {
-              prNumber = parseInt(fs.readFileSync('pull_request_number.txt').toString());
-              console.log('Read pull request number from file: ' + prNumber);
-            }
-          } else {
-            core.error('Unsupported workflow trigger event: ' + context.payload.workflow_run.event);
-            return;
-          }
-          if (!prNumber) {
-            core.error('No pull request found for commit ' + context.sha + ' and workflow triggered by: ' + context.payload.workflow_run.event);
-            return;
-          }
-          {
-            // Mark previous comments as outdated by minimizing them.
-            const { data: comments } = await github.rest.issues.listComments({
-              ...context.repo,
-              issue_number: prNumber,
-            });
-            for (const comment of comments) {
-              if (comment.user.login === 'github-actions[bot]' && /\[Test results\]\(https:\/\/.+?\) for "${{ github.event.workflow_run.name }}"/.test(comment.body)) {
-                await github.graphql(`
-                  mutation {
-                    minimizeComment(input: {subjectId: "${comment.node_id}", classifier: OUTDATED}) {
-                      clientMutationId
-                    }
-                  }
-                `);
-              }
-            }
-          }
-          const reportDir = 'run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}';
-          const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/${reportDir}/index.html#?q=s%3Afailed%20s%3Aflaky`;
-          core.notice('Report url: ' + reportUrl);
-          const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
-          const reportMd = await fs.promises.readFile('report.md', 'utf8');
-          function formatComment(lines) {
-            let body = lines.join('\n');
-            if (body.length > 65535)
-              body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;
-            return body;
-          }
-          const { data: response } = await github.rest.issues.createComment({
-            ...context.repo,
-            issue_number: prNumber,
-            body: formatComment([
-              `### [Test results](${reportUrl}) for "${{ github.event.workflow_run.name }}"`,
-              reportMd,
-              '',
-              `Merge [workflow run](${mergeWorkflowUrl}).`
-            ]),
-          });
-          core.info('Posted comment: ' + response.html_url);

.github/workflows/merge.config.ts

@@ -1,4 +1,4 @@
 export default {
   testDir: '../../tests',
-  reporter: [[require.resolve('../../packages/playwright/lib/reporters/markdown')], ['html']]
+  reporter: [[require.resolve('../../tests/config/ghaMarkdownReporter.ts')], ['html']]
 };

package-lock.json

@@ -13,6 +13,7 @@
       ],
       "devDependencies": {
         "@actions/core": "^1.10.0",
+        "@actions/github": "^6.0.0",
         "@babel/cli": "^7.26.4",
         "@babel/code-frame": "^7.26.2",
         "@babel/plugin-transform-class-properties": "^7.25.9",
@@ -96,6 +97,19 @@
         "@actions/io": "^1.0.1"
       }
     },
+    "node_modules/@actions/github": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz",
+      "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@actions/http-client": "^2.2.0",
+        "@octokit/core": "^5.0.1",
+        "@octokit/plugin-paginate-rest": "^9.0.0",
+        "@octokit/plugin-rest-endpoint-methods": "^10.0.0"
+      }
+    },
     "node_modules/@actions/http-client": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz",
@@ -1523,6 +1537,178 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@octokit/auth-token": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz",
+      "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/core": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz",
+      "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/auth-token": "^4.0.0",
+        "@octokit/graphql": "^7.1.0",
+        "@octokit/request": "^8.4.1",
+        "@octokit/request-error": "^5.1.1",
+        "@octokit/types": "^13.0.0",
+        "before-after-hook": "^2.2.0",
+        "universal-user-agent": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/endpoint": {
+      "version": "9.0.6",
+      "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz",
+      "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/types": "^13.1.0",
+        "universal-user-agent": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/graphql": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz",
+      "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/request": "^8.4.1",
+        "@octokit/types": "^13.0.0",
+        "universal-user-agent": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/openapi-types": {
+      "version": "24.2.0",
+      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz",
+      "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/@octokit/plugin-paginate-rest": {
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz",
+      "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/types": "^12.6.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "@octokit/core": "5"
+      }
+    },
+    "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": {
+      "version": "20.0.0",
+      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",
+      "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": {
+      "version": "12.6.0",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",
+      "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/openapi-types": "^20.0.0"
+      }
+    },
+    "node_modules/@octokit/plugin-rest-endpoint-methods": {
+      "version": "10.4.1",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz",
+      "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/types": "^12.6.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      },
+      "peerDependencies": {
+        "@octokit/core": "5"
+      }
+    },
+    "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": {
+      "version": "20.0.0",
+      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",
+      "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": {
+      "version": "12.6.0",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",
+      "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/openapi-types": "^20.0.0"
+      }
+    },
+    "node_modules/@octokit/request": {
+      "version": "8.4.1",
+      "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz",
+      "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/endpoint": "^9.0.6",
+        "@octokit/request-error": "^5.1.1",
+        "@octokit/types": "^13.1.0",
+        "universal-user-agent": "^6.0.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/request-error": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz",
+      "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/types": "^13.1.0",
+        "deprecation": "^2.0.0",
+        "once": "^1.4.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@octokit/types": {
+      "version": "13.10.0",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz",
+      "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/openapi-types": "^24.2.0"
+      }
+    },
     "node_modules/@playwright/browser-chromium": {
       "resolved": "packages/playwright-browser-chromium",
       "link": true
@@ -2804,6 +2990,13 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/before-after-hook": {
+      "version": "2.2.3",
+      "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz",
+      "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==",
+      "dev": true,
+      "license": "Apache-2.0"
+    },
     "node_modules/binary-extensions": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz",
@@ -3434,6 +3627,13 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/deprecation": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
+      "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==",
+      "dev": true,
+      "license": "ISC"
+    },
     "node_modules/detect-node": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
@@ -7662,6 +7862,13 @@
       "devOptional": true,
       "license": "MIT"
     },
+    "node_modules/universal-user-agent": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz",
+      "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==",
+      "dev": true,
+      "license": "ISC"
+    },
     "node_modules/universalify": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",

package.json

@@ -52,6 +52,7 @@
   ],
   "devDependencies": {
     "@actions/core": "^1.10.0",
+    "@actions/github": "^6.0.0",
     "@babel/cli": "^7.26.4",
     "@babel/code-frame": "^7.26.2",
     "@babel/plugin-transform-class-properties": "^7.25.9",

packages/playwright/src/internalsForTest.ts

@@ -17,6 +17,7 @@
 import path from 'path';
 
 import { fileDependenciesForTest } from './transform/compilationCache';
+export { default as MarkdownReporter } from './reporters/markdown';
 
 export function fileDependencies() {
   return Object.fromEntries([...fileDependenciesForTest().entries()].map(entry => (

packages/playwright/src/reporters/markdown.ts

@@ -69,9 +69,13 @@ class MarkdownReporter extends TerminalReporter {
     lines.push(`:heavy_check_mark::heavy_check_mark::heavy_check_mark:`);
     lines.push(``);
 
+    await this.publishReport(lines.join('\n'));
+  }
+
+  protected async publishReport(report: string): Promise<void> {
     const reportFile = resolveReporterOutputPath('report.md', this._options.configDir, this._options.outputFile);
     await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });
-    await fs.promises.writeFile(reportFile, lines.join('\n'));
+    await fs.promises.writeFile(reportFile, report);
   }
 
   private _printTestList(prefix: string, tests: TestCase[], lines: string[], suffix?: string) {

tests/config/ghaMarkdownReporter.ts

@@ -0,0 +1,103 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MarkdownReporter } from 'playwright/lib/internalsForTest';
+import { context, getOctokit } from '@actions/github';
+import * as core from '@actions/core';
+
+import type { MetadataWithCommitInfo } from 'playwright/src/isomorphic/types';
+
+function getGithubToken() {
+  const token = process.env.GITHUB_TOKEN || core.getInput('github-token');
+  if (!token) {
+    core.setFailed('Missing "github-token" input');
+    throw new Error('Missing "github-token" input');
+  }
+  return token;
+}
+
+const octokit = getOctokit(getGithubToken());
+
+const magicComment = '<!-- Generated by Playwright markdown reporter -->';
+
+class GHAMarkdownReporter extends MarkdownReporter {
+  // declare config: FullConfig;
+
+  async publishReport(report: string) {
+    core.info('Publishing report to PR.');
+    const { prNumber, prHref } = this.pullRequestFromMetadata();
+    if (!prNumber) {
+      core.info(`No PR number found, skipping GHA comment. PR href: ${prHref}`);
+      return;
+    }
+    core.info(`Posting comment to PR ${prHref}`);
+
+    await this.collapsePreviousComments(prNumber);
+    await this.addNewReportComment(prNumber, report);
+  }
+
+  private async collapsePreviousComments(prNumber: number) {
+    const { data: comments } = await octokit.rest.issues.listComments({
+      ...context.repo,
+      issue_number: prNumber,
+    });
+    for (const comment of comments) {
+      if (comment.user.login === 'github-actions[bot]' && comment.body.includes(magicComment)) {
+        core.info(`Minimizing comment: ${comment.html_url}`);
+        await octokit.graphql(`
+          mutation {
+            minimizeComment(input: {subjectId: "${comment.node_id}", classifier: OUTDATED}) {
+              clientMutationId
+            }
+          }
+        `);
+      }
+    }
+  }
+
+  private async addNewReportComment(prNumber: number, report: string) {
+    const reportUrl = process.env.HTML_REPORT_URL;
+    const mergeWorkflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+
+    const { data: response } = await octokit.rest.issues.createComment({
+      ...context.repo,
+      issue_number: prNumber,
+      body: formatComment([
+        magicComment,
+        `### [Test results](${reportUrl}) for "${context.payload.workflow_run?.name}"`,
+        report,
+        '',
+        `Merge [workflow run](${mergeWorkflowUrl}).`
+      ]),
+    });
+    core.info(`Posted comment:  ${response.html_url}`);
+  }
+
+  private pullRequestFromMetadata() {
+    const metadata = this.config.metadata as MetadataWithCommitInfo;
+    const prHref = metadata.ci?.prHref;
+    return { prNumber: parseInt(prHref?.split('/').pop(), 10), prHref };
+  }
+}
+
+function formatComment(lines) {
+  let body = lines.join('\n');
+  if (body.length > 65535)
+    body = body.substring(0, 65000) + `... ${body.length - 65000} more characters`;
+  return body;
+}
+
+export default GHAMarkdownReporter;