chore: introduce accessibility tab in recorder (#33235)

This commit is contained in:
Pavel Feldman 2024-10-22 16:36:03 -07:00 committed by GitHub
parent 6800fd45a2
commit 6bfdad068c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 95 additions and 102 deletions

View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { Mode, Source } from '@recorder/recorderTypes';
import type { ElementInfo, Mode, Source } from '@recorder/recorderTypes';
import { gracefullyProcessExitDoNotHang } from '../utils/processLauncher';
import type { Browser } from './browser';
import type { BrowserContext } from './browserContext';
@ -221,9 +221,9 @@ class InspectingRecorderApp extends EmptyRecorderApp {
this._debugController = debugController;
}
override async setSelector(selector: string): Promise<void> {
const locator: string = asLocator(this._debugController._sdkLanguage, selector);
this._debugController.emit(DebugController.Events.InspectRequested, { selector, locator });
override async elementPicked(elementInfo: ElementInfo): Promise<void> {
const locator: string = asLocator(this._debugController._sdkLanguage, elementInfo.selector);
this._debugController.emit(DebugController.Events.InspectRequested, { selector: elementInfo.selector, locator });
}
override async setSources(sources: Source[]): Promise<void> {

View File

@ -53,7 +53,7 @@ export function generateAriaTree(rootElement: Element): AriaNode {
return;
const element = node as Element;
if (roleUtils.isElementIgnoredForAria(element))
if (roleUtils.isElementHiddenForAria(element))
return;
const visible = isElementVisible(element);
@ -281,7 +281,7 @@ export function renderAriaTree(ariaNode: AriaNode, options?: { noText?: boolean
const visit = (ariaNode: AriaNode | string, indent: string) => {
if (typeof ariaNode === 'string') {
if (!options?.noText)
lines.push(indent + '- text: ' + escapeYamlString(ariaNode));
lines.push(indent + '- text: ' + quoteYamlString(ariaNode));
return;
}
let line = `${indent}- ${ariaNode.role}`;
@ -308,7 +308,7 @@ export function renderAriaTree(ariaNode: AriaNode, options?: { noText?: boolean
const stringValue = !ariaNode.children.length || (ariaNode.children?.length === 1 && typeof ariaNode.children[0] === 'string');
if (stringValue) {
if (!options?.noText && ariaNode.children.length)
line += ': ' + escapeYamlString(ariaNode.children?.[0] as string);
line += ': ' + quoteYamlString(ariaNode.children?.[0] as string);
lines.push(line);
return;
}
@ -328,36 +328,10 @@ export function renderAriaTree(ariaNode: AriaNode, options?: { noText?: boolean
return lines.join('\n');
}
function escapeYamlString(str: string) {
if (str === '')
return '""';
const needQuotes = (
// Starts or ends with whitespace
/^\s|\s$/.test(str) ||
// Contains control characters
/[\x00-\x1f]/.test(str) ||
// Contains special YAML characters that could cause parsing issues
/[\[\]{}&*!,|>%@`]/.test(str) ||
// Contains a colon followed by a space (could be interpreted as a key-value pair)
/:\s/.test(str) ||
// Is a YAML boolean or null value
/^(?:y|Y|yes|Yes|YES|n|N|no|No|NO|true|True|TRUE|false|False|FALSE|null|Null|NULL|~)$/.test(str) ||
// Could be interpreted as a number
/^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/.test(str) ||
// Contains a newline character
/\n/.test(str) ||
// Starts with a special character
/^[\-?:,>|%@"`]/.test(str)
);
if (needQuotes) {
return `"${str
.replace(/\\/g, '\\\\')
.replace(/"/g, '\\"')
.replace(/\n/g, '\\n')
.replace(/\r/g, '\\r')}"`;
}
return str;
function quoteYamlString(str: string) {
return `"${str
.replace(/\\/g, '\\\\')
.replace(/"/g, '\\"')
.replace(/\n/g, '\\n')
.replace(/\r/g, '\\r')}"`;
}

View File

@ -85,7 +85,11 @@ class ConsoleAPI {
inspect: (selector: string) => this._inspect(selector),
selector: (element: Element) => this._selector(element),
generateLocator: (element: Element, language?: Language) => this._generateLocator(element, language),
ariaSnapshot: (element?: Element) => this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body),
ariaSnapshot: (element?: Element) => {
const snapshot = this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body);
// eslint-disable-next-line no-console
console.log(snapshot);
},
resume: () => this._resume(),
...new Locator(injectedScript, ''),
};

View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { Mode, OverlayState, UIState } from '@recorder/recorderTypes';
import type { ElementInfo, Mode, OverlayState, UIState } from '@recorder/recorderTypes';
import type * as actions from '@recorder/actions';
import type { InjectedScript } from '../injectedScript';
import { Recorder } from './recorder';
@ -24,7 +24,7 @@ interface Embedder {
__pw_recorderPerformAction(action: actions.PerformOnRecordAction): Promise<void>;
__pw_recorderRecordAction(action: actions.Action): Promise<void>;
__pw_recorderState(): Promise<UIState>;
__pw_recorderSetSelector(selector: string): Promise<void>;
__pw_recorderElementPicked(element: { selector: string, ariaSnapshot?: string }): Promise<void>;
__pw_recorderSetMode(mode: Mode): Promise<void>;
__pw_recorderSetOverlayState(state: OverlayState): Promise<void>;
__pw_refreshOverlay(): void;
@ -75,8 +75,8 @@ export class PollingRecorder implements RecorderDelegate {
await this._embedder.__pw_recorderRecordAction(action);
}
async setSelector(selector: string): Promise<void> {
await this._embedder.__pw_recorderSetSelector(selector);
async elementPicked(elementInfo: ElementInfo): Promise<void> {
await this._embedder.__pw_recorderElementPicked(elementInfo);
}
async setMode(mode: Mode): Promise<void> {

View File

@ -17,7 +17,7 @@
import type * as actions from '@recorder/actions';
import type { InjectedScript } from '../injectedScript';
import type { Point } from '../../../common/types';
import type { Mode, OverlayState, UIState } from '@recorder/recorderTypes';
import type { ElementInfo, Mode, OverlayState, UIState } from '@recorder/recorderTypes';
import type { ElementText } from '../selectorUtils';
import type { Highlight, HighlightOptions } from '../highlight';
import clipPaths from './clipPaths';
@ -25,7 +25,7 @@ import clipPaths from './clipPaths';
export interface RecorderDelegate {
performAction?(action: actions.PerformOnRecordAction): Promise<void>;
recordAction?(action: actions.Action): Promise<void>;
setSelector?(selector: string): Promise<void>;
elementPicked?(elementInfo: ElementInfo): Promise<void>;
setMode?(mode: Mode): Promise<void>;
setOverlayState?(state: OverlayState): Promise<void>;
highlightUpdated?(): void;
@ -85,7 +85,7 @@ class InspectTool implements RecorderTool {
if (event.button !== 0)
return;
if (this._hoveredModel?.selector)
this._commit(this._hoveredModel.selector);
this._commit(this._hoveredModel.selector, this._hoveredModel);
}
onContextMenu(event: MouseEvent) {
@ -93,13 +93,14 @@ class InspectTool implements RecorderTool {
&& this._hoveredSelectors && this._hoveredSelectors.length > 1) {
consumeEvent(event);
const selectors = this._hoveredSelectors;
const hoveredModel = this._hoveredModel;
this._hoveredModel.tooltipFooter = undefined;
this._hoveredModel.tooltipList = selectors.map(selector => this._recorder.injectedScript.utils.asLocator(this._recorder.state.language, selector));
this._hoveredModel.tooltipListItemSelected = (index: number | undefined) => {
if (index === undefined)
this._reset(true);
else
this._commit(selectors[index]);
this._commit(selectors[index], hoveredModel);
};
this._recorder.updateHighlight(this._hoveredModel, true);
}
@ -181,7 +182,7 @@ class InspectTool implements RecorderTool {
this._reset(false);
}
private _commit(selector: string) {
private _commit(selector: string, model: HighlightModel) {
if (this._assertVisibility) {
this._recorder.recordAction({
name: 'assertVisible',
@ -191,7 +192,7 @@ class InspectTool implements RecorderTool {
this._recorder.setMode('recording');
this._recorder.overlay?.flashToolSucceeded('assertingVisibility');
} else {
this._recorder.setSelector(selector);
this._recorder.elementPicked(selector, model);
}
}
@ -1314,8 +1315,9 @@ export class Recorder {
void this._delegate.setOverlayState?.(state);
}
setSelector(selector: string) {
void this._delegate.setSelector?.(selector);
elementPicked(selector: string, model: HighlightModel) {
const ariaSnapshot = this.injectedScript.ariaSnapshot(model.elements[0]);
void this._delegate.elementPicked?.({ selector, ariaSnapshot });
}
}

View File

@ -15,7 +15,7 @@
*/
import type * as channels from '@protocol/channels';
import type { CallLog, CallLogStatus, EventData, Mode, OverlayState, Source, UIState } from '@recorder/recorderTypes';
import type { CallLog, CallLogStatus, ElementInfo, EventData, Mode, OverlayState, Source, UIState } from '@recorder/recorderTypes';
import * as fs from 'fs';
import type { Point } from '../common/types';
import * as consoleApiSource from '../generated/consoleApiSource';
@ -168,9 +168,9 @@ export class Recorder implements InstrumentationListener, IRecorder {
return uiState;
});
await this._context.exposeBinding('__pw_recorderSetSelector', false, async ({ frame }, selector: string) => {
await this._context.exposeBinding('__pw_recorderElementPicked', false, async ({ frame }, elementInfo: ElementInfo) => {
const selectorChain = await generateFrameSelector(frame);
await this._recorderApp?.setSelector(buildFullSelector(selectorChain, selector), true);
await this._recorderApp?.elementPicked({ selector: buildFullSelector(selectorChain, elementInfo.selector), ariaSnapshot: elementInfo.ariaSnapshot }, true);
});
await this._context.exposeBinding('__pw_recorderSetMode', false, async ({ frame }, mode: Mode) => {
@ -256,12 +256,10 @@ export class Recorder implements InstrumentationListener, IRecorder {
this._currentCallsMetadata.set(metadata, sdkObject);
this._updateUserSources();
this.updateCallLog([metadata]);
if (isScreenshotCommand(metadata)) {
if (isScreenshotCommand(metadata))
this.hideHighlightedSelector();
} else if (metadata.params && metadata.params.selector) {
else if (metadata.params && metadata.params.selector)
this._highlightedSelector = metadata.params.selector;
this._recorderApp?.setSelector(this._highlightedSelector).catch(() => {});
}
}
async onAfterCall(sdkObject: SdkObject, metadata: CallMetadata) {

View File

@ -20,7 +20,7 @@ import type { Page } from '../page';
import { ProgressController } from '../progress';
import { EventEmitter } from 'events';
import { serverSideCallMetadata } from '../instrumentation';
import type { CallLog, Mode, Source } from '@recorder/recorderTypes';
import type { CallLog, ElementInfo, Mode, Source } from '@recorder/recorderTypes';
import { isUnderTest } from '../../utils';
import { mime } from '../../utilsBundle';
import { syncLocalStorageWithSettings } from '../launchApp';
@ -35,7 +35,7 @@ export class EmptyRecorderApp extends EventEmitter implements IRecorderApp {
async setPaused(paused: boolean): Promise<void> {}
async setMode(mode: Mode): Promise<void> {}
async setRunningFile(file: string | undefined): Promise<void> {}
async setSelector(selector: string, userGesture?: boolean): Promise<void> {}
async elementPicked(elementInfo: ElementInfo, userGesture?: boolean): Promise<void> {}
async updateCallLogs(callLogs: CallLog[]): Promise<void> {}
async setSources(sources: Source[]): Promise<void> {}
async setActions(actions: actions.ActionInContext[], sources: Source[]): Promise<void> {}
@ -158,18 +158,12 @@ export class RecorderApp extends EventEmitter implements IRecorderApp {
async setActions(actions: actions.ActionInContext[], sources: Source[]): Promise<void> {
}
async setSelector(selector: string, userGesture?: boolean): Promise<void> {
if (userGesture) {
if (this._recorder?.mode() === 'inspecting') {
this._recorder.setMode('standby');
this._page.bringToFront();
} else {
this._recorder?.setMode('recording');
}
}
await this._page.mainFrame().evaluateExpression(((data: { selector: string, userGesture?: boolean }) => {
window.playwrightSetSelector(data.selector, data.userGesture);
}).toString(), { isFunction: true }, { selector, userGesture }).catch(() => {});
async elementPicked(elementInfo: ElementInfo, userGesture?: boolean): Promise<void> {
if (userGesture)
this._page.bringToFront();
await this._page.mainFrame().evaluateExpression(((param: { elementInfo: ElementInfo, userGesture?: boolean }) => {
window.playwrightElementPicked(param.elementInfo, param.userGesture);
}).toString(), { isFunction: true }, { elementInfo, userGesture }).catch(() => {});
}
async updateCallLogs(callLogs: CallLog[]): Promise<void> {

View File

@ -15,7 +15,7 @@
*/
import type * as actions from '@recorder/actions';
import type { CallLog, Mode, Source } from '@recorder/recorderTypes';
import type { CallLog, Mode, Source, ElementInfo } from '@recorder/recorderTypes';
import type { EventEmitter } from 'events';
export interface IRecorder {
@ -29,7 +29,7 @@ export interface IRecorderApp extends EventEmitter {
setPaused(paused: boolean): Promise<void>;
setMode(mode: Mode): Promise<void>;
setRunningFile(file: string | undefined): Promise<void>;
setSelector(selector: string, userGesture?: boolean): Promise<void>;
elementPicked(elementInfo: ElementInfo, userGesture?: boolean): Promise<void>;
updateCallLogs(callLogs: CallLog[]): Promise<void>;
setSources(sources: Source[]): Promise<void>;
setActions(actions: actions.ActionInContext[], sources: Source[]): Promise<void>;

View File

@ -15,7 +15,7 @@
*/
import path from 'path';
import type { CallLog, Mode, Source } from '@recorder/recorderTypes';
import type { CallLog, ElementInfo, Mode, Source } from '@recorder/recorderTypes';
import { EventEmitter } from 'events';
import type { IRecorder, IRecorderApp, IRecorderAppFactory } from './recorderFrontend';
import { installRootRedirect, openTraceViewerApp, startTraceViewerServer } from '../trace/viewer/traceViewer';
@ -70,8 +70,8 @@ export class RecorderInTraceViewer extends EventEmitter implements IRecorderApp
this._transport.deliverEvent('setRunningFile', { file });
}
async setSelector(selector: string, userGesture?: boolean): Promise<void> {
this._transport.deliverEvent('setSelector', { selector, userGesture });
async elementPicked(elementInfo: ElementInfo, userGesture?: boolean): Promise<void> {
this._transport.deliverEvent('elementPicked', { elementInfo, userGesture });
}
async updateCallLogs(callLogs: CallLog[]): Promise<void> {

View File

@ -14,7 +14,7 @@
limitations under the License.
*/
import type { CallLog, Mode, Source } from './recorderTypes';
import type { CallLog, ElementInfo, Mode, Source } from './recorderTypes';
import { CodeMirrorWrapper } from '@web/components/codeMirrorWrapper';
import { SplitView } from '@web/components/splitView';
import { TabbedPane } from '@web/components/tabbedPane';
@ -44,6 +44,7 @@ export const Recorder: React.FC<RecorderProps> = ({
const [selectedFileId, setSelectedFileId] = React.useState<string | undefined>();
const [runningFileId, setRunningFileId] = React.useState<string | undefined>();
const [selectedTab, setSelectedTab] = React.useState<string>('log');
const [ariaSnapshot, setAriaSnapshot] = React.useState<string | undefined>();
const fileId = selectedFileId || runningFileId || sources[0]?.id;
@ -57,11 +58,18 @@ export const Recorder: React.FC<RecorderProps> = ({
}, [sources, fileId]);
const [locator, setLocator] = React.useState('');
window.playwrightSetSelector = (selector: string, focus?: boolean) => {
window.playwrightElementPicked = (elementInfo: ElementInfo, userGesture?: boolean) => {
const language = source.language;
if (focus)
setLocator(asLocator(language, elementInfo.selector));
setAriaSnapshot(elementInfo.ariaSnapshot);
if (userGesture && selectedTab !== 'locator' && selectedTab !== 'aria')
setSelectedTab('locator');
setLocator(asLocator(language, selector));
if (mode === 'inspecting' && selectedTab === 'aria') {
// Keep exploring aria.
} else {
window.dispatch({ event: 'setMode', params: { mode: mode === 'inspecting' ? 'standby' : 'recording' } }).catch(() => { });
}
};
window.playwrightSetRunningFile = setRunningFileId;
@ -94,7 +102,7 @@ export const Recorder: React.FC<RecorderProps> = ({
}, [paused]);
const onEditorChange = React.useCallback((selector: string) => {
if (mode === 'none')
if (mode === 'none' || mode === 'inspecting')
window.dispatch({ event: 'setMode', params: { mode: 'standby' } });
setLocator(selector);
window.dispatch({ event: 'selectorUpdated', params: { selector } });
@ -157,7 +165,7 @@ export const Recorder: React.FC<RecorderProps> = ({
sidebarSize={200}
main={<CodeMirrorWrapper text={source.text} language={source.language} highlight={source.highlight} revealLine={source.revealLine} readOnly={true} lineNumbers={true} />}
sidebar={<TabbedPane
rightToolbar={selectedTab === 'locator' ? [<ToolbarButton key={1} icon='files' title='Copy' onClick={() => copy(locator)} />] : []}
rightToolbar={selectedTab === 'locator' || selectedTab === 'aria' ? [<ToolbarButton key={1} icon='files' title='Copy' onClick={() => copy((selectedTab === 'locator' ? locator : ariaSnapshot) || '')} />] : []}
tabs={[
{
id: 'locator',
@ -169,6 +177,11 @@ export const Recorder: React.FC<RecorderProps> = ({
title: 'Log',
render: () => <CallLogView language={source.language} log={Array.from(log.values())} />
},
{
id: 'aria',
title: 'Accessibility',
render: () => <CodeMirrorWrapper text={ariaSnapshot || ''} language={'python'} readOnly={true} wrapLines={true} />
},
]}
selectedTab={selectedTab}
setSelectedTab={setSelectedTab}

View File

@ -29,6 +29,11 @@ export type Mode =
| 'assertingValue'
| 'assertingSnapshot';
export type ElementInfo = {
selector: string;
ariaSnapshot: string;
};
export type EventData = {
event:
| 'clear'
@ -98,7 +103,7 @@ declare global {
playwrightSetOverlayVisible: (visible: boolean) => void;
playwrightUpdateLogs: (callLogs: CallLog[]) => void;
playwrightSetRunningFile: (file: string | undefined) => void;
playwrightSetSelector: (selector: string, focus?: boolean) => void;
playwrightElementPicked: (elementInfo: ElementInfo, userGesture?: boolean) => void;
playwrightSourcesEchoForTest: Source[];
dispatch(data: any): Promise<void>;
}

View File

@ -19,6 +19,7 @@
"paths": {
"@isomorphic/*": ["../playwright-core/src/utils/isomorphic/*"],
"@protocol/*": ["../protocol/src/*"],
"@recorder/*": ["../recorder/src/*"],
"@web/*": ["../web/src/*"],
}
},

View File

@ -30,6 +30,7 @@ import { locatorOrSelectorAsSelector } from '@isomorphic/locatorParser';
import { TabbedPaneTab } from '@web/components/tabbedPane';
import { BrowserFrame } from './browserFrame';
import { ClickPointer } from './clickPointer';
import type { ElementInfo } from '@recorder/recorderTypes';
function findClosest<T>(items: T[], metric: (v: T) => number, target: number) {
return items.find((item, index) => {
@ -291,8 +292,8 @@ export const InspectModeController: React.FunctionComponent<{
testIdAttributeName,
overlay: { offsetX: 0 },
}, {
async setSelector(selector: string) {
setHighlightedLocator(asLocator(sdkLanguage, frameSelector + selector));
async elementPicked(elementInfo: ElementInfo) {
setHighlightedLocator(asLocator(sdkLanguage, frameSelector + elementInfo.selector));
},
highlightUpdated() {
for (const r of recorders) {

View File

@ -25,6 +25,7 @@ import 'codemirror-shadow-1/mode/python/python';
import 'codemirror-shadow-1/mode/clike/clike';
import 'codemirror-shadow-1/mode/markdown/markdown';
import 'codemirror-shadow-1/addon/mode/simple';
import 'codemirror-shadow-1/mode/yaml/yaml';
export type CodeMirror = typeof codemirrorType;
export default codemirror;

View File

@ -64,8 +64,8 @@ it('should snapshot list with accessible name', async ({ page }) => {
`);
await checkAndMatchSnapshot(page.locator('body'), `
- list "my list":
- listitem: one
- listitem: two
- listitem: "one"
- listitem: "two"
`);
});
@ -92,7 +92,7 @@ it('should allow text nodes', async ({ page }) => {
await checkAndMatchSnapshot(page.locator('body'), `
- heading "Microsoft" [level=1]
- text: Open source projects and samples from Microsoft
- text: "Open source projects and samples from Microsoft"
`);
});
@ -105,7 +105,7 @@ it('should snapshot details visibility', async ({ page }) => {
`);
await checkAndMatchSnapshot(page.locator('body'), `
- group: Summary
- group: "Summary"
`);
});
@ -145,10 +145,10 @@ it('should snapshot integration', async ({ page }) => {
await checkAndMatchSnapshot(page.locator('body'), `
- heading "Microsoft" [level=1]
- text: Open source projects and samples from Microsoft
- text: "Open source projects and samples from Microsoft"
- list:
- listitem:
- group: Verified
- group: "Verified"
- listitem:
- link "Sponsor"
`);
@ -164,7 +164,7 @@ it('should support multiline text', async ({ page }) => {
`);
await checkAndMatchSnapshot(page.locator('body'), `
- paragraph: Line 1 Line 2 Line 3
- paragraph: "Line 1 Line 2 Line 3"
`);
await expect(page.locator('body')).toMatchAriaSnapshot(`
- paragraph: |
@ -180,7 +180,7 @@ it('should concatenate span text', async ({ page }) => {
`);
await checkAndMatchSnapshot(page.locator('body'), `
- text: One Two Three
- text: "One Two Three"
`);
});
@ -190,7 +190,7 @@ it('should concatenate span text 2', async ({ page }) => {
`);
await checkAndMatchSnapshot(page.locator('body'), `
- text: One Two Three
- text: "One Two Three"
`);
});
@ -200,7 +200,7 @@ it('should concatenate div text with spaces', async ({ page }) => {
`);
await checkAndMatchSnapshot(page.locator('body'), `
- text: One Two Three
- text: "One Two Three"
`);
});
@ -362,12 +362,12 @@ it('should snapshot inner text', async ({ page }) => {
await checkAndMatchSnapshot(page.locator('body'), `
- listitem:
- text: a.test.ts
- text: "a.test.ts"
- button "Run"
- button "Show source"
- button "Watch"
- listitem:
- text: snapshot 30ms
- text: "snapshot 30ms"
- button "Run"
- button "Show source"
- button "Watch"
@ -382,6 +382,6 @@ it('should include pseudo codepoints', async ({ page, server }) => {
`);
await checkAndMatchSnapshot(page.locator('body'), `
- paragraph: \ueab2hello
- paragraph: "\ueab2hello"
`);
});

View File

@ -283,7 +283,7 @@ for (const webPackage of ['html-reporter', 'recorder', 'trace-viewer']) {
'vite',
'build',
...(watchMode ? ['--watch', '--minify=false'] : []),
...(withSourceMaps ? ['--sourcemap'] : []),
...(withSourceMaps ? ['--sourcemap=inline'] : []),
],
shell: true,
cwd: path.join(__dirname, '..', '..', 'packages', webPackage),
@ -299,7 +299,7 @@ steps.push({
'vite.sw.config.ts',
'build',
...(watchMode ? ['--watch', '--minify=false'] : []),
...(withSourceMaps ? ['--sourcemap'] : []),
...(withSourceMaps ? ['--sourcemap=inline'] : []),
],
shell: true,
cwd: path.join(__dirname, '..', '..', 'packages', 'trace-viewer'),