Add discrepancy handling to A11yPanel

code/addons/a11y/src/components/A11YPanel.tsx

@@ -8,6 +8,7 @@ import { CheckIcon, SyncIcon } from '@storybook/icons';
 import { useA11yContext } from './A11yContext';
 import { Report } from './Report';
 import { Tabs } from './Tabs';
+import { TestDiscrepancyMessage } from './TestDiscrepancyMessage';
 
 export enum RuleType {
   VIOLATION,
@@ -43,7 +44,7 @@ const Centered = styled.span({
 });
 
 export const A11YPanel: React.FC = () => {
-  const { results, status, handleManual, error } = useA11yContext();
+  const { results, status, handleManual, error, discrepancy } = useA11yContext();
 
   const manualActionItems = useMemo(
     () => [{ title: 'Run test', onClick: handleManual }],
@@ -106,31 +107,35 @@ export const A11YPanel: React.FC = () => {
 
   return (
     <>
-      {status === 'initial' && <Centered>Initializing...</Centered>}
+      {discrepancy && <TestDiscrepancyMessage discrepancy={discrepancy} />}
-      {status === 'manual' && (
+      {status === 'ready' || status === 'ran' ? (
-        <>
-          <Centered>Manually run the accessibility scan.</Centered>
-          <ActionBar key="actionbar" actionItems={manualActionItems} />
-        </>
-      )}
-      {status === 'running' && (
-        <Centered>
-          <RotatingIcon size={12} /> Please wait while the accessibility scan is running ...
-        </Centered>
-      )}
-      {(status === 'ready' || status === 'ran') && (
         <>
           <ScrollArea vertical horizontal>
             <Tabs key="tabs" tabs={tabs} />
           </ScrollArea>
           <ActionBar key="actionbar" actionItems={readyActionItems} />
         </>
-      )}
+      ) : (
-      {status === 'error' && (
+        <Centered style={{ marginTop: discrepancy ? '1em' : 0 }}>
-        <Centered>
+          {status === 'initial' && 'Initializing...'}
-          The accessibility scan encountered an error.
+          {status === 'manual' && (
-          <br />
+            <>
-          {typeof error === 'string' ? error : JSON.stringify(error)}
+              <>Manually run the accessibility scan.</>
+              <ActionBar key="actionbar" actionItems={manualActionItems} />
+            </>
+          )}
+          {status === 'running' && (
+            <>
+              <RotatingIcon size={12} /> Please wait while the accessibility scan is running ...
+            </>
+          )}
+          {status === 'error' && (
+            <>
+              The accessibility scan encountered an error.
+              <br />
+              {typeof error === 'string' ? error : JSON.stringify(error)}
+            </>
+          )}
         </Centered>
       )}
     </>

code/addons/a11y/src/components/A11yContext.tsx

@@ -1,5 +1,5 @@
 import type { FC, PropsWithChildren } from 'react';
-import React, { createContext, useCallback, useContext, useEffect, useState } from 'react';
+import React, { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react';
 
 import {
   STORY_FINISHED,
@@ -10,6 +10,7 @@ import {
   useAddonState,
   useChannel,
   useParameter,
+  useStorybookApi,
   useStorybookState,
 } from 'storybook/internal/manager-api';
 import type { Report } from 'storybook/internal/preview-api';
@@ -19,9 +20,10 @@ import { HIGHLIGHT } from '@storybook/addon-highlight';
 
 import type { AxeResults, Result } from 'axe-core';
 
-import { ADDON_ID, EVENTS } from '../constants';
+import { ADDON_ID, EVENTS, TEST_PROVIDER_ID } from '../constants';
 import type { A11yParameters } from '../params';
 import type { A11YReport } from '../types';
+import type { TestDiscrepancy } from './TestDiscrepancyMessage';
 
 export interface Results {
   passes: Result[];
@@ -40,6 +42,7 @@ export interface A11yContextStore {
   setStatus: (status: Status) => void;
   error: unknown;
   handleManual: () => void;
+  discrepancy: TestDiscrepancy;
 }
 
 const colorsByType = [
@@ -63,6 +66,7 @@ export const A11yContext = createContext<A11yContextStore>({
   status: 'initial',
   error: undefined,
   handleManual: () => {},
+  discrepancy: null,
 });
 
 const defaultResult = {
@@ -80,12 +84,15 @@ export const A11yContextProvider: FC<PropsWithChildren> = (props) => {
 
   const getInitialStatus = useCallback((manual = false) => (manual ? 'manual' : 'initial'), []);
 
+  const api = useStorybookApi();
   const [results, setResults] = useAddonState<Results>(ADDON_ID, defaultResult);
   const [tab, setTab] = useState(0);
   const [error, setError] = React.useState<unknown>(undefined);
   const [status, setStatus] = useState<Status>(getInitialStatus(parameters.manual!));
   const [highlighted, setHighlighted] = useState<string[]>([]);
+
   const { storyId } = useStorybookState();
+  const storyStatus = api.getCurrentStoryStatus();
 
   const handleToggleHighlight = useCallback((target: string[], highlight: boolean) => {
     setHighlighted((prevHighlighted) =>
@@ -178,6 +185,28 @@ export const A11yContextProvider: FC<PropsWithChildren> = (props) => {
     emit(HIGHLIGHT, { elements: highlighted, color: colorsByType[tab] });
   }, [emit, highlighted, tab]);
 
+  const discrepancy: TestDiscrepancy = useMemo(() => {
+    const storyStatusA11y = storyStatus[TEST_PROVIDER_ID]?.status;
+
+    if (storyStatusA11y) {
+      if (storyStatusA11y === 'success' && results.violations.length > 0) {
+        return 'cliPassedBrowserFailed';
+      }
+
+      if (storyStatusA11y === 'error' && results.violations.length === 0) {
+        if (status === 'ready' || status === 'ran') {
+          return 'browserPassedCliFailed';
+        }
+
+        if (status === 'manual') {
+          return 'cliFailedButModeManual';
+        }
+      }
+    }
+
+    return null;
+  }, [results.violations.length, status, storyStatus]);
+
   return (
     <A11yContext.Provider
       value={{
@@ -191,6 +220,7 @@ export const A11yContextProvider: FC<PropsWithChildren> = (props) => {
         setStatus,
         error,
         handleManual,
+        discrepancy,
       }}
       {...props}
     />

code/addons/a11y/src/components/TestDiscrepancyMessage.tsx

@@ -0,0 +1,73 @@
+import React, { useMemo } from 'react';
+
+import { Link } from 'storybook/internal/components';
+import { useStorybookApi } from 'storybook/internal/manager-api';
+import { styled } from 'storybook/internal/theming';
+
+import { DOCUMENTATION_DISCREPANCY_LINK } from '../constants';
+
+const Wrapper = styled.div(({ theme: { color, typography, background } }) => ({
+  textAlign: 'start',
+  padding: '11px 15px',
+  fontSize: `${typography.size.s2}px`,
+  fontWeight: typography.weight.regular,
+  lineHeight: '1rem',
+  background: background.app,
+  borderBottom: `1px solid ${color.border}`,
+  color: color.defaultText,
+  backgroundClip: 'padding-box',
+  position: 'relative',
+  code: {
+    fontSize: `${typography.size.s1 - 1}px`,
+    color: 'inherit',
+    margin: '0 0.2em',
+    padding: '0 0.2em',
+    background: 'rgba(255, 255, 255, 0.8)',
+    borderRadius: '2px',
+    boxShadow: '0 0 0 1px rgba(0, 0, 0, 0.1)',
+  },
+}));
+
+export type TestDiscrepancy =
+  | 'browserPassedCliFailed'
+  | 'cliPassedBrowserFailed'
+  | 'cliFailedButModeManual'
+  | null;
+
+interface TestDiscrepancyMessageProps {
+  discrepancy: TestDiscrepancy;
+}
+export const TestDiscrepancyMessage = ({ discrepancy }: TestDiscrepancyMessageProps) => {
+  const api = useStorybookApi();
+  const docsUrl = api.getDocsUrl({
+    subpath: DOCUMENTATION_DISCREPANCY_LINK,
+    versioned: true,
+    renderer: true,
+  });
+
+  const message = useMemo(() => {
+    switch (discrepancy) {
+      case 'browserPassedCliFailed':
+        return 'Accessibility checks passed in this browser but failed in the CLI.';
+      case 'cliPassedBrowserFailed':
+        return 'Accessibility checks passed in the CLI but failed in this browser.';
+      case 'cliFailedButModeManual':
+        return 'Accessibility checks failed in the CLI. Run the tests manually to see the results.';
+      default:
+        return null;
+    }
+  }, [discrepancy]);
+
+  if (!message) {
+    return null;
+  }
+
+  return (
+    <Wrapper>
+      {message}{' '}
+      <Link href={docsUrl} target="_blank" withArrow>
+        Learn what could cause this
+      </Link>
+    </Wrapper>
+  );
+};

code/addons/a11y/src/constants.ts

@@ -7,4 +7,9 @@ const RUNNING = `${ADDON_ID}/running`;
 const ERROR = `${ADDON_ID}/error`;
 const MANUAL = `${ADDON_ID}/manual`;
 
+export const DOCUMENTATION_LINK = 'writing-tests/accessibility-testing';
+export const DOCUMENTATION_DISCREPANCY_LINK = `${DOCUMENTATION_LINK}#what-happens-when-there-are-different-results-in-multiple-environments`;
+
+export const TEST_PROVIDER_ID = 'storybook/addon-a11y/test-provider';
+
 export const EVENTS = { RESULT, REQUEST, RUNNING, ERROR, MANUAL };

code/addons/a11y/template/stories/A11YPanel.stories.tsx

@@ -1,22 +1,34 @@
 import React from 'react';
 
+import { ManagerContext } from 'storybook/internal/manager-api';
 import { ThemeProvider, convert, themes } from 'storybook/internal/theming';
 
 import type { Meta, StoryObj } from '@storybook/react';
 import { fn } from '@storybook/test';
 
+import type axe from 'axe-core';
+
 import { A11YPanel } from '../../src/components/A11YPanel';
 import { A11yContext } from '../../src/components/A11yContext';
 import type { A11yContextStore } from '../../src/components/A11yContext';
 
+const managerContext: any = {
+  state: {},
+  api: {
+    getDocsUrl: fn().mockName('api::getDocsUrl'),
+  },
+};
+
 const meta: Meta = {
   title: 'A11YPanel',
   component: A11YPanel,
   decorators: [
     (Story) => (
-      <ThemeProvider theme={convert(themes.light)}>
+      <ManagerContext.Provider value={managerContext}>
-        <Story />
+        <ThemeProvider theme={convert(themes.light)}>
-      </ThemeProvider>
+          <Story />
+        </ThemeProvider>
+      </ManagerContext.Provider>
     ),
   ],
 } satisfies Meta<typeof A11YPanel>;
@@ -25,7 +37,62 @@ export default meta;
 
 type Story = StoryObj<typeof meta>;
 
-const Template = (args: Pick<A11yContextStore, 'results' | 'error' | 'status'>) => (
+const violations: axe.Result[] = [
+  {
+    id: 'aria-command-name',
+    impact: 'serious',
+    tags: ['cat.aria', 'wcag2a', 'wcag412', 'TTv5', 'TT6.a', 'EN-301-549', 'EN-9.4.1.2', 'ACT'],
+    description: 'Ensures every ARIA button, link and menuitem has an accessible name',
+    help: 'ARIA commands must have an accessible name',
+    helpUrl: 'https://dequeuniversity.com/rules/axe/4.8/aria-command-name?application=axeAPI',
+    nodes: [
+      {
+        any: [
+          {
+            id: 'has-visible-text',
+            data: null,
+            relatedNodes: [],
+            impact: 'serious',
+            message: 'Element does not have text that is visible to screen readers',
+          },
+          {
+            id: 'aria-label',
+            data: null,
+            relatedNodes: [],
+            impact: 'serious',
+            message: 'aria-label attribute does not exist or is empty',
+          },
+          {
+            id: 'aria-labelledby',
+            data: null,
+            relatedNodes: [],
+            impact: 'serious',
+            message:
+              'aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty',
+          },
+          {
+            id: 'non-empty-title',
+            data: {
+              messageKey: 'noAttr',
+            },
+            relatedNodes: [],
+            impact: 'serious',
+            message: 'Element has no title attribute',
+          },
+        ],
+        all: [],
+        none: [],
+        impact: 'serious',
+        html: '<div role="button" class="css-12jpz5t">',
+        target: ['.css-12jpz5t'],
+        failureSummary:
+          'Fix any of the following:\n  Element does not have text that is visible to screen readers\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute',
+      },
+    ],
+  },
+];
+
+const Template = (args: Pick<A11yContextStore, 'results' | 'error' | 'status' | 'discrepancy'>) => (
   <A11yContext.Provider
     value={{
       handleManual: fn(),
@@ -49,6 +116,7 @@ export const Initializing: Story = {
         results={{ passes: [], incomplete: [], violations: [] }}
         status="initial"
         error={null}
+        discrepancy={null}
       />
     );
   },
@@ -61,6 +129,20 @@ export const Manual: Story = {
         results={{ passes: [], incomplete: [], violations: [] }}
         status="manual"
         error={null}
+        discrepancy={null}
+      />
+    );
+  },
+};
+
+export const ManualWithDiscrepancy: Story = {
+  render: () => {
+    return (
+      <Template
+        results={{ passes: [], incomplete: [], violations: [] }}
+        status="manual"
+        error={null}
+        discrepancy={'cliFailedButModeManual'}
       />
     );
   },
@@ -73,6 +155,7 @@ export const Running: Story = {
         results={{ passes: [], incomplete: [], violations: [] }}
         status="running"
         error={null}
+        discrepancy={null}
       />
     );
   },
@@ -85,73 +168,28 @@ export const ReadyWithResults: Story = {
         results={{
           passes: [],
           incomplete: [],
-          violations: [
+          violations,
-            {
-              id: 'aria-command-name',
-              impact: 'serious',
-              tags: [
-                'cat.aria',
-                'wcag2a',
-                'wcag412',
-                'TTv5',
-                'TT6.a',
-                'EN-301-549',
-                'EN-9.4.1.2',
-                'ACT',
-              ],
-              description: 'Ensures every ARIA button, link and menuitem has an accessible name',
-              help: 'ARIA commands must have an accessible name',
-              helpUrl:
-                'https://dequeuniversity.com/rules/axe/4.8/aria-command-name?application=axeAPI',
-              nodes: [
-                {
-                  any: [
-                    {
-                      id: 'has-visible-text',
-                      data: null,
-                      relatedNodes: [],
-                      impact: 'serious',
-                      message: 'Element does not have text that is visible to screen readers',
-                    },
-                    {
-                      id: 'aria-label',
-                      data: null,
-                      relatedNodes: [],
-                      impact: 'serious',
-                      message: 'aria-label attribute does not exist or is empty',
-                    },
-                    {
-                      id: 'aria-labelledby',
-                      data: null,
-                      relatedNodes: [],
-                      impact: 'serious',
-                      message:
-                        'aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty',
-                    },
-                    {
-                      id: 'non-empty-title',
-                      data: {
-                        messageKey: 'noAttr',
-                      },
-                      relatedNodes: [],
-                      impact: 'serious',
-                      message: 'Element has no title attribute',
-                    },
-                  ],
-                  all: [],
-                  none: [],
-                  impact: 'serious',
-                  html: '<div role="button" class="css-12jpz5t">',
-                  target: ['.css-12jpz5t'],
-                  failureSummary:
-                    'Fix any of the following:\n  Element does not have text that is visible to screen readers\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute',
-                },
-              ],
-            },
-          ],
         }}
         status="ready"
         error={null}
+        discrepancy={null}
+      />
+    );
+  },
+};
+
+export const ReadyWithResultsDiscrepancyCLIPassedBrowserFailed: Story = {
+  render: () => {
+    return (
+      <Template
+        results={{
+          passes: [],
+          incomplete: [],
+          violations,
+        }}
+        status="ready"
+        error={null}
+        discrepancy={'cliPassedBrowserFailed'}
       />
     );
   },
@@ -164,6 +202,7 @@ export const Error: Story = {
         results={{ passes: [], incomplete: [], violations: [] }}
         status="error"
         error="Test error message"
+        discrepancy={null}
       />
     );
   },
@@ -176,6 +215,7 @@ export const ErrorStateWithObject: Story = {
         results={{ passes: [], incomplete: [], violations: [] }}
         status="error"
         error={{ message: 'Test error object message' }}
+        discrepancy={null}
       />
     );
   },

code/addons/a11y/template/stories/TestDiscrepancyMessage.stories.tsx

@@ -0,0 +1,51 @@
+import React from 'react';
+
+import { ManagerContext } from 'storybook/internal/manager-api';
+
+import type { Meta, StoryObj } from '@storybook/react';
+import { fn } from '@storybook/test';
+
+import { TestDiscrepancyMessage } from '../../src/components/TestDiscrepancyMessage';
+
+type Story = StoryObj<typeof TestDiscrepancyMessage>;
+
+const managerContext: any = {
+  state: {},
+  api: {
+    getDocsUrl: fn().mockName('api::getDocsUrl'),
+  },
+};
+
+export default {
+  title: 'TestDiscrepancyMessage',
+  component: TestDiscrepancyMessage,
+  parameters: {
+    layout: 'fullscreen',
+  },
+  args: {
+    storyId: 'story-id',
+  },
+  decorators: [
+    (storyFn) => (
+      <ManagerContext.Provider value={managerContext}>{storyFn()}</ManagerContext.Provider>
+    ),
+  ],
+} as Meta<typeof TestDiscrepancyMessage>;
+
+export const BrowserPassedCliFailed: Story = {
+  args: {
+    discrepancy: 'browserPassedCliFailed',
+  },
+};
+
+export const CliPassedBrowserFailed: Story = {
+  args: {
+    discrepancy: 'cliPassedBrowserFailed',
+  },
+};
+
+export const CliFailedButModeManual: Story = {
+  args: {
+    discrepancy: 'cliFailedButModeManual',
+  },
+};

code/core/src/manager-api/modules/stories.ts

@@ -7,6 +7,7 @@ import type {
   API_LeafEntry,
   API_LoadedRefData,
   API_PreparedStoryIndex,
+  API_StatusObject,
   API_StatusState,
   API_StatusUpdate,
   API_StoryEntry,
@@ -268,6 +269,12 @@ export interface SubAPI {
    * @returns {Promise<void>} A promise that resolves when the preview has been set as initialized.
    */
   setPreviewInitialized: (ref?: ComposedRef) => Promise<void>;
+  /**
+   * Returns the current status of the stories.
+   *
+   * @returns {API_StatusState} The current status of the stories.
+   */
+  getCurrentStoryStatus: () => Record<string, API_StatusObject>;
   /**
    * Updates the status of a collection of stories.
    *
@@ -630,6 +637,11 @@ export const init: ModuleFn<SubAPI, SubState> = ({
       }
     },
 
+    getCurrentStoryStatus: () => {
+      const { status, storyId } = store.getState();
+      return status[storyId as StoryId];
+    },
+
     /* EXPERIMENTAL APIs */
     experimental_updateStatus: async (id, input) => {
       const { status, internal_index: index } = store.getState();