Merge pull request #30002 from storybookjs/29703-bug-test-discrepancy-handling-is-not-accurate

Addon Test: Wait for 2 seconds before showing result mismatch warning

code/addons/test/src/components/InteractionsPanel.stories.tsx

@@ -58,7 +58,6 @@ const meta = {
     endRef: null,
     // prop for the AddonPanel used as wrapper of Panel
     active: true,
-    storyId: 'story-id',
   },
 } as Meta<typeof InteractionsPanel>;
 

code/addons/test/src/components/InteractionsPanel.tsx

@@ -44,8 +44,6 @@ interface InteractionsPanelProps {
   onScrollToEnd?: () => void;
   hasResultMismatch?: boolean;
   browserTestStatus?: CallStates;
-  storyId: StoryId;
-  testRunId: string;
 }
 
 const Container = styled.div(({ theme }) => ({
@@ -105,20 +103,12 @@ export const InteractionsPanel: React.FC<InteractionsPanelProps> = React.memo(
     endRef,
     hasResultMismatch,
     browserTestStatus,
-    storyId,
-    testRunId,
   }) {
     const filter = useAnsiToHtmlFilter();
 
     return (
       <Container>
-        {hasResultMismatch && (
-          <TestDiscrepancyMessage
-            browserTestStatus={browserTestStatus}
-            storyId={storyId}
-            testRunId={testRunId}
-          />
-        )}
+        {hasResultMismatch && <TestDiscrepancyMessage browserTestStatus={browserTestStatus} />}
         {(interactions.length > 0 || hasException) && (
           <Subnav
             controls={controls}

code/addons/test/src/components/Panel.tsx

@@ -19,7 +19,7 @@ import { global } from '@storybook/global';
 import { type Call, CallStates, EVENTS, type LogItem } from '@storybook/instrumenter';
 import type { API_StatusValue } from '@storybook/types';
 
-import { ADDON_ID, TEST_PROVIDER_ID } from '../constants';
+import { ADDON_ID, STORYBOOK_ADDON_TEST_CHANNEL, TEST_PROVIDER_ID } from '../constants';
 import { InteractionsPanel } from './InteractionsPanel';
 
 interface Interaction extends Call {
@@ -114,6 +114,7 @@ export const Panel = memo<{ storyId: string }>(function PanelMemoized({ storyId
   // local state
   const [scrollTarget, setScrollTarget] = useState<HTMLElement | undefined>(undefined);
   const [collapsed, setCollapsed] = useState<Set<Call['id']>>(new Set());
+  const [hasResultMismatch, setResultMismatch] = useState(false);
 
   const {
     controlStates = INITIAL_CONTROL_STATES,
@@ -226,7 +227,7 @@ export const Panel = memo<{ storyId: string }>(function PanelMemoized({ storyId
         interactionsCount: list.filter(({ method }) => method !== 'step').length,
       };
     });
-  }, [collapsed]);
+  }, [set, collapsed]);
 
   const controls = useMemo(
     () => ({
@@ -239,7 +240,7 @@ export const Panel = memo<{ storyId: string }>(function PanelMemoized({ storyId
         emit(FORCE_REMOUNT, { storyId });
       },
     }),
-    [storyId]
+    [emit, storyId]
   );
 
   const storyFilePath = useParameter('fileName', '');
@@ -252,22 +253,48 @@ export const Panel = memo<{ storyId: string }>(function PanelMemoized({ storyId
     interactions.some((v) => v.status === CallStates.ERROR);
 
   const storyStatus = storyStatuses[storyId]?.[TEST_PROVIDER_ID];
+  const storyTestStatus = storyStatus?.status;
 
-  const browserTestStatus = React.useMemo<CallStates | null>(() => {
+  const browserTestStatus = useMemo<CallStates | null>(() => {
     if (!isPlaying && (interactions.length > 0 || hasException)) {
       return hasException ? CallStates.ERROR : CallStates.DONE;
     }
     return isPlaying ? CallStates.ACTIVE : null;
   }, [isPlaying, interactions, hasException]);
 
-  const hasResultMismatch = React.useMemo(() => {
-    return (
-      browserTestStatus !== null &&
-      browserTestStatus !== CallStates.ACTIVE &&
-      storyStatus?.status !== undefined &&
-      statusMap[browserTestStatus] !== storyStatus.status
-    );
-  }, [browserTestStatus, storyStatus]);
+  const { testRunId } = storyStatus?.data || {};
+
+  useEffect(() => {
+    const isMismatch =
+      browserTestStatus &&
+      storyTestStatus &&
+      storyTestStatus !== 'pending' &&
+      storyTestStatus !== statusMap[browserTestStatus];
+
+    if (isMismatch) {
+      const timeout = setTimeout(
+        () =>
+          setResultMismatch((currentValue) => {
+            if (!currentValue) {
+              emit(STORYBOOK_ADDON_TEST_CHANNEL, {
+                type: 'test-discrepancy',
+                payload: {
+                  browserStatus: browserTestStatus === CallStates.DONE ? 'PASS' : 'FAIL',
+                  cliStatus: browserTestStatus === CallStates.DONE ? 'FAIL' : 'PASS',
+                  storyId,
+                  testRunId,
+                },
+              });
+            }
+            return true;
+          }),
+        2000
+      );
+      return () => clearTimeout(timeout);
+    } else {
+      setResultMismatch(false);
+    }
+  }, [emit, browserTestStatus, storyTestStatus, storyId, testRunId]);
 
   if (isErrored) {
     return <Fragment key="component-tests" />;
@@ -290,8 +317,6 @@ export const Panel = memo<{ storyId: string }>(function PanelMemoized({ storyId
         pausedAt={pausedAt}
         endRef={endRef}
         onScrollToEnd={scrollTarget && scrollToTarget}
-        storyId={storyId}
-        testRunId={storyStatus?.data?.testRunId}
       />
     </Fragment>
   );

code/addons/test/src/components/TestDiscrepancyMessage.stories.tsx

@@ -23,9 +23,6 @@ export default {
   parameters: {
     layout: 'fullscreen',
   },
-  args: {
-    storyId: 'story-id',
-  },
   decorators: [
     (storyFn) => (
       <ManagerContext.Provider value={managerContext}>{storyFn()}</ManagerContext.Provider>

code/addons/test/src/components/TestDiscrepancyMessage.tsx

@@ -33,39 +33,23 @@ const Wrapper = styled.div(({ theme: { color, typography, background } }) => ({
 
 interface TestDiscrepancyMessageProps {
   browserTestStatus: CallStates;
-  storyId: StoryId;
-  testRunId: string;
 }
-export const TestDiscrepancyMessage = ({
-  browserTestStatus,
-  storyId,
-  testRunId,
-}: TestDiscrepancyMessageProps) => {
+
+export const TestDiscrepancyMessage = ({ browserTestStatus }: TestDiscrepancyMessageProps) => {
   const api = useStorybookApi();
   const docsUrl = api.getDocsUrl({
     subpath: DOCUMENTATION_DISCREPANCY_LINK,
     versioned: true,
     renderer: true,
   });
-  const message = `This component test passed in ${browserTestStatus === CallStates.DONE ? 'this browser' : 'CLI'}, but the tests failed in ${browserTestStatus === CallStates.ERROR ? 'this browser' : 'CLI'}.`;
-
-  useEffect(
-    () =>
-      api.emit(STORYBOOK_ADDON_TEST_CHANNEL, {
-        type: 'test-discrepancy',
-        payload: {
-          browserStatus: browserTestStatus === CallStates.DONE ? 'PASS' : 'FAIL',
-          cliStatus: browserTestStatus === CallStates.DONE ? 'FAIL' : 'PASS',
-          storyId,
-          testRunId,
-        },
-      }),
-    [api, browserTestStatus, storyId, testRunId]
-  );
+  const [passed, failed] =
+    browserTestStatus === CallStates.ERROR
+      ? ['the CLI', 'this browser']
+      : ['this browser', 'the CLI'];
 
   return (
     <Wrapper>
-      {message}{' '}
+      This component test passed in {passed}, but the tests failed in {failed}.{' '}
       <Link href={docsUrl} target="_blank" withArrow>
         Learn what could cause this
       </Link>

test-storybooks/portable-stories-kitchen-sink/react/e2e-tests/component-testing.spec.ts

@@ -96,7 +96,7 @@ test.describe("component testing", () => {
       "Test status: success"
     );
     await expect(sbPage.panelContent()).toContainText(
-      /This component test passed in CLI, but the tests failed in this browser./
+      /This component test passed in the CLI, but the tests failed in this browser/
     );
 
     // Assert discrepancy: CLI fail + Browser pass
@@ -109,7 +109,7 @@ test.describe("component testing", () => {
       "Test status: error"
     );
     await expect(sbPage.panelContent()).toContainText(
-      /This component test passed in this browser, but the tests failed in CLI/
+      /This component test passed in this browser, but the tests failed in the CLI/
     );
   });