First pass working

lib/api/src/lib/StoryIndexClient.ts

@@ -0,0 +1,18 @@
+import global from 'global';
+
+import { StoryIndex } from './stories';
+
+const { fetch, EventSource } = global;
+
+const PATH = './stories.json';
+
+export class StoryIndexClient extends EventSource {
+  constructor() {
+    super(PATH);
+  }
+
+  async fetch() {
+    const result = await fetch(PATH);
+    return result.json() as StoryIndex;
+  }
+}

lib/api/src/modules/stories.ts

@@ -34,6 +34,7 @@ import type {
 
 import { Args, ModuleFn } from '../index';
 import { ComposedRef } from './refs';
+import { StoryIndexClient } from '../lib/StoryIndexClient';
 
 const { DOCS_MODE } = global;
 
@@ -121,6 +122,8 @@ export const init: ModuleFn = ({
   storyId: initialStoryId,
   viewMode: initialViewMode,
 }) => {
+  let indexClient: StoryIndexClient;
+
   const api: SubAPI = {
     storyId: toId,
     getData: (storyId, refId) => {
@@ -350,9 +353,8 @@ export const init: ModuleFn = ({
       });
     },
     fetchStoryList: async () => {
-      // This needs some fleshing out as part of the stories list server project
-      const result = await global.fetch('/stories.json');
-      const storyIndex = (await result.json()) as StoryIndex;
+      console.log('fetchStoryList');
+      const storyIndex = await indexClient.fetch();
 
       // We can only do this if the stories.json is a proper storyIndex
       if (storyIndex.v !== 3) {
@@ -360,6 +362,7 @@ export const init: ModuleFn = ({
         return;
       }
 
+      console.log('got index');
       await fullAPI.setStoryList(storyIndex);
     },
     setStoryList: async (storyIndex: StoryIndex) => {
@@ -367,6 +370,7 @@ export const init: ModuleFn = ({
         provider,
       });
 
+      console.log(hash);
       await store.setState({
         storiesHash: hash,
         storiesConfigured: true,
@@ -496,6 +500,8 @@ export const init: ModuleFn = ({
       }
     );
 
+    indexClient = new StoryIndexClient();
+    indexClient.addEventListener('INVALIDATE', () => fullAPI.fetchStoryList());
     await fullAPI.fetchStoryList();
   };
 

lib/builder-webpack4/src/preview/virtualModuleModernEntry.js.handlebars

@@ -14,15 +14,10 @@ const getProjectAnnotations = () =>
     {{/each}}
   ]);
 
-const fetchStoryIndex = async () => {
-  const result = await fetch('./stories.json');
-  return result.json();
-}
-
 const channel = createChannel({ page: 'preview' });
 addons.setChannel(channel);
 
-const preview = new PreviewWeb({ importFn, fetchStoryIndex });
+const preview = new PreviewWeb({ importFn });
 
 window.__STORYBOOK_PREVIEW__ = preview;
 window.__STORYBOOK_STORY_STORE__ = preview.storyStore;

lib/builder-webpack5/src/preview/virtualModuleModernEntry.js.handlebars

@@ -14,15 +14,10 @@ const getProjectAnnotations = () =>
     {{/each}}
   ]);
 
-const fetchStoryIndex = async () => {
-  const result = await fetch('./stories.json');
-  return result.json();
-}
-
 const channel = createChannel({ page: 'preview' });
 addons.setChannel(channel);
 
-const preview = new PreviewWeb({ importFn, fetchStoryIndex });
+const preview = new PreviewWeb({ importFn });
 
 window.__STORYBOOK_PREVIEW__ = preview;
 window.__STORYBOOK_STORY_STORE__ = preview.storyStore;

lib/core-common/src/utils/progress-reporting.ts

@@ -31,6 +31,7 @@ export const useProgressReporting = async (
     reportProgress = (progress: any) => {
       if (closed || response.writableEnded) return;
       response.write(`data: ${JSON.stringify(progress)}\n\n`);
+      response.flush();
       if (progress.value === 1) close();
     };
   });

lib/core-server/package.json

@@ -74,6 +74,7 @@
     "serve-favicon": "^2.5.0",
     "ts-dedent": "^2.0.0",
     "util-deprecate": "^1.0.2",
+    "watchpack": "^2.2.0",
     "webpack": "4"
   },
   "devDependencies": {

lib/core-server/src/utils/StoryIndexGenerator.ts

@@ -124,6 +124,17 @@ export class StoryIndexGenerator {
     };
   }
 
+  invalidate(specifier: NormalizedStoriesSpecifier, filePath: Path, removed: boolean) {
+    const pathToEntries = this.storyIndexEntries.get(specifier);
+
+    console.log('onInvalidated', path, removed);
+    if (removed) {
+      delete pathToEntries[filePath];
+    } else {
+      pathToEntries[filePath] = false;
+    }
+  }
+
   async getStorySortParameter() {
     const previewFile = ['js', 'jsx', 'ts', 'tsx']
       .map((ext) => path.join(this.configDir, `preview.${ext}`))

lib/core-server/src/utils/stories-json.ts

@@ -1,6 +1,10 @@
 import fs from 'fs-extra';
+import EventEmitter from 'events';
 import { Options, normalizeStories, NormalizedStoriesSpecifier } from '@storybook/core-common';
 import { StoryIndexGenerator } from './StoryIndexGenerator';
+import { watchStorySpecifiers } from './watch-story-specifier';
+
+const eventName = 'INVALIDATE';
 
 export async function extractStoriesJson(
   outputFile: string,
@@ -15,21 +19,51 @@ export async function extractStoriesJson(
 }
 
 export async function useStoriesJson(router: any, options: Options) {
-  const normalized = normalizeStories(await options.presets.apply('stories'), {
+  const normalizedStories = normalizeStories(await options.presets.apply('stories'), {
     configDir: options.configDir,
     workingDir: process.cwd(),
   });
+  const generator = new StoryIndexGenerator(normalizedStories, options.configDir);
+  await generator.initialize();
 
-  router.use('/stories.json', async (_req: any, res: any) => {
-    const generator = new StoryIndexGenerator(normalized, options.configDir);
-    await generator.initialize();
+  const invalidationEmitter = new EventEmitter();
+  watchStorySpecifiers(normalizedStories, (specifier, path, removed) => {
+    generator.invalidate(specifier, path, removed);
+    console.log('emitting');
+    invalidationEmitter.emit(eventName);
+  });
+
+  router.use('/stories.json', async (req: any, res: any) => {
+    if (req.headers.accept === 'text/event-stream') {
+      let closed = false;
+      const watcher = () => {
+        if (closed || res.writableEnded) return;
+        res.write(`event:INVALIDATE\ndata:DATA\n\n`);
+        res.flush();
+      };
+      const close = () => {
+        invalidationEmitter.off(eventName, watcher);
+        closed = true;
+        res.end();
+      };
+      res.on('close', close);
+
+      if (closed || res.writableEnded) return;
+      res.setHeader('Cache-Control', 'no-cache');
+      res.setHeader('Content-Type', 'text/event-stream');
+      res.setHeader('Connection', 'keep-alive');
+      res.flushHeaders();
+
+      invalidationEmitter.on(eventName, watcher);
+      return;
+    }
 
     try {
       const index = await generator.getIndex();
       res.header('Content-Type', 'application/json');
-      return res.send(JSON.stringify(index));
+      res.send(JSON.stringify(index));
     } catch (err) {
-      return res.status(500).send(err.message);
+      res.status(500).send(err.message);
     }
   });
 }

lib/core-server/src/utils/watch-story-specifier.ts

@@ -0,0 +1,47 @@
+import Watchpack from 'watchpack';
+import { toRequireContext, NormalizedStoriesSpecifier } from '@storybook/core-common';
+import { Path } from '@storybook/store';
+
+// TODO -- deal with non "specified" specifiers
+export function watchStorySpecifiers(
+  specifiers: NormalizedStoriesSpecifier[],
+  onInvalidate: (specifier: NormalizedStoriesSpecifier, path: Path, removed: boolean) => void
+) {
+  // See https://www.npmjs.com/package/watchpack for full options.
+  // If you want less traffic, consider using aggregation with some interval
+  const wp = new Watchpack({
+    // poll: true, // Slow!!! Enable only in special cases
+    followSymlinks: false,
+    ignored: ['**/.git', 'node_modules'],
+  });
+  console.log(specifiers.map((ns) => ns.specifier.directory));
+  wp.watch({
+    directories: specifiers.map((ns) => ns.specifier.directory),
+  });
+
+  function onChangeOrRemove(path: Path, removed: boolean) {
+    console.log('onChangeOrRemove', path, removed);
+    const specifier = specifiers.find((ns) => {
+      const { path: base, regex } = toRequireContext(ns.glob);
+      console.log(
+        base,
+        regex,
+        !!path.startsWith(base.replace(/^\.\//, '')),
+        !!path.match(new RegExp(regex))
+      );
+      return path.startsWith(base.replace(/^\.\//, '')) && path.match(new RegExp(regex));
+    });
+    if (specifier) {
+      onInvalidate(specifier, path, removed);
+    }
+  }
+
+  wp.on('change', (path: Path, mtime: Date, explanation: string) => {
+    console.log('change', explanation);
+    onChangeOrRemove(path, false);
+  });
+  wp.on('remove', (path: Path, explanation: string) => {
+    console.log('remove', explanation);
+    onChangeOrRemove(path, true);
+  });
+}

lib/core-server/typings.d.ts

@@ -8,6 +8,7 @@ declare module '@storybook/ui/paths';
 declare module 'better-opn';
 declare module '@storybook/ui';
 declare module '@discoveryjs/json-ext';
+declare module 'watchpack';
 
 declare module 'file-system-cache' {
   export interface Options {

lib/preview-web/src/PreviewWeb.tsx

@@ -30,6 +30,7 @@ import { WebProjectAnnotations, DocsContextProps } from './types';
 import { UrlStore } from './UrlStore';
 import { WebView } from './WebView';
 import { NoDocs } from './NoDocs';
+import { StoryIndexClient } from './StoryIndexClient';
 
 const { window: globalWindow, AbortController, FEATURES } = global;
 
@@ -63,13 +64,23 @@ export class PreviewWeb<TFramework extends AnyFramework> {
     fetchStoryIndex,
   }: {
     importFn: ModuleImportFn;
-    fetchStoryIndex: ConstructorParameters<typeof StoryStore>[0]['fetchStoryIndex'];
+    fetchStoryIndex?: ConstructorParameters<typeof StoryStore>[0]['fetchStoryIndex'];
   }) {
     this.channel = addons.getChannel();
     this.view = new WebView();
 
     this.urlStore = new UrlStore();
-    this.storyStore = new StoryStore({ importFn, fetchStoryIndex });
+
+    if (FEATURES?.storyStoreV7) {
+      const indexClient = new StoryIndexClient();
+      this.storyStore = new StoryStore({ importFn, fetchStoryIndex: () => indexClient.fetch() });
+      indexClient.addEventListener('INVALIDATE', () => this.storyStore.onStoryIndexChanged());
+    } else {
+      if (!fetchStoryIndex) {
+        throw new Error('No `fetchStoryIndex` function defined in v6 mode');
+      }
+      this.storyStore = new StoryStore({ importFn, fetchStoryIndex });
+    }
 
     // Add deprecated APIs for back-compat
     // @ts-ignore

lib/preview-web/src/StoryIndexClient.ts

@@ -0,0 +1,18 @@
+import global from 'global';
+
+import { StoryIndex } from '@storybook/store';
+
+const { fetch, EventSource } = global;
+
+const PATH = './stories.json';
+
+export class StoryIndexClient extends EventSource {
+  constructor() {
+    super(PATH);
+  }
+
+  async fetch() {
+    const result = await fetch(PATH);
+    return result.json() as StoryIndex;
+  }
+}

lib/store/src/StoryStore.ts

@@ -156,6 +156,16 @@ export class StoryStore<TFramework extends AnyFramework> {
     }
   }
 
+  // TODO -- call this when calling `onImportFnChanged` rather than calling directly
+  async onStoryIndexChanged() {
+    // We need to refetch the stories list as it may have changed too
+    await this.storyIndex.cache(false);
+
+    if (this.cachedCSFFiles) {
+      await this.cacheAllCSFFiles(false);
+    }
+  }
+
   // To load a single CSF file to service a story we need to look up the importPath in the index
   loadCSFFileByStoryId(storyId: StoryId, options: { sync: false }): Promise<CSFFile<TFramework>>;
 

yarn.lock

@@ -8088,6 +8088,7 @@ __metadata:
     serve-favicon: ^2.5.0
     ts-dedent: ^2.0.0
     util-deprecate: ^1.0.2
+    watchpack: ^2.2.0
     webpack: 4
   peerDependencies:
     "@storybook/builder-webpack5": 6.4.0-alpha.39
@@ -45267,6 +45268,16 @@ resolve@1.19.0:
   languageName: node
   linkType: hard
 
+"watchpack@npm:^2.2.0":
+  version: 2.2.0
+  resolution: "watchpack@npm:2.2.0"
+  dependencies:
+    glob-to-regexp: ^0.4.1
+    graceful-fs: ^4.1.2
+  checksum: 4ea76d262f5f3110fdcb71a26c2ca0be39e7e7b566d30215f6a8f5a59501e2143a35bb075526848177aac29d04952f27272724e9342adbd4751e609ad4768124
+  languageName: node
+  linkType: hard
+
 "wbuf@npm:^1.1.0, wbuf@npm:^1.7.3":
   version: 1.7.3
   resolution: "wbuf@npm:1.7.3"