diff --git a/lib/api/src/lib/StoryIndexClient.ts b/lib/api/src/lib/StoryIndexClient.ts new file mode 100644 index 00000000000..7017328f097 --- /dev/null +++ b/lib/api/src/lib/StoryIndexClient.ts @@ -0,0 +1,18 @@ +import global from 'global'; + +import { StoryIndex } from './stories'; + +const { fetch, EventSource } = global; + +const PATH = './stories.json'; + +export class StoryIndexClient extends EventSource { + constructor() { + super(PATH); + } + + async fetch() { + const result = await fetch(PATH); + return result.json() as StoryIndex; + } +} diff --git a/lib/api/src/modules/stories.ts b/lib/api/src/modules/stories.ts index 78224e84013..bd902650fa0 100644 --- a/lib/api/src/modules/stories.ts +++ b/lib/api/src/modules/stories.ts @@ -34,6 +34,7 @@ import type { import { Args, ModuleFn } from '../index'; import { ComposedRef } from './refs'; +import { StoryIndexClient } from '../lib/StoryIndexClient'; const { DOCS_MODE } = global; @@ -121,6 +122,8 @@ export const init: ModuleFn = ({ storyId: initialStoryId, viewMode: initialViewMode, }) => { + let indexClient: StoryIndexClient; + const api: SubAPI = { storyId: toId, getData: (storyId, refId) => { @@ -350,9 +353,8 @@ export const init: ModuleFn = ({ }); }, fetchStoryList: async () => { - // This needs some fleshing out as part of the stories list server project - const result = await global.fetch('/stories.json'); - const storyIndex = (await result.json()) as StoryIndex; + console.log('fetchStoryList'); + const storyIndex = await indexClient.fetch(); // We can only do this if the stories.json is a proper storyIndex if (storyIndex.v !== 3) { @@ -360,6 +362,7 @@ export const init: ModuleFn = ({ return; } + console.log('got index'); await fullAPI.setStoryList(storyIndex); }, setStoryList: async (storyIndex: StoryIndex) => { @@ -367,6 +370,7 @@ export const init: ModuleFn = ({ provider, }); + console.log(hash); await store.setState({ storiesHash: hash, storiesConfigured: true, @@ -496,6 +500,8 @@ export const init: ModuleFn = ({ } ); + indexClient = new StoryIndexClient(); + indexClient.addEventListener('INVALIDATE', () => fullAPI.fetchStoryList()); await fullAPI.fetchStoryList(); }; diff --git a/lib/builder-webpack4/src/preview/virtualModuleModernEntry.js.handlebars b/lib/builder-webpack4/src/preview/virtualModuleModernEntry.js.handlebars index 0e7ff5e0500..07330175857 100644 --- a/lib/builder-webpack4/src/preview/virtualModuleModernEntry.js.handlebars +++ b/lib/builder-webpack4/src/preview/virtualModuleModernEntry.js.handlebars @@ -14,15 +14,10 @@ const getProjectAnnotations = () => {{/each}} ]); -const fetchStoryIndex = async () => { - const result = await fetch('./stories.json'); - return result.json(); -} - const channel = createChannel({ page: 'preview' }); addons.setChannel(channel); -const preview = new PreviewWeb({ importFn, fetchStoryIndex }); +const preview = new PreviewWeb({ importFn }); window.__STORYBOOK_PREVIEW__ = preview; window.__STORYBOOK_STORY_STORE__ = preview.storyStore; diff --git a/lib/builder-webpack5/src/preview/virtualModuleModernEntry.js.handlebars b/lib/builder-webpack5/src/preview/virtualModuleModernEntry.js.handlebars index 0e7ff5e0500..07330175857 100644 --- a/lib/builder-webpack5/src/preview/virtualModuleModernEntry.js.handlebars +++ b/lib/builder-webpack5/src/preview/virtualModuleModernEntry.js.handlebars @@ -14,15 +14,10 @@ const getProjectAnnotations = () => {{/each}} ]); -const fetchStoryIndex = async () => { - const result = await fetch('./stories.json'); - return result.json(); -} - const channel = createChannel({ page: 'preview' }); addons.setChannel(channel); -const preview = new PreviewWeb({ importFn, fetchStoryIndex }); +const preview = new PreviewWeb({ importFn }); window.__STORYBOOK_PREVIEW__ = preview; window.__STORYBOOK_STORY_STORE__ = preview.storyStore; diff --git a/lib/core-common/src/utils/progress-reporting.ts b/lib/core-common/src/utils/progress-reporting.ts index e6e1c60b86c..29f57185264 100644 --- a/lib/core-common/src/utils/progress-reporting.ts +++ b/lib/core-common/src/utils/progress-reporting.ts @@ -31,6 +31,7 @@ export const useProgressReporting = async ( reportProgress = (progress: any) => { if (closed || response.writableEnded) return; response.write(`data: ${JSON.stringify(progress)}\n\n`); + response.flush(); if (progress.value === 1) close(); }; }); diff --git a/lib/core-server/package.json b/lib/core-server/package.json index e021d0f158c..77a317654c8 100644 --- a/lib/core-server/package.json +++ b/lib/core-server/package.json @@ -74,6 +74,7 @@ "serve-favicon": "^2.5.0", "ts-dedent": "^2.0.0", "util-deprecate": "^1.0.2", + "watchpack": "^2.2.0", "webpack": "4" }, "devDependencies": { diff --git a/lib/core-server/src/utils/StoryIndexGenerator.ts b/lib/core-server/src/utils/StoryIndexGenerator.ts index bb91978b0e4..51267832518 100644 --- a/lib/core-server/src/utils/StoryIndexGenerator.ts +++ b/lib/core-server/src/utils/StoryIndexGenerator.ts @@ -124,6 +124,17 @@ export class StoryIndexGenerator { }; } + invalidate(specifier: NormalizedStoriesSpecifier, filePath: Path, removed: boolean) { + const pathToEntries = this.storyIndexEntries.get(specifier); + + console.log('onInvalidated', path, removed); + if (removed) { + delete pathToEntries[filePath]; + } else { + pathToEntries[filePath] = false; + } + } + async getStorySortParameter() { const previewFile = ['js', 'jsx', 'ts', 'tsx'] .map((ext) => path.join(this.configDir, `preview.${ext}`)) diff --git a/lib/core-server/src/utils/stories-json.ts b/lib/core-server/src/utils/stories-json.ts index fd993601d5e..da10757dcd2 100644 --- a/lib/core-server/src/utils/stories-json.ts +++ b/lib/core-server/src/utils/stories-json.ts @@ -1,6 +1,10 @@ import fs from 'fs-extra'; +import EventEmitter from 'events'; import { Options, normalizeStories, NormalizedStoriesSpecifier } from '@storybook/core-common'; import { StoryIndexGenerator } from './StoryIndexGenerator'; +import { watchStorySpecifiers } from './watch-story-specifier'; + +const eventName = 'INVALIDATE'; export async function extractStoriesJson( outputFile: string, @@ -15,21 +19,51 @@ export async function extractStoriesJson( } export async function useStoriesJson(router: any, options: Options) { - const normalized = normalizeStories(await options.presets.apply('stories'), { + const normalizedStories = normalizeStories(await options.presets.apply('stories'), { configDir: options.configDir, workingDir: process.cwd(), }); + const generator = new StoryIndexGenerator(normalizedStories, options.configDir); + await generator.initialize(); - router.use('/stories.json', async (_req: any, res: any) => { - const generator = new StoryIndexGenerator(normalized, options.configDir); - await generator.initialize(); + const invalidationEmitter = new EventEmitter(); + watchStorySpecifiers(normalizedStories, (specifier, path, removed) => { + generator.invalidate(specifier, path, removed); + console.log('emitting'); + invalidationEmitter.emit(eventName); + }); + + router.use('/stories.json', async (req: any, res: any) => { + if (req.headers.accept === 'text/event-stream') { + let closed = false; + const watcher = () => { + if (closed || res.writableEnded) return; + res.write(`event:INVALIDATE\ndata:DATA\n\n`); + res.flush(); + }; + const close = () => { + invalidationEmitter.off(eventName, watcher); + closed = true; + res.end(); + }; + res.on('close', close); + + if (closed || res.writableEnded) return; + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Connection', 'keep-alive'); + res.flushHeaders(); + + invalidationEmitter.on(eventName, watcher); + return; + } try { const index = await generator.getIndex(); res.header('Content-Type', 'application/json'); - return res.send(JSON.stringify(index)); + res.send(JSON.stringify(index)); } catch (err) { - return res.status(500).send(err.message); + res.status(500).send(err.message); } }); } diff --git a/lib/core-server/src/utils/watch-story-specifier.ts b/lib/core-server/src/utils/watch-story-specifier.ts new file mode 100644 index 00000000000..c05bd748a31 --- /dev/null +++ b/lib/core-server/src/utils/watch-story-specifier.ts @@ -0,0 +1,47 @@ +import Watchpack from 'watchpack'; +import { toRequireContext, NormalizedStoriesSpecifier } from '@storybook/core-common'; +import { Path } from '@storybook/store'; + +// TODO -- deal with non "specified" specifiers +export function watchStorySpecifiers( + specifiers: NormalizedStoriesSpecifier[], + onInvalidate: (specifier: NormalizedStoriesSpecifier, path: Path, removed: boolean) => void +) { + // See https://www.npmjs.com/package/watchpack for full options. + // If you want less traffic, consider using aggregation with some interval + const wp = new Watchpack({ + // poll: true, // Slow!!! Enable only in special cases + followSymlinks: false, + ignored: ['**/.git', 'node_modules'], + }); + console.log(specifiers.map((ns) => ns.specifier.directory)); + wp.watch({ + directories: specifiers.map((ns) => ns.specifier.directory), + }); + + function onChangeOrRemove(path: Path, removed: boolean) { + console.log('onChangeOrRemove', path, removed); + const specifier = specifiers.find((ns) => { + const { path: base, regex } = toRequireContext(ns.glob); + console.log( + base, + regex, + !!path.startsWith(base.replace(/^\.\//, '')), + !!path.match(new RegExp(regex)) + ); + return path.startsWith(base.replace(/^\.\//, '')) && path.match(new RegExp(regex)); + }); + if (specifier) { + onInvalidate(specifier, path, removed); + } + } + + wp.on('change', (path: Path, mtime: Date, explanation: string) => { + console.log('change', explanation); + onChangeOrRemove(path, false); + }); + wp.on('remove', (path: Path, explanation: string) => { + console.log('remove', explanation); + onChangeOrRemove(path, true); + }); +} diff --git a/lib/core-server/typings.d.ts b/lib/core-server/typings.d.ts index bf8f59bf50e..3a99ac672e7 100644 --- a/lib/core-server/typings.d.ts +++ b/lib/core-server/typings.d.ts @@ -8,6 +8,7 @@ declare module '@storybook/ui/paths'; declare module 'better-opn'; declare module '@storybook/ui'; declare module '@discoveryjs/json-ext'; +declare module 'watchpack'; declare module 'file-system-cache' { export interface Options { diff --git a/lib/preview-web/src/PreviewWeb.tsx b/lib/preview-web/src/PreviewWeb.tsx index 4037f2f1abf..fb0daf76e1f 100644 --- a/lib/preview-web/src/PreviewWeb.tsx +++ b/lib/preview-web/src/PreviewWeb.tsx @@ -30,6 +30,7 @@ import { WebProjectAnnotations, DocsContextProps } from './types'; import { UrlStore } from './UrlStore'; import { WebView } from './WebView'; import { NoDocs } from './NoDocs'; +import { StoryIndexClient } from './StoryIndexClient'; const { window: globalWindow, AbortController, FEATURES } = global; @@ -63,13 +64,23 @@ export class PreviewWeb { fetchStoryIndex, }: { importFn: ModuleImportFn; - fetchStoryIndex: ConstructorParameters[0]['fetchStoryIndex']; + fetchStoryIndex?: ConstructorParameters[0]['fetchStoryIndex']; }) { this.channel = addons.getChannel(); this.view = new WebView(); this.urlStore = new UrlStore(); - this.storyStore = new StoryStore({ importFn, fetchStoryIndex }); + + if (FEATURES?.storyStoreV7) { + const indexClient = new StoryIndexClient(); + this.storyStore = new StoryStore({ importFn, fetchStoryIndex: () => indexClient.fetch() }); + indexClient.addEventListener('INVALIDATE', () => this.storyStore.onStoryIndexChanged()); + } else { + if (!fetchStoryIndex) { + throw new Error('No `fetchStoryIndex` function defined in v6 mode'); + } + this.storyStore = new StoryStore({ importFn, fetchStoryIndex }); + } // Add deprecated APIs for back-compat // @ts-ignore diff --git a/lib/preview-web/src/StoryIndexClient.ts b/lib/preview-web/src/StoryIndexClient.ts new file mode 100644 index 00000000000..c9ed3408a96 --- /dev/null +++ b/lib/preview-web/src/StoryIndexClient.ts @@ -0,0 +1,18 @@ +import global from 'global'; + +import { StoryIndex } from '@storybook/store'; + +const { fetch, EventSource } = global; + +const PATH = './stories.json'; + +export class StoryIndexClient extends EventSource { + constructor() { + super(PATH); + } + + async fetch() { + const result = await fetch(PATH); + return result.json() as StoryIndex; + } +} diff --git a/lib/store/src/StoryStore.ts b/lib/store/src/StoryStore.ts index 46b823014ce..20ddf98e0ee 100644 --- a/lib/store/src/StoryStore.ts +++ b/lib/store/src/StoryStore.ts @@ -156,6 +156,16 @@ export class StoryStore { } } + // TODO -- call this when calling `onImportFnChanged` rather than calling directly + async onStoryIndexChanged() { + // We need to refetch the stories list as it may have changed too + await this.storyIndex.cache(false); + + if (this.cachedCSFFiles) { + await this.cacheAllCSFFiles(false); + } + } + // To load a single CSF file to service a story we need to look up the importPath in the index loadCSFFileByStoryId(storyId: StoryId, options: { sync: false }): Promise>; diff --git a/yarn.lock b/yarn.lock index b59bba0e4a9..1a936b748d1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8088,6 +8088,7 @@ __metadata: serve-favicon: ^2.5.0 ts-dedent: ^2.0.0 util-deprecate: ^1.0.2 + watchpack: ^2.2.0 webpack: 4 peerDependencies: "@storybook/builder-webpack5": 6.4.0-alpha.39 @@ -45267,6 +45268,16 @@ resolve@1.19.0: languageName: node linkType: hard +"watchpack@npm:^2.2.0": + version: 2.2.0 + resolution: "watchpack@npm:2.2.0" + dependencies: + glob-to-regexp: ^0.4.1 + graceful-fs: ^4.1.2 + checksum: 4ea76d262f5f3110fdcb71a26c2ca0be39e7e7b566d30215f6a8f5a59501e2143a35bb075526848177aac29d04952f27272724e9342adbd4751e609ad4768124 + languageName: node + linkType: hard + "wbuf@npm:^1.1.0, wbuf@npm:^1.7.3": version: 1.7.3 resolution: "wbuf@npm:1.7.3"