Merge pull request #18844 from storybookjs/tom/sb-593-start-verdaccio-publish-packages-install

Got verdaccio working, borrowing heavily from the old repro command

.gitignore

@@ -9,6 +9,7 @@ dist
 junit.xml
 /repros
 /sandbox
+.verdaccio-cache
 
 # Yarn stuff
 /**/.yarn/*

scripts/sandbox.ts

@@ -5,7 +5,8 @@ import prompts from 'prompts';
 
 import { getOptionsOrPrompt } from './utils/options';
 import { executeCLIStep } from './utils/cli-step';
-import { exec } from '../code/lib/cli/src/repro-generators/scripts';
+import { installYarn2, configureYarn2ForVerdaccio, addPackageResolutions } from './utils/yarn';
+import { exec } from './utils/exec';
 import { getInterpretedFile } from '../code/lib/core-common';
 import { ConfigFile, readConfig, writeConfig } from '../code/lib/csf-tools';
 import { babelParse } from '../code/lib/csf-tools/src/babelParse';
@@ -134,7 +135,7 @@ async function addPackageScripts({
   cwd: string;
   scripts: Record<string, string>;
 }) {
-  logger.info(`🔢 Adding package resolutions:`);
+  logger.info(`🔢 Adding package scripts:`);
   const packageJsonPath = path.join(cwd, 'package.json');
   const packageJson = await readJSON(packageJsonPath);
   packageJson.scripts = {
@@ -159,7 +160,7 @@ const webpackFinalCode = `
       ...config.modules,
       rules: [
         {
-          test: [/\\/node_modules\\/@storybook\\/[^/]*\\/template\\/stories\\//],
+          test: [/\\/code\\/[^/]*\\/[^/]*\\/template\\/stories\\//],
           loader: '${loaderPath}',
           options: {
             loader: 'tsx',
@@ -171,7 +172,7 @@ const webpackFinalCode = `
     },
   })`;
 
-// paths are of the form 'node_modules/@storybook/react'
+// paths are of the form 'renderers/react', 'addons/actions'
 async function addStories(paths: string[], { mainConfig }: { mainConfig: ConfigFile }) {
   const stories = mainConfig.getFieldValue(['stories']) as string[];
   const extraStoryDirsAndExistence = await Promise.all(
@@ -180,9 +181,10 @@ async function addStories(paths: string[], { mainConfig }: { mainConfig: ConfigF
       .map(async (p) => [p, await pathExists(path.resolve(codeDir, p))] as const)
   );
 
+  const relativeCodeDir = path.join('..', '..', '..', 'code');
   const extraStories = extraStoryDirsAndExistence
     .filter(([, exists]) => exists)
-    .map(([p]) => path.join('..', p, '*.stories.@(js|jsx|ts|tsx)'));
+    .map(([p]) => path.join(relativeCodeDir, p, '*.stories.@(js|jsx|ts|tsx)'));
   mainConfig.setFieldValue(['stories'], [...stories, ...extraStories]);
 
   mainConfig.setFieldNode(
@@ -228,7 +230,8 @@ async function main() {
     const storiesPath = await findFirstPath([path.join('src', 'stories'), 'stories'], { cwd });
 
     // Link in the template/components/index.js from the renderer
-    const rendererPath = path.join('node_modules', templateConfig.expected.renderer);
+    const rendererName = templateConfig.expected.renderer.split('/')[1];
+    const rendererPath = path.join('renderers', rendererName);
     await ensureSymlink(
       path.join(codeDir, rendererPath, 'template', 'components'),
       path.resolve(cwd, storiesPath, 'components')
@@ -252,34 +255,53 @@ async function main() {
     }
 
     for (const addon of [...defaultAddons, ...optionValues.addon]) {
-      storiesToAdd.push(path.join('node_modules', '@storybook', `addon-${addon}`));
+      storiesToAdd.push(path.join('addons', addon));
     }
     await addStories(storiesToAdd, { mainConfig });
 
     await writeConfig(mainConfig);
 
+    await installYarn2({ cwd, dryRun });
     if (link) {
-      await exec('yarn set version berry', { cwd }, { dryRun });
-      await exec('yarn config set enableGlobalCache true', { cwd }, { dryRun });
-      await exec('yarn config set nodeLinker node-modules', { cwd }, { dryRun });
-
       await executeCLIStep(steps.link, {
         argument: cwd,
         cwd: codeDir,
         dryRun,
         optionValues: { local: true, start: false },
       });
+    } else {
+      await exec('yarn local-registry --publish', { cwd: codeDir }, { dryRun });
 
-      await addPackageScripts({
-        cwd,
-        scripts: {
-          storybook:
-            'NODE_OPTIONS="--preserve-symlinks --preserve-symlinks-main" storybook dev -p 6006',
-          'build-storybook':
-            'NODE_OPTIONS="--preserve-symlinks --preserve-symlinks-main" storybook build',
-        },
-      });
+      // NOTE: this is a background task and will run forever (TODO: sort out logging/exiting)
+      exec('CI=true yarn local-registry --open', { cwd: codeDir }, { dryRun });
+      await exec('yarn wait-on http://localhost:6000', { cwd: codeDir }, { dryRun });
+
+      // We need to add package resolutions to ensure that we only ever install the latest version
+      // of any storybook packages as verdaccio is not able to both proxy to npm and publish over
+      // the top. In theory this could mask issues where different versions cause problems.
+      await addPackageResolutions({ cwd, dryRun });
+      await configureYarn2ForVerdaccio({ cwd, dryRun });
+
+      await exec(
+        'yarn install',
+        { cwd },
+        {
+          dryRun,
+          startMessage: `⬇️ Installing local dependencies`,
+          errorMessage: `🚨 Installing local dependencies failed`,
+        }
+      );
     }
+
+    await addPackageScripts({
+      cwd,
+      scripts: {
+        storybook:
+          'NODE_OPTIONS="--preserve-symlinks --preserve-symlinks-main" storybook dev -p 6006',
+        'build-storybook':
+          'NODE_OPTIONS="--preserve-symlinks --preserve-symlinks-main" storybook build',
+      },
+    });
   }
 
   const { start } = optionValues;

scripts/utils/cli-step.ts

@@ -1,5 +1,5 @@
 import { getCommand, OptionSpecifier, OptionValues } from './options';
-import { exec } from '../../code/lib/cli/src/repro-generators/scripts';
+import { exec } from './exec';
 
 const cliExecutable = require.resolve('../../code/lib/cli/bin/index.js');
 

scripts/utils/exec.ts

@@ -0,0 +1,46 @@
+import shell, { ExecOptions } from 'shelljs';
+import chalk from 'chalk';
+
+const logger = console;
+
+export const exec = async (
+  command: string,
+  options: ExecOptions = {},
+  {
+    startMessage,
+    errorMessage,
+    dryRun,
+  }: { startMessage?: string; errorMessage?: string; dryRun?: boolean } = {}
+) => {
+  if (startMessage) logger.info(startMessage);
+
+  if (dryRun) {
+    logger.info(`\n> ${command}\n`);
+    return undefined;
+  }
+
+  logger.debug(command);
+  return new Promise((resolve, reject) => {
+    const defaultOptions: ExecOptions = {
+      silent: false,
+    };
+    const child = shell.exec(command, {
+      ...defaultOptions,
+      ...options,
+      async: true,
+      silent: false,
+    });
+
+    child.stderr.pipe(process.stderr);
+
+    child.on('exit', (code) => {
+      if (code === 0) {
+        resolve(undefined);
+      } else {
+        logger.error(chalk.red(`An error occurred while executing: \`${command}\``));
+        logger.log(errorMessage);
+        reject(new Error(`command exited with code: ${code}: `));
+      }
+    });
+  });
+};

scripts/utils/yarn.ts

@@ -0,0 +1,63 @@
+import { readJSON, writeJSON } from 'fs-extra';
+import path from 'path';
+
+import { exec } from './exec';
+// TODO -- should we generate this file a second time outside of CLI?
+import storybookVersions from '../../code/lib/cli/src/versions';
+
+export type YarnOptions = {
+  cwd: string;
+  dryRun?: boolean;
+};
+
+const logger = console;
+
+export const addPackageResolutions = async ({ cwd, dryRun }: YarnOptions) => {
+  logger.info(`🔢 Adding package resolutions:`);
+  if (dryRun) return;
+
+  const packageJsonPath = path.join(cwd, 'package.json');
+  const packageJson = await readJSON(packageJsonPath);
+  packageJson.resolutions = storybookVersions;
+  await writeJSON(packageJsonPath, packageJson, { spaces: 2 });
+};
+
+export const installYarn2 = async ({ cwd, dryRun }: YarnOptions) => {
+  const command = [
+    `yarn set version berry`,
+    // Use the global cache so we aren't re-caching dependencies each time we run sandbox
+    `yarn config set enableGlobalCache true`,
+    `yarn config set nodeLinker node-modules`,
+  ];
+
+  await exec(
+    command.join(' && '),
+    { cwd },
+    { dryRun, startMessage: `🧶 Installing Yarn 2`, errorMessage: `🚨 Installing Yarn 2 failed` }
+  );
+};
+
+export const configureYarn2ForVerdaccio = async ({ cwd, dryRun }: YarnOptions) => {
+  const command = [
+    // We don't want to use the cache or we might get older copies of our built packages
+    // (with identical versions), as yarn (correctly I guess) assumes the same version hasn't changed
+    `yarn config set enableGlobalCache false`,
+    `yarn config set enableMirror false`,
+    // ⚠️ Need to set registry because Yarn 2 is not using the conf of Yarn 1 (URL is hardcoded in CircleCI config.yml)
+    `yarn config set npmScopes --json '{ "storybook": { "npmRegistryServer": "http://localhost:6000/" } }'`,
+    // Some required magic to be able to fetch deps from local registry
+    `yarn config set unsafeHttpWhitelist --json '["localhost"]'`,
+    // Disable fallback mode to make sure everything is required correctly
+    `yarn config set pnpFallbackMode none`,
+    // We need to be able to update lockfile when bootstrapping the examples
+    `yarn config set enableImmutableInstalls false`,
+    // Discard all YN0013 - FETCH_NOT_CACHED messages
+    `yarn config set logFilters --json '[ { "code": "YN0013", "level": "discard" } ]'`,
+  ].join(' && ');
+
+  await exec(
+    command,
+    { cwd },
+    { startMessage: `🎛 Configuring Yarn 2`, errorMessage: `🚨 Configuring Yarn 2 failed`, dryRun }
+  );
+};