Merge pull request #23839 from storybookjs/yann/add-parallelism-count-check

Build: Add parallelism count check to CI

.circleci/config.yml

@@ -183,7 +183,7 @@ jobs:
             git diff --exit-code
       - report-workflow-on-failure
       - cancel-workflow-on-failure
-  script-unit-tests:
+  script-checks:
     executor: sb_node_16_browsers
     steps:
       - git-shallow-clone/checkout_advanced:
@@ -191,7 +191,12 @@ jobs:
       - attach_workspace:
           at: .
       - run:
-          name: Test
+          name: Check parallelism count
+          command: |
+            cd scripts
+            yarn get-template --check
+      - run:
+          name: Run tests
           command: |
             cd scripts
             yarn test --coverage --ci
@@ -479,7 +484,7 @@ workflows:
       - unit-tests:
           requires:
             - build
-      - script-unit-tests:
+      - script-checks:
           requires:
             - build
       - chromatic-internal-storybooks:
@@ -535,7 +540,7 @@ workflows:
       - unit-tests:
           requires:
             - build
-      - script-unit-tests:
+      - script-checks:
           requires:
             - build
       - chromatic-internal-storybooks:
@@ -592,7 +597,7 @@ workflows:
       - unit-tests:
           requires:
             - build
-      - script-unit-tests:
+      - script-checks:
           requires:
             - build
       - chromatic-internal-storybooks:

scripts/get-template.ts

@@ -1,7 +1,9 @@
 import { readdir } from 'fs/promises';
-import { pathExists } from 'fs-extra';
+import { pathExists, readFile } from 'fs-extra';
 import { program } from 'commander';
 import dedent from 'ts-dedent';
+import chalk from 'chalk';
+import yaml from 'yaml';
 import {
   allTemplates,
   templatesByCadence,
@@ -70,23 +72,31 @@ export async function getTemplate(
   return potentialTemplateKeys[index];
 }
 
-const tasks = [
-  'sandbox',
-  'build',
-  'chromatic',
-  'e2e-tests',
-  'e2e-tests-dev',
-  'test-runner',
+const tasksMap = {
+  sandbox: 'create-sandboxes',
+  build: 'build-sandboxes',
+  chromatic: 'chromatic-sandboxes',
+  'e2e-tests': 'e2e-production',
+  'e2e-tests-dev': 'e2e-dev',
+  'test-runner': 'test-runner-production',
   // 'test-runner-dev', TODO: bring this back when the task is enabled again
-  'bench',
-];
+  bench: 'bench',
+};
+
+const tasks = Object.keys(tasksMap);
+
+const CONFIG_YML_FILE = '../.circleci/config.yml';
+
+async function checkParallelism(cadence?: Cadence, scriptName?: string) {
+  const configYml = await readFile(CONFIG_YML_FILE, 'utf-8');
+  const data = yaml.parse(configYml);
 
-async function getParallelismSummary(cadence?: Cadence, scriptName?: string) {
   let potentialTemplateKeys: TemplateKey[] = [];
   const cadences = cadence ? [cadence] : (Object.keys(templatesByCadence) as Cadence[]);
   const scripts = scriptName ? [scriptName] : tasks;
   const summary = [];
-  summary.push('These are the values you should have in .circleci/config.yml:');
+  let isIncorrect = false;
+
   cadences.forEach((cad) => {
     summary.push(`\n${cad}`);
     const cadenceTemplates = Object.entries(allTemplates).filter(([key]) =>
@@ -94,7 +104,7 @@ async function getParallelismSummary(cadence?: Cadence, scriptName?: string) {
     );
     potentialTemplateKeys = cadenceTemplates.map(([k]) => k) as TemplateKey[];
 
-    scripts.forEach((script) => {
+    scripts.forEach((script: keyof typeof tasksMap) => {
       const templateKeysPerScript = potentialTemplateKeys.filter((t) => {
         const currentTemplate = allTemplates[t] as Template;
         return (
@@ -102,31 +112,56 @@ async function getParallelismSummary(cadence?: Cadence, scriptName?: string) {
           !currentTemplate.skipTasks?.includes(script as SkippableTask)
         );
       });
-      if (templateKeysPerScript.length > 0) {
-        summary.push(
-          `-- ${script} - parallelism: ${templateKeysPerScript.length}${
-            templateKeysPerScript.length === 2 ? ' (default)' : ''
-          }`
-        );
+      const workflowJobsRaw: (string | { [key: string]: any })[] = data.workflows[cad].jobs;
+      const workflowJobs = workflowJobsRaw
+        .filter((item) => typeof item === 'object' && item !== null)
+        .reduce((result, item) => Object.assign(result, item), {}) as Record<string, any>;
+
+      if (templateKeysPerScript.length > 0 && workflowJobs[tasksMap[script]]) {
+        const currentParallelism = workflowJobs[tasksMap[script]].parallelism || 2;
+        const newParallelism = templateKeysPerScript.length;
+
+        if (newParallelism !== currentParallelism) {
+          summary.push(
+            `-- ❌ ${tasksMap[script]} - parallelism: ${currentParallelism} ${chalk.bgRed(
+              `(should be ${newParallelism})`
+            )}`
+          );
+          isIncorrect = true;
+        } else {
+          summary.push(
+            `-- ✅ ${tasksMap[script]} - parallelism: ${templateKeysPerScript.length}${
+              templateKeysPerScript.length === 2 ? ' (default)' : ''
+            }`
+          );
+        }
       } else {
         summary.push(`-- ${script} - this script is fully skipped for this cadence.`);
       }
     });
   });
 
-  return summary.concat('\n').join('\n');
+  if (isIncorrect) {
+    summary.unshift(
+      'The parellism count is incorrect for some jobs in .circleci/config.yml, you have to update them:'
+    );
+    throw new Error(summary.concat('\n').join('\n'));
+  } else {
+    summary.unshift('✅  The parallelism count is correct for all jobs in .circleci/config.yml:');
+    console.log(summary.concat('\n').join('\n'));
+  }
 }
 
-type RunOptions = { cadence?: Cadence; task?: string; debug: boolean };
-async function run({ cadence, task, debug }: RunOptions) {
-  if (debug) {
+type RunOptions = { cadence?: Cadence; task?: string; check: boolean };
+async function run({ cadence, task, check }: RunOptions) {
+  if (check) {
     if (task && !tasks.includes(task)) {
       throw new Error(
         dedent`The "${task}" task you provided is not valid. Valid tasks (found in .circleci/config.yml) are: 
         ${tasks.map((v) => `- ${v}`).join('\n')}`
       );
     }
-    console.log(await getParallelismSummary(cadence as Cadence, task));
+    await checkParallelism(cadence as Cadence, task);
     return;
   }
 
@@ -147,7 +182,11 @@ if (require.main === module) {
     .description('Retrieve the template to run for a given cadence and task')
     .option('--cadence <cadence>', 'Which cadence you want to run the script for')
     .option('--task <task>', 'Which task you want to run the script for')
-    .option('--debug', 'Whether to list the parallelism counts for tasks by cadence', false);
+    .option(
+      '--check',
+      'Throws an error when the parallelism counts for tasks are incorrect',
+      false
+    );
 
   program.parse(process.argv);
 

scripts/package.json

@@ -185,6 +185,7 @@
     "uuid": "^9.0.0",
     "wait-on": "^7.0.1",
     "window-size": "^1.1.1",
+    "yaml": "^2.3.1",
     "zod": "^3.21.4"
   },
   "optionalDependencies": {

scripts/yarn.lock

@@ -3046,6 +3046,7 @@ __metadata:
     verdaccio-auth-memory: ^10.2.0
     wait-on: ^7.0.1
     window-size: ^1.1.1
+    yaml: ^2.3.1
     zod: ^3.21.4
   dependenciesMeta:
     "@verdaccio/types":
@@ -17359,7 +17360,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"yaml@npm:^2.0.0":
+"yaml@npm:^2.0.0, yaml@npm:^2.3.1":
   version: 2.3.1
   resolution: "yaml@npm:2.3.1"
   checksum: ed4c21a907fb1cd60a25177612fa46d95064a144623d269199817908475fe85bef20fb17406e3bdc175351b6488056a6f84beb7836e8c262646546a0220188e3