diff --git a/docs/development-guide/08-testing.md b/docs/development-guide/08-testing.md index 48fcf586379..805729ae65d 100644 --- a/docs/development-guide/08-testing.md +++ b/docs/development-guide/08-testing.md @@ -199,24 +199,26 @@ These environment variables are used by Cloud Manager's UI tests to override reg | `CY_TEST_REGION` | ID of region to test (as used by Linode APIv4). | `us-east` | Unset; regions are selected at random | ###### Run Splitting -These environment variables facilitate splitting the Cypress run between multiple runners without the use of any third party services. This can be useful for improving Cypress test performance in some circumstances. +These environment variables facilitate splitting the Cypress run between multiple runners without the use of any third party services. This can be useful for improving Cypress test performance in some circumstances. For additional performance gains, an optional test weights file can be specified using `CY_TEST_SPLIT_RUN_WEIGHTS` (see `CY_TEST_GENWEIGHTS` to generate test weights). -| Environment Variable | Description | Example | Default | -|---------------------------|--------------------------------------------|----------------|----------------------------| -| `CY_TEST_SPLIT_RUN` | Enable run splitting | `1` | Unset; disabled by default | -| `CY_TEST_SPLIT_RUN_INDEX` | Numeric index for each Cypress test runner | `1`, `2`, etc. | Unset | -| `CY_TEST_SPLIT_RUN_TOTAL` | Total number of runners for the tests | `2` | Unset | +| Environment Variable | Description | Example | Default | +|-----------------------------|--------------------------------------------|------------------|----------------------------| +| `CY_TEST_SPLIT_RUN` | Enable run splitting | `1` | Unset; disabled by default | +| `CY_TEST_SPLIT_RUN_INDEX` | Numeric index for each Cypress test runner | `1`, `2`, etc. | Unset | +| `CY_TEST_SPLIT_RUN_TOTAL` | Total number of runners for the tests | `2` | Unset | +| `CY_TEST_SPLIT_RUN_WEIGHTS` | Path to test weights file | `./weights.json` | Unset; disabled by default | ###### Development, Logging, and Reporting Environment variables related to Cypress logging and reporting, as well as report generation. -| Environment Variable | Description | Example | Default | -|---------------------------------|-----------------------------------------------|-----------|----------------------------| -| `CY_TEST_USER_REPORT` | Log test account information when tests begin | `1` | Unset; disabled by default | -| `CY_TEST_JUNIT_REPORT` | Enable JUnit reporting | `1` | Unset; disabled by default | -| `CY_TEST_DISABLE_FILE_WATCHING` | Disable file watching in Cypress UI | `1` | Unset; disabled by default | -| `CY_TEST_DISABLE_RETRIES` | Disable test retries on failure in CI | `1` | Unset; disabled by default | -| `CY_TEST_FAIL_ON_MANAGED` | Fail affected tests when Managed is enabled | `1` | Unset; disabled by default | +| Environment Variable | Description | Example | Default | +|---------------------------------|----------------------------------------------------|------------------|----------------------------| +| `CY_TEST_USER_REPORT` | Log test account information when tests begin | `1` | Unset; disabled by default | +| `CY_TEST_JUNIT_REPORT` | Enable JUnit reporting | `1` | Unset; disabled by default | +| `CY_TEST_DISABLE_FILE_WATCHING` | Disable file watching in Cypress UI | `1` | Unset; disabled by default | +| `CY_TEST_DISABLE_RETRIES` | Disable test retries on failure in CI | `1` | Unset; disabled by default | +| `CY_TEST_FAIL_ON_MANAGED` | Fail affected tests when Managed is enabled | `1` | Unset; disabled by default | +| `CY_TEST_GENWEIGHTS` | Generate and output test weights to the given path | `./weights.json` | Unset; disabled by default | ### Writing End-to-End Tests diff --git a/packages/manager/cypress.config.ts b/packages/manager/cypress.config.ts index c0cc12e20a3..f5edf432f71 100644 --- a/packages/manager/cypress.config.ts +++ b/packages/manager/cypress.config.ts @@ -14,6 +14,7 @@ import { fetchAccount } from './cypress/support/plugins/fetch-account'; import { fetchLinodeRegions } from './cypress/support/plugins/fetch-linode-regions'; import { splitCypressRun } from './cypress/support/plugins/split-run'; import { enableJunitReport } from './cypress/support/plugins/junit-report'; +import { generateTestWeights } from './cypress/support/plugins/generate-weights'; import { logTestTagInfo } from './cypress/support/plugins/test-tagging-info'; /** @@ -70,6 +71,7 @@ export default defineConfig({ logTestTagInfo, splitCypressRun, enableJunitReport, + generateTestWeights, ]); }, }, diff --git a/packages/manager/cypress/support/plugins/generate-weights.ts b/packages/manager/cypress/support/plugins/generate-weights.ts new file mode 100644 index 00000000000..bfb13cc0b32 --- /dev/null +++ b/packages/manager/cypress/support/plugins/generate-weights.ts @@ -0,0 +1,155 @@ +import type { CypressPlugin } from './plugin'; +import { DateTime } from 'luxon'; +import { writeFileSync } from 'fs'; +import { resolve } from 'path'; +import { object, string, array, number, SchemaOf } from 'yup'; + +// The name of the environment variable to read to check if generation is enabled. +// The value should be a path to the weights file. +const envVarName = 'CY_TEST_GENWEIGHTS'; + +/** + * Describes spec file weights for a test suite. + */ +export interface SpecWeights { + /** + * Spec weight metadata. + */ + meta: { + /** + * Date and time that test spec weights were generated. + */ + datetime: string; + + /** + * Total test weight. + */ + totalWeight: number; + + /** + * Total test run duration in milliseconds. + */ + totalDuration: number; + }; + /** + * Array of spec weights. + */ + weights: SpecWeight[]; +} + +/** + * Describes the weight of an individual spec file. + */ +export interface SpecWeight extends SpecResult { + /** + * Spec weight. + */ + weight: number; +} + +/** + * Spec weights schema for JSON parsing, etc. + */ +export const specWeightsSchema: SchemaOf = object({ + meta: object({ + datetime: string().required(), + totalWeight: number().required(), + totalDuration: number().required(), + }).required(), + weights: array( + object({ + filepath: string().required(), + duration: number().required(), + weight: number().required(), + }) + ).required(), +}); + +/** + * Describes the duration of an individual spec file. + * + * Used in the process of calculating weights for each spec. + */ +interface SpecResult { + /** + * Relative path to spec file. + */ + filepath: string; + + /** + * Spec run duration in milliseconds. + */ + duration: number; +} + +/** + * Enables test weight generation when `CY_TEST_GENWEIGHTS` is defined. + * + * @returns Cypress configuration object. + */ +export const generateTestWeights: CypressPlugin = (on, config) => { + const specResults: SpecResult[] = []; + + if (!!config.env[envVarName]) { + const writeFilepath = config.env[envVarName]; + + // Capture duration after each spec runs. + on('after:spec', (spec, results) => { + const duration = results.stats.duration; + if (duration) { + specResults.push({ + filepath: spec.relative, + duration, + }); + } else { + console.warn( + `Failed to record test information for '${spec.relative}'` + ); + } + }); + + // Aggregate spec durations and save as a spec weights JSON file. + on( + 'after:run', + ( + results: + | CypressCommandLine.CypressRunResult + | CypressCommandLine.CypressFailedRunResult + ) => { + // Determine whether this is a failed run. "Failed" in this context means + // that Cypress itself failed to run, not that the test results contained failures. + const isFailedResult = ( + results: + | CypressCommandLine.CypressRunResult + | CypressCommandLine.CypressFailedRunResult + ): results is CypressCommandLine.CypressFailedRunResult => { + return 'failures' in results; + }; + + if (!isFailedResult(results)) { + const totalWeight = 100; + const totalDuration = results.totalDuration; + const weights: SpecWeights = { + meta: { + datetime: DateTime.now().toISO(), + totalWeight, + totalDuration, + }, + weights: specResults.map( + (specResult: SpecResult): SpecWeight => { + return { + filepath: specResult.filepath, + duration: specResult.duration, + weight: (specResult.duration / totalDuration) * totalWeight, + }; + } + ), + }; + + const resolvePath = resolve(writeFilepath); + writeFileSync(resolvePath, JSON.stringify(weights), 'utf-8'); + } + } + ); + } +}; diff --git a/packages/manager/cypress/support/plugins/split-run.ts b/packages/manager/cypress/support/plugins/split-run.ts index fe90671d701..eccb6c0e153 100644 --- a/packages/manager/cypress/support/plugins/split-run.ts +++ b/packages/manager/cypress/support/plugins/split-run.ts @@ -1,57 +1,251 @@ /** - * @file Implements naive parallelization without Cypress Cloud. + * @file Allows parallelization without Cypress Cloud. */ import { globSync } from 'glob'; - +import { resolve } from 'path'; +import { readFileSync } from 'fs'; +import { SpecWeight, SpecWeights, specWeightsSchema } from './generate-weights'; import type { CypressPlugin } from './plugin'; +/** + * Describes weighted specs for a single test runner. + */ +interface WeightedRunnerSpecs { + /** Array of spec filepaths for runner. */ + specs: string[]; + + /** Total test weight of specs in `specs`. */ + weight: number; +} + +/** + * Divides a run between separate Cypress processes. + * + * Optionally, a test weights file may be specified to optimize test distribution + * among runners. + */ export const splitCypressRun: CypressPlugin = (_on, config) => { - const splitRunEnabled = config?.env?.['CY_TEST_SPLIT_RUN']; - const splitRunTotalRunners = config?.env?.['CY_TEST_SPLIT_RUN_TOTAL']; - const splitRunRunnerIndex = config?.env?.['CY_TEST_SPLIT_RUN_INDEX']; + const { + CY_TEST_SPLIT_RUN: splitRunEnabled, + CY_TEST_SPLIT_RUN_TOTAL: splitRunTotalRunners, + CY_TEST_SPLIT_RUN_INDEX: splitRunRunnerIndex, + CY_TEST_SPLIT_RUN_WEIGHTS: splitRunWeightsPath, + } = config.env; + + // Short-circuit if split running is not enabled. + // In this case, return an unmodified config object. + if (!splitRunEnabled) { + return config; + } // If split running is enabled, total and index must be defined. - if (splitRunEnabled) { - if (!splitRunTotalRunners || !splitRunRunnerIndex) { - throw new Error( - 'CY_TEST_SPLIT_RUN is enabled, but CY_TEST_SPLIT_RUN_TOTAL and CY_TEST_SPLIT_RUN_INDEX are not defined.' - ); + // Otherwise, we'll throw an error that will be displayed to the user. + if (!splitRunTotalRunners || !splitRunRunnerIndex) { + throw new Error( + 'CY_TEST_SPLIT_RUN is enabled, but CY_TEST_SPLIT_RUN_TOTAL and CY_TEST_SPLIT_RUN_INDEX are not defined.' + ); + } + if (isNaN(splitRunTotalRunners) || isNaN(splitRunRunnerIndex)) { + throw new Error( + 'CY_TEST_SPLIT_RUN_TOTAL and CY_TEST_SPLIT_RUN_INDEX must be numeric.' + ); + } + + const totalRunners = parseInt(splitRunTotalRunners, 10); + const runner = parseInt(splitRunRunnerIndex, 10); + + // Override configuration spec pattern to reflect test subset for this runner... + const specs = globSync(config.specPattern); + + let totalWeight = 0; + let weightedSpecs: SpecWeight[] = []; + let unweightedSpecs: string[] = [...specs]; + + // If spec weights file path is specified, attempt to read its contents. + // If weights file does not exist, is inaccessible, or is malformed, weights + // data will be discarded and run splitting will fall back on round-robin + // distribution method. + if (splitRunWeightsPath) { + try { + const specWeights = readTestWeightsFile(splitRunWeightsPath); + weightedSpecs = getWeightedSpecs(specs, specWeights); + unweightedSpecs = getUnweightedSpecs(specs, specWeights); + totalWeight = specWeights.meta.totalWeight; + } catch (err) { + // Swallow error here; it's OK if test weights file doesn't exist / can't be read. + // Wrap messages in IIFEs to avoid issue where info messages get printed first. + (() => { + console.warn(`Failed to read weights file at '${splitRunWeightsPath}'`); + if ('message' in err) { + console.warn(`Error message: ${err.message}`); + } + })(); + (() => { + console.info( + 'You can optimize your CI run performance by generating a valid weights file' + ); + console.info( + `Example: CY_TEST_GENWEIGHTS='${splitRunWeightsPath}' yarn cy:run` + ); + })(); } - if (isNaN(splitRunTotalRunners) || isNaN(splitRunRunnerIndex)) { - throw new Error( - 'CY_TEST_SPLIT_RUN_TOTAL and CY_TEST_SPLIT_RUN_INDEX must be numeric.' - ); + } + + // Distribute specs based on their weights and get an array of weighted specs + // for this runner. + const weightedSpecsForRunner = getWeightedRunnerSpecs( + runner, + totalRunners, + weightedSpecs + ); + + // Distribute remaining unweighted specs round-robin style, if applicable. + // Sort spec filenames as deterministically as we easily can. + unweightedSpecs.sort((a: string, b: string): number => { + if (a.toLowerCase() < b.toLowerCase()) { + return -1; + } else if (a.toLowerCase() > b.toLowerCase()) { + return 1; } + return 0; + }); + + config.specPattern = [ + // Include weighted specs first. + ...weightedSpecsForRunner.specs, - const totalRunners = parseInt(splitRunTotalRunners, 10); - const runner = parseInt(splitRunRunnerIndex, 10); - - // Override configuration spec pattern to reflect test subset for this runner. - const specs = globSync(config.specPattern); - // Sort spec filenames deterministically. - // Or at least as deterministically as we can in a pinch... - specs.sort((a: string, b: string): number => { - if (a.toLowerCase() < b.toLowerCase()) { - return -1; - } else if (a.toLowerCase() > b.toLowerCase()) { - return 1; - } - return 0; - }); - - // Only include every Nth spec, where N is the total number of runners. - config.specPattern = specs.filter((spec: string, index: number) => { + // If there are any unweighted specs remaining, only include every Nth + // spec, where N is the total number of runners. + ...unweightedSpecs.filter((_spec: string, index: number) => { return (index + runner - 1) % totalRunners === 0; - }); - - console.info('Cypress split running is enabled.'); - console.table({ - '# of Specs Total': specs.length, - '# of Specs for This Run': config.specPattern.length, - Runner: runner, - 'Total Runners': totalRunners, - }); - } + }), + ]; + + const splitRunInfo = { + '# of Specs Total': specs.length, + '# of Specs for This Run': config.specPattern.length, + Runner: runner, + 'Total Runners': totalRunners, + }; + + const weightsInfo = (() => { + if (weightedSpecs.length < 1) { + return { + 'Test Weights': 'Unavailable', + }; + } + return { + 'Test Weights': splitRunWeightsPath, + 'Total Test Weight': `${Math.round(totalWeight * 100) / 100}%`, + 'Runner Test Weight': `${ + Math.round(weightedSpecsForRunner.weight * 100) / 100 + }%`, + 'Weighted Specs': weightedSpecs.length, + 'Unweighted Specs': unweightedSpecs.length, + }; + })(); + + console.info('Cypress split running is enabled.'); + console.table({ + ...splitRunInfo, + ...weightsInfo, + }); + return config; }; + +/** + * Reads a test weights file at the given path and returns its data. + * + * Weights data is sorted from highest to lowest weight. + * + * @param weightsFilepath - Path to weights file. + * + * @throws If `weightsFilepath` does not exist or is not readable. + * @throws If weights data is invalid. + * + * @returns Spec weights data. + */ +const readTestWeightsFile = (weightsFilepath: string): SpecWeights => { + const weightsContents = readFileSync(resolve(weightsFilepath), 'utf-8'); + const weightsData = JSON.parse(weightsContents) as SpecWeights; + specWeightsSchema.validateSync(weightsData); + + // Sort spec weights from highest weight to lowest. + weightsData.weights.sort((a, b) => b.weight - a.weight); + + return weightsData; +}; + +/** + * Returns an array of `SpecWeight` objects for each spec file with corresponding weight data. + * + * @param allSpecs - String of spec filepaths for this run. + * @param specWeights - Spec weights data. + * + * @returns Array of `SpecWeight` objects for each spec file that has weight data. + */ +const getWeightedSpecs = ( + allSpecs: string[], + specWeights: SpecWeights +): SpecWeight[] => { + return allSpecs + .map((specPath: string): SpecWeight | undefined => { + return specWeights.weights.find( + (specWeight) => specWeight.filepath === specPath + ); + }) + .filter((specWeight): specWeight is SpecWeight => !!specWeight); +}; + +/** + * Returns an array of spec filepaths for each spec file that does not have weight data. + * + * @param allSpecs - String of spec filepaths for this run. + * @param specWeights - Spec weights data. + * + * @returns Array of spec filepaths for each spec file that does not have corresponding weight data. + */ +const getUnweightedSpecs = ( + allSpecs: string[], + specWeights: SpecWeights +): string[] => { + return allSpecs.filter((specPath: string) => { + return !specWeights.weights.find( + (specWeight) => specWeight.filepath === specPath + ); + }); +}; + +/** + * Returns weighted specs for a single runner in a split run. + * + * @param runnerIndex - Index of the runner for which to retrieve specs. + * @param totalRunners - Total number of runners. + * @param weightedSpecs - Weighted spec data from which to retrieve specs. + * + * @returns Weighted specs for runner with index `runnerIndex`. + */ +const getWeightedRunnerSpecs = ( + runnerIndex: number, + totalRunners: number, + weightedSpecs: SpecWeight[] +): WeightedRunnerSpecs => { + const weightSimulationResults: { + specs: string[]; + weight: number; + }[] = Array.from({ length: totalRunners }, () => ({ + specs: [], + weight: 0, + })); + + weightedSpecs.forEach((weightedSpec) => { + // Ensure lowest weighed runner is at index 0. + weightSimulationResults.sort((a, b) => a.weight - b.weight); + weightSimulationResults[0].specs.push(weightedSpec.filepath); + weightSimulationResults[0].weight += weightedSpec.weight; + }); + + return weightSimulationResults[runnerIndex - 1]; +};