diff --git a/docs/guide/features.md b/docs/guide/features.md index 66381496d467..5e642597a55e 100644 --- a/docs/guide/features.md +++ b/docs/guide/features.md @@ -232,10 +232,12 @@ test('my types work properly', () => { ## Sharding Run tests on different machines using [`--shard`](/guide/cli#shard) and [`--reporter=blob`](/guide/reporters#blob-reporter) flags. -All test results can be merged at the end of your CI pipeline using `--merge-reports` command: +All test and coverage results can be merged at the end of your CI pipeline using `--merge-reports` command: ```bash vitest --shard=1/2 --reporter=blob vitest --shard=2/2 --reporter=blob -vitest --merge-reports --reporter=junit +vitest --merge-reports --reporter=junit --coverage.reporter=text ``` + +See [`Improving Performance | Sharding`](/guide/improving-performance#sharding) for more information. diff --git a/docs/guide/improving-performance.md b/docs/guide/improving-performance.md index 71c6a0edba3e..aa7b61119ea2 100644 --- a/docs/guide/improving-performance.md +++ b/docs/guide/improving-performance.md @@ -72,3 +72,116 @@ export default defineConfig({ }) ``` ::: + +## Sharding + +Test sharding means running a small subset of test cases at a time. It can be useful when you have multiple machines that could be used to run tests simultaneously. + +To split Vitest tests on multiple different runs, use [`--shard`](/guide/cli#shard) option with [`--reporter=blob`](/guide/reporters#blob-reporter) option: + +```sh +vitest run --reporter=blob --shard=1/3 # 1st machine +vitest run --reporter=blob --shard=2/3 # 2nd machine +vitest run --reporter=blob --shard=3/3 # 3rd machine +``` + +Collect the results stored in `.vitest-reports` directory from each machine and merge them with [`--merge-reports`](/guide/cli#merge-reports) option: + +```sh +vitest --merge-reports +``` + +
+ Github action example + +This setup is also used at https://github.com/vitest-tests/test-sharding. + +```yaml +# Inspired from https://playwright.dev/docs/test-sharding +name: Tests +on: + push: + branches: + - main +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + shardIndex: [1, 2, 3, 4] + shardTotal: [4] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install pnpm + uses: pnpm/action-setup@v4 + + - name: Install dependencies + run: pnpm i + + - name: Run tests + run: pnpm run test --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + + - name: Upload blob report to GitHub Actions Artifacts + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: blob-report-${{ matrix.shardIndex }} + path: .vitest-reports/* + retention-days: 1 + + merge-reports: + if: ${{ !cancelled() }} + needs: [tests] + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install pnpm + uses: pnpm/action-setup@v4 + + - name: Install dependencies + run: pnpm i + + - name: Download blob reports from GitHub Actions Artifacts + uses: actions/download-artifact@v4 + with: + path: .vitest-reports + pattern: blob-report-* + merge-multiple: true + + - name: Merge reports + run: npx vitest --merge-reports +``` + +
+ +:::tip +Test sharding can also become useful on high CPU-count machines. + +Vitest will run only a single Vite server in its main thread. Rest of the threads are used to run test files. +In a high CPU-count machine the main thread can become a bottleneck as it cannot handle all the requests coming from the threads. For example in 32 CPU machine the main thread is responsible to handle load coming from 31 test threads. + +To reduce the load from main thread's Vite server you can use test sharding. The load can be balanced on multiple Vite server. + +```sh +# Example for splitting tests on 32 CPU to 4 shards. +# As each process needs 1 main thread, there's 7 threads for test runners (1+7)*4 = 32 +# Use VITEST_MAX_THREADS or VITEST_MAX_FORKS depending on the pool: +VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=1/4 & \ +VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=2/4 & \ +VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=3/4 & \ +VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=4/4 & \ +wait # https://man7.org/linux/man-pages/man2/waitpid.2.html + +vitest --merge-reports +``` + +::: diff --git a/packages/coverage-istanbul/src/provider.ts b/packages/coverage-istanbul/src/provider.ts index 28cfd7a394e2..ee08224bfc8c 100644 --- a/packages/coverage-istanbul/src/provider.ts +++ b/packages/coverage-istanbul/src/provider.ts @@ -162,7 +162,7 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co this.pendingPromises = [] } - async reportCoverage({ allTestsRun }: ReportContext = {}) { + async generateCoverage({ allTestsRun }: ReportContext) { const coverageMap = libCoverage.createCoverageMap({}) let index = 0 const total = this.pendingPromises.length @@ -202,6 +202,29 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co coverageMap.merge(await transformCoverage(uncoveredCoverage)) } + return coverageMap + } + + async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) { + await this.generateReports( + coverageMap as CoverageMap || libCoverage.createCoverageMap({}), + allTestsRun, + ) + + // In watch mode we need to preserve the previous results if cleanOnRerun is disabled + const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch + + if (!keepResults) { + this.coverageFiles = new Map() + await fs.rm(this.coverageFilesDirectory, { recursive: true }) + + // Remove empty reports directory, e.g. when only text-reporter is used + if (readdirSync(this.options.reportsDirectory).length === 0) + await fs.rm(this.options.reportsDirectory, { recursive: true }) + } + } + + async generateReports(coverageMap: CoverageMap, allTestsRun: boolean | undefined) { const context = libReport.createContext({ dir: this.options.reportsDirectory, coverageMap, @@ -248,21 +271,18 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co }) } } + } - // In watch mode we need to preserve the previous results if cleanOnRerun is disabled - const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch + async mergeReports(coverageMaps: unknown[]) { + const coverageMap = libCoverage.createCoverageMap({}) - if (!keepResults) { - this.coverageFiles = new Map() - await fs.rm(this.coverageFilesDirectory, { recursive: true }) + for (const coverage of coverageMaps) + coverageMap.merge(coverage as CoverageMap) - // Remove empty reports directory, e.g. when only text-reporter is used - if (readdirSync(this.options.reportsDirectory).length === 0) - await fs.rm(this.options.reportsDirectory, { recursive: true }) - } + await this.generateReports(coverageMap, true) } - async getCoverageMapForUncoveredFiles(coveredFiles: string[]) { + private async getCoverageMapForUncoveredFiles(coveredFiles: string[]) { const allFiles = await this.testExclude.glob(this.ctx.config.root) let includedFiles = allFiles.map(file => resolve(this.ctx.config.root, file)) diff --git a/packages/coverage-v8/src/provider.ts b/packages/coverage-v8/src/provider.ts index cf4a342dfb6f..9b08b466c243 100644 --- a/packages/coverage-v8/src/provider.ts +++ b/packages/coverage-v8/src/provider.ts @@ -148,10 +148,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage this.pendingPromises.push(promise) } - async reportCoverage({ allTestsRun }: ReportContext = {}) { - if (provider === 'stackblitz') - this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.')) - + async generateCoverage({ allTestsRun }: ReportContext) { const coverageMap = libCoverage.createCoverageMap({}) let index = 0 const total = this.pendingPromises.length @@ -193,6 +190,32 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage coverageMap.merge(await transformCoverage(converted)) } + return coverageMap + } + + async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) { + if (provider === 'stackblitz') + this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.')) + + await this.generateReports( + coverageMap as CoverageMap || libCoverage.createCoverageMap({}), + allTestsRun, + ) + + // In watch mode we need to preserve the previous results if cleanOnRerun is disabled + const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch + + if (!keepResults) { + this.coverageFiles = new Map() + await fs.rm(this.coverageFilesDirectory, { recursive: true }) + + // Remove empty reports directory, e.g. when only text-reporter is used + if (readdirSync(this.options.reportsDirectory).length === 0) + await fs.rm(this.options.reportsDirectory, { recursive: true }) + } + } + + async generateReports(coverageMap: CoverageMap, allTestsRun?: boolean) { const context = libReport.createContext({ dir: this.options.reportsDirectory, coverageMap, @@ -239,18 +262,15 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage }) } } + } - // In watch mode we need to preserve the previous results if cleanOnRerun is disabled - const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch + async mergeReports(coverageMaps: unknown[]) { + const coverageMap = libCoverage.createCoverageMap({}) - if (!keepResults) { - this.coverageFiles = new Map() - await fs.rm(this.coverageFilesDirectory, { recursive: true }) + for (const coverage of coverageMaps) + coverageMap.merge(coverage as CoverageMap) - // Remove empty reports directory, e.g. when only text-reporter is used - if (readdirSync(this.options.reportsDirectory).length === 0) - await fs.rm(this.options.reportsDirectory, { recursive: true }) - } + await this.generateReports(coverageMap, true) } private async getUntestedFiles(testedFiles: string[]): Promise { diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index 6ef32188c4e0..83a4d05948e8 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -393,7 +393,7 @@ export class Vitest { if (this.reporters.some(r => r instanceof BlobReporter)) throw new Error('Cannot merge reports when `--reporter=blob` is used. Remove blob reporter from the config first.') - const { files, errors } = await readBlobs(this.config.mergeReports, this.projects) + const { files, errors, coverages } = await readBlobs(this.config.mergeReports, this.projects) await this.report('onInit', this) await this.report('onPathsCollected', files.flatMap(f => f.filepath)) @@ -439,6 +439,8 @@ export class Vitest { process.exitCode = 1 await this.report('onFinished', files, errors) + await this.initCoverageProvider() + await this.coverageProvider?.mergeReports?.(coverages) } async start(filters?: string[]) { @@ -459,7 +461,9 @@ export class Vitest { // if run with --changed, don't exit if no tests are found if (!files.length) { - await this.reportCoverage(true) + // Report coverage for uncovered files + const coverage = await this.coverageProvider?.generateCoverage?.({ allTestsRun: true }) + await this.reportCoverage(coverage, true) this.logger.printNoTestFound(filters) @@ -645,8 +649,10 @@ export class Vitest { .finally(async () => { // can be duplicate files if different projects are using the same file const files = Array.from(new Set(specs.map(([, p]) => p))) - await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors()) - await this.reportCoverage(allTestsRun) + const coverage = await this.coverageProvider?.generateCoverage({ allTestsRun }) + + await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors(), coverage) + await this.reportCoverage(coverage, allTestsRun) this.runningPromise = undefined this.isFirstRun = false @@ -946,12 +952,12 @@ export class Vitest { return Array.from(new Set(files)) } - private async reportCoverage(allTestsRun: boolean) { + private async reportCoverage(coverage: unknown, allTestsRun: boolean) { if (!this.config.coverage.reportOnFailure && this.state.getCountOfFailedTests() > 0) return if (this.coverageProvider) { - await this.coverageProvider.reportCoverage({ allTestsRun }) + await this.coverageProvider.reportCoverage(coverage, { allTestsRun }) // notify coverage iframe reload for (const reporter of this.reporters) { if (reporter instanceof WebSocketReporter) diff --git a/packages/vitest/src/node/reporters/blob.ts b/packages/vitest/src/node/reporters/blob.ts index bedf64b9bf79..cfd0810fa4f1 100644 --- a/packages/vitest/src/node/reporters/blob.ts +++ b/packages/vitest/src/node/reporters/blob.ts @@ -26,7 +26,7 @@ export class BlobReporter implements Reporter { this.ctx = ctx } - async onFinished(files: File[] = [], errors: unknown[] = []) { + async onFinished(files: File[] = [], errors: unknown[] = [], coverage: unknown) { let outputFile = this.options.outputFile ?? getOutputFile(this.ctx.config, 'blob') if (!outputFile) { const shard = this.ctx.config.shard @@ -39,7 +39,7 @@ export class BlobReporter implements Reporter { return [project.getName(), [...project.server.moduleGraph.idToModuleMap.keys()]] }) - const report = stringify([this.ctx.version, files, errors, moduleKeys] satisfies MergeReport) + const report = stringify([this.ctx.version, files, errors, moduleKeys, coverage] satisfies MergeReport) const reportFile = resolve(this.ctx.config.root, outputFile) @@ -62,8 +62,8 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace const blobsFiles = await readdir(resolvedDir) const promises = blobsFiles.map(async (file) => { const content = await readFile(resolve(resolvedDir, file), 'utf-8') - const [version, files, errors, moduleKeys] = parse(content) as MergeReport - return { version, files, errors, moduleKeys } + const [version, files, errors, moduleKeys, coverage] = parse(content) as MergeReport + return { version, files, errors, moduleKeys, coverage } }) const blobs = await Promise.all(promises) @@ -108,10 +108,12 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace return time1 - time2 }) const errors = blobs.flatMap(blob => blob.errors) + const coverages = blobs.map(blob => blob.coverage) return { files, errors, + coverages, } } @@ -120,6 +122,7 @@ type MergeReport = [ files: File[], errors: unknown[], moduleKeys: MergeReportModuleKeys[], + coverage: unknown, ] type MergeReportModuleKeys = [projectName: string, moduleIds: string[]] diff --git a/packages/vitest/src/types/coverage.ts b/packages/vitest/src/types/coverage.ts index 1f84a86bb96e..8206f8493c88 100644 --- a/packages/vitest/src/types/coverage.ts +++ b/packages/vitest/src/types/coverage.ts @@ -5,18 +5,33 @@ import type { Arrayable } from './general' import type { AfterSuiteRunMeta } from './worker' type TransformResult = string | Partial | undefined | null | void +type CoverageResults = unknown export interface CoverageProvider { name: string + + /** Called when provider is being initialized before tests run */ initialize: (ctx: Vitest) => Promise | void + /** Called when setting coverage options for Vitest context (`ctx.config.coverage`) */ resolveOptions: () => ResolvedCoverageOptions + + /** Callback to clean previous reports */ clean: (clean?: boolean) => void | Promise + /** Called with coverage results after a single test file has been run */ onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void | Promise - reportCoverage: (reportContext?: ReportContext) => void | Promise + /** Callback to generate final coverage results */ + generateCoverage: (reportContext: ReportContext) => CoverageResults | Promise + + /** Callback to convert coverage results to coverage reports. Called with results returned from `generateCoverage` */ + reportCoverage: (coverage: CoverageResults, reportContext: ReportContext) => void | Promise + + /** Callback for `--merge-reports` options. Called with multiple coverage results generated by `generateCoverage`. */ + mergeReports?: (coverages: CoverageResults[]) => void | Promise + /** Callback called for instrumenting files with coverage counters. */ onFileTransform?: ( sourceCode: string, id: string, diff --git a/packages/vitest/src/types/reporter.ts b/packages/vitest/src/types/reporter.ts index be0aa2fe8062..aa1baf1abdc6 100644 --- a/packages/vitest/src/types/reporter.ts +++ b/packages/vitest/src/types/reporter.ts @@ -7,7 +7,7 @@ export interface Reporter { onPathsCollected?: (paths?: string[]) => Awaitable onSpecsCollected?: (specs?: SerializableSpec[]) => Awaitable onCollected?: (files?: File[]) => Awaitable - onFinished?: (files?: File[], errors?: unknown[]) => Awaitable + onFinished?: (files?: File[], errors?: unknown[], coverage?: unknown) => Awaitable onTaskUpdate?: (packs: TaskResultPack[]) => Awaitable onTestRemoved?: (trigger?: string) => Awaitable diff --git a/test/cli/test/create-vitest.test.ts b/test/cli/test/create-vitest.test.ts index ff24ea211a43..a4719cc1d2e6 100644 --- a/test/cli/test/create-vitest.test.ts +++ b/test/cli/test/create-vitest.test.ts @@ -24,5 +24,6 @@ it(createVitest, async () => { }, ], [], + undefined, ]) }) diff --git a/test/coverage-test/coverage-report-tests/merge-reports.test.ts b/test/coverage-test/coverage-report-tests/merge-reports.test.ts new file mode 100644 index 000000000000..562cef2e7185 --- /dev/null +++ b/test/coverage-test/coverage-report-tests/merge-reports.test.ts @@ -0,0 +1,27 @@ +import { expect, test } from 'vitest' +import libCoverage from 'istanbul-lib-coverage' + +import { readCoverageJson } from '../coverage-report-tests/utils' + +test('reports are merged', async () => { + const json = await readCoverageJson('./coverage/coverage-final.json') + const coverageMap = libCoverage.createCoverageMap(json as any) + const files = coverageMap.files() + + // Two files were covered: 2/3 cases covered utils, 1/3 covered importEnv + expect(files).toMatchInlineSnapshot(` + [ + "/src/importEnv.ts", + "/src/utils.ts", + ] + `) + + const fileCoverage = coverageMap.fileCoverageFor('/src/utils.ts') + const lines = fileCoverage.getLineCoverage() + + // add() should be covered by one test file + expect(lines[2]).toBe(1) + + // multiply() should be covered by two test files + expect(lines[6]).toBe(2) +}) diff --git a/test/coverage-test/custom-provider.ts b/test/coverage-test/custom-provider.ts index b4e45c40539d..4ed22d087db0 100644 --- a/test/coverage-test/custom-provider.ts +++ b/test/coverage-test/custom-provider.ts @@ -69,7 +69,11 @@ class CustomCoverageProvider implements CoverageProvider { })) } - reportCoverage(reportContext?: ReportContext) { + generateCoverage(_reportContext: ReportContext) { + return {} + } + + reportCoverage(coverage: unknown, reportContext?: ReportContext) { this.calls.add(`reportCoverage with ${JSON.stringify(reportContext)}`) const jsonReport = JSON.stringify({ diff --git a/test/coverage-test/option-tests/merge-fixture-1.test.ts b/test/coverage-test/option-tests/merge-fixture-1.test.ts new file mode 100644 index 000000000000..c35cec13bb82 --- /dev/null +++ b/test/coverage-test/option-tests/merge-fixture-1.test.ts @@ -0,0 +1,6 @@ +import { test } from 'vitest' +import { add } from '../src/utils' + +test('cover add', () => { + add(1, 2) +}) diff --git a/test/coverage-test/option-tests/merge-fixture-2.test.ts b/test/coverage-test/option-tests/merge-fixture-2.test.ts new file mode 100644 index 000000000000..bcf1c731f677 --- /dev/null +++ b/test/coverage-test/option-tests/merge-fixture-2.test.ts @@ -0,0 +1,6 @@ +import { test } from 'vitest' +import { multiply } from '../src/utils' + +test('cover multiply', () => { + multiply(1, 2) +}) diff --git a/test/coverage-test/option-tests/merge-fixture-3.test.ts b/test/coverage-test/option-tests/merge-fixture-3.test.ts new file mode 100644 index 000000000000..0972de505f29 --- /dev/null +++ b/test/coverage-test/option-tests/merge-fixture-3.test.ts @@ -0,0 +1,11 @@ +import { test } from 'vitest' +import { multiply } from '../src/utils' +import { useImportEnv } from '../src/importEnv' + +test('cover multiply again', () => { + multiply(1, 2) +}) + +test('also cover another file', () => { + useImportEnv() +}) diff --git a/test/coverage-test/test/configuration-options.test-d.ts b/test/coverage-test/test/configuration-options.test-d.ts index e622ae91a078..5b3b8c1f7db3 100644 --- a/test/coverage-test/test/configuration-options.test-d.ts +++ b/test/coverage-test/test/configuration-options.test-d.ts @@ -106,6 +106,7 @@ test('provider module', () => { return { name: 'custom-provider', initialize(_: Vitest) {}, + generateCoverage() {}, resolveOptions(): ResolvedCoverageOptions { return { clean: true, diff --git a/test/coverage-test/testing-options.mjs b/test/coverage-test/testing-options.mjs index 690713f6d555..069632058a96 100644 --- a/test/coverage-test/testing-options.mjs +++ b/test/coverage-test/testing-options.mjs @@ -166,6 +166,34 @@ const testCases = [ } }, }, + ...[1, 2, 3].map(index => ({ + testConfig: { + name: `generate #${index} blob report`, + include: ['option-tests/merge-fixture-*.test.ts'], + reporters: 'blob', + shard: `${index}/3`, + coverage: { + reporter: [], + all: false, + include: ['src'], + }, + }, + })), + { + testConfig: { + name: 'merge blob reports', + // Pass default value - this option is publicly only available via CLI so it's a bit hacky usage here + mergeReports: '.vitest-reports', + reporter: 'dot', + coverage: { + reporter: 'json', + all: false, + }, + }, + assertionConfig: { + include: ['coverage-report-tests/merge-reports.test.ts'], + }, + }, ] for (const provider of ['v8', 'istanbul']) {