From 1fe2f095a0bc0aafb6b2dbd0cdaae79cc2e59ddd Mon Sep 17 00:00:00 2001 From: Cory Hall <43035978+corymhall@users.noreply.github.com> Date: Fri, 23 Jun 2023 05:56:55 -0400 Subject: [PATCH] feat(integ-runner): integ-runner --watch (#26087) This PR adds a new option `--watch` that runs a single integration test in watch mode. See README for more details - Full deploy ![watch-demo1](https://github.com/aws/aws-cdk/assets/43035978/2c1af717-acec-4761-8a1e-2362f5c8bc89) - Deploy without logs ![watch-demo2](https://github.com/aws/aws-cdk/assets/43035978/b859f1d3-634c-44dc-bd3d-54a0e48dc9a1) - Deploy on file change ![watch-demo3](https://github.com/aws/aws-cdk/assets/43035978/b283ddd5-7e33-4c61-8477-12f25f4996ee) ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../cdk-cli-wrapper/lib/cdk-wrapper.ts | 55 +++- .../cdk-cli-wrapper/lib/commands/deploy.ts | 47 +++ .../@aws-cdk/cdk-cli-wrapper/lib/utils.ts | 21 +- .../cdk-cli-wrapper/test/cdk-wrapper.test.ts | 34 +- packages/@aws-cdk/integ-runner/README.md | 54 ++++ .../integ-runner/THIRD_PARTY_LICENSES | 302 ++++++++++++++++++ packages/@aws-cdk/integ-runner/lib/cli.ts | 84 +++-- .../lib/runner/integ-test-runner.ts | 208 +++++++++++- .../lib/runner/integration-tests.ts | 14 +- .../integ-runner/lib/workers/common.ts | 7 + .../lib/workers/extract/extract_worker.ts | 33 +- .../lib/workers/integ-test-worker.ts | 1 + .../lib/workers/integ-watch-worker.ts | 14 + packages/@aws-cdk/integ-runner/package.json | 1 + .../@aws-cdk/integ-runner/test/helpers.ts | 19 +- .../test/runner/integ-test-runner.test.ts | 59 +++- 16 files changed, 912 insertions(+), 41 deletions(-) create mode 100644 packages/@aws-cdk/integ-runner/lib/workers/integ-watch-worker.ts diff --git a/packages/@aws-cdk/cdk-cli-wrapper/lib/cdk-wrapper.ts b/packages/@aws-cdk/cdk-cli-wrapper/lib/cdk-wrapper.ts index fc570be9f6481..78a3f749868b9 100644 --- a/packages/@aws-cdk/cdk-cli-wrapper/lib/cdk-wrapper.ts +++ b/packages/@aws-cdk/cdk-cli-wrapper/lib/cdk-wrapper.ts @@ -1,5 +1,6 @@ -import { DefaultCdkOptions, DeployOptions, DestroyOptions, SynthOptions, ListOptions, StackActivityProgress } from './commands'; -import { exec } from './utils'; +import { ChildProcess } from 'child_process'; +import { DefaultCdkOptions, DeployOptions, DestroyOptions, SynthOptions, ListOptions, StackActivityProgress, HotswapMode } from './commands'; +import { exec, watch } from './utils'; /** * AWS CDK CLI operations @@ -30,6 +31,11 @@ export interface ICdk { * cdk synth fast */ synthFast(options: SynthFastOptions): void; + + /** + * cdk watch + */ + watch(options: DeployOptions): ChildProcess; } /** @@ -176,6 +182,7 @@ export class CdkCliWrapper implements ICdk { ...options.changeSetName ? ['--change-set-name', options.changeSetName] : [], ...options.toolkitStackName ? ['--toolkit-stack-name', options.toolkitStackName] : [], ...options.progress ? ['--progress', options.progress] : ['--progress', StackActivityProgress.EVENTS], + ...options.deploymentMethod ? ['--method', options.deploymentMethod] : [], ...this.createDefaultArguments(options), ]; @@ -186,6 +193,50 @@ export class CdkCliWrapper implements ICdk { }); } + public watch(options: DeployOptions): ChildProcess { + let hotswap: string; + switch (options.hotswap) { + case HotswapMode.FALL_BACK: + hotswap = '--hotswap-fallback'; + break; + case HotswapMode.HOTSWAP_ONLY: + hotswap = '--hotswap'; + break; + default: + hotswap = '--hotswap-fallback'; + break; + } + const deployCommandArgs: string[] = [ + '--watch', + ...renderBooleanArg('ci', options.ci), + ...renderBooleanArg('execute', options.execute), + ...renderBooleanArg('exclusively', options.exclusively), + ...renderBooleanArg('force', options.force), + ...renderBooleanArg('previous-parameters', options.usePreviousParameters), + ...renderBooleanArg('rollback', options.rollback), + ...renderBooleanArg('staging', options.staging), + ...renderBooleanArg('logs', options.traceLogs), + hotswap, + ...options.reuseAssets ? renderArrayArg('--reuse-assets', options.reuseAssets) : [], + ...options.notificationArns ? renderArrayArg('--notification-arns', options.notificationArns) : [], + ...options.parameters ? renderMapArrayArg('--parameters', options.parameters) : [], + ...options.outputsFile ? ['--outputs-file', options.outputsFile] : [], + ...options.requireApproval ? ['--require-approval', options.requireApproval] : [], + ...options.changeSetName ? ['--change-set-name', options.changeSetName] : [], + ...options.toolkitStackName ? ['--toolkit-stack-name', options.toolkitStackName] : [], + ...options.progress ? ['--progress', options.progress] : ['--progress', StackActivityProgress.EVENTS], + ...options.deploymentMethod ? ['--method', options.deploymentMethod] : [], + ...this.createDefaultArguments(options), + ]; + + return watch([this.cdk, 'deploy', ...deployCommandArgs], { + cwd: this.directory, + verbose: this.showOutput, + env: this.env, + }); + + } + /** * cdk destroy */ diff --git a/packages/@aws-cdk/cdk-cli-wrapper/lib/commands/deploy.ts b/packages/@aws-cdk/cdk-cli-wrapper/lib/commands/deploy.ts index 6d05f0d9f1e6d..798b85d36d395 100644 --- a/packages/@aws-cdk/cdk-cli-wrapper/lib/commands/deploy.ts +++ b/packages/@aws-cdk/cdk-cli-wrapper/lib/commands/deploy.ts @@ -105,6 +105,53 @@ export interface DeployOptions extends DefaultCdkOptions { * @default StackActivityProgress.EVENTS */ readonly progress?: StackActivityProgress; + + /** + * Whether this 'deploy' command should actually delegate to the 'watch' command. + * + * @default false + */ + readonly watch?: boolean; + + /** + * Whether to perform a 'hotswap' deployment. + * A 'hotswap' deployment will attempt to short-circuit CloudFormation + * and update the affected resources like Lambda functions directly. + * + * @default - `HotswapMode.FALL_BACK` for regular deployments, `HotswapMode.HOTSWAP_ONLY` for 'watch' deployments + */ + readonly hotswap?: HotswapMode; + + /** + * Whether to show CloudWatch logs for hotswapped resources + * locally in the users terminal + * + * @default - false + */ + readonly traceLogs?: boolean; + + /** + * Deployment method + */ + readonly deploymentMethod?: DeploymentMethod; +} +export type DeploymentMethod = 'direct' | 'change-set'; + +export enum HotswapMode { + /** + * Will fall back to CloudFormation when a non-hotswappable change is detected + */ + FALL_BACK = 'fall-back', + + /** + * Will not fall back to CloudFormation when a non-hotswappable change is detected + */ + HOTSWAP_ONLY = 'hotswap-only', + + /** + * Will not attempt to hotswap anything and instead go straight to CloudFormation + */ + FULL_DEPLOYMENT = 'full-deployment', } /** diff --git a/packages/@aws-cdk/cdk-cli-wrapper/lib/utils.ts b/packages/@aws-cdk/cdk-cli-wrapper/lib/utils.ts index 28c52a16e23ed..0233af48fe898 100644 --- a/packages/@aws-cdk/cdk-cli-wrapper/lib/utils.ts +++ b/packages/@aws-cdk/cdk-cli-wrapper/lib/utils.ts @@ -1,5 +1,5 @@ // Helper functions for CDK Exec -import { spawnSync } from 'child_process'; +import { spawn, spawnSync } from 'child_process'; /** * Our own execute function which doesn't use shells and strings. @@ -37,3 +37,22 @@ export function exec(commandLine: string[], options: { cwd?: string, json?: bool throw new Error('Command output is not JSON'); } } + +/** + * For use with `cdk deploy --watch` + */ +export function watch(commandLine: string[], options: { cwd?: string, verbose?: boolean, env?: any } = { }) { + const proc = spawn(commandLine[0], commandLine.slice(1), { + stdio: ['ignore', 'pipe', options.verbose ? 'inherit' : 'pipe'], // inherit STDERR in verbose mode + env: { + ...process.env, + ...options.env, + }, + cwd: options.cwd, + }); + proc.on('error', (err: Error) => { + throw err; + }); + + return proc; +} diff --git a/packages/@aws-cdk/cdk-cli-wrapper/test/cdk-wrapper.test.ts b/packages/@aws-cdk/cdk-cli-wrapper/test/cdk-wrapper.test.ts index c16e94d069b07..266a7b9c81190 100644 --- a/packages/@aws-cdk/cdk-cli-wrapper/test/cdk-wrapper.test.ts +++ b/packages/@aws-cdk/cdk-cli-wrapper/test/cdk-wrapper.test.ts @@ -2,6 +2,7 @@ import * as child_process from 'child_process'; import { CdkCliWrapper } from '../lib/cdk-wrapper'; import { RequireApproval, StackActivityProgress } from '../lib/commands'; let spawnSyncMock: jest.SpyInstance; +let spawnMock: jest.SpyInstance; beforeEach(() => { spawnSyncMock = jest.spyOn(child_process, 'spawnSync').mockReturnValue({ @@ -12,6 +13,11 @@ beforeEach(() => { output: ['stdout', 'stderr'], signal: null, }); + spawnMock = jest.spyOn(child_process, 'spawn').mockImplementation(jest.fn(() => { + return { + on: jest.fn(() => {}), + } as unknown as child_process.ChildProcess; + })); }); afterEach(() => { @@ -317,7 +323,33 @@ test('default synth', () => { ); }); -test('synth arguments', () => { +test('watch arguments', () => { + // WHEN + const cdk = new CdkCliWrapper({ + directory: '/project', + env: { + KEY: 'value', + }, + }); + cdk.watch({ + app: 'node bin/my-app.js', + stacks: ['test-stack1'], + }); + + // THEN + expect(spawnMock).toHaveBeenCalledWith( + expect.stringMatching(/cdk/), + ['deploy', '--watch', '--hotswap-fallback', '--progress', 'events', '--app', 'node bin/my-app.js', 'test-stack1'], + expect.objectContaining({ + env: expect.objectContaining({ + KEY: 'value', + }), + cwd: '/project', + }), + ); +}); + +test('destroy arguments', () => { // WHEN const cdk = new CdkCliWrapper({ directory: '/project', diff --git a/packages/@aws-cdk/integ-runner/README.md b/packages/@aws-cdk/integ-runner/README.md index 350087e45bc8b..e38f23f393b38 100644 --- a/packages/@aws-cdk/integ-runner/README.md +++ b/packages/@aws-cdk/integ-runner/README.md @@ -75,6 +75,10 @@ to be a self contained CDK app. The runner will execute the following for each f - `--test-regex` Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected. +- `--watch` + Run a single integration test in watch mode. In watch mode the integ-runner + will not save any snapshots. + Use together with `--app` to fully customize how tests are run, or use with a single `--language` preset to change which files are detected for this language. - `--language` The language presets to use. You can discover and run tests written in multiple languages by passing this flag multiple times (`--language typescript --language python`). Defaults to all supported languages. Currently supported language presets are: @@ -221,6 +225,56 @@ integ-runner --update-on-failed --disable-update-workflow integ.new-test.js This is because for a new test we do not need to test the update workflow (there is nothing to update). +### watch + +It can be useful to run an integration test in watch mode when you are iterating +on a specific test. + +```console +integ-runner integ.new-test.js --watch +``` + +In watch mode the integ test will run similar to `cdk deploy --watch` with the +addition of also displaying the assertion results. By default the output will +only show the assertion results. + +- To show the console output from watch run with `-v` +- To also stream the CloudWatch logs (i.e. `cdk deploy --watch --logs`) run with `-vv` + +When running in watch mode most of the integ-runner functionality will be turned +off. + +- Snapshots will not be created +- Update workflow will not be run +- Stacks will not be cleaned up (you must manually clean up the stacks) +- Only a single test can be run + +Once you are done iterating using watch and want to create the snapshot you can +run the integ test like normal to create the snapshot and clean up the test. + +#### cdk.context.json + +cdk watch depends on a `cdk.context.json` file existing with a `watch` key. The +integ-runner will create a default `cdk.context.json` file if one does not +exist. + +```json +{ + "watch": {} +} +``` + +You can further edit this file after it is created and add additional `watch` +fields. For example: + +```json +{ + "watch": { + "include": ["**/*.js"] + } +} +``` + ### integ.json schema See [@aws-cdk/cloud-assembly-schema/lib/integ-tests/schema.ts](../cloud-assembly-schema/lib/integ-tests/schema.ts) diff --git a/packages/@aws-cdk/integ-runner/THIRD_PARTY_LICENSES b/packages/@aws-cdk/integ-runner/THIRD_PARTY_LICENSES index 1c0f1383e638b..1524800b29732 100644 --- a/packages/@aws-cdk/integ-runner/THIRD_PARTY_LICENSES +++ b/packages/@aws-cdk/integ-runner/THIRD_PARTY_LICENSES @@ -53,6 +53,26 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +---------------- + +** anymatch@3.1.3 - https://www.npmjs.com/package/anymatch/v/3.1.3 | ISC +The ISC License + +Copyright (c) 2019 Elan Shanker, Paul Miller (https://paulmillr.com) + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR +IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + ---------------- ** archiver-utils@2.1.0 - https://www.npmjs.com/package/archiver-utils/v/2.1.0 | MIT @@ -168,6 +188,20 @@ Amazon Web Services, Inc. (http://aws.amazon.com/). ** balanced-match@1.0.2 - https://www.npmjs.com/package/balanced-match/v/1.0.2 | MIT +---------------- + +** binary-extensions@2.2.0 - https://www.npmjs.com/package/binary-extensions/v/2.2.0 | MIT +MIT License + +Copyright (c) 2019 Sindre Sorhus (https://sindresorhus.com), Paul Miller (https://paulmillr.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + ---------------- ** bl@4.1.0 - https://www.npmjs.com/package/bl/v/4.1.0 | MIT @@ -224,6 +258,32 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +---------------- + +** braces@3.0.2 - https://www.npmjs.com/package/braces/v/3.0.2 | MIT +The MIT License (MIT) + +Copyright (c) 2014-2018, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + ---------------- ** buffer-crc32@0.2.13 - https://www.npmjs.com/package/buffer-crc32/v/0.2.13 | MIT @@ -262,6 +322,32 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +---------------- + +** chokidar@3.5.3 - https://www.npmjs.com/package/chokidar/v/3.5.3 | MIT +The MIT License (MIT) + +Copyright (c) 2012-2019 Paul Miller (https://paulmillr.com), Elan Shanker + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + ---------------- ** cliui@7.0.4 - https://www.npmjs.com/package/cliui/v/7.0.4 | ISC @@ -749,6 +835,32 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +---------------- + +** fill-range@7.0.1 - https://www.npmjs.com/package/fill-range/v/7.0.1 | MIT +The MIT License (MIT) + +Copyright (c) 2014-present, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + ---------------- ** fs-constants@1.0.0 - https://www.npmjs.com/package/fs-constants/v/1.0.0 | MIT @@ -847,6 +959,26 @@ the licensed code: ** get-caller-file@2.0.5 - https://www.npmjs.com/package/get-caller-file/v/2.0.5 | ISC +---------------- + +** glob-parent@5.1.2 - https://www.npmjs.com/package/glob-parent/v/5.1.2 | ISC +The ISC License + +Copyright (c) 2015, 2019 Elan Shanker + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR +IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + ---------------- ** glob@7.2.3 - https://www.npmjs.com/package/glob/v/7.2.3 | ISC @@ -948,6 +1080,46 @@ PERFORMANCE OF THIS SOFTWARE. +---------------- + +** is-binary-path@2.1.0 - https://www.npmjs.com/package/is-binary-path/v/2.1.0 | MIT +MIT License + +Copyright (c) 2019 Sindre Sorhus (https://sindresorhus.com), Paul Miller (https://paulmillr.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +---------------- + +** is-extglob@2.1.1 - https://www.npmjs.com/package/is-extglob/v/2.1.1 | MIT +The MIT License (MIT) + +Copyright (c) 2014-2016, Jon Schlinkert + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + ---------------- ** is-fullwidth-code-point@3.0.0 - https://www.npmjs.com/package/is-fullwidth-code-point/v/3.0.0 | MIT @@ -962,6 +1134,58 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +---------------- + +** is-glob@4.0.3 - https://www.npmjs.com/package/is-glob/v/4.0.3 | MIT +The MIT License (MIT) + +Copyright (c) 2014-2017, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +---------------- + +** is-number@7.0.0 - https://www.npmjs.com/package/is-number/v/7.0.0 | MIT +The MIT License (MIT) + +Copyright (c) 2014-present, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + ---------------- ** isarray@1.0.0 - https://www.npmjs.com/package/isarray/v/1.0.0 | MIT @@ -1528,6 +1752,32 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +---------------- + +** picomatch@2.3.1 - https://www.npmjs.com/package/picomatch/v/2.3.1 | MIT +The MIT License (MIT) + +Copyright (c) 2017-present, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + ---------------- ** process-nextick-args@2.0.1 - https://www.npmjs.com/package/process-nextick-args/v/2.0.1 | MIT @@ -1841,6 +2091,32 @@ IN THE SOFTWARE. See the License for the specific language governing permissions and limitations under the License. +---------------- + +** readdirp@3.6.0 - https://www.npmjs.com/package/readdirp/v/3.6.0 | MIT +MIT License + +Copyright (c) 2012-2019 Thorsten Lorenz, Paul Miller (https://paulmillr.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + ---------------- ** require-directory@2.1.1 - https://www.npmjs.com/package/require-directory/v/2.1.1 | MIT @@ -2203,6 +2479,32 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +---------------- + +** to-regex-range@5.0.1 - https://www.npmjs.com/package/to-regex-range/v/5.0.1 | MIT +The MIT License (MIT) + +Copyright (c) 2015-present, Jon Schlinkert. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + ---------------- ** universalify@2.0.0 - https://www.npmjs.com/package/universalify/v/2.0.0 | MIT diff --git a/packages/@aws-cdk/integ-runner/lib/cli.ts b/packages/@aws-cdk/integ-runner/lib/cli.ts index eb1a0134192a5..3334cd11e5435 100644 --- a/packages/@aws-cdk/integ-runner/lib/cli.ts +++ b/packages/@aws-cdk/integ-runner/lib/cli.ts @@ -4,8 +4,9 @@ import * as path from 'path'; import * as chalk from 'chalk'; import * as workerpool from 'workerpool'; import * as logger from './logger'; -import { IntegrationTests, IntegTestInfo } from './runner/integration-tests'; +import { IntegrationTests, IntegTest, IntegTestInfo } from './runner/integration-tests'; import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers'; +import { watchIntegrationTest } from './workers/integ-watch-worker'; // https://github.com/yargs/yargs/issues/1929 // https://github.com/evanw/esbuild/issues/1492 @@ -21,6 +22,7 @@ export function parseCliArgs(args: string[] = []) { default: 'integ.config.json', desc: 'Load options from a JSON config file. Options provided as CLI arguments take precedent.', }) + .option('watch', { type: 'boolean', default: false, desc: 'Perform integ tests in watch mode' }) .option('list', { type: 'boolean', default: false, desc: 'List tests instead of running them' }) .option('clean', { type: 'boolean', default: true, desc: 'Skips stack clean up after test is completed (use --no-clean to negate)' }) .option('verbose', { type: 'boolean', default: false, alias: 'v', count: true, desc: 'Verbose logs and metrics on integration tests durations (specify multiple times to increase verbosity)' }) @@ -73,6 +75,7 @@ export function parseCliArgs(args: string[] = []) { app: argv.app as (string | undefined), testRegex: arrayFromYargs(argv['test-regex']), testRegions, + originalRegions: parallelRegions, profiles, runUpdateOnFailed: (argv['update-on-failed'] ?? false) as boolean, fromFile, @@ -88,6 +91,7 @@ export function parseCliArgs(args: string[] = []) { dryRun: argv['dry-run'] as boolean, disableUpdateWorkflow: argv['disable-update-workflow'] as boolean, language: arrayFromYargs(argv.language), + watch: argv.watch as boolean, }; } @@ -96,38 +100,47 @@ export async function main(args: string[]) { const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliOptions(options); - // List only prints the discoverd tests + // List only prints the discovered tests if (options.list) { process.stdout.write(testsFromArgs.map(t => t.discoveryRelativeFileName).join('\n') + '\n'); return; } const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), { - maxWorkers: options.maxWorkers, + maxWorkers: options.watch ? 1 : options.maxWorkers, }); const testsToRun: IntegTestWorkerConfig[] = []; const destructiveChanges: DestructiveChange[] = []; let failedSnapshots: IntegTestWorkerConfig[] = []; let testsSucceeded = false; + validateWatchArgs({ + ...options, + testRegions: options.originalRegions, + tests: testsFromArgs, + }); try { - // always run snapshot tests, but if '--force' is passed then - // run integration tests on all failed tests, not just those that - // failed snapshot tests - failedSnapshots = await runSnapshotTests(pool, testsFromArgs, { - retain: options.inspectFailures, - verbose: options.verbose, - }); - for (const failure of failedSnapshots) { - destructiveChanges.push(...failure.destructiveChanges ?? []); - } - if (!options.force) { - testsToRun.push(...failedSnapshots); + if (!options.watch) { + // always run snapshot tests, but if '--force' is passed then + // run integration tests on all failed tests, not just those that + // failed snapshot tests + failedSnapshots = await runSnapshotTests(pool, testsFromArgs, { + retain: options.inspectFailures, + verbose: options.verbose, + }); + for (const failure of failedSnapshots) { + destructiveChanges.push(...failure.destructiveChanges ?? []); + } + if (!options.force) { + testsToRun.push(...failedSnapshots); + } else { + // if any of the test failed snapshot tests, keep those results + // and merge with the rest of the tests from args + testsToRun.push(...mergeTests(testsFromArgs.map(t => t.info), failedSnapshots)); + } } else { - // if any of the test failed snapshot tests, keep those results - // and merge with the rest of the tests from args - testsToRun.push(...mergeTests(testsFromArgs.map(t => t.info), failedSnapshots)); + testsToRun.push(...testsFromArgs.map(t => t.info)); } // run integration tests if `--update-on-failed` OR `--force` is used @@ -141,6 +154,7 @@ export async function main(args: string[]) { dryRun: options.dryRun, verbosity: options.verbosity, updateWorkflow: !options.disableUpdateWorkflow, + watch: options.watch, }); testsSucceeded = success; @@ -155,6 +169,14 @@ export async function main(args: string[]) { if (!success) { throw new Error('Some integration tests failed!'); } + } else if (options.watch) { + await watchIntegrationTest(pool, { + watch: true, + verbosity: options.verbosity, + ...testsToRun[0], + profile: options.profiles ? options.profiles[0] : undefined, + region: options.testRegions[0], + }); } } finally { void pool.terminate(); @@ -176,6 +198,32 @@ export async function main(args: string[]) { } +function validateWatchArgs(args: { + tests: IntegTest[], + testRegions?: string[], + profiles?: string[], + maxWorkers: number, + force: boolean, + dryRun: boolean, + disableUpdateWorkflow: boolean, + runUpdateOnFailed: boolean, + watch: boolean, +}) { + if (args.watch) { + if ( + (args.testRegions && args.testRegions.length > 1) + || (args.profiles && args.profiles.length > 1) + || args.tests.length > 1) { + throw new Error('Running with watch only supports a single test. Only provide a single option'+ + 'to `--profiles` `--parallel-regions` `--max-workers'); + } + + if (args.runUpdateOnFailed || args.disableUpdateWorkflow || args.force || args.dryRun) { + logger.warning('args `--update-on-failed`, `--disable-update-workflow`, `--force`, `--dry-run` have no effect when running with `--watch`'); + } + } +} + function printDestructiveChanges(changes: DestructiveChange[]): void { if (changes.length > 0) { logger.warning('!!! This test contains %s !!!', chalk.bold('destructive changes')); diff --git a/packages/@aws-cdk/integ-runner/lib/runner/integ-test-runner.ts b/packages/@aws-cdk/integ-runner/lib/runner/integ-test-runner.ts index d2799f5fe02a4..d25293036c906 100644 --- a/packages/@aws-cdk/integ-runner/lib/runner/integ-test-runner.ts +++ b/packages/@aws-cdk/integ-runner/lib/runner/integ-test-runner.ts @@ -1,21 +1,34 @@ import * as path from 'path'; -import { DeployOptions, DestroyOptions } from '@aws-cdk/cdk-cli-wrapper'; +import { DeployOptions, DestroyOptions, HotswapMode, StackActivityProgress } from '@aws-cdk/cdk-cli-wrapper'; import { RequireApproval } from '@aws-cdk/cloud-assembly-schema'; +import * as chokidar from 'chokidar'; import * as fs from 'fs-extra'; +import * as workerpool from 'workerpool'; import { IntegRunnerOptions, IntegRunner, DEFAULT_SYNTH_OPTIONS } from './runner-base'; import * as logger from '../logger'; import { chunks, exec } from '../utils'; -import { DestructiveChange, AssertionResults, AssertionResult } from '../workers/common'; +import { DestructiveChange, AssertionResults, AssertionResult, DiagnosticReason, formatAssertionResults } from '../workers/common'; -/** - * Options for the integration test runner - */ -export interface RunOptions { +export interface CommonOptions { /** * The name of the test case */ readonly testCaseName: string; + /** + * The level of verbosity for logging. + * + * @default 0 + */ + readonly verbosity?: number; +} + +export interface WatchOptions extends CommonOptions { } + +/** + * Options for the integration test runner + */ +export interface RunOptions extends CommonOptions { /** * Whether or not to run `cdk destroy` and cleanup the * integration test stacks. @@ -48,13 +61,6 @@ export interface RunOptions { * @default true */ readonly updateWorkflow?: boolean; - - /** - * The level of verbosity for logging. - * - * @default 0 - */ - readonly verbosity?: number; } /** @@ -77,6 +83,14 @@ export class IntegTestRunner extends IntegRunner { } } + public createCdkContextJson(): void { + if (!fs.existsSync(this.cdkContextPath)) { + fs.writeFileSync(this.cdkContextPath, JSON.stringify({ + watch: { }, + }, undefined, 2)); + } + } + /** * When running integration tests with the update path workflow * it is important that the snapshot that is deployed is the current snapshot @@ -135,6 +149,42 @@ export class IntegTestRunner extends IntegRunner { } } + /** + * Runs cdk deploy --watch for an integration test + * + * This is meant to be run on a single test and will not create a snapshot + */ + public async watchIntegTest(options: WatchOptions): Promise { + const actualTestCase = this.actualTestSuite.testSuite[options.testCaseName]; + if (!actualTestCase) { + throw new Error(`Did not find test case name '${options.testCaseName}' in '${Object.keys(this.actualTestSuite.testSuite)}'`); + } + const enableForVerbosityLevel = (needed = 1) => { + const verbosity = options.verbosity ?? 0; + return (verbosity >= needed) ? true : undefined; + }; + try { + await this.watch( + { + ...this.defaultArgs, + progress: StackActivityProgress.BAR, + hotswap: HotswapMode.FALL_BACK, + deploymentMethod: 'direct', + profile: this.profile, + requireApproval: RequireApproval.NEVER, + traceLogs: enableForVerbosityLevel(2) ?? false, + verbose: enableForVerbosityLevel(3), + debug: enableForVerbosityLevel(4), + watch: true, + }, + options.testCaseName, + options.verbosity ?? 0, + ); + } catch (e) { + throw e; + } + } + /** * Orchestrates running integration tests. Currently this includes * @@ -247,6 +297,138 @@ export class IntegTestRunner extends IntegRunner { } } + private async watch(watchArgs: DeployOptions, testCaseName: string, verbosity: number): Promise { + const actualTestCase = this.actualTestSuite.testSuite[testCaseName]; + if (actualTestCase.hooks?.preDeploy) { + actualTestCase.hooks.preDeploy.forEach(cmd => { + exec(chunks(cmd), { + cwd: path.dirname(this.snapshotDir), + }); + }); + } + const deployArgs = { + ...watchArgs, + lookups: this.actualTestSuite.enableLookups, + stacks: [ + ...actualTestCase.stacks, + ...actualTestCase.assertionStack ? [actualTestCase.assertionStack] : [], + ], + output: path.relative(this.directory, this.cdkOutDir), + outputsFile: path.relative(this.directory, path.join(this.cdkOutDir, 'assertion-results.json')), + ...actualTestCase?.cdkCommandOptions?.deploy?.args, + context: { + ...this.getContext(actualTestCase?.cdkCommandOptions?.deploy?.args?.context), + }, + app: this.cdkApp, + }; + const destroyMessage = { + additionalMessages: [ + 'After you are done you must manually destroy the deployed stacks', + ` ${[ + ...process.env.AWS_REGION ? [`AWS_REGION=${process.env.AWS_REGION}`] : [], + 'cdk destroy', + `-a '${this.cdkApp}'`, + deployArgs.stacks.join(' '), + `--profile ${deployArgs.profile}`, + ].join(' ')}`, + ], + }; + workerpool.workerEmit(destroyMessage); + if (watchArgs.verbose) { + // if `-vvv` (or above) is used then print out the command that was used + // this allows users to manually run the command + workerpool.workerEmit({ + additionalMessages: [ + 'Repro:', + ` ${[ + 'cdk synth', + `-a '${this.cdkApp}'`, + `-o '${this.cdkOutDir}'`, + ...Object.entries(this.getContext()).flatMap(([k, v]) => typeof v !== 'object' ? [`-c '${k}=${v}'`] : []), + deployArgs.stacks.join(' '), + `--outputs-file ${deployArgs.outputsFile}`, + `--profile ${deployArgs.profile}`, + '--hotswap-fallback', + ].join(' ')}`, + ], + }); + } + + const assertionResults = path.join(this.cdkOutDir, 'assertion-results.json'); + const watcher = chokidar.watch([this.cdkOutDir], { + cwd: this.directory, + }); + watcher.on('all', (event: 'add' | 'change', file: string) => { + // we only care about changes to the `assertion-results.json` file. If there + // are assertions then this will change on every deployment + if (assertionResults.endsWith(file) && (event === 'add' || event === 'change')) { + const start = Date.now(); + if (actualTestCase.hooks?.postDeploy) { + actualTestCase.hooks.postDeploy.forEach(cmd => { + exec(chunks(cmd), { + cwd: path.dirname(this.snapshotDir), + }); + }); + } + + if (actualTestCase.assertionStack && actualTestCase.assertionStackName) { + const res = this.processAssertionResults( + assertionResults, + actualTestCase.assertionStackName, + actualTestCase.assertionStack, + ); + if (res && Object.values(res).some(r => r.status === 'fail')) { + workerpool.workerEmit({ + reason: DiagnosticReason.ASSERTION_FAILED, + testName: `${testCaseName} (${watchArgs.profile}`, + message: formatAssertionResults(res), + duration: (Date.now() - start) / 1000, + }); + } else { + workerpool.workerEmit({ + reason: DiagnosticReason.TEST_SUCCESS, + testName: `${testCaseName}`, + message: res ? formatAssertionResults(res) : 'NO ASSERTIONS', + duration: (Date.now() - start) / 1000, + }); + } + // emit the destroy message after every run + // so that it's visible to the user + workerpool.workerEmit(destroyMessage); + } + } + }); + await new Promise(resolve => { + watcher.on('ready', async () => { + resolve({}); + }); + }); + + const child = this.cdk.watch(deployArgs); + // if `-v` (or above) is passed then stream the logs + child.stdout?.on('data', (message) => { + if (verbosity > 0) { + process.stdout.write(message); + } + }); + child.stderr?.on('data', (message) => { + if (verbosity > 0) { + process.stderr.write(message); + } + }); + + await new Promise(resolve => { + child.on('close', async (code) => { + if (code !== 0) { + throw new Error('Watch exited with error'); + } + child.stdin?.end(); + await watcher.close(); + resolve(code); + }); + }); + } + /** * Perform a integ test case deployment, including * peforming the update workflow diff --git a/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts b/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts index 9aa48874428b1..16702e5c26a4a 100644 --- a/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts +++ b/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts @@ -31,6 +31,13 @@ export interface IntegTestInfo { * @default - test run command will be `node {filePath}` */ readonly appCommand?: string; + + /** + * true if this test is running in watch mode + * + * @default false + */ + readonly watch?: boolean; } /** @@ -102,7 +109,8 @@ export class IntegTest { const parsed = path.parse(this.fileName); this.discoveryRelativeFileName = path.relative(info.discoveryRoot, info.fileName); - this.directory = parsed.dir; + // if `--watch` then we need the directory to be the cwd + this.directory = info.watch ? process.cwd() : parsed.dir; // if we are running in a package directory then just use the fileName // as the testname, but if we are running in a parent directory with @@ -115,8 +123,8 @@ export class IntegTest { : path.join(path.relative(this.info.discoveryRoot, parsed.dir), parsed.name); this.normalizedTestName = parsed.name; - this.snapshotDir = path.join(this.directory, `${parsed.base}.snapshot`); - this.temporaryOutputDir = path.join(this.directory, `${CDK_OUTDIR_PREFIX}.${parsed.base}.snapshot`); + this.snapshotDir = path.join(parsed.dir, `${parsed.base}.snapshot`); + this.temporaryOutputDir = path.join(parsed.dir, `${CDK_OUTDIR_PREFIX}.${parsed.base}.snapshot`); } /** diff --git a/packages/@aws-cdk/integ-runner/lib/workers/common.ts b/packages/@aws-cdk/integ-runner/lib/workers/common.ts index b42fc9da81360..e76c304b4f9f0 100644 --- a/packages/@aws-cdk/integ-runner/lib/workers/common.ts +++ b/packages/@aws-cdk/integ-runner/lib/workers/common.ts @@ -161,6 +161,13 @@ export interface IntegTestOptions { * @default true */ readonly updateWorkflow?: boolean; + + /** + * true if running in watch mode + * + * @default false + */ + readonly watch?: boolean; } /** diff --git a/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts b/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts index 5aad4af1ad495..b6de31f906602 100644 --- a/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts +++ b/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts @@ -3,6 +3,7 @@ import { IntegSnapshotRunner, IntegTestRunner } from '../../runner'; import { IntegTest, IntegTestInfo } from '../../runner/integration-tests'; import { DiagnosticReason, IntegTestWorkerConfig, SnapshotVerificationOptions, Diagnostic, formatAssertionResults } from '../common'; import { IntegTestBatchRequest } from '../integ-test-worker'; +import { IntegWatchOptions } from '../integ-watch-worker'; /** * Runs a single integration test batch request. @@ -17,7 +18,10 @@ export function integTestWorker(request: IntegTestBatchRequest): IntegTestWorker const verbosity = request.verbosity ?? 0; for (const testInfo of request.tests) { - const test = new IntegTest(testInfo); // Hydrate from data + const test = new IntegTest({ + ...testInfo, + watch: request.watch, + }); // Hydrate from data const start = Date.now(); try { @@ -85,6 +89,32 @@ export function integTestWorker(request: IntegTestBatchRequest): IntegTestWorker return failures; } +export async function watchTestWorker(options: IntegWatchOptions) { + const verbosity = options.verbosity ?? 0; + const test = new IntegTest(options); + const runner = new IntegTestRunner({ + test, + profile: options.profile, + env: { + AWS_REGION: options.region, + CDK_DOCKER: process.env.CDK_DOCKER ?? 'docker', + }, + showOutput: verbosity >= 2, + }); + runner.createCdkContextJson(); + const tests = runner.actualTests(); + + if (!tests || Object.keys(tests).length === 0) { + throw new Error(`No tests defined for ${runner.testName}`); + } + for (const testCaseName of Object.keys(tests)) { + await runner.watchIntegTest({ + testCaseName, + verbosity, + }); + } +} + /** * Runs a single snapshot test batch request. * For each integration test this will check to see @@ -153,4 +183,5 @@ export function snapshotTestWorker(testInfo: IntegTestInfo, options: SnapshotVer workerpool.worker({ snapshotTestWorker, integTestWorker, + watchTestWorker, }); diff --git a/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts b/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts index 13ec1ec2de283..1bbbe1f5b18e0 100644 --- a/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts +++ b/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts @@ -128,6 +128,7 @@ export async function runIntegrationTestsInParallel( const testStart = Date.now(); logger.highlight(`Running test ${test.fileName} in ${worker.profile ? worker.profile + '/' : ''}${worker.region}`); const response: IntegTestInfo[][] = await options.pool.exec('integTestWorker', [{ + watch: options.watch, region: worker.region, profile: worker.profile, tests: [test], diff --git a/packages/@aws-cdk/integ-runner/lib/workers/integ-watch-worker.ts b/packages/@aws-cdk/integ-runner/lib/workers/integ-watch-worker.ts new file mode 100644 index 0000000000000..16f5ff926e0b0 --- /dev/null +++ b/packages/@aws-cdk/integ-runner/lib/workers/integ-watch-worker.ts @@ -0,0 +1,14 @@ +import * as workerpool from 'workerpool'; +import { printResults } from './common'; +import { IntegTestInfo } from '../runner'; + +export interface IntegWatchOptions extends IntegTestInfo { + readonly region: string; + readonly profile?: string; + readonly verbosity?: number; +} +export async function watchIntegrationTest(pool: workerpool.WorkerPool, options: IntegWatchOptions): Promise { + await pool.exec('watchTestWorker', [options], { + on: printResults, + }); +} diff --git a/packages/@aws-cdk/integ-runner/package.json b/packages/@aws-cdk/integ-runner/package.json index 49aa7788c4c3a..e813abd0fe939 100644 --- a/packages/@aws-cdk/integ-runner/package.json +++ b/packages/@aws-cdk/integ-runner/package.json @@ -66,6 +66,7 @@ "ts-node": "^10.9.1" }, "dependencies": { + "chokidar": "^3.5.3", "@aws-cdk/cloud-assembly-schema": "0.0.0", "@aws-cdk/cloudformation-diff": "0.0.0", "@aws-cdk/cx-api": "0.0.0", diff --git a/packages/@aws-cdk/integ-runner/test/helpers.ts b/packages/@aws-cdk/integ-runner/test/helpers.ts index 7db689642694a..2a2c92b6523cb 100644 --- a/packages/@aws-cdk/integ-runner/test/helpers.ts +++ b/packages/@aws-cdk/integ-runner/test/helpers.ts @@ -1,9 +1,12 @@ -import { ICdk, CdkCliWrapper, CdkCliWrapperOptions, SynthFastOptions, DestroyOptions, ListOptions, SynthOptions, DeployOptions } from '@aws-cdk/cdk-cli-wrapper'; +import { ChildProcess } from 'child_process'; +import { Readable, Writable } from 'stream'; +import { CdkCliWrapper, CdkCliWrapperOptions, DeployOptions, DestroyOptions, ICdk, ListOptions, SynthFastOptions, SynthOptions } from '@aws-cdk/cdk-cli-wrapper'; import { IntegSnapshotRunner, IntegTest } from '../lib/runner'; import { DestructiveChange, Diagnostic } from '../lib/workers'; export interface MockCdkMocks { deploy?: jest.MockedFn<(options: DeployOptions) => void>; + watch?: jest.MockedFn<(options: DeployOptions) => ChildProcess>; synth?: jest.MockedFn<(options: SynthOptions) => void>; synthFast?: jest.MockedFn<(options: SynthFastOptions) => void>; destroy?: jest.MockedFn<(options: DestroyOptions) => void>; @@ -22,6 +25,19 @@ export class MockCdkProvider { this.mocks.deploy = mock ?? jest.fn().mockImplementation(); this.cdk.deploy = this.mocks.deploy; } + public mockWatch(mock?: MockCdkMocks['watch']) { + this.mocks.watch = mock ?? jest.fn().mockImplementation(jest.fn(() => { + return { + on: (_event: 'close', listener: (..._args: any[]) => void) => { + listener(0); + }, + stdout: new Readable({ read: jest.fn(() => {}) }), + stderr: new Readable({ read: jest.fn(() => {}) }), + stdin: new Writable({ write: jest.fn(() => {}), final: jest.fn(() => {}) }), + } as unknown as ChildProcess; + })); + this.cdk.watch = this.mocks.watch; + } public mockSynth(mock?: MockCdkMocks['synth']) { this.mocks.synth = mock ?? jest.fn().mockImplementation(); this.cdk.synth = this.mocks.synth; @@ -40,6 +56,7 @@ export class MockCdkProvider { } public mockAll(mocks: MockCdkMocks = {}): Required { this.mockDeploy(mocks.deploy); + this.mockWatch(mocks.watch); this.mockSynth(mocks.synth); this.mockSynthFast(mocks.synthFast); this.mockDestroy(mocks.destroy); diff --git a/packages/@aws-cdk/integ-runner/test/runner/integ-test-runner.test.ts b/packages/@aws-cdk/integ-runner/test/runner/integ-test-runner.test.ts index 8ffb8c6172e64..3e85a14be0db2 100644 --- a/packages/@aws-cdk/integ-runner/test/runner/integ-test-runner.test.ts +++ b/packages/@aws-cdk/integ-runner/test/runner/integ-test-runner.test.ts @@ -1,5 +1,6 @@ import * as child_process from 'child_process'; import * as builtinFs from 'fs'; +import { HotswapMode } from '@aws-cdk/cdk-cli-wrapper'; import { Manifest } from '@aws-cdk/cloud-assembly-schema'; import { AVAILABILITY_ZONE_FALLBACK_CONTEXT_KEY } from '@aws-cdk/cx-api'; import * as fs from 'fs-extra'; @@ -9,7 +10,6 @@ import { MockCdkProvider } from '../helpers'; let cdkMock: MockCdkProvider; let spawnSyncMock: jest.SpyInstance; let removeSyncMock: jest.SpyInstance; - beforeEach(() => { cdkMock = new MockCdkProvider({ directory: 'test/test-data' }); cdkMock.mockAll().list.mockImplementation(() => 'stackabc'); @@ -619,3 +619,60 @@ describe('IntegTest runIntegTests', () => { })); }); }); + +describe('IntegTest watchIntegTest', () => { + test('default watch', async () => { + // GIVEN + const integTest = new IntegTestRunner({ + cdk: cdkMock.cdk, + test: new IntegTest({ + fileName: 'test/test-data/xxxxx.test-with-snapshot.js', + discoveryRoot: 'test/test-data', + appCommand: 'node --no-warnings {filePath}', + }), + }); + + // WHEN + await integTest.watchIntegTest({ + testCaseName: 'xxxxx.test-with-snapshot', + }); + + // THEN + expect(cdkMock.mocks.watch).toHaveBeenCalledWith(expect.objectContaining({ + app: 'node --no-warnings xxxxx.test-with-snapshot.js', + hotswap: HotswapMode.FALL_BACK, + watch: true, + traceLogs: false, + deploymentMethod: 'direct', + verbose: undefined, + })); + }); + + test('verbose watch', async () => { + // GIVEN + const integTest = new IntegTestRunner({ + cdk: cdkMock.cdk, + test: new IntegTest({ + fileName: 'test/test-data/xxxxx.test-with-snapshot.js', + discoveryRoot: 'test/test-data', + appCommand: 'node --no-warnings {filePath}', + }), + }); + + // WHEN + await integTest.watchIntegTest({ + testCaseName: 'xxxxx.test-with-snapshot', + verbosity: 2, + }); + + // THEN + expect(cdkMock.mocks.watch).toHaveBeenCalledWith(expect.objectContaining({ + app: 'node --no-warnings xxxxx.test-with-snapshot.js', + hotswap: HotswapMode.FALL_BACK, + watch: true, + traceLogs: true, + deploymentMethod: 'direct', + verbose: undefined, + })); + }); +});