From eb0fb382308f18460844d646dada2e564f82353d Mon Sep 17 00:00:00 2001 From: Andrew Clark Date: Tue, 12 Jan 2021 11:32:32 -0600 Subject: [PATCH] Build stable and experimental with same command (#20573) The goal is to simplify our CI pipeline so that all configurations are built and tested in a single workflow. As a first step, this adds a new build script entry point that builds both the experimental and stable release channels into a single artifacts directory. The script works by wrapping the existing build script (which only builds a single release channel at a time), then post-processing the results to match the desired filesystem layout. A future version of the build script would output the files directly without post-processing. Because many parts of our infra depend on the existing layout of the build artifacts directory, I have left the old workflows untouched. We can incremental migrate to the new layout, then delete the old workflows after we've finished. --- .circleci/config.yml | 50 ++++++++ .gitignore | 1 + package.json | 1 + scripts/rollup/build-all-release-channels.js | 123 +++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 scripts/rollup/build-all-release-channels.js diff --git a/.circleci/config.yml b/.circleci/config.yml index f859b2c59a309..a532fb2e067a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -293,6 +293,45 @@ jobs: - dist - sizes/*.json + yarn_build_combined: + docker: *docker + environment: *environment + parallelism: 40 + steps: + - checkout + - run: yarn workspaces info | head -n -1 > workspace_info.txt + - *restore_node_modules + - run: + command: | + ./scripts/circleci/add_build_info_json.sh + ./scripts/circleci/update_package_versions.sh + yarn build-combined + - persist_to_workspace: + root: build2 + paths: + - facebook-www + - facebook-react-native + - facebook-relay + - oss-stable + - oss-experimental + - react-native + - dist + - sizes/*.json + + process_artifacts_combined: + docker: *docker + environment: *environment + steps: + - checkout + - attach_workspace: + at: build2 + - run: yarn workspaces info | head -n -1 > workspace_info.txt + - *restore_node_modules + # Compress build directory into a single tarball for easy download + - run: tar -zcvf ./build2.tgz ./build2 + - store_artifacts: + path: ./build2.tgz + build_devtools_and_process_artifacts: docker: *docker environment: *environment @@ -611,6 +650,17 @@ workflows: only: - master + # New workflow that will replace "stable" and "experimental" + combined: + jobs: + - setup + - yarn_build_combined: + requires: + - setup + - process_artifacts_combined: + requires: + - yarn_build_combined + fuzz_tests: triggers: - schedule: diff --git a/.gitignore b/.gitignore index 5a913ac6e5278..fed2a08aa5d9e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ scripts/flow/*/.flowconfig _SpecRunner.html __benchmarks__ build/ +build2/ remote-repo/ coverage/ .module-cache diff --git a/package.json b/package.json index bf225d9cdc81c..deb7b24882da6 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ }, "scripts": { "build": "node ./scripts/rollup/build.js", + "build-combined": "node ./scripts/rollup/build-all-release-channels.js", "build-for-devtools": "cross-env RELEASE_CHANNEL=experimental yarn build react/index,react-dom,react-is,react-debug-tools,scheduler,react-test-renderer,react-refresh", "build-for-devtools-dev": "yarn build-for-devtools --type=NODE_DEV", "build-for-devtools-prod": "yarn build-for-devtools --type=NODE_PROD", diff --git a/scripts/rollup/build-all-release-channels.js b/scripts/rollup/build-all-release-channels.js new file mode 100644 index 0000000000000..774f1011aa287 --- /dev/null +++ b/scripts/rollup/build-all-release-channels.js @@ -0,0 +1,123 @@ +'use strict'; + +/* eslint-disable no-for-of-loops/no-for-of-loops */ + +const fs = require('fs'); +const {spawnSync} = require('child_process'); +const tmp = require('tmp'); + +// Runs the build script for both stable and experimental release channels, +// by configuring an environment variable. + +if (process.env.CIRCLE_NODE_TOTAL) { + // In CI, we use multiple concurrent processes. Allocate half the processes to + // build the stable channel, and the other half for experimental. Override + // the environment variables to "trick" the underlying build script. + const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10); + const halfTotal = Math.floor(total / 2); + const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10); + if (index < halfTotal) { + const nodeTotal = halfTotal; + const nodeIndex = index; + buildForChannel('stable', nodeTotal, nodeIndex); + processStable('./build'); + } else { + const nodeTotal = total - halfTotal; + const nodeIndex = index - halfTotal; + buildForChannel('experimental', nodeTotal, nodeIndex); + processExperimental('./build'); + } + + // TODO: Currently storing artifacts as `./build2` so that it doesn't conflict + // with old build job. Remove once we migrate rest of build/test pipeline. + fs.renameSync('./build', './build2'); +} else { + // Running locally, no concurrency. Move each channel's build artifacts into + // a temporary directory so that they don't conflict. + buildForChannel('stable', '', ''); + const stableDir = tmp.dirSync().name; + fs.renameSync('./build', stableDir); + processStable(stableDir); + + buildForChannel('experimental', '', ''); + const experimentalDir = tmp.dirSync().name; + fs.renameSync('./build', experimentalDir); + processExperimental(experimentalDir); + + // Then merge the experimental folder into the stable one. processExperimental + // will have already removed conflicting files. + // + // In CI, merging is handled automatically by CircleCI's workspace feature. + spawnSync('rsync', ['-ar', experimentalDir + '/', stableDir + '/']); + + // Now restore the combined directory back to its original name + // TODO: Currently storing artifacts as `./build2` so that it doesn't conflict + // with old build job. Remove once we migrate rest of build/test pipeline. + fs.renameSync(stableDir, './build2'); +} + +function buildForChannel(channel, nodeTotal, nodeIndex) { + spawnSync('node', ['./scripts/rollup/build.js', ...process.argv.slice(2)], { + stdio: ['pipe', process.stdout, process.stderr], + env: { + ...process.env, + RELEASE_CHANNEL: channel, + CIRCLE_NODE_TOTAL: nodeTotal, + CIRCLE_NODE_INDEX: nodeIndex, + }, + }); +} + +function processStable(buildDir) { + if (fs.existsSync(buildDir + '/node_modules')) { + fs.renameSync(buildDir + '/node_modules', buildDir + '/oss-stable'); + } + + if (fs.existsSync(buildDir + '/facebook-www')) { + for (const fileName of fs.readdirSync(buildDir + '/facebook-www')) { + const filePath = buildDir + '/facebook-www/' + fileName; + const stats = fs.statSync(filePath); + if (!stats.isDirectory()) { + fs.renameSync(filePath, filePath.replace('.js', '.classic.js')); + } + } + } + + if (fs.existsSync(buildDir + '/sizes')) { + fs.renameSync(buildDir + '/sizes', buildDir + '/sizes-stable'); + } +} + +function processExperimental(buildDir) { + if (fs.existsSync(buildDir + '/node_modules')) { + fs.renameSync(buildDir + '/node_modules', buildDir + '/oss-experimental'); + } + + if (fs.existsSync(buildDir + '/facebook-www')) { + for (const fileName of fs.readdirSync(buildDir + '/facebook-www')) { + const filePath = buildDir + '/facebook-www/' + fileName; + const stats = fs.statSync(filePath); + if (!stats.isDirectory()) { + fs.renameSync(filePath, filePath.replace('.js', '.modern.js')); + } + } + } + + if (fs.existsSync(buildDir + '/sizes')) { + fs.renameSync(buildDir + '/sizes', buildDir + '/sizes-experimental'); + } + + // Delete all other artifacts that weren't handled above. We assume they are + // duplicates of the corresponding artifacts in the stable channel. Ideally, + // the underlying build script should not have produced these files in the + // first place. + for (const pathName of fs.readdirSync(buildDir)) { + if ( + pathName !== 'oss-experimental' && + pathName !== 'facebook-www' && + pathName !== 'sizes-experimental' + ) { + spawnSync('rm', ['-rm', buildDir + '/' + pathName]); + } + } +}