Skip to content

Commit

Permalink
chore(NA): build bazel projects all at once in the distributable buil…
Browse files Browse the repository at this point in the history 
…d process (elastic#90328)

* chore(NA): build bazel projects all at once in the distributable build process

* chore(NA): make sure bazelisk is installed

* chore(NA): install bazelisk using npm

* chore(NA): remove extra spac

* chore(NA): test yarn path exports

* chore(NA): add direct global dir

* chore(NA): some more debug steps

* chore(NA): remove one statement

* chore(NA): comment one more line out for testing purposes

* chore(NA): export the correct yarn bin location into the PATH

* chore(NA): cleaning implementation

* chore(NA): move installation process of bazelisk into npm

* chore(NA): add missing type
  • Loading branch information
@mistic
mistic committed Feb 5, 2021
1 parent d18bdb0 commit 1d14bdf
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 39 deletions.
60 changes: 37 additions & 23 deletions packages/kbn-pm/dist/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48106,23 +48106,34 @@ async function isBazelBinAvailable() {
}
}

async function isBazeliskInstalled(bazeliskVersion) {
try {
const {
stdout: bazeliskPkgInstallStdout
} = await Object(_child_process__WEBPACK_IMPORTED_MODULE_2__["spawn"])('npm', ['ls', '--global', '--parseable', '--long', `@bazel/bazelisk@${bazeliskVersion}`], {
stdio: 'pipe'
});
return bazeliskPkgInstallStdout.includes(`@bazel/bazelisk@${bazeliskVersion}`);
} catch {
return false;
}
}

async function installBazelTools(repoRootPath) {
_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] reading bazel tools versions from version files`);
const bazeliskVersion = await readBazelToolsVersionFile(repoRootPath, '.bazeliskversion');
const bazelVersion = await readBazelToolsVersionFile(repoRootPath, '.bazelversion'); // Check what globals are installed

_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] verify if bazelisk is installed`);
const {
stdout: bazeliskPkgInstallStdout
} = await Object(_child_process__WEBPACK_IMPORTED_MODULE_2__["spawn"])('yarn', ['global', 'list'], {
stdio: 'pipe'
});
_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] verify if bazelisk is installed`); // Test if bazelisk is already installed in the correct version

const isBazeliskPkgInstalled = await isBazeliskInstalled(bazeliskVersion); // Test if bazel bin is available

const isBazelBinAlreadyAvailable = await isBazelBinAvailable(); // Install bazelisk if not installed

if (!bazeliskPkgInstallStdout.includes(`@bazel/bazelisk@${bazeliskVersion}`) || !isBazelBinAlreadyAvailable) {
if (!isBazeliskPkgInstalled || !isBazelBinAlreadyAvailable) {
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] installing Bazel tools`);
_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] bazelisk is not installed. Installing @bazel/bazelisk@${bazeliskVersion} and bazel@${bazelVersion}`);
await Object(_child_process__WEBPACK_IMPORTED_MODULE_2__["spawn"])('yarn', ['global', 'add', `@bazel/bazelisk@${bazeliskVersion}`], {
await Object(_child_process__WEBPACK_IMPORTED_MODULE_2__["spawn"])('npm', ['install', '--global', `@bazel/bazelisk@${bazeliskVersion}`], {
env: {
USE_BAZEL_VERSION: bazelVersion
},
Expand All @@ -48132,7 +48143,7 @@ async function installBazelTools(repoRootPath) {

if (!isBazelBinAvailableAfterInstall) {
throw new Error(dedent__WEBPACK_IMPORTED_MODULE_0___default.a`
[bazel_tools] an error occurred when installing the Bazel tools. Please make sure 'yarn global bin' is on your $PATH, otherwise just add it there
[bazel_tools] an error occurred when installing the Bazel tools. Please make sure you have access to npm globally installed modules on your $PATH
`);
}
}
Expand Down Expand Up @@ -59771,10 +59782,11 @@ __webpack_require__.r(__webpack_exports__);
/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(4);
/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_2___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_2__);
/* harmony import */ var _build_non_bazel_production_projects__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(745);
/* harmony import */ var _utils_fs__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(131);
/* harmony import */ var _utils_log__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(246);
/* harmony import */ var _utils_package_json__WEBPACK_IMPORTED_MODULE_6__ = __webpack_require__(251);
/* harmony import */ var _utils_projects__WEBPACK_IMPORTED_MODULE_7__ = __webpack_require__(248);
/* harmony import */ var _utils_bazel_run__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(374);
/* harmony import */ var _utils_fs__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(131);
/* harmony import */ var _utils_log__WEBPACK_IMPORTED_MODULE_6__ = __webpack_require__(246);
/* harmony import */ var _utils_package_json__WEBPACK_IMPORTED_MODULE_7__ = __webpack_require__(251);
/* harmony import */ var _utils_projects__WEBPACK_IMPORTED_MODULE_8__ = __webpack_require__(248);
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
Expand All @@ -59790,17 +59802,19 @@ __webpack_require__.r(__webpack_exports__);




async function buildBazelProductionProjects({
kibanaRoot,
buildRoot,
onlyOSS
}) {
const projects = await Object(_utils_projects__WEBPACK_IMPORTED_MODULE_7__["getBazelProjectsOnly"])(await Object(_build_non_bazel_production_projects__WEBPACK_IMPORTED_MODULE_3__["getProductionProjects"])(kibanaRoot, onlyOSS));
const projects = await Object(_utils_projects__WEBPACK_IMPORTED_MODULE_8__["getBazelProjectsOnly"])(await Object(_build_non_bazel_production_projects__WEBPACK_IMPORTED_MODULE_3__["getProductionProjects"])(kibanaRoot, onlyOSS));
const projectNames = [...projects.values()].map(project => project.name);
_utils_log__WEBPACK_IMPORTED_MODULE_5__["log"].info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`);
_utils_log__WEBPACK_IMPORTED_MODULE_6__["log"].info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`);
await Object(_utils_bazel_run__WEBPACK_IMPORTED_MODULE_4__["runBazel"])(['build', '//packages:build']);
_utils_log__WEBPACK_IMPORTED_MODULE_6__["log"].info(`All Bazel projects production builds for [${projectNames.join(', ')}] are complete}]`);

for (const project of projects.values()) {
await Object(_build_non_bazel_production_projects__WEBPACK_IMPORTED_MODULE_3__["buildProject"])(project);
await copyToBuild(project, kibanaRoot, buildRoot);
await applyCorrectPermissions(project, kibanaRoot, buildRoot);
}
Expand Down Expand Up @@ -59835,9 +59849,9 @@ async function copyToBuild(project, kibanaRoot, buildRoot) {
// the intermediate build, we fall back to using the project's already defined
// `package.json`.

const packageJson = (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["isFile"])(Object(path__WEBPACK_IMPORTED_MODULE_2__["join"])(buildProjectPath, 'package.json'))) ? await Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_6__["readPackageJson"])(buildProjectPath) : project.json;
const preparedPackageJson = Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_6__["createProductionPackageJson"])(packageJson);
await Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_6__["writePackageJson"])(buildProjectPath, preparedPackageJson);
const packageJson = (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["isFile"])(Object(path__WEBPACK_IMPORTED_MODULE_2__["join"])(buildProjectPath, 'package.json'))) ? await Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_7__["readPackageJson"])(buildProjectPath) : project.json;
const preparedPackageJson = Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_7__["createProductionPackageJson"])(packageJson);
await Object(_utils_package_json__WEBPACK_IMPORTED_MODULE_7__["writePackageJson"])(buildProjectPath, preparedPackageJson);
}

async function applyCorrectPermissions(project, kibanaRoot, buildRoot) {
Expand All @@ -59852,12 +59866,12 @@ async function applyCorrectPermissions(project, kibanaRoot, buildRoot) {
for (const pluginPath of allPluginPaths) {
const resolvedPluginPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(buildRoot, pluginPath);

if (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["isFile"])(resolvedPluginPath)) {
await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["chmod"])(resolvedPluginPath, 0o644);
if (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["isFile"])(resolvedPluginPath)) {
await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["chmod"])(resolvedPluginPath, 0o644);
}

if (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["isDirectory"])(resolvedPluginPath)) {
await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_4__["chmod"])(resolvedPluginPath, 0o755);
if (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["isDirectory"])(resolvedPluginPath)) {
await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["chmod"])(resolvedPluginPath, 0o755);
}
}
}
Expand Down
7 changes: 5 additions & 2 deletions packages/kbn-pm/src/production/build_bazel_production_projects.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ import copy from 'cpy';
import globby from 'globby';
import { basename, join, relative, resolve } from 'path';

import { buildProject, getProductionProjects } from './build_non_bazel_production_projects';
import { getProductionProjects } from './build_non_bazel_production_projects';
import { runBazel } from '../utils/bazel/run';
import { chmod, isFile, isDirectory } from '../utils/fs';
import { log } from '../utils/log';
import {
Expand All @@ -35,8 +36,10 @@ export async function buildBazelProductionProjects({
const projectNames = [...projects.values()].map((project) => project.name);
log.info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`);

await runBazel(['build', '//packages:build']);
log.info(`All Bazel projects production builds for [${projectNames.join(', ')}] are complete}]`);

for (const project of projects.values()) {
await buildProject(project);
await copyToBuild(project, kibanaRoot, buildRoot);
await applyCorrectPermissions(project, kibanaRoot, buildRoot);
}
Expand Down
32 changes: 23 additions & 9 deletions packages/kbn-pm/src/utils/bazel/install_tools.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,30 +36,44 @@ async function isBazelBinAvailable() {
}
}

async function isBazeliskInstalled(bazeliskVersion: string) {
try {
const { stdout: bazeliskPkgInstallStdout } = await spawn(
'npm',
['ls', '--global', '--parseable', '--long', `@bazel/bazelisk@${bazeliskVersion}`],
{
stdio: 'pipe',
}
);

return bazeliskPkgInstallStdout.includes(`@bazel/bazelisk@${bazeliskVersion}`);
} catch {
return false;
}
}

export async function installBazelTools(repoRootPath: string) {
log.debug(`[bazel_tools] reading bazel tools versions from version files`);
const bazeliskVersion = await readBazelToolsVersionFile(repoRootPath, '.bazeliskversion');
const bazelVersion = await readBazelToolsVersionFile(repoRootPath, '.bazelversion');

// Check what globals are installed
log.debug(`[bazel_tools] verify if bazelisk is installed`);
const { stdout: bazeliskPkgInstallStdout } = await spawn('yarn', ['global', 'list'], {
stdio: 'pipe',
});

// Test if bazelisk is already installed in the correct version
const isBazeliskPkgInstalled = await isBazeliskInstalled(bazeliskVersion);

// Test if bazel bin is available
const isBazelBinAlreadyAvailable = await isBazelBinAvailable();

// Install bazelisk if not installed
if (
!bazeliskPkgInstallStdout.includes(`@bazel/bazelisk@${bazeliskVersion}`) ||
!isBazelBinAlreadyAvailable
) {
if (!isBazeliskPkgInstalled || !isBazelBinAlreadyAvailable) {
log.info(`[bazel_tools] installing Bazel tools`);

log.debug(
`[bazel_tools] bazelisk is not installed. Installing @bazel/bazelisk@${bazeliskVersion} and bazel@${bazelVersion}`
);
await spawn('yarn', ['global', 'add', `@bazel/bazelisk@${bazeliskVersion}`], {
await spawn('npm', ['install', '--global', `@bazel/bazelisk@${bazeliskVersion}`], {
env: {
USE_BAZEL_VERSION: bazelVersion,
},
Expand All @@ -69,7 +83,7 @@ export async function installBazelTools(repoRootPath: string) {
const isBazelBinAvailableAfterInstall = await isBazelBinAvailable();
if (!isBazelBinAvailableAfterInstall) {
throw new Error(dedent`
[bazel_tools] an error occurred when installing the Bazel tools. Please make sure 'yarn global bin' is on your $PATH, otherwise just add it there
[bazel_tools] an error occurred when installing the Bazel tools. Please make sure you have access to npm globally installed modules on your $PATH
`);
}
}
Expand Down
5 changes: 0 additions & 5 deletions src/dev/ci_setup/setup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,3 @@ if [ "$GIT_CHANGES" ]; then
echo -e "$GIT_CHANGES\n"
exit 1
fi

###
### copy .bazelrc-ci into $HOME/.bazelrc
###
cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc";
11 changes: 11 additions & 0 deletions src/dev/ci_setup/setup_env.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,4 +175,15 @@ if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then
export JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
fi

###
### copy .bazelrc-ci into $HOME/.bazelrc
###
cp -f "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc";

###
### append auth token to buildbuddy into "$HOME/.bazelrc";
###
echo "# Appended by $KIBANA_DIR/src/dev/ci_setup/setup.sh" >> "$HOME/.bazelrc"
echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc"

export CI_ENV_SETUP=true

0 comments on commit 1d14bdf

Please sign in to comment.