WIP: Introduce optional dependencies to streamline platform-specific installation

package.json

@@ -13,10 +13,10 @@
   "gypfile": true,
   "scripts": {
     "prepare": "tsc -p ./tsconfig.build.json",
-    "install": "node-gyp-build",
     "prebuild": "node ./scripts/prebuild.js",
     "build": "shx rm -rf ./dist && tsc -p ./tsconfig.build.json",
-    "postversion": "npm install --package-lock-only --ignore-scripts --silent",
+    "version": "node ./scripts/version.js",
+    "prepublishOnly": "node ./scripts/prepublishOnly.js",
     "ts-node": "ts-node",
     "test": "jest",
     "lint": "eslint '{src,tests,scripts,benches}/**/*.{js,ts}'",
@@ -34,9 +34,14 @@
     "@matrixai/logger": "^3.1.0",
     "@matrixai/resources": "^1.1.5",
     "@matrixai/workers": "^1.3.7",
-    "node-gyp-build": "4.4.0",
     "threads": "^1.6.5"
   },
+  "optionalDependencies": {
+    "@matrixai/db-darwin-arm64": "5.1.0",
+    "@matrixai/db-darwin-x64": "5.1.0",
+    "@matrixai/db-linux-x64": "5.1.0",
+    "@matrixai/db-win32-x64": "5.1.0"
+  },
   "devDependencies": {
     "@swc/core": "^1.3.62",
     "@swc/jest": "^0.2.26",

scripts/prepublishOnly.js

@@ -0,0 +1,133 @@
+#!/usr/bin/env node
+
+/**
+ * This runs before `npm publish` command.
+ * This will take the native objects in `prebuild/`
+ * and create native packages under `prepublishOnly/`.
+ *
+ * For example:
+ *
+ * /prepublishOnly
+ *   /@org
+ *     /name-linux-x64
+ *       /package.json
+ *       /node.napi.node
+ *       /README.md
+ */
+
+const os = require('os');
+const fs = require('fs');
+const path = require('path');
+const process = require('process');
+const childProcess = require('child_process');
+const packageJSON = require('../package.json');
+
+const platform = os.platform();
+
+/* eslint-disable no-console */
+async function main(argv = process.argv) {
+  argv = argv.slice(2);
+  let tag;
+  let dryRun = false;
+  const restArgs = [];
+  while (argv.length > 0) {
+    const option = argv.shift();
+    let match;
+    if ((match = option.match(/--tag(?:=(.+)|$)/))) {
+      tag = match[1] ?? argv.shift();
+    } else if ((match = option.match(/--dry-run$/))) {
+      dryRun = true;
+    } else {
+      restArgs.push(option);
+    }
+  }
+  if (tag == null) {
+    tag = process.env.npm_config_tag;
+  }
+  const projectRoot = path.join(__dirname, '..');
+  const prebuildPath = path.join(projectRoot, 'prebuild');
+  const prepublishOnlyPath = path.join(projectRoot, 'prepublishOnly');
+  const buildNames = (await fs.promises.readdir(prebuildPath)).filter(
+    (filename) => /^(?:[^-]+)-(?:[^-]+)-(?:[^-]+)$/.test(filename),
+  );
+  if (buildNames.length < 1) {
+    console.error(
+      'You must prebuild at least 1 native object with the filename of `name-platform-arch` before prepublish',
+    );
+    process.exitCode = 1;
+    return process.exitCode;
+  }
+  // Extract out the org name, this may be undefined
+  const orgName = packageJSON.name.match(/^@[^/]+/)?.[0];
+  for (const buildName of buildNames) {
+    // This is `name-platform-arch`
+    const name = path.basename(buildName, '.node');
+    // This is `@org/name-platform-arch`, uses `posix` to force usage of `/`
+    const packageName = path.posix.join(orgName ?? '', name);
+    const constraints = name.match(
+      /^(?:[^-]+)-(?<platform>[^-]+)-(?<arch>[^-]+)$/,
+    );
+    // This will be `prebuild/name-platform-arch.node`
+    const buildPath = path.join(prebuildPath, buildName);
+    // This will be `prepublishOnly/@org/name-platform-arch`
+    const packagePath = path.join(prepublishOnlyPath, packageName);
+    console.error('Packaging:', packagePath);
+    try {
+      await fs.promises.rm(packagePath, {
+        recursive: true,
+      });
+    } catch (e) {
+      if (e.code !== 'ENOENT') throw e;
+    }
+    await fs.promises.mkdir(packagePath, { recursive: true });
+    const nativePackageJSON = {
+      name: packageName,
+      version: packageJSON.version,
+      homepage: packageJSON.homepage,
+      author: packageJSON.author,
+      contributors: packageJSON.contributors,
+      description: packageJSON.description,
+      keywords: packageJSON.keywords,
+      license: packageJSON.license,
+      repository: packageJSON.repository,
+      main: 'node.napi.node',
+      os: [constraints.groups.platform],
+      cpu: [...constraints.groups.arch.split('+')],
+    };
+    const packageJSONPath = path.join(packagePath, 'package.json');
+    console.error(`Writing ${packageJSONPath}`);
+    const packageJSONString = JSON.stringify(nativePackageJSON, null, 2);
+    console.error(packageJSONString);
+    await fs.promises.writeFile(packageJSONPath, packageJSONString, {
+      encoding: 'utf-8',
+    });
+    const packageReadmePath = path.join(packagePath, 'README.md');
+    console.error(`Writing ${packageReadmePath}`);
+    const packageReadme = `# ${packageName}\n`;
+    console.error(packageReadme);
+    await fs.promises.writeFile(packageReadmePath, packageReadme, {
+      encoding: 'utf-8',
+    });
+    const packageBuildPath = path.join(packagePath, 'node.napi.node');
+    console.error(`Copying ${buildPath} to ${packageBuildPath}`);
+    await fs.promises.copyFile(buildPath, packageBuildPath);
+    const publishArgs = [
+      'publish',
+      packagePath,
+      ...(tag != null ? [`--tag=${tag}`] : []),
+      '--access=public',
+      ...(dryRun ? ['--dry-run'] : []),
+    ];
+    console.error('Running npm publish:');
+    console.error(['npm', ...publishArgs].join(' '));
+    childProcess.execFileSync('npm', publishArgs, {
+      stdio: ['inherit', 'inherit', 'inherit'],
+      windowsHide: true,
+      encoding: 'utf-8',
+      shell: platform === 'win32' ? true : false,
+    });
+  }
+}
+/* eslint-enable no-console */
+
+void main();

scripts/version.js

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+
+/**
+ * This runs after `npm version` command updates the version but before changes are commited.
+ * This will also update the `package.json` optional native dependencies
+ * to match the same version as the version of this package.
+ * This maintains the same version between this master package
+ * and the optional native dependencies.
+ * At the same time, the `package-lock.json` is also regenerated.
+ * Note that at this point, the new optional native dependencies have
+ * not yet been published, so the `--package-lock-only` flag is used
+ * to prevent `npm` from attempting to download unpublished packages.
+ */
+
+const path = require('path');
+const os = require('os');
+const childProcess = require('child_process');
+const packageJSON = require('../package.json');
+
+const platform = os.platform();
+
+/* eslint-disable no-console */
+async function main() {
+  console.error(
+    'Updating the package.json with optional native dependencies and package-lock.json',
+  );
+  const optionalDepsNative = [];
+  for (const key in packageJSON.optionalDependencies) {
+    if (key.startsWith(packageJSON.name)) {
+      optionalDepsNative.push(`${key}@${packageJSON.version}`);
+    }
+  }
+  if (optionalDepsNative.length > 0) {
+    const installArgs = [
+      'install',
+      '--ignore-scripts',
+      '--silent',
+      '--package-lock-only',
+      '--save-optional',
+      '--save-exact',
+      ...optionalDepsNative,
+    ];
+    console.error('Running npm install:');
+    console.error(['npm', ...installArgs].join(' '));
+    childProcess.execFileSync('npm', installArgs, {
+      stdio: ['inherit', 'inherit', 'inherit'],
+      windowsHide: true,
+      encoding: 'utf-8',
+      shell: platform === 'win32' ? true : false,
+    });
+  }
+}
+/* eslint-enable no-console */
+
+void main();

src/native/rocksdb.ts

@@ -19,7 +19,6 @@ import type {
   RocksDBCountOptions,
 } from './types';
 import path from 'path';
-import nodeGypBuild from 'node-gyp-build';
 
 interface RocksDB {
   dbInit(): RocksDBDatabase;
@@ -271,8 +270,102 @@ interface RocksDB {
   ): void;
 }
 
-const rocksdb: RocksDB = nodeGypBuild(path.join(__dirname, '../../'));
+const projectRoot = path.join(__dirname, '../../');
+const prebuildPath = path.join(projectRoot, 'prebuild');
 
-export default rocksdb;
+/**
+ * Try require on all prebuild targets first, then
+ * try require on all npm targets second.
+ */
+function requireBinding(targets: Array<string>): RocksDB {
+  const prebuildTargets = targets.map((target) =>
+    path.join(prebuildPath, `db-${target}.node`),
+  );
+  for (const prebuildTarget of prebuildTargets) {
+    try {
+      return require(prebuildTarget);
+    } catch (e) {
+      if (e.code !== 'MODULE_NOT_FOUND') throw e;
+    }
+  }
+  const npmTargets = targets.map((target) => `@matrixai/db-${target}`);
+  for (const npmTarget of npmTargets) {
+    try {
+      return require(npmTarget);
+    } catch (e) {
+      if (e.code !== 'MODULE_NOT_FOUND') throw e;
+    }
+  }
+  throw new Error(
+    `Failed requiring possible native bindings: ${prebuildTargets.concat(
+      npmTargets,
+    )}`,
+  );
+}
+
+let nativeBinding: RocksDB;
+
+/**
+ * For desktop we only support win32, darwin and linux.
+ * Mobile OS support is pending.
+ */
+switch (process.platform) {
+  case 'win32':
+    switch (process.arch) {
+      case 'x64':
+        nativeBinding = requireBinding(['win32-x64']);
+        break;
+      case 'ia32':
+        nativeBinding = requireBinding(['win32-ia32']);
+        break;
+      case 'arm64':
+        nativeBinding = requireBinding(['win32-arm64']);
+        break;
+      default:
+        throw new Error(`Unsupported architecture on Windows: ${process.arch}`);
+    }
+    break;
+  case 'darwin':
+    switch (process.arch) {
+      case 'x64':
+        nativeBinding = requireBinding([
+          'darwin-x64',
+          'darwin-x64+arm64',
+          'darwin-arm64+x64',
+        ]);
+        break;
+      case 'arm64':
+        nativeBinding = requireBinding([
+          'darwin-arm64',
+          'darwin-arm64+x64',
+          'darwin-x64+arm64',
+        ]);
+        break;
+      default:
+        throw new Error(`Unsupported architecture on macOS: ${process.arch}`);
+    }
+    break;
+  case 'linux':
+    switch (process.arch) {
+      case 'x64':
+        nativeBinding = requireBinding(['linux-x64']);
+        break;
+      case 'arm64':
+        nativeBinding = requireBinding(['linux-arm64']);
+        break;
+      case 'arm':
+        nativeBinding = requireBinding(['linux-arm']);
+        break;
+      default:
+        throw new Error(`Unsupported architecture on Linux: ${process.arch}`);
+    }
+    break;
+  default:
+    throw new Error(
+      `Unsupported OS: ${process.platform}, architecture: ${process.arch}`,
+    );
+}
+
+export default nativeBinding;
 
 export type { RocksDB };