Merge pull request #8 from serokell/zhenya/zt385-avoid-rebuilds

[ZT-385] buildNpmPackage: build node_modules in a separate derivation

.travis.nix

@@ -1,14 +1,8 @@
-# TODO:
-# * replace w/ more generic test
-# * also test npm packages
-# * unhide badge
-
-{ pkgs ? import <nixpkgs> {} }:
 let
-  bp = pkgs.callPackage ./default.nix {};
+  # we pin nixpkgs revision for tests
+  sources = import ./nix/sources.nix;
+  pkgs = import sources.nixpkgs {};
+
+  npm-buildpackage = pkgs.callPackage ./default.nix {};
 in
-  import (fetchGit {
-    url = "https://github.com/obfusk/nix-vault-with-ui.git";
-    rev = "664d3d9a1f30626d0277a62c520cf82df0e0a2a1";
-    ref = "664d3d9-tag-you-are-it"; # TODO: "v0.1.0"
-  }) { inherit pkgs; npm-buildpackage = bp; }
+  import ./tests/buildNpmPackage { inherit pkgs npm-buildpackage; }

.travis.yml

@@ -1,3 +1,3 @@
 language: nix
-nix: 2.1.3
+nix: 2.3.1
 script: nix-build .travis.nix

README.md

@@ -22,7 +22,7 @@ in ...
 ```
 
 ```nix
-bp.buildNpmPackage { src = ./.; }
+bp.buildNpmPackage { src = ./.; npmBuild = "npm run build"; }
 ```
 
 ```nix

default.nix

@@ -110,50 +110,73 @@ with stdenv.lib; let
     url  = "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz";
     sha1 = "e77a97fbd345b76d83245edcd17d393b1b41fb31";
   };
-in {
+in rec {
+  mkNodeModules = { packageJson, packageLockJson, extraEnvVars ? {} }:
+    let
+      info = fromJSON (readFile packageJson);
+      lock = fromJSON (readFile packageLockJson);
+    in stdenv.mkDerivation ({
+      name = "${info.name}-${info.version}-node-modules";
+
+      buildInputs = [ _nodejs ];
+
+      npmFlags = npmFlagsNpm;
+
+      buildCommand = ''
+        cp ${packageJson} ./package.json
+        cp ${packageLockJson} ./package-lock.json
+
+        echo 'building npm cache'
+        chmod u+w ./package-lock.json
+        NODE_PATH=${npmModules} node ${./mknpmcache.js} ${cacheInput "npm-cache-input.json" lock}
+
+        echo 'building node_modules'
+        npm $npmFlags ci
+        patchShebangs ./node_modules/
+
+        mkdir $out
+        mv ./node_modules $out/
+      '';
+    } // extraEnvVars);
+
   buildNpmPackage = args @ {
-    src, npmBuild ? "npm ci", npmBuildMore ? "",
-    buildInputs ? [], npmFlags ? [], ...
+    src, npmBuild, buildInputs ? [],
+
+    # allows to avoid rebuilding node_modules each time `src` changes (e.g. if you filter gitignored files)
+    packageJson ? src + "/package.json", packageLockJson ? src + "/package-lock.json",
+
+    # pass env variables to `npm install`
+    extraEnvVars ? {},
+    ...
   }:
     let
-      info  = npmInfo src;
-      lock  = fromJSON (readFile (src + "/package-lock.json"));
+      info = fromJSON (readFile packageJson);
+      lock = fromJSON (readFile packageLockJson);
+      name = "${info.name}-${info.version}";
+      nodeModules = mkNodeModules { inherit packageJson packageLockJson extraEnvVars; };
     in stdenv.mkDerivation ({
-      inherit (info) name;
-      inherit (info.info) version;
-
-      preBuildPhases    = [ "npmCachePhase" ];
-      preInstallPhases  = [ "npmPackPhase" ];
+      inherit name;
 
-      npmCachePhase = ''
-        addToSearchPath NODE_PATH ${npmModules}   # pacote
-        node ${./mknpmcache.js} ${cacheInput "npm-cache-input.json" lock}
+      configurePhase = ''
+        cp -r ${nodeModules}/node_modules ./node_modules
+        chmod -R u+w ./node_modules
       '';
 
       buildPhase = ''
-        ${npmAlias}
         runHook preBuild
         ${npmBuild}
-        ${npmBuildMore}
         runHook postBuild
       '';
 
-      # make a package .tgz (no way around it)
-      npmPackPhase = ''
-        ${npmAlias}
-        npm prune --production
-        rm -rf ./npm-cache
-        npm pack --ignore-scripts
-      '';
-
       installPhase = ''
         runHook preInstall
-        ${untarAndWrap info.name [npmCmd]}
+        npm prune --production
+        npm pack --ignore-scripts
+        ${untarAndWrap name [npmCmd]}
         runHook postInstall
       '';
-    } // commonEnv // args // {
+    } // commonEnv // extraEnvVars // removeAttrs args [ "extraEnvVars" ] // {
       buildInputs = commonBuildInputs ++ buildInputs;
-      npmFlags    = npmFlagsNpm ++ npmFlags;
     });
 
   buildYarnPackage = args @ {

mknpmcache.js

@@ -38,7 +38,7 @@ async function main(lockfile, nix, cache) {
     }
   })
   // rewrite lock file to use sha512 hashes from pacote
-  fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 2))
+  fs.writeFileSync(pkgLockFile, JSON.stringify(lockfile, null, 2))
 }
 
 process.on("unhandledRejection", error => {

nix/sources.json

@@ -0,0 +1,14 @@
+{
+    "nixpkgs": {
+        "branch": "master",
+        "description": "Nix Packages collection",
+        "homepage": null,
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a36b2925a5082e821ad80b3a7bfedcc6e488b91c",
+        "sha256": "14sx173cg1g1lpd5hcarxxz8n78jsffr6pv9l05a58aq0ilgk328",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/a36b2925a5082e821ad80b3a7bfedcc6e488b91c.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}

nix/sources.nix

@@ -0,0 +1,93 @@
+# This file has been generated by Niv.
+
+# A record, from name to path, of the third-party packages
+with rec
+{
+  pkgs =
+    if hasNixpkgsPath
+    then
+        if hasThisAsNixpkgsPath
+        then import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {}
+        else import <nixpkgs> {}
+    else
+        import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {};
+
+  sources_nixpkgs =
+    if builtins.hasAttr "nixpkgs" sources
+    then sources.nixpkgs
+    else abort
+    ''
+        Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+        add a package called "nixpkgs" to your sources.json.
+    '';
+
+  # fetchTarball version that is compatible between all the versions of Nix
+  builtins_fetchTarball =
+      { url, sha256 }@attrs:
+      let
+        inherit (builtins) lessThan nixVersion fetchTarball;
+      in
+        if lessThan nixVersion "1.12" then
+          fetchTarball { inherit url; }
+        else
+          fetchTarball attrs;
+
+  # fetchurl version that is compatible between all the versions of Nix
+  builtins_fetchurl =
+      { url, sha256 }@attrs:
+      let
+        inherit (builtins) lessThan nixVersion fetchurl;
+      in
+        if lessThan nixVersion "1.12" then
+          fetchurl { inherit url; }
+        else
+          fetchurl attrs;
+
+  # A wrapper around pkgs.fetchzip that has inspectable arguments,
+  # annoyingly this means we have to specify them
+  fetchzip = { url, sha256 }@attrs: pkgs.fetchzip attrs;
+
+  # A wrapper around pkgs.fetchurl that has inspectable arguments,
+  # annoyingly this means we have to specify them
+  fetchurl = { url, sha256 }@attrs: pkgs.fetchurl attrs;
+
+  hasNixpkgsPath = (builtins.tryEval <nixpkgs>).success;
+  hasThisAsNixpkgsPath =
+    (builtins.tryEval <nixpkgs>).success && <nixpkgs> == ./.;
+
+  sources = builtins.fromJSON (builtins.readFile ./sources.json);
+
+  mapAttrs = builtins.mapAttrs or
+    (f: set: with builtins;
+      listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)));
+
+  # borrowed from nixpkgs
+  functionArgs = f: f.__functionArgs or (builtins.functionArgs f);
+  callFunctionWith = autoArgs: f: args:
+    let auto = builtins.intersectAttrs (functionArgs f) autoArgs;
+    in f (auto // args);
+
+  getFetcher = spec:
+    let fetcherName =
+      if builtins.hasAttr "type" spec
+      then builtins.getAttr "type" spec
+      else "builtin-tarball";
+    in builtins.getAttr fetcherName {
+      "tarball" = fetchzip;
+      "builtin-tarball" = builtins_fetchTarball;
+      "file" = fetchurl;
+      "builtin-url" = builtins_fetchurl;
+    };
+};
+# NOTE: spec must _not_ have an "outPath" attribute
+mapAttrs (_: spec:
+  if builtins.hasAttr "outPath" spec
+  then abort
+    "The values in sources.json should not have an 'outPath' attribute"
+  else
+    if builtins.hasAttr "url" spec && builtins.hasAttr "sha256" spec
+    then
+      spec //
+      { outPath = callFunctionWith spec (getFetcher spec) { }; }
+    else spec
+  ) sources

tests/buildNpmPackage/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+out/

tests/buildNpmPackage/default.nix

@@ -0,0 +1,8 @@
+{ pkgs, npm-buildpackage }:
+
+npm-buildpackage.buildNpmPackage {
+  packageJson = ./package.json;
+  packageLockJson = ./package-lock.json;
+  src = builtins.filterSource (path: type: baseNameOf path != "node_modules" && baseNameOf path != "out") ./.;
+  npmBuild = "npm run build";
+}