Add cross-platform support for the new menuinst (#474)

* support alphas and other non-int minor versions * act on .version * fix conda version handling again * add shortcuts handling on linux / macos * change how --shortcuts-only is set * handle shortcus with --shortcuts-only now * simplify shortcut flags generation * conda-standalone always available as $PREFIX/_conda.exe * revert accidental error on rebase * make shortcuts optional on windows again * rework some options * fix macro conditional * ensure conda is in base env for extra_envs so we can remove the has_conda check from header.sh * update script path * escape curly braces * render docs * fix quotes * adjust cli syntax * define CONDA_ROOT_PREFIX so conda-standalone doesn't incorrectly use its own sys.prefix * set CONDA_ROOT_PREFIX on uninstaller too * add env shortcuts.txt to shar * remove unused import * ppd templating doesn't support nested ifs * add 'enable_shortcuts' to shellcheck * pre-commit * sync docs * fix shellcheck * relax test * only sh * revert to 'all' * create .nonadmin if not run as sudo * add menuinst v2 tests * Try with bundle_tools_2 and let's see what happens! * missing .exists() * fix expected paths * print windows install logs to stderr * move back to napari/label/bundle_tools_3 * darwin in lowercase * debug windows ci * try using cmd on windows? * move tmate * assert before uninstalling * restore workflow * force activate constructor-dev * do not initialize conda by default * restore main.yml * guard against undefined env vars * remove dup key * deprecate menuinst wrappers * revert bad merge * allow micromamba in some cases * better place for guards * add docs * pre-commit * rerender docs * process all envs under $INSTDIR * extend not append * this is already a list * enable micromamba tests on Windows * do not skip here either * do not test micromamba x windows yet * use regular conda-standalone nightly * test miniforge shortcuts too * pre-commit * debug * retrigger * revert * fix expected path * debug conda-standalone 23.10 * debug uninstall logs too * try more verbosity * fix SetDetailsPrint * logset on in uninstall * pass name * tmate again * revert debugging and require conda-standalone 23.11 * comment version out for now * remove libmamba specific test * add news * skip / xfail as needed * warn early about incompatible conda-exes * test menu_packages from extra_envs * debug * fix directory * revert debugging * allow None * pre-commit * win-select menu_packages too * revert -k selector * mark as incompatible, not disabled * fix shellcheck tests * fix conda_exe param * fix $env_shortcuts * rename conda.exe to _conda on Unix * pre-commit * pre-commit * compare enable_shortcuts to strings * return early --------- Co-authored-by: Daniel Bast <2790401+dbast@users.noreply.github.com>
conda · Jan 5, 2024 · 764ba8a · 764ba8a
1 parent 1fcb520
commit 764ba8a
Show file tree

Hide file tree

Showing 28 changed files with 509 additions and 130 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -79,11 +79,8 @@ jobs:
             conda-standalone: conda-standalone
           - os: windows
             python-version: "3.11"
+            # conda-standalone: micromamba
             conda-standalone: conda-standalone-nightly
-          # Micromamba doesn't support Windows yet (menuinst features missing)
-          # - os: windows
-          #   python-version: 3.11
-          #   conda-standalone: micromamba
 
     env:
       PYTHONUNBUFFERED: "1"
@@ -133,7 +130,6 @@ jobs:
       - name: Run examples
         env:
           CONSTRUCTOR_EXAMPLES_KEEP_ARTIFACTS: "${{ runner.temp }}/examples_artifacts"
-          # signtool only exists on Windows, but doesn't cause errors on unix when absent
           CONSTRUCTOR_SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.17763.0/x86/signtool.exe"
         run: |
           rm -rf coverage.json
@@ -144,10 +140,6 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: integration
-      - name: Test with conda-libmamba-solver
-        run: |
-          conda install -yq conda-libmamba-solver
-          CONDA_SOLVER=libmamba CONDA_VERBOSITY=1 pytest -vv tests/test_examples.py -k noconda
       - name: Check docs are up-to-date
         if: matrix.check-docs
         run: |

diff --git a/CONSTRUCT.md b/CONSTRUCT.md
@@ -128,9 +128,16 @@ _required:_ no<br/>
 _type:_ list<br/>
 
 A list of packages with menu items to be installed. The packages must have
-necessary metadata in `Menu/<package name>.json`). Menu items are currently
-only supported on Windows. By default, all menu items will be installed;
-supplying this list allows a subset to be selected instead.
+necessary metadata in `Menu/<package name>.json`). By default, all menu items
+found in the installation will be created; supplying this list allows a
+subset to be selected instead. If an empty list is supplied, no shortcuts will
+be created.
+
+If all environments (`extra_envs` included) set `menu_packages` to an empty list,
+no UI options about shortcuts will be offered to the user.
+
+Note: This option is not fully implemented when `micromamba` is used as
+the `--conda-exe` binary. The only accepted value is an empty list (`[]`).
 
 ### `ignore_duplicate_files`
 
@@ -220,12 +227,12 @@ name) to a dictionary of options:
   an empty list.
 - `user_requested_specs` (list of str): same as the global option, but for this env;
   if not provided, global value is _not_ used
+- `menu_packages` (list of str): same as the global option, for this env;
+  if not provided, the global value is _not_ used.
 
 Notes:
 - `ignore_duplicate_files` will always be considered `True` if `extra_envs` is in use.
 - `conda` needs to be present in the `base` environment (via `specs`)
-- support for `menu_packages` is planned, but not possible right now. For now, all packages
-  in an `extra_envs` config will be allowed to create their shortcuts.
 - If a global `exclude` option is used, it will have an effect on the environments created
   by `extra_envs` too. For example, if the global environment excludes `tk`, none of the
   extra environments will have it either. Unlike the global option, an error will not be

diff --git a/constructor/construct.py b/constructor/construct.py
@@ -83,11 +83,18 @@
 is contained as a result of resolving the specs for `python 2.7`.
 '''),
 
-    ('menu_packages',          False, list, '''
+    ('menu_packages',           False, list, '''
 A list of packages with menu items to be installed. The packages must have
-necessary metadata in `Menu/<package name>.json`). Menu items are currently
-only supported on Windows. By default, all menu items will be installed;
-supplying this list allows a subset to be selected instead.
+necessary metadata in `Menu/<package name>.json`). By default, all menu items
+found in the installation will be created; supplying this list allows a
+subset to be selected instead. If an empty list is supplied, no shortcuts will
+be created.
+
+If all environments (`extra_envs` included) set `menu_packages` to an empty list,
+no UI options about shortcuts will be offered to the user.
+
+Note: This option is not fully implemented when `micromamba` is used as
+the `--conda-exe` binary. The only accepted value is an empty list (`[]`).
 '''),
 
     ('ignore_duplicate_files',  False, bool, '''
@@ -152,12 +159,12 @@
   an empty list.
 - `user_requested_specs` (list of str): same as the global option, but for this env;
   if not provided, global value is _not_ used
+- `menu_packages` (list of str): same as the global option, for this env;
+  if not provided, the global value is _not_ used.
 
 Notes:
 - `ignore_duplicate_files` will always be considered `True` if `extra_envs` is in use.
 - `conda` needs to be present in the `base` environment (via `specs`)
-- support for `menu_packages` is planned, but not possible right now. For now, all packages
-  in an `extra_envs` config will be allowed to create their shortcuts.
 - If a global `exclude` option is used, it will have an effect on the environments created
   by `extra_envs` too. For example, if the global environment excludes `tk`, none of the
   extra environments will have it either. Unlike the global option, an error will not be
@@ -599,9 +606,7 @@
     "channels_remap": (list, tuple),
     "user_requested_specs": (list, tuple),
     "exclude": (list, tuple),
-    # TODO: we can't support menu_packages for extra envs yet
-    # will implement when the PR for new menuinst lands
-    # "menu_packages": (list, tuple),
+    "menu_packages": (list, tuple),
 }
 
 logger = logging.getLogger(__name__)
@@ -766,7 +771,7 @@ def verify(info):
             sys.exit(
                 f"Environment names (keys in 'extra_envs') cannot contain any of {disallowed}. "
                 f"You tried to use: {env_name}"
-                )
+            )
         for key, value in env_data.items():
             if key not in _EXTRA_ENVS_SCHEMA:
                 sys.exit(f"Key '{key}' not supported in 'extra_envs'.")

diff --git a/constructor/fcp.py b/constructor/fcp.py
@@ -56,6 +56,8 @@ def getsize(filename):
 
 
 def warn_menu_packages_missing(precs, menu_packages):
+    if not menu_packages:
+        return
     all_names = {prec.name for prec in precs}
     for name in menu_packages:
         if name not in all_names:
@@ -236,7 +238,7 @@ def _precs_from_environment(environment, input_dir):
 
 
 def _solve_precs(name, version, download_dir, platform, channel_urls=(), channels_remap=(),
-                 specs=(), exclude=(), menu_packages=(), environment=None, environment_file=None,
+                 specs=(), exclude=(), menu_packages=None, environment=None, environment_file=None,
                  verbose=True, conda_exe="conda.exe", extra_env=False, input_dir=""):
     # Add python to specs, since all installers need a python interpreter. In the future we'll
     # probably want to add conda too.
@@ -376,7 +378,7 @@ def _fetch_precs(precs, download_dir, transmute_file_type=''):
 
 
 def _main(name, version, download_dir, platform, channel_urls=(), channels_remap=(), specs=(),
-          exclude=(), menu_packages=(), ignore_duplicate_files=True, environment=None,
+          exclude=(), menu_packages=None, ignore_duplicate_files=True, environment=None,
           environment_file=None, verbose=True, dry_run=False, conda_exe="conda.exe",
           transmute_file_type='', extra_envs=None, check_path_spaces=True, input_dir=""):
     precs = _solve_precs(
@@ -408,7 +410,7 @@ def _main(name, version, download_dir, platform, channel_urls=(), channels_remap
             channels_remap=env_config.get("channels_remap", channels_remap),
             specs=env_config.get("specs", ()),
             exclude=env_config.get("exclude", exclude),
-            menu_packages=env_config.get("menu_packages", ()),
+            menu_packages=env_config.get("menu_packages"),
             environment=env_config.get("environment"),
             environment_file=env_config.get("environment_file"),
             verbose=verbose,
@@ -462,7 +464,7 @@ def main(info, verbose=True, dry_run=False, conda_exe="conda.exe"):
     channels_remap = info.get('channels_remap', ())
     specs = info.get("specs", ())
     exclude = info.get("exclude", ())
-    menu_packages = info.get("menu_packages", ())
+    menu_packages = info.get("menu_packages")
     ignore_duplicate_files = info.get("ignore_duplicate_files", True)
     environment = info.get("environment", None)
     environment_file = info.get("environment_file", None)

diff --git a/constructor/header.sh b/constructor/header.sh
@@ -46,6 +46,9 @@ KEEP_PKGS=1
 KEEP_PKGS=0
 #endif
 SKIP_SCRIPTS=0
+#if enable_shortcuts == "true"
+SKIP_SHORTCUTS=0
+#endif
 TEST=0
 REINSTALL=0
 USAGE="
@@ -70,6 +73,9 @@ Installs ${INSTALLER_NAME} ${INSTALLER_VER}
 -p PREFIX    install prefix, defaults to $PREFIX
 #endif
 -s           skip running pre/post-link/install scripts
+#if enable_shortcuts == 'true'
+-m           disable the creation of menu items / shortcuts
+#endif
 -u           update an existing installation
 #if has_conda
 -t           run package tests after installation (may install conda-build)
@@ -80,7 +86,11 @@ Installs ${INSTALLER_NAME} ${INSTALLER_VER}
 # However getopt is not standardized and the version on Mac has different
 # behaviour. getopts is good enough for what we need :)
 # More info: https://unix.stackexchange.com/questions/62950/
+#if enable_shortcuts == "true"
+while getopts "bifhkp:smut" x; do
+#else
 while getopts "bifhkp:sut" x; do
+#endif
     case "$x" in
         h)
             printf "%s\\n" "$USAGE"
@@ -104,6 +114,11 @@ while getopts "bifhkp:sut" x; do
         s)
             SKIP_SCRIPTS=1
             ;;
+#if enable_shortcuts == "true"
+        m)
+            SKIP_SHORTCUTS=1
+            ;;
+#endif
         u)
             FORCE=1
             ;;
@@ -391,14 +406,19 @@ cd "$PREFIX"
 unset PYTHON_SYSCONFIGDATA_NAME _CONDA_PYTHON_SYSCONFIGDATA_NAME
 
 # the first binary payload: the standalone conda executable
-CONDA_EXEC="$PREFIX/conda.exe"
+CONDA_EXEC="$PREFIX/_conda"
 extract_range "${boundary0}" "${boundary1}" > "$CONDA_EXEC"
 chmod +x "$CONDA_EXEC"
 
 export TMP_BACKUP="${TMP:-}"
 export TMP="$PREFIX/install_tmp"
 mkdir -p "$TMP"
 
+# Create $PREFIX/.nonadmin if the installation didn't require superuser permissions
+if [ "$(id -u)" -ne 0 ]; then
+    touch "$PREFIX/.nonadmin"
+fi
+
 # the second binary payload: the tarball of packages
 printf "Unpacking payload ...\n"
 extract_range $boundary1 $boundary2 | \
@@ -445,14 +465,31 @@ test -d ~/.conda || mkdir -p ~/.conda >/dev/null 2>/dev/null || test -d ~/.conda
 
 printf "\nInstalling base environment...\n\n"
 
+#if enable_shortcuts == "true"
+if [ "$SKIP_SHORTCUTS" = "1" ]; then
+    shortcuts="--no-shortcuts"
+else
+    shortcuts="__SHORTCUTS__"
+fi
+#endif
+#if enable_shortcuts == "false"
+shortcuts="--no-shortcuts"
+#endif
+#if enable_shortcuts == "incompatible"
+shortcuts=""
+#endif
+
+# shellcheck disable=SC2086
+CONDA_ROOT_PREFIX="$PREFIX" \
 CONDA_REGISTER_ENVS="__REGISTER_ENVS__" \
 CONDA_SAFETY_CHECKS=disabled \
 CONDA_EXTRA_SAFETY_CHECKS=no \
 CONDA_CHANNELS="__CHANNELS__" \
 CONDA_PKGS_DIRS="$PREFIX/pkgs" \
-"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" || exit 1
+"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts || exit 1
 rm -f "$PREFIX/pkgs/env.txt"
 
+#The templating doesn't support nested if statements
 #if has_conda
 mkdir -p "$PREFIX/envs"
 for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do
@@ -469,14 +506,31 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do
     else
         env_channels="__CHANNELS__"
     fi
-
-    # TODO: custom shortcuts per env?
+#endif
+#if has_conda and enable_shortcuts == "true"
+    if [ "$SKIP_SHORTCUTS" = "1" ]; then
+        env_shortcuts="--no-shortcuts"
+    else
+        # This file is guaranteed to exist, even if empty
+        env_shortcuts=$(cat "${env_pkgs}shortcuts.txt")
+        rm -f "${env_pkgs}shortcuts.txt"
+    fi
+#endif
+#if has_conda and enable_shortcuts == "false"
+    env_shortcuts="--no-shortcuts"
+#endif
+#if has_conda and enable_shortcuts == "incompatible"
+    env_shortcuts=""
+#endif
+#if has_conda
+    # shellcheck disable=SC2086
+    CONDA_ROOT_PREFIX="$PREFIX" \
     CONDA_REGISTER_ENVS="__REGISTER_ENVS__" \
     CONDA_SAFETY_CHECKS=disabled \
     CONDA_EXTRA_SAFETY_CHECKS=no \
     CONDA_CHANNELS="$env_channels" \
     CONDA_PKGS_DIRS="$PREFIX/pkgs" \
-    "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" || exit 1
+    "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts || exit 1
     rm -f "${env_pkgs}env.txt"
 done
 #endif
@@ -486,9 +540,6 @@ __INSTALL_COMMANDS__
 POSTCONDA="$PREFIX/postconda.tar.bz2"
 "$CONDA_EXEC" constructor --prefix "$PREFIX" --extract-tarball < "$POSTCONDA" || exit 1
 rm -f "$POSTCONDA"
-
-rm -f "$CONDA_EXEC"
-
 rm -rf "$PREFIX/install_tmp"
 export TMP="$TMP_BACKUP"
 
@@ -630,4 +681,5 @@ fi
 #endif
 
 exit 0
+# shellcheck disable=SC2317
 @@END_HEADER@@
diff --git a/constructor/main.py b/constructor/main.py
@@ -14,12 +14,14 @@
 
 from . import __version__
 from .build_outputs import process_build_outputs
-from .conda_interface import SUPPORTED_PLATFORMS, cc_platform
+from .conda_interface import SUPPORTED_PLATFORMS
+from .conda_interface import VersionOrder as Version
+from .conda_interface import cc_platform
 from .construct import generate_key_info_list, ns_platform
 from .construct import parse as construct_parse
 from .construct import verify as construct_verify
 from .fcp import main as fcp_main
-from .utils import normalize_path, yield_lines
+from .utils import identify_conda_exe, normalize_path, yield_lines
 
 DEFAULT_CACHE_DIR = os.getenv('CONSTRUCTOR_CACHE', '~/.conda/constructor')
 
@@ -90,7 +92,7 @@ def main_build(dir_path, output_dir='.', platform=cc_platform,
     if platform != cc_platform and 'pkg' in itypes and not cc_platform.startswith('osx-'):
         sys.exit("Error: cannot construct a macOS 'pkg' installer on '%s'" % cc_platform)
     if osname == "win" and "micromamba" in os.path.basename(info['_conda_exe']):
-        # TODO: Remove when shortcut creation is implemented on micromamba
+        # TODO: Investigate errors on Windows and re-enable
         sys.exit("Error: micromamba is not supported on Windows installers.")
 
     logger.debug('conda packages download: %s', info['_download_dir'])
@@ -145,6 +147,23 @@ def main_build(dir_path, output_dir='.', platform=cc_platform,
             if config_key == "environment_file":
                 env_config[config_key] = abspath(join(dir_path, value))
 
+    exe_name, exe_version = identify_conda_exe(info.get("_conda_exe"))
+    if sys.platform != "win32" and (
+        exe_name == "micromamba" or Version(exe_version) < Version("23.11.0")
+    ):
+        logger.warning("conda-standalone 23.11.0 or above is required for shortcuts on Unix.")
+        info['_enable_shortcuts'] = "incompatible"
+    else:
+        # Installers will provide shortcut options and features only if the user
+        # didn't opt-out by setting every `menu_packages` item to an empty list
+        info['_enable_shortcuts'] = bool(
+            info.get("menu_packages", True)
+            or any(
+                env.get("menu_packages", True)
+                for env in info.get("extra_envs", {}).values()
+            )
+        )
+
     info['installer_type'] = itypes[0]
     fcp_main(info, verbose=verbose, dry_run=dry_run, conda_exe=conda_exe)
     if dry_run: