Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve standalone executable type detection and handling #864

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
24 changes: 13 additions & 11 deletions constructor/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from .construct import parse as construct_parse
from .construct import verify as construct_verify
from .fcp import main as fcp_main
from .utils import identify_conda_exe, normalize_path, yield_lines
from .utils import StandaloneExe, identify_conda_exe, normalize_path, yield_lines

DEFAULT_CACHE_DIR = os.getenv('CONSTRUCTOR_CACHE', '~/.conda/constructor')

Expand Down Expand Up @@ -92,7 +92,13 @@ def main_build(dir_path, output_dir='.', platform=cc_platform,

if platform != cc_platform and 'pkg' in itypes and not cc_platform.startswith('osx-'):
sys.exit("Error: cannot construct a macOS 'pkg' installer on '%s'" % cc_platform)
if osname == "win" and "micromamba" in os.path.basename(info['_conda_exe']):

exe_type, exe_version = identify_conda_exe(info.get("_conda_exe"))
if exe_version is not None:
exe_version = Version(exe_version)
info["_conda_exe_type"] = exe_type
info["_conda_exe_version"] = exe_version
if osname == "win" and exe_type == StandaloneExe.MAMBA:
# TODO: Investigate errors on Windows and re-enable
sys.exit("Error: micromamba is not supported on Windows installers.")

Expand Down Expand Up @@ -172,17 +178,13 @@ def main_build(dir_path, output_dir='.', platform=cc_platform,
else:
env_config[config_key] = value

try:
exe_name, exe_version = identify_conda_exe(info.get("_conda_exe"))
except OSError as exc:
if exe_type is None or exe_version is None:
logger.warning(
"Could not identify conda-standalone / micromamba version (%s). "
"Will assume it is compatible with shortcuts.",
exc,
)
exe_name, exe_version = None, None
if sys.platform != "win32" and exe_name is not None and (
exe_name == "micromamba" or Version(exe_version) < Version("23.11.0")
"Will assume it is compatible with shortcuts."
)
if sys.platform != "win32" and exe_type is not None and (
exe_type == StandaloneExe.MAMBA or exe_version < Version("23.11.0")
):
logger.warning("conda-standalone 23.11.0 or above is required for shortcuts on Unix.")
info['_enable_shortcuts'] = "incompatible"
Expand Down
47 changes: 29 additions & 18 deletions constructor/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@
import sys
from io import StringIO
from os import environ, sep, unlink
from os.path import basename, isdir, isfile, islink, join, normpath
from os.path import isdir, isfile, islink, join, normpath
from pathlib import Path
from shutil import rmtree
from subprocess import check_call, check_output
from subprocess import CalledProcessError, check_call, check_output
from typing import Tuple, Union

from ruamel.yaml import YAML

Expand All @@ -23,6 +25,11 @@
yaml.indent(mapping=2, sequence=4, offset=2)


class StandaloneExe:
CONDA = "conda"
MAMBA = "mamba"


def explained_check_call(args):
"""
Execute a system process and debug the invocation
Expand Down Expand Up @@ -153,7 +160,7 @@ def ensure_transmuted_ext(info, url):
"""
if (
info.get("transmute_file_type") == ".conda"
and "micromamba" in basename(info.get("_conda_exe", ""))
and info.get("_conda_exe_type") == StandaloneExe.MAMBA
):
if url.lower().endswith(".tar.bz2"):
url = url[:-8] + ".conda"
Expand Down Expand Up @@ -215,15 +222,13 @@ def yield_lines(path):
yield line


def shortcuts_flags(info, conda_exe=None):
def shortcuts_flags(info) -> str:
menu_packages = info.get("menu_packages")
conda_exe = conda_exe or info.get("_conda_exe", "")
is_micromamba = "micromamba" in basename(conda_exe).lower()
if menu_packages is None:
# not set: we create all shortcuts (default behaviour)
return ""
if menu_packages:
if is_micromamba:
if info.get("_conda_exe_type") == StandaloneExe.MAMBA:
logger.warning(
"Micromamba does not support '--shortcuts-only'. "
"Will install all shortcuts."
Expand Down Expand Up @@ -252,19 +257,25 @@ def approx_size_kb(info, which="pkgs"):
return int(math.ceil(size_bytes/1000))


def identify_conda_exe(conda_exe=None):
def identify_conda_exe(conda_exe: Union[str, Path] = None) -> Tuple[StandaloneExe, str]:
if conda_exe is None:
conda_exe = normalize_path(join(sys.prefix, "standalone_conda", "conda.exe"))
output = check_output([conda_exe, "--version"], text=True)
output = output.strip()
fields = output.split()
if "conda" in fields:
name = "conda-standalone"
version = fields[1]
else:
name = "micromamba"
version = output.strip()
return name, version
if isinstance(conda_exe, Path):
conda_exe = str(conda_exe)
try:
output_version = check_output([conda_exe, "--version"], text=True)
output_version = output_version.strip()
fields = output_version.split()
if "conda" in fields:
return StandaloneExe.CONDA, fields[1]
# micromamba only returns the version number
output_help = check_output([conda_exe, "--help"], text=True)
if "mamba" in output_help:
return StandaloneExe.MAMBA, output_version
except CalledProcessError as exc:
logger.warning(f"Could not identify standalone binary {exc}.")
pass
return None, None


def win_str_esc(s, newlines=True):
Expand Down
5 changes: 4 additions & 1 deletion constructor/winexe.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,9 @@ def setup_envs_commands(info, dir_path):
for env_name in info.get("_extra_envs_info", {}):
lines += ["", ""]
env_info = info["extra_envs"][env_name]
# Needed for shortcuts_flags function
if "_conda_exe_type" not in env_info:
env_info["_conda_exe_type"] = info.get("_conda_exe_type")
channel_info = {
"channels": env_info.get("channels", info.get("channels", ())),
"channels_remap": env_info.get("channels_remap", info.get("channels_remap", ()))
Expand All @@ -185,7 +188,7 @@ def setup_envs_commands(info, dir_path):
conda_meta=join("$INSTDIR", "envs", env_name, "conda-meta"),
history_abspath=join(dir_path, "envs", env_name, "conda-meta", "history"),
channels=",".join(get_final_channels(channel_info)),
shortcuts=shortcuts_flags(env_info, conda_exe=info.get("_conda_exe")),
shortcuts=shortcuts_flags(env_info),
register_envs=str(info.get("register_envs", True)).lower(),
).splitlines()

Expand Down
19 changes: 19 additions & 0 deletions news/864-improve-standalone-binary-detection
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
### Enhancements

* Improve detection and handling of standalone executable type. (#864)

### Bug fixes

* <news item>

### Deprecations

* <news item>

### Docs

* <news item>

### Other

* <news item>
21 changes: 16 additions & 5 deletions tests/test_examples.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from conda.core.prefix_data import PrefixData
from conda.models.version import VersionOrder as Version

from constructor.utils import identify_conda_exe
from constructor.utils import StandaloneExe, identify_conda_exe

if sys.platform == "darwin":
from constructor.osxpkg import calculate_install_dir
Expand All @@ -36,6 +36,8 @@
ON_CI = os.environ.get("CI")
CONSTRUCTOR_CONDA_EXE = os.environ.get("CONSTRUCTOR_CONDA_EXE")
CONDA_EXE, CONDA_EXE_VERSION = identify_conda_exe(CONSTRUCTOR_CONDA_EXE)
if CONDA_EXE_VERSION is not None:
CONDA_EXE_VERSION = Version(CONDA_EXE_VERSION)
CONSTRUCTOR_DEBUG = bool(os.environ.get("CONSTRUCTOR_DEBUG"))
if artifacts_path := os.environ.get("CONSTRUCTOR_EXAMPLES_KEEP_ARTIFACTS"):
KEEP_ARTIFACTS_PATH = Path(artifacts_path)
Expand Down Expand Up @@ -355,8 +357,9 @@ def _example_path(example_name):
return REPO_DIR / "examples" / example_name


def _is_micromamba(path):
return "micromamba" in Path(path).stem
def _is_micromamba(path) -> bool:
name, _ = identify_conda_exe(path)
return name == StandaloneExe.MAMBA


def test_example_customize_controls(tmp_path, request):
Expand Down Expand Up @@ -391,7 +394,11 @@ def test_example_extra_files(tmp_path, request):


@pytest.mark.xfail(
CONDA_EXE == "conda-standalone" and Version(CONDA_EXE_VERSION) < Version("23.11.0a0"),
(
CONDA_EXE == StandaloneExe.CONDA
and CONDA_EXE_VERSION is not None
and CONDA_EXE_VERSION < Version("23.11.0a0")
),
reason="Known issue with conda-standalone<=23.10: shortcuts are created but not removed.",
)
def test_example_miniforge(tmp_path, request):
Expand Down Expand Up @@ -565,7 +572,11 @@ def test_example_scripts(tmp_path, request):


@pytest.mark.skipif(
CONDA_EXE == "micromamba" or Version(CONDA_EXE_VERSION) < Version("23.11.0a0"),
(
CONDA_EXE == StandaloneExe.MAMBA
or CONDA_EXE_VERSION is None
or CONDA_EXE_VERSION < Version("23.11.0a0")
),
reason="menuinst v2 requires conda-standalone>=23.11.0; micromamba is not supported yet",
)
def test_example_shortcuts(tmp_path, request):
Expand Down
Loading