Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: read feedstock name from rattler-build feedstock #2003

Merged
merged 17 commits into from
Aug 5, 2024
Merged

fix: read feedstock name from rattler-build feedstock #2003

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
df43fa3
fix: use right feedstock name
nichmor Jul 30, 2024
2a22c1d
misc: add news entry
nichmor Jul 30, 2024
72fdbe0
misc: change imports
nichmor Jul 30, 2024
79f19cf
misc: remove comment
nichmor Jul 30, 2024
4cb789b
Update conda_smithy/utils.py
nichmor Jul 30, 2024
77f9779
misc: update doc comments
nichmor Jul 30, 2024
f3f494f
misc: add tests that will fail
nichmor Aug 2, 2024
2cc1c34
misc: add test and update version bound
nichmor Aug 2, 2024
a60d716
misc: put upper bound to <2
nichmor Aug 2, 2024
fb4440a
misc: change upperbound
nichmor Aug 2, 2024
0afac05
Update environment.yml
nichmor Aug 2, 2024
1927c5b
misc: revert files as in old way
nichmor Aug 5, 2024
d354d99
Merge branch 'main' into fix/use-right-feedstock-name
nichmor Aug 5, 2024
5f877d1
Merge branch 'main' into fix/use-right-feedstock-name
beckermr Aug 5, 2024
7f226ae
misc: remove indirection
nichmor Aug 5, 2024
9ed6b9b
Update conda_smithy/utils.py
nichmor Aug 5, 2024
c901838
Merge branch 'main' into fix/use-right-feedstock-name
beckermr Aug 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions conda_smithy/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from typing import Optional, Union

import conda # noqa
import conda_build.api
from conda_build.metadata import MetaData
from rattler_build_conda_compat.render import MetaData as RattlerMetaData
from rattler_build_conda_compat.utils import has_recipe as has_rattler_recipe
Expand All @@ -21,7 +20,9 @@
from conda_smithy.utils import (
CONDA_BUILD,
RATTLER_BUILD,
get_feedstock_name_from_meta,
_get_metadata_from_feedstock_dir,
_load_forge_config,
get_feedstock_name_from_metadata,
beckermr marked this conversation as resolved.
Show resolved Hide resolved
merge_dict,
)

Expand Down Expand Up @@ -328,14 +329,15 @@ def __call__(self, args):
from conda_smithy import ci_register

owner = args.user or args.organization
meta = conda_build.api.render(
args.feedstock_directory,
permit_undefined_jinja=True,
finalize=False,
bypass_env_check=True,
trim_skip=False,
)[0][0]
feedstock_name = get_feedstock_name_from_meta(meta)

# Load the conda-forge config and read metadata from the feedstock recipe
forge_config = _load_forge_config(args.feedstock_directory, None)
metadata = _get_metadata_from_feedstock_dir(
args.feedstock_directory, forge_config
)

feedstock_name = get_feedstock_name_from_metadata(metadata)

repo = f"{feedstock_name}-feedstock"

if args.feedstock_config is None:
Expand Down
268 changes: 1 addition & 267 deletions conda_smithy/configure_feedstock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import sys
import textwrap
import time
import warnings
from collections import Counter, OrderedDict, namedtuple
from copy import deepcopy
from functools import lru_cache
Expand All @@ -30,12 +29,10 @@
import json

import conda_build.api
import conda_build.conda_interface
import conda_build.render
import conda_build.utils
import conda_build.variants
from conda.exceptions import InvalidVersionSpec
from conda.models.match_spec import MatchSpec
from conda.models.version import VersionOrder
from conda_build import __version__ as conda_build_version
from conda_build.metadata import get_selectors
Expand All @@ -55,13 +52,10 @@
from conda_smithy.utils import (
RATTLER_BUILD,
HashableDict,
_load_forge_config,
get_feedstock_about_from_meta,
get_feedstock_name_from_meta,
)
from conda_smithy.validate_schema import (
CONDA_FORGE_YAML_DEFAULTS_FILE,
validate_json_schema,
)

conda_forge_content = os.path.abspath(os.path.dirname(__file__))

Expand Down Expand Up @@ -656,18 +650,6 @@ def _yaml_represent_ordereddict(yaml_representer, data):
)


def _santize_remote_ci_setup(remote_ci_setup):
remote_ci_setup_ = conda_build.utils.ensure_list(remote_ci_setup)
remote_ci_setup = []
for package in remote_ci_setup_:
if package.startswith(("'", '"')):
pass
elif ("<" in package) or (">" in package) or ("|" in package):
package = '"' + package + '"'
remote_ci_setup.append(package)
return remote_ci_setup


def finalize_config(config, platform, arch, forge_config):
"""For configs without essential parameters like docker_image
add fallback value.
Expand Down Expand Up @@ -2169,254 +2151,6 @@ def copy_feedstock_content(forge_config, forge_dir):
copytree(feedstock_content, forge_dir, skip_files)


def _update_dict_within_dict(items, config):
"""recursively update dict within dict, if any"""
for key, value in items:
if isinstance(value, dict):
config[key] = _update_dict_within_dict(
value.items(), config.get(key, {})
)
else:
config[key] = value
return config


def _read_forge_config(forge_dir, forge_yml=None):
# Load default values from the conda-forge.yml file
with open(CONDA_FORGE_YAML_DEFAULTS_FILE) as fh:
default_config = yaml.safe_load(fh.read())

if forge_yml is None:
forge_yml = os.path.join(forge_dir, "conda-forge.yml")

if not os.path.exists(forge_yml):
raise RuntimeError(
f"Could not find config file {forge_yml}."
" Either you are not rerendering inside the feedstock root (likely)"
" or there's no `conda-forge.yml` in the feedstock root (unlikely)."
" Add an empty `conda-forge.yml` file in"
" feedstock root if it's the latter."
)

with open(forge_yml) as fh:
documents = list(yaml.safe_load_all(fh))
file_config = (documents or [None])[0] or {}

# Validate loaded configuration against a JSON schema.
validate_lints, validate_hints = validate_json_schema(file_config)
for err in chain(validate_lints, validate_hints):
logger.warning(
"%s: %s = %s -> %s",
os.path.relpath(forge_yml, forge_dir),
err.json_path,
err.instance,
err.message,
)
logger.debug("Relevant schema:\n%s", json.dumps(err.schema, indent=2))

# The config is just the union of the defaults, and the overridden
# values.
config = _update_dict_within_dict(file_config.items(), default_config)

# check for conda-smithy 2.x matrix which we can't auto-migrate
# to conda_build_config
if file_config.get("matrix") and not os.path.exists(
os.path.join(
forge_dir, config["recipe_dir"], "conda_build_config.yaml"
)
):
raise ValueError(
"Cannot rerender with matrix in conda-forge.yml."
" Please migrate matrix to conda_build_config.yaml and try again."
" See https://github.com/conda-forge/conda-smithy/wiki/Release-Notes-3.0.0.rc1"
" for more info."
)

if file_config.get("docker") and file_config.get("docker").get("image"):
raise ValueError(
"Setting docker image in conda-forge.yml is removed now."
" Use conda_build_config.yaml instead"
)

if (
"build_with_mambabuild" in file_config
and "conda_build_tool" not in file_config
):
warnings.warn(
"build_with_mambabuild is deprecated, use conda_build_tool instead",
DeprecationWarning,
)
config["conda_build_tool"] = (
"mambabuild" if config["build_with_mambabuild"] else "conda-build"
)
if file_config.get("conda_build_tool_deps"):
raise ValueError(
"Cannot set 'conda_build_tool_deps' directly. "
"Use 'conda_build_tool' instead."
)

return config


def _legacy_compatibility_checks(config: dict, forge_dir):
# An older conda-smithy used to have some files which should no longer exist,
# remove those now.
old_files = [
"disabled_appveyor.yml",
os.path.join("ci_support", "upload_or_check_non_existence.py"),
"circle.yml",
"appveyor.yml",
os.path.join("ci_support", "checkout_merge_commit.sh"),
os.path.join("ci_support", "fast_finish_ci_pr_build.sh"),
os.path.join("ci_support", "run_docker_build.sh"),
"LICENSE",
"__pycache__",
os.path.join(".github", "CONTRIBUTING.md"),
os.path.join(".github", "ISSUE_TEMPLATE.md"),
os.path.join(".github", "PULL_REQUEST_TEMPLATE.md"),
os.path.join(".github", "workflows", "main.yml"),
]

for old_file in old_files:
if old_file.replace(os.sep, "/") in config["skip_render"]:
continue
remove_file_or_dir(os.path.join(forge_dir, old_file))

# Older conda-smithy versions supported this with only one
# entry. To avoid breakage, we are converting single elements
# to a list of length one.
for platform, providers in config["provider"].items():
providers = conda_build.utils.ensure_list(providers)
config["provider"][platform] = providers

return config


def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None):
config = _read_forge_config(forge_dir, forge_yml=forge_yml)

for plat in ["linux", "osx", "win"]:
if config["azure"]["timeout_minutes"] is not None:
# fmt: off
config["azure"][f"settings_{plat}"]["timeoutInMinutes"] \
= config["azure"]["timeout_minutes"]
# fmt: on
if "name" in config["azure"][f"settings_{plat}"]["pool"]:
del config["azure"][f"settings_{plat}"]["pool"]["vmImage"]

if config["conda_forge_output_validation"]:
config["secrets"] = sorted(
set(
config["secrets"]
+ ["FEEDSTOCK_TOKEN", "STAGING_BINSTAR_TOKEN"]
)
)

target_platforms = sorted(config["build_platform"].keys())

for platform_arch in target_platforms:
config[platform_arch] = {"enabled": "True"}
if platform_arch not in config["provider"]:
config["provider"][platform_arch] = None

config["noarch_platforms"] = conda_build.utils.ensure_list(
config["noarch_platforms"]
)

# NOTE: Currently assuming these dependencies are name-only (no version constraints)
if config["conda_build_tool"] == "mambabuild":
config["conda_build_tool_deps"] = "conda-build boa"
elif config["conda_build_tool"] == "conda-build+conda-libmamba-solver":
config["conda_build_tool_deps"] = "conda-build conda-libmamba-solver"
elif config["conda_build_tool"] == "rattler-build":
config["conda_build_tool_deps"] = "rattler-build"
else:
config["conda_build_tool_deps"] = "conda-build"

# NOTE: Currently assuming these dependencies are name-only (no version constraints)
if config["conda_install_tool"] == "mamba":
config["conda_install_tool_deps"] = "mamba"
elif config["conda_install_tool"] in "conda":
config["conda_install_tool_deps"] = "conda"
if config.get("conda_solver") == "libmamba":
config["conda_install_tool_deps"] += " conda-libmamba-solver"

config["secrets"] = sorted(set(config["secrets"] + ["BINSTAR_TOKEN"]))

if config["test_on_native_only"]:
config["test"] = "native_and_emulated"

if config["test"] is None:
config["test"] = "all"

# Set some more azure defaults
config["azure"].setdefault("user_or_org", config["github"]["user_or_org"])

log = yaml.safe_dump(config)
logger.debug("## CONFIGURATION USED\n")
logger.debug(log)
logger.debug("## END CONFIGURATION\n")

if config["provider"]["linux_aarch64"] == "default":
config["provider"]["linux_aarch64"] = ["travis"]

if config["provider"]["linux_aarch64"] == "native":
config["provider"]["linux_aarch64"] = ["travis"]

if config["provider"]["linux_ppc64le"] == "default":
config["provider"]["linux_ppc64le"] = ["travis"]

if config["provider"]["linux_ppc64le"] == "native":
config["provider"]["linux_ppc64le"] = ["travis"]

if config["provider"]["linux_s390x"] in {"default", "native"}:
config["provider"]["linux_s390x"] = ["travis"]

config["remote_ci_setup"] = _santize_remote_ci_setup(
config["remote_ci_setup"]
)
if config["conda_install_tool"] == "conda":
config["remote_ci_setup_update"] = [
MatchSpec(pkg.strip('"').strip("'")).name
for pkg in config["remote_ci_setup"]
]
else:
config["remote_ci_setup_update"] = config["remote_ci_setup"]

if not config["github_actions"]["triggers"]:
self_hosted = config["github_actions"]["self_hosted"]
config["github_actions"]["triggers"] = (
["push"] if self_hosted else ["push", "pull_request"]
)

# Run the legacy checks for backwards compatibility
config = _legacy_compatibility_checks(config, forge_dir)

# Fallback handling set to azure, for platforms that are not fully specified by this time
for platform, providers in config["provider"].items():
for i, provider in enumerate(providers):
if provider in {"default", "emulated"}:
providers[i] = "azure"

# Set the environment variable for the compiler stack
os.environ["CF_COMPILER_STACK"] = config["compiler_stack"]
# Set valid ranger for the supported platforms
os.environ["CF_MIN_PY_VER"] = config["min_py_ver"]
os.environ["CF_MAX_PY_VER"] = config["max_py_ver"]
os.environ["CF_MIN_R_VER"] = config["min_r_ver"]
os.environ["CF_MAX_R_VER"] = config["max_r_ver"]

config["package"] = os.path.basename(forge_dir)
if not config["github"]["repo_name"]:
feedstock_name = os.path.basename(forge_dir)
if not feedstock_name.endswith("-feedstock"):
feedstock_name += "-feedstock"
config["github"]["repo_name"] = feedstock_name
config["exclusive_config_file"] = exclusive_config_file

return config


def get_most_recent_version(name, include_broken=False):
request = requests.get(
"https://api.anaconda.org/package/conda-forge/" + name
Expand Down
Loading