langchain-ai/staged-recipes
1
0
Fork 0
mirror of https://github.com/langchain-ai/staged-recipes.git synced 2026-07-01 20:54:22 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add pre-commit config (#29567)

Browse Source 
* Add pre-commit config

* Apply pre-commit

* rstrip env contents to not annoy pre-commit

* remove dup
This commit is contained in:
jaimergp
2025-04-04 00:15:53 +02:00
committed by GitHub
parent 233cfd59d5
commit bddf77d301
15 changed files with 680 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci_support/build_all.py
+104 -70
View File
@@ -26,8 +26,10 @@ except ImportError:
EXAMPLE_RECIPE_FOLDERS = ["example", "example-v1"]
LOCAL_CHANNELS = os.environ.get("CONDA_BLD_PATH", "local").split(",")
def get_host_platform():
from sys import platform
if platform == "linux" or platform == "linux2":
return "linux"
elif platform == "darwin":
@@ -54,7 +56,9 @@ def build_all(recipes_dir, arch):
platform = get_host_platform()
script_dir = os.path.dirname(os.path.realpath(__file__))
variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch)))
variant_config_file = os.path.join(
script_dir, "{}.yaml".format(get_config_name(arch))
)
has_meta_yaml = False
has_recipe_yaml = False
@@ -65,11 +69,11 @@ def build_all(recipes_dir, arch):
meta_yaml = os.path.join(recipes_dir, folder, "meta.yaml")
if os.path.exists(meta_yaml):
has_meta_yaml = True
with(open(meta_yaml, "r", encoding="utf-8")) as f:
text = ''.join(f.readlines())
if 'cuda' in text:
with open(meta_yaml, "r", encoding="utf-8") as f:
text = "".join(f.readlines())
if "cuda" in text:
found_cuda = True
if 'sysroot_linux-64' in text:
if "sysroot_linux-64" in text:
found_centos7 = True
recipe_yaml = os.path.join(recipes_dir, folder, "recipe.yaml")
@@ -86,16 +90,18 @@ def build_all(recipes_dir, arch):
if os.path.exists(cbc):
with open(cbc, "r") as f:
lines = f.readlines()
pat = re.compile(r"^([^\#]*?)\s+\#\s\[.*(not\s(linux|unix)|(?<!not\s)(osx|win)).*\]\s*$")
pat = re.compile(
r"^([^\#]*?)\s+\#\s\[.*(not\s(linux|unix)|(?<!not\s)(osx|win)).*\]\s*$"
)
# remove lines with selectors that don't apply to linux, i.e. if they contain
# "not linux", "not unix", "osx" or "win"; this also removes trailing newlines
lines = [pat.sub("", x) for x in lines]
text = "\n".join(lines)
if platform == 'linux' and ('c_stdlib_version' in text):
if platform == "linux" and ("c_stdlib_version" in text):
config = load(text, Loader=BaseLoader)
if 'c_stdlib_version' in config:
for version in config['c_stdlib_version']:
version = tuple([int(x) for x in version.split('.')])
if "c_stdlib_version" in config:
for version in config["c_stdlib_version"]:
version = tuple([int(x) for x in version.split(".")])
print(f"Found c_stdlib_version for linux: {version=}")
found_centos7 |= version == (2, 17)
@@ -105,7 +111,7 @@ def build_all(recipes_dir, arch):
raise ValueError("Neither a meta.yaml or a recipe.yaml recipes was found")
if found_cuda:
print('##vso[task.setvariable variable=NEED_CUDA;isOutput=true]1')
print("##vso[task.setvariable variable=NEED_CUDA;isOutput=true]1")
if found_centos7:
os.environ["DEFAULT_LINUX_VERSION"] = "cos7"
print("Overriding DEFAULT_LINUX_VERSION to be cos7")
@@ -118,70 +124,78 @@ def build_all(recipes_dir, arch):
if os.path.exists(cbc):
with open(cbc, "r") as f:
lines = f.readlines()
pat = re.compile(r"^([^\#]*?)\s+\#\s\[.*(not\s(osx|unix)|(?<!not\s)(linux|win)).*\]\s*$")
pat = re.compile(
r"^([^\#]*?)\s+\#\s\[.*(not\s(osx|unix)|(?<!not\s)(linux|win)).*\]\s*$"
)
# remove lines with selectors that don't apply to osx, i.e. if they contain
# "not osx", "not unix", "linux" or "win"; this also removes trailing newlines
lines = [pat.sub("", x) for x in lines]
text = "\n".join(lines)
if platform == 'osx' and (
'MACOSX_DEPLOYMENT_TARGET' in text or
'MACOSX_SDK_VERSION' in text or
'c_stdlib_version' in text):
if platform == "osx" and (
"MACOSX_DEPLOYMENT_TARGET" in text
or "MACOSX_SDK_VERSION" in text
or "c_stdlib_version" in text
):
config = load(text, Loader=BaseLoader)
if 'MACOSX_DEPLOYMENT_TARGET' in config:
for version in config['MACOSX_DEPLOYMENT_TARGET']:
version = tuple([int(x) for x in version.split('.')])
if "MACOSX_DEPLOYMENT_TARGET" in config:
for version in config["MACOSX_DEPLOYMENT_TARGET"]:
version = tuple([int(x) for x in version.split(".")])
deployment_version = max(deployment_version, version)
if 'c_stdlib_version' in config:
for version in config['c_stdlib_version']:
version = tuple([int(x) for x in version.split('.')])
if "c_stdlib_version" in config:
for version in config["c_stdlib_version"]:
version = tuple([int(x) for x in version.split(".")])
print(f"Found c_stdlib_version for osx: {version=}")
deployment_version = max(deployment_version, version)
if 'MACOSX_SDK_VERSION' in config:
for version in config['MACOSX_SDK_VERSION']:
version = tuple([int(x) for x in version.split('.')])
if "MACOSX_SDK_VERSION" in config:
for version in config["MACOSX_SDK_VERSION"]:
version = tuple([int(x) for x in version.split(".")])
sdk_version = max(sdk_version, deployment_version, version)
if 'channel_sources' not in text:
new_channel_urls = [*LOCAL_CHANNELS, 'conda-forge']
if "channel_sources" not in text:
new_channel_urls = [*LOCAL_CHANNELS, "conda-forge"]
else:
config = load(text, Loader=BaseLoader)
new_channel_urls = [*LOCAL_CHANNELS, *config['channel_sources'][0].split(',')]
new_channel_urls = [
*LOCAL_CHANNELS,
*config["channel_sources"][0].split(","),
]
if channel_urls is None:
channel_urls = new_channel_urls
elif channel_urls != new_channel_urls:
raise ValueError(f'Detected different channel_sources in the recipes: {channel_urls} vs. {new_channel_urls}. Consider submitting them in separate PRs')
raise ValueError(
f"Detected different channel_sources in the recipes: {channel_urls} vs. {new_channel_urls}. Consider submitting them in separate PRs"
)
if channel_urls is None:
channel_urls = [*LOCAL_CHANNELS, 'conda-forge']
channel_urls = [*LOCAL_CHANNELS, "conda-forge"]
with open(variant_config_file, 'r') as f:
variant_text = ''.join(f.readlines())
with open(variant_config_file, "r") as f:
variant_text = "".join(f.readlines())
if deployment_version != (0, 0):
deployment_version = '.'.join([str(x) for x in deployment_version])
deployment_version = ".".join([str(x) for x in deployment_version])
print("Overriding MACOSX_DEPLOYMENT_TARGET to be ", deployment_version)
variant_text += '\nMACOSX_DEPLOYMENT_TARGET:\n'
variant_text += "\nMACOSX_DEPLOYMENT_TARGET:\n"
variant_text += f'- "{deployment_version}"\n'
if sdk_version != (0, 0):
sdk_version = '.'.join([str(x) for x in sdk_version])
sdk_version = ".".join([str(x) for x in sdk_version])
print("Overriding MACOSX_SDK_VERSION to be ", sdk_version)
variant_text += '\nMACOSX_SDK_VERSION:\n'
variant_text += "\nMACOSX_SDK_VERSION:\n"
variant_text += f'- "{sdk_version}"\n'
with open(variant_config_file, 'w') as f:
with open(variant_config_file, "w") as f:
f.write(variant_text)
if platform == "osx" and (sdk_version != (0, 0) or deployment_version != (0, 0)):
subprocess.run("run_conda_forge_build_setup", shell=True, check=True)
if 'conda-forge' not in channel_urls:
raise ValueError('conda-forge needs to be part of channel_sources')
if "conda-forge" not in channel_urls:
raise ValueError("conda-forge needs to be part of channel_sources")
if has_meta_yaml:
print("Building {} with {}".format(','.join(folders), ','.join(channel_urls)))
print("Building {} with {}".format(",".join(folders), ",".join(channel_urls)))
build_folders(recipes_dir, folders, arch, channel_urls)
elif has_recipe_yaml:
print(
@@ -193,28 +207,31 @@ def build_all(recipes_dir, arch):
def get_config(arch, channel_urls):
exclusive_config_files = [os.path.join(conda.base.context.context.root_prefix,
'conda_build_config.yaml')]
exclusive_config_files = [
os.path.join(conda.base.context.context.root_prefix, "conda_build_config.yaml")
]
script_dir = os.path.dirname(os.path.realpath(__file__))
# since variant builds override recipe/conda_build_config.yaml, see
# https://github.com/conda/conda-build/blob/3.21.8/conda_build/variants.py#L175-L181
# we need to make sure not to use variant_configs here, otherwise
# staged-recipes PRs cannot override anything using the recipe-cbc.
exclusive_config_file = os.path.join(script_dir, '{}.yaml'.format(
get_config_name(arch)))
exclusive_config_file = os.path.join(
script_dir, "{}.yaml".format(get_config_name(arch))
)
if os.path.exists(exclusive_config_file):
exclusive_config_files.append(exclusive_config_file)
config = conda_build.api.Config(
arch=arch, exclusive_config_files=exclusive_config_files,
channel_urls=channel_urls, error_overlinking=True,
arch=arch,
exclusive_config_files=exclusive_config_files,
channel_urls=channel_urls,
error_overlinking=True,
)
return config
def build_folders(recipes_dir, folders, arch, channel_urls):
index_path = os.path.join(sys.exec_prefix, 'conda-bld')
index_path = os.path.join(sys.exec_prefix, "conda-bld")
os.makedirs(index_path, exist_ok=True)
conda_index.api.update_index(index_path)
index = conda.core.index.get_index(channel_urls=channel_urls)
@@ -223,26 +240,38 @@ def build_folders(recipes_dir, folders, arch, channel_urls):
config = get_config(arch, channel_urls)
platform = get_host_platform()
worker = {'platform': platform, 'arch': arch,
'label': '{}-{}'.format(platform, arch)}
worker = {
"platform": platform,
"arch": arch,
"label": "{}-{}".format(platform, arch),
}
G = construct_graph(recipes_dir, worker=worker, run='build',
conda_resolve=conda_resolve, folders=folders,
config=config, finalize=False)
G = construct_graph(
recipes_dir,
worker=worker,
run="build",
conda_resolve=conda_resolve,
folders=folders,
config=config,
finalize=False,
)
order = list(nx.topological_sort(G))
order.reverse()
print('Computed that there are {} distributions to build from {} recipes'
.format(len(order), len(folders)))
print(
"Computed that there are {} distributions to build from {} recipes".format(
len(order), len(folders)
)
)
if not order:
print('Nothing to do')
print("Nothing to do")
return
print("Resolved dependencies, will be built in the following order:")
print(' '+'\n '.join(order))
print(" " + "\n ".join(order))
d = OrderedDict()
for node in order:
d[G.nodes[node]['meta'].meta_path] = 1
d[G.nodes[node]["meta"].meta_path] = 1
for recipe in d.keys():
conda_build.api.build([recipe], config=get_config(arch, channel_urls))
@@ -285,7 +314,7 @@ def build_folders_rattler_build(
# Construct a temporary file where we write the combined variant config. We can then pass that
# to rattler-build.
with tempfile.NamedTemporaryFile(delete=False) as fp:
fp.write(variant_config.encode("utf-8"))
fp.write(variant_config.encode("utf-8"))
atexit.register(os.unlink, fp.name)
# Execute rattler-build.
@@ -295,17 +324,21 @@ def build_folders_rattler_build(
def check_recipes_in_correct_dir(root_dir, correct_dir):
for path in Path(root_dir).glob("*"):
path = Path(path)
if path.is_dir() and path.name.lower() in ('.pixi', 'build_artifacts', 'miniforge3'):
if path.is_dir() and path.name.lower() in (
".pixi",
"build_artifacts",
"miniforge3",
):
# ignore pkg_cache in build_artifacts
continue
for recipe_path in path.rglob('*.yaml'):
for recipe_path in path.rglob("*.yaml"):
if recipe_path.name not in ("meta.yaml", "recipe.yaml"):
continue
recipe_path = recipe_path.absolute().relative_to(root_dir)
if (
(recipe_path.parts[0] != correct_dir and recipe_path.parts[0] != "broken-recipes")
or len(recipe_path.parts) != 3
):
recipe_path.parts[0] != correct_dir
and recipe_path.parts[0] != "broken-recipes"
) or len(recipe_path.parts) != 3:
raise RuntimeError(
f"recipe {recipe_path} in wrong directory; "
f"must be under {correct_dir}/<name>/"
@@ -336,21 +369,22 @@ def read_mambabuild(recipes_dir):
def use_mambabuild():
from boa.cli.mambabuild import prepare
prepare()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
'--arch',
default='64',
help='target architecture (second component of a subdir; e.g. 64, arm64, ppc64le)'
"--arch",
default="64",
help="target architecture (second component of a subdir; e.g. 64, arm64, ppc64le)",
)
args = parser.parse_args()
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
check_recipes_in_correct_dir(root_dir, "recipes")
use_mamba = read_mambabuild(os.path.join(root_dir, "recipes"))
if use_mamba:
use_mambabuild()
subprocess.run("conda clean --all --yes", shell=True, check=True)
use_mambabuild()
subprocess.run("conda clean --all --yes", shell=True, check=True)
build_all(os.path.join(root_dir, "recipes"), args.arch)
.ci_support/compute_build_graph.py
+298 -149
View File
@@ -29,6 +29,7 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
from __future__ import print_function, division
import logging
@@ -61,39 +62,57 @@ def freezeargs(func):
@functools.wraps(func)
def wrapped(*args, **kwargs):
args = (frozendict(arg) if isinstance(arg, dict) else arg for arg in args)
kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()}
kwargs = {
k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()
}
return func(*args, **kwargs)
return wrapped
def package_key(metadata, worker_label, run='build'):
def package_key(metadata, worker_label, run="build"):
# get the build string from whatever conda-build makes of the configuration
used_loop_vars = metadata.get_used_loop_vars()
build_vars = '-'.join([k + '_' + str(metadata.config.variant[k]) for k in used_loop_vars
if k != 'target_platform'])
build_vars = "-".join(
[
k + "_" + str(metadata.config.variant[k])
for k in used_loop_vars
if k != "target_platform"
]
)
# kind of a special case. Target platform determines a lot of output behavior, but may not be
# explicitly listed in the recipe.
tp = metadata.config.variant.get('target_platform')
if tp and tp != metadata.config.subdir and 'target_platform' not in build_vars:
build_vars += '-target_' + tp
tp = metadata.config.variant.get("target_platform")
if tp and tp != metadata.config.subdir and "target_platform" not in build_vars:
build_vars += "-target_" + tp
key = [metadata.name(), metadata.version()]
if build_vars:
key.append(build_vars)
key.extend(['on', worker_label])
key.extend(["on", worker_label])
key = "-".join(key)
if run == 'test':
key = '-'.join(('c3itest', key))
if run == "test":
key = "-".join(("c3itest", key))
return key
def _git_changed_files(git_rev, stop_rev=None, git_root=''):
def _git_changed_files(git_rev, stop_rev=None, git_root=""):
if not git_root:
git_root = os.getcwd()
if stop_rev:
git_rev = "{0}..{1}".format(git_rev, stop_rev)
print("Changed files from:", git_rev, stop_rev, git_root)
output = subprocess.check_output(['git', '-C', git_root, 'diff-tree',
'--no-commit-id', '--name-only', '-r', git_rev])
output = subprocess.check_output(
[
"git",
"-C",
git_root,
"diff-tree",
"--no-commit-id",
"--name-only",
"-r",
git_rev,
]
)
files = output.decode().splitlines()
return files
@@ -102,8 +121,8 @@ def _get_base_folders(base_dir, changed_files):
recipe_dirs = []
for f in changed_files:
# only consider files that come from folders
if '/' in f:
f = f.split('/')[0]
if "/" in f:
f = f.split("/")[0]
try:
find_recipe(os.path.join(base_dir, f))
recipe_dirs.append(f)
@@ -112,52 +131,59 @@ def _get_base_folders(base_dir, changed_files):
return recipe_dirs
def git_changed_submodules(git_rev='HEAD@{1}', stop_rev=None, git_root='.'):
def git_changed_submodules(git_rev="HEAD@{1}", stop_rev=None, git_root="."):
if stop_rev is not None:
git_rev = "{0}..{1}".format(git_rev, stop_rev)
diff_script = pkg_resources.resource_filename('conda_concourse_ci', 'diff-script.sh')
diff_script = pkg_resources.resource_filename(
"conda_concourse_ci", "diff-script.sh"
)
diff = subprocess.check_output(['bash', diff_script, git_rev],
cwd=git_root, universal_newlines=True)
diff = subprocess.check_output(
["bash", diff_script, git_rev], cwd=git_root, universal_newlines=True
)
submodule_changed_files = [line.split() for line in diff.splitlines()]
submodules_with_recipe_changes = []
for submodule in submodule_changed_files:
for file in submodule:
if 'recipe/' in file and submodule[0] not in submodules_with_recipe_changes:
if "recipe/" in file and submodule[0] not in submodules_with_recipe_changes:
submodules_with_recipe_changes.append(submodule[0])
return submodules_with_recipe_changes
def git_new_submodules(git_rev='HEAD@{1}', stop_rev=None, git_root='.'):
def git_new_submodules(git_rev="HEAD@{1}", stop_rev=None, git_root="."):
if stop_rev is not None:
git_rev = "{0}..{1}".format(git_rev, stop_rev)
new_submodule_script = pkg_resources.resource_filename('conda_concourse_ci',
'new-submodule-script.sh')
new_submodule_script = pkg_resources.resource_filename(
"conda_concourse_ci", "new-submodule-script.sh"
)
diff = subprocess.check_output(['bash', new_submodule_script, git_rev],
cwd=git_root, universal_newlines=True)
diff = subprocess.check_output(
["bash", new_submodule_script, git_rev], cwd=git_root, universal_newlines=True
)
return diff.splitlines()
def git_renamed_folders(git_rev='HEAD@{1}', stop_rev=None, git_root='.'):
def git_renamed_folders(git_rev="HEAD@{1}", stop_rev=None, git_root="."):
if stop_rev is not None:
git_rev = "{0}..{1}".format(git_rev, stop_rev)
rename_script = pkg_resources.resource_filename('conda_concourse_ci',
'rename-script.sh')
rename_script = pkg_resources.resource_filename(
"conda_concourse_ci", "rename-script.sh"
)
renamed_files = subprocess.check_output(['bash', rename_script], cwd=git_root,
universal_newlines=True).splitlines()
renamed_files = subprocess.check_output(
["bash", rename_script], cwd=git_root, universal_newlines=True
).splitlines()
return renamed_files
def git_changed_recipes(git_rev='HEAD@{1}', stop_rev=None, git_root='.'):
def git_changed_recipes(git_rev="HEAD@{1}", stop_rev=None, git_root="."):
"""
Get the list of files changed in a git revision and return a list of
package directories that have been modified.
@@ -190,24 +216,24 @@ def _deps_to_version_dict(deps):
if len(x) == 3:
d[x[0]] = (x[1], x[2])
elif len(x) == 2:
d[x[0]] = (x[1], 'any')
d[x[0]] = (x[1], "any")
else:
d[x[0]] = ('any', 'any')
d[x[0]] = ("any", "any")
return d
def get_build_deps(meta):
build_reqs = meta.get_value('requirements/build')
build_reqs = meta.get_value("requirements/build")
if not build_reqs:
build_reqs = []
return _deps_to_version_dict(build_reqs)
def get_run_test_deps(meta):
run_reqs = meta.get_value('requirements/run')
run_reqs = meta.get_value("requirements/run")
if not run_reqs:
run_reqs = []
test_reqs = meta.get_value('test/requires')
test_reqs = meta.get_value("test/requires")
if not test_reqs:
test_reqs = []
return _deps_to_version_dict(run_reqs + test_reqs)
@@ -220,37 +246,61 @@ _rendered_recipes = {}
@lru_cache(maxsize=None)
def _get_or_render_metadata(meta_file_or_recipe_dir, worker, finalize, config=None):
global _rendered_recipes
platform = worker['platform']
arch = str(worker['arch'])
platform = worker["platform"]
arch = str(worker["arch"])
if (meta_file_or_recipe_dir, platform, arch) not in _rendered_recipes:
print("rendering {0} for {1}".format(meta_file_or_recipe_dir, worker['label']))
_rendered_recipes[(meta_file_or_recipe_dir, platform, arch)] = \
api.render(meta_file_or_recipe_dir, platform=platform, arch=arch,
verbose=False, permit_undefined_jinja=True,
bypass_env_check=True, config=config, finalize=finalize)
print("rendering {0} for {1}".format(meta_file_or_recipe_dir, worker["label"]))
_rendered_recipes[(meta_file_or_recipe_dir, platform, arch)] = api.render(
meta_file_or_recipe_dir,
platform=platform,
arch=arch,
verbose=False,
permit_undefined_jinja=True,
bypass_env_check=True,
config=config,
finalize=finalize,
)
return _rendered_recipes[(meta_file_or_recipe_dir, platform, arch)]
def add_recipe_to_graph(recipe_dir, graph, run, worker, conda_resolve,
recipes_dir=None, config=None, finalize=False):
def add_recipe_to_graph(
recipe_dir,
graph,
run,
worker,
conda_resolve,
recipes_dir=None,
config=None,
finalize=False,
):
try:
print(recipe_dir, worker, config, finalize, flush=True)
rendered = _get_or_render_metadata(recipe_dir, worker, config=config, finalize=finalize)
except (IOError, SystemExit) as e:
log.exception('invalid recipe dir: %s', recipe_dir)
rendered = _get_or_render_metadata(
recipe_dir, worker, config=config, finalize=finalize
)
except (IOError, SystemExit):
log.exception("invalid recipe dir: %s", recipe_dir)
raise
name = None
for (metadata, _, _) in rendered:
name = package_key(metadata, worker['label'], run)
for metadata, _, _ in rendered:
name = package_key(metadata, worker["label"], run)
if metadata.skip():
continue
if name not in graph.nodes():
graph.add_node(name, meta=metadata, worker=worker)
add_dependency_nodes_and_edges(name, graph, run, worker, conda_resolve, config=config,
recipes_dir=recipes_dir, finalize=finalize)
add_dependency_nodes_and_edges(
name,
graph,
run,
worker,
conda_resolve,
config=config,
recipes_dir=recipes_dir,
finalize=finalize,
)
# # add the test equivalent at the same time. This is so that expanding can find it.
# if run == 'build':
@@ -269,13 +319,15 @@ def match_peer_job(target_matchspec, other_m, this_m=None):
"""target_matchspec comes from the recipe. target_variant is the variant from the recipe whose
deps we are matching. m is the peer job, which must satisfy conda and also have matching keys
for any keys that are shared between target_variant and m.config.variant"""
match_dict = {'name': other_m.name(),
'version': other_m.version(),
'build': _fix_any(other_m.build_id(), other_m.config), }
match_dict = {
"name": other_m.name(),
"version": other_m.version(),
"build": _fix_any(other_m.build_id(), other_m.config),
}
match_record = PackageRecord(
name=match_dict['name'],
version=match_dict['version'],
build=match_dict['build'],
name=match_dict["name"],
version=match_dict["version"],
build=match_dict["build"],
build_number=int(other_m.build_number() or 0),
channel=None,
)
@@ -294,34 +346,42 @@ def add_intradependencies(graph):
"""ensure that downstream packages wait for upstream build/test (not use existing
available packages)"""
for node in graph.nodes():
if 'meta' not in graph.nodes[node]:
if "meta" not in graph.nodes[node]:
continue
# get build dependencies
m = graph.nodes[node]['meta']
m = graph.nodes[node]["meta"]
# this is pretty hard. Realistically, we would want to know
# what the build and host platforms are on the build machine.
# However, all we know right now is what machine we're actually
# on (the one calculating the graph).
test_requires = m.meta.get('test', {}).get('requires', [])
test_requires = m.meta.get("test", {}).get("requires", [])
log.info("node: {}".format(node))
log.info(" build: {}".format(m.ms_depends('build')))
log.info(" host: {}".format(m.ms_depends('host')))
log.info(" run: {}".format(m.ms_depends('run')))
log.info(" build: {}".format(m.ms_depends("build")))
log.info(" host: {}".format(m.ms_depends("host")))
log.info(" run: {}".format(m.ms_depends("run")))
log.info(" test: {}".format(test_requires))
deps = set(m.ms_depends('build') + m.ms_depends('host') + m.ms_depends('run') +
[MatchSpec(dep) for dep in test_requires or []])
deps = set(
m.ms_depends("build")
+ m.ms_depends("host")
+ m.ms_depends("run")
+ [MatchSpec(dep) for dep in test_requires or []]
)
for dep in deps:
name_matches = (n for n in graph.nodes() if graph.nodes[n]['meta'].name() == dep.name)
name_matches = (
n for n in graph.nodes() if graph.nodes[n]["meta"].name() == dep.name
)
for matching_node in name_matches:
# are any of these build dependencies also nodes in our graph?
if (match_peer_job(MatchSpec(dep),
graph.nodes[matching_node]['meta'],
m) and
(node, matching_node) not in graph.edges()):
if (
match_peer_job(
MatchSpec(dep), graph.nodes[matching_node]["meta"], m
)
and (node, matching_node) not in graph.edges()
):
# add edges if they don't already exist
graph.add_edge(node, matching_node)
@@ -336,9 +396,9 @@ def collapse_subpackage_nodes(graph):
# group nodes by their recipe path first, then within those groups by their variant
node_groups = {}
for node in graph.nodes():
if 'meta' in graph.nodes[node]:
meta = graph.nodes[node]['meta']
meta_path = meta.meta_path or meta.meta['extra']['parent_recipe']['path']
if "meta" in graph.nodes[node]:
meta = graph.nodes[node]["meta"]
meta_path = meta.meta_path or meta.meta["extra"]["parent_recipe"]["path"]
master = False
master_meta = MetaData(meta_path, config=meta.config)
@@ -347,13 +407,15 @@ def collapse_subpackage_nodes(graph):
group = node_groups.get(meta_path, {})
subgroup = group.get(deepfreeze(meta.config.variant), {})
if master:
if 'master' in subgroup:
raise ValueError("tried to set more than one node in a group as master")
subgroup['master'] = node
if "master" in subgroup:
raise ValueError(
"tried to set more than one node in a group as master"
)
subgroup["master"] = node
else:
sps = subgroup.get('subpackages', [])
sps = subgroup.get("subpackages", [])
sps.append(node)
subgroup['subpackages'] = sps
subgroup["subpackages"] = sps
group[deepfreeze(meta.config.variant)] = subgroup
node_groups[meta_path] = group
@@ -361,18 +423,19 @@ def collapse_subpackage_nodes(graph):
for variant, subgroup in group.items():
# if no node is the top-level recipe (only outputs, no top-level output), need to obtain
# package/name from recipe given by common recipe path.
subpackages = subgroup.get('subpackages')
if 'master' not in subgroup:
subpackages = subgroup.get("subpackages")
if "master" not in subgroup:
sp0 = graph.nodes[subpackages[0]]
master_meta = MetaData(recipe_path, config=sp0['meta'].config)
worker = sp0['worker']
master_key = package_key(master_meta, worker['label'])
master_meta = MetaData(recipe_path, config=sp0["meta"].config)
worker = sp0["worker"]
master_key = package_key(master_meta, worker["label"])
graph.add_node(master_key, meta=master_meta, worker=worker)
master = graph.nodes[master_key]
else:
master = subgroup['master']
master_key = package_key(graph.nodes[master]['meta'],
graph.nodes[master]['worker']['label'])
master = subgroup["master"]
master_key = package_key(
graph.nodes[master]["meta"], graph.nodes[master]["worker"]["label"]
)
# fold in dependencies for all of the other subpackages within a group. This is just
# the intersection of the edges between all nodes. Store this on the "master" node.
if subpackages:
@@ -388,16 +451,25 @@ def collapse_subpackage_nodes(graph):
graph.remove_node(subnode)
def construct_graph(recipes_dir, worker, run, conda_resolve, folders=(),
git_rev=None, stop_rev=None, matrix_base_dir=None,
config=None, finalize=False):
'''
def construct_graph(
recipes_dir,
worker,
run,
conda_resolve,
folders=(),
git_rev=None,
stop_rev=None,
matrix_base_dir=None,
config=None,
finalize=False,
):
"""
Construct a directed graph of dependencies from a directory of recipes
run: whether to use build or run/test requirements for the graph. Avoids cycles.
values: 'build' or 'test'. Actually, only 'build' matters - otherwise, it's
run/test for any other value.
'''
"""
matrix_base_dir = matrix_base_dir or recipes_dir
if not os.path.isabs(recipes_dir):
recipes_dir = os.path.normpath(os.path.join(os.getcwd(), recipes_dir))
@@ -405,25 +477,32 @@ def construct_graph(recipes_dir, worker, run, conda_resolve, folders=(),
if not folders:
if not git_rev:
git_rev = 'HEAD'
git_rev = "HEAD"
folders = git_changed_recipes(git_rev, stop_rev=stop_rev,
git_root=recipes_dir)
folders = git_changed_recipes(git_rev, stop_rev=stop_rev, git_root=recipes_dir)
graph = nx.DiGraph()
for folder in folders:
recipe_dir = os.path.join(recipes_dir, folder)
if not os.path.isdir(recipe_dir):
raise ValueError("Specified folder {} does not exist".format(recipe_dir))
add_recipe_to_graph(recipe_dir, graph, run, worker, conda_resolve,
recipes_dir, config=config, finalize=finalize)
add_recipe_to_graph(
recipe_dir,
graph,
run,
worker,
conda_resolve,
recipes_dir,
config=config,
finalize=finalize,
)
add_intradependencies(graph)
collapse_subpackage_nodes(graph)
return graph
def _fix_any(value, config):
value = re.sub('any(?:h[0-9a-f]{%d})?' % config.hash_length, '', value)
value = re.sub("any(?:h[0-9a-f]{%d})?" % config.hash_length, "", value)
return value
@@ -431,29 +510,41 @@ def _fix_any(value, config):
def _installable(name, version, build_string, config, conda_resolve):
"""Can Conda install the package we need?"""
ms = MatchSpec(
" ".join(
[name, _fix_any(version, config), _fix_any(build_string, config)]
)
" ".join([name, _fix_any(version, config), _fix_any(build_string, config)])
)
installable = conda_resolve.find_matches(ms)
if not installable:
log.warn("Dependency {name}, version {ver} is not installable from your "
"channels: {channels} with subdir {subdir}. Seeing if we can build it..."
.format(name=name, ver=version, channels=config.channel_urls,
subdir=config.host_subdir))
log.warn(
"Dependency {name}, version {ver} is not installable from your "
"channels: {channels} with subdir {subdir}. Seeing if we can build it...".format(
name=name,
ver=version,
channels=config.channel_urls,
subdir=config.host_subdir,
)
)
return installable
def _buildable(name, version, recipes_dir, worker, config, finalize):
"""Does the recipe that we have available produce the package we need?"""
possible_dirs = os.listdir(recipes_dir)
packagename_re = re.compile(r'%s(?:\-[0-9]+[\.0-9\_\-a-zA-Z]*)?$' % name)
likely_dirs = (dirname for dirname in possible_dirs if
(os.path.isdir(os.path.join(recipes_dir, dirname)) and
packagename_re.match(dirname)))
metadata_tuples = [m for path in likely_dirs
for (m, _, _) in _get_or_render_metadata(os.path.join(recipes_dir,
path), worker, finalize=finalize)]
packagename_re = re.compile(r"%s(?:\-[0-9]+[\.0-9\_\-a-zA-Z]*)?$" % name)
likely_dirs = (
dirname
for dirname in possible_dirs
if (
os.path.isdir(os.path.join(recipes_dir, dirname))
and packagename_re.match(dirname)
)
)
metadata_tuples = [
m
for path in likely_dirs
for (m, _, _) in _get_or_render_metadata(
os.path.join(recipes_dir, path), worker, finalize=finalize
)
]
# this is our target match
ms = MatchSpec(" ".join([name, _fix_any(version, config)]))
@@ -465,46 +556,82 @@ def _buildable(name, version, recipes_dir, worker, config, finalize):
return m.meta_path if available else False
def add_dependency_nodes_and_edges(node, graph, run, worker, conda_resolve, recipes_dir=None,
finalize=False, config=None):
'''add build nodes for any upstream deps that are not yet installable
def add_dependency_nodes_and_edges(
node,
graph,
run,
worker,
conda_resolve,
recipes_dir=None,
finalize=False,
config=None,
):
"""add build nodes for any upstream deps that are not yet installable
changes graph in place.
'''
metadata = graph.nodes[node]['meta']
"""
metadata = graph.nodes[node]["meta"]
# for plain test runs, ignore build reqs.
deps = get_run_test_deps(metadata)
recipes_dir = recipes_dir or os.getcwd()
# cross: need to distinguish between build_subdir (build reqs) and host_subdir
if run == 'build':
if run == "build":
deps.update(get_build_deps(metadata))
for dep, (version, build_str) in deps.items():
# we don't need worker info in _installable because it is already part of conda_resolve
if not _installable(dep, version, build_str, metadata.config, conda_resolve):
recipe_dir = _buildable(dep, version, recipes_dir, worker, metadata.config,
finalize=finalize)
recipe_dir = _buildable(
dep, version, recipes_dir, worker, metadata.config, finalize=finalize
)
if not recipe_dir:
continue
# raise ValueError("Dependency {} is not installable, and recipe (if "
# " available) can't produce desired version ({})."
# .format(dep, version))
dep_name = add_recipe_to_graph(recipe_dir, graph, 'build', worker,
conda_resolve, recipes_dir, config=config, finalize=finalize)
dep_name = add_recipe_to_graph(
recipe_dir,
graph,
"build",
worker,
conda_resolve,
recipes_dir,
config=config,
finalize=finalize,
)
if not dep_name:
raise ValueError("Tried to build recipe {0} as dependency, which is skipped "
"in meta.yaml".format(recipe_dir))
raise ValueError(
"Tried to build recipe {0} as dependency, which is skipped "
"in meta.yaml".format(recipe_dir)
)
graph.add_edge(node, dep_name)
def expand_run_upstream(graph, conda_resolve, worker, run, steps=0, max_downstream=5,
recipes_dir=None, matrix_base_dir=None):
def expand_run_upstream(
graph,
conda_resolve,
worker,
run,
steps=0,
max_downstream=5,
recipes_dir=None,
matrix_base_dir=None,
):
pass
def expand_run(graph, conda_resolve, worker, run, steps=0, max_downstream=5,
recipes_dir=None, matrix_base_dir=None, finalize=False):
def expand_run(
graph,
conda_resolve,
worker,
run,
steps=0,
max_downstream=5,
recipes_dir=None,
matrix_base_dir=None,
finalize=False,
):
"""Apply the build label to any nodes that need (re)building or testing.
"need rebuilding" means both packages that our target package depends on,
@@ -527,9 +654,16 @@ def expand_run(graph, conda_resolve, worker, run, steps=0, max_downstream=5,
for predecessor in full_graph.predecessors(node):
if max_downstream < 0 or (downstream - initial_nodes) < max_downstream:
add_recipe_to_graph(
os.path.dirname(full_graph.nodes[predecessor]['meta'].meta_path),
task_graph, run=run, worker=worker, conda_resolve=conda_resolve,
recipes_dir=recipes_dir, finalize=finalize)
os.path.dirname(
full_graph.nodes[predecessor]["meta"].meta_path
),
task_graph,
run=run,
worker=worker,
conda_resolve=conda_resolve,
recipes_dir=recipes_dir,
finalize=finalize,
)
downstream += 1
return len(graph.nodes())
@@ -539,15 +673,19 @@ def expand_run(graph, conda_resolve, worker, run, steps=0, max_downstream=5,
if steps != 0:
if not recipes_dir:
raise ValueError("recipes_dir is necessary if steps != 0. "
"Please pass it as an argument.")
raise ValueError(
"recipes_dir is necessary if steps != 0. "
"Please pass it as an argument."
)
# here we need to fully populate a graph that has the right build or run/test deps.
# We don't create this elsewhere because it is unnecessary and costly.
# get all immediate subdirectories
other_top_dirs = [d for d in os.listdir(recipes_dir)
if os.path.isdir(os.path.join(recipes_dir, d)) and
not d.startswith('.')]
other_top_dirs = [
d
for d in os.listdir(recipes_dir)
if os.path.isdir(os.path.join(recipes_dir, d)) and not d.startswith(".")
]
recipe_dirs = []
for recipe_dir in other_top_dirs:
try:
@@ -557,8 +695,14 @@ def expand_run(graph, conda_resolve, worker, run, steps=0, max_downstream=5,
pass
# constructing the graph for build will automatically also include the test deps
full_graph = construct_graph(recipes_dir, worker, 'build', folders=recipe_dirs,
matrix_base_dir=matrix_base_dir, conda_resolve=conda_resolve)
full_graph = construct_graph(
recipes_dir,
worker,
"build",
folders=recipe_dirs,
matrix_base_dir=matrix_base_dir,
conda_resolve=conda_resolve,
)
if steps >= 0:
for step in range(steps):
@@ -572,20 +716,21 @@ def expand_run(graph, conda_resolve, worker, run, steps=0, max_downstream=5,
def order_build(graph):
'''
"""
Assumes that packages are in graph.
Builds a temporary graph of relevant nodes and returns it topological sort.
Relevant nodes selected in a breadth first traversal sourced at each pkg
in packages.
'''
"""
reorder_cyclical_test_dependencies(graph)
try:
order = list(nx.topological_sort(graph))
order.reverse()
except nx.exception.NetworkXUnfeasible:
raise ValueError("Cycles detected in graph: %s", nx.find_cycle(graph,
orientation='reverse'))
raise ValueError(
"Cycles detected in graph: %s", nx.find_cycle(graph, orientation="reverse")
)
return order
@@ -608,19 +753,23 @@ def reorder_cyclical_test_dependencies(graph):
build A <-- build B <-- test A <-- test B
"""
# find all test nodes with edges to build nodes
test_nodes = [node for node in graph.nodes() if node.startswith('test-')]
edges_from_test_to_build = [edge for edge in graph.edges() if edge[0] in test_nodes and
edge[1].startswith('build-')]
test_nodes = [node for node in graph.nodes() if node.startswith("test-")]
edges_from_test_to_build = [
edge
for edge in graph.edges()
if edge[0] in test_nodes and edge[1].startswith("build-")
]
# find any of their inverses. Entries here are of the form (test-A, build-B)
circular_deps = [edge for edge in edges_from_test_to_build
if (edge[1], edge[0]) in graph.edges()]
circular_deps = [
edge for edge in edges_from_test_to_build if (edge[1], edge[0]) in graph.edges()
]
for (testA, buildB) in circular_deps:
for testA, buildB in circular_deps:
# remove build B dependence on test A
graph.remove_edge(testA, buildB)
# remove test B dependence on build B
testB = buildB.replace('build-', 'test-', 1)
testB = buildB.replace("build-", "test-", 1)
graph.remove_edge(buildB, testB)
# Add test B dependence on test A
graph.add_edge(testA, testB)
.github/ISSUE_TEMPLATE/bugs_template.yml
+2 -2
View File
@@ -6,13 +6,13 @@ body:
attributes:
value: |
_Please note that conda-forge team doesn't follow this repo because of the high volume of notifications._
If you would like to get the attention of the conda-forge team, please do one of the following:
1. If the issue is related to the staged-recipes infrastructure, ping the `@conda-forge/staged-recipes` team in this issue.
_Note:_ If you're not a member of the conda-forge GitHub organization, this will be disabled by GitHub and you can ask the bot to ping the team for you by entering the following command in a comment: `@conda-forge-admin, please ping conda-forge/staged-recipes`
2. If the issue is related to conda-forge, please open an issue in the [general conda-forge repo](https://github.com/conda-forge/conda-forge.github.io).
3. If you need help, join our [Zulip](https://conda-forge.zulipchat.com) community chat.
3. If you need help, join our [Zulip](https://conda-forge.zulipchat.com) community chat.
- type: textarea
id: comment
.github/ISSUE_TEMPLATE/issue_template.yml
+2 -2
View File
@@ -6,13 +6,13 @@ body:
attributes:
value: |
_Please note that conda-forge team doesn't follow this repo because of the high volume of notifications._
If you would like to get the attention of the conda-forge team, please do one of the following:
1. If the issue is related to the staged-recipes infrastructure, ping the `@conda-forge/staged-recipes` team in this issue.
_Note:_ If you're not a member of the conda-forge GitHub organization, this will be disabled by GitHub and you can ask the bot to ping the team for you by entering the following command in a comment: `@conda-forge-admin, please ping conda-forge/staged-recipes`
2. If the issue is related to conda-forge, please open an issue in the [general conda-forge repo](https://github.com/conda-forge/conda-forge.github.io).
3. If you need help, join our [Zulip](https://conda-forge.zulipchat.com) community chat.
3. If you need help, join our [Zulip](https://conda-forge.zulipchat.com) community chat.
- type: textarea
id: comment
.github/ISSUE_TEMPLATE/package-request.yml
+2 -3
View File
@@ -19,7 +19,7 @@ body:
- [The conda r skeleton helpers](https://github.com/bgruening/conda_r_skeleton_helper) - to automatically generate a recipe for packages on CRAN
---
If none of these options are helpful, please fill out the following form:
- type: input
@@ -58,7 +58,7 @@ body:
For example: This package is a dependency for ...
- type: checkboxes
id: Duplicates
id: Duplicates
attributes:
label: Package is not available
description: |
@@ -76,4 +76,3 @@ body:
options:
- label: No previous issue exists and no PR has been opened.
required: true
.github/stale.yml
+1 -1
View File
@@ -22,7 +22,7 @@ markComment: >
If you'd like to keep it open, please comment/push and we will be happy to oblige!
Note that very old PRs will likely need to be rebased on `main` so that they can
Note that very old PRs will likely need to be rebased on `main` so that they can
be rebuilt with the most recent CI scripts. If you have any trouble, or we missed
reviewing this PR in the first place (sorry!), feel free
to [ping the team](https://conda-forge.org/docs/maintainer/infrastructure.html#conda-forge-admin-please-ping-team)
.github/workflows/scripts/create_feedstocks.py
+202 -115
View File
@@ -35,7 +35,7 @@ import github
import requests
from ruamel.yaml import YAML
from conda_forge_feedstock_ops.parse_package_and_feedstock_names import (
parse_package_and_feedstock_names
parse_package_and_feedstock_names,
)
from conda_forge_metadata.feedstock_outputs import sharded_path as _get_sharded_path
from conda_build.utils import create_file_with_permissions
@@ -76,7 +76,10 @@ def _register_package_for_feedstock(feedstock, pkg_name, gh):
# we proceed anyways and do not raise since it could be a rerun of staged recipes
# print a warning for the users
data = json.loads(contents.decoded_content.decode("utf-8"))
print(f" WARNING: output {pkg_name} already exists from feedstock(s) {data["feedstocks"]}", flush=True)
print(
f" WARNING: output {pkg_name} already exists from feedstock(s) {data['feedstocks']}",
flush=True,
)
def list_recipes() -> Iterator[tuple[str, str]]:
@@ -212,8 +215,11 @@ def print_rate_limiting_info(gh, user):
print("GitHub API Rate Limit Info:")
print("---------------------------")
print("token: ", user)
print("Currently remaining {remaining} out of {total}.".format(
remaining=gh_api_remaining, total=gh_api_total))
print(
"Currently remaining {remaining} out of {total}.".format(
remaining=gh_api_remaining, total=gh_api_total
)
)
print("Will reset in {time}.".format(time=gh_api_reset_time))
print("")
return gh_api_remaining
@@ -236,39 +242,42 @@ def sleep_until_reset(gh):
print("Sleeping until GitHub API resets.")
for i in range(mins_to_sleep):
time.sleep(60)
print("slept for minute {curr} out of {tot}.".format(
curr=i+1, tot=mins_to_sleep))
print(
"slept for minute {curr} out of {tot}.".format(
curr=i + 1, tot=mins_to_sleep
)
)
return True
else:
return False
if __name__ == '__main__':
if __name__ == "__main__":
exit_code = 0
is_merged_pr = os.environ.get('CF_CURRENT_BRANCH') == 'main'
is_merged_pr = os.environ.get("CF_CURRENT_BRANCH") == "main"
smithy_conf = os.path.expanduser('~/.conda-smithy')
smithy_conf = os.path.expanduser("~/.conda-smithy")
if not os.path.exists(smithy_conf):
os.mkdir(smithy_conf)
def write_token(name, token):
path = os.path.join(smithy_conf, name + '.token')
path = os.path.join(smithy_conf, name + ".token")
with create_file_with_permissions(path, 0o600) as fh:
fh.write(token)
if 'APPVEYOR_TOKEN' in os.environ:
write_token('appveyor', os.environ['APPVEYOR_TOKEN'])
if 'CIRCLE_TOKEN' in os.environ:
write_token('circle', os.environ['CIRCLE_TOKEN'])
if 'AZURE_TOKEN' in os.environ:
write_token('azure', os.environ['AZURE_TOKEN'])
if 'DRONE_TOKEN' in os.environ:
write_token('drone', os.environ['DRONE_TOKEN'])
if 'TRAVIS_TOKEN' in os.environ:
write_token('travis', os.environ['TRAVIS_TOKEN'])
if 'STAGING_BINSTAR_TOKEN' in os.environ:
write_token('anaconda', os.environ['STAGING_BINSTAR_TOKEN'])
if "APPVEYOR_TOKEN" in os.environ:
write_token("appveyor", os.environ["APPVEYOR_TOKEN"])
if "CIRCLE_TOKEN" in os.environ:
write_token("circle", os.environ["CIRCLE_TOKEN"])
if "AZURE_TOKEN" in os.environ:
write_token("azure", os.environ["AZURE_TOKEN"])
if "DRONE_TOKEN" in os.environ:
write_token("drone", os.environ["DRONE_TOKEN"])
if "TRAVIS_TOKEN" in os.environ:
write_token("travis", os.environ["TRAVIS_TOKEN"])
if "STAGING_BINSTAR_TOKEN" in os.environ:
write_token("anaconda", os.environ["STAGING_BINSTAR_TOKEN"])
# gh_drone = Github(os.environ['GH_DRONE_TOKEN'])
# gh_drone_remaining = print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
@@ -277,12 +286,12 @@ if __name__ == '__main__':
gh_travis = None
gh = None
if 'GH_TOKEN' in os.environ:
write_token('github', os.environ['GH_TOKEN'])
gh = Github(os.environ['GH_TOKEN'])
if "GH_TOKEN" in os.environ:
write_token("github", os.environ["GH_TOKEN"])
gh = Github(os.environ["GH_TOKEN"])
# Get our initial rate limit info.
gh_remaining = print_rate_limiting_info(gh, 'GH_TOKEN')
gh_remaining = print_rate_limiting_info(gh, "GH_TOKEN")
# if we are out, exit early
# if sleep_until_reset(gh):
@@ -293,10 +302,10 @@ if __name__ == '__main__':
# write_token('github', os.environ['GH_DRONE_TOKEN'])
# gh = Github(os.environ['GH_DRONE_TOKEN'])
owner_info = ['--organization', 'conda-forge']
owner_info = ["--organization", "conda-forge"]
print('Calculating the recipes which need to be turned into feedstocks.')
with tmp_dir('__feedstocks') as feedstocks_dir:
print("Calculating the recipes which need to be turned into feedstocks.")
with tmp_dir("__feedstocks") as feedstocks_dir:
feedstock_dirs = []
for recipe_dir, name in list_recipes():
if name.lower() in REPO_SKIP_LIST:
@@ -304,12 +313,19 @@ if __name__ == '__main__':
if name.lower() == "ctx":
sys.exit(1)
feedstock_dir = os.path.join(feedstocks_dir, name + '-feedstock')
print('Making feedstock for {}'.format(name))
feedstock_dir = os.path.join(feedstocks_dir, name + "-feedstock")
print("Making feedstock for {}".format(name))
try:
subprocess.check_call(
['conda', 'smithy', 'init', recipe_dir,
'--feedstock-directory', feedstock_dir])
[
"conda",
"smithy",
"init",
recipe_dir,
"--feedstock-directory",
feedstock_dir,
]
)
except subprocess.CalledProcessError:
traceback.print_exception(*sys.exc_info())
continue
@@ -319,40 +335,48 @@ if __name__ == '__main__':
# thing without having any metadata issues.
continue
subprocess.check_call([
'git', 'remote', 'add', 'upstream_with_token',
'https://conda-forge-manager:{}@github.com/'
'conda-forge/{}-feedstock'.format(
os.environ['GH_TOKEN'],
name
)
subprocess.check_call(
[
"git",
"remote",
"add",
"upstream_with_token",
"https://conda-forge-manager:{}@github.com/"
"conda-forge/{}-feedstock".format(os.environ["GH_TOKEN"], name),
],
cwd=feedstock_dir
cwd=feedstock_dir,
)
# print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
# Sometimes we already have the feedstock created. We need to
# deal with that case.
if repo_exists(gh, 'conda-forge', name + '-feedstock'):
if repo_exists(gh, "conda-forge", name + "-feedstock"):
default_branch = repo_default_branch(
gh, 'conda-forge', name + '-feedstock'
gh, "conda-forge", name + "-feedstock"
)
subprocess.check_call(
['git', 'fetch', 'upstream_with_token'], cwd=feedstock_dir)
["git", "fetch", "upstream_with_token"], cwd=feedstock_dir
)
subprocess.check_call(
['git', 'branch', '-m', default_branch, 'old'], cwd=feedstock_dir)
["git", "branch", "-m", default_branch, "old"], cwd=feedstock_dir
)
try:
subprocess.check_call(
[
'git', 'checkout', '-b', default_branch,
'upstream_with_token/%s' % default_branch
"git",
"checkout",
"-b",
default_branch,
"upstream_with_token/%s" % default_branch,
],
cwd=feedstock_dir)
cwd=feedstock_dir,
)
except subprocess.CalledProcessError:
# Sometimes, we have a repo, but there are no commits on
# it! Just catch that case.
subprocess.check_call(
['git', 'checkout', '-b', default_branch], cwd=feedstock_dir)
["git", "checkout", "-b", default_branch], cwd=feedstock_dir
)
else:
default_branch = "main"
@@ -365,8 +389,7 @@ if __name__ == '__main__':
# now register with github
subprocess.check_call(
['conda', 'smithy', 'register-github', feedstock_dir]
+ owner_info
["conda", "smithy", "register-github", feedstock_dir] + owner_info
# hack to help travis work
# + ['--extra-admin-users', gh_travis.get_user().login]
# end of hack
@@ -375,7 +398,7 @@ if __name__ == '__main__':
if gh:
# Get our final rate limit info.
print_rate_limiting_info(gh, 'GH_TOKEN')
print_rate_limiting_info(gh, "GH_TOKEN")
# drone doesn't run our jobs any more so no reason to do this
# from conda_smithy.ci_register import drone_sync
@@ -417,12 +440,24 @@ if __name__ == '__main__':
try:
subprocess.check_call(
['conda', 'smithy', 'register-ci', '--without-appveyor',
'--without-circle', '--without-drone', '--without-cirun',
'--without-webservice', '--feedstock_directory',
feedstock_dir] + owner_info)
[
"conda",
"smithy",
"register-ci",
"--without-appveyor",
"--without-circle",
"--without-drone",
"--without-cirun",
"--without-webservice",
"--feedstock_directory",
feedstock_dir,
]
+ owner_info
)
subprocess.check_call(
['conda', 'smithy', 'rerender', '--no-check-uptodate'], cwd=feedstock_dir)
["conda", "smithy", "rerender", "--no-check-uptodate"],
cwd=feedstock_dir,
)
except subprocess.CalledProcessError:
exit_code = 0
traceback.print_exception(*sys.exc_info())
@@ -431,31 +466,55 @@ if __name__ == '__main__':
# slow down so we make sure we are registered
for i in range(1, 13):
time.sleep(10)
print("Waiting for registration: {i} s".format(i=i*10))
print("Waiting for registration: {i} s".format(i=i * 10))
# if we get here, now we make the feedstock token and add the staging token
print("making the feedstock token and adding the staging binstar token")
try:
if not feedstock_token_exists("conda-forge", name + "-feedstock"):
subprocess.check_call(
['conda', 'smithy', 'generate-feedstock-token',
'--unique-token-per-provider',
'--feedstock_directory', feedstock_dir] + owner_info)
[
"conda",
"smithy",
"generate-feedstock-token",
"--unique-token-per-provider",
"--feedstock_directory",
feedstock_dir,
]
+ owner_info
)
subprocess.check_call(
['conda', 'smithy', 'register-feedstock-token',
'--unique-token-per-provider',
'--without-circle', '--without-drone',
'--feedstock_directory', feedstock_dir] + owner_info)
[
"conda",
"smithy",
"register-feedstock-token",
"--unique-token-per-provider",
"--without-circle",
"--without-drone",
"--feedstock_directory",
feedstock_dir,
]
+ owner_info
)
# add staging token env var to all CI probiders except appveyor
# and azure
# azure has it by default and appveyor is not used
subprocess.check_call(
['conda', 'smithy', 'rotate-binstar-token',
'--without-appveyor', '--without-azure',
"--without-github-actions", '--without-circle', '--without-drone',
'--token_name', 'STAGING_BINSTAR_TOKEN'],
cwd=feedstock_dir)
[
"conda",
"smithy",
"rotate-binstar-token",
"--without-appveyor",
"--without-azure",
"--without-github-actions",
"--without-circle",
"--without-drone",
"--token_name",
"STAGING_BINSTAR_TOKEN",
],
cwd=feedstock_dir,
)
yaml = YAML()
with open(os.path.join(feedstock_dir, "conda-forge.yml"), "r") as fp:
@@ -464,15 +523,18 @@ if __name__ == '__main__':
with open(os.path.join(feedstock_dir, "conda-forge.yml"), "w") as fp:
yaml.dump(_cfg, fp)
subprocess.check_call(
["git", "add", "conda-forge.yml"],
cwd=feedstock_dir
["git", "add", "conda-forge.yml"], cwd=feedstock_dir
)
subprocess.check_call(
['conda', 'smithy', 'rerender', '--no-check-uptodate'], cwd=feedstock_dir)
["conda", "smithy", "rerender", "--no-check-uptodate"],
cwd=feedstock_dir,
)
# pre-register outputs
print("registering outputs...")
_, pkg_names, _ = parse_package_and_feedstock_names(feedstock_dir, use_container=False)
_, pkg_names, _ = parse_package_and_feedstock_names(
feedstock_dir, use_container=False
)
for pkg_name in pkg_names:
_register_package_for_feedstock(name, pkg_name, gh)
except subprocess.CalledProcessError:
@@ -482,18 +544,28 @@ if __name__ == '__main__':
print("making a commit and pushing...")
subprocess.check_call(
['git', 'commit', '--allow-empty', '-am',
"Re-render the feedstock after CI registration."], cwd=feedstock_dir)
[
"git",
"commit",
"--allow-empty",
"-am",
"Re-render the feedstock after CI registration.",
],
cwd=feedstock_dir,
)
for i in range(5):
try:
# Capture the output, as it may contain the GH_TOKEN.
out = subprocess.check_output(
[
'git', 'push', 'upstream_with_token',
'HEAD:%s' % default_branch
"git",
"push",
"upstream_with_token",
"HEAD:%s" % default_branch,
],
cwd=feedstock_dir,
stderr=subprocess.STDOUT)
stderr=subprocess.STDOUT,
)
break
except subprocess.CalledProcessError:
pass
@@ -501,27 +573,33 @@ if __name__ == '__main__':
# Likely another job has already pushed to this repo.
# Place our changes on top of theirs and try again.
out = subprocess.check_output(
['git', 'fetch', 'upstream_with_token', default_branch],
["git", "fetch", "upstream_with_token", default_branch],
cwd=feedstock_dir,
stderr=subprocess.STDOUT)
stderr=subprocess.STDOUT,
)
try:
subprocess.check_call(
[
'git', 'rebase',
'upstream_with_token/%s' % default_branch, default_branch
"git",
"rebase",
"upstream_with_token/%s" % default_branch,
default_branch,
],
cwd=feedstock_dir)
cwd=feedstock_dir,
)
except subprocess.CalledProcessError:
# Handle rebase failure by choosing the changes in default_branch.
subprocess.check_call(
['git', 'checkout', default_branch, '--', '.'],
cwd=feedstock_dir)
["git", "checkout", default_branch, "--", "."],
cwd=feedstock_dir,
)
subprocess.check_call(
['git', 'rebase', '--continue'], cwd=feedstock_dir)
["git", "rebase", "--continue"], cwd=feedstock_dir
)
# Remove this recipe from the repo.
if is_merged_pr:
subprocess.check_call(['git', 'rm', '-rf', recipe_dir])
subprocess.check_call(["git", "rm", "-rf", recipe_dir])
# hack to help travis work
# from conda_smithy.ci_register import travis_cleanup
# travis_cleanup("conda-forge", name + "-feedstock")
@@ -529,17 +607,17 @@ if __name__ == '__main__':
if gh:
# Get our final rate limit info.
print_rate_limiting_info(gh, 'GH_TOKEN')
print_rate_limiting_info(gh, "GH_TOKEN")
# Update status based on the remote.
subprocess.check_call(['git', 'stash', '--keep-index', '--include-untracked'])
subprocess.check_call(['git', 'fetch'])
subprocess.check_call(["git", "stash", "--keep-index", "--include-untracked"])
subprocess.check_call(["git", "fetch"])
# CBURR: Debugging
subprocess.check_call(['git', 'status'])
subprocess.check_call(['git', 'rebase', '--autostash'])
subprocess.check_call(['git', 'add', '.'])
subprocess.check_call(["git", "status"])
subprocess.check_call(["git", "rebase", "--autostash"])
subprocess.check_call(["git", "add", "."])
try:
subprocess.check_call(['git', 'stash', 'pop'])
subprocess.check_call(["git", "stash", "pop"])
except subprocess.CalledProcessError:
# In case there was nothing to stash.
# Finish quietly.
@@ -548,8 +626,8 @@ if __name__ == '__main__':
# Parse `git status --porcelain` to handle some merge conflicts and
# generate the removed recipe list.
changed_files = subprocess.check_output(
['git', 'status', '--porcelain', recipe_directory_name],
universal_newlines=True)
["git", "status", "--porcelain", recipe_directory_name], universal_newlines=True
)
changed_files = changed_files.splitlines()
# Add all files from AU conflicts. They are new files that we
@@ -557,9 +635,10 @@ if __name__ == '__main__':
# Adding them resolves the conflict and doesn't actually add anything to the index.
new_file_conflicts = filter(lambda _: _.startswith("AU "), changed_files)
new_file_conflicts = map(
lambda _: _.replace("AU", "", 1).lstrip(), new_file_conflicts)
lambda _: _.replace("AU", "", 1).lstrip(), new_file_conflicts
)
for each_new_file in new_file_conflicts:
subprocess.check_call(['git', 'add', each_new_file])
subprocess.check_call(["git", "add", each_new_file])
# Generate a fresh listing of recipes removed.
#
@@ -570,36 +649,44 @@ if __name__ == '__main__':
removed_recipes = filter(lambda _: _.startswith("D "), changed_files)
removed_recipes = map(lambda _: _.replace("D", "", 1).lstrip(), removed_recipes)
removed_recipes = map(
lambda _: os.path.relpath(_, recipe_directory_name), removed_recipes)
lambda _: os.path.relpath(_, recipe_directory_name), removed_recipes
)
removed_recipes = map(lambda _: _.split(os.path.sep)[0], removed_recipes)
removed_recipes = sorted(set(removed_recipes))
# Commit any removed packages.
subprocess.check_call(['git', 'status'])
subprocess.check_call(["git", "status"])
if removed_recipes:
msg = ('Removed recipe{s} ({}) after converting into feedstock{s}.'
''.format(', '.join(removed_recipes),
s=('s' if len(removed_recipes) > 1 else '')))
msg += ' [ci skip]'
msg = "Removed recipe{s} ({}) after converting into feedstock{s}.".format(
", ".join(removed_recipes), s=("s" if len(removed_recipes) > 1 else "")
)
msg += " [ci skip]"
if is_merged_pr:
# Capture the output, as it may contain the GH_TOKEN.
out = subprocess.check_output(
['git', 'remote', 'add', 'upstream_with_token',
'https://x-access-token:{}@github.com/'
'conda-forge/staged-recipes'.format(os.environ['GH_TOKEN'])],
stderr=subprocess.STDOUT)
subprocess.check_call(['git', 'commit', '-m', msg])
[
"git",
"remote",
"add",
"upstream_with_token",
"https://x-access-token:{}@github.com/"
"conda-forge/staged-recipes".format(os.environ["GH_TOKEN"]),
],
stderr=subprocess.STDOUT,
)
subprocess.check_call(["git", "commit", "-m", msg])
# Capture the output, as it may contain the GH_TOKEN.
branch = os.environ.get('CF_CURRENT_BRANCH')
branch = os.environ.get("CF_CURRENT_BRANCH")
out = subprocess.check_output(
['git', 'push', 'upstream_with_token', 'HEAD:%s' % branch],
stderr=subprocess.STDOUT)
["git", "push", "upstream_with_token", "HEAD:%s" % branch],
stderr=subprocess.STDOUT,
)
else:
print('Would git commit, with the following message: \n {}'.format(msg))
print("Would git commit, with the following message: \n {}".format(msg))
if gh:
# Get our final rate limit info.
print_rate_limiting_info(gh, 'GH_TOKEN')
print_rate_limiting_info(gh, "GH_TOKEN")
# if gh_drone:
# print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
# if gh_travis:
.github/workflows/scripts/linter.py
+19 -14
View File
@@ -38,10 +38,14 @@ def _lint_recipes(gh, pr):
if "maintenance" not in labels:
for fname in fnames:
if fname in example_recipes:
lints[fname].append("Do not edit or delete example recipes in `recipes/example/` or `recipe/example-v1/`.")
lints[fname].append(
"Do not edit or delete example recipes in `recipes/example/` or `recipe/example-v1/`."
)
extra_edits = True
if not fname.startswith("recipes/"):
lints[fname].append("Do not edit files outside of the `recipes/` directory.")
lints[fname].append(
"Do not edit files outside of the `recipes/` directory."
)
extra_edits = True
# 2. Make sure the new recipe is in the right directory
@@ -58,18 +62,18 @@ def _lint_recipes(gh, pr):
)
# 3. Ensure environment.yaml and pixi.toml are in sync
original_environment_yaml = (ROOT / "environment.yaml").read_text()
original_environment_yaml = (ROOT / "environment.yaml").read_text().rstrip()
pixi_exported_env_yaml = check_output(
["pixi", "project", "export", "conda-environment", "-e", "build"],
text=True,
)
).rstrip()
if original_environment_yaml != pixi_exported_env_yaml:
import difflib
_orig_lines = original_environment_yaml.splitlines(keepends=True)
_expt_lines = pixi_exported_env_yaml.splitlines(keepends=True)
print("environment diff:", flush=True)
print(''.join(difflib.unified_diff(_orig_lines, _expt_lines)), flush=True)
print("".join(difflib.unified_diff(_orig_lines, _expt_lines)), flush=True)
lints["environment.yaml"].append(
"The `environment.yaml` file is out of sync with `pixi.toml`. "
"Fix by running `pixi project export conda-environment -e build > environment.yaml`."
@@ -103,9 +107,7 @@ def _lint_recipes(gh, pr):
outputs_section = get_section(
meta, "outputs", lints, recipe_version=recipe_version
)
extra_section = get_section(
meta, "extra", lints, recipe_version=recipe_version
)
extra_section = get_section(meta, "extra", lints, recipe_version=recipe_version)
maintainers = extra_section.get("recipe-maintainers", [])
if recipe_version == 1:
@@ -138,7 +140,9 @@ def _lint_recipes(gh, pr):
feedstock_exists = False
if feedstock_exists and existing_recipe_name == recipe_name:
lints[fname].append("Feedstock with the same name exists in conda-forge.")
lints[fname].append(
"Feedstock with the same name exists in conda-forge."
)
elif feedstock_exists:
hints[fname].append(
f"Feedstock with the name {existing_recipe_name} exists in conda-forge. "
@@ -159,7 +163,9 @@ def _lint_recipes(gh, pr):
url = None
if recipe_version == 1:
for source_section in sources_section:
if str(source_section.get("url")).startswith("https://pypi.io/packages/source/"):
if str(source_section.get("url")).startswith(
"https://pypi.io/packages/source/"
):
url = source_section["url"]
else:
for source_section in sources_section:
@@ -291,9 +297,9 @@ please add a `maintenance` label to the PR.\n"""
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Lint staged recipes.')
parser.add_argument('--owner', type=str, required=True, help='the repo owner')
parser.add_argument('--pr-num', type=int, required=True, help='the PR number')
parser = argparse.ArgumentParser(description="Lint staged recipes.")
parser.add_argument("--owner", type=str, required=True, help="the repo owner")
parser.add_argument("--pr-num", type=int, required=True, help="the PR number")
args = parser.parse_args()
@@ -305,4 +311,3 @@ if __name__ == "__main__":
_comment_on_pr(pr, lints, hints, extra_edits)
if lints:
sys.exit(1)
.github/workflows/scripts/linter_make_comment.py
+10 -4
View File
@@ -33,10 +33,16 @@ def _get_latest_run_summary(repo, workflow_run_id):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Lint staged recipes.')
parser.add_argument('--head-repo-owner', type=str, required=True, help='the head repo owner')
parser.add_argument('--workflow-run-id', type=int, required=True, help='the ID of the workflor run')
parser.add_argument('--head-sha', type=str, required=True, help='the head SHA of the PR')
parser = argparse.ArgumentParser(description="Lint staged recipes.")
parser.add_argument(
"--head-repo-owner", type=str, required=True, help="the head repo owner"
)
parser.add_argument(
"--workflow-run-id", type=int, required=True, help="the ID of the workflor run"
)
parser.add_argument(
"--head-sha", type=str, required=True, help="the head SHA of the PR"
)
args = parser.parse_args()
.github/workflows/scripts/print_tokens.py
+1 -1
View File
@@ -5,7 +5,7 @@ for token in [
"GH_TOKEN",
"GH_TRAVIS_TOKEN",
"GH_DRONE_TOKEN",
"ORGWIDE_GH_TRAVIS_TOKEN"
"ORGWIDE_GH_TRAVIS_TOKEN",
]:
try:
gh = github.Github(os.environ[token])
.pre-commit-config.yaml
+34
View File
@@ -0,0 +1,34 @@
# disable autofixing PRs, commenting "pre-commit.ci autofix" on a pull request triggers a autofix
ci:
autofix_prs: false
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
# standard end of line/end of file cleanup
- id: mixed-line-ending
- id: end-of-file-fixer
- id: trailing-whitespace
# ensure syntaxes are valid
- id: check-toml
- id: check-yaml
- repo: meta
# see https://pre-commit.com/#meta-hooks
hooks:
- id: check-hooks-apply
- id: check-useless-excludes
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.11.0
hooks:
# lint & attempt to correct failures (e.g. pyupgrade)
- id: ruff
args: [--fix]
# compatible replacement for black
- id: ruff-format
- repo: https://github.com/python-jsonschema/check-jsonschema
rev: 0.32.1
hooks:
# verify github syntaxes
- id: check-github-workflows
- id: check-dependabot
exclude: ^(broken-)?recipes/.*$
.scripts/build_steps.sh
+1 -1
View File
@@ -37,7 +37,7 @@ solver: libmamba
CONDARC
# Workaround for errors related to "unsafe" directories:
# https://github.blog/2022-04-12-git-security-vulnerability-announced/#cve-2022-24765
# https://github.blog/2022-04-12-git-security-vulnerability-announced/#cve-2022-24765
git config --global --add safe.directory "${FEEDSTOCK_ROOT}"
# Copy the host recipes folder so we don't ever muck with it
LICENSE.txt
+1 -1
View File
@@ -24,4 +24,4 @@ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
build-locally.py
+3 -2
View File
@@ -12,9 +12,10 @@ from argparse import ArgumentParser
from subprocess import check_output
def verify_system():
branch_name = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], text=True).strip()
branch_name = check_output(
["git", "rev-parse", "--abbrev-ref", "HEAD"], text=True
).strip()
if branch_name == "main":
raise RuntimeError(
"You should run build-locally from a new branch, not 'main'. "
environment.yaml
-1
View File
@@ -13,4 +13,3 @@ dependencies:
- frozendict *
- networkx 2.4.*
- rattler-build-conda-compat >=1.2.0,<2.0.0a0