diff --git a/CHANGELOG.md b/CHANGELOG.md index b54eac0a..ba5f3e68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ --- # Unreleased +- **[Improvement]** Bits searches for package recipes at each checkout root first, then in `recipes/`. Flat and mixed checkouts work, and root recipes win on name clashes. Defaults, patches and other repository files stay at the root. - **[Change]** bits detects the platform name the CI and the builder images use: `x86_64-el9` on RHEL, AlmaLinux and Rocky 9, `x86_64-ubuntu2404` on Ubuntu 24.04 (macOS keeps `osx_arm64`). The `aliBuild` wrapper keeps ALICE's names (`slc9_x86-64`) by setting `BITS_LEGACY_ARCH=1`; native bits for ALICE records it with `bits use --architecture slc9_x86-64`. ALICE's store is read only for ALICE's names exactly (USERGUIDE "Platforms"). - **[Fix]** `bits init stacks.bits` (any `.bits`) no longer creates `sw` in the current directory: the work dir belongs inside the checkout, where you then build. - **[Improvement]** `bits --version` of an installed bits whose `_version.py` holds `git describe` (as bits.bits writes it) shows the tag, the commits since and the commit, e.g. `bits 0.7-3-g1a2b3c4 (tag 0.7 +3, commit 1a2b3c4)`. diff --git a/bits_helpers/brew.py b/bits_helpers/brew.py index 3ef1b944..726735d5 100644 --- a/bits_helpers/brew.py +++ b/bits_helpers/brew.py @@ -51,6 +51,7 @@ import glob from bits_helpers.log import debug, error, info +from bits_helpers.paths import recipe_files from bits_helpers.recipe import parseRecipe, FileReader @@ -120,12 +121,12 @@ def collect_homebrew(configDirs, architecture): configDirs = [configDirs] seen = set() for configDir in configDirs: - for path in sorted(glob.glob(os.path.join(configDir, "*.sh"))): + for path in recipe_files(configDir, include_defaults=True): real = os.path.realpath(path) if real in seen: continue seen.add(real) - err, spec, _ = parseRecipe(FileReader(path)) + err, spec, _ = parseRecipe(FileReader(path, repo_dir=configDir)) if err or not spec: # Not a parseable recipe (e.g. a defaults-*.sh helper). Skip quietly. continue @@ -143,13 +144,8 @@ def _standalone_scan_dirs(configDir, work_dir): """ dirs = [configDir] repos = os.path.join(os.path.abspath(work_dir), "REPOS") - if os.path.isdir(repos): - # Any directory under REPOS that contains recipes. Duplicates across - # provider versions are harmless — formulae are collected into a set. - found = set() - for path in glob.glob(os.path.join(repos, "**", "*.sh"), recursive=True): - found.add(os.path.dirname(path)) - dirs.extend(sorted(found)) + dirs.extend(sorted(path for path in glob.glob(os.path.join(repos, "*", "*")) + if os.path.isdir(path) and not os.path.islink(path))) return dirs diff --git a/bits_helpers/checksums_cmd.py b/bits_helpers/checksums_cmd.py index bf3e4e01..81558371 100644 --- a/bits_helpers/checksums_cmd.py +++ b/bits_helpers/checksums_cmd.py @@ -45,6 +45,7 @@ from bits_helpers.defaults import asDict, merge_dicts from bits_helpers.log import error, info, warning from bits_helpers.matchers import _parse_patch_entry, resolve_variables +from bits_helpers.paths import recipe_file_name, recipe_files from bits_helpers.recipe import getRecipeReader, parseRecipe from bits_helpers.utilities import apply_version_from, resolve_spec_data from bits_helpers.workarea import _resolve_source_entry, _split_arch_prefix @@ -57,9 +58,9 @@ # ── Recipes and profiles ────────────────────────────────────────────────────── -def _read(path): +def _read(path, repo_dir=None): """Parsed YAML header of recipe *path*, or ``None`` (warned) if unparsable.""" - err, meta, _ = parseRecipe(getRecipeReader(path)) + err, meta, _ = parseRecipe(getRecipeReader(path, repo_dir=repo_dir)) if err or not isinstance(meta, dict): warning("Skipping %s: %s", path, err or "not a recipe") return None @@ -68,12 +69,7 @@ def _read(path): def recipes_in(repo): """``{lowercase package name: recipe path}`` for the recipes of *repo*.""" - out = {} - for f in sorted(os.listdir(repo)): - path = os.path.join(repo, f) - if f.endswith(".sh") and not f.startswith("defaults-") and os.path.isfile(path): - out[f[:-3].lower()] = path - return out + return {recipe_file_name(path): path for path in recipe_files(repo, strict=True)} def profiles_in(repo, want): @@ -261,7 +257,7 @@ def plan(repo, recipe_dirs, profile_names, packages, arch): search = [repo] + [d for d in recipe_dirs if os.path.abspath(d) != repo] found = {} for d in reversed(search): # earlier repositories win, as in a build - found.update(recipes_in(d)) + found.update((name, (path, d)) for name, path in recipes_in(d).items()) own = recipes_in(repo) want = {p.lower() for p in packages} profiles = profiles_in(repo, profile_names) if profile_names else [] @@ -298,14 +294,15 @@ def add(pkg, patches_dir, got, origin): for name in sorted(set(own) | set(found if profiles else ())): if want and name not in want: continue - meta = _read(found[name]) + path, recipe_repo = found[name] + meta = _read(path, repo_dir=recipe_repo) if meta is None: continue pkg = str(meta.get("package", name)) if pkg.lower() != name: # the build refuses such a recipe too - warning("Skipping %s: its package field is %r", found[name], pkg) + warning("Skipping %s: its package field is %r", path, pkg) continue - patches_dir = os.path.join(os.path.dirname(found[name]), "patches") + patches_dir = os.path.join(recipe_repo, "patches") try: base = wants(meta, ["release"], rel_vars, arch, source_mode(release)) except ValueError as exc: diff --git a/bits_helpers/compliance.py b/bits_helpers/compliance.py index 495b190e..e4429b78 100644 --- a/bits_helpers/compliance.py +++ b/bits_helpers/compliance.py @@ -27,6 +27,7 @@ import urllib.request from bits_helpers.log import banner, debug, error, info, warning +from bits_helpers.paths import recipe_files from bits_helpers import site # Front-matter keys read from each recipe. Values may be quoted (the header @@ -71,10 +72,9 @@ def scan_recipes(recipes_dir): out = {"total": 0, "missing_license": [], "licenseref": [], "noassertion": [], "restricted": [], "restricted_sources": [], "by_package": {}} - for name in sorted(os.listdir(recipes_dir)): - if not name.endswith(".sh"): - continue - meta = _recipe_meta(os.path.join(recipes_dir, name)) + for path in recipe_files(recipes_dir, include_defaults=True, strict=True): + name = os.path.relpath(path, recipes_dir) + meta = _recipe_meta(path) pkg = meta.get("package") if not pkg: continue # not a recipe (helper script) @@ -631,7 +631,7 @@ def doCompliance(args, parser): "distributed)", len(system_pkgs)) else: recipes_dir = os.path.abspath(getattr(args, "recipesDir", None) or ".") - if not any(f.endswith(".sh") for f in os.listdir(recipes_dir)): + if not recipe_files(recipes_dir, include_defaults=True): parser.error("no recipes (*.sh) found in %s — point --recipes at a " "recipe repository (e.g. a lcg.bits checkout), or give " "PACKAGE roots for a group-wide audit" % recipes_dir) diff --git a/bits_helpers/doctor.py b/bits_helpers/doctor.py index 3c7ae062..d0283e03 100644 --- a/bits_helpers/doctor.py +++ b/bits_helpers/doctor.py @@ -54,6 +54,7 @@ from bits_helpers.defaults import (parseDefaults, readDefaults, validateDefaults, incompatibleFlavorDefaults) from bits_helpers.packages import getPackageList +from bits_helpers.paths import checkForFilename from bits_helpers.arch import effective_arch # ── Status constants ─────────────────────────────────────────────────────────── @@ -839,7 +840,7 @@ def doDoctor(args, parser): packages = [] for p in args.packages: - recipe_path = "%s/%s.sh" % (args.configDir, p.lower()) + recipe_path = checkForFilename({}, p.lower(), args.configDir) if not exists(recipe_path): error("Cannot find recipe %s for package %s.", recipe_path, p) exitcode = 1 diff --git a/bits_helpers/packages.py b/bits_helpers/packages.py index 9882db16..1cc929ea 100644 --- a/bits_helpers/packages.py +++ b/bits_helpers/packages.py @@ -256,7 +256,8 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, "is the intended one.", p, recipeSourceLabel(pkgdir, provider_dirs), recipeSourceLabel(_later, provider_dirs), ", ".join(_defaults_dirs[_later])) - err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir, generatedPackages[pkgdir]), generatedPackages) + reader = getRecipeReader(filename, configDir, generatedPackages[pkgdir], repo_dir=pkgdir) + err, spec, recipe = parseRecipe(reader, generatedPackages) dieOnError(err, err) # Unless there was an error, both spec and recipe should be valid. # otherwise the error should have been caught above. diff --git a/bits_helpers/paths.py b/bits_helpers/paths.py index 3b4785c5..4e4f5d32 100644 --- a/bits_helpers/paths.py +++ b/bits_helpers/paths.py @@ -7,18 +7,57 @@ layers above it can share these without an import cycle.""" import os -from os.path import exists, join +from os.path import basename, dirname, exists, isdir, isfile, join from bits_helpers.log import dieOnError, error -def checkForFilename(taps, pkg, d, ext=".sh"): - filename = taps.get(pkg, "{}/{}{}".format(d, pkg, ext)) - if not exists(filename): - if "/" in pkg: - filename = taps.get(pkg, "{}/{}".format(d, pkg)) - else: - filename = taps.get(pkg, "{}/{}/latest".format(d, pkg)) - return filename +RECIPE_SUBDIR = "recipes" + + +def recipe_file_name(path): + """Package lookup name of a listed .sh file or versioned latest recipe.""" + name = basename(path) + return (basename(dirname(path)) if name == "latest" else name[:-3]).lower() + + +def recipe_files(repo, include_defaults=False, strict=False): + """List .sh recipes from both locations, with root copies taking precedence.""" + def scan(directory, defaults=False, optional=False): + try: + with os.scandir(directory) as entries: + return sorted(e.path for e in entries if e.name.endswith(".sh") + and (defaults or not e.name.startswith("defaults-")) and e.is_file()) + except OSError as exc: + if strict and not (optional and isinstance(exc, (FileNotFoundError, NotADirectoryError))): + raise + return [] + + root = scan(repo, defaults=include_defaults) + names = {recipe_file_name(path) for path in root} + for path in scan(join(repo, RECIPE_SUBDIR), optional=True): + name = recipe_file_name(path) + if name not in names: + latest = join(repo, name, "latest") + root.append(latest if isfile(latest) else path) + return root + + +def checkForFilename(taps, pkg, d, ext=".sh", *, nested=True): + if pkg in taps: + return taps[pkg] + + def candidates(directory): + yield "{}/{}{}".format(directory, pkg, ext) + yield "{}/{}".format(directory, pkg) if "/" in pkg else "{}/{}/latest".format(directory, pkg) + + root, fallback = candidates(d) + if exists(root) and (ext != ".sh" or not isdir(root)): + return root + if nested and ext == ".sh" and not pkg.startswith("defaults-") and not isfile(fallback): + for recipe in candidates(join(d, RECIPE_SUBDIR)): + if isfile(recipe): + return recipe + return fallback def resolveLocalPath(configDir, s): """ diff --git a/bits_helpers/preload_cmd.py b/bits_helpers/preload_cmd.py index 32553f16..dbd4d38b 100644 --- a/bits_helpers/preload_cmd.py +++ b/bits_helpers/preload_cmd.py @@ -399,12 +399,13 @@ def main(argv=None): ap.error("--docker requires --docker-image IMAGE") def recipe_reader(pkg): + from bits_helpers.paths import checkForFilename from bits_helpers.recipe import parseRecipe, FileReader - path = os.path.join(a.config_dir, pkg + ".sh") + path = checkForFilename({}, pkg, a.config_dir) if not os.path.isfile(path): return None try: - _err, spec, _body = parseRecipe(FileReader(path)) + _err, spec, _body = parseRecipe(FileReader(path, repo_dir=a.config_dir)) return spec except Exception: return None diff --git a/bits_helpers/recipe.py b/bits_helpers/recipe.py index 607f6736..c3644e02 100644 --- a/bits_helpers/recipe.py +++ b/bits_helpers/recipe.py @@ -13,7 +13,7 @@ import re import sys from collections import OrderedDict -from glob import glob +from glob import escape, glob from os.path import join from typing import Any, IO @@ -22,7 +22,7 @@ from bits_helpers.cmd import getoutput from bits_helpers.git import git from bits_helpers.log import dieOnError -from bits_helpers.paths import getConfigPaths, resolveFilename +from bits_helpers.paths import RECIPE_SUBDIR, checkForFilename, getConfigPaths, resolveFilename class SpecError(Exception): pass @@ -43,7 +43,7 @@ def validateSpec(spec): spec["version"] = "0" -def getRecipeReader(url: str, dist=None, genPackages={}): +def getRecipeReader(url: str, dist=None, genPackages={}, repo_dir=None): m = re.search(r'^(dist|generate):(.*)@([^@]+)$', url) if m and m.group(1) == "generate": pkg, version = m.group(2), m.group(3) @@ -54,7 +54,7 @@ def getRecipeReader(url: str, dist=None, genPackages={}): elif m and dist: return GitReader(url, dist) else: - return FileReader(url) + return FileReader(url, repo_dir) # Generate a recipe of package class GeneratedPackage: @@ -66,8 +66,14 @@ def __call__(self): # Read a recipe from a file class FileReader: - def __init__(self, url) -> None: + def __init__(self, url, repo_dir=None) -> None: self.url = url + self.repo_dir = None + if repo_dir: + nested = join(os.path.abspath(repo_dir), RECIPE_SUBDIR) + path = os.path.abspath(url) + if path.startswith(nested + os.sep) and path != join(nested, "latest"): + self.repo_dir = repo_dir def __call__(self): with open(self.url) as f: return f.read() @@ -79,8 +85,10 @@ def __init__(self, url, configDir) -> None: def __call__(self): m = re.search(r'^dist:(.*)@([^@]+)$', self.url) fn, gh = m.groups() - err, d = git(("show", f"{gh}:{fn.lower()}.sh"), - directory=self.configDir) + fn = fn.lower() + err, d = git(("show", f"{gh}:{fn}.sh"), directory=self.configDir, check=False) + if err and not fn.startswith("defaults-"): + err, d = git(("show", f"{gh}:{RECIPE_SUBDIR}/{fn}.sh"), directory=self.configDir, check=False) if err: raise RuntimeError("Cannot read recipe {fn} from reference {gh}.\n" "Make sure you run first (this will not alter your recipes):\n" @@ -238,7 +246,8 @@ def parseRecipe(reader, generatePackages=None, visited=None): header, recipe = d[:_m.start()], d[_m.end():] # Splice any `#!include` directives in the body before anything else sees it, # so the included text is variable-expanded and hashed as if written inline. - recipe = resolveIncludes(recipe, getattr(reader, "url", "") or "") + repo_dir = getattr(reader, "repo_dir", None) + recipe = resolveIncludes(recipe, getattr(reader, "url", "") or "", repo_dir) # YAML forbids '%' as the first character of a plain (unquoted) scalar because # it is reserved for directives (e.g. %YAML, %TAG). Recipe authors may want # to write "- %(name)s-%(version)s.patch" in patches: (and similar lists) @@ -271,15 +280,15 @@ def parseRecipe(reader, generatePackages=None, visited=None): if spec and "from" in spec: basename = os.path.basename(getattr(reader, "url", "") or "") filename = basename[:-3] if basename.endswith(".sh") else basename - repoDir = os.environ.get("BITS_REPO_DIR") + repoDir = repo_dir or os.environ.get("BITS_REPO_DIR") if visited is None: visited = [] if spec["from"] in visited: raise RuntimeError(f" Cyclic Dependency: {' -> '.join(list(visited) + [spec['from']])}") visited.append(spec["from"]) parent_dir = os.path.join(repoDir, spec["from"]) - base_filename, pkgdir = resolveFilename({}, filename, parent_dir, generatePackages) - base_reader = getRecipeReader(base_filename, repoDir, generatePackages[parent_dir]) + base_filename, pkgdir = resolveFilename({}, filename, parent_dir, generatePackages or {}) + base_reader = getRecipeReader(base_filename, repoDir, (generatePackages or {}).get(pkgdir, {}), pkgdir) err, base_spec, base_recipe = parseRecipe(base_reader, generatePackages, visited) spec, recipe_append = handleMergePolicy(spec, base_spec) recipe = recipe + base_recipe if recipe_append else recipe @@ -302,11 +311,10 @@ def parseRecipe(reader, generatePackages=None, visited=None): def getGeneratedPackages(configDir): all_pkgs = {} pkgDirs = getConfigPaths(configDir) - for pkgdir in pkgDirs: + def generated_in(directory, pkgdir): dir_pkgs = {} - for vp in [x.split(os.sep)[-2] for x in glob(join(pkgdir, "*", "packages.py"))]: - packages_py = join(pkgdir, vp, "packages.py") - sys.path.insert(0, join(pkgdir, vp)) + for packages_py in glob(join(escape(directory), "*", "packages.py")): + sys.path.insert(0, os.path.dirname(packages_py)) try: pkg = __import__("packages") except (ImportError, SyntaxError) as e: @@ -319,6 +327,14 @@ def getGeneratedPackages(configDir): dieOnError(True, "Error running getPackages() in %r: %s" % (packages_py, e)) sys.modules.pop("packages") sys.path.pop(0) + return dir_pkgs + + for pkgdir in pkgDirs: + dir_pkgs = generated_in(pkgdir, pkgdir) + nested_pkgs = generated_in(join(pkgdir, RECIPE_SUBDIR), pkgdir) + for name, meta in nested_pkgs.items(): + if name not in dir_pkgs and not os.path.exists(checkForFilename({}, name, pkgdir, nested=False)): + dir_pkgs[name] = meta all_pkgs[pkgdir] = dir_pkgs return all_pkgs diff --git a/bits_helpers/repo_provider.py b/bits_helpers/repo_provider.py index 84a6d6e5..c8e7a5f5 100644 --- a/bits_helpers/repo_provider.py +++ b/bits_helpers/repo_provider.py @@ -50,7 +50,6 @@ """ import fcntl -import glob import os import re import shutil @@ -64,7 +63,7 @@ from bits_helpers.workarea import updateReferenceRepoSpec, logged_scm from bits_helpers.utilities import resolve_spec_data, symlink from bits_helpers.recipe import getGeneratedPackages, getRecipeReader, parseRecipe -from bits_helpers.paths import checkForFilename, getConfigPaths +from bits_helpers.paths import checkForFilename, getConfigPaths, recipe_file_name, recipe_files from bits_helpers.matchers import _parse_req_matcher # Maximum provider-discovery iterations (guards against run-away recursion) @@ -117,8 +116,7 @@ def _check_for_shadows( return # nothing on BITS_PATH yet — no shadowing possible incoming_recipes = { - os.path.splitext(os.path.basename(f))[0].lower() - for f in glob.glob(os.path.join(incoming_dir, "*.sh")) + recipe_file_name(f) for f in recipe_files(incoming_dir, include_defaults=True) } if not incoming_recipes: return # empty provider directory — nothing to shadow @@ -130,8 +128,7 @@ def _check_for_shadows( ) for existing_dir in existing_dirs: existing_recipes = { - os.path.splitext(os.path.basename(f))[0].lower() - for f in glob.glob(os.path.join(existing_dir, "*.sh")) + recipe_file_name(f) for f in recipe_files(existing_dir, include_defaults=True) } shadowed = incoming_recipes & existing_recipes if shadowed: @@ -268,7 +265,7 @@ def _try_read_spec(pkg_lower: str, config_dir: str, taps: dict): continue gen_pkgs = {} - err, spec, _ = parseRecipe(getRecipeReader(filename, config_dir, gen_pkgs)) + err, spec, _ = parseRecipe(getRecipeReader(filename, config_dir, gen_pkgs, repo_dir=pkg_dir)) if err or spec is None: continue return spec @@ -555,9 +552,9 @@ def load_always_on_providers( ) # ── 2. ``always_load: true`` recipes in the primary config dir ────────── - for sh_path in sorted(glob.glob(os.path.join(abspath(config_dir), "*.sh"))): + for sh_path in recipe_files(abspath(config_dir), include_defaults=True): try: - err, spec, _ = parseRecipe(getRecipeReader(sh_path)) + err, spec, _ = parseRecipe(getRecipeReader(sh_path, repo_dir=config_dir)) except Exception: continue if err or spec is None: @@ -667,7 +664,7 @@ def _local_provider_dir(config_dir, package): unchanged (the local dir simply takes the clone's place on ``BITS_PATH``). """ d = join(abspath(config_dir), package) - if os.path.isdir(d) and glob.glob(join(d, "*.sh")): + if recipe_files(d, include_defaults=True): return d return None @@ -741,13 +738,13 @@ def bootstrap_default_config(args, work_dir: str) -> Optional[str]: organisation = (getattr(args, "organisation", None) or "").lower() candidates = [] if organisation: - candidates.append(("%s.bits.sh" % organisation, organisation)) - candidates.append(("default.bits.sh", "default")) + candidates.append(("%s.bits" % organisation, organisation)) + candidates.append(("default.bits", "default")) chosen_sh = None chosen_label = None - for filename, label in candidates: - path = join(providers_checkout, filename) + for name, label in candidates: + path = checkForFilename({}, name, providers_checkout) if exists(path): chosen_sh = path chosen_label = label @@ -765,7 +762,7 @@ def bootstrap_default_config(args, work_dir: str) -> Optional[str]: return None try: - err, default_spec, _ = parseRecipe(getRecipeReader(chosen_sh)) + err, default_spec, _ = parseRecipe(getRecipeReader(chosen_sh, repo_dir=providers_checkout)) except Exception as exc: warning("Bootstrap: could not parse %s.bits.sh: %s", chosen_label, exc) return None @@ -842,12 +839,12 @@ def _warn(*a): _warn("Could not fetch the provider registry from %s.", url) return None - sh = join(providers_checkout, name + ".sh") + sh = checkForFilename({}, name, providers_checkout) if not exists(sh): _warn("No '%s.sh' in the provider registry %s.", name, url) return None try: - err, reg_spec, _ = parseRecipe(getRecipeReader(sh)) + err, reg_spec, _ = parseRecipe(getRecipeReader(sh, repo_dir=providers_checkout)) except Exception as exc: # pylint: disable=broad-except _warn("Could not parse %s.sh from the registry: %s", name, exc) return None diff --git a/bits_helpers/upgrade_cmd.py b/bits_helpers/upgrade_cmd.py index e695978c..f0068e4a 100644 --- a/bits_helpers/upgrade_cmd.py +++ b/bits_helpers/upgrade_cmd.py @@ -33,6 +33,7 @@ import yaml from bits_helpers.log import banner, info, warning +from bits_helpers.paths import RECIPE_SUBDIR, checkForFilename TIMEOUT_S = 20 @@ -631,9 +632,15 @@ def _status_args(args): sys.argv = saved -def _cached(path): - """Is *path* in a repository provider's checkout (bits' cache, not to edit)?""" - return os.path.exists(os.path.join(os.path.dirname(os.path.realpath(path)), ".bits_provider_ok")) +def _cached_repo(path): + """Return the root of a cached provider checkout containing *path*, if any.""" + directory = os.path.dirname(os.path.realpath(path)) + parent = os.path.dirname(directory) + if os.path.exists(os.path.join(directory, ".bits_provider_ok")): + return directory + if os.path.basename(directory) == RECIPE_SUBDIR and os.path.exists(os.path.join(parent, ".bits_provider_ok")): + return parent + return None def _patches_of(text): @@ -652,8 +659,11 @@ def _bring_in(path, config_dir, package, dry): """A file of a provider's cached checkout: its copy in *config_dir* (which is searched first), with the patches and the checksum file it uses. Returns (the copy, [what was copied]); NeedsHand when a copy would replace a file.""" - src_dir = os.path.dirname(path) - dest = os.path.join(config_dir, os.path.basename(path)) + src_dir = _cached_repo(path) or os.path.dirname(path) + dest = checkForFilename({}, os.path.basename(path)[:-3], config_dir) + if not os.path.exists(dest): + recipes = os.path.join(config_dir, RECIPE_SUBDIR) + dest = os.path.join(recipes if os.path.isdir(recipes) else config_dir, os.path.basename(path)) if os.path.exists(dest): raise NeedsHand("%s exists: it would be replaced by %s's" % (dest, path)) with open(path) as f: @@ -677,7 +687,6 @@ def _bring_in(path, config_dir, package, dry): def _recipe_file(taps, spec): """The recipe file *spec* was read from (in its own directory), or None.""" - from bits_helpers.paths import checkForFilename path = checkForFilename(taps, spec["package"].lower(), spec.get("pkgdir") or "") return path if os.path.isfile(path) and path.endswith(".sh") else None @@ -686,7 +695,8 @@ def _origin(path, provider_dirs): """The repository a file was read from: its provider's name, else its git root's.""" d = os.path.dirname(os.path.abspath(path)) for pd, (name, commit) in (provider_dirs or {}).items(): - if os.path.abspath(pd) == d: + root = os.path.abspath(pd) + if d in (root, os.path.join(root, RECIPE_SUBDIR)): return "%s@%s" % (name, str(commit)[:10]) r = subprocess.run(["git", "-C", d, "rev-parse", "--show-toplevel"], capture_output=True, text=True) return os.path.basename(r.stdout.strip()) if r.returncode == 0 else d @@ -750,7 +760,7 @@ def resolve(spec, text): try: if pin.startswith("defaults:"): path = resolveDefaultsFilename(pin.split(":", 1)[1], sargs.configDir, failOnError=False) - if path and _cached(path): + if path and _cached_repo(path): raise NeedsHand("its profile is in a provider's cached checkout (%s): check that" " repository out and build with it" % _origin(path, r["provider_dirs"])) def edit(t): @@ -775,7 +785,7 @@ def edit(t): where = _origin(path, r["provider_dirs"]) if path in copies: row["detail"] = "in the copy made above" - elif _cached(path): # a provider's cached checkout: work on a copy + elif _cached_repo(path): # a provider's cached checkout: work on a copy target, copied = _bring_in(path, sargs.configDir, os.path.basename(path)[:-3], dry) copies[path], texts[target] = target, texts.pop(path) row["detail"] = "copied %s from %s" % (", ".join(copied), where) diff --git a/docs/COOKBOOK.md b/docs/COOKBOOK.md index 6d7c3ceb..9f188596 100644 --- a/docs/COOKBOOK.md +++ b/docs/COOKBOOK.md @@ -280,6 +280,8 @@ The **aliBuild** front-end is the legacy path instead: `aliBuild init` checks ou ### Use a private recipe repository alongside the defaults +Package recipes can live at the root or in `recipes/`. Keep `defaults-*.sh`, `cvmfs.yaml`, `patches/` and `checksums/` at the repository root, and pass that root (not `recipes/`) to `-c`. If you copy a recipe into `recipes/`, Bits still uses the root copy until you remove it. + Set `BITS_PATH` to add a repository to the recipe search path. A plain name such as `myorg` means `/myorg.bits`; an absolute path is used as is. These directories are searched *after* the main recipe directory: ```bash diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 74d621e1..3353544c 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -696,7 +696,7 @@ repository_position: append # commit: 1a2b3c4d ``` -The `source` URL must point to a git repository whose top-level directory contains `*.sh` recipe files (the same layout as any other `*.bits` directory). `tag` selects the branch, tag or commit to check out (default: `version`). A provider repository is loaded once per build; a `requires` entry that asks for a different version of an already-loaded provider is ignored, with a warning. +The `source` URL must point to a git repository with `*.sh` package recipes at the root, in `recipes/`, or both. `tag` selects the branch, tag or commit to check out (default: `version`). A provider repository is loaded once per build; a `requires` entry that asks for a different version of an already-loaded provider is ignored, with a warning. ### Always-on providers (`always_load: true`) @@ -870,7 +870,7 @@ During the walk, an `overrides:` entry for a provider in the active defaults (`s #### Using a local checkout of a provider -To build with your own copy of a provider's recipes, for example while editing `lcg.bits`, check it out in the config directory under the provider's name. Before cloning a declared provider `` (in either phase, except the registry itself), bits looks for `//`; if that directory contains `*.sh` recipes it is used instead of the remote, with its git `HEAD` recorded as provenance (plus `-dirty` if it has uncommitted changes). So a provider you are editing (e.g. `lcg.bits/` next to `lcg.bits.sh`) is picked up as-is, just as a local package checkout shadows its `source`. Only declared providers are shadowed; bits never picks up undeclared `*.bits/` directories this way. `--force-tracked` disables this (and all local-checkout pickup), forcing the remote clone. For an *undeclared* local sub-repo, put it on `BITS_PATH` with `--search-path NAMES`. +To build with your own copy of a provider's recipes, for example while editing `lcg.bits`, check it out in the config directory under the provider's name. Before cloning a declared provider `` (in either phase, except the registry itself), bits looks for `//`; if that checkout contains package recipes (under `recipes/` or in a legacy flat layout) or defaults profiles, it is used instead of the remote, with its git `HEAD` recorded as provenance (plus `-dirty` if it has uncommitted changes). So a provider you are editing (e.g. `lcg.bits/` next to `lcg.bits.sh`) is picked up as-is, just as a local package checkout shadows its `source`. Only declared providers are shadowed; bits never picks up undeclared `*.bits/` directories this way. `--force-tracked` disables this (and all local-checkout pickup), forcing the remote clone. For an *undeclared* local sub-repo, put it on `BITS_PATH` with `--search-path NAMES`. ### Triggering providers from a defaults file @@ -1086,11 +1086,13 @@ Which packages, for which platform and defaults profiles, with which flavours an Where the work directory and the recipes are, and which local checkouts are built: +Point `--config-dir` and `BITS_PATH` entries at repository roots. Bits looks for packages at each checkout root first, then in `recipes/` before trying the next checkout. A root recipe or generator still takes precedence over any copy in `recipes/`. When moving a recipe, the root copy stays active until removed. Keep shell helpers outside the recipe search locations. Defaults profiles, `patches/`, `checksums/`, `cvmfs.yaml`, and repository-relative file sources and hooks stay at the root. + | Option | Description | |--------|-------------| | `-C DIR`, `--chdir DIR` | Change to DIR first (or set `$BITS_CHDIR`). | | `-w DIR`, `--work-dir DIR` | Work/output directory. Default: `sw`. | -| `-c DIR`, `--config-dir DIR` | Directory containing recipe files. Default: `.`. | +| `-c DIR`, `--config-dir DIR` | Recipe checkout root. Default: `.`. | | `--search-path NAMES` | Comma-separated recipe sub-repos to search besides the config dir. A relative NAME resolves to `/NAME.bits`; absolute paths are used as-is. Seeds `BITS_PATH` (an explicit `$BITS_PATH` wins). | | `--reference-sources DIR` | Directory for the local mirrors of git repositories. Default: `/MIRROR`. | | `--provider-policy POLICY` | Control where each repository provider is inserted into `BITS_PATH`. Format: comma-separated `name:prepend\|append` pairs; every provider defaults to `append`, and this flag is the only way to grant `prepend`. See [§13 Provider policy](#provider-policy). | diff --git a/docs/USERGUIDE.md b/docs/USERGUIDE.md index 62eaef91..ca3de911 100644 --- a/docs/USERGUIDE.md +++ b/docs/USERGUIDE.md @@ -448,7 +448,7 @@ Bits resolves the full transitive dependency graph of each requested package, co ### How a build proceeds -1. **Recipe discovery** — Bits locates `.sh` in the recipe directory, then in each `BITS_PATH` entry (`--search-path`; a relative NAME means `/NAME.bits`, an absolute path is used as is). Repository-provider packages (see [§13](REFERENCE.md#13-repository-provider-feature)) are cloned first to extend the search path before the main resolution pass. +1. **Recipe discovery** — Bits searches the main checkout, then each `BITS_PATH` entry (`--search-path`; a relative NAME means `/NAME.bits`, an absolute path is used as is). In each checkout it looks at the root first, then in `recipes/` if the recipe is not at the root. Defaults profiles stay at the root. Repository-provider packages (see [§13](REFERENCE.md#13-repository-provider-feature)) are cloned first to extend the search path before the main resolution pass. 2. **Dependency resolution** — `requires`, `build_requires`, and `runtime_requires` fields are read recursively, forming a DAG. Cycles are reported as errors. 3. **Hash computation** — A hash is computed for each package from its recipe text, source commit, dependency hashes, and environment. Packages with a matching hash in a store are downloaded instead of rebuilt. 4. **Source fetching** — Source repositories are cloned into a local mirror and then checked out into a build area. Up to 8 repositories are fetched in parallel. diff --git a/tests/test_packagelist.py b/tests/test_packagelist.py index f00b18b6..da7c3f38 100644 --- a/tests/test_packagelist.py +++ b/tests/test_packagelist.py @@ -167,7 +167,7 @@ } class MockReader: - def __init__(self, url, dist=None, genPackages=None): + def __init__(self, url, dist=None, genPackages=None, repo_dir=None): self._contents = RECIPES[url] self.url = "mock://" + url diff --git a/tests/test_recipe_layout.py b/tests/test_recipe_layout.py new file mode 100644 index 00000000..594f5087 --- /dev/null +++ b/tests/test_recipe_layout.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +import os +import subprocess +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from bits_helpers.checksums_cmd import plan, recipes_in +from bits_helpers.compliance import scan_recipes +from bits_helpers.paths import resolveDefaultsFilename, resolveFilename, resolveLocalPath +from bits_helpers.recipe import GitReader, getGeneratedPackages, getRecipeReader, parseRecipe +from bits_helpers.repo_provider import ( + _local_provider_dir, load_always_on_providers, resolve_registry_repo, +) +from bits_helpers.upgrade_cmd import _bring_in, _cached_repo + + +def recipe(name): + return f'package: {name}\nversion: "1"\n---\n' + + +class TestRecipeLayout(unittest.TestCase): + def setUp(self): + directory = TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.directory = Path(directory.name).resolve() + env = patch.dict(os.environ) + env.start() + self.addCleanup(env.stop) + os.environ.pop('BITS_PATH', None) + os.environ.pop('BITS_REPO_DIR', None) + + def test_layout_mixes_new_and_legacy_repositories(self): + repo = self.directory / 'group[dev].bits' + recipes = repo / 'recipes' + recipes.mkdir(parents=True) + provider = repo / 'base.bits' + provider.mkdir() + os.environ['BITS_PATH'] = 'base' + (repo / 'defaults-release.sh').write_text(recipe('defaults-release')) + (repo / 'foo.sh').write_text(recipe('Foo')) + (repo / 'root-only.sh').write_text(recipe('Root-Only')) + (recipes / 'foo.sh').write_text(recipe('Shadowed')) + (recipes / 'nested-only.sh').write_text(recipe('Nested-Only')) + (repo / 'nested-only.sh').mkdir() + (recipes / 'pinned.sh').write_text(recipe('Shadowed')) + (repo / 'pinned').mkdir() + (repo / 'pinned' / 'latest').write_text(recipe('Pinned')) + (provider / 'bar.sh').write_text(recipe('Bar')) + (provider / 'packages').mkdir() + (provider / 'packages' / 'latest').write_text(recipe('Packages')) + (repo / 'extra.tar').touch() + (recipes / 'latest').mkdir() + (recipes / 'latest' / 'latest').write_text(recipe('Latest')) + root = str(repo) + for name, path, checkout in ( + ('foo', repo / 'foo.sh', repo), + ('root-only', repo / 'root-only.sh', repo), + ('nested-only', recipes / 'nested-only.sh', repo), + ('pinned', repo / 'pinned' / 'latest', repo), + ('latest', recipes / 'latest' / 'latest', repo), + ('bar', provider / 'bar.sh', provider), + ('packages', provider / 'packages' / 'latest', provider), + ('defaults-release', repo / 'defaults-release.sh', repo), + ): + with self.subTest(name=name): + self.assertEqual(resolveFilename({}, name, root, {}), (str(path), str(checkout))) + self.assertEqual(resolveDefaultsFilename('release', root), str(repo / 'defaults-release.sh')) + self.assertEqual(resolveLocalPath(root, 'file://extra.tar'), 'file:/' + str(repo / 'extra.tar')) + self.assertEqual(recipes_in(root), { + 'foo': str(repo / 'foo.sh'), 'root-only': str(repo / 'root-only.sh'), + 'nested-only': str(recipes / 'nested-only.sh'), 'pinned': str(repo / 'pinned' / 'latest'), + }) + self.assertEqual(set(scan_recipes(root)['by_package']), + {'foo', 'root-only', 'nested-only', 'pinned', 'defaults-release'}) + (repo / 'foo.sh').unlink() + self.assertEqual(resolveFilename({}, 'foo', root, {}), (str(recipes / 'foo.sh'), root)) + + def test_provider_discovery_uses_recipes_but_keeps_checkout_root(self): + recipes = self.directory / 'recipes' + recipes.mkdir() + provider = self.directory / 'base.bits' + (provider / 'recipes').mkdir(parents=True) + (provider / 'recipes' / 'foo.sh').write_text(recipe('Foo')) + text = 'package: base.bits\nversion: "1"\nsource: example\nprovides_repository: true\nalways_load: true\n---\n' + (recipes / 'base.bits.sh').write_text(text) + root = str(self.directory) + self.assertEqual(_local_provider_dir(root, 'base.bits'), str(provider)) + self.assertEqual(load_always_on_providers(root, root, '', True), {str(provider): ('base.bits', 'local')}) + self.assertEqual(resolveFilename({}, 'foo', root, {}), (str(provider / 'recipes' / 'foo.sh'), str(provider))) + args = type('Args', (), {'bits_providers': 'https://example.invalid/repo'})() + with patch('bits_helpers.repo_provider.clone_or_update_provider', return_value=(root, 'hash')): + self.assertEqual(resolve_registry_repo(args, 'base.bits', root)['source'], 'example') + + def test_git_reader_uses_layout_of_requested_reference(self): + root = str(self.directory) + + def git(*args): + return subprocess.check_output(['git', '-C', root, *args], text=True).strip() + + git('init', '-q') + git('config', 'user.email', 'test@example.invalid') + git('config', 'user.name', 'Recipe Layout Test') + git('config', 'commit.gpgsign', 'false') + (self.directory / 'foo.sh').write_text(recipe('Foo')) + git('add', 'foo.sh') + git('commit', '-qm', 'Flat layout') + old = git('rev-parse', 'HEAD') + recipes = self.directory / 'recipes' + recipes.mkdir() + (recipes / 'README.md').write_text('Notes for the flat checkout.') + git('add', 'recipes/README.md') + git('commit', '-qm', 'Add notes under recipes in the flat layout') + assets = git('rev-parse', 'HEAD') + self.assertEqual(recipes_in(root), {'foo': str(self.directory / 'foo.sh')}) + (recipes / 'latest').write_text(recipe('Recipes')) + git('add', 'recipes/latest') + git('commit', '-qm', 'Add a versioned legacy Recipes recipe') + legacy = git('rev-parse', 'HEAD') + self.assertEqual(resolveFilename({}, 'foo', root, {}), (str(self.directory / 'foo.sh'), root)) + self.assertEqual(resolveFilename({}, 'recipes', root, {}), (str(recipes / 'latest'), root)) + (recipes / 'foo.sh').write_text(recipe('FromNested')) + git('add', 'recipes/foo.sh') + git('commit', '-qm', 'Mixed layout') + mixed = git('rev-parse', 'HEAD') + git('rm', 'foo.sh') + git('commit', '-qm', 'Remove the shadowing root copy') + for ref, name in ((old, 'Foo'), (assets, 'Foo'), (legacy, 'Foo'), (mixed, 'Foo'), ('HEAD', 'FromNested')): + with self.subTest(ref=ref): + self.assertEqual(GitReader(f'dist:Foo@{ref}', root)().strip(), recipe(name).strip()) + + def test_generated_packages_and_includes_use_repo_root(self): + primary = self.directory / 'primary' + primary.mkdir() + (primary / 'root.inc').write_text('echo wrong repository') + os.environ['BITS_REPO_DIR'] = str(primary) + recipes = self.directory / 'recipes' + generators = recipes / 'generated' + generators.mkdir(parents=True) + (generators / 'packages.py').write_text( + 'def getPackages(virtual_packages, config_dir):\n' + ' for name in ("generated", "root-file", "root-generated"):\n' + ' virtual_packages[name] = {"pkgdir": config_dir, "version": "1"}\n') + root_generators = self.directory / 'generated' + root_generators.mkdir() + (root_generators / 'packages.py').write_text( + 'def getPackages(virtual_packages, config_dir):\n' + ' virtual_packages["root-generated"] = {"pkgdir": config_dir, "version": "2"}\n') + root_file = self.directory / 'root-file.sh' + root_file.write_text(recipe('Root-File')) + root = str(self.directory) + generated = getGeneratedPackages(root) + self.assertEqual(resolveFilename({}, 'generated', root, generated), ('generate:generated@1', root)) + self.assertEqual(resolveFilename({}, 'root-generated', root, generated), ('generate:root-generated@2', root)) + self.assertEqual(resolveFilename({}, 'root-file', root, generated), (str(root_file), root)) + path = recipes / 'foo.sh' + path.write_text(recipe('Foo') + '#!include \n#!include "local.inc"\n') + (self.directory / 'root.inc').write_text('echo root') + (recipes / 'local.inc').write_text('echo local') + err, _, body = parseRecipe(getRecipeReader(str(path), repo_dir=root)) + self.assertIsNone(err) + self.assertEqual(body.strip().splitlines(), ['echo root', 'echo local']) + + def test_checksums_resolves_patches_at_root_and_rejects_missing_checkout(self): + (self.directory / 'recipes').mkdir() + (self.directory / 'recipes' / 'foo.sh').write_text( + 'package: Foo\nversion: "1"\npatches: [foo.patch]\n---\n') + root = str(self.directory) + checks = plan(root, [], None, [], 'x86_64-el9') + self.assertEqual(checks['Foo']['patches']['foo.patch'], (None, str(self.directory / 'patches'))) + with self.assertRaises(FileNotFoundError): + plan(root, [str(self.directory / 'missing.bits')], 'all', [], 'x86_64-el9') + scandir = os.scandir + + def deny_recipes(path): + if Path(path) == self.directory / 'recipes': + raise PermissionError(path) + return scandir(path) + + with patch('bits_helpers.paths.os.scandir', side_effect=deny_recipes): + with self.assertRaises(PermissionError): + recipes_in(root) + + def test_upgrade_copies_recipe_under_recipes_and_patches_at_root(self): + provider = self.directory / 'provider' + (provider / 'recipes').mkdir(parents=True) + (provider / 'patches').mkdir() + (provider / '.bits_provider_ok').touch() + (provider / 'patches' / 'foo.patch').write_text('patch') + path = provider / 'recipes' / 'foo.sh' + path.write_text('package: Foo\nversion: "1"\npatches: [foo.patch]\n---\n') + config = self.directory / 'group.bits' + (config / 'recipes').mkdir(parents=True) + (config / 'recipes' / 'existing.sh').write_text(recipe('Existing')) + self.assertEqual(_cached_repo(str(path)), str(provider)) + dest, copied = _bring_in(str(path), str(config), 'Foo', dry=False) + self.assertEqual(dest, str(config / 'recipes' / 'foo.sh')) + self.assertEqual(copied, ['recipes/foo.sh', 'patches/foo.patch']) + self.assertEqual((config / 'patches' / 'foo.patch').read_text(), 'patch') diff --git a/tests/test_repo_provider.py b/tests/test_repo_provider.py index 1dc6568c..b01ecfb8 100644 --- a/tests/test_repo_provider.py +++ b/tests/test_repo_provider.py @@ -750,7 +750,7 @@ def clone(spec, *a, **kw): class MockReaderPkgList: - def __init__(self, url, dist=None, genPackages=None): + def __init__(self, url, dist=None, genPackages=None, repo_dir=None): self._contents = _PKGLIST_RECIPES[url] self.url = "mock://" + url @@ -913,20 +913,24 @@ def _make_clone_side_effect(self, providers_checkout, config_checkout): ]) return lambda *a, **kw: next(calls) + def _registry(self, filename): + checkout = tempfile.TemporaryDirectory() + self.addCleanup(checkout.cleanup) + with open(os.path.join(checkout.name, filename), "w"): + pass + return checkout.name + @patch("bits_helpers.repo_provider.info") # avoid LogFormatter double-format bug @patch("bits_helpers.repo_provider.clone_or_update_provider") @patch("bits_helpers.repo_provider.parseRecipe") @patch("bits_helpers.repo_provider.getRecipeReader") - @patch("bits_helpers.repo_provider.exists") # module-level name, not os.path.exists def test_returns_config_checkout_when_default_sh_found( - self, mock_exists, mock_reader, mock_parse, mock_clone, _info, + self, mock_reader, mock_parse, mock_clone, _info, ): """Happy path: bits-providers has default.bits.sh → returns config checkout.""" - providers_checkout = "/work/REPOS/bits-providers/abc1234567" + providers_checkout = self._registry("default.bits.sh") config_checkout = "/work/REPOS/default.bits/def4567890" - # exists() returns True only for the default.bits.sh path - mock_exists.side_effect = lambda p: p.endswith("default.bits.sh") mock_reader.return_value = "reader" mock_parse.return_value = ( None, @@ -981,13 +985,11 @@ def test_returns_none_when_bits_providers_clone_fails(self, mock_clone, _warn): @patch("bits_helpers.repo_provider.clone_or_update_provider") @patch("bits_helpers.repo_provider.parseRecipe") @patch("bits_helpers.repo_provider.getRecipeReader") - @patch("bits_helpers.repo_provider.exists") # module-level name, not os.path.exists def test_returns_none_when_config_clone_fails( - self, mock_exists, mock_reader, mock_parse, mock_clone, _info, _warn, + self, mock_reader, mock_parse, mock_clone, _info, _warn, ): """A clone failure for the default config repo is caught and returns None.""" - providers_checkout = "/work/REPOS/bits-providers/abc1234567" - mock_exists.side_effect = lambda p: p.endswith("default.bits.sh") + providers_checkout = self._registry("default.bits.sh") mock_reader.return_value = "reader" mock_parse.return_value = ( None, @@ -1014,17 +1016,13 @@ def test_returns_none_when_config_clone_fails( @patch("bits_helpers.repo_provider.clone_or_update_provider") @patch("bits_helpers.repo_provider.parseRecipe") @patch("bits_helpers.repo_provider.getRecipeReader") - @patch("bits_helpers.repo_provider.exists") def test_organisation_recipe_preferred_over_default( - self, mock_exists, mock_reader, mock_parse, mock_clone, _info, + self, mock_reader, mock_parse, mock_clone, _info, ): """Organisation stored uppercase (LHCB) → resolves to lhcb.bits.sh (lowercase).""" - providers_checkout = "/work/REPOS/bits-providers/abc1234567" + providers_checkout = self._registry("lhcb.bits.sh") config_checkout = "/work/REPOS/lhcb.bits/def4567890" - # Only lhcb.bits.sh (lowercase) exists — default.bits.sh is absent. - # args.organisation is uppercase "LHCB" as stored in bits.rc. - mock_exists.side_effect = lambda p: p.endswith("lhcb.bits.sh") mock_reader.return_value = "reader" mock_parse.return_value = ( None, @@ -1056,16 +1054,13 @@ def test_organisation_recipe_preferred_over_default( @patch("bits_helpers.repo_provider.clone_or_update_provider") @patch("bits_helpers.repo_provider.parseRecipe") @patch("bits_helpers.repo_provider.getRecipeReader") - @patch("bits_helpers.repo_provider.exists") def test_organisation_falls_back_to_default_sh_when_org_sh_absent( - self, mock_exists, mock_reader, mock_parse, mock_clone, _info, + self, mock_reader, mock_parse, mock_clone, _info, ): """When .bits.sh is absent, bootstrap falls back to default.bits.sh.""" - providers_checkout = "/work/REPOS/bits-providers/abc1234567" + providers_checkout = self._registry("default.bits.sh") config_checkout = "/work/REPOS/default.bits/def4567890" - # LHCB.bits.sh / lhcb.bits.sh absent, default.bits.sh present - mock_exists.side_effect = lambda p: p.endswith("default.bits.sh") mock_reader.return_value = "reader" mock_parse.return_value = ( None,