From dd701dc990c4ad910c2109e0c1496200baeb471a Mon Sep 17 00:00:00 2001 From: Juan Miguel Carceller Date: Wed, 18 Nov 2026 21:01:48 +0100 Subject: [PATCH 1/4] Find package recipes under packages/ Keep repository assets at the checkout root and retain flat-layout support. --- CHANGELOG.md | 1 + bits_helpers/brew.py | 14 ++-- bits_helpers/checksums_cmd.py | 21 +++-- bits_helpers/compliance.py | 10 +-- bits_helpers/doctor.py | 3 +- bits_helpers/packages.py | 3 +- bits_helpers/paths.py | 35 ++++++++- bits_helpers/preload_cmd.py | 5 +- bits_helpers/recipe.py | 35 +++++---- bits_helpers/repo_provider.py | 23 +++--- bits_helpers/upgrade_cmd.py | 27 ++++--- docs/COOKBOOK.md | 2 + docs/REFERENCE.md | 8 +- docs/USERGUIDE.md | 2 +- tests/test_packagelist.py | 2 +- tests/test_recipe_layout.py | 144 ++++++++++++++++++++++++++++++++++ tests/test_repo_provider.py | 2 +- 17 files changed, 265 insertions(+), 72 deletions(-) create mode 100644 tests/test_recipe_layout.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b54eac0a..c641f840 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ --- # Unreleased +- **[Improvement]** Recipe repositories can put package recipes under `packages/` and keep defaults, patches and other repository files at the root. Existing flat repositories still work. - **[Change]** bits detects the platform name the CI and the builder images use: `x86_64-el9` on RHEL, AlmaLinux and Rocky 9, `x86_64-ubuntu2404` on Ubuntu 24.04 (macOS keeps `osx_arm64`). The `aliBuild` wrapper keeps ALICE's names (`slc9_x86-64`) by setting `BITS_LEGACY_ARCH=1`; native bits for ALICE records it with `bits use --architecture slc9_x86-64`. ALICE's store is read only for ALICE's names exactly (USERGUIDE "Platforms"). - **[Fix]** `bits init stacks.bits` (any `.bits`) no longer creates `sw` in the current directory: the work dir belongs inside the checkout, where you then build. - **[Improvement]** `bits --version` of an installed bits whose `_version.py` holds `git describe` (as bits.bits writes it) shows the tag, the commits since and the commit, e.g. `bits 0.7-3-g1a2b3c4 (tag 0.7 +3, commit 1a2b3c4)`. diff --git a/bits_helpers/brew.py b/bits_helpers/brew.py index 3ef1b944..726735d5 100644 --- a/bits_helpers/brew.py +++ b/bits_helpers/brew.py @@ -51,6 +51,7 @@ import glob from bits_helpers.log import debug, error, info +from bits_helpers.paths import recipe_files from bits_helpers.recipe import parseRecipe, FileReader @@ -120,12 +121,12 @@ def collect_homebrew(configDirs, architecture): configDirs = [configDirs] seen = set() for configDir in configDirs: - for path in sorted(glob.glob(os.path.join(configDir, "*.sh"))): + for path in recipe_files(configDir, include_defaults=True): real = os.path.realpath(path) if real in seen: continue seen.add(real) - err, spec, _ = parseRecipe(FileReader(path)) + err, spec, _ = parseRecipe(FileReader(path, repo_dir=configDir)) if err or not spec: # Not a parseable recipe (e.g. a defaults-*.sh helper). Skip quietly. continue @@ -143,13 +144,8 @@ def _standalone_scan_dirs(configDir, work_dir): """ dirs = [configDir] repos = os.path.join(os.path.abspath(work_dir), "REPOS") - if os.path.isdir(repos): - # Any directory under REPOS that contains recipes. Duplicates across - # provider versions are harmless — formulae are collected into a set. - found = set() - for path in glob.glob(os.path.join(repos, "**", "*.sh"), recursive=True): - found.add(os.path.dirname(path)) - dirs.extend(sorted(found)) + dirs.extend(sorted(path for path in glob.glob(os.path.join(repos, "*", "*")) + if os.path.isdir(path) and not os.path.islink(path))) return dirs diff --git a/bits_helpers/checksums_cmd.py b/bits_helpers/checksums_cmd.py index bf3e4e01..5e13397a 100644 --- a/bits_helpers/checksums_cmd.py +++ b/bits_helpers/checksums_cmd.py @@ -45,6 +45,7 @@ from bits_helpers.defaults import asDict, merge_dicts from bits_helpers.log import error, info, warning from bits_helpers.matchers import _parse_patch_entry, resolve_variables +from bits_helpers.paths import recipe_files from bits_helpers.recipe import getRecipeReader, parseRecipe from bits_helpers.utilities import apply_version_from, resolve_spec_data from bits_helpers.workarea import _resolve_source_entry, _split_arch_prefix @@ -57,9 +58,9 @@ # ── Recipes and profiles ────────────────────────────────────────────────────── -def _read(path): +def _read(path, repo_dir=None): """Parsed YAML header of recipe *path*, or ``None`` (warned) if unparsable.""" - err, meta, _ = parseRecipe(getRecipeReader(path)) + err, meta, _ = parseRecipe(getRecipeReader(path, repo_dir=repo_dir)) if err or not isinstance(meta, dict): warning("Skipping %s: %s", path, err or "not a recipe") return None @@ -68,12 +69,7 @@ def _read(path): def recipes_in(repo): """``{lowercase package name: recipe path}`` for the recipes of *repo*.""" - out = {} - for f in sorted(os.listdir(repo)): - path = os.path.join(repo, f) - if f.endswith(".sh") and not f.startswith("defaults-") and os.path.isfile(path): - out[f[:-3].lower()] = path - return out + return {os.path.basename(path)[:-3].lower(): path for path in recipe_files(repo, strict=True)} def profiles_in(repo, want): @@ -261,7 +257,7 @@ def plan(repo, recipe_dirs, profile_names, packages, arch): search = [repo] + [d for d in recipe_dirs if os.path.abspath(d) != repo] found = {} for d in reversed(search): # earlier repositories win, as in a build - found.update(recipes_in(d)) + found.update((name, (path, d)) for name, path in recipes_in(d).items()) own = recipes_in(repo) want = {p.lower() for p in packages} profiles = profiles_in(repo, profile_names) if profile_names else [] @@ -298,14 +294,15 @@ def add(pkg, patches_dir, got, origin): for name in sorted(set(own) | set(found if profiles else ())): if want and name not in want: continue - meta = _read(found[name]) + path, recipe_repo = found[name] + meta = _read(path, repo_dir=recipe_repo) if meta is None: continue pkg = str(meta.get("package", name)) if pkg.lower() != name: # the build refuses such a recipe too - warning("Skipping %s: its package field is %r", found[name], pkg) + warning("Skipping %s: its package field is %r", path, pkg) continue - patches_dir = os.path.join(os.path.dirname(found[name]), "patches") + patches_dir = os.path.join(recipe_repo, "patches") try: base = wants(meta, ["release"], rel_vars, arch, source_mode(release)) except ValueError as exc: diff --git a/bits_helpers/compliance.py b/bits_helpers/compliance.py index 495b190e..e4429b78 100644 --- a/bits_helpers/compliance.py +++ b/bits_helpers/compliance.py @@ -27,6 +27,7 @@ import urllib.request from bits_helpers.log import banner, debug, error, info, warning +from bits_helpers.paths import recipe_files from bits_helpers import site # Front-matter keys read from each recipe. Values may be quoted (the header @@ -71,10 +72,9 @@ def scan_recipes(recipes_dir): out = {"total": 0, "missing_license": [], "licenseref": [], "noassertion": [], "restricted": [], "restricted_sources": [], "by_package": {}} - for name in sorted(os.listdir(recipes_dir)): - if not name.endswith(".sh"): - continue - meta = _recipe_meta(os.path.join(recipes_dir, name)) + for path in recipe_files(recipes_dir, include_defaults=True, strict=True): + name = os.path.relpath(path, recipes_dir) + meta = _recipe_meta(path) pkg = meta.get("package") if not pkg: continue # not a recipe (helper script) @@ -631,7 +631,7 @@ def doCompliance(args, parser): "distributed)", len(system_pkgs)) else: recipes_dir = os.path.abspath(getattr(args, "recipesDir", None) or ".") - if not any(f.endswith(".sh") for f in os.listdir(recipes_dir)): + if not recipe_files(recipes_dir, include_defaults=True): parser.error("no recipes (*.sh) found in %s — point --recipes at a " "recipe repository (e.g. a lcg.bits checkout), or give " "PACKAGE roots for a group-wide audit" % recipes_dir) diff --git a/bits_helpers/doctor.py b/bits_helpers/doctor.py index 3c7ae062..d0283e03 100644 --- a/bits_helpers/doctor.py +++ b/bits_helpers/doctor.py @@ -54,6 +54,7 @@ from bits_helpers.defaults import (parseDefaults, readDefaults, validateDefaults, incompatibleFlavorDefaults) from bits_helpers.packages import getPackageList +from bits_helpers.paths import checkForFilename from bits_helpers.arch import effective_arch # ── Status constants ─────────────────────────────────────────────────────────── @@ -839,7 +840,7 @@ def doDoctor(args, parser): packages = [] for p in args.packages: - recipe_path = "%s/%s.sh" % (args.configDir, p.lower()) + recipe_path = checkForFilename({}, p.lower(), args.configDir) if not exists(recipe_path): error("Cannot find recipe %s for package %s.", recipe_path, p) exitcode = 1 diff --git a/bits_helpers/packages.py b/bits_helpers/packages.py index 9882db16..1cc929ea 100644 --- a/bits_helpers/packages.py +++ b/bits_helpers/packages.py @@ -256,7 +256,8 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, "is the intended one.", p, recipeSourceLabel(pkgdir, provider_dirs), recipeSourceLabel(_later, provider_dirs), ", ".join(_defaults_dirs[_later])) - err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir, generatedPackages[pkgdir]), generatedPackages) + reader = getRecipeReader(filename, configDir, generatedPackages[pkgdir], repo_dir=pkgdir) + err, spec, recipe = parseRecipe(reader, generatedPackages) dieOnError(err, err) # Unless there was an error, both spec and recipe should be valid. # otherwise the error should have been caught above. diff --git a/bits_helpers/paths.py b/bits_helpers/paths.py index 3b4785c5..8e17f757 100644 --- a/bits_helpers/paths.py +++ b/bits_helpers/paths.py @@ -7,11 +7,44 @@ layers above it can share these without an import cycle.""" import os -from os.path import exists, join +from os.path import exists, isdir, join, lexists from bits_helpers.log import dieOnError, error +# A legacy checkout can already use packages/ for a recipe or generator. +LEGACY_PACKAGES_ENTRIES = ("latest", "packages.py") + + +def package_recipe_dir(repo): + """Return the package directory, or the checkout root for a flat repository.""" + packages = join(repo, "packages") + if isdir(packages) and not any(lexists(join(packages, f)) for f in LEGACY_PACKAGES_ENTRIES): + return packages + return repo + + +def recipe_files(repo, include_defaults=False, strict=False): + """List package recipe files; optionally include root-level defaults profiles.""" + def scan(directory, defaults): + try: + with os.scandir(directory) as entries: + return [e.path for e in entries if e.name.endswith(".sh") + and e.name.startswith("defaults-") == defaults and e.is_file()] + except OSError: + if strict: + raise + return [] + + packages = package_recipe_dir(repo) + files = scan(packages, defaults=False) + if include_defaults: + files.extend(scan(repo, defaults=True)) + return sorted(files) + + def checkForFilename(taps, pkg, d, ext=".sh"): + if ext == ".sh" and not pkg.startswith("defaults-"): + d = package_recipe_dir(d) filename = taps.get(pkg, "{}/{}{}".format(d, pkg, ext)) if not exists(filename): if "/" in pkg: diff --git a/bits_helpers/preload_cmd.py b/bits_helpers/preload_cmd.py index 32553f16..dbd4d38b 100644 --- a/bits_helpers/preload_cmd.py +++ b/bits_helpers/preload_cmd.py @@ -399,12 +399,13 @@ def main(argv=None): ap.error("--docker requires --docker-image IMAGE") def recipe_reader(pkg): + from bits_helpers.paths import checkForFilename from bits_helpers.recipe import parseRecipe, FileReader - path = os.path.join(a.config_dir, pkg + ".sh") + path = checkForFilename({}, pkg, a.config_dir) if not os.path.isfile(path): return None try: - _err, spec, _body = parseRecipe(FileReader(path)) + _err, spec, _body = parseRecipe(FileReader(path, repo_dir=a.config_dir)) return spec except Exception: return None diff --git a/bits_helpers/recipe.py b/bits_helpers/recipe.py index 607f6736..754c1815 100644 --- a/bits_helpers/recipe.py +++ b/bits_helpers/recipe.py @@ -13,7 +13,7 @@ import re import sys from collections import OrderedDict -from glob import glob +from glob import escape, glob from os.path import join from typing import Any, IO @@ -22,7 +22,7 @@ from bits_helpers.cmd import getoutput from bits_helpers.git import git from bits_helpers.log import dieOnError -from bits_helpers.paths import getConfigPaths, resolveFilename +from bits_helpers.paths import LEGACY_PACKAGES_ENTRIES, getConfigPaths, package_recipe_dir, resolveFilename class SpecError(Exception): pass @@ -43,7 +43,7 @@ def validateSpec(spec): spec["version"] = "0" -def getRecipeReader(url: str, dist=None, genPackages={}): +def getRecipeReader(url: str, dist=None, genPackages={}, repo_dir=None): m = re.search(r'^(dist|generate):(.*)@([^@]+)$', url) if m and m.group(1) == "generate": pkg, version = m.group(2), m.group(3) @@ -54,7 +54,7 @@ def getRecipeReader(url: str, dist=None, genPackages={}): elif m and dist: return GitReader(url, dist) else: - return FileReader(url) + return FileReader(url, repo_dir) # Generate a recipe of package class GeneratedPackage: @@ -66,8 +66,9 @@ def __call__(self): # Read a recipe from a file class FileReader: - def __init__(self, url) -> None: + def __init__(self, url, repo_dir=None) -> None: self.url = url + self.repo_dir = repo_dir if repo_dir and package_recipe_dir(repo_dir) != repo_dir else None def __call__(self): with open(self.url) as f: return f.read() @@ -79,8 +80,14 @@ def __init__(self, url, configDir) -> None: def __call__(self): m = re.search(r'^dist:(.*)@([^@]+)$', self.url) fn, gh = m.groups() - err, d = git(("show", f"{gh}:{fn.lower()}.sh"), - directory=self.configDir) + fn = fn.lower() + if not fn.startswith("defaults-"): + err, entries = git(("ls-tree", "--full-tree", "-z", "--name-only", gh, "--", "packages/"), + directory=self.configDir, check=False) + paths = set(entries.split("\0")) - {""} + if not err and paths and not paths.intersection("packages/" + f for f in LEGACY_PACKAGES_ENTRIES): + fn = "packages/" + fn + err, d = git(("show", f"{gh}:{fn}.sh"), directory=self.configDir, check=False) if err: raise RuntimeError("Cannot read recipe {fn} from reference {gh}.\n" "Make sure you run first (this will not alter your recipes):\n" @@ -238,7 +245,8 @@ def parseRecipe(reader, generatePackages=None, visited=None): header, recipe = d[:_m.start()], d[_m.end():] # Splice any `#!include` directives in the body before anything else sees it, # so the included text is variable-expanded and hashed as if written inline. - recipe = resolveIncludes(recipe, getattr(reader, "url", "") or "") + repo_dir = getattr(reader, "repo_dir", None) + recipe = resolveIncludes(recipe, getattr(reader, "url", "") or "", repo_dir) # YAML forbids '%' as the first character of a plain (unquoted) scalar because # it is reserved for directives (e.g. %YAML, %TAG). Recipe authors may want # to write "- %(name)s-%(version)s.patch" in patches: (and similar lists) @@ -271,15 +279,15 @@ def parseRecipe(reader, generatePackages=None, visited=None): if spec and "from" in spec: basename = os.path.basename(getattr(reader, "url", "") or "") filename = basename[:-3] if basename.endswith(".sh") else basename - repoDir = os.environ.get("BITS_REPO_DIR") + repoDir = repo_dir or os.environ.get("BITS_REPO_DIR") if visited is None: visited = [] if spec["from"] in visited: raise RuntimeError(f" Cyclic Dependency: {' -> '.join(list(visited) + [spec['from']])}") visited.append(spec["from"]) parent_dir = os.path.join(repoDir, spec["from"]) - base_filename, pkgdir = resolveFilename({}, filename, parent_dir, generatePackages) - base_reader = getRecipeReader(base_filename, repoDir, generatePackages[parent_dir]) + base_filename, pkgdir = resolveFilename({}, filename, parent_dir, generatePackages or {}) + base_reader = getRecipeReader(base_filename, repoDir, (generatePackages or {}).get(pkgdir, {}), pkgdir) err, base_spec, base_recipe = parseRecipe(base_reader, generatePackages, visited) spec, recipe_append = handleMergePolicy(spec, base_spec) recipe = recipe + base_recipe if recipe_append else recipe @@ -304,9 +312,8 @@ def getGeneratedPackages(configDir): pkgDirs = getConfigPaths(configDir) for pkgdir in pkgDirs: dir_pkgs = {} - for vp in [x.split(os.sep)[-2] for x in glob(join(pkgdir, "*", "packages.py"))]: - packages_py = join(pkgdir, vp, "packages.py") - sys.path.insert(0, join(pkgdir, vp)) + for packages_py in glob(join(escape(package_recipe_dir(pkgdir)), "*", "packages.py")): + sys.path.insert(0, os.path.dirname(packages_py)) try: pkg = __import__("packages") except (ImportError, SyntaxError) as e: diff --git a/bits_helpers/repo_provider.py b/bits_helpers/repo_provider.py index 84a6d6e5..653eea1e 100644 --- a/bits_helpers/repo_provider.py +++ b/bits_helpers/repo_provider.py @@ -50,7 +50,6 @@ """ import fcntl -import glob import os import re import shutil @@ -64,7 +63,7 @@ from bits_helpers.workarea import updateReferenceRepoSpec, logged_scm from bits_helpers.utilities import resolve_spec_data, symlink from bits_helpers.recipe import getGeneratedPackages, getRecipeReader, parseRecipe -from bits_helpers.paths import checkForFilename, getConfigPaths +from bits_helpers.paths import checkForFilename, getConfigPaths, package_recipe_dir, recipe_files from bits_helpers.matchers import _parse_req_matcher # Maximum provider-discovery iterations (guards against run-away recursion) @@ -118,7 +117,7 @@ def _check_for_shadows( incoming_recipes = { os.path.splitext(os.path.basename(f))[0].lower() - for f in glob.glob(os.path.join(incoming_dir, "*.sh")) + for f in recipe_files(incoming_dir, include_defaults=True) } if not incoming_recipes: return # empty provider directory — nothing to shadow @@ -131,7 +130,7 @@ def _check_for_shadows( for existing_dir in existing_dirs: existing_recipes = { os.path.splitext(os.path.basename(f))[0].lower() - for f in glob.glob(os.path.join(existing_dir, "*.sh")) + for f in recipe_files(existing_dir, include_defaults=True) } shadowed = incoming_recipes & existing_recipes if shadowed: @@ -268,7 +267,7 @@ def _try_read_spec(pkg_lower: str, config_dir: str, taps: dict): continue gen_pkgs = {} - err, spec, _ = parseRecipe(getRecipeReader(filename, config_dir, gen_pkgs)) + err, spec, _ = parseRecipe(getRecipeReader(filename, config_dir, gen_pkgs, repo_dir=pkg_dir)) if err or spec is None: continue return spec @@ -555,9 +554,9 @@ def load_always_on_providers( ) # ── 2. ``always_load: true`` recipes in the primary config dir ────────── - for sh_path in sorted(glob.glob(os.path.join(abspath(config_dir), "*.sh"))): + for sh_path in recipe_files(abspath(config_dir), include_defaults=True): try: - err, spec, _ = parseRecipe(getRecipeReader(sh_path)) + err, spec, _ = parseRecipe(getRecipeReader(sh_path, repo_dir=config_dir)) except Exception: continue if err or spec is None: @@ -667,7 +666,7 @@ def _local_provider_dir(config_dir, package): unchanged (the local dir simply takes the clone's place on ``BITS_PATH``). """ d = join(abspath(config_dir), package) - if os.path.isdir(d) and glob.glob(join(d, "*.sh")): + if recipe_files(d, include_defaults=True): return d return None @@ -747,7 +746,7 @@ def bootstrap_default_config(args, work_dir: str) -> Optional[str]: chosen_sh = None chosen_label = None for filename, label in candidates: - path = join(providers_checkout, filename) + path = join(package_recipe_dir(providers_checkout), filename) if exists(path): chosen_sh = path chosen_label = label @@ -765,7 +764,7 @@ def bootstrap_default_config(args, work_dir: str) -> Optional[str]: return None try: - err, default_spec, _ = parseRecipe(getRecipeReader(chosen_sh)) + err, default_spec, _ = parseRecipe(getRecipeReader(chosen_sh, repo_dir=providers_checkout)) except Exception as exc: warning("Bootstrap: could not parse %s.bits.sh: %s", chosen_label, exc) return None @@ -842,12 +841,12 @@ def _warn(*a): _warn("Could not fetch the provider registry from %s.", url) return None - sh = join(providers_checkout, name + ".sh") + sh = join(package_recipe_dir(providers_checkout), name + ".sh") if not exists(sh): _warn("No '%s.sh' in the provider registry %s.", name, url) return None try: - err, reg_spec, _ = parseRecipe(getRecipeReader(sh)) + err, reg_spec, _ = parseRecipe(getRecipeReader(sh, repo_dir=providers_checkout)) except Exception as exc: # pylint: disable=broad-except _warn("Could not parse %s.sh from the registry: %s", name, exc) return None diff --git a/bits_helpers/upgrade_cmd.py b/bits_helpers/upgrade_cmd.py index e695978c..192a75a7 100644 --- a/bits_helpers/upgrade_cmd.py +++ b/bits_helpers/upgrade_cmd.py @@ -33,6 +33,7 @@ import yaml from bits_helpers.log import banner, info, warning +from bits_helpers.paths import checkForFilename, package_recipe_dir TIMEOUT_S = 20 @@ -631,9 +632,15 @@ def _status_args(args): sys.argv = saved -def _cached(path): - """Is *path* in a repository provider's checkout (bits' cache, not to edit)?""" - return os.path.exists(os.path.join(os.path.dirname(os.path.realpath(path)), ".bits_provider_ok")) +def _cached_repo(path): + """Return the root of a cached provider checkout containing *path*, if any.""" + directory = os.path.dirname(os.path.realpath(path)) + parent = os.path.dirname(directory) + if os.path.exists(os.path.join(directory, ".bits_provider_ok")): + return directory + if directory == package_recipe_dir(parent) and os.path.exists(os.path.join(parent, ".bits_provider_ok")): + return parent + return None def _patches_of(text): @@ -652,8 +659,10 @@ def _bring_in(path, config_dir, package, dry): """A file of a provider's cached checkout: its copy in *config_dir* (which is searched first), with the patches and the checksum file it uses. Returns (the copy, [what was copied]); NeedsHand when a copy would replace a file.""" - src_dir = os.path.dirname(path) - dest = os.path.join(config_dir, os.path.basename(path)) + src_dir = _cached_repo(path) or os.path.dirname(path) + dest = checkForFilename({}, os.path.basename(path)[:-3], config_dir) + if not os.path.exists(dest): + dest = os.path.join(package_recipe_dir(config_dir), os.path.basename(path)) if os.path.exists(dest): raise NeedsHand("%s exists: it would be replaced by %s's" % (dest, path)) with open(path) as f: @@ -677,7 +686,6 @@ def _bring_in(path, config_dir, package, dry): def _recipe_file(taps, spec): """The recipe file *spec* was read from (in its own directory), or None.""" - from bits_helpers.paths import checkForFilename path = checkForFilename(taps, spec["package"].lower(), spec.get("pkgdir") or "") return path if os.path.isfile(path) and path.endswith(".sh") else None @@ -686,7 +694,8 @@ def _origin(path, provider_dirs): """The repository a file was read from: its provider's name, else its git root's.""" d = os.path.dirname(os.path.abspath(path)) for pd, (name, commit) in (provider_dirs or {}).items(): - if os.path.abspath(pd) == d: + root = os.path.abspath(pd) + if d in (root, package_recipe_dir(root)): return "%s@%s" % (name, str(commit)[:10]) r = subprocess.run(["git", "-C", d, "rev-parse", "--show-toplevel"], capture_output=True, text=True) return os.path.basename(r.stdout.strip()) if r.returncode == 0 else d @@ -750,7 +759,7 @@ def resolve(spec, text): try: if pin.startswith("defaults:"): path = resolveDefaultsFilename(pin.split(":", 1)[1], sargs.configDir, failOnError=False) - if path and _cached(path): + if path and _cached_repo(path): raise NeedsHand("its profile is in a provider's cached checkout (%s): check that" " repository out and build with it" % _origin(path, r["provider_dirs"])) def edit(t): @@ -775,7 +784,7 @@ def edit(t): where = _origin(path, r["provider_dirs"]) if path in copies: row["detail"] = "in the copy made above" - elif _cached(path): # a provider's cached checkout: work on a copy + elif _cached_repo(path): # a provider's cached checkout: work on a copy target, copied = _bring_in(path, sargs.configDir, os.path.basename(path)[:-3], dry) copies[path], texts[target] = target, texts.pop(path) row["detail"] = "copied %s from %s" % (", ".join(copied), where) diff --git a/docs/COOKBOOK.md b/docs/COOKBOOK.md index 6d7c3ceb..b348c8e0 100644 --- a/docs/COOKBOOK.md +++ b/docs/COOKBOOK.md @@ -280,6 +280,8 @@ The **aliBuild** front-end is the legacy path instead: `aliBuild init` checks ou ### Use a private recipe repository alongside the defaults +Put package recipes in `packages/`. Keep `defaults-*.sh`, `cvmfs.yaml`, `patches/` and `checksums/` at the repository root, and pass that root (not `packages/`) to `-c`. Existing flat repositories still work; once you add `packages/`, move all package recipes there, not just the new ones. + Set `BITS_PATH` to add a repository to the recipe search path. A plain name such as `myorg` means `/myorg.bits`; an absolute path is used as is. These directories are searched *after* the main recipe directory: ```bash diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 74d621e1..06a138d4 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -696,7 +696,7 @@ repository_position: append # commit: 1a2b3c4d ``` -The `source` URL must point to a git repository whose top-level directory contains `*.sh` recipe files (the same layout as any other `*.bits` directory). `tag` selects the branch, tag or commit to check out (default: `version`). A provider repository is loaded once per build; a `requires` entry that asks for a different version of an already-loaded provider is ignored, with a warning. +The `source` URL must point to a git repository with `*.sh` package recipes in `packages/`, or at the root for a legacy flat checkout. `tag` selects the branch, tag or commit to check out (default: `version`). A provider repository is loaded once per build; a `requires` entry that asks for a different version of an already-loaded provider is ignored, with a warning. ### Always-on providers (`always_load: true`) @@ -870,7 +870,7 @@ During the walk, an `overrides:` entry for a provider in the active defaults (`s #### Using a local checkout of a provider -To build with your own copy of a provider's recipes, for example while editing `lcg.bits`, check it out in the config directory under the provider's name. Before cloning a declared provider `` (in either phase, except the registry itself), bits looks for `//`; if that directory contains `*.sh` recipes it is used instead of the remote, with its git `HEAD` recorded as provenance (plus `-dirty` if it has uncommitted changes). So a provider you are editing (e.g. `lcg.bits/` next to `lcg.bits.sh`) is picked up as-is, just as a local package checkout shadows its `source`. Only declared providers are shadowed; bits never picks up undeclared `*.bits/` directories this way. `--force-tracked` disables this (and all local-checkout pickup), forcing the remote clone. For an *undeclared* local sub-repo, put it on `BITS_PATH` with `--search-path NAMES`. +To build with your own copy of a provider's recipes, for example while editing `lcg.bits`, check it out in the config directory under the provider's name. Before cloning a declared provider `` (in either phase, except the registry itself), bits looks for `//`; if that checkout contains package recipes (under `packages/` or in a legacy flat layout) or defaults profiles, it is used instead of the remote, with its git `HEAD` recorded as provenance (plus `-dirty` if it has uncommitted changes). So a provider you are editing (e.g. `lcg.bits/` next to `lcg.bits.sh`) is picked up as-is, just as a local package checkout shadows its `source`. Only declared providers are shadowed; bits never picks up undeclared `*.bits/` directories this way. `--force-tracked` disables this (and all local-checkout pickup), forcing the remote clone. For an *undeclared* local sub-repo, put it on `BITS_PATH` with `--search-path NAMES`. ### Triggering providers from a defaults file @@ -1086,11 +1086,13 @@ Which packages, for which platform and defaults profiles, with which flavours an Where the work directory and the recipes are, and which local checkouts are built: +Point `--config-dir` and `BITS_PATH` entries at repository roots. When a checkout has a `packages/` directory, only that directory supplies package recipes (including versioned recipes and package generators). Without it, Bits looks for package recipes at the root as before. A `packages/` directory with a `latest` entry or a `packages.py` directly inside it is treated as a legacy versioned recipe or generator directory. Defaults profiles, `patches/`, `checksums/`, `cvmfs.yaml`, and repository-relative file sources and hooks stay at the root. + | Option | Description | |--------|-------------| | `-C DIR`, `--chdir DIR` | Change to DIR first (or set `$BITS_CHDIR`). | | `-w DIR`, `--work-dir DIR` | Work/output directory. Default: `sw`. | -| `-c DIR`, `--config-dir DIR` | Directory containing recipe files. Default: `.`. | +| `-c DIR`, `--config-dir DIR` | Recipe checkout root. Default: `.`. | | `--search-path NAMES` | Comma-separated recipe sub-repos to search besides the config dir. A relative NAME resolves to `/NAME.bits`; absolute paths are used as-is. Seeds `BITS_PATH` (an explicit `$BITS_PATH` wins). | | `--reference-sources DIR` | Directory for the local mirrors of git repositories. Default: `/MIRROR`. | | `--provider-policy POLICY` | Control where each repository provider is inserted into `BITS_PATH`. Format: comma-separated `name:prepend\|append` pairs; every provider defaults to `append`, and this flag is the only way to grant `prepend`. See [§13 Provider policy](#provider-policy). | diff --git a/docs/USERGUIDE.md b/docs/USERGUIDE.md index 62eaef91..87050b25 100644 --- a/docs/USERGUIDE.md +++ b/docs/USERGUIDE.md @@ -448,7 +448,7 @@ Bits resolves the full transitive dependency graph of each requested package, co ### How a build proceeds -1. **Recipe discovery** — Bits locates `.sh` in the recipe directory, then in each `BITS_PATH` entry (`--search-path`; a relative NAME means `/NAME.bits`, an absolute path is used as is). Repository-provider packages (see [§13](REFERENCE.md#13-repository-provider-feature)) are cloned first to extend the search path before the main resolution pass. +1. **Recipe discovery** — Bits searches the main checkout, then each `BITS_PATH` entry (`--search-path`; a relative NAME means `/NAME.bits`, an absolute path is used as is). In each checkout it looks for `.sh` under `packages/`, or at the root if there is no `packages/`. Defaults profiles stay at the root. Repository-provider packages (see [§13](REFERENCE.md#13-repository-provider-feature)) are cloned first to extend the search path before the main resolution pass. 2. **Dependency resolution** — `requires`, `build_requires`, and `runtime_requires` fields are read recursively, forming a DAG. Cycles are reported as errors. 3. **Hash computation** — A hash is computed for each package from its recipe text, source commit, dependency hashes, and environment. Packages with a matching hash in a store are downloaded instead of rebuilt. 4. **Source fetching** — Source repositories are cloned into a local mirror and then checked out into a build area. Up to 8 repositories are fetched in parallel. diff --git a/tests/test_packagelist.py b/tests/test_packagelist.py index f00b18b6..da7c3f38 100644 --- a/tests/test_packagelist.py +++ b/tests/test_packagelist.py @@ -167,7 +167,7 @@ } class MockReader: - def __init__(self, url, dist=None, genPackages=None): + def __init__(self, url, dist=None, genPackages=None, repo_dir=None): self._contents = RECIPES[url] self.url = "mock://" + url diff --git a/tests/test_recipe_layout.py b/tests/test_recipe_layout.py new file mode 100644 index 00000000..4693f01d --- /dev/null +++ b/tests/test_recipe_layout.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +import subprocess +from unittest.mock import patch + +import pytest + +from bits_helpers.checksums_cmd import plan, recipes_in +from bits_helpers.compliance import scan_recipes +from bits_helpers.paths import resolveDefaultsFilename, resolveFilename, resolveLocalPath +from bits_helpers.recipe import GitReader, getGeneratedPackages, getRecipeReader, parseRecipe +from bits_helpers.repo_provider import ( + _local_provider_dir, load_always_on_providers, resolve_registry_repo, +) +from bits_helpers.upgrade_cmd import _bring_in, _cached_repo + + +def recipe(name): + return f'package: {name}\nversion: "1"\n---\n' + + +def test_layout_mixes_new_and_legacy_repositories(tmp_path, monkeypatch): + repo = tmp_path / 'group[dev].bits' + packages = repo / 'packages' + packages.mkdir(parents=True) + provider = repo / 'base.bits' + provider.mkdir() + monkeypatch.setenv('BITS_PATH', 'base') + (repo / 'defaults-release.sh').write_text(recipe('defaults-release')) + (repo / 'foo.sh').write_text(recipe('NotARecipeInThisLayout')) + (packages / 'foo.sh').write_text(recipe('Foo')) + (provider / 'bar.sh').write_text(recipe('Bar')) + (repo / 'extra.tar').touch() + (packages / 'versioned').mkdir() + (packages / 'versioned' / 'latest').write_text(recipe('Versioned')) + root = str(repo) + assert resolveFilename({}, 'foo', root, {}) == (str(packages / 'foo.sh'), root) + assert resolveFilename({}, 'versioned', root, {}) == (str(packages / 'versioned' / 'latest'), root) + assert resolveFilename({}, 'bar', root, {}) == (str(provider / 'bar.sh'), str(provider)) + assert resolveFilename({}, 'defaults-release', root, {}) == (str(repo / 'defaults-release.sh'), root) + assert resolveDefaultsFilename('release', root) == str(repo / 'defaults-release.sh') + assert resolveLocalPath(root, 'file://extra.tar') == 'file:/' + str(repo / 'extra.tar') + assert recipes_in(root) == {'foo': str(packages / 'foo.sh')} + assert set(scan_recipes(root)['by_package']) == {'foo', 'defaults-release'} + + +def test_provider_discovery_uses_packages_but_keeps_checkout_root(tmp_path, monkeypatch): + packages = tmp_path / 'packages' + packages.mkdir() + provider = tmp_path / 'base.bits' + (provider / 'packages').mkdir(parents=True) + (provider / 'packages' / 'foo.sh').write_text(recipe('Foo')) + text = 'package: base.bits\nversion: "1"\nsource: example\nprovides_repository: true\nalways_load: true\n---\n' + (packages / 'base.bits.sh').write_text(text) + monkeypatch.delenv('BITS_PATH', raising=False) + assert _local_provider_dir(tmp_path, 'base.bits') == str(provider) + assert load_always_on_providers(str(tmp_path), str(tmp_path), '', True) == { + str(provider): ('base.bits', 'local')} + assert resolveFilename({}, 'foo', str(tmp_path), {}) == ( + str(provider / 'packages' / 'foo.sh'), str(provider)) + args = type('Args', (), {'bits_providers': 'https://example.invalid/repo'})() + with patch('bits_helpers.repo_provider.clone_or_update_provider', return_value=(str(tmp_path), 'hash')): + assert resolve_registry_repo(args, 'base.bits', str(tmp_path))['source'] == 'example' + + +def test_git_reader_uses_layout_of_requested_reference(tmp_path): + root = str(tmp_path) + + def git(*args): + return subprocess.check_output(['git', '-C', root, *args], text=True).strip() + + git('init', '-q') + git('config', 'user.email', 'test@example.invalid') + git('config', 'user.name', 'Recipe Layout Test') + (tmp_path / 'foo.sh').write_text(recipe('Foo')) + git('add', 'foo.sh') + git('commit', '-qm', 'Flat layout') + old = git('rev-parse', 'HEAD') + (tmp_path / 'packages').mkdir() + (tmp_path / 'packages' / 'latest').write_text(recipe('Packages')) + git('add', 'packages/latest') + git('commit', '-qm', 'Add a versioned legacy Packages recipe') + legacy = git('rev-parse', 'HEAD') + assert resolveFilename({}, 'foo', root, {}) == (str(tmp_path / 'foo.sh'), root) + assert resolveFilename({}, 'packages', root, {}) == (str(tmp_path / 'packages' / 'latest'), root) + git('rm', 'packages/latest') + (tmp_path / 'packages').mkdir(exist_ok=True) + git('mv', 'foo.sh', 'packages/foo.sh') + git('commit', '-qm', 'New layout') + assert GitReader(f'dist:Foo@{old}', root)().strip() == recipe('Foo').strip() + assert GitReader(f'dist:Foo@{legacy}', root)().strip() == recipe('Foo').strip() + assert GitReader('dist:Foo@HEAD', root)().strip() == recipe('Foo').strip() + + +def test_generated_packages_and_includes_use_repo_root(tmp_path, monkeypatch): + monkeypatch.delenv('BITS_PATH', raising=False) + primary = tmp_path / 'primary' + primary.mkdir() + (primary / 'root.inc').write_text('echo wrong repository') + monkeypatch.setenv('BITS_REPO_DIR', str(primary)) + generators = tmp_path / 'packages' / 'generated' + generators.mkdir(parents=True) + (generators / 'packages.py').write_text( + 'def getPackages(virtual_packages, config_dir):\n' + ' virtual_packages["generated"] = {"pkgdir": config_dir, "version": "1"}\n') + root = str(tmp_path) + assert resolveFilename({}, 'generated', root, getGeneratedPackages(root)) == ( + 'generate:generated@1', root) + path = tmp_path / 'packages' / 'foo.sh' + path.write_text(recipe('Foo') + '#!include \n#!include "local.inc"\n') + (tmp_path / 'root.inc').write_text('echo root') + (path.parent / 'local.inc').write_text('echo local') + err, _, body = parseRecipe(getRecipeReader(str(path), repo_dir=root)) + assert err is None + assert body.strip().splitlines() == ['echo root', 'echo local'] + + +def test_checksums_resolves_patches_at_root_and_rejects_missing_checkout(tmp_path): + (tmp_path / 'packages').mkdir() + (tmp_path / 'packages' / 'foo.sh').write_text( + 'package: Foo\nversion: "1"\npatches: [foo.patch]\n---\n') + root = str(tmp_path) + checks = plan(root, [], None, [], 'x86_64-el9') + assert checks['Foo']['patches']['foo.patch'] == (None, str(tmp_path / 'patches')) + with pytest.raises(FileNotFoundError): + plan(root, [str(tmp_path / 'missing.bits')], 'all', [], 'x86_64-el9') + + +def test_upgrade_copies_recipe_under_packages_and_patches_at_root(tmp_path): + provider = tmp_path / 'provider' + (provider / 'packages').mkdir(parents=True) + (provider / 'patches').mkdir() + (provider / '.bits_provider_ok').touch() + (provider / 'patches' / 'foo.patch').write_text('patch') + path = provider / 'packages' / 'foo.sh' + path.write_text('package: Foo\nversion: "1"\npatches: [foo.patch]\n---\n') + config = tmp_path / 'group.bits' + (config / 'packages').mkdir(parents=True) + assert _cached_repo(str(path)) == str(provider) + dest, copied = _bring_in(str(path), str(config), 'Foo', dry=False) + assert dest == str(config / 'packages' / 'foo.sh') + assert copied == ['packages/foo.sh', 'patches/foo.patch'] + assert (config / 'patches' / 'foo.patch').read_text() == 'patch' diff --git a/tests/test_repo_provider.py b/tests/test_repo_provider.py index 1dc6568c..3afb7635 100644 --- a/tests/test_repo_provider.py +++ b/tests/test_repo_provider.py @@ -750,7 +750,7 @@ def clone(spec, *a, **kw): class MockReaderPkgList: - def __init__(self, url, dist=None, genPackages=None): + def __init__(self, url, dist=None, genPackages=None, repo_dir=None): self._contents = _PKGLIST_RECIPES[url] self.url = "mock://" + url From c10a3bff4b5b7e2ed4c5a1558d4836109d29afee Mon Sep 17 00:00:00 2001 From: Juan Miguel Carceller Date: Thu, 19 Nov 2026 16:03:05 +0100 Subject: [PATCH 2/4] Use recipes/ for the nested recipe layout Keep asset-only directories compatible with flat lookups, and run the layout tests under unittest in CI. --- CHANGELOG.md | 2 +- bits_helpers/paths.py | 31 +++-- bits_helpers/recipe.py | 19 ++- docs/COOKBOOK.md | 2 +- docs/REFERENCE.md | 6 +- docs/USERGUIDE.md | 2 +- tests/test_recipe_layout.py | 269 +++++++++++++++++++----------------- 7 files changed, 187 insertions(+), 144 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c641f840..496dfe12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ --- # Unreleased -- **[Improvement]** Recipe repositories can put package recipes under `packages/` and keep defaults, patches and other repository files at the root. Existing flat repositories still work. +- **[Improvement]** Recipe repositories can put package recipes under `recipes/` and keep defaults, patches and other repository files at the root. Existing flat repositories still work. - **[Change]** bits detects the platform name the CI and the builder images use: `x86_64-el9` on RHEL, AlmaLinux and Rocky 9, `x86_64-ubuntu2404` on Ubuntu 24.04 (macOS keeps `osx_arm64`). The `aliBuild` wrapper keeps ALICE's names (`slc9_x86-64`) by setting `BITS_LEGACY_ARCH=1`; native bits for ALICE records it with `bits use --architecture slc9_x86-64`. ALICE's store is read only for ALICE's names exactly (USERGUIDE "Platforms"). - **[Fix]** `bits init stacks.bits` (any `.bits`) no longer creates `sw` in the current directory: the work dir belongs inside the checkout, where you then build. - **[Improvement]** `bits --version` of an installed bits whose `_version.py` holds `git describe` (as bits.bits writes it) shows the tag, the commits since and the commit, e.g. `bits 0.7-3-g1a2b3c4 (tag 0.7 +3, commit 1a2b3c4)`. diff --git a/bits_helpers/paths.py b/bits_helpers/paths.py index 8e17f757..dc91ffe8 100644 --- a/bits_helpers/paths.py +++ b/bits_helpers/paths.py @@ -7,19 +7,33 @@ layers above it can share these without an import cycle.""" import os -from os.path import exists, isdir, join, lexists +from os.path import exists, isdir, isfile, join, lexists from bits_helpers.log import dieOnError, error -# A legacy checkout can already use packages/ for a recipe or generator. -LEGACY_PACKAGES_ENTRIES = ("latest", "packages.py") +RECIPE_SUBDIR = "recipes" +# A legacy checkout can already use recipes/ for a recipe or generator. +LEGACY_RECIPE_DIR_ENTRIES = ("latest", "packages.py") + + +def is_package_recipe(filename): + return filename.endswith(".sh") and not filename.startswith("defaults-") def package_recipe_dir(repo): - """Return the package directory, or the checkout root for a flat repository.""" - packages = join(repo, "packages") - if isdir(packages) and not any(lexists(join(packages, f)) for f in LEGACY_PACKAGES_ENTRIES): - return packages + """Return the recipe directory, or the checkout root for a flat repository.""" + recipes = join(repo, RECIPE_SUBDIR) + if not isdir(recipes) or any(lexists(join(recipes, f)) for f in LEGACY_RECIPE_DIR_ENTRIES): + return repo + try: + with os.scandir(recipes) as entries: + for entry in entries: + if is_package_recipe(entry.name) and entry.is_file(): + return recipes + if entry.is_dir() and any(isfile(join(entry.path, f)) for f in LEGACY_RECIPE_DIR_ENTRIES): + return recipes + except OSError: + pass return repo @@ -35,8 +49,7 @@ def scan(directory, defaults): raise return [] - packages = package_recipe_dir(repo) - files = scan(packages, defaults=False) + files = scan(package_recipe_dir(repo), defaults=False) if include_defaults: files.extend(scan(repo, defaults=True)) return sorted(files) diff --git a/bits_helpers/recipe.py b/bits_helpers/recipe.py index 754c1815..2c003320 100644 --- a/bits_helpers/recipe.py +++ b/bits_helpers/recipe.py @@ -22,7 +22,10 @@ from bits_helpers.cmd import getoutput from bits_helpers.git import git from bits_helpers.log import dieOnError -from bits_helpers.paths import LEGACY_PACKAGES_ENTRIES, getConfigPaths, package_recipe_dir, resolveFilename +from bits_helpers.paths import ( + LEGACY_RECIPE_DIR_ENTRIES, RECIPE_SUBDIR, getConfigPaths, is_package_recipe, + package_recipe_dir, resolveFilename, +) class SpecError(Exception): pass @@ -82,11 +85,17 @@ def __call__(self): fn, gh = m.groups() fn = fn.lower() if not fn.startswith("defaults-"): - err, entries = git(("ls-tree", "--full-tree", "-z", "--name-only", gh, "--", "packages/"), + prefix = RECIPE_SUBDIR + "/" + err, entries = git(("ls-tree", "-r", "--full-tree", "-z", "--name-only", gh, "--", prefix), directory=self.configDir, check=False) - paths = set(entries.split("\0")) - {""} - if not err and paths and not paths.intersection("packages/" + f for f in LEGACY_PACKAGES_ENTRIES): - fn = "packages/" + fn + paths = {p[len(prefix):] for p in entries.split("\0") if p.startswith(prefix)} + if not err and not paths.intersection(LEGACY_RECIPE_DIR_ENTRIES): + for path in paths: + parts = path.split("/") + if ((len(parts) == 1 and is_package_recipe(parts[0])) or + (len(parts) == 2 and parts[1] in LEGACY_RECIPE_DIR_ENTRIES)): + fn = prefix + fn + break err, d = git(("show", f"{gh}:{fn}.sh"), directory=self.configDir, check=False) if err: raise RuntimeError("Cannot read recipe {fn} from reference {gh}.\n" diff --git a/docs/COOKBOOK.md b/docs/COOKBOOK.md index b348c8e0..8a83f4a9 100644 --- a/docs/COOKBOOK.md +++ b/docs/COOKBOOK.md @@ -280,7 +280,7 @@ The **aliBuild** front-end is the legacy path instead: `aliBuild init` checks ou ### Use a private recipe repository alongside the defaults -Put package recipes in `packages/`. Keep `defaults-*.sh`, `cvmfs.yaml`, `patches/` and `checksums/` at the repository root, and pass that root (not `packages/`) to `-c`. Existing flat repositories still work; once you add `packages/`, move all package recipes there, not just the new ones. +Put package recipes in `recipes/`. Keep `defaults-*.sh`, `cvmfs.yaml`, `patches/` and `checksums/` at the repository root, and pass that root (not `recipes/`) to `-c`. Existing flat repositories still work; once you add `recipes/`, move all package recipes there, not just the new ones. Set `BITS_PATH` to add a repository to the recipe search path. A plain name such as `myorg` means `/myorg.bits`; an absolute path is used as is. These directories are searched *after* the main recipe directory: diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 06a138d4..13974377 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -696,7 +696,7 @@ repository_position: append # commit: 1a2b3c4d ``` -The `source` URL must point to a git repository with `*.sh` package recipes in `packages/`, or at the root for a legacy flat checkout. `tag` selects the branch, tag or commit to check out (default: `version`). A provider repository is loaded once per build; a `requires` entry that asks for a different version of an already-loaded provider is ignored, with a warning. +The `source` URL must point to a git repository with `*.sh` package recipes in `recipes/`, or at the root for a legacy flat checkout. `tag` selects the branch, tag or commit to check out (default: `version`). A provider repository is loaded once per build; a `requires` entry that asks for a different version of an already-loaded provider is ignored, with a warning. ### Always-on providers (`always_load: true`) @@ -870,7 +870,7 @@ During the walk, an `overrides:` entry for a provider in the active defaults (`s #### Using a local checkout of a provider -To build with your own copy of a provider's recipes, for example while editing `lcg.bits`, check it out in the config directory under the provider's name. Before cloning a declared provider `` (in either phase, except the registry itself), bits looks for `//`; if that checkout contains package recipes (under `packages/` or in a legacy flat layout) or defaults profiles, it is used instead of the remote, with its git `HEAD` recorded as provenance (plus `-dirty` if it has uncommitted changes). So a provider you are editing (e.g. `lcg.bits/` next to `lcg.bits.sh`) is picked up as-is, just as a local package checkout shadows its `source`. Only declared providers are shadowed; bits never picks up undeclared `*.bits/` directories this way. `--force-tracked` disables this (and all local-checkout pickup), forcing the remote clone. For an *undeclared* local sub-repo, put it on `BITS_PATH` with `--search-path NAMES`. +To build with your own copy of a provider's recipes, for example while editing `lcg.bits`, check it out in the config directory under the provider's name. Before cloning a declared provider `` (in either phase, except the registry itself), bits looks for `//`; if that checkout contains package recipes (under `recipes/` or in a legacy flat layout) or defaults profiles, it is used instead of the remote, with its git `HEAD` recorded as provenance (plus `-dirty` if it has uncommitted changes). So a provider you are editing (e.g. `lcg.bits/` next to `lcg.bits.sh`) is picked up as-is, just as a local package checkout shadows its `source`. Only declared providers are shadowed; bits never picks up undeclared `*.bits/` directories this way. `--force-tracked` disables this (and all local-checkout pickup), forcing the remote clone. For an *undeclared* local sub-repo, put it on `BITS_PATH` with `--search-path NAMES`. ### Triggering providers from a defaults file @@ -1086,7 +1086,7 @@ Which packages, for which platform and defaults profiles, with which flavours an Where the work directory and the recipes are, and which local checkouts are built: -Point `--config-dir` and `BITS_PATH` entries at repository roots. When a checkout has a `packages/` directory, only that directory supplies package recipes (including versioned recipes and package generators). Without it, Bits looks for package recipes at the root as before. A `packages/` directory with a `latest` entry or a `packages.py` directly inside it is treated as a legacy versioned recipe or generator directory. Defaults profiles, `patches/`, `checksums/`, `cvmfs.yaml`, and repository-relative file sources and hooks stay at the root. +Point `--config-dir` and `BITS_PATH` entries at repository roots. When `recipes/` contains package recipes, versioned recipes, or package generators, Bits uses it instead of the root for package lookup. A `recipes/` directory with only support files, or with `latest` or `packages.py` directly inside it (a legacy versioned recipe or generator), does not switch the checkout to the new layout. Defaults profiles, `patches/`, `checksums/`, `cvmfs.yaml`, and repository-relative file sources and hooks stay at the root. | Option | Description | |--------|-------------| diff --git a/docs/USERGUIDE.md b/docs/USERGUIDE.md index 87050b25..437397d5 100644 --- a/docs/USERGUIDE.md +++ b/docs/USERGUIDE.md @@ -448,7 +448,7 @@ Bits resolves the full transitive dependency graph of each requested package, co ### How a build proceeds -1. **Recipe discovery** — Bits searches the main checkout, then each `BITS_PATH` entry (`--search-path`; a relative NAME means `/NAME.bits`, an absolute path is used as is). In each checkout it looks for `.sh` under `packages/`, or at the root if there is no `packages/`. Defaults profiles stay at the root. Repository-provider packages (see [§13](REFERENCE.md#13-repository-provider-feature)) are cloned first to extend the search path before the main resolution pass. +1. **Recipe discovery** — Bits searches the main checkout, then each `BITS_PATH` entry (`--search-path`; a relative NAME means `/NAME.bits`, an absolute path is used as is). In each checkout it looks for `.sh` under `recipes/` if that directory holds package recipes, or at the root for a flat checkout. Defaults profiles stay at the root. Repository-provider packages (see [§13](REFERENCE.md#13-repository-provider-feature)) are cloned first to extend the search path before the main resolution pass. 2. **Dependency resolution** — `requires`, `build_requires`, and `runtime_requires` fields are read recursively, forming a DAG. Cycles are reported as errors. 3. **Hash computation** — A hash is computed for each package from its recipe text, source commit, dependency hashes, and environment. Packages with a matching hash in a store are downloaded instead of rebuilt. 4. **Source fetching** — Source repositories are cloned into a local mirror and then checked out into a build area. Up to 8 repositories are fetched in parallel. diff --git a/tests/test_recipe_layout.py b/tests/test_recipe_layout.py index 4693f01d..4ece462e 100644 --- a/tests/test_recipe_layout.py +++ b/tests/test_recipe_layout.py @@ -1,11 +1,13 @@ # SPDX-FileCopyrightText: 2026 CERN # SPDX-License-Identifier: GPL-3.0-or-later +import os import subprocess +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory from unittest.mock import patch -import pytest - from bits_helpers.checksums_cmd import plan, recipes_in from bits_helpers.compliance import scan_recipes from bits_helpers.paths import resolveDefaultsFilename, resolveFilename, resolveLocalPath @@ -20,125 +22,144 @@ def recipe(name): return f'package: {name}\nversion: "1"\n---\n' -def test_layout_mixes_new_and_legacy_repositories(tmp_path, monkeypatch): - repo = tmp_path / 'group[dev].bits' - packages = repo / 'packages' - packages.mkdir(parents=True) - provider = repo / 'base.bits' - provider.mkdir() - monkeypatch.setenv('BITS_PATH', 'base') - (repo / 'defaults-release.sh').write_text(recipe('defaults-release')) - (repo / 'foo.sh').write_text(recipe('NotARecipeInThisLayout')) - (packages / 'foo.sh').write_text(recipe('Foo')) - (provider / 'bar.sh').write_text(recipe('Bar')) - (repo / 'extra.tar').touch() - (packages / 'versioned').mkdir() - (packages / 'versioned' / 'latest').write_text(recipe('Versioned')) - root = str(repo) - assert resolveFilename({}, 'foo', root, {}) == (str(packages / 'foo.sh'), root) - assert resolveFilename({}, 'versioned', root, {}) == (str(packages / 'versioned' / 'latest'), root) - assert resolveFilename({}, 'bar', root, {}) == (str(provider / 'bar.sh'), str(provider)) - assert resolveFilename({}, 'defaults-release', root, {}) == (str(repo / 'defaults-release.sh'), root) - assert resolveDefaultsFilename('release', root) == str(repo / 'defaults-release.sh') - assert resolveLocalPath(root, 'file://extra.tar') == 'file:/' + str(repo / 'extra.tar') - assert recipes_in(root) == {'foo': str(packages / 'foo.sh')} - assert set(scan_recipes(root)['by_package']) == {'foo', 'defaults-release'} - - -def test_provider_discovery_uses_packages_but_keeps_checkout_root(tmp_path, monkeypatch): - packages = tmp_path / 'packages' - packages.mkdir() - provider = tmp_path / 'base.bits' - (provider / 'packages').mkdir(parents=True) - (provider / 'packages' / 'foo.sh').write_text(recipe('Foo')) - text = 'package: base.bits\nversion: "1"\nsource: example\nprovides_repository: true\nalways_load: true\n---\n' - (packages / 'base.bits.sh').write_text(text) - monkeypatch.delenv('BITS_PATH', raising=False) - assert _local_provider_dir(tmp_path, 'base.bits') == str(provider) - assert load_always_on_providers(str(tmp_path), str(tmp_path), '', True) == { - str(provider): ('base.bits', 'local')} - assert resolveFilename({}, 'foo', str(tmp_path), {}) == ( - str(provider / 'packages' / 'foo.sh'), str(provider)) - args = type('Args', (), {'bits_providers': 'https://example.invalid/repo'})() - with patch('bits_helpers.repo_provider.clone_or_update_provider', return_value=(str(tmp_path), 'hash')): - assert resolve_registry_repo(args, 'base.bits', str(tmp_path))['source'] == 'example' - - -def test_git_reader_uses_layout_of_requested_reference(tmp_path): - root = str(tmp_path) - - def git(*args): - return subprocess.check_output(['git', '-C', root, *args], text=True).strip() - - git('init', '-q') - git('config', 'user.email', 'test@example.invalid') - git('config', 'user.name', 'Recipe Layout Test') - (tmp_path / 'foo.sh').write_text(recipe('Foo')) - git('add', 'foo.sh') - git('commit', '-qm', 'Flat layout') - old = git('rev-parse', 'HEAD') - (tmp_path / 'packages').mkdir() - (tmp_path / 'packages' / 'latest').write_text(recipe('Packages')) - git('add', 'packages/latest') - git('commit', '-qm', 'Add a versioned legacy Packages recipe') - legacy = git('rev-parse', 'HEAD') - assert resolveFilename({}, 'foo', root, {}) == (str(tmp_path / 'foo.sh'), root) - assert resolveFilename({}, 'packages', root, {}) == (str(tmp_path / 'packages' / 'latest'), root) - git('rm', 'packages/latest') - (tmp_path / 'packages').mkdir(exist_ok=True) - git('mv', 'foo.sh', 'packages/foo.sh') - git('commit', '-qm', 'New layout') - assert GitReader(f'dist:Foo@{old}', root)().strip() == recipe('Foo').strip() - assert GitReader(f'dist:Foo@{legacy}', root)().strip() == recipe('Foo').strip() - assert GitReader('dist:Foo@HEAD', root)().strip() == recipe('Foo').strip() - - -def test_generated_packages_and_includes_use_repo_root(tmp_path, monkeypatch): - monkeypatch.delenv('BITS_PATH', raising=False) - primary = tmp_path / 'primary' - primary.mkdir() - (primary / 'root.inc').write_text('echo wrong repository') - monkeypatch.setenv('BITS_REPO_DIR', str(primary)) - generators = tmp_path / 'packages' / 'generated' - generators.mkdir(parents=True) - (generators / 'packages.py').write_text( - 'def getPackages(virtual_packages, config_dir):\n' - ' virtual_packages["generated"] = {"pkgdir": config_dir, "version": "1"}\n') - root = str(tmp_path) - assert resolveFilename({}, 'generated', root, getGeneratedPackages(root)) == ( - 'generate:generated@1', root) - path = tmp_path / 'packages' / 'foo.sh' - path.write_text(recipe('Foo') + '#!include \n#!include "local.inc"\n') - (tmp_path / 'root.inc').write_text('echo root') - (path.parent / 'local.inc').write_text('echo local') - err, _, body = parseRecipe(getRecipeReader(str(path), repo_dir=root)) - assert err is None - assert body.strip().splitlines() == ['echo root', 'echo local'] - - -def test_checksums_resolves_patches_at_root_and_rejects_missing_checkout(tmp_path): - (tmp_path / 'packages').mkdir() - (tmp_path / 'packages' / 'foo.sh').write_text( - 'package: Foo\nversion: "1"\npatches: [foo.patch]\n---\n') - root = str(tmp_path) - checks = plan(root, [], None, [], 'x86_64-el9') - assert checks['Foo']['patches']['foo.patch'] == (None, str(tmp_path / 'patches')) - with pytest.raises(FileNotFoundError): - plan(root, [str(tmp_path / 'missing.bits')], 'all', [], 'x86_64-el9') - - -def test_upgrade_copies_recipe_under_packages_and_patches_at_root(tmp_path): - provider = tmp_path / 'provider' - (provider / 'packages').mkdir(parents=True) - (provider / 'patches').mkdir() - (provider / '.bits_provider_ok').touch() - (provider / 'patches' / 'foo.patch').write_text('patch') - path = provider / 'packages' / 'foo.sh' - path.write_text('package: Foo\nversion: "1"\npatches: [foo.patch]\n---\n') - config = tmp_path / 'group.bits' - (config / 'packages').mkdir(parents=True) - assert _cached_repo(str(path)) == str(provider) - dest, copied = _bring_in(str(path), str(config), 'Foo', dry=False) - assert dest == str(config / 'packages' / 'foo.sh') - assert copied == ['packages/foo.sh', 'patches/foo.patch'] - assert (config / 'patches' / 'foo.patch').read_text() == 'patch' +class TestRecipeLayout(unittest.TestCase): + def setUp(self): + directory = TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.directory = Path(directory.name).resolve() + env = patch.dict(os.environ) + env.start() + self.addCleanup(env.stop) + os.environ.pop('BITS_PATH', None) + os.environ.pop('BITS_REPO_DIR', None) + + def test_layout_mixes_new_and_legacy_repositories(self): + repo = self.directory / 'group[dev].bits' + recipes = repo / 'recipes' + recipes.mkdir(parents=True) + provider = repo / 'base.bits' + provider.mkdir() + os.environ['BITS_PATH'] = 'base' + (repo / 'defaults-release.sh').write_text(recipe('defaults-release')) + (repo / 'foo.sh').write_text(recipe('NotARecipeInThisLayout')) + (recipes / 'foo.sh').write_text(recipe('Foo')) + (provider / 'bar.sh').write_text(recipe('Bar')) + (provider / 'packages').mkdir() + (provider / 'packages' / 'latest').write_text(recipe('Packages')) + (repo / 'extra.tar').touch() + (recipes / 'versioned').mkdir() + (recipes / 'versioned' / 'latest').write_text(recipe('Versioned')) + root = str(repo) + for name, path, checkout in ( + ('foo', recipes / 'foo.sh', repo), + ('versioned', recipes / 'versioned' / 'latest', repo), + ('bar', provider / 'bar.sh', provider), + ('packages', provider / 'packages' / 'latest', provider), + ('defaults-release', repo / 'defaults-release.sh', repo), + ): + with self.subTest(name=name): + self.assertEqual(resolveFilename({}, name, root, {}), (str(path), str(checkout))) + self.assertEqual(resolveDefaultsFilename('release', root), str(repo / 'defaults-release.sh')) + self.assertEqual(resolveLocalPath(root, 'file://extra.tar'), 'file:/' + str(repo / 'extra.tar')) + self.assertEqual(recipes_in(root), {'foo': str(recipes / 'foo.sh')}) + self.assertEqual(set(scan_recipes(root)['by_package']), {'foo', 'defaults-release'}) + + def test_provider_discovery_uses_recipes_but_keeps_checkout_root(self): + recipes = self.directory / 'recipes' + recipes.mkdir() + provider = self.directory / 'base.bits' + (provider / 'recipes').mkdir(parents=True) + (provider / 'recipes' / 'foo.sh').write_text(recipe('Foo')) + text = 'package: base.bits\nversion: "1"\nsource: example\nprovides_repository: true\nalways_load: true\n---\n' + (recipes / 'base.bits.sh').write_text(text) + root = str(self.directory) + self.assertEqual(_local_provider_dir(root, 'base.bits'), str(provider)) + self.assertEqual(load_always_on_providers(root, root, '', True), {str(provider): ('base.bits', 'local')}) + self.assertEqual(resolveFilename({}, 'foo', root, {}), (str(provider / 'recipes' / 'foo.sh'), str(provider))) + args = type('Args', (), {'bits_providers': 'https://example.invalid/repo'})() + with patch('bits_helpers.repo_provider.clone_or_update_provider', return_value=(root, 'hash')): + self.assertEqual(resolve_registry_repo(args, 'base.bits', root)['source'], 'example') + + def test_git_reader_uses_layout_of_requested_reference(self): + root = str(self.directory) + + def git(*args): + return subprocess.check_output(['git', '-C', root, *args], text=True).strip() + + git('init', '-q') + git('config', 'user.email', 'test@example.invalid') + git('config', 'user.name', 'Recipe Layout Test') + git('config', 'commit.gpgsign', 'false') + (self.directory / 'foo.sh').write_text(recipe('Foo')) + git('add', 'foo.sh') + git('commit', '-qm', 'Flat layout') + old = git('rev-parse', 'HEAD') + recipes = self.directory / 'recipes' + recipes.mkdir() + (recipes / 'README.md').write_text('Notes for the flat checkout.') + git('add', 'recipes/README.md') + git('commit', '-qm', 'Add notes under recipes in the flat layout') + assets = git('rev-parse', 'HEAD') + self.assertEqual(recipes_in(root), {'foo': str(self.directory / 'foo.sh')}) + (recipes / 'latest').write_text(recipe('Recipes')) + git('add', 'recipes/latest') + git('commit', '-qm', 'Add a versioned legacy Recipes recipe') + legacy = git('rev-parse', 'HEAD') + self.assertEqual(resolveFilename({}, 'foo', root, {}), (str(self.directory / 'foo.sh'), root)) + self.assertEqual(resolveFilename({}, 'recipes', root, {}), (str(recipes / 'latest'), root)) + git('rm', 'recipes/latest') + recipes.mkdir(exist_ok=True) + git('mv', 'foo.sh', 'recipes/foo.sh') + git('commit', '-qm', 'New layout') + for ref in (old, assets, legacy, 'HEAD'): + with self.subTest(ref=ref): + self.assertEqual(GitReader(f'dist:Foo@{ref}', root)().strip(), recipe('Foo').strip()) + + def test_generated_packages_and_includes_use_repo_root(self): + primary = self.directory / 'primary' + primary.mkdir() + (primary / 'root.inc').write_text('echo wrong repository') + os.environ['BITS_REPO_DIR'] = str(primary) + recipes = self.directory / 'recipes' + generators = recipes / 'generated' + generators.mkdir(parents=True) + (generators / 'packages.py').write_text( + 'def getPackages(virtual_packages, config_dir):\n' + ' virtual_packages["generated"] = {"pkgdir": config_dir, "version": "1"}\n') + root = str(self.directory) + self.assertEqual(resolveFilename({}, 'generated', root, getGeneratedPackages(root)), + ('generate:generated@1', root)) + path = recipes / 'foo.sh' + path.write_text(recipe('Foo') + '#!include \n#!include "local.inc"\n') + (self.directory / 'root.inc').write_text('echo root') + (recipes / 'local.inc').write_text('echo local') + err, _, body = parseRecipe(getRecipeReader(str(path), repo_dir=root)) + self.assertIsNone(err) + self.assertEqual(body.strip().splitlines(), ['echo root', 'echo local']) + + def test_checksums_resolves_patches_at_root_and_rejects_missing_checkout(self): + (self.directory / 'recipes').mkdir() + (self.directory / 'recipes' / 'foo.sh').write_text( + 'package: Foo\nversion: "1"\npatches: [foo.patch]\n---\n') + root = str(self.directory) + checks = plan(root, [], None, [], 'x86_64-el9') + self.assertEqual(checks['Foo']['patches']['foo.patch'], (None, str(self.directory / 'patches'))) + with self.assertRaises(FileNotFoundError): + plan(root, [str(self.directory / 'missing.bits')], 'all', [], 'x86_64-el9') + + def test_upgrade_copies_recipe_under_recipes_and_patches_at_root(self): + provider = self.directory / 'provider' + (provider / 'recipes').mkdir(parents=True) + (provider / 'patches').mkdir() + (provider / '.bits_provider_ok').touch() + (provider / 'patches' / 'foo.patch').write_text('patch') + path = provider / 'recipes' / 'foo.sh' + path.write_text('package: Foo\nversion: "1"\npatches: [foo.patch]\n---\n') + config = self.directory / 'group.bits' + (config / 'recipes').mkdir(parents=True) + (config / 'recipes' / 'existing.sh').write_text(recipe('Existing')) + self.assertEqual(_cached_repo(str(path)), str(provider)) + dest, copied = _bring_in(str(path), str(config), 'Foo', dry=False) + self.assertEqual(dest, str(config / 'recipes' / 'foo.sh')) + self.assertEqual(copied, ['recipes/foo.sh', 'patches/foo.patch']) + self.assertEqual((config / 'patches' / 'foo.patch').read_text(), 'patch') From 68e32b137fb71ac6ea6d3e2876b26a9d919eff67 Mon Sep 17 00:00:00 2001 From: Juan Miguel Carceller Date: Thu, 19 Nov 2026 16:07:21 +0100 Subject: [PATCH 3/4] Keep recipe discovery errors visible in strict scans Do not mistake a nested versioned Latest recipe for a flat-layout marker. --- bits_helpers/paths.py | 17 +++++++++++------ docs/REFERENCE.md | 2 +- tests/test_recipe_layout.py | 16 +++++++++++++--- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/bits_helpers/paths.py b/bits_helpers/paths.py index dc91ffe8..b4509241 100644 --- a/bits_helpers/paths.py +++ b/bits_helpers/paths.py @@ -7,7 +7,7 @@ layers above it can share these without an import cycle.""" import os -from os.path import exists, isdir, isfile, join, lexists +from os.path import exists, isfile, islink, join from bits_helpers.log import dieOnError, error @@ -20,11 +20,13 @@ def is_package_recipe(filename): return filename.endswith(".sh") and not filename.startswith("defaults-") -def package_recipe_dir(repo): +def package_recipe_dir(repo, strict=False): """Return the recipe directory, or the checkout root for a flat repository.""" recipes = join(repo, RECIPE_SUBDIR) - if not isdir(recipes) or any(lexists(join(recipes, f)) for f in LEGACY_RECIPE_DIR_ENTRIES): - return repo + for name in LEGACY_RECIPE_DIR_ENTRIES: + path = join(recipes, name) + if isfile(path) or islink(path): + return repo try: with os.scandir(recipes) as entries: for entry in entries: @@ -32,8 +34,11 @@ def package_recipe_dir(repo): return recipes if entry.is_dir() and any(isfile(join(entry.path, f)) for f in LEGACY_RECIPE_DIR_ENTRIES): return recipes - except OSError: + except (FileNotFoundError, NotADirectoryError): pass + except OSError: + if strict: + raise return repo @@ -49,7 +54,7 @@ def scan(directory, defaults): raise return [] - files = scan(package_recipe_dir(repo), defaults=False) + files = scan(package_recipe_dir(repo, strict=strict), defaults=False) if include_defaults: files.extend(scan(repo, defaults=True)) return sorted(files) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 13974377..5375f97b 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -1086,7 +1086,7 @@ Which packages, for which platform and defaults profiles, with which flavours an Where the work directory and the recipes are, and which local checkouts are built: -Point `--config-dir` and `BITS_PATH` entries at repository roots. When `recipes/` contains package recipes, versioned recipes, or package generators, Bits uses it instead of the root for package lookup. A `recipes/` directory with only support files, or with `latest` or `packages.py` directly inside it (a legacy versioned recipe or generator), does not switch the checkout to the new layout. Defaults profiles, `patches/`, `checksums/`, `cvmfs.yaml`, and repository-relative file sources and hooks stay at the root. +Point `--config-dir` and `BITS_PATH` entries at repository roots. When `recipes/` contains `.sh` package recipes, versioned recipes, or package generators, Bits uses it instead of the root for package lookup. Keep shell helpers outside `recipes/` so they are not treated as recipes. A `recipes/` directory with the legacy marker file `latest` or `packages.py` directly inside it does not switch the checkout to the new layout. Defaults profiles, `patches/`, `checksums/`, `cvmfs.yaml`, and repository-relative file sources and hooks stay at the root. | Option | Description | |--------|-------------| diff --git a/tests/test_recipe_layout.py b/tests/test_recipe_layout.py index 4ece462e..71c1d2c5 100644 --- a/tests/test_recipe_layout.py +++ b/tests/test_recipe_layout.py @@ -47,12 +47,12 @@ def test_layout_mixes_new_and_legacy_repositories(self): (provider / 'packages').mkdir() (provider / 'packages' / 'latest').write_text(recipe('Packages')) (repo / 'extra.tar').touch() - (recipes / 'versioned').mkdir() - (recipes / 'versioned' / 'latest').write_text(recipe('Versioned')) + (recipes / 'latest').mkdir() + (recipes / 'latest' / 'latest').write_text(recipe('Latest')) root = str(repo) for name, path, checkout in ( ('foo', recipes / 'foo.sh', repo), - ('versioned', recipes / 'versioned' / 'latest', repo), + ('latest', recipes / 'latest' / 'latest', repo), ('bar', provider / 'bar.sh', provider), ('packages', provider / 'packages' / 'latest', provider), ('defaults-release', repo / 'defaults-release.sh', repo), @@ -146,6 +146,16 @@ def test_checksums_resolves_patches_at_root_and_rejects_missing_checkout(self): self.assertEqual(checks['Foo']['patches']['foo.patch'], (None, str(self.directory / 'patches'))) with self.assertRaises(FileNotFoundError): plan(root, [str(self.directory / 'missing.bits')], 'all', [], 'x86_64-el9') + scandir = os.scandir + + def deny_recipes(path): + if Path(path) == self.directory / 'recipes': + raise PermissionError(path) + return scandir(path) + + with patch('bits_helpers.paths.os.scandir', side_effect=deny_recipes): + with self.assertRaises(PermissionError): + recipes_in(root) def test_upgrade_copies_recipe_under_recipes_and_patches_at_root(self): provider = self.directory / 'provider' From 7a6245cc948ca3dc5bdc0d356ecc9b38089ccdd1 Mon Sep 17 00:00:00 2001 From: Juan Miguel Carceller Date: Thu, 19 Nov 2026 18:09:32 +0100 Subject: [PATCH 4/4] Search the checkout root before recipes/ Allow flat and nested recipes to coexist without replacing root recipes, including in pinned refs. --- CHANGELOG.md | 2 +- bits_helpers/checksums_cmd.py | 4 +- bits_helpers/paths.py | 90 +++++++++++++++-------------------- bits_helpers/recipe.py | 38 +++++++-------- bits_helpers/repo_provider.py | 18 ++++--- bits_helpers/upgrade_cmd.py | 9 ++-- docs/COOKBOOK.md | 2 +- docs/REFERENCE.md | 4 +- docs/USERGUIDE.md | 2 +- tests/test_recipe_layout.py | 55 +++++++++++++++------ tests/test_repo_provider.py | 35 ++++++-------- 11 files changed, 134 insertions(+), 125 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 496dfe12..ba5f3e68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ --- # Unreleased -- **[Improvement]** Recipe repositories can put package recipes under `recipes/` and keep defaults, patches and other repository files at the root. Existing flat repositories still work. +- **[Improvement]** Bits searches for package recipes at each checkout root first, then in `recipes/`. Flat and mixed checkouts work, and root recipes win on name clashes. Defaults, patches and other repository files stay at the root. - **[Change]** bits detects the platform name the CI and the builder images use: `x86_64-el9` on RHEL, AlmaLinux and Rocky 9, `x86_64-ubuntu2404` on Ubuntu 24.04 (macOS keeps `osx_arm64`). The `aliBuild` wrapper keeps ALICE's names (`slc9_x86-64`) by setting `BITS_LEGACY_ARCH=1`; native bits for ALICE records it with `bits use --architecture slc9_x86-64`. ALICE's store is read only for ALICE's names exactly (USERGUIDE "Platforms"). - **[Fix]** `bits init stacks.bits` (any `.bits`) no longer creates `sw` in the current directory: the work dir belongs inside the checkout, where you then build. - **[Improvement]** `bits --version` of an installed bits whose `_version.py` holds `git describe` (as bits.bits writes it) shows the tag, the commits since and the commit, e.g. `bits 0.7-3-g1a2b3c4 (tag 0.7 +3, commit 1a2b3c4)`. diff --git a/bits_helpers/checksums_cmd.py b/bits_helpers/checksums_cmd.py index 5e13397a..81558371 100644 --- a/bits_helpers/checksums_cmd.py +++ b/bits_helpers/checksums_cmd.py @@ -45,7 +45,7 @@ from bits_helpers.defaults import asDict, merge_dicts from bits_helpers.log import error, info, warning from bits_helpers.matchers import _parse_patch_entry, resolve_variables -from bits_helpers.paths import recipe_files +from bits_helpers.paths import recipe_file_name, recipe_files from bits_helpers.recipe import getRecipeReader, parseRecipe from bits_helpers.utilities import apply_version_from, resolve_spec_data from bits_helpers.workarea import _resolve_source_entry, _split_arch_prefix @@ -69,7 +69,7 @@ def _read(path, repo_dir=None): def recipes_in(repo): """``{lowercase package name: recipe path}`` for the recipes of *repo*.""" - return {os.path.basename(path)[:-3].lower(): path for path in recipe_files(repo, strict=True)} + return {recipe_file_name(path): path for path in recipe_files(repo, strict=True)} def profiles_in(repo, want): diff --git a/bits_helpers/paths.py b/bits_helpers/paths.py index b4509241..4e4f5d32 100644 --- a/bits_helpers/paths.py +++ b/bits_helpers/paths.py @@ -7,69 +7,57 @@ layers above it can share these without an import cycle.""" import os -from os.path import exists, isfile, islink, join +from os.path import basename, dirname, exists, isdir, isfile, join from bits_helpers.log import dieOnError, error RECIPE_SUBDIR = "recipes" -# A legacy checkout can already use recipes/ for a recipe or generator. -LEGACY_RECIPE_DIR_ENTRIES = ("latest", "packages.py") - - -def is_package_recipe(filename): - return filename.endswith(".sh") and not filename.startswith("defaults-") - - -def package_recipe_dir(repo, strict=False): - """Return the recipe directory, or the checkout root for a flat repository.""" - recipes = join(repo, RECIPE_SUBDIR) - for name in LEGACY_RECIPE_DIR_ENTRIES: - path = join(recipes, name) - if isfile(path) or islink(path): - return repo - try: - with os.scandir(recipes) as entries: - for entry in entries: - if is_package_recipe(entry.name) and entry.is_file(): - return recipes - if entry.is_dir() and any(isfile(join(entry.path, f)) for f in LEGACY_RECIPE_DIR_ENTRIES): - return recipes - except (FileNotFoundError, NotADirectoryError): - pass - except OSError: - if strict: - raise - return repo + + +def recipe_file_name(path): + """Package lookup name of a listed .sh file or versioned latest recipe.""" + name = basename(path) + return (basename(dirname(path)) if name == "latest" else name[:-3]).lower() def recipe_files(repo, include_defaults=False, strict=False): - """List package recipe files; optionally include root-level defaults profiles.""" - def scan(directory, defaults): + """List .sh recipes from both locations, with root copies taking precedence.""" + def scan(directory, defaults=False, optional=False): try: with os.scandir(directory) as entries: - return [e.path for e in entries if e.name.endswith(".sh") - and e.name.startswith("defaults-") == defaults and e.is_file()] - except OSError: - if strict: + return sorted(e.path for e in entries if e.name.endswith(".sh") + and (defaults or not e.name.startswith("defaults-")) and e.is_file()) + except OSError as exc: + if strict and not (optional and isinstance(exc, (FileNotFoundError, NotADirectoryError))): raise return [] - files = scan(package_recipe_dir(repo, strict=strict), defaults=False) - if include_defaults: - files.extend(scan(repo, defaults=True)) - return sorted(files) - - -def checkForFilename(taps, pkg, d, ext=".sh"): - if ext == ".sh" and not pkg.startswith("defaults-"): - d = package_recipe_dir(d) - filename = taps.get(pkg, "{}/{}{}".format(d, pkg, ext)) - if not exists(filename): - if "/" in pkg: - filename = taps.get(pkg, "{}/{}".format(d, pkg)) - else: - filename = taps.get(pkg, "{}/{}/latest".format(d, pkg)) - return filename + root = scan(repo, defaults=include_defaults) + names = {recipe_file_name(path) for path in root} + for path in scan(join(repo, RECIPE_SUBDIR), optional=True): + name = recipe_file_name(path) + if name not in names: + latest = join(repo, name, "latest") + root.append(latest if isfile(latest) else path) + return root + + +def checkForFilename(taps, pkg, d, ext=".sh", *, nested=True): + if pkg in taps: + return taps[pkg] + + def candidates(directory): + yield "{}/{}{}".format(directory, pkg, ext) + yield "{}/{}".format(directory, pkg) if "/" in pkg else "{}/{}/latest".format(directory, pkg) + + root, fallback = candidates(d) + if exists(root) and (ext != ".sh" or not isdir(root)): + return root + if nested and ext == ".sh" and not pkg.startswith("defaults-") and not isfile(fallback): + for recipe in candidates(join(d, RECIPE_SUBDIR)): + if isfile(recipe): + return recipe + return fallback def resolveLocalPath(configDir, s): """ diff --git a/bits_helpers/recipe.py b/bits_helpers/recipe.py index 2c003320..c3644e02 100644 --- a/bits_helpers/recipe.py +++ b/bits_helpers/recipe.py @@ -22,10 +22,7 @@ from bits_helpers.cmd import getoutput from bits_helpers.git import git from bits_helpers.log import dieOnError -from bits_helpers.paths import ( - LEGACY_RECIPE_DIR_ENTRIES, RECIPE_SUBDIR, getConfigPaths, is_package_recipe, - package_recipe_dir, resolveFilename, -) +from bits_helpers.paths import RECIPE_SUBDIR, checkForFilename, getConfigPaths, resolveFilename class SpecError(Exception): pass @@ -71,7 +68,12 @@ def __call__(self): class FileReader: def __init__(self, url, repo_dir=None) -> None: self.url = url - self.repo_dir = repo_dir if repo_dir and package_recipe_dir(repo_dir) != repo_dir else None + self.repo_dir = None + if repo_dir: + nested = join(os.path.abspath(repo_dir), RECIPE_SUBDIR) + path = os.path.abspath(url) + if path.startswith(nested + os.sep) and path != join(nested, "latest"): + self.repo_dir = repo_dir def __call__(self): with open(self.url) as f: return f.read() @@ -84,19 +86,9 @@ def __call__(self): m = re.search(r'^dist:(.*)@([^@]+)$', self.url) fn, gh = m.groups() fn = fn.lower() - if not fn.startswith("defaults-"): - prefix = RECIPE_SUBDIR + "/" - err, entries = git(("ls-tree", "-r", "--full-tree", "-z", "--name-only", gh, "--", prefix), - directory=self.configDir, check=False) - paths = {p[len(prefix):] for p in entries.split("\0") if p.startswith(prefix)} - if not err and not paths.intersection(LEGACY_RECIPE_DIR_ENTRIES): - for path in paths: - parts = path.split("/") - if ((len(parts) == 1 and is_package_recipe(parts[0])) or - (len(parts) == 2 and parts[1] in LEGACY_RECIPE_DIR_ENTRIES)): - fn = prefix + fn - break err, d = git(("show", f"{gh}:{fn}.sh"), directory=self.configDir, check=False) + if err and not fn.startswith("defaults-"): + err, d = git(("show", f"{gh}:{RECIPE_SUBDIR}/{fn}.sh"), directory=self.configDir, check=False) if err: raise RuntimeError("Cannot read recipe {fn} from reference {gh}.\n" "Make sure you run first (this will not alter your recipes):\n" @@ -319,9 +311,9 @@ def parseRecipe(reader, generatePackages=None, visited=None): def getGeneratedPackages(configDir): all_pkgs = {} pkgDirs = getConfigPaths(configDir) - for pkgdir in pkgDirs: + def generated_in(directory, pkgdir): dir_pkgs = {} - for packages_py in glob(join(escape(package_recipe_dir(pkgdir)), "*", "packages.py")): + for packages_py in glob(join(escape(directory), "*", "packages.py")): sys.path.insert(0, os.path.dirname(packages_py)) try: pkg = __import__("packages") @@ -335,6 +327,14 @@ def getGeneratedPackages(configDir): dieOnError(True, "Error running getPackages() in %r: %s" % (packages_py, e)) sys.modules.pop("packages") sys.path.pop(0) + return dir_pkgs + + for pkgdir in pkgDirs: + dir_pkgs = generated_in(pkgdir, pkgdir) + nested_pkgs = generated_in(join(pkgdir, RECIPE_SUBDIR), pkgdir) + for name, meta in nested_pkgs.items(): + if name not in dir_pkgs and not os.path.exists(checkForFilename({}, name, pkgdir, nested=False)): + dir_pkgs[name] = meta all_pkgs[pkgdir] = dir_pkgs return all_pkgs diff --git a/bits_helpers/repo_provider.py b/bits_helpers/repo_provider.py index 653eea1e..c8e7a5f5 100644 --- a/bits_helpers/repo_provider.py +++ b/bits_helpers/repo_provider.py @@ -63,7 +63,7 @@ from bits_helpers.workarea import updateReferenceRepoSpec, logged_scm from bits_helpers.utilities import resolve_spec_data, symlink from bits_helpers.recipe import getGeneratedPackages, getRecipeReader, parseRecipe -from bits_helpers.paths import checkForFilename, getConfigPaths, package_recipe_dir, recipe_files +from bits_helpers.paths import checkForFilename, getConfigPaths, recipe_file_name, recipe_files from bits_helpers.matchers import _parse_req_matcher # Maximum provider-discovery iterations (guards against run-away recursion) @@ -116,8 +116,7 @@ def _check_for_shadows( return # nothing on BITS_PATH yet — no shadowing possible incoming_recipes = { - os.path.splitext(os.path.basename(f))[0].lower() - for f in recipe_files(incoming_dir, include_defaults=True) + recipe_file_name(f) for f in recipe_files(incoming_dir, include_defaults=True) } if not incoming_recipes: return # empty provider directory — nothing to shadow @@ -129,8 +128,7 @@ def _check_for_shadows( ) for existing_dir in existing_dirs: existing_recipes = { - os.path.splitext(os.path.basename(f))[0].lower() - for f in recipe_files(existing_dir, include_defaults=True) + recipe_file_name(f) for f in recipe_files(existing_dir, include_defaults=True) } shadowed = incoming_recipes & existing_recipes if shadowed: @@ -740,13 +738,13 @@ def bootstrap_default_config(args, work_dir: str) -> Optional[str]: organisation = (getattr(args, "organisation", None) or "").lower() candidates = [] if organisation: - candidates.append(("%s.bits.sh" % organisation, organisation)) - candidates.append(("default.bits.sh", "default")) + candidates.append(("%s.bits" % organisation, organisation)) + candidates.append(("default.bits", "default")) chosen_sh = None chosen_label = None - for filename, label in candidates: - path = join(package_recipe_dir(providers_checkout), filename) + for name, label in candidates: + path = checkForFilename({}, name, providers_checkout) if exists(path): chosen_sh = path chosen_label = label @@ -841,7 +839,7 @@ def _warn(*a): _warn("Could not fetch the provider registry from %s.", url) return None - sh = join(package_recipe_dir(providers_checkout), name + ".sh") + sh = checkForFilename({}, name, providers_checkout) if not exists(sh): _warn("No '%s.sh' in the provider registry %s.", name, url) return None diff --git a/bits_helpers/upgrade_cmd.py b/bits_helpers/upgrade_cmd.py index 192a75a7..f0068e4a 100644 --- a/bits_helpers/upgrade_cmd.py +++ b/bits_helpers/upgrade_cmd.py @@ -33,7 +33,7 @@ import yaml from bits_helpers.log import banner, info, warning -from bits_helpers.paths import checkForFilename, package_recipe_dir +from bits_helpers.paths import RECIPE_SUBDIR, checkForFilename TIMEOUT_S = 20 @@ -638,7 +638,7 @@ def _cached_repo(path): parent = os.path.dirname(directory) if os.path.exists(os.path.join(directory, ".bits_provider_ok")): return directory - if directory == package_recipe_dir(parent) and os.path.exists(os.path.join(parent, ".bits_provider_ok")): + if os.path.basename(directory) == RECIPE_SUBDIR and os.path.exists(os.path.join(parent, ".bits_provider_ok")): return parent return None @@ -662,7 +662,8 @@ def _bring_in(path, config_dir, package, dry): src_dir = _cached_repo(path) or os.path.dirname(path) dest = checkForFilename({}, os.path.basename(path)[:-3], config_dir) if not os.path.exists(dest): - dest = os.path.join(package_recipe_dir(config_dir), os.path.basename(path)) + recipes = os.path.join(config_dir, RECIPE_SUBDIR) + dest = os.path.join(recipes if os.path.isdir(recipes) else config_dir, os.path.basename(path)) if os.path.exists(dest): raise NeedsHand("%s exists: it would be replaced by %s's" % (dest, path)) with open(path) as f: @@ -695,7 +696,7 @@ def _origin(path, provider_dirs): d = os.path.dirname(os.path.abspath(path)) for pd, (name, commit) in (provider_dirs or {}).items(): root = os.path.abspath(pd) - if d in (root, package_recipe_dir(root)): + if d in (root, os.path.join(root, RECIPE_SUBDIR)): return "%s@%s" % (name, str(commit)[:10]) r = subprocess.run(["git", "-C", d, "rev-parse", "--show-toplevel"], capture_output=True, text=True) return os.path.basename(r.stdout.strip()) if r.returncode == 0 else d diff --git a/docs/COOKBOOK.md b/docs/COOKBOOK.md index 8a83f4a9..9f188596 100644 --- a/docs/COOKBOOK.md +++ b/docs/COOKBOOK.md @@ -280,7 +280,7 @@ The **aliBuild** front-end is the legacy path instead: `aliBuild init` checks ou ### Use a private recipe repository alongside the defaults -Put package recipes in `recipes/`. Keep `defaults-*.sh`, `cvmfs.yaml`, `patches/` and `checksums/` at the repository root, and pass that root (not `recipes/`) to `-c`. Existing flat repositories still work; once you add `recipes/`, move all package recipes there, not just the new ones. +Package recipes can live at the root or in `recipes/`. Keep `defaults-*.sh`, `cvmfs.yaml`, `patches/` and `checksums/` at the repository root, and pass that root (not `recipes/`) to `-c`. If you copy a recipe into `recipes/`, Bits still uses the root copy until you remove it. Set `BITS_PATH` to add a repository to the recipe search path. A plain name such as `myorg` means `/myorg.bits`; an absolute path is used as is. These directories are searched *after* the main recipe directory: diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 5375f97b..3353544c 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -696,7 +696,7 @@ repository_position: append # commit: 1a2b3c4d ``` -The `source` URL must point to a git repository with `*.sh` package recipes in `recipes/`, or at the root for a legacy flat checkout. `tag` selects the branch, tag or commit to check out (default: `version`). A provider repository is loaded once per build; a `requires` entry that asks for a different version of an already-loaded provider is ignored, with a warning. +The `source` URL must point to a git repository with `*.sh` package recipes at the root, in `recipes/`, or both. `tag` selects the branch, tag or commit to check out (default: `version`). A provider repository is loaded once per build; a `requires` entry that asks for a different version of an already-loaded provider is ignored, with a warning. ### Always-on providers (`always_load: true`) @@ -1086,7 +1086,7 @@ Which packages, for which platform and defaults profiles, with which flavours an Where the work directory and the recipes are, and which local checkouts are built: -Point `--config-dir` and `BITS_PATH` entries at repository roots. When `recipes/` contains `.sh` package recipes, versioned recipes, or package generators, Bits uses it instead of the root for package lookup. Keep shell helpers outside `recipes/` so they are not treated as recipes. A `recipes/` directory with the legacy marker file `latest` or `packages.py` directly inside it does not switch the checkout to the new layout. Defaults profiles, `patches/`, `checksums/`, `cvmfs.yaml`, and repository-relative file sources and hooks stay at the root. +Point `--config-dir` and `BITS_PATH` entries at repository roots. Bits looks for packages at each checkout root first, then in `recipes/` before trying the next checkout. A root recipe or generator still takes precedence over any copy in `recipes/`. When moving a recipe, the root copy stays active until removed. Keep shell helpers outside the recipe search locations. Defaults profiles, `patches/`, `checksums/`, `cvmfs.yaml`, and repository-relative file sources and hooks stay at the root. | Option | Description | |--------|-------------| diff --git a/docs/USERGUIDE.md b/docs/USERGUIDE.md index 437397d5..ca3de911 100644 --- a/docs/USERGUIDE.md +++ b/docs/USERGUIDE.md @@ -448,7 +448,7 @@ Bits resolves the full transitive dependency graph of each requested package, co ### How a build proceeds -1. **Recipe discovery** — Bits searches the main checkout, then each `BITS_PATH` entry (`--search-path`; a relative NAME means `/NAME.bits`, an absolute path is used as is). In each checkout it looks for `.sh` under `recipes/` if that directory holds package recipes, or at the root for a flat checkout. Defaults profiles stay at the root. Repository-provider packages (see [§13](REFERENCE.md#13-repository-provider-feature)) are cloned first to extend the search path before the main resolution pass. +1. **Recipe discovery** — Bits searches the main checkout, then each `BITS_PATH` entry (`--search-path`; a relative NAME means `/NAME.bits`, an absolute path is used as is). In each checkout it looks at the root first, then in `recipes/` if the recipe is not at the root. Defaults profiles stay at the root. Repository-provider packages (see [§13](REFERENCE.md#13-repository-provider-feature)) are cloned first to extend the search path before the main resolution pass. 2. **Dependency resolution** — `requires`, `build_requires`, and `runtime_requires` fields are read recursively, forming a DAG. Cycles are reported as errors. 3. **Hash computation** — A hash is computed for each package from its recipe text, source commit, dependency hashes, and environment. Packages with a matching hash in a store are downloaded instead of rebuilt. 4. **Source fetching** — Source repositories are cloned into a local mirror and then checked out into a build area. Up to 8 repositories are fetched in parallel. diff --git a/tests/test_recipe_layout.py b/tests/test_recipe_layout.py index 71c1d2c5..594f5087 100644 --- a/tests/test_recipe_layout.py +++ b/tests/test_recipe_layout.py @@ -41,8 +41,14 @@ def test_layout_mixes_new_and_legacy_repositories(self): provider.mkdir() os.environ['BITS_PATH'] = 'base' (repo / 'defaults-release.sh').write_text(recipe('defaults-release')) - (repo / 'foo.sh').write_text(recipe('NotARecipeInThisLayout')) - (recipes / 'foo.sh').write_text(recipe('Foo')) + (repo / 'foo.sh').write_text(recipe('Foo')) + (repo / 'root-only.sh').write_text(recipe('Root-Only')) + (recipes / 'foo.sh').write_text(recipe('Shadowed')) + (recipes / 'nested-only.sh').write_text(recipe('Nested-Only')) + (repo / 'nested-only.sh').mkdir() + (recipes / 'pinned.sh').write_text(recipe('Shadowed')) + (repo / 'pinned').mkdir() + (repo / 'pinned' / 'latest').write_text(recipe('Pinned')) (provider / 'bar.sh').write_text(recipe('Bar')) (provider / 'packages').mkdir() (provider / 'packages' / 'latest').write_text(recipe('Packages')) @@ -51,7 +57,10 @@ def test_layout_mixes_new_and_legacy_repositories(self): (recipes / 'latest' / 'latest').write_text(recipe('Latest')) root = str(repo) for name, path, checkout in ( - ('foo', recipes / 'foo.sh', repo), + ('foo', repo / 'foo.sh', repo), + ('root-only', repo / 'root-only.sh', repo), + ('nested-only', recipes / 'nested-only.sh', repo), + ('pinned', repo / 'pinned' / 'latest', repo), ('latest', recipes / 'latest' / 'latest', repo), ('bar', provider / 'bar.sh', provider), ('packages', provider / 'packages' / 'latest', provider), @@ -61,8 +70,14 @@ def test_layout_mixes_new_and_legacy_repositories(self): self.assertEqual(resolveFilename({}, name, root, {}), (str(path), str(checkout))) self.assertEqual(resolveDefaultsFilename('release', root), str(repo / 'defaults-release.sh')) self.assertEqual(resolveLocalPath(root, 'file://extra.tar'), 'file:/' + str(repo / 'extra.tar')) - self.assertEqual(recipes_in(root), {'foo': str(recipes / 'foo.sh')}) - self.assertEqual(set(scan_recipes(root)['by_package']), {'foo', 'defaults-release'}) + self.assertEqual(recipes_in(root), { + 'foo': str(repo / 'foo.sh'), 'root-only': str(repo / 'root-only.sh'), + 'nested-only': str(recipes / 'nested-only.sh'), 'pinned': str(repo / 'pinned' / 'latest'), + }) + self.assertEqual(set(scan_recipes(root)['by_package']), + {'foo', 'root-only', 'nested-only', 'pinned', 'defaults-release'}) + (repo / 'foo.sh').unlink() + self.assertEqual(resolveFilename({}, 'foo', root, {}), (str(recipes / 'foo.sh'), root)) def test_provider_discovery_uses_recipes_but_keeps_checkout_root(self): recipes = self.directory / 'recipes' @@ -107,13 +122,15 @@ def git(*args): legacy = git('rev-parse', 'HEAD') self.assertEqual(resolveFilename({}, 'foo', root, {}), (str(self.directory / 'foo.sh'), root)) self.assertEqual(resolveFilename({}, 'recipes', root, {}), (str(recipes / 'latest'), root)) - git('rm', 'recipes/latest') - recipes.mkdir(exist_ok=True) - git('mv', 'foo.sh', 'recipes/foo.sh') - git('commit', '-qm', 'New layout') - for ref in (old, assets, legacy, 'HEAD'): + (recipes / 'foo.sh').write_text(recipe('FromNested')) + git('add', 'recipes/foo.sh') + git('commit', '-qm', 'Mixed layout') + mixed = git('rev-parse', 'HEAD') + git('rm', 'foo.sh') + git('commit', '-qm', 'Remove the shadowing root copy') + for ref, name in ((old, 'Foo'), (assets, 'Foo'), (legacy, 'Foo'), (mixed, 'Foo'), ('HEAD', 'FromNested')): with self.subTest(ref=ref): - self.assertEqual(GitReader(f'dist:Foo@{ref}', root)().strip(), recipe('Foo').strip()) + self.assertEqual(GitReader(f'dist:Foo@{ref}', root)().strip(), recipe(name).strip()) def test_generated_packages_and_includes_use_repo_root(self): primary = self.directory / 'primary' @@ -125,10 +142,20 @@ def test_generated_packages_and_includes_use_repo_root(self): generators.mkdir(parents=True) (generators / 'packages.py').write_text( 'def getPackages(virtual_packages, config_dir):\n' - ' virtual_packages["generated"] = {"pkgdir": config_dir, "version": "1"}\n') + ' for name in ("generated", "root-file", "root-generated"):\n' + ' virtual_packages[name] = {"pkgdir": config_dir, "version": "1"}\n') + root_generators = self.directory / 'generated' + root_generators.mkdir() + (root_generators / 'packages.py').write_text( + 'def getPackages(virtual_packages, config_dir):\n' + ' virtual_packages["root-generated"] = {"pkgdir": config_dir, "version": "2"}\n') + root_file = self.directory / 'root-file.sh' + root_file.write_text(recipe('Root-File')) root = str(self.directory) - self.assertEqual(resolveFilename({}, 'generated', root, getGeneratedPackages(root)), - ('generate:generated@1', root)) + generated = getGeneratedPackages(root) + self.assertEqual(resolveFilename({}, 'generated', root, generated), ('generate:generated@1', root)) + self.assertEqual(resolveFilename({}, 'root-generated', root, generated), ('generate:root-generated@2', root)) + self.assertEqual(resolveFilename({}, 'root-file', root, generated), (str(root_file), root)) path = recipes / 'foo.sh' path.write_text(recipe('Foo') + '#!include \n#!include "local.inc"\n') (self.directory / 'root.inc').write_text('echo root') diff --git a/tests/test_repo_provider.py b/tests/test_repo_provider.py index 3afb7635..b01ecfb8 100644 --- a/tests/test_repo_provider.py +++ b/tests/test_repo_provider.py @@ -913,20 +913,24 @@ def _make_clone_side_effect(self, providers_checkout, config_checkout): ]) return lambda *a, **kw: next(calls) + def _registry(self, filename): + checkout = tempfile.TemporaryDirectory() + self.addCleanup(checkout.cleanup) + with open(os.path.join(checkout.name, filename), "w"): + pass + return checkout.name + @patch("bits_helpers.repo_provider.info") # avoid LogFormatter double-format bug @patch("bits_helpers.repo_provider.clone_or_update_provider") @patch("bits_helpers.repo_provider.parseRecipe") @patch("bits_helpers.repo_provider.getRecipeReader") - @patch("bits_helpers.repo_provider.exists") # module-level name, not os.path.exists def test_returns_config_checkout_when_default_sh_found( - self, mock_exists, mock_reader, mock_parse, mock_clone, _info, + self, mock_reader, mock_parse, mock_clone, _info, ): """Happy path: bits-providers has default.bits.sh → returns config checkout.""" - providers_checkout = "/work/REPOS/bits-providers/abc1234567" + providers_checkout = self._registry("default.bits.sh") config_checkout = "/work/REPOS/default.bits/def4567890" - # exists() returns True only for the default.bits.sh path - mock_exists.side_effect = lambda p: p.endswith("default.bits.sh") mock_reader.return_value = "reader" mock_parse.return_value = ( None, @@ -981,13 +985,11 @@ def test_returns_none_when_bits_providers_clone_fails(self, mock_clone, _warn): @patch("bits_helpers.repo_provider.clone_or_update_provider") @patch("bits_helpers.repo_provider.parseRecipe") @patch("bits_helpers.repo_provider.getRecipeReader") - @patch("bits_helpers.repo_provider.exists") # module-level name, not os.path.exists def test_returns_none_when_config_clone_fails( - self, mock_exists, mock_reader, mock_parse, mock_clone, _info, _warn, + self, mock_reader, mock_parse, mock_clone, _info, _warn, ): """A clone failure for the default config repo is caught and returns None.""" - providers_checkout = "/work/REPOS/bits-providers/abc1234567" - mock_exists.side_effect = lambda p: p.endswith("default.bits.sh") + providers_checkout = self._registry("default.bits.sh") mock_reader.return_value = "reader" mock_parse.return_value = ( None, @@ -1014,17 +1016,13 @@ def test_returns_none_when_config_clone_fails( @patch("bits_helpers.repo_provider.clone_or_update_provider") @patch("bits_helpers.repo_provider.parseRecipe") @patch("bits_helpers.repo_provider.getRecipeReader") - @patch("bits_helpers.repo_provider.exists") def test_organisation_recipe_preferred_over_default( - self, mock_exists, mock_reader, mock_parse, mock_clone, _info, + self, mock_reader, mock_parse, mock_clone, _info, ): """Organisation stored uppercase (LHCB) → resolves to lhcb.bits.sh (lowercase).""" - providers_checkout = "/work/REPOS/bits-providers/abc1234567" + providers_checkout = self._registry("lhcb.bits.sh") config_checkout = "/work/REPOS/lhcb.bits/def4567890" - # Only lhcb.bits.sh (lowercase) exists — default.bits.sh is absent. - # args.organisation is uppercase "LHCB" as stored in bits.rc. - mock_exists.side_effect = lambda p: p.endswith("lhcb.bits.sh") mock_reader.return_value = "reader" mock_parse.return_value = ( None, @@ -1056,16 +1054,13 @@ def test_organisation_recipe_preferred_over_default( @patch("bits_helpers.repo_provider.clone_or_update_provider") @patch("bits_helpers.repo_provider.parseRecipe") @patch("bits_helpers.repo_provider.getRecipeReader") - @patch("bits_helpers.repo_provider.exists") def test_organisation_falls_back_to_default_sh_when_org_sh_absent( - self, mock_exists, mock_reader, mock_parse, mock_clone, _info, + self, mock_reader, mock_parse, mock_clone, _info, ): """When .bits.sh is absent, bootstrap falls back to default.bits.sh.""" - providers_checkout = "/work/REPOS/bits-providers/abc1234567" + providers_checkout = self._registry("default.bits.sh") config_checkout = "/work/REPOS/default.bits/def4567890" - # LHCB.bits.sh / lhcb.bits.sh absent, default.bits.sh present - mock_exists.side_effect = lambda p: p.endswith("default.bits.sh") mock_reader.return_value = "reader" mock_parse.return_value = ( None,