Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[
---
# Unreleased

- **[Improvement]** Bits searches for package recipes at each checkout root first, then in `recipes/`. Flat and mixed checkouts work, and root recipes win on name clashes. Defaults, patches and other repository files stay at the root.
- **[Change]** bits detects the platform name the CI and the builder images use: `x86_64-el9` on RHEL, AlmaLinux and Rocky 9, `x86_64-ubuntu2404` on Ubuntu 24.04 (macOS keeps `osx_arm64`). The `aliBuild` wrapper keeps ALICE's names (`slc9_x86-64`) by setting `BITS_LEGACY_ARCH=1`; native bits for ALICE records it with `bits use --architecture slc9_x86-64`. ALICE's store is read only for ALICE's names exactly (USERGUIDE "Platforms").
- **[Fix]** `bits init stacks.bits` (any `<group>.bits`) no longer creates `sw` in the current directory: the work dir belongs inside the checkout, where you then build.
- **[Improvement]** `bits --version` of an installed bits whose `_version.py` holds `git describe` (as bits.bits writes it) shows the tag, the commits since and the commit, e.g. `bits 0.7-3-g1a2b3c4 (tag 0.7 +3, commit 1a2b3c4)`.
Expand Down
14 changes: 5 additions & 9 deletions bits_helpers/brew.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import glob

from bits_helpers.log import debug, error, info
from bits_helpers.paths import recipe_files
from bits_helpers.recipe import parseRecipe, FileReader


Expand Down Expand Up @@ -120,12 +121,12 @@ def collect_homebrew(configDirs, architecture):
configDirs = [configDirs]
seen = set()
for configDir in configDirs:
for path in sorted(glob.glob(os.path.join(configDir, "*.sh"))):
for path in recipe_files(configDir, include_defaults=True):
real = os.path.realpath(path)
if real in seen:
continue
seen.add(real)
err, spec, _ = parseRecipe(FileReader(path))
err, spec, _ = parseRecipe(FileReader(path, repo_dir=configDir))
if err or not spec:
# Not a parseable recipe (e.g. a defaults-*.sh helper). Skip quietly.
continue
Expand All @@ -143,13 +144,8 @@ def _standalone_scan_dirs(configDir, work_dir):
"""
dirs = [configDir]
repos = os.path.join(os.path.abspath(work_dir), "REPOS")
if os.path.isdir(repos):
# Any directory under REPOS that contains recipes. Duplicates across
# provider versions are harmless — formulae are collected into a set.
found = set()
for path in glob.glob(os.path.join(repos, "**", "*.sh"), recursive=True):
found.add(os.path.dirname(path))
dirs.extend(sorted(found))
dirs.extend(sorted(path for path in glob.glob(os.path.join(repos, "*", "*"))
if os.path.isdir(path) and not os.path.islink(path)))
return dirs


Expand Down
21 changes: 9 additions & 12 deletions bits_helpers/checksums_cmd.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
from bits_helpers.defaults import asDict, merge_dicts
from bits_helpers.log import error, info, warning
from bits_helpers.matchers import _parse_patch_entry, resolve_variables
from bits_helpers.paths import recipe_file_name, recipe_files
from bits_helpers.recipe import getRecipeReader, parseRecipe
from bits_helpers.utilities import apply_version_from, resolve_spec_data
from bits_helpers.workarea import _resolve_source_entry, _split_arch_prefix
Expand All @@ -57,9 +58,9 @@

# ── Recipes and profiles ──────────────────────────────────────────────────────

def _read(path):
def _read(path, repo_dir=None):
"""Parsed YAML header of recipe *path*, or ``None`` (warned) if unparsable."""
err, meta, _ = parseRecipe(getRecipeReader(path))
err, meta, _ = parseRecipe(getRecipeReader(path, repo_dir=repo_dir))
if err or not isinstance(meta, dict):
warning("Skipping %s: %s", path, err or "not a recipe")
return None
Expand All @@ -68,12 +69,7 @@ def _read(path):

def recipes_in(repo):
"""``{lowercase package name: recipe path}`` for the recipes of *repo*."""
out = {}
for f in sorted(os.listdir(repo)):
path = os.path.join(repo, f)
if f.endswith(".sh") and not f.startswith("defaults-") and os.path.isfile(path):
out[f[:-3].lower()] = path
return out
return {recipe_file_name(path): path for path in recipe_files(repo, strict=True)}


def profiles_in(repo, want):
Expand Down Expand Up @@ -261,7 +257,7 @@ def plan(repo, recipe_dirs, profile_names, packages, arch):
search = [repo] + [d for d in recipe_dirs if os.path.abspath(d) != repo]
found = {}
for d in reversed(search): # earlier repositories win, as in a build
found.update(recipes_in(d))
found.update((name, (path, d)) for name, path in recipes_in(d).items())
own = recipes_in(repo)
want = {p.lower() for p in packages}
profiles = profiles_in(repo, profile_names) if profile_names else []
Expand Down Expand Up @@ -298,14 +294,15 @@ def add(pkg, patches_dir, got, origin):
for name in sorted(set(own) | set(found if profiles else ())):
if want and name not in want:
continue
meta = _read(found[name])
path, recipe_repo = found[name]
meta = _read(path, repo_dir=recipe_repo)
if meta is None:
continue
pkg = str(meta.get("package", name))
if pkg.lower() != name: # the build refuses such a recipe too
warning("Skipping %s: its package field is %r", found[name], pkg)
warning("Skipping %s: its package field is %r", path, pkg)
continue
patches_dir = os.path.join(os.path.dirname(found[name]), "patches")
patches_dir = os.path.join(recipe_repo, "patches")
try:
base = wants(meta, ["release"], rel_vars, arch, source_mode(release))
except ValueError as exc:
Expand Down
10 changes: 5 additions & 5 deletions bits_helpers/compliance.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import urllib.request

from bits_helpers.log import banner, debug, error, info, warning
from bits_helpers.paths import recipe_files
from bits_helpers import site

# Front-matter keys read from each recipe. Values may be quoted (the header
Expand Down Expand Up @@ -71,10 +72,9 @@ def scan_recipes(recipes_dir):
out = {"total": 0, "missing_license": [], "licenseref": [],
"noassertion": [], "restricted": [], "restricted_sources": [],
"by_package": {}}
for name in sorted(os.listdir(recipes_dir)):
if not name.endswith(".sh"):
continue
meta = _recipe_meta(os.path.join(recipes_dir, name))
for path in recipe_files(recipes_dir, include_defaults=True, strict=True):
name = os.path.relpath(path, recipes_dir)
meta = _recipe_meta(path)
pkg = meta.get("package")
if not pkg:
continue # not a recipe (helper script)
Expand Down Expand Up @@ -631,7 +631,7 @@ def doCompliance(args, parser):
"distributed)", len(system_pkgs))
else:
recipes_dir = os.path.abspath(getattr(args, "recipesDir", None) or ".")
if not any(f.endswith(".sh") for f in os.listdir(recipes_dir)):
if not recipe_files(recipes_dir, include_defaults=True):
parser.error("no recipes (*.sh) found in %s — point --recipes at a "
"recipe repository (e.g. a lcg.bits checkout), or give "
"PACKAGE roots for a group-wide audit" % recipes_dir)
Expand Down
3 changes: 2 additions & 1 deletion bits_helpers/doctor.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
from bits_helpers.defaults import (parseDefaults, readDefaults, validateDefaults,
incompatibleFlavorDefaults)
from bits_helpers.packages import getPackageList
from bits_helpers.paths import checkForFilename
from bits_helpers.arch import effective_arch

# ── Status constants ───────────────────────────────────────────────────────────
Expand Down Expand Up @@ -839,7 +840,7 @@ def doDoctor(args, parser):

packages = []
for p in args.packages:
recipe_path = "%s/%s.sh" % (args.configDir, p.lower())
recipe_path = checkForFilename({}, p.lower(), args.configDir)
if not exists(recipe_path):
error("Cannot find recipe %s for package %s.", recipe_path, p)
exitcode = 1
Expand Down
3 changes: 2 additions & 1 deletion bits_helpers/packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,8 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem,
"is the intended one.", p, recipeSourceLabel(pkgdir, provider_dirs),
recipeSourceLabel(_later, provider_dirs), ", ".join(_defaults_dirs[_later]))

err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir, generatedPackages[pkgdir]), generatedPackages)
reader = getRecipeReader(filename, configDir, generatedPackages[pkgdir], repo_dir=pkgdir)
err, spec, recipe = parseRecipe(reader, generatedPackages)
dieOnError(err, err)
# Unless there was an error, both spec and recipe should be valid.
# otherwise the error should have been caught above.
Expand Down
57 changes: 48 additions & 9 deletions bits_helpers/paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,57 @@
layers above it can share these without an import cycle."""

import os
from os.path import exists, join
from os.path import basename, dirname, exists, isdir, isfile, join

from bits_helpers.log import dieOnError, error

def checkForFilename(taps, pkg, d, ext=".sh"):
filename = taps.get(pkg, "{}/{}{}".format(d, pkg, ext))
if not exists(filename):
if "/" in pkg:
filename = taps.get(pkg, "{}/{}".format(d, pkg))
else:
filename = taps.get(pkg, "{}/{}/latest".format(d, pkg))
return filename
RECIPE_SUBDIR = "recipes"


def recipe_file_name(path):
"""Package lookup name of a listed .sh file or versioned latest recipe."""
name = basename(path)
return (basename(dirname(path)) if name == "latest" else name[:-3]).lower()


def recipe_files(repo, include_defaults=False, strict=False):
"""List .sh recipes from both locations, with root copies taking precedence."""
def scan(directory, defaults=False, optional=False):
try:
with os.scandir(directory) as entries:
return sorted(e.path for e in entries if e.name.endswith(".sh")
and (defaults or not e.name.startswith("defaults-")) and e.is_file())
except OSError as exc:
if strict and not (optional and isinstance(exc, (FileNotFoundError, NotADirectoryError))):
raise
return []

root = scan(repo, defaults=include_defaults)
names = {recipe_file_name(path) for path in root}
for path in scan(join(repo, RECIPE_SUBDIR), optional=True):
name = recipe_file_name(path)
if name not in names:
latest = join(repo, name, "latest")
root.append(latest if isfile(latest) else path)
return root


def checkForFilename(taps, pkg, d, ext=".sh", *, nested=True):
if pkg in taps:
return taps[pkg]

def candidates(directory):
yield "{}/{}{}".format(directory, pkg, ext)
yield "{}/{}".format(directory, pkg) if "/" in pkg else "{}/{}/latest".format(directory, pkg)

root, fallback = candidates(d)
if exists(root) and (ext != ".sh" or not isdir(root)):
return root
if nested and ext == ".sh" and not pkg.startswith("defaults-") and not isfile(fallback):
for recipe in candidates(join(d, RECIPE_SUBDIR)):
if isfile(recipe):
return recipe
return fallback

def resolveLocalPath(configDir, s):
"""
Expand Down
5 changes: 3 additions & 2 deletions bits_helpers/preload_cmd.py
Original file line number Diff line number Diff line change
Expand Up @@ -399,12 +399,13 @@ def main(argv=None):
ap.error("--docker requires --docker-image IMAGE")

def recipe_reader(pkg):
from bits_helpers.paths import checkForFilename
from bits_helpers.recipe import parseRecipe, FileReader
path = os.path.join(a.config_dir, pkg + ".sh")
path = checkForFilename({}, pkg, a.config_dir)
if not os.path.isfile(path):
return None
try:
_err, spec, _body = parseRecipe(FileReader(path))
_err, spec, _body = parseRecipe(FileReader(path, repo_dir=a.config_dir))
return spec
except Exception:
return None
Expand Down
46 changes: 31 additions & 15 deletions bits_helpers/recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import re
import sys
from collections import OrderedDict
from glob import glob
from glob import escape, glob
from os.path import join
from typing import Any, IO

Expand All @@ -22,7 +22,7 @@
from bits_helpers.cmd import getoutput
from bits_helpers.git import git
from bits_helpers.log import dieOnError
from bits_helpers.paths import getConfigPaths, resolveFilename
from bits_helpers.paths import RECIPE_SUBDIR, checkForFilename, getConfigPaths, resolveFilename

class SpecError(Exception):
pass
Expand All @@ -43,7 +43,7 @@ def validateSpec(spec):
spec["version"] = "0"


def getRecipeReader(url: str, dist=None, genPackages={}):
def getRecipeReader(url: str, dist=None, genPackages={}, repo_dir=None):
m = re.search(r'^(dist|generate):(.*)@([^@]+)$', url)
if m and m.group(1) == "generate":
pkg, version = m.group(2), m.group(3)
Expand All @@ -54,7 +54,7 @@ def getRecipeReader(url: str, dist=None, genPackages={}):
elif m and dist:
return GitReader(url, dist)
else:
return FileReader(url)
return FileReader(url, repo_dir)

# Generate a recipe of package
class GeneratedPackage:
Expand All @@ -66,8 +66,14 @@ def __call__(self):

# Read a recipe from a file
class FileReader:
def __init__(self, url) -> None:
def __init__(self, url, repo_dir=None) -> None:
self.url = url
self.repo_dir = None
if repo_dir:
nested = join(os.path.abspath(repo_dir), RECIPE_SUBDIR)
path = os.path.abspath(url)
if path.startswith(nested + os.sep) and path != join(nested, "latest"):
self.repo_dir = repo_dir
def __call__(self):
with open(self.url) as f:
return f.read()
Expand All @@ -79,8 +85,10 @@ def __init__(self, url, configDir) -> None:
def __call__(self):
m = re.search(r'^dist:(.*)@([^@]+)$', self.url)
fn, gh = m.groups()
err, d = git(("show", f"{gh}:{fn.lower()}.sh"),
directory=self.configDir)
fn = fn.lower()
err, d = git(("show", f"{gh}:{fn}.sh"), directory=self.configDir, check=False)
if err and not fn.startswith("defaults-"):
err, d = git(("show", f"{gh}:{RECIPE_SUBDIR}/{fn}.sh"), directory=self.configDir, check=False)
if err:
raise RuntimeError("Cannot read recipe {fn} from reference {gh}.\n"
"Make sure you run first (this will not alter your recipes):\n"
Expand Down Expand Up @@ -238,7 +246,8 @@ def parseRecipe(reader, generatePackages=None, visited=None):
header, recipe = d[:_m.start()], d[_m.end():]
# Splice any `#!include` directives in the body before anything else sees it,
# so the included text is variable-expanded and hashed as if written inline.
recipe = resolveIncludes(recipe, getattr(reader, "url", "") or "")
repo_dir = getattr(reader, "repo_dir", None)
recipe = resolveIncludes(recipe, getattr(reader, "url", "") or "", repo_dir)
# YAML forbids '%' as the first character of a plain (unquoted) scalar because
# it is reserved for directives (e.g. %YAML, %TAG). Recipe authors may want
# to write "- %(name)s-%(version)s.patch" in patches: (and similar lists)
Expand Down Expand Up @@ -271,15 +280,15 @@ def parseRecipe(reader, generatePackages=None, visited=None):
if spec and "from" in spec:
basename = os.path.basename(getattr(reader, "url", "") or "")
filename = basename[:-3] if basename.endswith(".sh") else basename
repoDir = os.environ.get("BITS_REPO_DIR")
repoDir = repo_dir or os.environ.get("BITS_REPO_DIR")
if visited is None:
visited = []
if spec["from"] in visited:
raise RuntimeError(f" Cyclic Dependency: {' -> '.join(list(visited) + [spec['from']])}")
visited.append(spec["from"])
parent_dir = os.path.join(repoDir, spec["from"])
base_filename, pkgdir = resolveFilename({}, filename, parent_dir, generatePackages)
base_reader = getRecipeReader(base_filename, repoDir, generatePackages[parent_dir])
base_filename, pkgdir = resolveFilename({}, filename, parent_dir, generatePackages or {})
base_reader = getRecipeReader(base_filename, repoDir, (generatePackages or {}).get(pkgdir, {}), pkgdir)
err, base_spec, base_recipe = parseRecipe(base_reader, generatePackages, visited)
spec, recipe_append = handleMergePolicy(spec, base_spec)
recipe = recipe + base_recipe if recipe_append else recipe
Expand All @@ -302,11 +311,10 @@ def parseRecipe(reader, generatePackages=None, visited=None):
def getGeneratedPackages(configDir):
all_pkgs = {}
pkgDirs = getConfigPaths(configDir)
for pkgdir in pkgDirs:
def generated_in(directory, pkgdir):
dir_pkgs = {}
for vp in [x.split(os.sep)[-2] for x in glob(join(pkgdir, "*", "packages.py"))]:
packages_py = join(pkgdir, vp, "packages.py")
sys.path.insert(0, join(pkgdir, vp))
for packages_py in glob(join(escape(directory), "*", "packages.py")):
sys.path.insert(0, os.path.dirname(packages_py))
try:
pkg = __import__("packages")
except (ImportError, SyntaxError) as e:
Expand All @@ -319,6 +327,14 @@ def getGeneratedPackages(configDir):
dieOnError(True, "Error running getPackages() in %r: %s" % (packages_py, e))
sys.modules.pop("packages")
sys.path.pop(0)
return dir_pkgs

for pkgdir in pkgDirs:
dir_pkgs = generated_in(pkgdir, pkgdir)
nested_pkgs = generated_in(join(pkgdir, RECIPE_SUBDIR), pkgdir)
for name, meta in nested_pkgs.items():
if name not in dir_pkgs and not os.path.exists(checkForFilename({}, name, pkgdir, nested=False)):
dir_pkgs[name] = meta
all_pkgs[pkgdir] = dir_pkgs
return all_pkgs

Expand Down
Loading
Loading