From eebdcee708510c4f9e2f08894e0305dda759acde Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Thu, 12 Nov 2026 09:37:55 +0900 Subject: [PATCH] Pre-commit: fix the shellcheck and hadolint findings and run every hook in CI (closes #368, refs #365) The Pre-commit workflow skipped shellcheck and hadolint-docker because the repository had findings for both, so neither protected anything. This fixes the findings and removes the SKIP. Shell scripts (build.sh, build-native.sh, run_examples.sh, native/scripts/build-image.sh): ls parsing becomes find or a glob loop (newest_wheel() compares mtimes with -nt), word-split argument strings become arrays (DOCKER_BUILD_ARGS, read -r -a for the example arguments and the EXAMPLES globs), the javac source list goes through find -print0 | xargs -0, expansions are quoted. run_examples.sh run against a stub interpreter gives the same calls, outputs and GitHub outputs as before. Dockerfile.build: each stage that runs pipes sets SHELL bash -o pipefail (DL4006); the jdeps, grep and paste steps keep their old tolerance with || true, as build-native.sh has it; jlink gets its --module-path through the positional parameters; the smoke test script is written with printf (bash echo does not expand \n, dash echo did; the bytes are identical); --no-install-recommends (DL3015). DL3008 (pinned apt versions) is ignored per instruction with the reason: the base images are rolling tags. .shellcheckrc follows the source lines (external-sources, SCRIPTDIR). #365 code and config items: the local hook runs with language python instead of the system python3, the pyproject.toml comment names a README section that exists, and the version examples of build.sh and build_wheel.py name 26.12.1 and 26.11.1. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/pre-commit.yml | 3 -- .pre-commit-config.yaml | 4 +-- .shellcheckrc | 4 +++ CLAUDE.md | 2 +- CONTRIBUTING.md | 2 +- native/scripts/build-image.sh | 3 +- native/wheel/build_wheel.py | 4 +-- pyproject.toml | 2 +- scripts/Dockerfile.build | 57 ++++++++++++++++++++++---------- scripts/build-native.sh | 16 +++++---- scripts/build.sh | 48 +++++++++++++++------------ scripts/run_examples.sh | 41 +++++++++++++++++------ 12 files changed, 119 insertions(+), 67 deletions(-) create mode 100644 .shellcheckrc diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 1d5e047c..b1358f34 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -46,7 +46,4 @@ jobs: uv --version - name: Run pre-commit on all files - env: - # shellcheck and hadolint findings are fixed in the follow-up PR. - SKIP: shellcheck,hadolint-docker run: uvx pre-commit run --all-files --show-diff-on-failure diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 255ce3d4..f9320f53 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -77,7 +77,7 @@ repos: hooks: - id: no-old-package-name name: the old package name does not come back - entry: python3 scripts/check_no_embedded_name.py - language: system + entry: python scripts/check_no_embedded_name.py + language: python pass_filenames: false always_run: true diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 00000000..84c8eeda --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,4 @@ +# Picked up by the shellcheck pre-commit hook (.pre-commit-config.yaml) and by editors. +# Follow `source` lines; a `# shellcheck source=` directive names the file relative to the script's directory. +external-sources=true +source-path=SCRIPTDIR diff --git a/CLAUDE.md b/CLAUDE.md index ad47f782..25e1e243 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,7 +52,7 @@ uv run --no-project --with "$W[test]" python -m pytest -m server ## Lint / format -Pre-commit: black, isort (`--profile black`), shfmt (`-i 4 -ci -sr`), pretty-format-yaml (not on `.github/`), actionlint. CI also runs Bandit, which must be clean: +Pre-commit (CI runs every hook on all files, none is skipped): ruff, black, isort (`--profile black`), prettier, shfmt (`-i 4 -ci -sr`), shellcheck and hadolint (both in Docker), pretty-format-yaml (not on `.github/`), actionlint. CI also runs Bandit, which must be clean: ```bash uvx bandit==1.9.4 -c pyproject.toml -r src tests --severity-level low --confidence-level low diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 03fb7ad1..db3d3bf5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,7 +87,7 @@ uv run --no-project --python 3.12 python native/wheel/build_wheel.py --lib-dir < ### Pre-commit -This project uses [pre-commit](https://pre-commit.com/): ruff (lint, default rules E4/E7/E9/F at line length 88), black and isort (Python formatting), prettier (Java, XML, Markdown, JSON), shfmt and shellcheck (shell), hadolint (Dockerfile), pretty-format-yaml and the basic hygiene hooks (trailing whitespace, end-of-file, merge-conflict markers, large files, private keys), and actionlint (GitHub Actions). The settings live in `pyproject.toml` (`[tool.ruff]`) and `.prettierrc.json`. +This project uses [pre-commit](https://pre-commit.com/): ruff (lint, default rules E4/E7/E9/F at line length 88), black and isort (Python formatting), prettier (Java, XML, Markdown, JSON), shfmt and shellcheck (shell), hadolint (Dockerfile), pretty-format-yaml and the basic hygiene hooks (trailing whitespace, end-of-file, merge-conflict markers, large files, private keys), and actionlint (GitHub Actions). The settings live in `pyproject.toml` (`[tool.ruff]`), `.prettierrc.json` and `.shellcheckrc`. Install the hooks once so they run on every commit: diff --git a/native/scripts/build-image.sh b/native/scripts/build-image.sh index fe88cd1d..fe2a0575 100755 --- a/native/scripts/build-image.sh +++ b/native/scripts/build-image.sh @@ -65,7 +65,8 @@ for f in target/reports/used_classes_*.txt reports/used_classes_*.txt; do done if [ -z "$UC" ]; then echo "no used_classes_*.txt analysis report after the build (-H:+PrintAnalysisCallTree); found:" >&2 - ls -R target 2> /dev/null | grep -i -E "report|used_" >&2 || true + target_listing=$(ls -R target 2> /dev/null || true) + printf '%s\n' "$target_listing" | grep -i -E "report|used_" >&2 || true exit 1 fi cp "$UC" "$OUT/used_classes.txt" diff --git a/native/wheel/build_wheel.py b/native/wheel/build_wheel.py index 7c44c24a..8f55c2b5 100644 --- a/native/wheel/build_wheel.py +++ b/native/wheel/build_wheel.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Build the `arcadedb` wheel (the native build) from the shared Python sources and a built native image. - python native/wheel/build_wheel.py --lib-dir native/target/lib --version 26.11.1.dev0 --engine 26.10.2 --out + python native/wheel/build_wheel.py --lib-dir native/target/lib --version 26.12.1.dev0 --engine 26.11.1 --out Steps: copy src/arcadedb to /stage/src/arcadedb (the shared tree is not duplicated in git; only the install hints of the extras are renamed from arcadedb-jvm[...] to arcadedb[...]); rewrite `import jpype` to the bundled stub; flip the default backend to native; add the library of the build platform (libarcadedb.so on Linux, @@ -116,7 +116,7 @@ def main(): ap.add_argument( "--engine", required=True, - help="engine release the image was built from, e.g. 26.10.2", + help="engine release the image was built from, e.g. 26.11.1", ) ap.add_argument("--out", required=True) ap.add_argument( diff --git a/pyproject.toml b/pyproject.toml index c029dfab..5f811e70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ dependencies = [ # This pyproject is the *package definition* consumed by the wheel build # (scripts/build.sh). It is not a uv project: tests run against the built wheel, # with its `test` extra, in a throwaway `uv run --no-project --with` environment -# (see README, Testing). `managed = false` makes uv skip it during project +# (see README, "Build and test from source"). `managed = false` makes uv skip it during project # discovery, so `uv run` never tries to sync this package. managed = false diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build index d62576c9..339702bd 100644 --- a/scripts/Dockerfile.build +++ b/scripts/Dockerfile.build @@ -39,7 +39,14 @@ FROM arcadedata/arcadedb:${ARCADEDB_TAG} AS java-builder # JDK it runs, so the bundled JRE has the same floor. Both are GPLv2 with the Classpath Exception. FROM eclipse-temurin:25-jdk AS jre-builder +# bash with pipefail for every RUN of this stage: a failing command in the middle of a pipe stops the build +# (hadolint DL4006). The RUNs below were written for sh, so they avoid bashisms that differ (echo with \n). +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + # Install required tooling (findutils for jar exclusion, binutils for jlink) +# DL3008: the package versions are not pinned on purpose. The base image is a rolling Ubuntu tag, and a pinned +# version disappears from the mirror at the next point release, which would break this build for no reason. +# hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends findutils binutils \ && rm -rf /var/lib/apt/lists/* @@ -66,7 +73,7 @@ RUN echo "LOCAL_JARS_HASH=${LOCAL_JARS_HASH}" # Select jar source: local when requested and available; otherwise fall back to upstream image RUN if [ "$USE_LOCAL_JARS" = "1" ]; then \ - if [ -d /build/local-jars ] && [ "$(ls -1 /build/local-jars | wc -l)" -gt 0 ]; then \ + if [ -d /build/local-jars ] && [ "$(find /build/local-jars -mindepth 1 -maxdepth 1 -name '[!.]*' | wc -l)" -gt 0 ]; then \ echo "📦 Using local jars from build context" && cp /build/local-jars/* /build/jars/; \ else \ echo "❌ USE_LOCAL_JARS=1 but no jars found in local-jars/lib" && exit 1; \ @@ -82,7 +89,7 @@ COPY scripts/jre_module_exclusions.txt /build/jre_module_exclusions.txt # Remove excluded JARs based on jar_exclusions.txt RUN echo "🗑️ Removing excluded JARs..." && \ - echo "📋 JAR count before exclusion: $(ls -1 /build/jars/*.jar | wc -l)" && \ + echo "📋 JAR count before exclusion: $(find /build/jars -maxdepth 1 -name '*.jar' | wc -l)" && \ while IFS= read -r pattern; do \ # Skip empty lines and comments if [ -n "$pattern" ] && [ "${pattern#\#}" = "$pattern" ]; then \ @@ -91,15 +98,15 @@ RUN echo "🗑️ Removing excluded JARs..." && \ find /build/jars -name "$pattern" -type f -exec echo " - Removing: {}" \; -delete; \ fi \ done < /build/jar_exclusions.txt && \ - echo "📋 JAR count after exclusion: $(ls -1 /build/jars/*.jar | wc -l)" + echo "📋 JAR count after exclusion: $(find /build/jars -maxdepth 1 -name '*.jar' | wc -l)" # Build the Python-bindings bridge jar (batched row transport; see # src/java/com/arcadedb/python/RowBatcher.java) COPY src/java /build/bridge-src RUN echo "🔨 Building arcadedb-python-bridge.jar..." && \ mkdir -p /build/bridge-classes && \ - javac -cp "/build/jars/*" -d /build/bridge-classes \ - $(find /build/bridge-src -name "*.java") && \ + find /build/bridge-src -name "*.java" -print0 \ + | xargs -0 javac -cp "/build/jars/*" -d /build/bridge-classes && \ jar cf /build/jars/arcadedb-python-bridge.jar -C /build/bridge-classes . && \ echo "✅ Bridge jar built: $(ls -la /build/jars/arcadedb-python-bridge.jar)" @@ -115,16 +122,17 @@ RUN echo "🔨 Building arcadedb-python-bridge.jar..." && \ RUN echo "🔍 Analyzing JARs with jdeps..." && \ JMODS_DIR="${JAVA_HOME}/jmods" && \ if [ ! -d "$JMODS_DIR" ]; then JMODS_DIR="${JAVA_HOME}/lib/jmods"; fi && \ - DETECTED_MODULES=$(find /build/jars -name "*.jar" | grep -v "jboss" | grep -v "wildfly" | grep -v "smallrye" | grep -v "spatial-extras" | xargs jdeps --print-module-deps --ignore-missing-deps --multi-release 25 2>&1 | tr ',' '\n' | sed 's/^ *//;s/ *$//' | grep -E '^[a-zA-Z0-9_.]+$' | sort -u | paste -sd "," -) && \ + RAW_JDEPS_OUTPUT=$(find /build/jars -name "*.jar" | grep -v "jboss" | grep -v "wildfly" | grep -v "smallrye" | grep -v "spatial-extras" | xargs jdeps --print-module-deps --ignore-missing-deps --multi-release 25 2>&1 || true) && \ + DETECTED_MODULES=$(echo "${RAW_JDEPS_OUTPUT}" | tr ',' '\n' | sed 's/^ *//;s/ *$//' | grep -E '^[a-zA-Z0-9_.]+$' | sort -u | paste -sd "," - || true) && \ if [ -d "$JMODS_DIR" ]; then \ AVAILABLE_MODULES=$(find "$JMODS_DIR" -name "*.jmod" -printf "%f\n" | sed 's/\.jmod$//' | sort -u | paste -sd "|" -) ; \ - FILTERED_MODULES=$(echo "${DETECTED_MODULES}" | tr ',' '\n' | sed '/^$/d' | grep -E "^(${AVAILABLE_MODULES})$" | sort -u | paste -sd "," -) ; \ + FILTERED_MODULES=$(echo "${DETECTED_MODULES}" | tr ',' '\n' | sed '/^$/d' | grep -E "^(${AVAILABLE_MODULES})$" | sort -u | paste -sd "," - || true) ; \ else \ FILTERED_MODULES="${DETECTED_MODULES}" ; \ fi && \ grep -v '^#' /build/jre_module_exclusions.txt | tr -d '\r' | sed '/^[[:space:]]*$/d' > /build/jre_excluded_modules.list && \ echo "📦 Excluded modules: $(paste -sd ',' /build/jre_excluded_modules.list)" && \ - FILTERED_MODULES=$(echo "${FILTERED_MODULES}" | tr ',' '\n' | sed '/^$/d' | grep -vxFf /build/jre_excluded_modules.list | paste -sd "," -) && \ + FILTERED_MODULES=$(echo "${FILTERED_MODULES}" | tr ',' '\n' | sed '/^$/d' | grep -vxFf /build/jre_excluded_modules.list | paste -sd "," - || true) && \ if [ -z "${FILTERED_MODULES}" ]; then \ echo "❌ jdeps found no modules; the java.se fallback would bring back the excluded modules, so the build stops" && \ exit 1 ; \ @@ -139,14 +147,14 @@ RUN echo "🔍 Analyzing JARs with jdeps..." && \ echo "" && \ echo "🔨 Running jlink..." && \ if [ -d "$JMODS_DIR" ]; then \ - JLINK_SOURCE="--module-path $JMODS_DIR" ; \ + set -- --module-path "$JMODS_DIR" ; \ echo " modules from ${JMODS_DIR}" ; \ else \ - JLINK_SOURCE="" ; \ + set -- ; \ echo " no jmods directory: linking from the run-time image of ${JAVA_HOME} (JEP 493)" ; \ fi && \ if ! jlink \ - ${JLINK_SOURCE} \ + "$@" \ --add-modules "${REQUIRED_MODULES}" \ --ignore-signing-information \ --strip-debug \ @@ -189,8 +197,14 @@ RUN bash /build/dump_cds_archive.sh /build/jre FROM python:${PYTHON_VERSION}-slim AS python-builder +# bash with pipefail for every RUN of this stage: a failing command in the middle of a pipe stops the build +# (hadolint DL4006). The RUNs below were written for sh, so they avoid bashisms that differ (echo with \n). +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + # Install minimal build tooling (no external JDK needed; we use bundled JRE) -RUN apt-get update && apt-get install -y \ +# DL3008: versions are not pinned on purpose (rolling Debian base image, see the jre-builder stage). +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ curl \ && rm -rf /var/lib/apt/lists/* @@ -256,7 +270,8 @@ RUN if [ -n "${BUILD_VERSION}" ]; then \ export ARCADEDB_VERSION="${BUILD_VERSION}"; \ echo "📌 Using provided version: ${ARCADEDB_VERSION}"; \ else \ - export ARCADEDB_VERSION=$(python3 scripts/extract_version.py --format=pep440 /build/jars); \ + ARCADEDB_VERSION=$(python3 scripts/extract_version.py --format=pep440 /build/jars) && \ + export ARCADEDB_VERSION && \ echo "📦 Extracted version from engine jar: ${ARCADEDB_VERSION}"; \ fi && \ echo "📦 Python package version: ${ARCADEDB_VERSION}" && \ @@ -275,7 +290,7 @@ RUN if [ -n "${BUILD_VERSION}" ]; then \ fi; \ if [ -n "${WHEEL_PLAT}" ]; then \ echo "🏷️ Building wheel with platform tag: ${WHEEL_PLAT}"; \ - uvx --from "build[uv]==1.6.1" pyproject-build --installer uv --wheel --config-setting=--build-option=--plat-name=${WHEEL_PLAT}; \ + uvx --from "build[uv]==1.6.1" pyproject-build --installer uv --wheel "--config-setting=--build-option=--plat-name=${WHEEL_PLAT}"; \ else \ uvx --from "build[uv]==1.6.1" pyproject-build --installer uv --wheel; \ fi; \ @@ -283,12 +298,12 @@ RUN if [ -n "${BUILD_VERSION}" ]; then \ uvx --from "build[uv]==1.6.1" pyproject-build --installer uv --wheel; \ fi && \ if echo "${TARGET_PLATFORM}" | grep -q '^linux-'; then \ - if [ -n "${WHEEL_PLAT}" ] && ! ls /build/dist/*${WHEEL_PLAT}*.whl 1> /dev/null 2>&1; then \ + if [ -n "${WHEEL_PLAT}" ] && ! ls /build/dist/*"${WHEEL_PLAT}"*.whl 1> /dev/null 2>&1; then \ echo "❌ Expected manylinux wheel (${WHEEL_PLAT}) not found"; \ ls -lh /build/dist; \ exit 1; \ fi; \ - WHEEL_FILE=$(ls /build/dist/*.whl | head -n1); \ + WHEEL_FILE=$(find /build/dist -maxdepth 1 -name '*.whl' | sort | head -n1); \ if [ -n "${WHEEL_FILE}" ]; then \ python3 scripts/verify_wheel_platform_tag.py "${WHEEL_FILE}" /build/jre || exit 1; \ fi; \ @@ -304,9 +319,15 @@ FROM python-builder AS export # Stage 5: Test the built wheel FROM python:${PYTHON_VERSION}-slim AS tester +# bash with pipefail for every RUN of this stage: a failing command in the middle of a pipe stops the build +# (hadolint DL4006). The RUNs below were written for sh, so they avoid bashisms that differ (echo with \n). +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + # No external JRE needed in tester; the wheel includes a bundled JRE # Keep build-essential in case pytest or wheels need compilation on some platforms -RUN apt-get update && apt-get install -y \ +# DL3008: versions are not pinned on purpose (rolling Debian base image, see the jre-builder stage). +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ curl \ && rm -rf /var/lib/apt/lists/* @@ -329,7 +350,7 @@ ENV UV_PYTHON_DOWNLOADS=never COPY --from=python-builder /build/tests ./tests/ # Create a quick installation test -RUN echo '#!/usr/bin/env python3\n\ +RUN printf '#!/usr/bin/env python3\n\ import arcadedb\n\ import tempfile\n\ import shutil\n\ diff --git a/scripts/build-native.sh b/scripts/build-native.sh index 3505eaab..b565d7df 100755 --- a/scripts/build-native.sh +++ b/scripts/build-native.sh @@ -161,7 +161,7 @@ JARS_DIR="$PY_BINDINGS_DIR/src/arcadedb/jars" # shellcheck source=stage_jars.sh source "$SCRIPT_DIR/stage_jars.sh" stage_jars "$JARS_DIR" "$JARS_SOURCE" "$ARCADEDB_TAG" -echo -e "${CYAN}📦 $(ls -1 "$JARS_DIR"/*.jar | wc -l | tr -d ' ') JAR files${NC}" +echo -e "${CYAN}📦 $(find "$JARS_DIR" -maxdepth 1 -name '*.jar' | wc -l | tr -d ' ') JAR files${NC}" # Always filter excluded jars, whichever source staged them. apply_jar_exclusions "$JARS_DIR" "$JAR_EXCLUSIONS_FILE" @@ -182,7 +182,7 @@ case "$(uname -s)" in BRIDGE_CP=$(find "$JARS_DIR" -name '*.jar' | paste -sd ':' -) ;; esac -javac -cp "$BRIDGE_CP" -d "$BRIDGE_CLASSES" $(find "$BRIDGE_SRC" -name "*.java") +find "$BRIDGE_SRC" -name "*.java" -print0 | xargs -0 javac -cp "$BRIDGE_CP" -d "$BRIDGE_CLASSES" jar cf "$JARS_DIR/arcadedb-python-bridge.jar" -C "$BRIDGE_CLASSES" . rm -rf "$BRIDGE_CLASSES" echo -e "${GREEN}✅ Bridge jar built${NC}" @@ -206,7 +206,7 @@ if [ ! -d "$JMODS_DIR" ]; then fi if [ -d "$JMODS_DIR" ]; then - AVAILABLE_MODULES=$(cd "$JMODS_DIR" && ls *.jmod 2> /dev/null | sed 's/\.jmod$//' | sort -u | paste -sd "|" -) + AVAILABLE_MODULES=$(for jmod in "$JMODS_DIR"/*.jmod; do if [ -e "$jmod" ]; then basename "$jmod" .jmod; fi; done | sort -u | paste -sd "|" -) if [ -n "$AVAILABLE_MODULES" ]; then FILTERED_MODULES=$(echo "$DETECTED_MODULES" | tr ',' '\n' | sed '/^$/d' | grep -E "^(${AVAILABLE_MODULES})$" | sort -u | paste -sd "," - || true) else @@ -286,7 +286,7 @@ echo -e "${CYAN}📦 Preparing package...${NC}" # Build and copy JRE copy_tree_with_symlinks "$PY_BINDINGS_DIR/temp_jre" "$PY_BINDINGS_DIR/src/arcadedb/jre" "$PYTHON_BIN" -JAR_COUNT=$(ls -1 "$JARS_DIR"/*.jar | wc -l) +JAR_COUNT=$(find "$JARS_DIR" -maxdepth 1 -name '*.jar' | wc -l) echo -e "${GREEN}✅ Package prepared (${JAR_COUNT} JARs + JRE)${NC}" # Step 4: Write version to pyproject.toml @@ -364,12 +364,16 @@ rm -f "$DIST_DIR"/*.whl # Rename wheel to have correct platform tag if needed # (the build may not set it correctly for cross-platform builds) -WHEEL_FILE=$(ls "$DIST_DIR"/*.whl | head -n1) +WHEEL_FILE="" +for wheel in "$DIST_DIR"/*.whl; do + [[ -e "$wheel" ]] && WHEEL_FILE="$wheel" + break +done if [[ -n "$WHEEL_FILE" ]]; then # Extract components from wheel filename WHEEL_NAME=$(basename "$WHEEL_FILE") # arcadedb_jvm-25.10.1-py3-none-any.whl -> arcadedb_jvm-25.10.1-py3-none-PLAT_NAME.whl - NEW_WHEEL_NAME=$(echo "$WHEEL_NAME" | sed "s|-py3-none-any\.whl|-py3-none-${PLAT_NAME}.whl|") + NEW_WHEEL_NAME="${WHEEL_NAME/-py3-none-any.whl/-py3-none-${PLAT_NAME}.whl}" if [[ "$WHEEL_NAME" != "$NEW_WHEEL_NAME" ]]; then mv "$WHEEL_FILE" "$DIST_DIR/$NEW_WHEEL_NAME" echo -e "${CYAN}🏷️ Renamed wheel to: ${YELLOW}${NEW_WHEEL_NAME}${NC}" diff --git a/scripts/build.sh b/scripts/build.sh index 8aa15f3d..3db32d4b 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -150,7 +150,7 @@ if [[ -z "$PLATFORM" ]]; then fi # ArcadeDB image tag: `latest` follows upstream's current snapshot; set -# ARCADEDB_TAG (e.g. 26.11.1-SNAPSHOT) to pin one. The package version is read +# ARCADEDB_TAG (e.g. 26.12.1-SNAPSHOT) to pin one. The package version is read # from the engine jar inside the image, not from this tag. DOCKER_TAG="${ARCADEDB_TAG:-latest}" echo -e "${CYAN}📌 Docker tag: ${YELLOW}${DOCKER_TAG}${NC}" @@ -158,8 +158,8 @@ echo "" # Select jar source: explicit directory when provided; otherwise pull from ArcadeDB image LOCAL_JARS_DIR="$PY_BINDINGS_DIR/local-jars/lib" -USE_LOCAL_JARS_ARG="" -LOCAL_JARS_HASH_ARG="" +# Extra build arguments of the two docker build calls below (empty: the jars of the ArcadeDB image, no BUILD_VERSION) +DOCKER_BUILD_ARGS=() JAR_SOURCE_DESC="ArcadeDB image" mkdir -p "$LOCAL_JARS_DIR" @@ -192,13 +192,24 @@ if [[ -n "$JAR_LIB_DIR" ]]; then JAR_SOURCE_DESC="${JAR_LIB_DIR} (staged into local-jars)" fi - USE_LOCAL_JARS_ARG="--build-arg USE_LOCAL_JARS=1" + DOCKER_BUILD_ARGS+=(--build-arg USE_LOCAL_JARS=1) LOCAL_JARS_HASH=$(find "$LOCAL_JARS_DIR" -maxdepth 1 -name "*.jar" -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}') - LOCAL_JARS_HASH_ARG="--build-arg LOCAL_JARS_HASH=${LOCAL_JARS_HASH}" + DOCKER_BUILD_ARGS+=(--build-arg "LOCAL_JARS_HASH=${LOCAL_JARS_HASH}") else echo -e "${CYAN}📦 Using JARs from ArcadeDB image (no JAR_LIB_DIR provided)${NC}" fi +# The newest wheel of dist/ by modification time, without parsing `ls`. Fails when dist/ holds no wheel. +newest_wheel() { + local newest="" wheel + for wheel in dist/*.whl; do + [[ -e "$wheel" ]] || continue + if [[ -z "$newest" || "$wheel" -nt "$newest" ]]; then newest="$wheel"; fi + done + [[ -n "$newest" ]] || return 1 + printf '%s\n' "$newest" +} + # Determine build method: native or Docker # Use native build if we're already on the target platform CURRENT_OS="$(uname -s)" @@ -296,10 +307,9 @@ else echo -e "${YELLOW} 🐳 Building in Docker...${NC}" # Check if BUILD_VERSION is set (from CI/CD) - BUILD_VERSION_ARG="" if [ -n "${BUILD_VERSION:-}" ]; then echo -e "${CYAN}📌 Using specified version: ${YELLOW}${BUILD_VERSION}${NC}" - BUILD_VERSION_ARG="--build-arg BUILD_VERSION=$BUILD_VERSION" + DOCKER_BUILD_ARGS+=(--build-arg "BUILD_VERSION=$BUILD_VERSION") fi # Convert platform format: linux/amd64 -> linux-x64, linux/arm64 -> linux-arm64, etc. @@ -329,9 +339,7 @@ else --build-arg PACKAGE_DESCRIPTION="$DESCRIPTION" \ --build-arg ARCADEDB_TAG="$DOCKER_TAG" \ --build-arg TARGET_PLATFORM="$TARGET_PLATFORM" \ - $USE_LOCAL_JARS_ARG \ - $LOCAL_JARS_HASH_ARG \ - $BUILD_VERSION_ARG \ + ${DOCKER_BUILD_ARGS[@]+"${DOCKER_BUILD_ARGS[@]}"} \ --target export \ -t arcadedb-python-package-export \ -f "$SCRIPT_DIR/Dockerfile.build" \ @@ -346,9 +354,7 @@ else --build-arg PACKAGE_DESCRIPTION="$DESCRIPTION" \ --build-arg ARCADEDB_TAG="$DOCKER_TAG" \ --build-arg TARGET_PLATFORM="$TARGET_PLATFORM" \ - $USE_LOCAL_JARS_ARG \ - $LOCAL_JARS_HASH_ARG \ - $BUILD_VERSION_ARG \ + ${DOCKER_BUILD_ARGS[@]+"${DOCKER_BUILD_ARGS[@]}"} \ --target tester \ -t arcadedb-python-package \ -f "$SCRIPT_DIR/Dockerfile.build" \ @@ -366,23 +372,23 @@ else # this by clearing the directory first; the Docker path had no equivalent. # Compare identities rather than clearing, so deliberately kept wheels for # other platforms survive a build. - WHEELS_BEFORE=$(ls -1 dist/*.whl 2> /dev/null | sort || true) + WHEELS_BEFORE=$(find dist -maxdepth 1 -name '*.whl' 2> /dev/null | sort || true) # Extract the wheel from the export container echo -e "${CYAN}📋 Extracting wheel file...${NC}" CONTAINER_ID=$(docker create arcadedb-python-package-export) - docker cp ${CONTAINER_ID}:/build/dist/. ./dist/ - docker rm ${CONTAINER_ID} + docker cp "${CONTAINER_ID}:/build/dist/." ./dist/ + docker rm "${CONTAINER_ID}" # Verify THIS build produced a wheel, not merely that dist/ contains one - WHEELS_AFTER=$(ls -1 dist/*.whl 2> /dev/null | sort || true) + WHEELS_AFTER=$(find dist -maxdepth 1 -name '*.whl' 2> /dev/null | sort || true) NEW_WHEELS=$(comm -13 <(echo "$WHEELS_BEFORE") <(echo "$WHEELS_AFTER") || true) if [[ -n "$NEW_WHEELS" ]]; then echo -e "${GREEN}✅ Wheel file created successfully!${NC}" elif [[ -n "$WHEELS_AFTER" ]]; then # Same filename as before: only trust it if the export actually rewrote # it. A rebuild of the same version legitimately lands here. - NEWEST=$(ls -t dist/*.whl | head -n1) + NEWEST=$(newest_wheel) if [[ "$NEWEST" -nt "$BUILD_START_MARKER" ]]; then echo -e "${GREEN}✅ Wheel file created successfully!${NC}" elif [[ -n "$JAR_LIB_DIR" ]]; then @@ -436,7 +442,7 @@ ARCADEDB_VERSION = os.environ["ARCADEDB_VERSION"] # (BUILD_VERSION when set, as the wheel's own version follows it), and assert # the version, so picking the wrong file fails loudly. # The wheel version's base, by extract_version.py's maven_to_pep440 rules: -# 26.11.1-SNAPSHOT -> 26.11.1 (.devN follows), 26.11.1-RC1 -> 26.11.1rc1 (#7). +# 26.12.1-SNAPSHOT -> 26.12.1 (.devN follows), 26.12.1-RC1 -> 26.12.1rc1 (#7). _want = (os.environ.get("BUILD_VERSION") or ARCADEDB_VERSION).replace("-SNAPSHOT", "") _want = _want.replace("-RC", "rc").replace("-", "") @@ -503,7 +509,7 @@ echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━ if [[ -n "${NEW_WHEELS:-}" ]]; then NEWEST_WHEEL=$(echo "$NEW_WHEELS" | head -n1) else - NEWEST_WHEEL=$(ls -t dist/*.whl | head -n1) + NEWEST_WHEEL=$(newest_wheel) fi NEWEST_TAG=$(basename "$NEWEST_WHEEL" | sed -E 's/^[^-]+-[^-]+-//') for old in dist/*.whl; do @@ -542,7 +548,7 @@ echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━ echo "" echo -e "${CYAN}📦 Built package:${NC}" if [ -d "dist" ]; then - ls -lh dist/*.whl 2> /dev/null | awk '{print " " $9 " (" $5 ")"}' + find dist -maxdepth 1 -name '*.whl' -exec ls -lh {} + 2> /dev/null | awk '{print " " $9 " (" $5 ")"}' fi echo "" diff --git a/scripts/run_examples.sh b/scripts/run_examples.sh index df82014a..92e2eab8 100755 --- a/scripts/run_examples.sh +++ b/scripts/run_examples.sh @@ -20,10 +20,22 @@ set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT/examples" +# The files that the space-separated globs in EXAMPLES match, one per line (a glob without a match adds nothing). +list_examples() { + local -a globs + local glob + # EXAMPLES is an input of the workflows (see the header), not a misspelling of the local $examples + # shellcheck disable=SC2153 + read -r -a globs <<< "$EXAMPLES" + for glob in "${globs[@]}"; do + compgen -G "$glob" || true + done +} + datasets() { # Determine which examples are selected - examples=$(ls $EXAMPLES 2> /dev/null || true) + examples=$(list_examples) # EXIT 75 MEANS "THE DATASET HOST IS DOWN", NOT "THE BINDINGS BROKE". # @@ -86,8 +98,9 @@ datasets() { # 04_csv_import_documents.py --dataset movielens-small against data # that was never fetched -- the job still fails, just later and with a # worse error. Caught in review on #6889. - echo "SKIPPED_DATASETS=$(echo $SKIPPED_DATASETS | xargs)" >> "$GITHUB_ENV" - if [ -n "$(echo $SKIPPED_DATASETS | xargs)" ]; then + skipped_datasets="$(echo "$SKIPPED_DATASETS" | xargs)" + echo "SKIPPED_DATASETS=$skipped_datasets" >> "$GITHUB_ENV" + if [ -n "$skipped_datasets" ]; then echo "::notice title=Datasets unavailable::skipping examples that need:$SKIPPED_DATASETS" fi } @@ -108,7 +121,7 @@ run_examples() { true > "$results_file" # Select example files by pattern(s) in EXAMPLES (space-separated globs) - examples=$(ls $EXAMPLES 2> /dev/null | sort) + examples=$(list_examples | sort) if [ -z "$examples" ]; then echo "❌ No example files found!" @@ -339,15 +352,18 @@ run_examples() { echo "📝 Running: $example_name" echo "----------------------------------------" + # The argument string of the example, split on spaces into words (no globbing) + read -r -a example_argv <<< "$example_args" + # Run the example with appropriate parameters if [ -n "$example_jvm_args" ]; then - if ARCADEDB_JVM_ARGS="$example_jvm_args" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" $example_args > "$log_file" 2>&1; then + if ARCADEDB_JVM_ARGS="$example_jvm_args" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" "${example_argv[@]}" > "$log_file" 2>&1; then exit_code=0 else exit_code=$? fi else - if ARCADEDB_JVM_ARGS="" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" $example_args > "$log_file" 2>&1; then + if ARCADEDB_JVM_ARGS="" JAVA_TOOL_OPTIONS="" _JAVA_OPTIONS="" $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" "${example_argv[@]}" > "$log_file" 2>&1; then exit_code=0 else exit_code=$? @@ -360,8 +376,9 @@ run_examples() { if [ $exit_code -eq 0 ] && [ "$example" = "10_stackoverflow_graph_olap.py" ]; then ref_args="${example_args/--db arcadedb_cypher/--db python_memory}" ref_args="${ref_args/--run-label ci10/--run-label ci10ref}" + read -r -a ref_argv <<< "$ref_args" ref_log="${example%.py}-reference.log" - if ! $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" $ref_args > "$ref_log" 2>&1; then + if ! $TIMEOUT_CMD $timeout_duration "$EX_PY" "$example" "${ref_argv[@]}" > "$ref_log" 2>&1; then echo "❌ reference run (python_memory) of $example failed; last 20 lines:" tail -n 20 "$ref_log" exit_code=1 @@ -402,10 +419,12 @@ run_examples() { echo "Examples pattern(s): $EXAMPLES" # Output to GitHub Actions - echo "total=$total" >> $GITHUB_OUTPUT - echo "passed=$passed" >> $GITHUB_OUTPUT - echo "failed=$failed" >> $GITHUB_OUTPUT - echo "skipped=$skipped" >> $GITHUB_OUTPUT + { + echo "total=$total" + echo "passed=$passed" + echo "failed=$failed" + echo "skipped=$skipped" + } >> "$GITHUB_OUTPUT" # Show detailed results echo "Detailed Results:"