diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 47ee26c91..d451f9ab2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,21 +20,12 @@ jobs: with: persist-credentials: false - - name: 'Validate that template/* are used to generate Dockerfiles' + name: 'Validate that the generated files are up to date' run: |- - docker run --rm -v $PWD:/app -w /app webdevops/dockerfile-build-env make provision - git diff --exit-code --color=always - - - name: 'Validate .github/workflows/build.yaml is up to date' - run: |- - docker run --rm -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine composer install --no-dev - docker run --rm -v $PWD:/app -w /app webdevops/php:8.4-alpine ci/console github:generate-ci - git diff --exit-code --color=always - - - name: 'Validate the README banner is up to date' - run: |- - docker run --rm -v $PWD:/app -w /app webdevops/php:8.4-alpine ci/console readme-banner:generate + make generate + git status --porcelain git diff --exit-code --color=always + test -z "$(git status --porcelain)" check-ci-code: name: 'Check ci/ (GrumPHP and PHPUnit)' runs-on: ubuntu-latest @@ -45,9 +36,7 @@ jobs: persist-credentials: false - name: 'Run GrumPHP' - run: |- - docker run --rm -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine composer install - docker run --rm -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine sh -c 'apk add --no-cache file && git config --global --add safe.directory /app && vendor/bin/grumphp run --no-interaction' + run: 'make check' apache_alpine: strategy: fail-fast: false diff --git a/.gitignore b/.gitignore index d64af3b3b..a983e610a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,9 @@ /.idea +.DS_Store /.cache /.vscode /tests/serverspec/vendor /tests/serverspec/.bundle/ /tests/serverspec/Dockerfile* -/BLACKLIST +/ci/BLACKLIST *.log -*.pyc -.doit.db diff --git a/AGENTS.md b/AGENTS.md index 6b3777400..696ed27ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,32 +4,31 @@ | Path | Content | | --- | --- | -| `template/` | Shared Jinja2 templates for the Dockerfiles (source) | -| `docker///Dockerfile.jinja2` | Per-image template (source) | +| `template/` | Shared Twig templates for the Dockerfiles: the layout, which writes the header, and the macros (source) | +| `docker///Dockerfile.twig` | Per-image Twig template; the layout includes it, and it imports the macros it uses from `template/Dockerfile/` (source) | | `provisioning/` | Scripts and configuration files copied into the images (source); `conf/provision.yml` says which go where | -| `docker///Dockerfile`, `docker///conf/` | Generated by `make provision`, never edited by hand | -| `bin/` | Old Python 2 tools behind `make provision` | -| `ci/` | PHP generators for `.github/workflows/build.yaml` (`ci/console github:generate-ci`) and the README banner (`ci/console readme-banner:generate`) | +| `docker///Dockerfile`, `docker///conf/` | Generated by `ci/console generate`, never edited by hand | +| `ci/` | PHP generators behind `ci/console generate`: the Dockerfiles (`generate:dockerfile`), their `conf/` (`generate:provision`), `.github/workflows/build.yaml` (`generate:github-ci`) and the README banner (`generate:readme-banner`) | | `ci/tests/` | PHPUnit tests of `ci/src`; `ci/tests/Fixtures/repository` is a small repository they run against | | `ci/grumphp.yml`, `ci/phpstan.neon`, `ci/rector.php`, `ci/fractor.php` | Checks of the PHP code in `ci/`, from pluswerk/grumphp-config | | `conf/readme-banner.yml`, `ci/resources/readme-banner/` | Texts and font outlines of the README banner (source) | -| `docs/assets/readme-banner-*.svg` | README banner, generated by `ci/console readme-banner:generate`, never edited by hand | +| `docs/assets/readme-banner-*.svg` | README banner, generated by `ci/console generate:readme-banner`, never edited by hand | | `tests/` | serverspec (Ruby) and container-structure-test definitions run by the CI | | `docs/` | User documentation | -`docker/php/` are symlinks to `docker/php-official/`; both images are built from the same Dockerfiles. The tags that are built are the `-t "webdevops/..."` lines in `.github/workflows/build.yaml`. +`docker/php/` are symlinks to `docker/php-official/`, kept by `ci/console generate:dockerfile`; both images are built from the same Dockerfiles. The tags that are built are the `-t "webdevops/..."` lines in `.github/workflows/build.yaml`. ## Build and test Only these commands are known to work: -- `docker run --rm -v $PWD:/app -w /app webdevops/dockerfile-build-env make provision` regenerates `docker/`. Run it after every change to `template/`, `provisioning/` or a `Dockerfile.jinja2`, and commit the result. +- `make generate` installs the PHP tools of `ci/` and runs `ci/console generate` in `webdevops/php:8.4-alpine`, which regenerates `docker/`, `.github/workflows/build.yaml` and the README banner. Run it after every change to `template/`, `provisioning/`, `conf/` or a `Dockerfile.twig`, and commit the result; the CI fails if a generated file is out of date or missing. - `docker build -t webdevops/: docker//` builds one image; its parent and `webdevops/toolbox` come from local images or Docker Hub. - `./build-local.sh [version]` builds all PHP images of one version. -- `docker run --rm -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine composer install` installs the PHP tools of `ci/`. -- `docker run --rm -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine sh -c 'apk add --no-cache file && git config --global --add safe.directory /app && vendor/bin/grumphp run'` checks the PHP code in `ci/`: coding standard, PHPStan, Rector, `composer audit` and the PHPUnit tests. Run it after every change to `ci/`. GrumPHP and PHPStan only check files that Git knows (`git ls-files`), so `git add` new files first; PHPUnit runs every test anyway. `vendor/bin/phpunit` instead of `vendor/bin/grumphp run` runs only the tests. Changes to `ci/src` come with tests. +- `make check` installs the PHP tools of `ci/` and checks the PHP code in `ci/` with GrumPHP: coding standard, PHPStan, Rector, `composer audit` and the PHPUnit tests. Run it after every change to `ci/`. GrumPHP and PHPStan only check files that Git knows (`git ls-files`), so `git add` new files first; PHPUnit runs every test anyway. `make test` runs only the tests. Changes to `ci/src` come with tests. +- `make generate`, `make check` and `make test` run as your user, so the files they write belong to you. Do not write into the checkout as root. -Besides `provision`, the Makefile has `structure-test` and `setup`, which the `webdevops/dockerfile-build-env` image runs while it is built. `make structure-test` runs container-structure-test against your local images of `bootstrap`, `base`, `base-app`, `php`, `php-dev` and `php-nginx`, the images that have a suite in `tests/structure-test`. It tests only the tags in `.github/workflows/build.yaml` and fails if a test fails, if `docker` fails or if no such image exists locally; run it inside `webdevops/dockerfile-build-env` with the Docker socket mounted. The CI runs serverspec for every image except `toolbox` and `dockerfile-build-env`, and container-structure-test for the six images above; to check a change by hand, start the image and try it as described below. +Besides `generate`, `check` and `test`, the Makefile has `structure-test`. `make structure-test` runs container-structure-test against your local images of `bootstrap`, `base`, `base-app`, `php`, `php-dev` and `php-nginx`, the images that have a suite in `tests/structure-test`. It tests only the tags in `.github/workflows/build.yaml` and fails if a test fails, if `docker` fails or if no such image exists locally; run it inside `webdevops/dockerfile-build-env` with the Docker socket mounted. The CI runs serverspec for every image except `toolbox` and `dockerfile-build-env`, and container-structure-test for the six images above; to check a change by hand, start the image and try it as described below. ### Packages come with tests @@ -53,7 +52,7 @@ Where the documentation lives: | `docs/environment-variables.md` | Every environment variable with its default and effect; uploads and long requests | | `docs/customization.md` | Commands, scripts at start and build time, tools for your Dockerfile, services, cron jobs and their time zone, healthcheck, configuration files and placeholders, supervisord, locales, application user and changing its UID | | `docs/images/*.md` | One page per image family: tags, base image, services, settings, development variant. `php.md` also has Debian or Alpine, production, installing extensions, mail and Xdebug with an IDE | -| `docs/known-issues.md` | Options and tools that do not work, with affected images and workaround; migration from older images; build tools that no longer work | +| `docs/known-issues.md` | Options and tools that do not work, with affected images and workaround; migration from older images | | `docs/discontinued-images.md` | Replacements for images that are no longer built (`storage`, `vsftp`, `ssh`) | ### When to update it @@ -73,12 +72,12 @@ Update the documentation whenever a change touches any of these: | Question | Where to look | | --- | --- | - | Published images and tags | `-t "webdevops/..."` tags in `.github/workflows/build.yaml` (generated by `ci/console github:generate-ci` from `conf/console.yml` and `ci/`) | + | Published images and tags | `-t "webdevops/..."` tags in `.github/workflows/build.yaml` (generated by `ci/console generate:github-ci` from `conf/console.yml` and `ci/`) | | Base image | `FROM` line in `docker///Dockerfile` | - | Variables and defaults | `ENV` in `template/Dockerfile/**` and `docker/**/Dockerfile.jinja2`, scripts in `provisioning/**/bin` and `provisioning/**/provision` | + | Variables and defaults | `ENV` in `template/Dockerfile/**` and `docker/**/Dockerfile.twig`, scripts in `provisioning/**/bin` and `provisioning/**/provision` | | Which files end up in which image | `conf/provision.yml` | -2. **Change sources, not generated files.** `Dockerfile` and `conf/` under `docker/` are generated. Edit `Dockerfile.jinja2`, `template/` or `provisioning/` and run `make provision`. +2. **Change sources, not generated files.** `Dockerfile` and `conf/` under `docker/` are generated. Edit `Dockerfile.twig`, `template/` or `provisioning/` and run `make generate`. 3. **Verify every statement in a running container.** Build the image (or use the image the CI built) and try the option. Do not rely on reading the scripts alone. Examples: - PHP settings: `php -r 'echo ini_get("memory_limit");'`, or `php-fpm -i` for values the CLI overrides @@ -91,7 +90,7 @@ Update the documentation whenever a change touches any of these: 4. **Document only what works.** Never describe an option that has no effect as working. Fix it, or leave it out and list it in `docs/known-issues.md`. If it works only in some images, say exactly which. Anything `docs/known-issues.md` lists as not working must never appear as working anywhere else. An `ENV` line or a script that reads a variable is no proof that it works: `SERVICE_POSTFIX_OPTS` is passed to `postfix start`, which ignores it. When you fix a known issue, remove its entry in the same pull request. -5. **Keep lists in sync.** The tag lists in `README.md` and in `docs/images/*.md` must match the tags pushed by `.github/workflows/build.yaml`. The README banner follows the built tags by itself when you run `ci/console readme-banner:generate`; change its texts in `conf/readme-banner.yml`. `webdevops/php-official` is not listed in the README; it is built from the same Dockerfiles as `webdevops/php`. The defaults in `docs/environment-variables.md` must match what a container reports. +5. **Keep lists in sync.** The tag lists in `README.md` and in `docs/images/*.md` must match the tags pushed by `.github/workflows/build.yaml`. The README banner follows the built tags by itself when you run `ci/console generate:readme-banner`; change its texts in `conf/readme-banner.yml`. `webdevops/php-official` is not listed in the README; it is built from the same Dockerfiles as `webdevops/php`. The defaults in `docs/environment-variables.md` must match what a container reports. 6. **Follow the style.** - English, GitHub Markdown, relative links, diagrams as Mermaid. No documentation tooling. diff --git a/Makefile b/Makefile index 486234952..c87d3d859 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,27 @@ MAKEFLAGS += --silent -.PHONY: list setup requirements structure-test provision +.PHONY: generate check test structure-test -list: - sh -c "echo; $(MAKE) -p no_targets__ 2>/dev/null |\ - awk -F':' '/^[a-zA-Z0-9][^\$$#\\t=]*:([^=]|$$)/ {split(\$$1,A,/ /);\ - for(i in A)print A[i]}' | \ - grep -v '__\$$\|Makefile\|%' | \ - sort" +# The containers run as the user who calls make, so that the files they write belong to them. In rootless Docker, +# root in the container already is that user. sh replaces the entrypoint of webdevops/php, which would switch to root. +ROOTLESS = $(findstring rootless,$(shell docker info --format '{{.SecurityOptions}}' 2>/dev/null)) +USER_ID = $(shell id -u):$(shell id -g) +DOCKER_USER = $(if $(ROOTLESS),,--user $(USER_ID)) +GOSU = $(if $(ROOTLESS),,gosu $(USER_ID)) +# git trusts the checkout without a write to a .gitconfig +GIT_TRUST = -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=/app -setup: requirements +# Installs the PHP tools of ci/ and writes every generated file, in webdevops/php:8.4-alpine +generate: + docker run --rm $(DOCKER_USER) -e COMPOSER_HOME=/tmp/composer -v "$(CURDIR)":/app -w /app --entrypoint sh webdevops/php:8.4-alpine -c 'composer install --working-dir=ci && ci/console generate' -requirements: - pip install --upgrade -I -r ./bin/requirements.txt - cd tests/serverspec && bundle install --path=vendor +# Installs the PHP tools of ci/ and checks the PHP code in ci/ with GrumPHP. Only apk runs as root, for the "file" +# command of GrumPHP's BOM check. +check: + docker run --rm $(GIT_TRUST) -e COMPOSER_HOME=/tmp/composer -v "$(CURDIR)":/app -w /app/ci --entrypoint sh webdevops/php:8.4-alpine -c 'apk add --no-cache file >/dev/null && exec $(GOSU) sh -c "composer install && vendor/bin/grumphp run --no-interaction"' + +# Runs only the PHPUnit tests of ci/ +test: + docker run --rm $(DOCKER_USER) -e COMPOSER_HOME=/tmp/composer -v "$(CURDIR)":/app -w /app/ci --entrypoint sh webdevops/php:8.4-alpine -c 'composer install && vendor/bin/phpunit' structure-test: cd tests/structure-test && ./run.sh - -provision: - python2 bin/console generate:dockerfile - python2 bin/console generate:provision diff --git a/README.md b/README.md index 930a1053f..d82f3f5a0 100644 --- a/README.md +++ b/README.md @@ -45,43 +45,46 @@ Or join our discord https://discord.gg/BpBZtXAgJz ### File generation -The general build process is currently a mixture of python jinja2 legacy and PHP. - -First we build the files inside the `docker` directory using this command: +The files inside the `docker` directory, the GitHub Actions workflow (`.github/workflows/build.yaml`) and the banner at the top of this README are generated with PHP: ``` -docker run --rm -ti -v $PWD:/app -w /app webdevops/dockerfile-build-env make provision -``` - -It will use the Jinja2 templates inside of the `template` directory and the -config files from the `provisioning` directory. - -**The generated files in the `docker` directory (`Dockerfile` and `conf/`) are never modified manually.** -Edit the `Dockerfile.jinja2` templates next to them, the shared templates in `template` or the files in `provisioning` instead. -`make provision` copies the `provisioning` files into the `conf/` directories as configured in `conf/provision.yml`. +make generate +``` -*This will be replaced with PHP twig templates in the future to streamline everything.* +It installs the PHP tools of `ci/` with Composer and runs `ci/console generate` in `webdevops/php:8.4-alpine`, so you only need `make` and Docker. +The container runs as your user, so the files it writes belong to you. +`ci/console generate` runs these commands in this order; each of them also runs on its own: + +| Command | Writes | +| --- | --- | +| `ci/console generate:dockerfile` | `docker///Dockerfile` from the `Dockerfile.twig` next to it and the Twig templates in the `template` directory, and the links `docker/php/` to `docker/php-official/` | +| `ci/console generate:provision` | `docker///conf/` from the `provisioning` directory, as configured in `conf/provision.yml` | +| `ci/console generate:github-ci` | `.github/workflows/build.yaml`, a build and a publish job for every Dockerfile | +| `ci/console generate:readme-banner` | `docs/assets/readme-banner-*.svg` | + +**The generated files are never modified manually.** +Edit the `Dockerfile.twig` templates, the shared templates in `template`, the files in `provisioning` or the settings in `conf` instead. +`template/Dockerfile/layout.twig` writes the header of every Dockerfile and includes its `Dockerfile.twig`. +A `Dockerfile.twig` imports the macros it uses from `template/Dockerfile`, e.g. `{% import 'docker.twig' as docker %}`; a macro imports the macros it calls itself. +`generate:provision` empties the `conf/` directories of an image first, so a file removed from `provisioning` disappears there too. +It checks all of `conf/provision.yml` before it changes a file and stops if a source directory or a tag does not exist, or if an image has another setting than `configuration`. +`generate:dockerfile` stops if a directory `docker//` has files but no `Dockerfile.twig`; it first removes the `.DS_Store` files of the macOS Finder under `docker`. +To stop building a tag, remove its directory and those of the tags that build on it; `generate:github-ci` names any it finds. `generate:dockerfile` adds and removes the links in `docker/php` along with the `php-official` tags. ### Building in CI -We generate the GitHub Actions workflow (`.github/workflows/build.yaml`) and the banner at the top of this README using PHP: - -``` -docker run --rm -ti -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine composer install -docker run --rm -ti -v $PWD:/app -w /app webdevops/php:8.4-alpine ci/console github:generate-ci -docker run --rm -ti -v $PWD:/app -w /app webdevops/php:8.4-alpine ci/console readme-banner:generate -``` - The banner (`docs/assets/readme-banner-*.svg`) takes the PHP versions, web servers, base OS suffixes, Debian releases and platforms from the images that are built. -Its texts are in `conf/readme-banner.yml`; the SVG files are never modified manually. The CI fails if the workflow or the banner is out of date. +Its texts are in `conf/readme-banner.yml`. The CI fails if a generated file is out of date or missing. The PHP code in `ci/` follows the rules of [pluswerk/grumphp-config](https://github.com/andersundsehr/grumphp-config) (PSR-12, PHPStan level 8, Rector, `composer audit`) and has PHPUnit tests in `ci/tests`. GrumPHP runs all of them; after a change to `ci/`, run it with: ``` -docker run --rm -ti -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine sh -c 'apk add --no-cache file && git config --global --add safe.directory /app && vendor/bin/grumphp run' +make check ``` +It installs the PHP tools of `ci/` and runs GrumPHP as your user. `make test` runs only the PHPUnit tests. + The CI job "Check ci/" runs the same checks. No image build waits for it. GitHub Actions builds every image independently and runs the serverspec and structure tests @@ -102,8 +105,16 @@ Build all PHP images of one version (default `8.5`), in the order they depend on ./build-local.sh 8.4 ``` -Then try an image with the quick start above. `make provision` calls `bin/console`, which needs Python 2; -run it inside `webdevops/dockerfile-build-env`. See [Known issues](docs/known-issues.md#build-tools). +Then try an image with the quick start above. + +The CI runs the serverspec tests for every image except `toolbox` and `dockerfile-build-env`, and the structure tests for `bootstrap`, `base`, `base-app`, `php`, `php-dev` and `php-nginx`. +To run the structure tests against the images you built locally: + +``` +docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:/app -w /app webdevops/dockerfile-build-env make structure-test +``` + +It tests the local tags that the CI builds and fails if no such image exists. ### PHP extension builds diff --git a/bin/__init__.py b/bin/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/bin/command/__init__.py b/bin/command/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/bin/command/docker_build_command.py b/bin/command/docker_build_command.py deleted file mode 100644 index 0a0e08f99..000000000 --- a/bin/command/docker_build_command.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from cleo import Output -from webdevops import Dockerfile -from webdevops.command import DoitCommand -from webdevops.taskloader import DockerBuildTaskLoader - -class DockerBuildCommand(DoitCommand): - """ - Build images - - docker:build - {docker images?* : Docker images (whitelist)} - {--dry-run : show only which images will be build} - {--no-cache : build without caching} - {--t|threads=0 : threads} - {--r|retry=0 : retry} - {--whitelist=?* : image/tag whitelist } - {--blacklist=?* : image/tag blacklist } - """ - - def run_task(self, configuration): - configuration.set('dockerBuild.noCache', self.option('no-cache')) - - return self.run_doit( - task_loader=DockerBuildTaskLoader(configuration), - configuration=configuration - ) - - - diff --git a/bin/command/docker_exec_command.py b/bin/command/docker_exec_command.py deleted file mode 100644 index 0f84f7627..000000000 --- a/bin/command/docker_exec_command.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from cleo import Output -from termcolor import colored -from webdevops.command import DoitCommand -from webdevops.taskloader import DockerPushTaskLoader -from webdevops import Command, DockerfileUtility - -class DockerExecCommand(DoitCommand): - """ - Push images to registry/hub - - docker:exec - {docker command* : Command} - {--dry-run : show only which images will be build} - {--whitelist=?* : image/tag whitelist } - {--blacklist=?* : image/tag blacklist } - """ - - def run_task(self, configuration): - dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( - base_path=configuration.get('dockerPath'), - path_regex=configuration.get('docker.pathRegex'), - image_prefix=configuration.get('docker.imagePrefix'), - whitelist=configuration.get('whitelist'), - blacklist=configuration.get('blacklist'), - ) - - image_fail_list = [] - - docker_command = self.argument('docker command') - - for dockerfile in dockerfile_list: - title = dockerfile['image']['fullname'] - - print title - print '~' * len(title) - - if configuration['dryRun']: - print ' exec: %s' % (docker_command) - else: - - cmd = [ - 'docker', - 'run', - '-t', - '--rm', - dockerfile['image']['fullname'], - 'bash', - '-c', - '%s' % (' '.join(docker_command)) - ] - status = Command.execute(cmd) - - if status: - print colored(' -> successfull', 'green') - else: - print colored(' -> failed', 'red') - image_fail_list.append(dockerfile['image']['fullname']) - print '' - print '' - - if len(image_fail_list) >= 1: - print '' - print colored(' => failed images (%s):' % (str(len(image_fail_list))), 'red') - for image in image_fail_list: - print ' - %s ' % image - print '' - - return False - else: - return True - - - - diff --git a/bin/command/docker_pull_command.py b/bin/command/docker_pull_command.py deleted file mode 100644 index e6f960b47..000000000 --- a/bin/command/docker_pull_command.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from cleo import Output -from webdevops.command import DoitCommand -from webdevops.taskloader import DockerPullTaskLoader - -class DockerPullCommand(DoitCommand): - """ - Pull all built images from registry/hub - - docker:pull - {docker images?* : Docker images (whitelist)} - {--dry-run : show only which images will be build} - {--t|threads=0 : threads} - {--r|retry=0 : retry} - {--whitelist=?* : image/tag whitelist } - {--blacklist=?* : image/tag blacklist } - """ - - def run_task(self, configuration): - return self.run_doit( - task_loader=DockerPullTaskLoader(configuration), - configuration=configuration - ) - - - diff --git a/bin/command/docker_push_command.py b/bin/command/docker_push_command.py deleted file mode 100644 index 85912b5b7..000000000 --- a/bin/command/docker_push_command.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from cleo import Output -from webdevops.command import DoitCommand -from webdevops.taskloader import DockerPushTaskLoader - -class DockerPushCommand(DoitCommand): - """ - Push images to registry/hub - - docker:push - {docker images?* : Docker images (whitelist)} - {--dry-run : show only which images will be build} - {--t|threads=0 : threads} - {--r|retry=0 : retry} - {--whitelist=?* : image/tag whitelist } - {--blacklist=?* : image/tag blacklist } - """ - - def run_task(self, configuration): - return self.run_doit( - task_loader=DockerPushTaskLoader(configuration), - configuration=configuration - ) - - - - diff --git a/bin/command/generate_dockerfile_command.py b/bin/command/generate_dockerfile_command.py deleted file mode 100644 index b5a3710d8..000000000 --- a/bin/command/generate_dockerfile_command.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os -from cleo import Output -from jinja2 import Environment, FileSystemLoader -from webdevops import DockerfileUtility -from webdevops.command import BaseCommand - -class GenerateDockerfileCommand(BaseCommand): - """ - Build Dockerfile containers - - generate:dockerfile - {docker images?* : Docker images (whitelist)} - {--whitelist=?* : image/tag whitelist } - {--blacklist=?* : image/tag blacklist } - """ - - template = '' - - template_header = '{% extends "Dockerfile/layout.jinja2" %}\n{% block content %}' - template_footer = '{% endblock %}' - - def run_task(self, configuration): - template_path = configuration.get('templatePath') - dockerfile_path = configuration.get('dockerPath') - whitelist = self.get_whitelist() - blacklist = self.get_blacklist() - - if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): - self.line('-> docker path : %s' % dockerfile_path) - self.line('-> template path : %s' % template_path) - - if whitelist: - self.line('-> whitelist :') - for crit in whitelist: - self.line("\t * %s" % crit) - - if blacklist: - self.line('-> blacklist :') - for crit in blacklist: - self.line("\t * %s" % crit) - - self.template = Environment( - autoescape=False, - loader=FileSystemLoader([template_path]), - trim_blocks=False - ) - - for file in DockerfileUtility.find_file_in_path(dockerfile_path=dockerfile_path, filename="Dockerfile.jinja2", whitelist=whitelist, blacklist=blacklist): - self.process_dockerfile(file) - - def process_dockerfile(self, input_file): - """ - :param input_file: Input File - :type input_file: str - """ - output_file = os.path.splitext(input_file) - output_file = os.path.join(os.path.dirname(output_file[0]), os.path.basename(output_file[0])) - - docker_image = os.path.basename(os.path.dirname(os.path.dirname(output_file))) - docker_tag = os.path.basename(os.path.dirname(output_file)) - - context = { - 'Dockerfile': { - 'image': docker_image, - 'tag': docker_tag - } - } - - if Output.VERBOSITY_NORMAL <= self.output.get_verbosity(): - self.line("* Processing Dockerfile for %s:%s" % (docker_image,docker_tag)) - - with open(input_file, 'r') as fileInput: - template_content = fileInput.read() - - template_content = self.template_header + template_content + self.template_footer - - rendered_content = self.template.from_string(template_content).render(context) - rendered_content = rendered_content.lstrip() - - with open(output_file, 'w') as file_output: - file_output.write(rendered_content) diff --git a/bin/command/generate_provision_command.py b/bin/command/generate_provision_command.py deleted file mode 100644 index cdf3cec44..000000000 --- a/bin/command/generate_provision_command.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os -import yaml -import yamlordereddictloader -import time -import Queue -import shutil -import grp -from cleo import Output -from webdevops import Provisioner -from webdevops.command import BaseCommand - -class GenerateProvisionCommand(BaseCommand): - """ - Provisionning docker images - - generate:provision - {--image=?* : filter on images name } - {--baselayout : Build the baselayout } - {--t|threads=0 : threads} - """ - - conf = '' - - __threads = [] - - __queue = '' - - def run_task(self, configuration): - self.__queue = Queue.Queue() - if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): - self.line('provision : %s' % configuration.get('provisionPath')) - self.line('dockerfile : %s' % configuration.get('dockerPath')) - self.line('baselayout : %s' % self.option('baselayout')) - if 0 < len(self.option('image')): - self.line('images :') - for crit in self.option('image'): - self.line("\t * %s" % crit) - self.__load_configuration() - self.__build_base_layout() - self.__create_thread() - for image_name in self.conf['provision']: - if 0 == len(self.option('image')) or image_name in self.option('image'): - self.__queue.put({'image_name': image_name, 'image_config': self.conf['provision'][image_name]}) - self.__queue.join() - if os.path.exists('baselayout.tar'): - os.remove('baselayout.tar') - - def __create_thread(self): - for i in range(self.get_threads()): - thread_name = "Pixie_%d" % i - if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): - self.line("* -> Create thread %s" % thread_name) - provisioner = Provisioner( - self.configuration.get('dockerPath'), - self.configuration.get('provisionPath'), - self.__queue, - self.output - ) - provisioner.setDaemon(True) - provisioner.setName(thread_name) - provisioner.start() - # self.__threads.append(provisioner) - - def __load_configuration(self): - """ - Load the configuration for provisioning image - """ - configuration_file = os.path.join(self.configuration.get('confPath'), 'provision.yml') - stream = open(configuration_file, "r") - self.conf = yaml.load(stream, Loader=yamlordereddictloader.Loader) - - def __build_base_layout(self): - """ - Build tar file from _localscripts for bootstrap containers - """ - if self.option('baselayout'): - if Output.VERBOSITY_NORMAL <= self.output.get_verbosity(): - self.line('* Building localscipts') - base_path = self.configuration.get('baselayoutPath') - - root_group = grp.getgrgid(0) - - shutil.make_archive( - base_name='baselayout', - format='bztar', - root_dir=base_path, - owner='root', - group=root_group.gr_name - ) - os.rename('baselayout.tar.bz2', 'baselayout.tar') - - - diff --git a/bin/command/test_serverspec_command.py b/bin/command/test_serverspec_command.py deleted file mode 100644 index 69c5ac5ad..000000000 --- a/bin/command/test_serverspec_command.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, glob -from cleo import Output -from webdevops import Dockerfile, DockerfileUtility -from webdevops.command import DoitCommand -from webdevops.taskloader import DockerTestServerspecTaskLoader - -class TestServerspecCommand(DoitCommand): - """ - Test docker images with Serverspec - - test:serverspec - {docker images?* : Docker images (whitelist)} - {--dry-run : show only which images will be build} - {--t|threads=0 : threads} - {--r|retry=0 : retry} - {--whitelist=?* : image/tag whitelist } - {--blacklist=?* : image/tag blacklist } - """ - - serverspec_path = False - - def run_task(self, configuration): - self.serverspec_path = configuration.get('serverspecPath') - - self.cleanup_dockerfiles() - - return self.run_doit( - task_loader=DockerTestServerspecTaskLoader(configuration), - configuration=configuration - ) - - def cleanup_dockerfiles(self): - """ - Cleanup old dockerfiles in test directory - """ - for file in glob.glob(os.path.join(self.serverspec_path, 'Dockerfile.*.tmp')): - os.remove(file) - - def teardown(self, exitcode): - self.cleanup_dockerfiles() - diff --git a/bin/command/test_testinfra_command.py b/bin/command/test_testinfra_command.py deleted file mode 100644 index 39508bd45..000000000 --- a/bin/command/test_testinfra_command.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from cleo import Output -from webdevops.command import DoitCommand -from webdevops.taskloader import DockerTestTestinfraTaskLoader - -class TestTestinfraCommand(DoitCommand): - """ - Test docker images with Testinfra - - test:testinfra - {docker images?* : Docker images (whitelist)} - {--dry-run : show only which images will be build} - {--t|threads=0 : threads} - {--whitelist=?* : image/tag whitelist } - {--blacklist=?* : image/tag blacklist } - """ - - def run_task(self, configuration): - return self.run_doit( - task_loader=DockerTestTestinfraTaskLoader(configuration), - configuration=configuration - ) - diff --git a/bin/console b/bin/console deleted file mode 100755 index ebf5b10b2..000000000 --- a/bin/console +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os, sys - -# prevent bytecode -sys.dont_write_bytecode = True - -# unbuffered stdout / stderr -sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) -sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0) - -import re, yaml -from cleo import Application - -from webdevops import Configuration -from webdevops.docker.DockerBaseClient import DockerBaseClient -from webdevops.docker.DockerPyClient import DockerPyClient -from webdevops.docker.DockerCliClient import DockerCliClient -from command.docker_build_command import DockerBuildCommand -from command.docker_push_command import DockerPushCommand -from command.docker_pull_command import DockerPullCommand -from command.docker_exec_command import DockerExecCommand -from command.test_testinfra_command import TestTestinfraCommand -from command.test_serverspec_command import TestServerspecCommand -from command.generate_dockerfile_command import GenerateDockerfileCommand -from command.generate_provision_command import GenerateProvisionCommand - -if __name__ == '__main__': - # Generate common paths - script_path = os.path.dirname(os.path.realpath(__file__)) - root_path = os.path.dirname(script_path) - conf_path = os.path.join(root_path, 'conf') - - - def generatePath(path): - """ - Generate full path based on root path - """ - return os.path.abspath(os.path.join(root_path, (path))) - - # Read console.yml for configuration - with open(os.path.join(conf_path, 'console.yml'), 'r') as stream: - try: - configuration = yaml.load(stream) - configuration['confPath'] = conf_path - except yaml.YAMLError as e: - configuration = None - print ' !!! Exception while loading configuration from %s:' % conf_path - print '' - print e - print '' - sys.exit(1) - - # Check if configuration is valid - if configuration is None: - print ' !!! Configuration not found' - sys.exit(1) - - # generate full paths - path_entries = [ - 'dockerPath', - 'templatePath', - 'provisionPath', - 'baselayoutPath', - 'testinfraPath', - 'serverspecPath', - 'blacklistFile', - ] - for key in path_entries: - if key in configuration: - configuration[key] = generatePath(configuration[key]) - - # Translate regexp - if 'docker' in configuration: - if 'pathRegex' in configuration['docker']: - configuration['docker']['pathRegex'] = re.compile(configuration['docker']['pathRegex']) - - if 'pathRegex' in configuration['docker']: - configuration['docker']['autoPullBlacklist'] = re.compile(configuration['docker']['autoPullBlacklist']) - - if 'dockerTest' in configuration: - if 'configuration' in configuration['dockerTest']: - if 'imageConfigurationRegex' in configuration['dockerTest']['configuration']: - configuration['dockerTest']['configuration']['imageConfigurationRegex'] = re.compile(configuration['dockerTest']['configuration']['imageConfigurationRegex']) - - configuration = Configuration.merge(configuration) - configuration = Configuration.dotdictify(configuration) - - # Init application - application = Application() - application.add(DockerBuildCommand(configuration=configuration)) - application.add(DockerPushCommand(configuration=configuration)) - application.add(DockerPullCommand(configuration=configuration)) - application.add(DockerExecCommand(configuration=configuration)) - application.add(TestTestinfraCommand(configuration=configuration)) - application.add(TestServerspecCommand(configuration=configuration)) - application.add(GenerateDockerfileCommand(configuration=configuration)) - application.add(GenerateProvisionCommand(configuration=configuration)) - - application.run() diff --git a/bin/requirements.txt b/bin/requirements.txt deleted file mode 100644 index 4ca4bcc6d..000000000 --- a/bin/requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -Jinja2>=2.8 -cleo<0.7.0 -yamlordereddictloader>=0.1.0 -testinfra>=1.4.2 -doit==0.29.0 -termcolor>=1.1.0 -pytest-timeout>=1.0.0 -pytest-rerunfailures>=1.0.0 -MarkupSafe -funcsigs -pbr -six -mock -psutil -pylev -py -appdirs -pyparsing -packaging -pytest -simplejson -backpack -PyYAML -cloudpickle -pyinotify -configparser diff --git a/bin/webdevops/Command.py b/bin/webdevops/Command.py deleted file mode 100644 index c7979e8d3..000000000 --- a/bin/webdevops/Command.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, subprocess, tempfile, copy, time - -def execute(cmd, cwd=False, env=None): - """ - Execute cmd and output stdout/stderr - """ - - print 'Execute: %s' % ' '.join(cmd) - - if env is not None: - env = copy.deepcopy(env) - env['PWD'] = os.environ['PWD'] - env['LC_CTYPE'] = os.environ['LC_CTYPE'] - env['SHELL'] = os.environ['SHELL'] - - if '_' in env: - del env['_'] - - # add system env vars - system_env_vars = ['PWD', 'PATH', 'LC_CTYPE', 'SHELL', 'DOCKER_HOST', 'DOCKER_CERT_PATH', 'DOCKER_MACHINE_NAME', 'DOCKER_TLS_VERIFY'] - for var_name in system_env_vars: - if not var_name in env and var_name in os.environ: - env[var_name] = os.environ[var_name] - - # set current working directory - path_current = os.getcwd() - if cwd: - os.chdir(cwd) - - # stdout file - # (stdout and stderr will be redirected to it, pieping both isn't possible) - file_stdout = tempfile.NamedTemporaryFile() - - # create subprocess - proc = subprocess.Popen( - cmd, - stdout=file_stdout, - stderr=file_stdout, - bufsize=-1, - env=env - ) - - # wait for process end - while proc.poll() is None: - time.sleep(1) - - # output stdout - with open(file_stdout.name, 'r') as f: - for line in f: - print line.rstrip('\n') - - # restore current work directory - os.chdir(path_current) - - if proc.returncode == 0: - return True - else: - print '>> failed command with return code %s' % proc.returncode - return False diff --git a/bin/webdevops/Configuration.py b/bin/webdevops/Configuration.py deleted file mode 100644 index e9f69d61f..000000000 --- a/bin/webdevops/Configuration.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import re -from webdevops.doit.DoitReporter import DoitReporter -from webdevops.docker.DockerCliClient import DockerCliClient - -default = { - 'dockerPath': False, - 'templatePath': False, - 'provisionPath': False, - 'confPath': False, - 'baselayoutPath': False, - 'testPath': False, - - 'dockerClient': DockerCliClient(), - - 'blacklistFile': False, - - 'doitConfig': { - 'GLOBAL': { - 'reporter': DoitReporter, - } - }, - - 'docker': { - 'imagePrefix': '', - 'autoLatestTag': False, - 'fromRegExp': re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P.+))?', re.MULTILINE), - 'pathRegex': False, - 'autoPull': False, - 'autoPullWhitelist': False, - 'autoPullBlacklist': False, - }, - - 'dockerTest': { - 'toolImages': {}, - 'serverspec': { - }, - - 'env': {}, - }, - - 'dockerBuild': { - 'noCache': False, - }, - - 'dockerPush': { - }, - - 'filter': { - 'whitelist': False, - 'blacklist': False, - }, - - 'verbosity': 0, - 'threads': 1, - 'retry': 1, - 'dryRun': False, -} - -def merge(configuration): - def dictmerge(original, update): - """ - Recursively update a dict. - Subdict's won't be overwritten but also updated. - """ - for key, value in original.iteritems(): - if key not in update: - update[key] = value - elif isinstance(value, dict): - dictmerge(value, update[key]) - return update - - return dictmerge(default, configuration) - - -class dotdictify(dict): - def __init__(self, value=None): - if value is None: - pass - elif isinstance(value, dict): - for key in value: - self.__setitem_internal__(key, value[key]) - else: - raise TypeError, 'expected dict' - - def __setitem_internal__(self, key, value): - """ - Set dict as raw value (preserv key with dots) - """ - if isinstance(value, dict) and not isinstance(value, dotdictify): - value = dotdictify(value) - dict.__setitem__(self, key, value) - - def __setitem__(self, key, value): - if key is not None and '.' in key: - myKey, restOfKey = key.split('.', 1) - target = self.setdefault(myKey, dotdictify()) - if not isinstance(target, dotdictify): - raise KeyError, 'cannot set "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target)) - target[restOfKey] = value - else: - if isinstance(value, dict) and not isinstance(value, dotdictify): - value = dotdictify(value) - dict.__setitem__(self, key, value) - - def __getitem__(self, key, raw=False): - if key is None or '.' not in key or raw: - return dict.get(self, key, None) - myKey, restOfKey = key.split('.', 1) - target = dict.get(self, myKey, None) - if not isinstance(target, dotdictify): - raise KeyError, 'cannot get "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target)) - return target[restOfKey] - - def __contains__(self, key): - """ - Check if element is contained in tree - """ - if key is None or '.' not in key: - return dict.__contains__(self, key) - myKey, restOfKey = key.split('.', 1) - if not dict.__contains__(self, myKey): - return False - target = dict.__getitem__(self, myKey) - if not isinstance(target, dotdictify): - return False - return restOfKey in target - - def setdefault(self, key, default): - """ - Set default value by using dotted notation - """ - if key not in self: - self[key] = default - return self[key] - - def to_dict(self): - """ - Convert to dict - :return: dict - """ - ret = {} - for key in self: - if key is not None: - ret[key] = self.__getitem__(key, True) - return ret - - def get(self, k, d=None): - """ - Get element by using dotted notation - """ - if dotdictify.__contains__(self, k): - return dotdictify.__getitem__(self, k) - return d - - def set(self, key, value): - """ - Set value by using dotted notation - """ - self[key] = value - return self[key] - - __setattr__ = __setitem__ - __getattr__ = __getitem__ diff --git a/bin/webdevops/Dockerfile.py b/bin/webdevops/Dockerfile.py deleted file mode 100644 index 8955a0597..000000000 --- a/bin/webdevops/Dockerfile.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os -import re - - -def finder(dockerfile_path, filename="Dockerfile", filter=[]): - """ - Search all file un dockerfile_path with filename ends with "filename" - And match filter - - :param dockerfile_path: path where to search file - :type dockerfile_path: str - - :param filename: pattern which the file must be validate - :type filename: str - - :param filter: list of term must be match in path - :type filter: list - - :return: list of path - :rtype: list - """ - dockerfile_stack = [] - filter_regex = re.compile(ur'.*(%s).*' % "|".join(filter), re.IGNORECASE) - # pprint(filter_regex.pattern) - for root, dirs, files in os.walk(dockerfile_path): - for file in files: - if file.endswith(filename): - if filter_regex.match(root): - dockerfile_stack.append(os.path.join(root, file)) - return dockerfile_stack - - -def find_by_image(dockerfile_path, filename="Dockerfile", image=[]): - """ - Similar that finder but adds a constraint of search on the name of the image docker - - :param dockerfile_path: path where to search file - :type dockerfile_path: str - - :param filename: pattern which the file must be validate - :type filename: str - - :param image: list of term must be match in path - :type image: list - - :return: list of path - :rtype: list - """ - updated_filter = [] - for i, v in enumerate(image): - updated_filter.append("/docker/(%s)/" % v) - return finder(dockerfile_path, filename, updated_filter) - - -def find_by_image_and_tag(dockerfile_path, image, tag): - """ - - :param dockerfile_path: - :type dockerfile_path: str - - :param image: - :type image: str - - :param tag: - :type tag: str - - :return: - :rtype: list - """ - if "*" == tag: - filter = ["/docker/%s/" % image] - else: - filter = ["/docker/%s/%s" % (image, tag.replace('*', '.+'))] - return finder(dockerfile_path, "Dockerfile", filter) - - -def find_by_tags(dockerfile_path, filename="Dockerfile", tags=[]): - """ - Similar that finder but adds a constraint of search on the tag of the image docker - - :param dockerfile_path: path where to search file - :type dockerfile_path: str - - :param filename: pattern which the file must be validate - :type filename: str - - :param tags: list of tags must be match in path - :type tags: list - - :return: list of path - :rtype: list - """ - updated_filter = [] - for i, v in enumerate(tags): - updated_filter.append("/docker/[^/]+/%s" % v) - return finder(dockerfile_path, filename, updated_filter) diff --git a/bin/webdevops/DockerfileUtility.py b/bin/webdevops/DockerfileUtility.py deleted file mode 100644 index 32b32a966..000000000 --- a/bin/webdevops/DockerfileUtility.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os -import re - -DOCKERFILE_STATEMENT_FROM_RE = re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P[^\s:]+))?(?!.*\s+AS)', re.MULTILINE) -DOCKERFILE_STATEMENT_FROM_MULTISTAGE_RE = re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P[^\s:]+))?(\s+AS)', re.MULTILINE) - -def find_file_in_path(dockerfile_path, filename="Dockerfile", whitelist=False, blacklist=False): - """ - Search all file un dockerfile_path with filename ends with "filename" - And match filter - - :param dockerfile_path: path where to search file - :type dockerfile_path: str - - :param filename: pattern which the file must be validate - :type filename: str - - :param whitelist: list of term must be match in path - :type whitelist: list - - :param blacklist: list of term must not be match in path - :type blacklist: list - - :return: list of path - :rtype: list - """ - file_list = [] - - # build list of files - for root, dirs, files in os.walk(dockerfile_path): - for file in files: - if file.endswith(filename): - file_list.append(os.path.join(root, file)) - - # filter by whitelist - if whitelist: - tmp = [] - for file in file_list: - for term in whitelist: - if term in file: - tmp.append(file) - break - file_list = tmp - - if blacklist: - for term in blacklist: - file_list = filter(lambda x: term not in x, file_list) - - return file_list - -def find_dockerfiles_in_path(base_path, path_regex, image_prefix, whitelist=False, blacklist=False): - """ - Find all Dockerfiles in path (and even in symlinks and build dependencies) - """ - - def parse_docker_info_from_path(path): - image_name_info = ([m.groupdict() for m in path_regex.finditer(os.path.abspath(path))])[0] - - image_repository = (image_name_info['repository'] if 'repository' in image_name_info else '') - image_name = (image_name_info['image'] if 'image' in image_name_info else '') - image_tag = (image_name_info['tag'] if 'tag' in image_name_info else '') - - image_is_duplicate = False - - # check if path is linked - if os.path.islink(os.path.dirname(path)): - linked_image_name_info = ([m.groupdict() for m in path_regex.finditer(os.path.realpath(path))])[0] - - linked_image_repository = (linked_image_name_info['repository'] if 'repository' in linked_image_name_info else '') - linked_image_name = (linked_image_name_info['image'] if 'image' in linked_image_name_info else '') - linked_image_tag = (linked_image_name_info['tag'] if 'tag' in linked_image_name_info else '') - - image_from = image_prefix + linked_image_repository + '/' + linked_image_name + ':' + linked_image_tag - image_is_duplicate = True - else: - image_from = parse_dockerfile_from_statement(path) - - imageInfo = { - 'fullname': image_prefix + image_repository + '/' + image_name + ':' + image_tag, - 'name': image_prefix + image_repository + '/' + image_name, - 'tag': image_tag, - 'repository': image_prefix + image_repository, - 'imageName': image_name, - 'from': image_from, - 'duplicate': image_is_duplicate, - 'multiStageImages': parse_dockerfile_multistage_images(path) - } - return imageInfo - - ret = [] - for path in find_dockerfile_in_path_recursive(base_path): - base_path = os.path.dirname(path) - if os.path.isfile(path) and os.path.basename(path) == 'Dockerfile': - dockerfile = { - 'path': path, - 'basePath': base_path, - 'abspath': os.path.abspath(path), - 'image': parse_docker_info_from_path(path), - } - ret.append(dockerfile) - - ret = filter_dockerfile( - dockerfile_list=ret, - whitelist=whitelist, - blacklist = blacklist - ) - - return ret - - -def filter_dockerfile(dockerfile_list, whitelist=False, blacklist=False): - """ - Filter Dockerfiles by white- and blacklist - """ - if whitelist: - tmp = [] - for dockerfile in dockerfile_list: - for term in whitelist: - if term in dockerfile['image']['fullname']: - tmp.append(dockerfile) - break - dockerfile_list = tmp - - if blacklist: - for term in blacklist: - dockerfile_list = filter(lambda x: term not in x['image']['fullname'], dockerfile_list) - - return dockerfile_list - - -def find_dockerfile_in_path_recursive(basePath): - """ - Find all Dockerfiles paths recursive in path - """ - - ret = [] - for root, subFolders, files in os.walk(basePath, followlinks=True): - for file in files: - if os.path.basename(file) == 'Dockerfile': - ret.append(os.path.join(root, file)) - return ret - -def create_imagename_from_regex_result(data): - """ - Create imagename from regex result (parsed Dockerfile) - """ - ret = data['image'] - - if data['tag']: - ret += ':%s' % data['tag'] - - ret = image_full_name(ret) - - return ret - -def parse_dockerfile_from_statement(path): - """ - Extract docker image name from FROM statement - """ - with open(path, 'r') as fileInput: - DockerfileContent = fileInput.read() - data = ([m.groupdict() for m in DOCKERFILE_STATEMENT_FROM_RE.finditer(DockerfileContent)])[0] - ret = create_imagename_from_regex_result(data) - return ret - -def parse_dockerfile_multistage_images(path): - """ - Extract docker image name from FROM statement - """ - ret = [] - - with open(path, 'r') as fileInput: - DockerfileContent = fileInput.read() - - for data in ([m.groupdict() for m in DOCKERFILE_STATEMENT_FROM_MULTISTAGE_RE.finditer(DockerfileContent)]): - ret.append(create_imagename_from_regex_result(data)) - return ret - -def image_full_name(image_name): - """ - Return full name image if just short form (eg. alpine instead of alpine:latest) - """ - if not re.search(':[^:]+$', image_name): - image_name = '%s:latest' % image_name - return image_name - -def generate_image_name_with_tag_latest(image_name): - """ - Prepare dockerfile list with dependency and also add "auto latest tag" images - """ - if re.search(':[^:]+$', image_name): - ret = re.sub('(:[^:]+)$', ':latest', image_name) - else: - ret = '%s:latest' % image_name - return ret - -def image_basename(image_name): - """ - Get image name without tag - """ - if re.search(':[^:]+$', image_name): - image_name = re.sub('(:[^:]+)$', '', image_name) - return image_name - -def extract_image_name_tag(image_name): - """ - Get tag from image name - """ - if re.search('^(.*):', image_name): - ret = re.sub('^(.*):', '', image_name) - else: - ret = 'latest' - return ret - -def check_if_base_image_needs_pull(image, configuration): - ret = False - - if configuration.get('docker.autoPull'): - if configuration.get('docker.autoPullWhitelist') and configuration.get('docker.autoPullWhitelist').search(image): - """ - Matched whitelist - """ - ret = True - else: - """ - No whitelist, we need to pull every image - """ - ret = True - - if configuration.get('docker.autoPullBlacklist') and configuration.get('docker.autoPullBlacklist').match(image): - """ - Matched blacklist - """ - ret = False - return ret diff --git a/bin/webdevops/Provisioner.py b/bin/webdevops/Provisioner.py deleted file mode 100644 index 08dc1ad7a..000000000 --- a/bin/webdevops/Provisioner.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -from cleo import Output -from webdevops import Dockerfile -import os -from distutils.dir_util import copy_tree, remove_tree -from threading import Thread -import Queue -import shutil - - -class Provisioner(Thread): - """ - Asked to provisioning an image - """ - - image_name = '' - - image_config = {} - - output = '' - - dockerfile = '' - - provision = '' - - queue = '' - - # def __init__(self, dockerfile, provision, image_name, image_config, output): - def __init__(self, dockerfile, provision, queue, output): - """ - Construct - - :param dockerfile: path to the folder containing dockerfile analyze - :type dockerfile: str - - :param queue: Stack of image config - :type queue: Queue.Queue - - :param output: stdout - :type output: Output - - """ - Thread.__init__(self) - self.dockerfile = dockerfile - self.provision = provision - self.queue = queue - self.output = output - - def __get_item(self): - """ Get a item in the Queue """ - if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): - self.line('Looking for the next item') - item = self.queue.get(True, 5.0) - self.image_name = item['image_name'] - self.image_config = item['image_config'] - - def __done_item(self): - """ Set current item of the Queue to done """ - self.queue.task_done() - self.image_name = '' - self.image_config = {} - - def run(self): - while True: - try: - self.__get_item() - if Output.VERBOSITY_NORMAL <= self.output.get_verbosity(): - self.line( - 'Building configuration for webdevops/%s' % self.image_name - ) - self.__clear_configuration() - if 'configuration' in self.image_config: - self.__deploy_configuration() - self.__done_item() - except Queue.Empty: - if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): - self.line("End") - break - - def __clear_configuration(self): - """ - Remove the old configuration - """ - dockerfiles = Dockerfile.find_by_image_and_tag(self.dockerfile, self.image_name, '*') - dockerfiles = [os.path.dirname(image_path) for image_path in dockerfiles] - for dest in dockerfiles: - dest = os.path.abspath(os.path.join(dest, 'conf/')) - if os.path.exists(dest): - if Output.VERBOSITY_VERY_VERBOSE <= self.output.get_verbosity(): - self.line('delete configuration : %s' % dest) - remove_tree(dest, 0) - - def __deploy_configuration(self): - """ - Deploy the configuration to the container - """ - for src, tag in self.image_config['configuration'].iteritems(): - if Output.VERBOSITY_NORMAL <= self.output.get_verbosity(): - self.line("%s => %s:%s" % (src, self.image_name, tag)) - if isinstance(tag, list): - dockerfiles = [] - for t in tag: - dockerfiles.extend(Dockerfile.find_by_image_and_tag(self.dockerfile, self.image_name, t)) - else: - dockerfiles = Dockerfile.find_by_image_and_tag(self.dockerfile, self.image_name, tag) - dockerfiles = [os.path.dirname(image_path) for image_path in dockerfiles] - self.__copy_configuration(dockerfiles, src) - - - def __copy_configuration(self, dockerfiles, src): - """ - Copy the different configs to the container - - :param dockerfiles: List of path's images to provisioning - :type dockerfiles: list - - :param src: sub-path of the provisioning directory - :type src: str - """ - src = os.path.abspath(os.path.join(self.provision, src)) + "/." - if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): - self.line('src : %s' % src) - for dest in dockerfiles: - dest = os.path.abspath(os.path.join(dest, 'conf/')) - if not os.path.exists(dest): - if Output.VERBOSITY_DEBUG <= self.output.get_verbosity(): - self.line('create : %s' % dest) - os.mkdir(dest) - if Output.VERBOSITY_VERBOSE <= self.output.get_verbosity(): - self.line('dest : %s' % dest) - copy_tree(src, dest, 1, 1, 0, 0, 0) - - def line(self, msg): - self.output.writeln( - '({0:^8}) [{1:.<20}] {2}'.format(self.name, self.image_name, msg) - ) diff --git a/bin/webdevops/__init__.py b/bin/webdevops/__init__.py deleted file mode 100644 index 0688bc71a..000000000 --- a/bin/webdevops/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from Provisioner import Provisioner - -__all__ = [ - 'Provisioner', -] diff --git a/bin/webdevops/command/BaseCommand.py b/bin/webdevops/command/BaseCommand.py deleted file mode 100644 index 6734f5917..000000000 --- a/bin/webdevops/command/BaseCommand.py +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, sys, re, traceback -import time, datetime -import multiprocessing -from cleo import Command -from webdevops import Configuration -from ..doit.DoitReporter import DoitReporter - -class BaseCommand(Command): - configuration = False - - time_startup = False - time_finish = False - - def __init__(self, configuration): - """ - Constructor - """ - Command.__init__(self) - self.configuration = configuration - - def handle(self): - """ - Main command method which will be called by Cleo - """ - self.build_configuration() - - self.startup() - - try: - exitcode = self.run_task(configuration=self.configuration) - except KeyboardInterrupt as e: - print ' !!! Execution aborted by user' - exitcode = 1 - except SystemExit as e: - print ' !!! Execution aborted by SystemExit' - print '' - traceback.print_exc(file=sys.stdout) - exitcode = 1 - - if exitcode == True or exitcode == 0 or exitcode == '' or exitcode is None: - exitcode = 0 - elif exitcode == False: - exitcode = 255 - - self.shutdown(exitcode=exitcode) - sys.exit(exitcode) - - def run_task(self, configuration): - """ - Run task - """ - return - - def startup(self): - """ - Show startup message - """ - self.time_startup = time.time() - - options = [] - - if 'threads' in self.configuration: - options.append('%s threads' % self.configuration.get('threads')) - - if 'retry' in self.configuration: - options.append('%s retries' % self.configuration.get('retry', 1)) - - if 'dryRun' in self.configuration and self.configuration.get('dryRun'): - options.append('dry-run') - DoitReporter.simulation_mode = True - - - print 'Executing %s (%s)' % (self.name, ', '.join(options)) - print '' - - try: - whitelist = self.get_whitelist() - if whitelist: - print 'WHITELIST active:' - for item in whitelist: - print ' - %s' % item - print '' - except: - pass - - try: - blacklist = self.get_blacklist() - if blacklist: - print 'BLACKLIST active:' - for item in blacklist: - print ' - %s' % item - print '' - except: - pass - - def teardown(self,exitcode): - pass - - def shutdown(self, exitcode=0): - """ - Show shutdown message - """ - - self.time_finish = time.time() - - duration = self.time_finish - self.time_startup - duration = str(datetime.timedelta(seconds=int(duration))) - - self.teardown(exitcode) - - print '' - if exitcode == 0: - print '> finished execution in %s successfully' % (duration) - else: - print '> finished execution in %s with errors (exitcode %s)' % (duration, exitcode) - - def build_configuration(self): - """ - Get configuration - """ - configuration = self.configuration - - # threads - try: - configuration.set('threads', self.get_threads()) - except (Exception): - configuration.set('threads', 1) - - # whitelist - try: - configuration.set('whitelist', self.get_whitelist()) - except (Exception): - pass - - # blacklist - try: - configuration.set('blacklist', self.get_blacklist()) - except (Exception): - pass - - # dryrun - try: - configuration.set('dryRun', self.get_dry_run()) - except (Exception): - pass - - # retry - try: - configuration.set('retry', self.get_retry()) - except (Exception): - configuration.set('retry', 1) - - # verbosity - if self.output.is_verbose(): - configuration.set('verbosity', 2) - - self.configuration = configuration - - def get_configuration(self): - """ - Get configuration - """ - return self.configuration - - def get_whitelist(self): - """ - Get whitelist - """ - - # add whitelist from --whitelist - ret = list(self.option('whitelist')) - - # add whitelist from argument list - try: - ret.extend(self.argument('docker images')) - except (Exception): - pass - - return ret - - def get_blacklist(self): - """ - Get blacklist - """ - ret = list(self.option('blacklist')) - - # static BLACKLIST file - if os.path.isfile(self.configuration.get('blacklistFile')): - lines = [line.rstrip('\n').lstrip('\n') for line in open(self.configuration.get('blacklistFile'))] - lines = filter(bool, lines) - - if lines: - ret.extend(lines) - - return ret - - def get_threads(self): - """ - Get processing thread count - """ - threads = os.getenv('THREADS', self.option('threads')) - - if threads == '0' or threads == '' or threads is None: - # use configuration value - threads = self.configuration.get('threads') - - match = re.match('auto(([-*+/])([0-9]+\.?[0-9]?))?', str(threads)) - if match is not None: - ret = multiprocessing.cpu_count() - - if match.group(2) and match.group(3): - math_sign = match.group(2) - math_value = float(match.group(3)) - - if math_sign == "*": - ret = int(ret * math_value) - elif math_sign == "/": - ret = int(ret / math_value) - elif math_sign == "+": - ret = int(ret + math_value) - elif math_sign == "-": - ret = int(ret - math_value) - else: - ret = max(1, int(self.option('threads'))) - - return int(ret) - - def get_dry_run(self): - """ - Get if dry run is enabled - """ - return bool(self.option('dry-run')) - - def get_retry(self): - """ - Get number of retries - """ - default = 1 - retry = os.getenv('RETRY', self.option('retry')) - retry = max(0, int(retry)) - - if retry > 0: - # user value - return retry - elif 'retry' in self.configuration: - # configuration value - return self.configuration.get('retry') - else: - # defaults - return default diff --git a/bin/webdevops/command/DoitCommand.py b/bin/webdevops/command/DoitCommand.py deleted file mode 100644 index c7d1dc3ed..000000000 --- a/bin/webdevops/command/DoitCommand.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import sys -from .BaseCommand import BaseCommand -from doit.doit_cmd import DoitMain - -class DoitCommand(BaseCommand): - def run_doit(self, task_loader, configuration): - arguments = [] - extra_configuration = {} - - if 'threads' in configuration and configuration.get('threads') > 1: - arguments.extend(['-n', str(configuration.get('threads')), '--parallel-type', 'process']) - - if 'doitConfig' in configuration: - extra_configuration = configuration.get('doitConfig') - - return DoitMain( - task_loader=task_loader, - extra_config=extra_configuration - ).run(arguments) diff --git a/bin/webdevops/command/__init__.py b/bin/webdevops/command/__init__.py deleted file mode 100644 index 9be4e15cb..000000000 --- a/bin/webdevops/command/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from BaseCommand import BaseCommand -from DoitCommand import DoitCommand - -__all__ = [ - 'BaseCommand', - 'DoitCommand', -] diff --git a/bin/webdevops/docker/DockerBaseClient.py b/bin/webdevops/docker/DockerBaseClient.py deleted file mode 100644 index b8b22cd9c..000000000 --- a/bin/webdevops/docker/DockerBaseClient.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -class DockerBaseClient: - - def __init__(self): - """ - Constructor - """ - - def pull_image(self, name, tag): - """ - Build dockerfile - """ - print 'Pull image %s:%s' % (name, tag) - return True - - def build_dockerfile(self, path, name, nocache=False): - """ - Build dockerfile - """ - print 'Build Dockerfile %s with name %s' % (path, name) - return True - - def push_image(self, name): - """ - Push one Docker image to registry - """ - print 'Build image %s' % (name) - return True diff --git a/bin/webdevops/docker/DockerCliClient.py b/bin/webdevops/docker/DockerCliClient.py deleted file mode 100644 index 2f59b5171..000000000 --- a/bin/webdevops/docker/DockerCliClient.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, subprocess, tempfile -from .DockerBaseClient import DockerBaseClient -from webdevops import Command - -class DockerCliClient(DockerBaseClient): - - def pull_image(self, name, tag): - """ - Build dockerfile - """ - cmd = ['docker', 'pull', '%s:%s' % (name, tag)] - return Command.execute(cmd) - - def build_dockerfile(self, path, name, nocache=False): - """ - Build dockerfile - """ - cmd = ['docker', 'build', '--tag', name, os.path.dirname(path)] - - if nocache: - cmd.append('--no-cache') - - return Command.execute(cmd) - - def push_image(self, name): - """ - Push one Docker image to registry - """ - cmd = ['docker', 'push', name] - return Command.execute(cmd) diff --git a/bin/webdevops/docker/DockerPyClient.py b/bin/webdevops/docker/DockerPyClient.py deleted file mode 100644 index 507a190f0..000000000 --- a/bin/webdevops/docker/DockerPyClient.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os -from .DockerBaseClient import DockerBaseClient - -class DockerPyClient(DockerBaseClient): - - docker_client = False - last_output_line = False - - def __init__(self, configuration): - """ - Constrcutor - """ - DockerBaseClient.__init__(self) - - import docker - - # Init docker client - self.docker_client = docker.from_env(assert_hostname=False) - - def pull_image(self, name, tag): - """ - Build dockerfile - """ - response = self.docker_client.pull( - repository=name, - tag=tag, - stream=True, - decode=True - ) - - return self.process_client_response(response) - - def build_dockerfile(self, path, name, nocache=False): - """ - Build dockerfile - """ - response = docker_client.build( - path=os.path.dirname(path), - tag=name, - pull=False, - nocache=nocache, - quiet=False, - decode=True - ) - - return self.process_client_response(response) - - def push_image(self, name): - """ - Push one Docker image to registry - """ - response = docker_client.push( - name, - stream=True, - decode=True - ) - return self.process_client_response(response) - - def process_client_response(self, response): - ret = True - last_message = False - - def output_message(message, prevent_repeat=False): - # Prevent repeating messages - if prevent_repeat: - if self.last_output_line and self.last_output_line == message: - return - self.last_output_line = message - else: - self.last_output_line = False - - sys.stdout.write(message.strip(' \t\n\r') + '\n') - - if not response: - return False - - for line in response: - # Keys - # - error - # - stream - # - status, progressDetail, id - # - progressDetail | aux [ tag, digest, size ] - if 'error' in line: - output_message(line['error']) - ret = False - if 'stream' in line: - output_message(line['stream'], prevent_repeat=True) - if 'status' in line: - message = line['status'] - if 'id' in line: - message += ' ' + line['id'] - output_message(message) - print '' - return ret diff --git a/bin/webdevops/docker/__init__.py b/bin/webdevops/docker/__init__.py deleted file mode 100644 index 9124d6a99..000000000 --- a/bin/webdevops/docker/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from DockerBaseClient import DockerBaseClient -from DockerPyClient import DockerPyClient -from DockerCliClient import DockerCliClient - -__all__ = [ - 'DockerBaseClient', - 'DockerPyClient', - 'DockerCliClient', -] diff --git a/bin/webdevops/doit/DoitReporter.py b/bin/webdevops/doit/DoitReporter.py deleted file mode 100644 index 84c85ee53..000000000 --- a/bin/webdevops/doit/DoitReporter.py +++ /dev/null @@ -1,352 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, sys, time, datetime, StringIO -import termcolor -from termcolor import colored -from ..taskloader.BaseTaskLoader import BaseTaskLoader - -class TaskResult(object): - """ - result object used by DoitReporter - """ - # FIXME what about returned value from python-actions ? - def __init__(self, task): - self.task = task - self.result = None # fail, success, up-to-date, ignore - self.out = None # stdout from task - self.err = None # stderr from task - self.error = None # error from doit (exception traceback) - self.started = None # datetime when task execution started - self.elapsed = None # time (in secs) taken to execute task - self._started_on = None # timestamp - self._finished_on = None # timestamp - - def start(self): - """ - called when task starts its execution - """ - self._started_on = time.time() - - def set_result(self, result, error=None): - """ - called when task finishes its execution - """ - self._finished_on = time.time() - self.result = result - - self.out = ''.join([a.out for a in self.task.actions if a.out]) - self.err = ''.join([a.err for a in self.task.actions if a.err]) - self.error = error - - self.calculate_elapsed() - - def calculate_elapsed(self): - """ - calculate elapsed time - """ - if self._started_on is not None and self.elapsed is None: - started = datetime.datetime.utcfromtimestamp(self._started_on) - - if self._finished_on is None: - self._finished_on = time.time() - - self.started = str(started) - self.elapsed = self._finished_on - self._started_on - - def to_dict(self): - """ - convert result data to dictionary - """ - - self.calculate_elapsed() - - ret = { - 'name': self.task.name, - 'result': self.result, - 'out': self.out, - 'err': self.err, - 'error': self.error, - 'started': self.started, - 'elapsed': self.elapsed - } - return ret - - -class DoitReporter(object): - """ - output results after finish - """ - - skip_detection = True - simulation_mode = False - - desc = 'output after finish' - - show_out = False - show_err = False - task_finished = 0 - task_total = 0 - - COLOR_SCHEMA = { - 'skipped': 'yellow', - 'simulated': 'blue', - 'success': 'green', - 'success2': 'cyan', - 'failed': 'red', - } - - def __init__(self, outstream, options=None): #pylint: disable=W0613 - # result is sent to stdout when doit finishes running - self.t_results = {} - self.failures = [] - # when using this reporter output can not contain any other output - # than the data. so anything that is sent to stdout/err needs to - # be captured. - self._old_out = sys.stdout - sys.stdout = StringIO.StringIO() - self._old_err = sys.stderr - sys.stderr = StringIO.StringIO() - self.outstream = outstream - # runtime and cleanup errors - self.errors = [] - - self.show_out = options.get('show_out', True) - self.show_err = options.get('show_err', True) - - def collect_task_result(self, task): - task_status = BaseTaskLoader.task_get_statusfile(task) - if task_status: - BaseTaskLoader.TASK_RESULTS[task.name] = task_status - - def get_status(self, task): - """ - called when task is selected (check if up-to-date) - """ - self.t_results[task.name] = TaskResult(task) - - def execute_task(self, task): - """ - called when excution starts - """ - self.t_results[task.name].start() - - - def add_failure(self, task, exception): - """ - called when excution finishes with a failure - """ - self.collect_task_result(task) - self.t_results[task.name].set_result('fail', exception.get_msg()) - - self.task_finished += 1 - - if task.actions and (task.name[0] != '_'): - duration = self.duration(self.t_results[task.name].elapsed) - progress = self.calc_progress() - self.writeln(colored('. %s FAILED (%s, #%s)' % (BaseTaskLoader.human_task_name(task.title()), duration, progress), DoitReporter.COLOR_SCHEMA['failed'])) - self.failures.append({'task': task, 'exception': exception}) - - def add_success(self, task): - """ - called when excution finishes successfuly - """ - self.collect_task_result(task) - self.t_results[task.name].set_result('success') - - self.task_finished += 1 - - if task.actions and (task.name[0] != '_'): - durationSeconds = self.t_results[task.name].elapsed - duration = self.duration(durationSeconds) - progress = self.calc_progress() - - if DoitReporter.simulation_mode: - output_status = 'simulated' - output_color = DoitReporter.COLOR_SCHEMA['simulated'] - else: - if DoitReporter.skip_detection and durationSeconds < 1: - output_status = 'SKIPPED' - output_color = DoitReporter.COLOR_SCHEMA['skipped'] - else: - output_status = 'finished' - output_color = DoitReporter.COLOR_SCHEMA['success'] - - # custom status - task_status = BaseTaskLoader.get_task_status(task.name) - if task_status: - if task_status['color'] and task_status['color'] in DoitReporter.COLOR_SCHEMA: - output_color = DoitReporter.COLOR_SCHEMA[task_status['color']] - if task_status['status']: - output_status = task_status['status'] - - self.writeln( - colored('. %s %s (%s, %s)' % (BaseTaskLoader.human_task_name(task.title()), output_status, duration, progress), output_color) - ) - - def skip_uptodate(self, task): - """ - skipped up-to-date task - """ - self.t_results[task.name].set_result('up-to-date') - - def skip_ignore(self, task): - """ - skipped ignored task - """ - self.collect_task_result(task) - self.t_results[task.name].set_result('ignore') - - def cleanup_error(self, exception): - """ - error during cleanup - """ - self.errors.append(exception.get_msg()) - - def runtime_error(self, msg): - """ - error from doit (not from a task execution) - """ - self.errors.append(msg) - - def teardown_task(self, task): - """ - called when starts the execution of teardown action - """ - pass - - def complete_run(self): - """ - called when finshed running all tasks - """ - # restore stdout - log_out = sys.stdout.getvalue() - sys.stdout = self._old_out - log_err = sys.stderr.getvalue() - sys.stderr = self._old_err - - # add errors together with stderr output - if self.errors: - log_err += "\n".join(self.errors) - - task_result_list = [tr.to_dict() for tr in self.t_results.values()] - - self.writeln('') - self.writeln('-> finished %s tasks' % (len(task_result_list))) - self.writeln('') - - # sort task list by task name - task_result_list = sorted(task_result_list, key=lambda task: task['name']) - - # show tasks if verbosity == 2 - if not self.show_out: - for task in task_result_list: - # Skip finish chain task (no content, just finish tasks) - if 'FinishChain|' in task['name']: - continue - - if task['result'] != 'fail': - self.task_stdout( - title=task['name'], - duration=task['elapsed'], - stdout=task['out'], - stderr=task['err'], - error=task['error'] - ) - - # show failed tasks (at the end) - for task in task_result_list: - # Skip finish chain task (no content, just finish tasks) - if 'FinishChain|' in task['name']: - continue - - if task['result'] == 'fail': - self.task_stdout( - title=task['name'], - duration=task['elapsed'], - stdout=task['out'], - stderr=task['err'], - error=task['error'] - ) - - if self.errors: - self.writeln("#" * 40 + "\n") - self.writeln("Execution aborted.\n") - self.writeln("\n".join(self.errors)) - self.writeln("\n") - - def task_stdout(self, title, duration=False, stdout=False, stderr=False, error=False, exception=False): - """ - Show task output - """ - - text_duration = '' - if duration: - text_duration = ' (%s)' % self.duration(duration) - - title_full = 'Task %s%s:' % (BaseTaskLoader.human_task_name(title), text_duration) - - self.writeln(title_full) - self.writeln('~' * len(title_full)) - - if stdout: - self.writeln() - self.writeln('%s' % stdout) - - if stderr: - self.writeln() - self.writeln(colored('-- STDERR OUTPUT --', 'red')) - self.write('%s' % stderr) - - if error: - self.writeln() - self.writeln(colored('-- ERROR OUTPUT --', 'red')) - self.write('%s' % error) - - if exception: - self.writeln() - self.writeln(colored('-- EXCEPTION --', 'red')) - self.write('%s' % exception.get_msg()) - - self.writeln() - self.writeln(':: end of output "%s"' % (BaseTaskLoader.human_task_name(title))) - self.writeln() - - def duration(self, duration): - """ - Calculate duration (seconds) to human readable time - """ - return 'duration: %s' % str(datetime.timedelta(seconds=int(duration))) - - def calc_progress(self): - percentage = 100 * float(self.task_finished)/float(BaseTaskLoader.TASK_COUNT) - return 'task %s/%s, progress %d%%' % (self.task_finished, BaseTaskLoader.TASK_COUNT, percentage) - - def writeln(self, text=''): - """ - Output - """ - self.outstream.write('%s\n' % text) - - def write(self, text): - """ - Output - """ - self.outstream.write(text) diff --git a/bin/webdevops/doit/__init__.py b/bin/webdevops/doit/__init__.py deleted file mode 100644 index 29fb79a7c..000000000 --- a/bin/webdevops/doit/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from DoitReporter import DoitReporter - -__all__ = [ - 'DoitReporter', -] diff --git a/bin/webdevops/taskloader/BaseDockerTaskLoader.py b/bin/webdevops/taskloader/BaseDockerTaskLoader.py deleted file mode 100644 index 42404c5a6..000000000 --- a/bin/webdevops/taskloader/BaseDockerTaskLoader.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, sys, re, copy -from .BaseTaskLoader import BaseTaskLoader -from webdevops import DockerfileUtility -from doit.task import dict_to_task - - -class BaseDockerTaskLoader(BaseTaskLoader): - - docker_client = False - - def __init__(self, configuration): - """ - Constrcutor - """ - BaseTaskLoader.__init__(self, configuration) - - # Init docker client - self.docker_client = configuration.get('dockerClient') - - def load_tasks(self, cmd, opt_values, pos_args): - """ - DOIT task list generator - """ - config = {'verbosity': self.configuration.get('verbosity')} - - dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( - base_path=self.configuration.get('dockerPath'), - path_regex=self.configuration.get('docker.pathRegex'), - image_prefix=self.configuration.get('docker.imagePrefix'), - whitelist=self.configuration.get('whitelist'), - blacklist=self.configuration.get('blacklist'), - ) - dockerfile_list = self.process_dockerfile_list(dockerfile_list) - - #import json,sys;print json.dumps(dockerfile_list, sort_keys=True, indent = 4, separators = (',', ': '));sys.exit(0); - - tasklist = self.generate_task_list(dockerfile_list) - - if not tasklist or len(tasklist) == 0: - raise Exception('No tasks found') - - tasklist = self.process_tasklist(tasklist) - - return tasklist, config - - def process_dockerfile_list(self, dockerfile_list): - """ - Prepare dockerfile list with dependency and also add "auto latest tag" images - """ - - # Process auto latest tag - autoLatestTagImageList = [] - image_list = [x['image']['fullname'] for x in dockerfile_list if x['image']['fullname']] - for dockerfile in dockerfile_list: - if self.configuration.get('docker.autoLatestTag') and dockerfile['image']['tag'] == self.configuration.get('docker.autoLatestTag'): - imageNameLatest = DockerfileUtility.generate_image_name_with_tag_latest(dockerfile['image']['fullname']) - if imageNameLatest not in image_list: - autoLatestTagImage = copy.deepcopy(dockerfile) - autoLatestTagImage['image']['fullname'] = imageNameLatest - autoLatestTagImage['image']['tag'] = 'latest' - autoLatestTagImage['image']['duplicate'] = True - - if not 'dependency' in autoLatestTagImage: - autoLatestTagImage['dependency'] = [] - autoLatestTagImage['dependency'].append(dockerfile['image']['fullname']) - autoLatestTagImageList.append(autoLatestTagImage) - # Add auto latest tag images to dockerfile list - dockerfile_list.extend(autoLatestTagImageList) - - # Calculate dependency - image_list = [x['image']['fullname'] for x in dockerfile_list if x['image']['fullname']] - for dockerfile in dockerfile_list: - if not 'dependency' in dockerfile: - dockerfile['dependency'] = [] - - # add image from if it is a dependency - if dockerfile['image']['from'] and dockerfile['image']['from'] in image_list: - dockerfile['dependency'].append(dockerfile['image']['from']) - - # add multi stage image if it is a dependency - for multiStageImage in dockerfile['image']['multiStageImages']: - if multiStageImage in image_list: - dockerfile['dependency'].append(multiStageImage) - - # unique list - unique_dep_list = [] - [unique_dep_list.append(item) for item in dockerfile['dependency'] if item not in unique_dep_list] - dockerfile['dependency'] = unique_dep_list - - return dockerfile_list - - def generate_task_list(self, dockerfile_list): - return [] - diff --git a/bin/webdevops/taskloader/BaseTaskLoader.py b/bin/webdevops/taskloader/BaseTaskLoader.py deleted file mode 100644 index a919fb934..000000000 --- a/bin/webdevops/taskloader/BaseTaskLoader.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import sys, re, time, StringIO, tempfile, json, base64, os -from webdevops import DockerfileUtility -from doit.cmd_base import TaskLoader -from doit.task import dict_to_task - - -class BaseTaskLoader(TaskLoader): - configuration = False - reporter = False - TASK_COUNT = 0 - TASK_RESULTS = {} - - def __init__(self, configuration): - """ - Constrcutor - """ - # Build configuration as namespace object - self.configuration = configuration - - def process_tasklist(self, tasklist): - """ - Process task list and create task objects - """ - ret = [] - for task in tasklist: - ret.append(dict_to_task(task)) - - print 'Starting execution of %s tasks...' % (len(ret)) - - BaseTaskLoader.TASK_COUNT = len(ret) - - return ret - - @staticmethod - def set_task_status(task, status, color): - """ - Set task status - """ - BaseTaskLoader.TASK_RESULTS[task.name] = { - 'task': task.name, - 'status': status, - 'color': color - } - BaseTaskLoader.task_write_statusfile(task, BaseTaskLoader.TASK_RESULTS[task.name]) - - @staticmethod - def get_task_status(task): - ret = False - if task in BaseTaskLoader.TASK_RESULTS: - ret = BaseTaskLoader.TASK_RESULTS[task] - return ret - - @staticmethod - def human_task_name(name): - """ - Translate internal task name to human readable name - """ - res = re.search('^.*\|(.*)', name) - - if res: - return re.search('^.*\|(.*)', name).group(1) - else: - return name - - - @staticmethod - def human_task_name_list(list): - """ - Translate list of internal task names to human readable names - """ - ret = [] - for name in list: - ret.append(BaseTaskLoader.human_task_name(name)) - return ', '.join(ret) - - - @staticmethod - def action_chain_finish(title): - """ - Action of finish chain - """ - return - - - @staticmethod - def task_title_finish(task): - """ - Finish task title function - """ - return "Finished chain %s" % (BaseTaskLoader.human_task_name(task.name)) - - @staticmethod - def task_runner(func, args, task): - """ - Wrapper for task runner - - Will return the stdout if task fails as exception - """ - backup = sys.stdout - sys.stdout = StringIO.StringIO() - status = func(task=task, *args) - output = sys.stdout.getvalue().strip() - sys.stdout.close() - sys.stdout = backup - - if not status: - raise Exception(output) - else: - print output - - return status - - - - @staticmethod - def task_statusfile(task): - return '%s/%s' % (tempfile.gettempdir(), base64.b64encode(task.name)) - - @staticmethod - def task_write_statusfile(task, data): - filename = BaseTaskLoader.task_statusfile(task) - with open(filename, "w") as f: - f.write(json.dumps(data)) - f.flush() - os.fsync(f.fileno()) - f.close() - - @staticmethod - def task_get_statusfile(task, remove=True): - ret = False - filename = BaseTaskLoader.task_statusfile(task) - - if os.path.isfile(filename): - with open(filename, "r") as f: - try: - json_data = f.read() - ret = json.loads(json_data) - except ValueError: - pass - f.close() - if remove: - BaseTaskLoader.task_remove_statusfile(task) - return ret - - @staticmethod - def task_remove_statusfile(task): - filename = '%s/%s' % (tempfile.gettempdir(), base64.b64encode(task.name)) - os.remove(filename) diff --git a/bin/webdevops/taskloader/DockerBuildTaskLoader.py b/bin/webdevops/taskloader/DockerBuildTaskLoader.py deleted file mode 100644 index 761e7cbe3..000000000 --- a/bin/webdevops/taskloader/DockerBuildTaskLoader.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, sys, re, copy, time -from .BaseTaskLoader import BaseTaskLoader -from .BaseDockerTaskLoader import BaseDockerTaskLoader -from webdevops import DockerfileUtility - -class DockerBuildTaskLoader(BaseDockerTaskLoader): - - def generate_task_list(self, dockerfileList): - """ - Generate task list for docker build - """ - tasklist = [] - - # TASK: dependency puller - task = { - 'name': 'DockerBuild|DependencyPuller', - 'title': DockerBuildTaskLoader.task_title_dependency_puller, - 'actions': [(BaseTaskLoader.task_runner, - [DockerBuildTaskLoader.task_dependency_puller, [self.docker_client, dockerfileList, self.configuration]])], - 'task_dep': [] - } - tasklist.append(task) - - # TASK: dockerfile build - for dockerfile in dockerfileList: - task = { - 'name': 'DockerBuild|%s' % dockerfile['image']['fullname'], - 'title': DockerBuildTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerBuildTaskLoader.task_run, [self.docker_client, dockerfile, self.configuration]])], - 'task_dep': ['DockerBuild|DependencyPuller'] - } - - for dep in dockerfile['dependency']: - task['task_dep'].append('DockerBuild|%s' % dep) - - - tasklist.append(task) - - # TASK: finisher - # task = { - # 'name': 'FinishChain|DockerBuild', - # 'title': DockerBuildTaskLoader.task_title_finish, - # 'actions': [(DockerBuildTaskLoader.action_chain_finish, ['docker build'])], - # 'task_dep': [task.name for task in taskList] - # } - # tasklist.append(task) - - return tasklist - - @staticmethod - def task_dependency_puller(docker_client, dockerfileList, configuration, task): - """ - Pulls dependency images before building - """ - def pull_image(image): - print ' -> Pull base image %s ' % image - - if configuration.get('dryRun'): - return True - - pull_image_name = DockerfileUtility.image_basename(image) - pull_image_tag = DockerfileUtility.extract_image_name_tag(image) - - pull_status = False - for retry_count in range(0, configuration.get('retry')): - pull_status = docker_client.pull_image( - name=pull_image_name, - tag=pull_image_tag, - ) - - if pull_status: - break - elif retry_count < (configuration.get('retry') - 1): - print ' failed, retrying... (try %s)' % (retry_count + 1) - else: - print ' failed, giving up' - - if not pull_status: - return False - - return True - - image_list = [] - for dockerfile in dockerfileList: - # Pull base image (FROM: xxx) first - if DockerfileUtility.check_if_base_image_needs_pull(dockerfile['image']['from'], configuration): - image_list.append(dockerfile['image']['from']) - - # Pull straged images (multi-stage dockerfiles) - for multiStageImage in dockerfile['image']['multiStageImages']: - if DockerfileUtility.check_if_base_image_needs_pull(multiStageImage, configuration): - image_list.append(multiStageImage) - - # filter only unique image names - image_list = set(image_list) - - # pull images - for image in set(image_list): - if not pull_image(image): - return False - - return True - - @staticmethod - def task_run(docker_client, dockerfile, configuration, task): - """ - Build one Dockerfile - """ - - # check if dockerfile is symlink, skipping tests if just a duplicate image - # image is using the same hashes - if dockerfile['image']['duplicate'] and not task.task_dep: - print ' Docker image %s is build from symlink but not included in build chain, please include %s' % (dockerfile['image']['fullname'], dockerfile['image']['from']) - print ' -> failing build' - return False - - if configuration.get('dryRun'): - print ' path: %s' % dockerfile['path'] - print ' dep: %s' % (DockerBuildTaskLoader.human_task_name_list(task.task_dep) if task.task_dep else 'none') - return True - - ## Build image - print ' -> Building image %s ' % dockerfile['image']['fullname'] - build_status = False - for retry_count in range(0, configuration.get('retry')): - build_status = docker_client.build_dockerfile( - path=dockerfile['path'], - name=dockerfile['image']['fullname'], - nocache=configuration.get('dockerBuild.noCache'), - ) - - if build_status: - break - elif retry_count < (configuration.get('retry')-1): - print ' failed, retrying... (try %s)' % (retry_count+1) - else: - print ' failed, giving up' - - if build_status and dockerfile['image']['duplicate']: - BaseTaskLoader.set_task_status(task, 'finished (duplicate)', 'success2') - - return build_status - - @staticmethod - def task_title(task): - """ - Build task title function - """ - return "Docker build %s" % (BaseTaskLoader.human_task_name(task.name)) - - @staticmethod - def task_title_dependency_puller(task): - """ - Build task title function - """ - return "Pulling dependency images" diff --git a/bin/webdevops/taskloader/DockerPullTaskLoader.py b/bin/webdevops/taskloader/DockerPullTaskLoader.py deleted file mode 100644 index 4bed18a51..000000000 --- a/bin/webdevops/taskloader/DockerPullTaskLoader.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, sys, re, copy -from .BaseTaskLoader import BaseTaskLoader -from .BaseDockerTaskLoader import BaseDockerTaskLoader -from webdevops import DockerfileUtility - -class DockerPullTaskLoader(BaseDockerTaskLoader): - - def generate_task_list(self, dockerfile_list): - """ - Generate task list for docker pull - """ - tasklist = [] - - for dockerfile in dockerfile_list: - task = { - 'name': 'DockerPull|%s' % dockerfile['image']['fullname'], - 'title': DockerPullTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerPullTaskLoader.task_run, [self.docker_client, dockerfile, self.configuration]])], - 'task_dep': [] - } - tasklist.append(task) - - # task = { - # 'name': 'FinishChain|DockerPush', - # 'title': DockerPullTaskLoader.task_title_finish, - # 'actions': [(DockerPullTaskLoader.action_chain_finish, ['docker push'])], - # 'task_dep': [task.name for task in taskList] - # } - # tasklist.append(task) - - return tasklist - - @staticmethod - def task_run(docker_client, dockerfile, configuration, task): - """ - Pull one Docker image from registry - """ - if configuration.get('dryRun'): - print ' pull: %s' % (dockerfile['image']['fullname']) - return True - - pull_status = False - for retry_count in range(0, configuration.get('retry')): - pull_status = docker_client.pull_image( - name=dockerfile['image']['name'], - tag=dockerfile['image']['tag'] - ) - - if pull_status: - break - elif retry_count < (configuration.get('retry') - 1): - print ' failed, retrying... (try %s)' % (retry_count+1) - else: - print ' failed, giving up' - - return pull_status - - @staticmethod - def task_title(task): - """ - Pull task title function - """ - return "Docker pull %s" % (BaseTaskLoader.human_task_name(task.name)) -0 diff --git a/bin/webdevops/taskloader/DockerPushTaskLoader.py b/bin/webdevops/taskloader/DockerPushTaskLoader.py deleted file mode 100644 index 974bdea7f..000000000 --- a/bin/webdevops/taskloader/DockerPushTaskLoader.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, sys, re, copy, time -from random import randint -from .BaseTaskLoader import BaseTaskLoader -from .BaseDockerTaskLoader import BaseDockerTaskLoader -from webdevops import DockerfileUtility - -class DockerPushTaskLoader(BaseDockerTaskLoader): - - def generate_task_list(self, dockerfile_list): - """ - Generate task list for docker push - """ - tasklist = [] - - for dockerfile in dockerfile_list: - task = { - 'name': 'DockerPush|%s' % dockerfile['image']['fullname'], - 'title': DockerPushTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerPushTaskLoader.task_run, [self.docker_client, dockerfile, self.configuration]])], - 'task_dep': [] - } - - for dep in dockerfile['dependency']: - task['task_dep'].append('DockerPush|%s' % dep) - - tasklist.append(task) - - # task = { - # 'name': 'FinishChain|DockerPush', - # 'title': DockerPushTaskLoader.task_title_finish, - # 'actions': [(DockerPushTaskLoader.action_chain_finish, ['docker push'])], - # 'task_dep': [task.name for task in taskList] - # } - # tasklist.append(task) - - return tasklist - - @staticmethod - def task_run(docker_client, dockerfile, configuration, task): - """ - Push one Docker image to registry - """ - if configuration.get('dryRun'): - print ' push: %s' % (dockerfile['image']['fullname']) - return True - - push_status = False - for retry_count in range(0, configuration.get('retry')): - push_status = docker_client.push_image( - name=dockerfile['image']['fullname'], - ) - - if push_status: - break - elif retry_count < (configuration.get('retry') - 1): - print ' failed, retrying... (try %s)' % (retry_count+1) - time.sleep(randint(10, 30)) - else: - print ' failed, giving up' - - return push_status - - @staticmethod - def task_title(task): - """ - Push task title function - """ - return "Docker push %s" % (BaseTaskLoader.human_task_name(task.name)) -0 diff --git a/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py b/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py deleted file mode 100644 index 0c11aff10..000000000 --- a/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, re, tempfile, json, base64 -from webdevops import Command -from .BaseDockerTaskLoader import BaseDockerTaskLoader -from .BaseTaskLoader import BaseTaskLoader -from doit.task import dict_to_task -import pytest - -class DockerTestServerspecTaskLoader(BaseDockerTaskLoader): - - def generate_task_list(self, dockerfile_list): - """ - Generate task list for docker push - """ - tasklist = [] - - for dockerfile in dockerfile_list: - task = { - 'name': 'DockerTestServerspec|%s' % dockerfile['image']['fullname'], - 'title': DockerTestServerspecTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerTestServerspecTaskLoader.task_run, [dockerfile, self.configuration]])], - 'task_dep': [] - } - - #if dockerfile['dependency']: - # task['task_dep'].append('DockerTestServerspec|%s' % dockerfile['dependency']) - - tasklist.append(task) - - # task = { - # 'name': 'FinishChain|DockerTestServerspec', - # 'title': DockerTestTestinfraTaskLoader.task_title_finish, - # 'actions': [(DockerTestTestinfraTaskLoader.action_chain_finish, ['docker test'])], - # 'task_dep': [task.name for task in taskList] - # } - # tasklist.append(task) - - return tasklist - - @staticmethod - def task_run(dockerfile, configuration, task): - """ - Run test - """ - - # check if dockerfile is symlink, skipping tests if just a duplicate image - # image is using the same hashes - if dockerfile['image']['duplicate']: - print ' Docker image %s is build from symlink and duplicate of %s' % (dockerfile['image']['fullname'], dockerfile['image']['from']) - print ' -> skipping tests' - BaseTaskLoader.set_task_status(task, 'skipped (symlink)', 'skipped') - return True - - # Check if current image is a toolimage (no daemon) - is_toolimage = False - for term in configuration.get('dockerTest.toolImages', {}): - if term in dockerfile['image']['fullname']: - is_toolimage = True - - # rspec spec file settings - spec_path = configuration.get('dockerTest.serverspec.specPath') % dockerfile['image']['imageName'] - spec_abs_path = os.path.join(configuration.get('serverspecPath'), spec_path) - - # create dockerfile - tmp_suffix = '.%s_%s_%s.tmp' % (dockerfile['image']['repository'], dockerfile['image']['imageName'], dockerfile['image']['tag']) - tmp_suffix = tmp_suffix.replace('/', '_') - test_dockerfile = tempfile.NamedTemporaryFile(prefix='Dockerfile.', suffix=tmp_suffix, dir=configuration.get('serverspecPath'), bufsize=0, delete=False) - - # serverspec conf - serverspec_conf = DockerTestServerspecTaskLoader.generate_serverspec_configuration( - path=os.path.basename(test_dockerfile.name), - dockerfile=dockerfile, - configuration=configuration, - is_toolimage=is_toolimage - ) - - # serverspec options - serverspec_opts = [] - serverspec_opts.extend([spec_path, dockerfile['image']['fullname'], base64.b64encode(json.dumps(serverspec_conf)), os.path.basename(test_dockerfile.name)]) - - # dockerfile content - dockerfile_content = DockerTestServerspecTaskLoader.generate_dockerfile( - dockerfile=dockerfile, - configuration=configuration, - is_toolimage=is_toolimage - ) - - # DryRun - if configuration.get('dryRun'): - if not os.path.isfile(spec_abs_path): - print ' no tests found' - - print ' image: %s' % (dockerfile['image']['fullname']) - print ' path: %s' % (spec_path) - print ' args: %s' % (' '.join(serverspec_opts)) - print '' - print 'spec configuration:' - print '-------------------' - print json.dumps(serverspec_conf, indent=4, sort_keys=True) - print '' - print 'Dockerfile:' - print '-----------' - print dockerfile_content - return True - - # check if we have any tests - if not os.path.isfile(spec_abs_path): - print ' no tests defined (%s)' % (spec_path) - BaseTaskLoader.set_task_status(task, 'skipped (no test)', 'skipped') - return True - - # build rspec/serverspec command - cmd = ['bash', 'serverspec.sh'] - cmd.extend(serverspec_opts) - - # create Dockerfile - with open(test_dockerfile.name, mode='w', buffering=0) as f: - f.write(dockerfile_content) - f.flush() - os.fsync(f.fileno()) - f.close() - - test_status = False - for retry_count in range(0, configuration.get('retry')): - try: - test_status = Command.execute(cmd, cwd=configuration.get('serverspecPath')) - except Exception as e: - print e - pass - - if test_status: - break - elif retry_count < (configuration.get('retry') - 1): - print ' failed, retrying... (try %s)' % (retry_count + 1) - else: - print ' failed, giving up' - - return test_status - - @staticmethod - def generate_serverspec_configuration(path, dockerfile, configuration, is_toolimage=False): - """ - Generate serverspec configuration dict - """ - ret = {} - - # add default vars - default_env_list = configuration.get('dockerTest.configuration.default', False) - if default_env_list: - ret = default_env_list.to_dict().copy() - - # parse configuration by regexp - image_configuration_regex = configuration.get('dockerTest.configuration.imageConfigurationRegex', False) - if image_configuration_regex: - parsed_configuration = ([m.groupdict() for m in image_configuration_regex.finditer(dockerfile['image']['fullname'])]) - if parsed_configuration: - ret.update(parsed_configuration[0]) - - # add docker image specific vars - image_env_list = configuration.get('dockerTest.configuration.image') - if image_env_list: - image_env_list = image_env_list.to_dict().copy() - for term in image_env_list: - regex = r'.*%s.*' % term - match = re.match(regex, dockerfile['image']['fullname'], re.IGNORECASE) - if match: - for key in image_env_list[term]: - ret[key] = str(image_env_list[term][key]) - - # add spec specific vars - ret['DOCKER_IMAGE'] = dockerfile['image']['fullname'] - ret['DOCKER_TAG'] = dockerfile['image']['tag'] - ret['DOCKERFILE'] = path - - return ret - - - @staticmethod - def generate_dockerfile(dockerfile, configuration, is_toolimage=False): - """ - Generate Dockerfile content - """ - ret = [] - - ret.append('FROM %s' % dockerfile['image']['fullname']) - ret.append('COPY conf/ /') - - if is_toolimage: - ret.append('RUN chmod +x /loop-entrypoint.sh') - ret.append('ENTRYPOINT /loop-entrypoint.sh') - - for term in configuration.get('dockerTest.dockerfile', {}): - if term in dockerfile['image']['fullname']: - ret.extend( configuration.get('dockerTest.dockerfile').get(term)) - return '\n'.join(ret) - - @staticmethod - def task_title(task): - """ - Build task title function - """ - return "Run serverspec for %s" % (BaseTaskLoader.human_task_name(task.name)) diff --git a/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py b/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py deleted file mode 100644 index a9bfe37db..000000000 --- a/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import os, re -from .BaseDockerTaskLoader import BaseDockerTaskLoader -from .BaseTaskLoader import BaseTaskLoader -from webdevops.testinfra import TestinfraDockerPlugin -from doit.task import dict_to_task -import pytest - -class DockerTestTestinfraTaskLoader(BaseDockerTaskLoader): - - def generate_task_list(self, dockerfile_list): - """ - Generate task list for docker push - """ - tasklist = [] - - for dockerfile in dockerfile_list: - task = { - 'name': 'DockerTestTestinfra|%s' % dockerfile['image']['fullname'], - 'title': DockerTestTestinfraTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerTestTestinfraTaskLoader.task_run, [dockerfile, self.configuration]])], - 'task_dep': [] - } - - for dep in dockerfile['dependency']: - task['task_dep'].append('DockerTestTestinfra|%s' % dep) - - tasklist.append(task) - - # task = { - # 'name': 'FinishChain|DockerTest', - # 'title': DockerTestTestinfraTaskLoader.task_title_finish, - # 'actions': [(DockerTestTestinfraTaskLoader.action_chain_finish, ['docker test'])], - # 'task_dep': [task.name for task in taskList] - # } - # tasklist.append(task) - - return tasklist - - @staticmethod - def task_run(dockerfile, configuration, task): - """ - Run test - """ - - test_opts = [] - - test_opts.extend(['-x', configuration.get('testinfraPath')]) - - if configuration.get('verbosity') > 1: - test_opts.extend(['-v']) - - if configuration.get('dryRun'): - print ' image: %s' % (dockerfile['image']['fullname']) - print ' args: %s' % (' '.join(test_opts)) - return True - - exitcode = pytest.main(test_opts, plugins=[TestinfraDockerPlugin(configuration=configuration, docker_image=dockerfile['image']['fullname'])]) - - if exitcode == 0: - return True - else: - return False - - @staticmethod - def task_title(task): - """ - Build task title function - """ - return "Run pytest for %s" % (BaseTaskLoader.human_task_name(task.name)) diff --git a/bin/webdevops/taskloader/__init__.py b/bin/webdevops/taskloader/__init__.py deleted file mode 100644 index e333eaa0a..000000000 --- a/bin/webdevops/taskloader/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from BaseTaskLoader import BaseTaskLoader -from BaseDockerTaskLoader import BaseDockerTaskLoader -from DockerBuildTaskLoader import DockerBuildTaskLoader -from DockerPushTaskLoader import DockerPushTaskLoader -from DockerPullTaskLoader import DockerPullTaskLoader -from DockerTestTestinfraTaskLoader import DockerTestTestinfraTaskLoader -from DockerTestServerspecTaskLoader import DockerTestServerspecTaskLoader - -__all__ = [ - 'BaseTaskLoader', - 'DockerBuildTaskLoader', - 'DockerPushTaskLoader', - 'DockerTestTestinfraTaskLoader', - 'DockerTestServerspecTaskLoader', - 'DockerPullTaskLoader', -] diff --git a/bin/webdevops/testinfra/TestinfraDockerPlugin.py b/bin/webdevops/testinfra/TestinfraDockerPlugin.py deleted file mode 100644 index ffe9ed322..000000000 --- a/bin/webdevops/testinfra/TestinfraDockerPlugin.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import pytest -import re -from webdevops import DockerfileUtility - -class TestinfraDockerPlugin: - docker_image_list = [] - configuration = False - - def __init__(self, configuration, docker_image=False): - """ - Constructor - """ - self.docker_image_list = [] - self.configuration = configuration - self.init_docker_image_list(docker_image) - - def init_docker_image_list(self, docker_image=False): - """ - Init and build list of available docker images - """ - if docker_image: - self.docker_image_list.append(docker_image) - else: - dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( - base_path=self.configuration.get('dockerPath'), - path_regex=self.configuration.get('docker.pathRegex'), - image_prefix=self.configuration.get('docker.imagePrefix'), - whitelist=self.configuration.get('whitelist'), - blacklist=self.configuration.get('blacklist'), - ) - - for image in dockerfile_list: - self.docker_image_list.append(image['image']['fullname']) - - def get_image_list_by_regexp(self, filter_regexp): - """ - Get image list by filtering via filter regexp - """ - ret = [] - - filter_regexp = re.compile(filter_regexp, re.IGNORECASE) - - for image_name in self.docker_image_list: - if filter_regexp.search(image_name): - ret.append(image_name) - - return ret - - def filter_list_by_term(self, list, term): - """ - Filter list by using blacklist term - """ - tmp = [] - for item in list: - if not term in item: - tmp.append(item) - return tmp - - def pytest_generate_tests(self, metafunc): - """ - Generate tests - """ - if "TestinfraBackend" in metafunc.fixturenames: - images = [] - - # Lookup "docker_images" marker - marker = getattr(metafunc.function, "docker_images", None) - if marker is not None: - for marker_image_name in marker.args: - images.extend(self.get_image_list_by_regexp(marker_image_name)) - - # Lookup "docker_images.blacklist" marker - marker = getattr(metafunc.function, "docker_images_blacklist", None) - if marker is not None: - for blacklist_term in marker.args: - images = self.filter_list_by_term( - list=images, - term=blacklist_term - ) - - # Check for infinite loop - marker = getattr(metafunc.function, "docker_loop", None) - if marker is not None: - images = ['{}#loop'.format(item) for item in images] - - # If the test has a destructive marker, we scope TestinfraBackend - # at function level (i.e. executing for each test). If not we scope - # at session level (i.e. all tests will share the same container) - if getattr(metafunc.function, "destructive", None) is not None: - scope = "function" - else: - scope = "session" - - metafunc.parametrize( - "TestinfraBackend", images, indirect=True, scope=scope - ) diff --git a/bin/webdevops/testinfra/__init__.py b/bin/webdevops/testinfra/__init__.py deleted file mode 100644 index 7a919e2a8..000000000 --- a/bin/webdevops/testinfra/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env/python -# -*- coding: utf-8 -*- -# -# (c) 2016 WebDevOps.io -# -# This file is part of Dockerfile Repository. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions -# of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from TestinfraDockerPlugin import TestinfraDockerPlugin - -__all__ = [ - 'TestinfraDockerPlugin', -] diff --git a/build-local.sh b/build-local.sh index 604365667..83755a21c 100755 --- a/build-local.sh +++ b/build-local.sh @@ -5,8 +5,6 @@ set -exuo pipefail ROOT_DIR=$(pwd) PHP_VERSION=8.5 -#docker run --rm -ti -v $PWD:/app -w /app webdevops/dockerfile-build-env make provision - # if first parameter is present overwrite php version + make sanity check if [ "$#" -gt 0 ]; then diff --git a/ci/composer.json b/ci/composer.json index 034ef659a..cc635821d 100644 --- a/ci/composer.json +++ b/ci/composer.json @@ -13,7 +13,8 @@ "bluem/tree": "^4.0", "symfony/console": "^6.0", "symfony/process": "^6.0", - "symfony/yaml": "^6.0" + "symfony/yaml": "^6.0", + "twig/twig": "^3.22" }, "require-dev": { "phpunit/phpunit": "^12", diff --git a/ci/composer.lock b/ci/composer.lock index 5c446f4f4..d3ba40ffe 100644 --- a/ci/composer.lock +++ b/ci/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "42a2cea5863d51343ea53957a23559a3", + "content-hash": "5afcbea67b03cbf34850d82a5a2688af", "packages": [ { "name": "bluem/tree", @@ -910,6 +910,87 @@ ], "time": "2026-11-04T07:42:47+00:00", "published-time": "2026-11-04T11:25:11+00:00" + }, + { + "name": "twig/twig", + "version": "v3.30.0", + "source": { + "type": "git", + "url": "https://github.com/twigphp/Twig.git", + "reference": "8c737079b726af72ff8ef3c595be9f6a810ea1ef" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/8c737079b726af72ff8ef3c595be9f6a810ea1ef", + "reference": "8c737079b726af72ff8ef3c595be9f6a810ea1ef", + "shasum": "" + }, + "require": { + "php": ">=8.1.0", + "symfony/deprecation-contracts": "^2.5|^3", + "symfony/polyfill-ctype": "^1.8", + "symfony/polyfill-mbstring": "^1.3" + }, + "require-dev": { + "php-cs-fixer/shim": "^3.0@stable", + "phpstan/phpstan": "^2.0@stable", + "psr/container": "^1.0|^2.0", + "symfony/phpunit-bridge": "^5.4.9|^6.4|^7.0" + }, + "type": "library", + "autoload": { + "files": [ + "src/Resources/core.php", + "src/Resources/debug.php", + "src/Resources/escaper.php", + "src/Resources/string_loader.php" + ], + "psr-4": { + "Twig\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com", + "homepage": "http://fabien.potencier.org", + "role": "Lead Developer" + }, + { + "name": "Twig Team", + "role": "Contributors" + }, + { + "name": "Armin Ronacher", + "email": "armin.ronacher@active-4.com", + "role": "Project Founder" + } + ], + "description": "Twig, the flexible, fast, and secure template language for PHP", + "homepage": "https://twig.symfony.com", + "keywords": [ + "templating" + ], + "support": { + "issues": "https://github.com/twigphp/Twig/issues", + "source": "https://github.com/twigphp/Twig/tree/v3.30.0" + }, + "funding": [ + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/twig/twig", + "type": "tidelift" + } + ], + "time": "2026-09-25T13:20:01+00:00", + "published-time": "2026-09-25T13:20:14+00:00" } ], "packages-dev": [ diff --git a/ci/console b/ci/console index 322b33915..c2b7e25f0 100755 --- a/ci/console +++ b/ci/console @@ -1,19 +1,23 @@ #!/usr/bin/env php add(new GithubCommand()); -$application->add(new ReadmeBannerCommand()); +$application->add(new GenerateCommand()); +$application->add(new GenerateDockerfileCommand()); +$application->add(new GenerateProvisionCommand()); +$application->add(new GenerateGithubCiCommand()); +$application->add(new GenerateReadmeBannerCommand()); $application->add(new BuildGlyphsCommand()); $application->run(); diff --git a/ci/resources/readme-banner/README.md b/ci/resources/readme-banner/README.md index e5b52bd10..67d8a5706 100644 --- a/ci/resources/readme-banner/README.md +++ b/ci/resources/readme-banner/README.md @@ -1,6 +1,6 @@ # README banner fonts -GitHub does not load web fonts in SVG images, so `ci/console readme-banner:generate` draws every text of the +GitHub does not load web fonts in SVG images, so `ci/console generate:readme-banner` draws every text of the README banner as outlines. `glyphs.json` holds the outlines, widths and kerning of the characters it can use, taken from [JetBrains Mono](https://github.com/JetBrains/JetBrainsMono) and [Inter](https://github.com/rsms/inter). Both fonts are licensed under the SIL Open Font License 1.1 (`OFL-JetBrainsMono.txt`, `OFL-Inter.txt`), @@ -20,14 +20,14 @@ and so is `glyphs.json`. `JetBrainsMono-Bold.ttf`. The current `glyphs.json` was built from Inter v20 and JetBrains Mono v24, the versions in the URLs. -2. Write `glyphs.json` and render the banner again: +2. Write `glyphs.json` and render the banner again. `ci/console` needs the PHP tools that `make generate` installs: ``` docker run --rm -ti -v $PWD:/app -w /app webdevops/php:8.4-alpine ci/console readme-banner:build-glyphs \ mono-bold=ci/var/fonts/JetBrainsMono-Bold.ttf mono-medium=ci/var/fonts/JetBrainsMono-Medium.ttf \ sans-regular=ci/var/fonts/Inter-Regular.ttf sans-medium=ci/var/fonts/Inter-Medium.ttf \ sans-semibold=ci/var/fonts/Inter-SemiBold.ttf - docker run --rm -ti -v $PWD:/app -w /app webdevops/php:8.4-alpine ci/console readme-banner:generate + make generate ``` `readme-banner:build-glyphs` reads fonts with TrueType outlines (`.ttf`). For files it cannot read, such as `.otf` diff --git a/ci/src/Commands/GenerateCommand.php b/ci/src/Commands/GenerateCommand.php new file mode 100644 index 000000000..069f1d13a --- /dev/null +++ b/ci/src/Commands/GenerateCommand.php @@ -0,0 +1,45 @@ +getApplication(); + if ($application === null) { + throw new RuntimeException('generate runs the other commands and needs the console application.'); + } + + foreach (self::COMMANDS as $name) { + $output->writeln(sprintf('%s', $name)); + $status = $application->find($name)->run(new ArrayInput([]), $output); + if ($status !== Command::SUCCESS) { + return $status; + } + } + + return Command::SUCCESS; + } +} diff --git a/ci/src/Commands/GenerateDockerfileCommand.php b/ci/src/Commands/GenerateDockerfileCommand.php new file mode 100644 index 000000000..b7006117b --- /dev/null +++ b/ci/src/Commands/GenerateDockerfileCommand.php @@ -0,0 +1,32 @@ +//Dockerfile from the templates. + */ +#[AsCommand(name: 'generate:dockerfile', description: 'Renders the Dockerfiles under docker/ from the Dockerfile.twig next to them')] +class GenerateDockerfileCommand extends Command +{ + /** + * @param string $rootPath the repository root + */ + public function __construct(private readonly string $rootPath = __DIR__ . '/../../..') + { + parent::__construct(); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + new DockerfileGenerator($this->rootPath)->generate($output->writeln(...)); + return Command::SUCCESS; + } +} diff --git a/ci/src/Commands/GithubCommand.php b/ci/src/Commands/GenerateGithubCiCommand.php similarity index 76% rename from ci/src/Commands/GithubCommand.php rename to ci/src/Commands/GenerateGithubCiCommand.php index 00f6dc69d..88d1d167f 100644 --- a/ci/src/Commands/GithubCommand.php +++ b/ci/src/Commands/GenerateGithubCiCommand.php @@ -4,6 +4,7 @@ use BlueM\Tree; use BlueM\Tree\Node; +use RuntimeException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; @@ -13,14 +14,19 @@ use Webdevops\Build\FileReader; use Webdevops\Build\GithubJobBuilder; +use function array_column; use function array_filter; use function array_map; use function array_values; +use function dirname; use function file; use function file_exists; use function file_put_contents; +use function in_array; +use function is_array; use function is_string; use function ksort; +use function sprintf; use function str_contains; use function str_pad; use function trim; @@ -28,8 +34,8 @@ /** * Writes .github/workflows/build.yaml: a build and a publish job for every Dockerfile under docker/. */ -#[AsCommand(name: 'github:generate-ci')] -class GithubCommand extends Command +#[AsCommand(name: 'generate:github-ci', description: 'Writes .github/workflows/build.yaml: a build and a publish job for every Dockerfile under docker/')] +class GenerateGithubCiCommand extends Command { private OutputInterface $output; @@ -163,6 +169,37 @@ private function buildTree(): Tree $data[] = $this->fileReader->getInfo($dockerfile); } + $this->checkInternalImages($data); return new Tree($data); } + + /** + * Every webdevops image that an image builds on (FROM) or copies from (COPY --from) has to be built here too. + * The tree would otherwise only report a node that "points to non-existent parent". + * + * @param list> $nodes + */ + private function checkInternalImages(array $nodes): void + { + $built = array_column($nodes, 'id'); + foreach ($nodes as $node) { + // an image with an outside base image is built after the toolbox, its parent in the tree + $needed = is_string($node['parent']) ? [$node['parent'] => $node['imageParent'] === 0 ? 'is built after' : 'builds on'] : []; + foreach (is_array($node['imageDependencies']) ? $node['imageDependencies'] : [] as $image) { + $needed[(string)$image] ??= 'copies files from'; + } + + foreach ($needed as $image => $relation) { + if (!in_array($image, $built, true)) { + throw new RuntimeException(sprintf( + '%1$s (%2$s) %3$s %4$s, which is not built any more. Remove %2$s too, or keep building %4$s.', + $node['id'], + dirname((string)$node['file']), + $relation, + $image, + )); + } + } + } + } } diff --git a/ci/src/Commands/GenerateProvisionCommand.php b/ci/src/Commands/GenerateProvisionCommand.php new file mode 100644 index 000000000..6e8f8a930 --- /dev/null +++ b/ci/src/Commands/GenerateProvisionCommand.php @@ -0,0 +1,32 @@ +//conf/, as conf/provision.yml says. + */ +#[AsCommand(name: 'generate:provision', description: 'Copies provisioning/ into the conf/ directories under docker/, as conf/provision.yml says')] +class GenerateProvisionCommand extends Command +{ + /** + * @param string $rootPath the repository root + */ + public function __construct(private readonly string $rootPath = __DIR__ . '/../../..') + { + parent::__construct(); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + new Provisioner($this->rootPath)->provision($output->writeln(...)); + return Command::SUCCESS; + } +} diff --git a/ci/src/Commands/ReadmeBannerCommand.php b/ci/src/Commands/GenerateReadmeBannerCommand.php similarity index 94% rename from ci/src/Commands/ReadmeBannerCommand.php rename to ci/src/Commands/GenerateReadmeBannerCommand.php index a8ce43146..5e7a2d292 100644 --- a/ci/src/Commands/ReadmeBannerCommand.php +++ b/ci/src/Commands/GenerateReadmeBannerCommand.php @@ -16,8 +16,8 @@ * Renders the banner at the top of README.md from the images that are built. * Its texts are in conf/readme-banner.yml. */ -#[AsCommand(name: 'readme-banner:generate')] -class ReadmeBannerCommand extends Command +#[AsCommand(name: 'generate:readme-banner')] +class GenerateReadmeBannerCommand extends Command { /** * @param string $rootPath the repository root diff --git a/ci/src/Dockerfile/DockerfileGenerator.php b/ci/src/Dockerfile/DockerfileGenerator.php new file mode 100644 index 000000000..ae146d306 --- /dev/null +++ b/ci/src/Dockerfile/DockerfileGenerator.php @@ -0,0 +1,79 @@ +//Dockerfile from the Dockerfile.twig next to it. + * + * template/Dockerfile/layout.twig writes the header and includes the Dockerfile.twig, which imports the macros it + * uses from template/Dockerfile/, e.g. {% import 'docker.twig' as docker %}. Directories that are symbolic links, + * such as docker/php/8.5, are left out; ImageLinks keeps them up to date. + */ +final readonly class DockerfileGenerator +{ + /** + * @param string $rootPath the repository root + */ + public function __construct(private string $rootPath = __DIR__ . '/../../..') + { + } + + /** + * Renders every Dockerfile and updates the links of ImageLinks. A tag directory without Dockerfile.twig is an + * error: nothing would update its files, but generate:github-ci would still build and publish a Dockerfile in it. + * The .DS_Store files that the macOS Finder leaves under docker/ are removed first. + * + * @param ?callable(string): void $log receives the image and tag of each Dockerfile + */ + public function generate(?callable $log = null): void + { + // a .DS_Store would keep the directory of a removed tag from being empty, and get into a local build + FinderFile::removeUnder($this->rootPath . '/docker'); + $tagDirectories = new TagDirectories($this->rootPath); + $withoutTemplate = $tagDirectories->withoutTemplate(); + if ($withoutTemplate !== []) { + throw new RuntimeException(sprintf( + 'No %s in %s. Add the template, or remove the directory to stop building the tag.', + TagDirectories::TEMPLATE, + implode(', ', $withoutTemplate), + )); + } + + new ImageLinks($this->rootPath)->update($tagDirectories); + + $loader = new FilesystemLoader($this->rootPath . '/template/Dockerfile'); + $loader->addPath($this->rootPath . '/docker', 'docker'); + + $twig = new Environment($loader, ['autoescape' => false, 'strict_variables' => true]); + + foreach ($tagDirectories->tags() as $directory) { + [, $image, $tag] = explode('/', $directory); + $dockerfile = $twig->render('layout.twig', [ + 'Dockerfile' => ['image' => $image, 'tag' => $tag], + // docker///Dockerfile.twig is @docker///Dockerfile.twig + 'template' => '@' . $directory . '/' . TagDirectories::TEMPLATE, + ]); + + if (file_put_contents($this->rootPath . '/' . $directory . '/Dockerfile', $dockerfile) === false) { + throw new RuntimeException(sprintf('Cannot write %s/Dockerfile.', $directory)); + } + + if ($log !== null) { + $log(sprintf('webdevops/%s:%s', $image, $tag)); + } + } + } +} diff --git a/ci/src/Dockerfile/ImageLinks.php b/ci/src/Dockerfile/ImageLinks.php new file mode 100644 index 000000000..b4ae6be32 --- /dev/null +++ b/ci/src/Dockerfile/ImageLinks.php @@ -0,0 +1,89 @@ + links to docker/php-official/ for every tag of php-official, and no other link stays there. + */ +final readonly class ImageLinks +{ + /** + * The images whose tags are links, each with the image they link to. + */ + public const array LINKS = ['php' => 'php-official']; + + /** + * @param string $rootPath the repository root + */ + public function __construct(private string $rootPath) + { + } + + public function update(TagDirectories $tagDirectories): void + { + foreach (self::LINKS as $image => $target) { + $directory = $this->rootPath . '/docker/' . $image; + if (!is_dir($directory) && !mkdir($directory) && !is_dir($directory)) { + throw new RuntimeException(sprintf('Cannot create docker/%s.', $image)); + } + + $tags = array_map(basename(...), $tagDirectories->tags($target)); + foreach (scandir($directory) ?: [] as $entry) { + if (is_link($directory . '/' . $entry) && !in_array($entry, $tags, true) && !unlink($directory . '/' . $entry)) { + throw new RuntimeException(sprintf('Cannot remove docker/%s/%s.', $image, $entry)); + } + } + + foreach ($tags as $tag) { + $this->link($image . '/' . $tag, '../' . $target . '/' . $tag); + } + } + } + + /** + * Makes docker/$path a symbolic link to $target. + */ + private function link(string $path, string $target): void + { + $link = $this->rootPath . '/docker/' . $path; + if (is_link($link)) { + if (readlink($link) === $target) { + return; + } + + if (!unlink($link)) { + throw new RuntimeException(sprintf('Cannot remove docker/%s.', $path)); + } + } elseif (is_file($link) && file_get_contents($link) === $target) { + // Git without symbolic links (core.symlinks=false) checks a link out as a file that holds its target + return; + } elseif (file_exists($link)) { + throw new RuntimeException(sprintf('docker/%s has to be a symbolic link to %s, but is a file or a directory.', $path, $target)); + } + + if (!symlink($target, $link)) { + throw new RuntimeException(sprintf('Cannot link docker/%s to %s.', $path, $target)); + } + } +} diff --git a/ci/src/FileReader.php b/ci/src/FileReader.php index 26db64504..1c9c96a6b 100644 --- a/ci/src/FileReader.php +++ b/ci/src/FileReader.php @@ -2,10 +2,7 @@ namespace Webdevops\Build; -use RecursiveDirectoryIterator; -use RecursiveIteratorIterator; use RuntimeException; -use SplFileInfo; use Symfony\Component\Yaml\Yaml; class FileReader @@ -38,22 +35,21 @@ public function autoLatestTag(): string } /** - * Every Dockerfile under docker/, relative to the repository root and sorted, e.g. docker/base/alpine/Dockerfile. - * Directories that are symbolic links, such as docker/php/8.5, are left out: they build the same Dockerfile. + * The Dockerfile of every tag under docker/ (see TagDirectories), relative to the repository root and sorted, e.g. + * docker/base/alpine/Dockerfile. Tags whose Dockerfile was not generated yet are left out. Directories that are + * symbolic links, such as docker/php/8.5, are left out too: they build the same Dockerfile. * * @return list */ public function collectDockerfiles(): array { $dockerfiles = []; - $files = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($this->rootPath . '/docker')); - foreach ($files as $file) { - if ($file instanceof SplFileInfo && preg_match('/^.*Dockerfile$/i', $file->getPathname())) { - $dockerfiles[] = substr($file->getPathname(), strlen($this->rootPath) + 1); + foreach (new TagDirectories($this->rootPath)->tags() as $directory) { + if (is_file($this->rootPath . '/' . $directory . '/Dockerfile')) { + $dockerfiles[] = $directory . '/Dockerfile'; } } - sort($dockerfiles); return $dockerfiles; } diff --git a/ci/src/FinderFile.php b/ci/src/FinderFile.php new file mode 100644 index 000000000..1c1363011 --- /dev/null +++ b/ci/src/FinderFile.php @@ -0,0 +1,41 @@ +getFilename() === self::NAME && $file->isFile() && !unlink($file->getPathname())) { + throw new RuntimeException(sprintf('Cannot remove %s.', $file->getPathname())); + } + } + } +} diff --git a/ci/src/GithubJobBuilder.php b/ci/src/GithubJobBuilder.php index e60ee0121..9fab5485d 100644 --- a/ci/src/GithubJobBuilder.php +++ b/ci/src/GithubJobBuilder.php @@ -441,12 +441,8 @@ public function getCodeCheckConfig(): array ], [ 'name' => 'Run GrumPHP', - 'run' => implode("\n", [ - 'docker run --rm -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine composer install', - // GrumPHP's BOM check needs "file"; git needs to trust the checkout, which belongs to another user - 'docker run --rm -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine sh -c \'' - . "apk add --no-cache file && git config --global --add safe.directory /app && vendor/bin/grumphp run --no-interaction'", - ]), + // The command of the documentation + 'run' => 'make check', ], ], ]; @@ -468,26 +464,14 @@ public function getValidationConfig(): array 'with' => ['persist-credentials' => false], ], [ - 'name' => 'Validate that template/* are used to generate Dockerfiles', - 'run' => implode("\n", [ - 'docker run --rm -v $PWD:/app -w /app webdevops/dockerfile-build-env make provision', - 'git diff --exit-code --color=always', - ]), - ], - [ - 'name' => 'Validate .github/workflows/build.yaml is up to date', - 'run' => implode("\n", [ - // The generators need no dev tools; check-ci-code installs those - 'docker run --rm -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine composer install --no-dev', - 'docker run --rm -v $PWD:/app -w /app webdevops/php:8.4-alpine ci/console github:generate-ci', - 'git diff --exit-code --color=always', - ]), - ], - [ - 'name' => 'Validate the README banner is up to date', + 'name' => 'Validate that the generated files are up to date', 'run' => implode("\n", [ - 'docker run --rm -v $PWD:/app -w /app webdevops/php:8.4-alpine ci/console readme-banner:generate', + // The command of the documentation + 'make generate', + // A changed or a new file fails the step + 'git status --porcelain', 'git diff --exit-code --color=always', + 'test -z "$(git status --porcelain)"', ]), ], ], diff --git a/ci/src/Provision/Provisioner.php b/ci/src/Provision/Provisioner.php new file mode 100644 index 000000000..caf444557 --- /dev/null +++ b/ci/src/Provision/Provisioner.php @@ -0,0 +1,235 @@ +, sources: array, directories: list}>}> + */ +final readonly class Provisioner +{ + /** + * @param string $rootPath the repository root + */ + public function __construct(private string $rootPath = __DIR__ . '/../../..') + { + } + + /** + * Provisions every image of conf/provision.yml. + * + * @param ?callable(string): void $log receives a line for each source that is copied + */ + public function provision(?callable $log = null): void + { + foreach ($this->plan() as $image => $plan) { + foreach ($plan['directories'] as $directory) { + $this->remove($this->rootPath . '/' . $directory . '/conf'); + } + + foreach ($plan['sources'] as $source => $copy) { + foreach ($copy['directories'] as $directory) { + $this->copyTree($this->rootPath . '/provisioning/' . $source, $this->rootPath . '/' . $directory . '/conf'); + } + + if ($log !== null) { + $log(sprintf('%s => webdevops/%s:%s', $source, $image, implode(', ', $copy['tags']))); + } + } + } + } + + /** + * What conf/provision.yml copies, in its order: for each image the tag directories whose conf/ is emptied, and for + * each source directory its tag patterns and the tag directories they match. + * + * @return Plan + */ + private function plan(): array + { + $images = Yaml::parseFile($this->rootPath . '/conf/provision.yml')['provision'] ?? null; + if (!is_array($images)) { + throw new RuntimeException('conf/provision.yml has no "provision" section.'); + } + + $tagDirectories = new TagDirectories($this->rootPath); + $plan = []; + foreach ($images as $image => $settings) { + $image = (string)$image; + // only "configuration", so that a misspelt key does not just empty the conf/ directories + if (!is_array($settings) || array_keys($settings) !== ['configuration'] || !is_array($settings['configuration'])) { + throw new RuntimeException(sprintf('conf/provision.yml: webdevops/%s needs "configuration", which maps directories of provisioning/ to tags, and no other setting.', $image)); + } + + $sources = $settings['configuration']; + + $directories = $tagDirectories->tags($image); + $plan[$image] = ['directories' => $directories, 'sources' => []]; + foreach ($sources as $source => $tags) { + $source = (string)$source; + if (!is_dir($this->rootPath . '/provisioning/' . $source)) { + throw new RuntimeException(sprintf('conf/provision.yml copies provisioning/%s into webdevops/%s, but it does not exist.', $source, $image)); + } + + $tags = is_array($tags) ? array_values($tags) : [$tags]; + if ($tags === []) { + throw new RuntimeException(sprintf('conf/provision.yml has no tag for provisioning/%s in webdevops/%s.', $source, $image)); + } + + $matches = []; + foreach ($tags as $tag) { + if (!is_string($tag) || $tag === '') { + throw new RuntimeException(sprintf('conf/provision.yml has an invalid tag for provisioning/%s in webdevops/%s.', $source, $image)); + } + + $tagMatches = $this->matching($directories, $tag); + if ($tagMatches === []) { + throw new RuntimeException(sprintf('conf/provision.yml copies provisioning/%s into webdevops/%s:%s, but webdevops/%s has no such tag under docker/.', $source, $image, $tag, $image)); + } + + $matches = array_merge($matches, $tagMatches); + } + + $plan[$image]['sources'][$source] = ['tags' => $tags, 'directories' => array_values(array_unique($matches))]; + } + } + + return $plan; + } + + /** + * The directories whose tag matches $tag, regardless of case. * in $tag stands for one or more characters. + * + * @param list $directories docker// + * @return list + */ + private function matching(array $directories, string $tag): array + { + $pattern = '#^' . implode('[^/]+', array_map(fn (string $part): string => preg_quote($part, '#'), explode('*', $tag))) . '$#i'; + return array_values(array_filter($directories, fn (string $directory): bool => preg_match($pattern, basename($directory)) === 1)); + } + + /** + * Copies the content of $source into $target, following symbolic links and keeping the permissions of the files. + * The .DS_Store files of the macOS Finder are left out, so that they do not end up in an image. + */ + private function copyTree(string $source, string $target): void + { + if (!is_dir($target) && !mkdir($target, 0777, true) && !is_dir($target)) { + throw new RuntimeException(sprintf('Cannot create %s.', $this->relative($target))); + } + + foreach ($this->entries($source) as $entry) { + if ($entry === FinderFile::NAME) { + continue; + } + + $from = $source . '/' . $entry; + $to = $target . '/' . $entry; + if (is_dir($from)) { + $this->copyTree($from, $to); + continue; + } + + if (!copy($from, $to) || !chmod($to, fileperms($from) & 0777)) { + throw new RuntimeException(sprintf('Cannot copy %s to %s.', $this->relative($from), $this->relative($to))); + } + } + } + + private function remove(string $path): void + { + if (!is_dir($path) || is_link($path)) { + return; + } + + $files = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS), + RecursiveIteratorIterator::CHILD_FIRST, + ); + foreach ($files as $file) { + if ($file instanceof SplFileInfo) { + $removed = $file->isDir() && !$file->isLink() ? rmdir($file->getPathname()) : unlink($file->getPathname()); + if (!$removed) { + throw new RuntimeException(sprintf('Cannot remove %s.', $this->relative($file->getPathname()))); + } + } + } + + if (!rmdir($path)) { + throw new RuntimeException(sprintf('Cannot remove %s.', $this->relative($path))); + } + } + + /** + * The names in a directory without . and .., sorted. + * + * @return list + */ + private function entries(string $directory): array + { + $entries = []; + foreach (scandir($directory) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + $entries[] = $entry; + } + } + + sort($entries); + return $entries; + } + + /** + * $path relative to the repository root, for messages. + */ + private function relative(string $path): string + { + $root = $this->rootPath . '/'; + return str_starts_with($path, $root) ? substr($path, strlen($root)) : $path; + } +} diff --git a/ci/src/TagDirectories.php b/ci/src/TagDirectories.php new file mode 100644 index 000000000..b13e2b4b4 --- /dev/null +++ b/ci/src/TagDirectories.php @@ -0,0 +1,118 @@ +// with a Dockerfile.twig, the template its + * Dockerfile is rendered from. Directories that are symbolic links, such as docker/php/8.5, are left out: they show + * a tag of the image they link to. docker/ is read once, when the object is created. + */ +final readonly class TagDirectories +{ + /** + * The template next to each Dockerfile. + */ + public const string TEMPLATE = 'Dockerfile.twig'; + + /** + * @var list every docker// directory, sorted like the Dockerfiles in them + */ + private array $directories; + + /** + * @param string $rootPath the repository root + */ + public function __construct(private string $rootPath) + { + $this->directories = $this->scan(); + } + + /** + * docker// of every tag, or of every tag of $image, relative to the repository root. + * Sorted like the Dockerfiles in them, so docker/php/8.5-alpine comes before docker/php/8.5. + * + * @return list + */ + public function tags(?string $image = null): array + { + return array_values(array_filter( + $this->directories, + fn (string $directory): bool => ($image === null || str_starts_with($directory, 'docker/' . $image . '/')) + && $this->hasTemplate($directory), + )); + } + + /** + * docker// of every directory that is not empty but has no template, sorted like tags(): files that + * are left over, such as a Dockerfile or conf/ of a tag whose template was removed. + * + * @return list + */ + public function withoutTemplate(): array + { + return array_values(array_filter( + $this->directories, + fn (string $directory): bool => !$this->hasTemplate($directory) + && array_diff(scandir($this->rootPath . '/' . $directory) ?: [], ['.', '..']) !== [], + )); + } + + private function hasTemplate(string $directory): bool + { + return is_file($this->rootPath . '/' . $directory . '/' . self::TEMPLATE); + } + + /** + * @return list + */ + private function scan(): array + { + $directories = []; + foreach ($this->subdirectories('docker') as $image) { + foreach ($this->subdirectories('docker/' . $image) as $tag) { + // with the slash, sort() orders the directories like the files in them + $directories[] = 'docker/' . $image . '/' . $tag . '/'; + } + } + + sort($directories); + return array_map(fn (string $directory): string => rtrim($directory, '/'), $directories); + } + + /** + * The names of the subdirectories of $directory that are no symbolic links. Empty if $directory does not exist + * or is a symbolic link itself. + * + * @return list + */ + private function subdirectories(string $directory): array + { + $path = $this->rootPath . '/' . $directory; + if (!is_dir($path) || is_link($path)) { + return []; + } + + $names = []; + foreach (scandir($path) ?: [] as $name) { + if ($name !== '.' && $name !== '..' && is_dir($path . '/' . $name) && !is_link($path . '/' . $name)) { + $names[] = $name; + } + } + + return $names; + } +} diff --git a/ci/tests/Commands/GenerateCommandTest.php b/ci/tests/Commands/GenerateCommandTest.php new file mode 100644 index 000000000..102eee8c4 --- /dev/null +++ b/ci/tests/Commands/GenerateCommandTest.php @@ -0,0 +1,95 @@ +repository = new TemporaryRepository(); + $this->repository->write('.github/workflows/build.yaml', ''); + } + + protected function tearDown(): void + { + $this->repository->remove(); + } + + public function testWritesEveryGeneratedFile(): void + { + $this->repository->write('docker/toolbox/latest/Dockerfile', ''); + + $tester = new CommandTester($this->application()->find('generate')); + + self::assertSame(Command::SUCCESS, $tester->execute([])); + self::assertStringContainsString('RUN mkdir /baselayout', $this->repository->read('docker/toolbox/latest/Dockerfile')); + self::assertSame("alpine\n", $this->repository->read('docker/base/alpine/conf/etc/os')); + self::assertStringContainsString('toolbox_latest:', $this->repository->read('.github/workflows/build.yaml')); + self::assertStringContainsString('repository->read('docs/assets/banner-light.svg')); + } + + public function testRunsTheGeneratorsInOrder(): void + { + $tester = new CommandTester($this->application()->find('generate')); + $tester->execute([]); + + self::assertMatchesRegularExpression( + '/^generate:dockerfile\n(webdevops\/.*\n)+generate:provision\n(.* => .*\n)+generate:github-ci\n(.*\n)+generate:readme-banner\nWrote /', + $tester->getDisplay(), + ); + } + + public function testStopsAtTheFirstFailure(): void + { + $application = $this->application(); + $application->add(new #[AsCommand(name: 'generate:provision')] class () extends Command { + protected function execute(InputInterface $input, OutputInterface $output): int + { + return Command::FAILURE; + } + }); + $tester = new CommandTester($application->find('generate')); + + self::assertSame(Command::FAILURE, $tester->execute([])); + self::assertStringNotContainsString('generate:github-ci', $tester->getDisplay()); + } + + public function testNeedsTheApplication(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('generate runs the other commands and needs the console application.'); + new CommandTester(new GenerateCommand())->execute([]); + } + + private function application(): Application + { + $application = new Application(); + $application->add(new GenerateCommand()); + $application->add(new GenerateDockerfileCommand($this->repository->path)); + $application->add(new GenerateProvisionCommand($this->repository->path)); + $application->add(new GenerateGithubCiCommand($this->repository->path)); + $application->add(new GenerateReadmeBannerCommand($this->repository->path)); + return $application; + } +} diff --git a/ci/tests/Commands/GenerateDockerfileCommandTest.php b/ci/tests/Commands/GenerateDockerfileCommandTest.php new file mode 100644 index 000000000..a56b896f1 --- /dev/null +++ b/ci/tests/Commands/GenerateDockerfileCommandTest.php @@ -0,0 +1,31 @@ +write('docker/toolbox/latest/Dockerfile', ''); + $tester = new CommandTester(new GenerateDockerfileCommand($repository->path)); + + self::assertSame(Command::SUCCESS, $tester->execute([])); + self::assertStringContainsString("webdevops/toolbox:latest\n", $tester->getDisplay()); + self::assertStringContainsString("FROM alpine:3.22\n\nRUN mkdir /baselayout\n", $repository->read('docker/toolbox/latest/Dockerfile')); + } finally { + $repository->remove(); + } + } +} diff --git a/ci/tests/Commands/GithubCommandTest.php b/ci/tests/Commands/GenerateGithubCiCommandTest.php similarity index 77% rename from ci/tests/Commands/GithubCommandTest.php rename to ci/tests/Commands/GenerateGithubCiCommandTest.php index 5ee9a604d..37bbea61f 100644 --- a/ci/tests/Commands/GithubCommandTest.php +++ b/ci/tests/Commands/GenerateGithubCiCommandTest.php @@ -7,19 +7,22 @@ use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +use RuntimeException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Tester\CommandTester; use Symfony\Component\Yaml\Yaml; -use Webdevops\Build\Commands\GithubCommand; +use Webdevops\Build\Commands\GenerateGithubCiCommand; use Webdevops\Build\Tests\TemporaryRepository; use function array_column; use function array_diff; use function array_keys; use function array_values; +use function strpos; +use function substr; -#[CoversClass(GithubCommand::class)] -final class GithubCommandTest extends TestCase +#[CoversClass(GenerateGithubCiCommand::class)] +final class GenerateGithubCiCommandTest extends TestCase { private const array ALL_JOBS = [ 'validate-automation', @@ -110,7 +113,7 @@ public function testPublishesTheAliases(): void public function testPrintsTheImageTree(): void { - $tester = new CommandTester(new GithubCommand($this->repository->path)); + $tester = new CommandTester(new GenerateGithubCiCommand($this->repository->path)); $tester->execute([]); $display = $tester->getDisplay(); @@ -119,6 +122,37 @@ public function testPrintsTheImageTree(): void self::assertStringContainsString("\t\t\tProcessing webdevops/php-nginx-dev:8.5-alpine\n", $display); } + /** + * @return iterable + */ + public static function missingInternalImages(): iterable + { + // e.g. php-official 8.5-alpine was removed, but not php-nginx 8.5-alpine + yield 'base image' => [ + 'docker/php-nginx/8.5-alpine/Dockerfile', + "FROM webdevops/php:8.6-alpine\n", + 'webdevops/php-nginx:8.5-alpine (docker/php-nginx/8.5-alpine) builds on webdevops/php:8.6-alpine, which is not built any more. ' + . 'Remove docker/php-nginx/8.5-alpine too, or keep building webdevops/php:8.6-alpine.', + ]; + yield 'image to copy from' => [ + 'docker/base/alpine/Dockerfile', + "FROM alpine:3.22\nCOPY --from=webdevops/tools:latest /bin/tool /bin/\n", + 'webdevops/base:alpine (docker/base/alpine) copies files from webdevops/tools:latest, which is not built any more. ' + . 'Remove docker/base/alpine too, or keep building webdevops/tools:latest.', + ]; + } + + #[DataProvider('missingInternalImages')] + public function testMissingInternalImageIsAnError(string $dockerfile, string $instructions, string $message): void + { + $header = $this->repository->read($dockerfile); + $this->repository->write($dockerfile, substr($header, 0, (int)strpos($header, 'FROM ')) . $instructions); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage($message); + new CommandTester(new GenerateGithubCiCommand($this->repository->path))->execute([]); + } + /** * @return iterable, 1: list}> */ @@ -139,7 +173,7 @@ public static function blacklists(): iterable #[DataProvider('blacklists')] public function testBlacklistedImagesAreLeftOut(array $blacklist, array $removed): void { - $tester = new CommandTester(new GithubCommand($this->repository->path)); + $tester = new CommandTester(new GenerateGithubCiCommand($this->repository->path)); $tester->execute(['--blacklist' => $blacklist]); self::assertSame($this->jobsWithout($removed), array_keys($this->workflow()['jobs'])); @@ -162,7 +196,7 @@ public static function emptyBlacklists(): iterable #[DataProvider('emptyBlacklists')] public function testEmptyBlacklistEntriesMatchNothing(array $blacklist): void { - $tester = new CommandTester(new GithubCommand($this->repository->path)); + $tester = new CommandTester(new GenerateGithubCiCommand($this->repository->path)); self::assertSame(Command::SUCCESS, $tester->execute(['--blacklist' => $blacklist])); self::assertSame(self::ALL_JOBS, array_keys($this->workflow()['jobs'])); @@ -179,7 +213,7 @@ public function testBlacklistFile(): void public function testBlacklistOptionReplacesTheFile(): void { $this->repository->write('ci/BLACKLIST', "php-nginx-dev\n"); - $tester = new CommandTester(new GithubCommand($this->repository->path)); + $tester = new CommandTester(new GenerateGithubCiCommand($this->repository->path)); $tester->execute(['--blacklist' => ['php:8.4']]); self::assertSame($this->jobsWithout(['php_8-4']), array_keys($this->workflow()['jobs'])); @@ -190,7 +224,7 @@ public function testBlacklistOptionReplacesTheFile(): void */ private function generate(): array { - $tester = new CommandTester(new GithubCommand($this->repository->path)); + $tester = new CommandTester(new GenerateGithubCiCommand($this->repository->path)); self::assertSame(Command::SUCCESS, $tester->execute([])); return $this->workflow(); } diff --git a/ci/tests/Commands/GenerateProvisionCommandTest.php b/ci/tests/Commands/GenerateProvisionCommandTest.php new file mode 100644 index 000000000..a6783b99b --- /dev/null +++ b/ci/tests/Commands/GenerateProvisionCommandTest.php @@ -0,0 +1,30 @@ +path)); + + self::assertSame(Command::SUCCESS, $tester->execute([])); + self::assertStringContainsString("php/official-alpine => webdevops/php-official:*-alpine\n", $tester->getDisplay()); + self::assertSame("alpine\n", $repository->read('docker/base/alpine/conf/etc/os')); + } finally { + $repository->remove(); + } + } +} diff --git a/ci/tests/Commands/ReadmeBannerCommandTest.php b/ci/tests/Commands/GenerateReadmeBannerCommandTest.php similarity index 84% rename from ci/tests/Commands/ReadmeBannerCommandTest.php rename to ci/tests/Commands/GenerateReadmeBannerCommandTest.php index a18d3681f..76e3776fb 100644 --- a/ci/tests/Commands/ReadmeBannerCommandTest.php +++ b/ci/tests/Commands/GenerateReadmeBannerCommandTest.php @@ -9,11 +9,11 @@ use SimpleXMLElement; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Tester\CommandTester; -use Webdevops\Build\Commands\ReadmeBannerCommand; +use Webdevops\Build\Commands\GenerateReadmeBannerCommand; use Webdevops\Build\Tests\TemporaryRepository; -#[CoversClass(ReadmeBannerCommand::class)] -final class ReadmeBannerCommandTest extends TestCase +#[CoversClass(GenerateReadmeBannerCommand::class)] +final class GenerateReadmeBannerCommandTest extends TestCase { private TemporaryRepository $repository; @@ -29,7 +29,7 @@ protected function tearDown(): void public function testRendersTheBannerFromTheDockerfiles(): void { - $tester = new CommandTester(new ReadmeBannerCommand($this->repository->path)); + $tester = new CommandTester(new GenerateReadmeBannerCommand($this->repository->path)); self::assertSame(Command::SUCCESS, $tester->execute([])); self::assertSame("Wrote docs/assets/banner-light.svg\nWrote docs/assets/banner-dark.svg\n", $tester->getDisplay()); diff --git a/ci/tests/Dockerfile/DockerfileGeneratorTest.php b/ci/tests/Dockerfile/DockerfileGeneratorTest.php new file mode 100644 index 000000000..721633757 --- /dev/null +++ b/ci/tests/Dockerfile/DockerfileGeneratorTest.php @@ -0,0 +1,163 @@ +repository = new TemporaryRepository(); + } + + protected function tearDown(): void + { + $this->repository->remove(); + } + + /** + * The Dockerfiles of the test repository were rendered by Jinja from the templates before they became Twig templates; + * only docker/base/alpine lost the trailing space after its FROM line, when the comment there moved above it. + * docker/base/alpine uses a comment, if/elseif/else, a macro that imports another file and a macro whose output + * starts with a line break. + */ + public function testRendersTheDockerfiles(): void + { + $dockerfiles = new FileReader($this->repository->path)->collectDockerfiles(); + foreach ($dockerfiles as $dockerfile) { + $this->repository->write($dockerfile, ''); + } + + $this->generate(); + + self::assertCount(9, $dockerfiles); + foreach ($dockerfiles as $dockerfile) { + self::assertSame(file_get_contents(TemporaryRepository::FIXTURE . '/' . $dockerfile), $this->repository->read($dockerfile), $dockerfile); + } + } + + public function testLogsEachDockerfileAndLeavesOutLinkedDirectories(): void + { + // docker/php/8.4 links to docker/php-official/8.4 + self::assertSame([ + 'webdevops/base-app:ubuntu-22.04', + 'webdevops/base:alpine', + 'webdevops/base:ubuntu-22.04', + 'webdevops/php-nginx-dev:8.5-alpine', + 'webdevops/php-nginx:8.5-alpine', + 'webdevops/php-official:8.4', + 'webdevops/php-official:8.5-alpine', + 'webdevops/php-official:8.5', + 'webdevops/toolbox:latest', + ], $this->generate()); + } + + public function testTemplatesOutsideAnImageTagDirectoryAreLeftOut(): void + { + $this->repository->write('docker/base/Dockerfile.twig', '{{ undefined.macro() }}'); + $this->repository->write('docker/base/alpine/conf/Dockerfile.twig', '{{ undefined.macro() }}'); + + self::assertCount(9, $this->generate()); + } + + public function testUnknownMacroIsAnError(): void + { + $this->repository->write('docker/base/alpine/Dockerfile.twig', "{% import 'docker.twig' as docker %}\n{{ docker.missing() }}\n"); + + $this->expectException(RuntimeError::class); + $this->generate(); + } + + public function testUnknownVariableIsAnError(): void + { + $this->repository->write('docker/base/alpine/Dockerfile.twig', "{{ Dockerfile.version }}\n"); + + $this->expectException(RuntimeError::class); + $this->expectExceptionMessage('Key "version" for sequence/mapping with keys "image, tag" does not exist'); + $this->generate(); + } + + public function testDirectoryWithoutTemplateIsAnError(): void + { + // e.g. the template was removed to stop building the tag, but not the generated files + $this->repository->write('docker/base/alpine/Dockerfile', ''); + $this->repository->write('docker/base/alpine-edge/Dockerfile', "# Dockerfile for webdevops/base:alpine-edge\n"); + $this->repository->write('docker/php-official/8.3/conf/etc/php.conf', "php\n"); + + try { + $this->generate(); + self::fail('the directories without template were accepted'); + } catch (RuntimeException $runtimeException) { + self::assertSame( + 'No Dockerfile.twig in docker/base/alpine-edge, docker/php-official/8.3. ' + . 'Add the template, or remove the directory to stop building the tag.', + $runtimeException->getMessage(), + ); + self::assertSame('', $this->repository->read('docker/base/alpine/Dockerfile'), 'checked before any Dockerfile is written'); + } + } + + public function testRemovesTheFilesOfTheFinder(): void + { + // a directory of a removed tag that only the Finder's file keeps + $this->repository->write('docker/base/alpine-old/.DS_Store', ''); + $this->repository->write('docker/base/alpine/.DS_Store', ''); + $this->repository->write('docker/.DS_Store', ''); + $this->repository->write('docker/toolbox/latest/conf/.DS_Store', ''); + + $this->generate(); + + self::assertFileDoesNotExist($this->repository->path . '/docker/base/alpine-old/.DS_Store'); + self::assertFileDoesNotExist($this->repository->path . '/docker/base/alpine/.DS_Store'); + self::assertFileDoesNotExist($this->repository->path . '/docker/.DS_Store'); + self::assertFileDoesNotExist($this->repository->path . '/docker/toolbox/latest/conf/.DS_Store', 'it would get into a local build'); + } + + public function testUpdatesTheLinksOfPhp(): void + { + unlink($this->repository->path . '/docker/php/8.5'); + + $this->generate(); + + self::assertSame('../php-official/8.5', readlink($this->repository->path . '/docker/php/8.5')); + } + + public function testNewTagIsRendered(): void + { + $this->repository->write('docker/base/alpine-edge/Dockerfile.twig', $this->repository->read('docker/base/alpine/Dockerfile.twig')); + + self::assertContains('webdevops/base:alpine-edge', $this->generate()); + self::assertStringStartsWith( + "#+++++++++++++++++++++++++++++++++++++++\n# Dockerfile for webdevops/base:alpine-edge\n", + $this->repository->read('docker/base/alpine-edge/Dockerfile'), + ); + } + + /** + * @return list the log lines + */ + private function generate(): array + { + $lines = []; + new DockerfileGenerator($this->repository->path)->generate(function (string $line) use (&$lines): void { + $lines[] = $line; + }); + return $lines; + } +} diff --git a/ci/tests/Dockerfile/ImageLinksTest.php b/ci/tests/Dockerfile/ImageLinksTest.php new file mode 100644 index 000000000..65ccf5956 --- /dev/null +++ b/ci/tests/Dockerfile/ImageLinksTest.php @@ -0,0 +1,89 @@ +repository = new TemporaryRepository(); + } + + protected function tearDown(): void + { + $this->repository->remove(); + } + + public function testLinksEveryTagOfPhpOfficial(): void + { + unlink($this->repository->path . '/docker/php/8.5'); + + $this->update(); + + self::assertSame(['.', '..', '8.4', '8.5', '8.5-alpine'], scandir($this->repository->path . '/docker/php')); + self::assertSame('../php-official/8.5', readlink($this->repository->path . '/docker/php/8.5')); + } + + public function testRemovesLinksOfRemovedTags(): void + { + // e.g. php-official 8.3 was removed, its link not + symlink('../php-official/8.3', $this->repository->path . '/docker/php/8.3'); + + $this->update(); + + self::assertFalse(is_link($this->repository->path . '/docker/php/8.3')); + } + + public function testCorrectsALinkWithAnotherTarget(): void + { + unlink($this->repository->path . '/docker/php/8.5'); + symlink('../php-official/8.4', $this->repository->path . '/docker/php/8.5'); + + $this->update(); + + self::assertSame('../php-official/8.5', readlink($this->repository->path . '/docker/php/8.5')); + } + + public function testKeepsALinkThatGitCheckedOutAsAFile(): void + { + unlink($this->repository->path . '/docker/php/8.5'); + $this->repository->write('docker/php/8.5', '../php-official/8.5'); + + $this->update(); + + self::assertSame('../php-official/8.5', $this->repository->read('docker/php/8.5')); + } + + public function testADirectoryInThePlaceOfALinkIsAnError(): void + { + unlink($this->repository->path . '/docker/php/8.5'); + $this->repository->write('docker/php/8.5/Dockerfile', ''); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('docker/php/8.5 has to be a symbolic link to ../php-official/8.5, but is a file or a directory.'); + $this->update(); + } + + private function update(): void + { + new ImageLinks($this->repository->path)->update(new TagDirectories($this->repository->path)); + } +} diff --git a/ci/tests/FileReaderTest.php b/ci/tests/FileReaderTest.php index a46ee0af3..13d9fdd95 100644 --- a/ci/tests/FileReaderTest.php +++ b/ci/tests/FileReaderTest.php @@ -33,6 +33,24 @@ public function testCollectsTheGeneratedDockerfilesSorted(): void ], new FileReader(self::ROOT)->collectDockerfiles()); } + public function testCollectsOnlyTheDockerfilesOfTags(): void + { + $repository = new TemporaryRepository(); + // a tag whose Dockerfile was not generated yet, a Dockerfile in conf/ and one without a template + $repository->write('docker/base/alpine-edge/Dockerfile.twig', ''); + $repository->write('docker/base/alpine/conf/Dockerfile', ''); + $repository->write('docker/toolbox/edge/Dockerfile', ''); + + try { + self::assertSame( + new FileReader(self::ROOT)->collectDockerfiles(), + new FileReader($repository->path)->collectDockerfiles(), + ); + } finally { + $repository->remove(); + } + } + public function testImageWithAnOutsideBaseImageIsBuiltAfterTheToolbox(): void { $node = $this->info('base/ubuntu-22.04'); diff --git a/ci/tests/FinderFileTest.php b/ci/tests/FinderFileTest.php new file mode 100644 index 000000000..5e26621c9 --- /dev/null +++ b/ci/tests/FinderFileTest.php @@ -0,0 +1,58 @@ +repository = new TemporaryRepository(); + } + + protected function tearDown(): void + { + $this->repository->remove(); + } + + public function testRemovesEveryFinderFileUnderTheDirectory(): void + { + $this->repository->write('docker/.DS_Store', ''); + $this->repository->write('docker/base/alpine/.DS_Store', ''); + $this->repository->write('docker/toolbox/latest/conf/etc/.DS_Store', ''); + + FinderFile::removeUnder($this->repository->path . '/docker'); + + self::assertFileDoesNotExist($this->repository->path . '/docker/.DS_Store'); + self::assertFileDoesNotExist($this->repository->path . '/docker/base/alpine/.DS_Store'); + self::assertFileDoesNotExist($this->repository->path . '/docker/toolbox/latest/conf/etc/.DS_Store', 'a hand-made conf/ is copied into the image'); + self::assertSame("not provisioned\n", $this->repository->read('docker/toolbox/latest/conf/hand-made')); + } + + public function testDoesNotFollowSymbolicLinks(): void + { + $this->repository->write('provisioning/base/general/.DS_Store', ''); + symlink('../provisioning', $this->repository->path . '/docker/provisioning'); + + FinderFile::removeUnder($this->repository->path . '/docker'); + + self::assertFileExists($this->repository->path . '/provisioning/base/general/.DS_Store'); + } + + public function testMissingDirectoryIsNoError(): void + { + FinderFile::removeUnder($this->repository->path . '/missing'); + + self::assertDirectoryDoesNotExist($this->repository->path . '/missing'); + } +} diff --git a/ci/tests/Fixtures/repository/conf/provision.yml b/ci/tests/Fixtures/repository/conf/provision.yml new file mode 100644 index 000000000..0e43d6808 --- /dev/null +++ b/ci/tests/Fixtures/repository/conf/provision.yml @@ -0,0 +1,13 @@ +--- +# Test copy of conf/provision.yml + +provision: + base: + configuration: + base/general: '*' + base/alpine: alpine + php-official: + configuration: + base/general: '*' + php/general: '*' + php/official-alpine: '*-alpine' diff --git a/ci/tests/Fixtures/repository/docker/base-app/ubuntu-22.04/Dockerfile.twig b/ci/tests/Fixtures/repository/docker/base-app/ubuntu-22.04/Dockerfile.twig new file mode 100644 index 000000000..ca87a1b75 --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/base-app/ubuntu-22.04/Dockerfile.twig @@ -0,0 +1,2 @@ +{% import 'docker.twig' as docker %} +{{ docker.from("base", "latest") }} diff --git a/ci/tests/Fixtures/repository/docker/base/alpine/Dockerfile b/ci/tests/Fixtures/repository/docker/base/alpine/Dockerfile index 3f341516e..37f82806c 100644 --- a/ci/tests/Fixtures/repository/docker/base/alpine/Dockerfile +++ b/ci/tests/Fixtures/repository/docker/base/alpine/Dockerfile @@ -6,3 +6,17 @@ FROM alpine:3.22 COPY --from=webdevops/toolbox:latest /baselayout/sbin/* /sbin/ + +# Installs a package by version + +RUN set -x \ + # Install the packages + && apk-install \ + two \ + && docker-image-cleanup + +RUN set -x \ + # Install the packages + && apk-install \ + other \ + && docker-image-cleanup diff --git a/ci/tests/Fixtures/repository/docker/base/alpine/Dockerfile.twig b/ci/tests/Fixtures/repository/docker/base/alpine/Dockerfile.twig new file mode 100644 index 000000000..7971ed810 --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/base/alpine/Dockerfile.twig @@ -0,0 +1,12 @@ +{% import 'docker.twig' as docker %} +{% import 'images/base.twig' as base %} +{# Alpine 3.22 #} +{{ docker.fromOfficial("alpine", "3.22") }} + +{{ docker.copyFromToolbox("latest") }} +{{ base.note("Installs a package by version") }} +RUN set -x \ + {{ base.install(version="2") }} + +RUN set -x \ + {{ base.install() }} diff --git a/ci/tests/Fixtures/repository/docker/base/ubuntu-22.04/Dockerfile.jinja2 b/ci/tests/Fixtures/repository/docker/base/ubuntu-22.04/Dockerfile.jinja2 deleted file mode 100644 index bd365936c..000000000 --- a/ci/tests/Fixtures/repository/docker/base/ubuntu-22.04/Dockerfile.jinja2 +++ /dev/null @@ -1 +0,0 @@ -{% extends "base.jinja2" %} diff --git a/ci/tests/Fixtures/repository/docker/base/ubuntu-22.04/Dockerfile.twig b/ci/tests/Fixtures/repository/docker/base/ubuntu-22.04/Dockerfile.twig new file mode 100644 index 000000000..2c1e0241c --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/base/ubuntu-22.04/Dockerfile.twig @@ -0,0 +1,4 @@ +{% import 'docker.twig' as docker %} +{{ docker.fromOfficial("ubuntu", "22.04") }} + +{{ docker.copyFromToolbox() }} diff --git a/ci/tests/Fixtures/repository/docker/php-nginx-dev/8.5-alpine/Dockerfile.twig b/ci/tests/Fixtures/repository/docker/php-nginx-dev/8.5-alpine/Dockerfile.twig new file mode 100644 index 000000000..ecfe2f5cc --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/php-nginx-dev/8.5-alpine/Dockerfile.twig @@ -0,0 +1,2 @@ +{% import 'docker.twig' as docker %} +{{ docker.from("php-nginx", "8.5-alpine") }} diff --git a/ci/tests/Fixtures/repository/docker/php-nginx/8.5-alpine/Dockerfile.twig b/ci/tests/Fixtures/repository/docker/php-nginx/8.5-alpine/Dockerfile.twig new file mode 100644 index 000000000..4c7a5bf08 --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/php-nginx/8.5-alpine/Dockerfile.twig @@ -0,0 +1,2 @@ +{% import 'docker.twig' as docker %} +{{ docker.from("php", "8.5-alpine") }} diff --git a/ci/tests/Fixtures/repository/docker/php-official/8.4/Dockerfile.twig b/ci/tests/Fixtures/repository/docker/php-official/8.4/Dockerfile.twig new file mode 100644 index 000000000..acc934b41 --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/php-official/8.4/Dockerfile.twig @@ -0,0 +1,4 @@ +{% import 'docker.twig' as docker %} +{{ docker.fromOfficial("php", "8.4-fpm-bookworm") }} + +{{ docker.copyFromToolbox() }} diff --git a/ci/tests/Fixtures/repository/docker/php-official/8.5-alpine/Dockerfile.twig b/ci/tests/Fixtures/repository/docker/php-official/8.5-alpine/Dockerfile.twig new file mode 100644 index 000000000..63b63fc48 --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/php-official/8.5-alpine/Dockerfile.twig @@ -0,0 +1,4 @@ +{% import 'docker.twig' as docker %} +{{ docker.fromOfficial("php", "8.5-fpm-alpine") }} + +{{ docker.copyFromToolbox() }} diff --git a/ci/tests/Fixtures/repository/docker/php-official/8.5/Dockerfile.twig b/ci/tests/Fixtures/repository/docker/php-official/8.5/Dockerfile.twig new file mode 100644 index 000000000..759cc66a4 --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/php-official/8.5/Dockerfile.twig @@ -0,0 +1,4 @@ +{% import 'docker.twig' as docker %} +{{ docker.fromOfficial("php", "8.5-fpm-trixie") }} + +{{ docker.copyFromToolbox() }} diff --git a/ci/tests/Fixtures/repository/docker/php/8.5 b/ci/tests/Fixtures/repository/docker/php/8.5 new file mode 120000 index 000000000..a0754b593 --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/php/8.5 @@ -0,0 +1 @@ +../php-official/8.5 \ No newline at end of file diff --git a/ci/tests/Fixtures/repository/docker/php/8.5-alpine b/ci/tests/Fixtures/repository/docker/php/8.5-alpine new file mode 120000 index 000000000..d3fc8eeba --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/php/8.5-alpine @@ -0,0 +1 @@ +../php-official/8.5-alpine \ No newline at end of file diff --git a/ci/tests/Fixtures/repository/docker/toolbox/latest/Dockerfile.twig b/ci/tests/Fixtures/repository/docker/toolbox/latest/Dockerfile.twig new file mode 100644 index 000000000..1fcf64b10 --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/toolbox/latest/Dockerfile.twig @@ -0,0 +1,4 @@ +{% import 'docker.twig' as docker %} +{{ docker.fromOfficial("alpine", "3.22") }} + +RUN mkdir /baselayout diff --git a/ci/tests/Fixtures/repository/docker/toolbox/latest/conf/hand-made b/ci/tests/Fixtures/repository/docker/toolbox/latest/conf/hand-made new file mode 100644 index 000000000..f8468cb38 --- /dev/null +++ b/ci/tests/Fixtures/repository/docker/toolbox/latest/conf/hand-made @@ -0,0 +1 @@ +not provisioned diff --git a/ci/tests/Fixtures/repository/provisioning/base/alpine/etc/os b/ci/tests/Fixtures/repository/provisioning/base/alpine/etc/os new file mode 100644 index 000000000..e0b8ab210 --- /dev/null +++ b/ci/tests/Fixtures/repository/provisioning/base/alpine/etc/os @@ -0,0 +1 @@ +alpine diff --git a/ci/tests/Fixtures/repository/provisioning/base/general/bin/service b/ci/tests/Fixtures/repository/provisioning/base/general/bin/service new file mode 100755 index 000000000..e81a82cdb --- /dev/null +++ b/ci/tests/Fixtures/repository/provisioning/base/general/bin/service @@ -0,0 +1,2 @@ +#!/bin/sh +echo service diff --git a/ci/tests/Fixtures/repository/provisioning/base/general/etc/os b/ci/tests/Fixtures/repository/provisioning/base/general/etc/os new file mode 100644 index 000000000..8265cfab4 --- /dev/null +++ b/ci/tests/Fixtures/repository/provisioning/base/general/etc/os @@ -0,0 +1 @@ +general diff --git a/ci/tests/Fixtures/repository/provisioning/php/general/etc/php.conf b/ci/tests/Fixtures/repository/provisioning/php/general/etc/php.conf new file mode 100644 index 000000000..2c0ea7b5a --- /dev/null +++ b/ci/tests/Fixtures/repository/provisioning/php/general/etc/php.conf @@ -0,0 +1 @@ +php diff --git a/ci/tests/Fixtures/repository/provisioning/php/official-alpine/etc/php.conf b/ci/tests/Fixtures/repository/provisioning/php/official-alpine/etc/php.conf new file mode 100644 index 000000000..fa89d3e0d --- /dev/null +++ b/ci/tests/Fixtures/repository/provisioning/php/official-alpine/etc/php.conf @@ -0,0 +1 @@ +php alpine diff --git a/ci/tests/Fixtures/repository/template/Dockerfile/docker.twig b/ci/tests/Fixtures/repository/template/Dockerfile/docker.twig new file mode 100644 index 000000000..d61796390 --- /dev/null +++ b/ci/tests/Fixtures/repository/template/Dockerfile/docker.twig @@ -0,0 +1,17 @@ +{# Test copy of template/Dockerfile/docker.twig with a few of its macros #} + +{% macro from(image, tag) -%} +FROM webdevops/{{ image }}:{{ tag }} +{%- endmacro %} + +{% macro fromOfficial(image, tag) -%} +FROM {{ image }}:{{ tag }} +{%- endmacro %} + +{% macro copyFromToolbox(tag='') -%} +COPY --from=webdevops/toolbox{% if tag %}:{{ tag }}{% endif %} /baselayout/sbin/* /sbin/ +{%- endmacro %} + +{% macro cleanup() -%} + && docker-image-cleanup +{%- endmacro %} diff --git a/ci/tests/Fixtures/repository/template/Dockerfile/images/base.twig b/ci/tests/Fixtures/repository/template/Dockerfile/images/base.twig new file mode 100644 index 000000000..70450821a --- /dev/null +++ b/ci/tests/Fixtures/repository/template/Dockerfile/images/base.twig @@ -0,0 +1,19 @@ +{# A macro sees only the imports in its own body #} +{% macro install(version='') -%} +{% import 'docker.twig' as docker -%} + # Install the packages + && apk-install \ +{% if version == '1' %} + one \ +{% elseif version == '2' %} + two \ +{% else %} + other \ +{% endif %} + {{ docker.cleanup() }} +{%- endmacro %} + +{% macro note(text) %} + +# {{ text }} +{% endmacro %} diff --git a/ci/tests/Fixtures/repository/template/Dockerfile/layout.twig b/ci/tests/Fixtures/repository/template/Dockerfile/layout.twig new file mode 100644 index 000000000..1b1d76a6b --- /dev/null +++ b/ci/tests/Fixtures/repository/template/Dockerfile/layout.twig @@ -0,0 +1,6 @@ +#+++++++++++++++++++++++++++++++++++++++ +# Dockerfile for webdevops/{{ Dockerfile.image }}:{{ Dockerfile.tag }} +# -- automatically generated -- +#+++++++++++++++++++++++++++++++++++++++ + +{{ include(template) -}} diff --git a/ci/tests/GithubJobBuilderTest.php b/ci/tests/GithubJobBuilderTest.php index e44b0e0d4..eef5fb3e2 100644 --- a/ci/tests/GithubJobBuilderTest.php +++ b/ci/tests/GithubJobBuilderTest.php @@ -227,18 +227,17 @@ public function testPublishPushesEveryTagToBothRegistries(): void public function testValidationJobChecksTheGeneratedFiles(): void { $job = new GithubJobBuilder(self::ROOT)->getValidationConfig(); + $steps = $this->steps(['job' => $job], 'job'); self::assertSame('Validate Automation', $job['name']); - self::assertSame([ - 'actions/checkout@v7', - 'Validate that template/* are used to generate Dockerfiles', - 'Validate .github/workflows/build.yaml is up to date', - 'Validate the README banner is up to date', - ], array_keys($this->steps(['job' => $job], 'job'))); - self::assertStringStartsWith( - 'docker run --rm -v $PWD:/app -w /app/ci webdevops/php:8.4-alpine composer install --no-dev' . "\n", - $this->steps(['job' => $job], 'job')['Validate .github/workflows/build.yaml is up to date']['run'], - 'generating needs no dev tools', + self::assertSame(['actions/checkout@v7', 'Validate that the generated files are up to date'], array_keys($steps)); + self::assertSame( + "make generate\n" + . "git status --porcelain\n" + . "git diff --exit-code --color=always\n" + . 'test -z "$(git status --porcelain)"', + $steps['Validate that the generated files are up to date']['run'], + 'the command of the documentation, and a new file fails the step too', ); } @@ -249,11 +248,11 @@ public function testCodeCheckRunsGrumphp(): void self::assertArrayNotHasKey('needs', $job); self::assertSame(['actions/checkout@v7', 'Run GrumPHP'], array_keys($steps)); - self::assertStringEndsWith("vendor/bin/grumphp run --no-interaction'", $steps['Run GrumPHP']['run']); + self::assertSame('make check', $steps['Run GrumPHP']['run'], 'the command of the documentation'); } /** - * Jobs of the fixture's webdevops/base:alpine, as FileReader and GithubCommand describe it. + * Jobs of the fixture's webdevops/base:alpine, as FileReader and GenerateGithubCiCommand describe it. * * @param array $node replaces values of the node * @return array> diff --git a/ci/tests/Provision/ProvisionerTest.php b/ci/tests/Provision/ProvisionerTest.php new file mode 100644 index 000000000..f1bfc838a --- /dev/null +++ b/ci/tests/Provision/ProvisionerTest.php @@ -0,0 +1,216 @@ +repository = new TemporaryRepository(); + } + + protected function tearDown(): void + { + $this->repository->remove(); + } + + public function testCopiesTheSourcesIntoTheConfDirectoryOfTheirTags(): void + { + $this->provision(); + + self::assertSame("general\n", $this->repository->read('docker/base/ubuntu-22.04/conf/etc/os')); + self::assertSame("php\n", $this->repository->read('docker/php-official/8.4/conf/etc/php.conf')); + self::assertSame("php\n", $this->repository->read('docker/php-official/8.5/conf/etc/php.conf'), '8.5 does not match *-alpine'); + self::assertSame("#!/bin/sh\necho service\n", $this->repository->read('docker/php-official/8.5-alpine/conf/bin/service')); + } + + public function testLaterSourcesReplaceFilesOfEarlierOnes(): void + { + $this->provision(); + + self::assertSame("alpine\n", $this->repository->read('docker/base/alpine/conf/etc/os')); + self::assertSame("php alpine\n", $this->repository->read('docker/php-official/8.5-alpine/conf/etc/php.conf')); + } + + public function testKeepsThePermissionsOfTheFiles(): void + { + $this->provision(); + + self::assertSame(0755, fileperms($this->repository->path . '/docker/base/alpine/conf/bin/service') & 0777); + self::assertSame(0644, fileperms($this->repository->path . '/docker/base/alpine/conf/etc/os') & 0777); + } + + public function testLeavesOutTheFilesOfTheFinder(): void + { + $this->repository->write('provisioning/base/general/etc/.DS_Store', ''); + + $this->provision(); + + self::assertFileDoesNotExist($this->repository->path . '/docker/base/alpine/conf/etc/.DS_Store'); + self::assertSame("alpine\n", $this->repository->read('docker/base/alpine/conf/etc/os')); + } + + public function testEmptiesTheConfDirectoriesFirst(): void + { + $this->repository->write('docker/base/alpine/conf/etc/removed-from-provisioning', 'old'); + $this->repository->write('docker/php-official/8.4/conf/etc/os', 'old'); + + $this->provision(); + + self::assertFileDoesNotExist($this->repository->path . '/docker/base/alpine/conf/etc/removed-from-provisioning'); + self::assertSame("general\n", $this->repository->read('docker/php-official/8.4/conf/etc/os')); + } + + public function testLeavesImagesAloneThatConfProvisionYmlDoesNotName(): void + { + $this->repository->write('docker/base-app/ubuntu-22.04/conf/etc/own', 'own'); + + $this->provision(); + + self::assertSame("not provisioned\n", $this->repository->read('docker/toolbox/latest/conf/hand-made')); + self::assertSame('own', $this->repository->read('docker/base-app/ubuntu-22.04/conf/etc/own')); + } + + public function testTagListAndPatterns(): void + { + $this->configure("provision:\n base:\n configuration:\n base/alpine: ['ALPINE', 'ubuntu-*']\n"); + + $this->provision(); + + self::assertSame("alpine\n", $this->repository->read('docker/base/alpine/conf/etc/os'), 'tags match regardless of case'); + self::assertSame("alpine\n", $this->repository->read('docker/base/ubuntu-22.04/conf/etc/os')); + } + + public function testTagMatchesTheWholeDirectoryName(): void + { + $this->repository->write('docker/base/alpine-edge/Dockerfile.twig', ''); + + $this->provision(); + + self::assertSame("general\n", $this->repository->read('docker/base/alpine-edge/conf/etc/os'), 'alpine does not match alpine-edge'); + } + + public function testNewTagIsProvisionedBeforeItsDockerfileIsGenerated(): void + { + $this->repository->write('docker/base/alpine-edge/Dockerfile.twig', ''); + + $this->provision(); + + self::assertSame("general\n", $this->repository->read('docker/base/alpine-edge/conf/etc/os')); + } + + public function testDirectoryWithoutTemplateIsNoTag(): void + { + // generate:dockerfile reports such a directory + $this->repository->write('docker/base/alpine-edge/conf/etc/os', 'old'); + + $this->provision(); + + self::assertSame('old', $this->repository->read('docker/base/alpine-edge/conf/etc/os')); + } + + public function testChecksTheWholeConfigurationBeforeChangingFiles(): void + { + $this->repository->write('docker/base/alpine/conf/etc/os', 'old'); + $this->configure("provision:\n base:\n configuration:\n base/general: '*'\n php-official:\n configuration:\n php/general: '8.6'\n"); + + try { + $this->provision(); + self::fail('8.6 is no tag of php-official'); + } catch (RuntimeException $runtimeException) { + self::assertSame('conf/provision.yml copies provisioning/php/general into webdevops/php-official:8.6, but webdevops/php-official has no such tag under docker/.', $runtimeException->getMessage()); + self::assertSame('old', $this->repository->read('docker/base/alpine/conf/etc/os')); + } + } + + public function testLogsEachSource(): void + { + self::assertSame([ + 'base/general => webdevops/base:*', + 'base/alpine => webdevops/base:alpine', + 'base/general => webdevops/php-official:*', + 'php/general => webdevops/php-official:*', + 'php/official-alpine => webdevops/php-official:*-alpine', + ], $this->provision()); + } + + /** + * @return iterable + */ + public static function invalidConfigurations(): iterable + { + yield 'missing source' => [ + "provision:\n base:\n configuration:\n base/missing: '*'\n", + 'conf/provision.yml copies provisioning/base/missing into webdevops/base, but it does not exist.', + ]; + yield 'no provision section' => ["images: []\n", 'conf/provision.yml has no "provision" section.']; + yield 'tag without a directory' => [ + "provision:\n base:\n configuration:\n base/alpine: alpin\n", + 'conf/provision.yml copies provisioning/base/alpine into webdevops/base:alpin, but webdevops/base has no such tag under docker/.', + ]; + yield 'image without tags' => [ + "provision:\n missing-image:\n configuration:\n base/general: '*'\n", + 'conf/provision.yml copies provisioning/base/general into webdevops/missing-image:*, but webdevops/missing-image has no such tag under docker/.', + ]; + yield 'image whose tags are symbolic links' => [ + // docker/php/8.4 links to docker/php-official/8.4 + "provision:\n php:\n configuration:\n php/general: '*'\n", + 'conf/provision.yml copies provisioning/php/general into webdevops/php:*, but webdevops/php has no such tag under docker/.', + ]; + yield 'empty tag list' => [ + "provision:\n base:\n configuration:\n base/general: []\n", + 'conf/provision.yml has no tag for provisioning/base/general in webdevops/base.', + ]; + $settings = 'conf/provision.yml: webdevops/base needs "configuration", which maps directories of provisioning/ to tags, and no other setting.'; + yield 'misspelt configuration' => ["provision:\n base:\n configuraton:\n base/general: '*'\n", $settings]; + yield 'another setting' => ["provision:\n base:\n configuration:\n base/general: '*'\n tags: '*'\n", $settings]; + yield 'configuration that is no map' => ["provision:\n base:\n configuration: base/general\n", $settings]; + yield 'no settings' => ["provision:\n base: ~\n", $settings]; + yield 'settings that are no map' => ["provision:\n base: base/general\n", $settings]; + yield 'tag that is no text' => [ + "provision:\n base:\n configuration:\n base/general: 8\n", + 'conf/provision.yml has an invalid tag for provisioning/base/general in webdevops/base.', + ]; + } + + #[DataProvider('invalidConfigurations')] + public function testInvalidConfigurationIsAnError(string $configuration, string $message): void + { + $this->configure($configuration); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage($message); + $this->provision(); + } + + private function configure(string $configuration): void + { + $this->repository->write('conf/provision.yml', $configuration); + } + + /** + * @return list the log lines + */ + private function provision(): array + { + $lines = []; + new Provisioner($this->repository->path)->provision(function (string $line) use (&$lines): void { + $lines[] = $line; + }); + return $lines; + } +} diff --git a/ci/tests/TagDirectoriesTest.php b/ci/tests/TagDirectoriesTest.php new file mode 100644 index 000000000..a19695a66 --- /dev/null +++ b/ci/tests/TagDirectoriesTest.php @@ -0,0 +1,72 @@ +repository = new TemporaryRepository(); + } + + protected function tearDown(): void + { + $this->repository->remove(); + } + + public function testTagsAreTheDirectoriesWithATemplateSortedLikeTheirDockerfiles(): void + { + // docker/php/8.4 links to docker/php-official/8.4 and is left out + self::assertSame([ + 'docker/base-app/ubuntu-22.04', + 'docker/base/alpine', + 'docker/base/ubuntu-22.04', + 'docker/php-nginx-dev/8.5-alpine', + 'docker/php-nginx/8.5-alpine', + 'docker/php-official/8.4', + 'docker/php-official/8.5-alpine', + 'docker/php-official/8.5', + 'docker/toolbox/latest', + ], $this->tagDirectories()->tags()); + } + + public function testTagsOfOneImage(): void + { + self::assertSame(['docker/base/alpine', 'docker/base/ubuntu-22.04'], $this->tagDirectories()->tags('base')); + self::assertSame([], $this->tagDirectories()->tags('php'), 'every tag of php is a symbolic link'); + self::assertSame([], $this->tagDirectories()->tags('missing')); + } + + public function testDirectoriesWithoutTemplateAreNoTags(): void + { + $this->repository->write('docker/base/alpine-edge/Dockerfile', ''); + $this->repository->write('docker/base/alpine-old/conf/etc/os', ''); + $this->repository->write('docker/base/Dockerfile.twig', ''); + $this->repository->write('docker/base/alpine/conf/sub/Dockerfile.twig', ''); + mkdir($this->repository->path . '/docker/base/empty'); + + self::assertSame(['docker/base/alpine', 'docker/base/ubuntu-22.04'], $this->tagDirectories()->tags('base'), 'a template in docker/base/ is no tag, one in alpine/conf/sub is too deep'); + self::assertSame(['docker/base/alpine-edge', 'docker/base/alpine-old'], $this->tagDirectories()->withoutTemplate(), 'an empty directory is left alone'); + } + + public function testNoDirectoryWithoutTemplateInTheFixture(): void + { + self::assertSame([], $this->tagDirectories()->withoutTemplate()); + } + + private function tagDirectories(): TagDirectories + { + return new TagDirectories($this->repository->path); + } +} diff --git a/ci/tests/TemporaryRepository.php b/ci/tests/TemporaryRepository.php index c80f3b9f6..af520b8e4 100644 --- a/ci/tests/TemporaryRepository.php +++ b/ci/tests/TemporaryRepository.php @@ -10,10 +10,12 @@ use SplFileInfo; use function bin2hex; +use function chmod; use function copy; use function dirname; use function file_get_contents; use function file_put_contents; +use function fileperms; use function is_dir; use function mkdir; use function random_bytes; @@ -57,6 +59,7 @@ public function __construct() mkdir($target); } else { copy($file->getPathname(), $target); + chmod($target, fileperms($file->getPathname()) & 0777); } } diff --git a/conf/console.yml b/conf/console.yml index 8456af4c7..a649dd17a 100644 --- a/conf/console.yml +++ b/conf/console.yml @@ -1,25 +1,12 @@ --- - -dockerPath: 'docker' -templatePath: 'template' -provisionPath: 'provisioning' -baselayoutPath: 'baselayout' -testinfraPath: 'tests/testinfra' -serverspecPath: 'tests/serverspec' -testDockerfilePath: 'tests/dockerfile' -blacklistFile: 'BLACKLIST' +# Settings of ci/console generate:github-ci and generate:readme-banner docker: - imagePrefix: 'webdevops' + # This tag is also published as latest autoLatestTag: 'ubuntu-22.04' - pathRegex: '/(?P[^/]+)/(?P[^/]+)/Dockerfile$' - autoPull: True - autoPullBlacklist: '^webdevops/.*' +# Settings of the serverspec tests, per image tag dockerTest: - serverspec: - specPath: 'spec/docker/%s_spec.rb' - configuration: imageConfigurationRegex: ':(?Palpine|debian|ubuntu|centos)-?(?P[^-]+)?' default: diff --git a/conf/readme-banner.yml b/conf/readme-banner.yml index 51eac4912..96039134e 100644 --- a/conf/readme-banner.yml +++ b/conf/readme-banner.yml @@ -1,7 +1,7 @@ --- # Texts of the banner at the top of README.md. # The PHP versions, web servers, base OS releases and platforms come from the -# images that are built. Run `ci/console readme-banner:generate` after a change. +# images that are built. Run `make generate` after a change. files: light: 'docs/assets/readme-banner-light.svg' diff --git a/docker/apache/alpine/Dockerfile.jinja2 b/docker/apache/alpine/Dockerfile.twig similarity index 58% rename from docker/apache/alpine/Dockerfile.jinja2 rename to docker/apache/alpine/Dockerfile.twig index 51384b363..cf7edd34d 100644 --- a/docker/apache/alpine/Dockerfile.jinja2 +++ b/docker/apache/alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/apache.twig' as apache %} {{ docker.from("base", "alpine") }} {{ environment.web() }} diff --git a/docker/base-app/alpine/Dockerfile.jinja2 b/docker/base-app/alpine/Dockerfile.twig similarity index 55% rename from docker/base-app/alpine/Dockerfile.jinja2 rename to docker/base-app/alpine/Dockerfile.twig index e0bac3426..2df78890b 100644 --- a/docker/base-app/alpine/Dockerfile.jinja2 +++ b/docker/base-app/alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/baseapp.twig' as baseapp %} {{ docker.from("base", "alpine") }} {{ environment.baseApp() }} diff --git a/docker/base-app/ubuntu-22.04/Dockerfile.jinja2 b/docker/base-app/ubuntu-22.04/Dockerfile.twig similarity index 56% rename from docker/base-app/ubuntu-22.04/Dockerfile.jinja2 rename to docker/base-app/ubuntu-22.04/Dockerfile.twig index 6d988225e..43c67709a 100644 --- a/docker/base-app/ubuntu-22.04/Dockerfile.jinja2 +++ b/docker/base-app/ubuntu-22.04/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/baseapp.twig' as baseapp %} {{ docker.from("base", "ubuntu-22.04") }} {{ environment.baseApp() }} diff --git a/docker/base/alpine/Dockerfile.jinja2 b/docker/base/alpine/Dockerfile.twig similarity index 63% rename from docker/base/alpine/Dockerfile.jinja2 rename to docker/base/alpine/Dockerfile.twig index 84af4cc8d..db629ab1e 100644 --- a/docker/base/alpine/Dockerfile.jinja2 +++ b/docker/base/alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/base.twig' as base %} {{ docker.from("bootstrap", "alpine") }} {{ environment.base() }} diff --git a/docker/base/ubuntu-22.04/Dockerfile.jinja2 b/docker/base/ubuntu-22.04/Dockerfile.twig similarity index 63% rename from docker/base/ubuntu-22.04/Dockerfile.jinja2 rename to docker/base/ubuntu-22.04/Dockerfile.twig index 29241a475..9bc947d18 100644 --- a/docker/base/ubuntu-22.04/Dockerfile.jinja2 +++ b/docker/base/ubuntu-22.04/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/base.twig' as base %} {{ docker.from("bootstrap", "ubuntu-22.04") }} {{ environment.base() }} diff --git a/docker/bootstrap/alpine/Dockerfile.jinja2 b/docker/bootstrap/alpine/Dockerfile.twig similarity index 51% rename from docker/bootstrap/alpine/Dockerfile.jinja2 rename to docker/bootstrap/alpine/Dockerfile.twig index d0279bf3b..cc3a4c55e 100644 --- a/docker/bootstrap/alpine/Dockerfile.jinja2 +++ b/docker/bootstrap/alpine/Dockerfile.twig @@ -1,3 +1,8 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} {{ docker.fromOfficial("alpine", "3.11") }} diff --git a/docker/bootstrap/ubuntu-22.04/Dockerfile.jinja2 b/docker/bootstrap/ubuntu-22.04/Dockerfile.twig similarity index 51% rename from docker/bootstrap/ubuntu-22.04/Dockerfile.jinja2 rename to docker/bootstrap/ubuntu-22.04/Dockerfile.twig index c9f9ae216..44c80d66f 100644 --- a/docker/bootstrap/ubuntu-22.04/Dockerfile.jinja2 +++ b/docker/bootstrap/ubuntu-22.04/Dockerfile.twig @@ -1,3 +1,8 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} {{ docker.fromOfficial("ubuntu", "22.04") }} diff --git a/docker/dockerfile-build-env/latest/Dockerfile b/docker/dockerfile-build-env/latest/Dockerfile index e1c2b3f26..602983f92 100644 --- a/docker/dockerfile-build-env/latest/Dockerfile +++ b/docker/dockerfile-build-env/latest/Dockerfile @@ -13,15 +13,10 @@ RUN apt-install \ ca-certificates \ software-properties-common \ git \ - python2 \ - python2-dev \ - python-setuptools \ ruby \ ruby-dev \ ruby-bundler \ libyaml-dev \ - && curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \ - && python2 get-pip.py \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ && add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ && apt-install docker-ce \ @@ -29,16 +24,7 @@ RUN apt-install \ && curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-$TARGETARCH \ && chmod +x container-structure-test-linux-$TARGETARCH \ && mv container-structure-test-linux-$TARGETARCH /usr/local/bin/container-structure-test \ - && pip install --upgrade pip \ - && hash -r pip \ - && pip install --upgrade setuptools \ - && pip install Cython==0.29.21 wheel \ - && git clone --depth 1 https://github.com/webdevops/Dockerfile.git /tmp/Dockerfile \ && gem install bundler -v 2.1.4 --no-document \ - && cd /tmp/Dockerfile/tests/serverspec \ - && bundle _2.1.4_ update --bundler \ - && cd /tmp/Dockerfile \ - && make setup \ && docker-image-cleanup WORKDIR /app diff --git a/docker/dockerfile-build-env/latest/Dockerfile.jinja2 b/docker/dockerfile-build-env/latest/Dockerfile.twig similarity index 64% rename from docker/dockerfile-build-env/latest/Dockerfile.jinja2 rename to docker/dockerfile-build-env/latest/Dockerfile.twig index a16d2e34a..1ce4cbd9b 100644 --- a/docker/dockerfile-build-env/latest/Dockerfile.jinja2 +++ b/docker/dockerfile-build-env/latest/Dockerfile.twig @@ -1,3 +1,4 @@ +{% import 'docker.twig' as docker %} {{ docker.from("base-app","ubuntu-22.04") }} ARG TARGETARCH @@ -8,15 +9,10 @@ RUN apt-install \ ca-certificates \ software-properties-common \ git \ - python2 \ - python2-dev \ - python-setuptools \ ruby \ ruby-dev \ ruby-bundler \ libyaml-dev \ - && curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \ - && python2 get-pip.py \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ && add-apt-repository "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ && apt-install docker-ce \ @@ -24,16 +20,7 @@ RUN apt-install \ && curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-$TARGETARCH \ && chmod +x container-structure-test-linux-$TARGETARCH \ && mv container-structure-test-linux-$TARGETARCH /usr/local/bin/container-structure-test \ - && pip install --upgrade pip \ - && hash -r pip \ - && pip install --upgrade setuptools \ - && pip install Cython==0.29.21 wheel \ - && git clone --depth 1 https://github.com/webdevops/Dockerfile.git /tmp/Dockerfile \ && gem install bundler -v 2.1.4 --no-document \ - && cd /tmp/Dockerfile/tests/serverspec \ - && bundle _2.1.4_ update --bundler \ - && cd /tmp/Dockerfile \ - && make setup \ {{ docker.cleanup() }} WORKDIR /app diff --git a/docker/nginx/alpine/Dockerfile.jinja2 b/docker/nginx/alpine/Dockerfile.twig similarity index 61% rename from docker/nginx/alpine/Dockerfile.jinja2 rename to docker/nginx/alpine/Dockerfile.twig index c7033f9b9..4d83f199b 100644 --- a/docker/nginx/alpine/Dockerfile.jinja2 +++ b/docker/nginx/alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/nginx.twig' as nginx %} {{ docker.from("base", "alpine") }} {{ environment.web() }} diff --git a/docker/php-apache-dev/8.2-alpine/Dockerfile.jinja2 b/docker/php-apache-dev/8.2-alpine/Dockerfile.twig similarity index 69% rename from docker/php-apache-dev/8.2-alpine/Dockerfile.jinja2 rename to docker/php-apache-dev/8.2-alpine/Dockerfile.twig index e8f8ecffc..b6eb757fa 100644 --- a/docker/php-apache-dev/8.2-alpine/Dockerfile.jinja2 +++ b/docker/php-apache-dev/8.2-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-apache", "8.2-alpine") }} {{ environment.web() }} diff --git a/docker/php-apache-dev/8.2/Dockerfile.jinja2 b/docker/php-apache-dev/8.2/Dockerfile.twig similarity index 70% rename from docker/php-apache-dev/8.2/Dockerfile.jinja2 rename to docker/php-apache-dev/8.2/Dockerfile.twig index 25591be37..94e219fe7 100644 --- a/docker/php-apache-dev/8.2/Dockerfile.jinja2 +++ b/docker/php-apache-dev/8.2/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-apache", "8.2") }} {{ environment.web() }} diff --git a/docker/php-apache-dev/8.3-alpine/Dockerfile.jinja2 b/docker/php-apache-dev/8.3-alpine/Dockerfile.twig similarity index 69% rename from docker/php-apache-dev/8.3-alpine/Dockerfile.jinja2 rename to docker/php-apache-dev/8.3-alpine/Dockerfile.twig index 58c3075fb..689a6ebb9 100644 --- a/docker/php-apache-dev/8.3-alpine/Dockerfile.jinja2 +++ b/docker/php-apache-dev/8.3-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-apache", "8.3-alpine") }} {{ environment.web() }} diff --git a/docker/php-apache-dev/8.3/Dockerfile.jinja2 b/docker/php-apache-dev/8.3/Dockerfile.twig similarity index 70% rename from docker/php-apache-dev/8.3/Dockerfile.jinja2 rename to docker/php-apache-dev/8.3/Dockerfile.twig index 30fb6e529..a53ee945c 100644 --- a/docker/php-apache-dev/8.3/Dockerfile.jinja2 +++ b/docker/php-apache-dev/8.3/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-apache", "8.3") }} {{ environment.web() }} diff --git a/docker/php-apache-dev/8.4-alpine/Dockerfile.jinja2 b/docker/php-apache-dev/8.4-alpine/Dockerfile.twig similarity index 69% rename from docker/php-apache-dev/8.4-alpine/Dockerfile.jinja2 rename to docker/php-apache-dev/8.4-alpine/Dockerfile.twig index d5cf71c04..e2ce6f8ff 100644 --- a/docker/php-apache-dev/8.4-alpine/Dockerfile.jinja2 +++ b/docker/php-apache-dev/8.4-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-apache", "8.4-alpine") }} {{ environment.web() }} diff --git a/docker/php-apache-dev/8.4/Dockerfile.jinja2 b/docker/php-apache-dev/8.4/Dockerfile.twig similarity index 70% rename from docker/php-apache-dev/8.4/Dockerfile.jinja2 rename to docker/php-apache-dev/8.4/Dockerfile.twig index 128eda8df..ba0e8c0be 100644 --- a/docker/php-apache-dev/8.4/Dockerfile.jinja2 +++ b/docker/php-apache-dev/8.4/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-apache", "8.4") }} {{ environment.web() }} diff --git a/docker/php-apache-dev/8.5-alpine/Dockerfile.jinja2 b/docker/php-apache-dev/8.5-alpine/Dockerfile.twig similarity index 69% rename from docker/php-apache-dev/8.5-alpine/Dockerfile.jinja2 rename to docker/php-apache-dev/8.5-alpine/Dockerfile.twig index 61b37d837..9d6c7c4ff 100644 --- a/docker/php-apache-dev/8.5-alpine/Dockerfile.jinja2 +++ b/docker/php-apache-dev/8.5-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-apache", "8.5-alpine") }} {{ environment.web() }} diff --git a/docker/php-apache-dev/8.5/Dockerfile.jinja2 b/docker/php-apache-dev/8.5/Dockerfile.twig similarity index 70% rename from docker/php-apache-dev/8.5/Dockerfile.jinja2 rename to docker/php-apache-dev/8.5/Dockerfile.twig index 65948c009..bbff0ae04 100644 --- a/docker/php-apache-dev/8.5/Dockerfile.jinja2 +++ b/docker/php-apache-dev/8.5/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-apache", "8.5") }} {{ environment.web() }} diff --git a/docker/php-apache/8.2-alpine/Dockerfile.jinja2 b/docker/php-apache/8.2-alpine/Dockerfile.twig similarity index 58% rename from docker/php-apache/8.2-alpine/Dockerfile.jinja2 rename to docker/php-apache/8.2-alpine/Dockerfile.twig index 9d86aecba..4bd6f26db 100644 --- a/docker/php-apache/8.2-alpine/Dockerfile.jinja2 +++ b/docker/php-apache/8.2-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/apache.twig' as apache %} {{ docker.from("php", "8.2-alpine") }} {{ environment.web() }} diff --git a/docker/php-apache/8.2/Dockerfile.jinja2 b/docker/php-apache/8.2/Dockerfile.twig similarity index 60% rename from docker/php-apache/8.2/Dockerfile.jinja2 rename to docker/php-apache/8.2/Dockerfile.twig index 1e1fcfd21..3f839787e 100644 --- a/docker/php-apache/8.2/Dockerfile.jinja2 +++ b/docker/php-apache/8.2/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/apache.twig' as apache %} {{ docker.from("php", "8.2") }} {{ environment.web() }} diff --git a/docker/php-apache/8.3-alpine/Dockerfile.jinja2 b/docker/php-apache/8.3-alpine/Dockerfile.twig similarity index 58% rename from docker/php-apache/8.3-alpine/Dockerfile.jinja2 rename to docker/php-apache/8.3-alpine/Dockerfile.twig index 59e5ec3ff..d08802b51 100644 --- a/docker/php-apache/8.3-alpine/Dockerfile.jinja2 +++ b/docker/php-apache/8.3-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/apache.twig' as apache %} {{ docker.from("php", "8.3-alpine") }} {{ environment.web() }} diff --git a/docker/php-apache/8.3/Dockerfile.jinja2 b/docker/php-apache/8.3/Dockerfile.twig similarity index 60% rename from docker/php-apache/8.3/Dockerfile.jinja2 rename to docker/php-apache/8.3/Dockerfile.twig index 75fd9ae07..5747d0a2d 100644 --- a/docker/php-apache/8.3/Dockerfile.jinja2 +++ b/docker/php-apache/8.3/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/apache.twig' as apache %} {{ docker.from("php", "8.3") }} {{ environment.web() }} diff --git a/docker/php-apache/8.4-alpine/Dockerfile.jinja2 b/docker/php-apache/8.4-alpine/Dockerfile.twig similarity index 58% rename from docker/php-apache/8.4-alpine/Dockerfile.jinja2 rename to docker/php-apache/8.4-alpine/Dockerfile.twig index aeb26b9a1..9649a726c 100644 --- a/docker/php-apache/8.4-alpine/Dockerfile.jinja2 +++ b/docker/php-apache/8.4-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/apache.twig' as apache %} {{ docker.from("php", "8.4-alpine") }} {{ environment.web() }} diff --git a/docker/php-apache/8.4/Dockerfile.jinja2 b/docker/php-apache/8.4/Dockerfile.twig similarity index 60% rename from docker/php-apache/8.4/Dockerfile.jinja2 rename to docker/php-apache/8.4/Dockerfile.twig index c1aa1c7ff..8709d3f84 100644 --- a/docker/php-apache/8.4/Dockerfile.jinja2 +++ b/docker/php-apache/8.4/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/apache.twig' as apache %} {{ docker.from("php", "8.4") }} {{ environment.web() }} diff --git a/docker/php-apache/8.5-alpine/Dockerfile.jinja2 b/docker/php-apache/8.5-alpine/Dockerfile.twig similarity index 58% rename from docker/php-apache/8.5-alpine/Dockerfile.jinja2 rename to docker/php-apache/8.5-alpine/Dockerfile.twig index ee9f6d967..afc66a6b7 100644 --- a/docker/php-apache/8.5-alpine/Dockerfile.jinja2 +++ b/docker/php-apache/8.5-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/apache.twig' as apache %} {{ docker.from("php", "8.5-alpine") }} {{ environment.web() }} diff --git a/docker/php-apache/8.5/Dockerfile.jinja2 b/docker/php-apache/8.5/Dockerfile.twig similarity index 60% rename from docker/php-apache/8.5/Dockerfile.jinja2 rename to docker/php-apache/8.5/Dockerfile.twig index 5455b122f..a81023211 100644 --- a/docker/php-apache/8.5/Dockerfile.jinja2 +++ b/docker/php-apache/8.5/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/apache.twig' as apache %} {{ docker.from("php", "8.5") }} {{ environment.web() }} diff --git a/docker/php-dev/8.2-alpine/Dockerfile.jinja2 b/docker/php-dev/8.2-alpine/Dockerfile.twig similarity index 70% rename from docker/php-dev/8.2-alpine/Dockerfile.jinja2 rename to docker/php-dev/8.2-alpine/Dockerfile.twig index 3bcc38dd1..100f44415 100644 --- a/docker/php-dev/8.2-alpine/Dockerfile.jinja2 +++ b/docker/php-dev/8.2-alpine/Dockerfile.twig @@ -1,3 +1,6 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'images/php.twig' as php %} {{ docker.from("php", "8.2-alpine") }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/php-dev/8.2/Dockerfile.jinja2 b/docker/php-dev/8.2/Dockerfile.twig similarity index 71% rename from docker/php-dev/8.2/Dockerfile.jinja2 rename to docker/php-dev/8.2/Dockerfile.twig index b4aaa5f83..531ba4ee6 100644 --- a/docker/php-dev/8.2/Dockerfile.jinja2 +++ b/docker/php-dev/8.2/Dockerfile.twig @@ -1,3 +1,6 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'images/php.twig' as php %} {{ docker.from("php", "8.2") }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/php-dev/8.3-alpine/Dockerfile.jinja2 b/docker/php-dev/8.3-alpine/Dockerfile.twig similarity index 70% rename from docker/php-dev/8.3-alpine/Dockerfile.jinja2 rename to docker/php-dev/8.3-alpine/Dockerfile.twig index 2aace93be..af76fa649 100644 --- a/docker/php-dev/8.3-alpine/Dockerfile.jinja2 +++ b/docker/php-dev/8.3-alpine/Dockerfile.twig @@ -1,3 +1,6 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'images/php.twig' as php %} {{ docker.from("php", "8.3-alpine") }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/php-dev/8.3/Dockerfile.jinja2 b/docker/php-dev/8.3/Dockerfile.twig similarity index 71% rename from docker/php-dev/8.3/Dockerfile.jinja2 rename to docker/php-dev/8.3/Dockerfile.twig index a74afb508..99aecaeea 100644 --- a/docker/php-dev/8.3/Dockerfile.jinja2 +++ b/docker/php-dev/8.3/Dockerfile.twig @@ -1,3 +1,6 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'images/php.twig' as php %} {{ docker.from("php", "8.3") }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/php-dev/8.4-alpine/Dockerfile.jinja2 b/docker/php-dev/8.4-alpine/Dockerfile.twig similarity index 70% rename from docker/php-dev/8.4-alpine/Dockerfile.jinja2 rename to docker/php-dev/8.4-alpine/Dockerfile.twig index 22311a6ed..6458f6927 100644 --- a/docker/php-dev/8.4-alpine/Dockerfile.jinja2 +++ b/docker/php-dev/8.4-alpine/Dockerfile.twig @@ -1,3 +1,6 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'images/php.twig' as php %} {{ docker.from("php", "8.4-alpine") }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/php-dev/8.4/Dockerfile.jinja2 b/docker/php-dev/8.4/Dockerfile.twig similarity index 71% rename from docker/php-dev/8.4/Dockerfile.jinja2 rename to docker/php-dev/8.4/Dockerfile.twig index 6f5b9ecff..844ed7b1a 100644 --- a/docker/php-dev/8.4/Dockerfile.jinja2 +++ b/docker/php-dev/8.4/Dockerfile.twig @@ -1,3 +1,6 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'images/php.twig' as php %} {{ docker.from("php", "8.4") }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/php-dev/8.5-alpine/Dockerfile.jinja2 b/docker/php-dev/8.5-alpine/Dockerfile.twig similarity index 70% rename from docker/php-dev/8.5-alpine/Dockerfile.jinja2 rename to docker/php-dev/8.5-alpine/Dockerfile.twig index 4807811e9..3fb284e3a 100644 --- a/docker/php-dev/8.5-alpine/Dockerfile.jinja2 +++ b/docker/php-dev/8.5-alpine/Dockerfile.twig @@ -1,3 +1,6 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'images/php.twig' as php %} {{ docker.from("php", "8.5-alpine") }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/php-dev/8.5/Dockerfile.jinja2 b/docker/php-dev/8.5/Dockerfile.twig similarity index 71% rename from docker/php-dev/8.5/Dockerfile.jinja2 rename to docker/php-dev/8.5/Dockerfile.twig index d3fb6ccca..5fa26cad5 100644 --- a/docker/php-dev/8.5/Dockerfile.jinja2 +++ b/docker/php-dev/8.5/Dockerfile.twig @@ -1,3 +1,6 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'images/php.twig' as php %} {{ docker.from("php", "8.5") }} {{ docker.copy('conf/', '/opt/docker/') }} diff --git a/docker/php-nginx-dev/8.2-alpine/Dockerfile.jinja2 b/docker/php-nginx-dev/8.2-alpine/Dockerfile.twig similarity index 69% rename from docker/php-nginx-dev/8.2-alpine/Dockerfile.jinja2 rename to docker/php-nginx-dev/8.2-alpine/Dockerfile.twig index fce559a63..3088d24e3 100644 --- a/docker/php-nginx-dev/8.2-alpine/Dockerfile.jinja2 +++ b/docker/php-nginx-dev/8.2-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-nginx", "8.2-alpine") }} {{ environment.web() }} diff --git a/docker/php-nginx-dev/8.2/Dockerfile.jinja2 b/docker/php-nginx-dev/8.2/Dockerfile.twig similarity index 70% rename from docker/php-nginx-dev/8.2/Dockerfile.jinja2 rename to docker/php-nginx-dev/8.2/Dockerfile.twig index 512ef8d0f..585646bc1 100644 --- a/docker/php-nginx-dev/8.2/Dockerfile.jinja2 +++ b/docker/php-nginx-dev/8.2/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-nginx", "8.2") }} {{ environment.web() }} diff --git a/docker/php-nginx-dev/8.3-alpine/Dockerfile.jinja2 b/docker/php-nginx-dev/8.3-alpine/Dockerfile.twig similarity index 69% rename from docker/php-nginx-dev/8.3-alpine/Dockerfile.jinja2 rename to docker/php-nginx-dev/8.3-alpine/Dockerfile.twig index d8ef1fcae..d4ef447bb 100644 --- a/docker/php-nginx-dev/8.3-alpine/Dockerfile.jinja2 +++ b/docker/php-nginx-dev/8.3-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-nginx", "8.3-alpine") }} {{ environment.web() }} diff --git a/docker/php-nginx-dev/8.3/Dockerfile.jinja2 b/docker/php-nginx-dev/8.3/Dockerfile.twig similarity index 70% rename from docker/php-nginx-dev/8.3/Dockerfile.jinja2 rename to docker/php-nginx-dev/8.3/Dockerfile.twig index 0dec7c23b..05b8366b9 100644 --- a/docker/php-nginx-dev/8.3/Dockerfile.jinja2 +++ b/docker/php-nginx-dev/8.3/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-nginx", "8.3") }} {{ environment.web() }} diff --git a/docker/php-nginx-dev/8.4-alpine/Dockerfile.jinja2 b/docker/php-nginx-dev/8.4-alpine/Dockerfile.twig similarity index 69% rename from docker/php-nginx-dev/8.4-alpine/Dockerfile.jinja2 rename to docker/php-nginx-dev/8.4-alpine/Dockerfile.twig index be938992b..03f00f048 100644 --- a/docker/php-nginx-dev/8.4-alpine/Dockerfile.jinja2 +++ b/docker/php-nginx-dev/8.4-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-nginx", "8.4-alpine") }} {{ environment.web() }} diff --git a/docker/php-nginx-dev/8.4/Dockerfile.jinja2 b/docker/php-nginx-dev/8.4/Dockerfile.twig similarity index 70% rename from docker/php-nginx-dev/8.4/Dockerfile.jinja2 rename to docker/php-nginx-dev/8.4/Dockerfile.twig index dc57542f8..9cedabbe7 100644 --- a/docker/php-nginx-dev/8.4/Dockerfile.jinja2 +++ b/docker/php-nginx-dev/8.4/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-nginx", "8.4") }} {{ environment.web() }} diff --git a/docker/php-nginx-dev/8.5-alpine/Dockerfile.jinja2 b/docker/php-nginx-dev/8.5-alpine/Dockerfile.twig similarity index 69% rename from docker/php-nginx-dev/8.5-alpine/Dockerfile.jinja2 rename to docker/php-nginx-dev/8.5-alpine/Dockerfile.twig index 490164887..b1d027639 100644 --- a/docker/php-nginx-dev/8.5-alpine/Dockerfile.jinja2 +++ b/docker/php-nginx-dev/8.5-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-nginx", "8.5-alpine") }} {{ environment.web() }} diff --git a/docker/php-nginx-dev/8.5/Dockerfile.jinja2 b/docker/php-nginx-dev/8.5/Dockerfile.twig similarity index 70% rename from docker/php-nginx-dev/8.5/Dockerfile.jinja2 rename to docker/php-nginx-dev/8.5/Dockerfile.twig index 8d049f9a0..c8cdb75c8 100644 --- a/docker/php-nginx-dev/8.5/Dockerfile.jinja2 +++ b/docker/php-nginx-dev/8.5/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/php.twig' as php %} {{ docker.from("php-nginx", "8.5") }} {{ environment.web() }} diff --git a/docker/php-nginx/8.2-alpine/Dockerfile.jinja2 b/docker/php-nginx/8.2-alpine/Dockerfile.twig similarity index 61% rename from docker/php-nginx/8.2-alpine/Dockerfile.jinja2 rename to docker/php-nginx/8.2-alpine/Dockerfile.twig index e6291274f..dd499a71d 100644 --- a/docker/php-nginx/8.2-alpine/Dockerfile.jinja2 +++ b/docker/php-nginx/8.2-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/nginx.twig' as nginx %} {{ docker.from("php", "8.2-alpine") }} {{ environment.web() }} diff --git a/docker/php-nginx/8.2/Dockerfile.jinja2 b/docker/php-nginx/8.2/Dockerfile.twig similarity index 63% rename from docker/php-nginx/8.2/Dockerfile.jinja2 rename to docker/php-nginx/8.2/Dockerfile.twig index 14e0efc61..103e90f20 100644 --- a/docker/php-nginx/8.2/Dockerfile.jinja2 +++ b/docker/php-nginx/8.2/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/nginx.twig' as nginx %} {{ docker.from("php", "8.2") }} {{ environment.web() }} diff --git a/docker/php-nginx/8.3-alpine/Dockerfile.jinja2 b/docker/php-nginx/8.3-alpine/Dockerfile.twig similarity index 61% rename from docker/php-nginx/8.3-alpine/Dockerfile.jinja2 rename to docker/php-nginx/8.3-alpine/Dockerfile.twig index 1fa397bae..8998fb46c 100644 --- a/docker/php-nginx/8.3-alpine/Dockerfile.jinja2 +++ b/docker/php-nginx/8.3-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/nginx.twig' as nginx %} {{ docker.from("php", "8.3-alpine") }} {{ environment.web() }} diff --git a/docker/php-nginx/8.3/Dockerfile.jinja2 b/docker/php-nginx/8.3/Dockerfile.twig similarity index 63% rename from docker/php-nginx/8.3/Dockerfile.jinja2 rename to docker/php-nginx/8.3/Dockerfile.twig index 2683460dc..4e6cf9164 100644 --- a/docker/php-nginx/8.3/Dockerfile.jinja2 +++ b/docker/php-nginx/8.3/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/nginx.twig' as nginx %} {{ docker.from("php", "8.3") }} {{ environment.web() }} diff --git a/docker/php-nginx/8.4-alpine/Dockerfile.jinja2 b/docker/php-nginx/8.4-alpine/Dockerfile.twig similarity index 61% rename from docker/php-nginx/8.4-alpine/Dockerfile.jinja2 rename to docker/php-nginx/8.4-alpine/Dockerfile.twig index e916082f0..dfd1da29b 100644 --- a/docker/php-nginx/8.4-alpine/Dockerfile.jinja2 +++ b/docker/php-nginx/8.4-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/nginx.twig' as nginx %} {{ docker.from("php", "8.4-alpine") }} {{ environment.web() }} diff --git a/docker/php-nginx/8.4/Dockerfile.jinja2 b/docker/php-nginx/8.4/Dockerfile.twig similarity index 63% rename from docker/php-nginx/8.4/Dockerfile.jinja2 rename to docker/php-nginx/8.4/Dockerfile.twig index 96ac60a31..d54209c26 100644 --- a/docker/php-nginx/8.4/Dockerfile.jinja2 +++ b/docker/php-nginx/8.4/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/nginx.twig' as nginx %} {{ docker.from("php", "8.4") }} {{ environment.web() }} diff --git a/docker/php-nginx/8.5-alpine/Dockerfile.jinja2 b/docker/php-nginx/8.5-alpine/Dockerfile.twig similarity index 61% rename from docker/php-nginx/8.5-alpine/Dockerfile.jinja2 rename to docker/php-nginx/8.5-alpine/Dockerfile.twig index ed91b9f30..1e5f803b8 100644 --- a/docker/php-nginx/8.5-alpine/Dockerfile.jinja2 +++ b/docker/php-nginx/8.5-alpine/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/nginx.twig' as nginx %} {{ docker.from("php", "8.5-alpine") }} {{ environment.web() }} diff --git a/docker/php-nginx/8.5/Dockerfile.jinja2 b/docker/php-nginx/8.5/Dockerfile.twig similarity index 63% rename from docker/php-nginx/8.5/Dockerfile.jinja2 rename to docker/php-nginx/8.5/Dockerfile.twig index 19f483fee..2e8d4e463 100644 --- a/docker/php-nginx/8.5/Dockerfile.jinja2 +++ b/docker/php-nginx/8.5/Dockerfile.twig @@ -1,3 +1,7 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'images/nginx.twig' as nginx %} {{ docker.from("php", "8.5") }} {{ environment.web() }} diff --git a/docker/php-official/8.2-alpine/Dockerfile b/docker/php-official/8.2-alpine/Dockerfile index 16c664007..72a4bf0ee 100644 --- a/docker/php-official/8.2-alpine/Dockerfile +++ b/docker/php-official/8.2-alpine/Dockerfile @@ -22,7 +22,7 @@ ENV APPLICATION_USER=application \ APPLICATION_PATH=/app \ APPLICATION_UID=1000 \ APPLICATION_GID=1000 -ENV PHP_SENDMAIL_PATH="/usr/sbin/sendmail -t -i" +ENV PHP_SENDMAIL_PATH="/usr/sbin/sendmail -t -i" ENV LD_PRELOAD="/usr/lib/preloadable_libiconv.so" ENV COMPOSER_VERSION="2" diff --git a/docker/php-official/8.2-alpine/Dockerfile.jinja2 b/docker/php-official/8.2-alpine/Dockerfile.twig similarity index 66% rename from docker/php-official/8.2-alpine/Dockerfile.jinja2 rename to docker/php-official/8.2-alpine/Dockerfile.twig index 7c8770169..4ce2df54f 100644 --- a/docker/php-official/8.2-alpine/Dockerfile.jinja2 +++ b/docker/php-official/8.2-alpine/Dockerfile.twig @@ -1,3 +1,11 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} +{% import 'images/base.twig' as base %} +{% import 'images/baseapp.twig' as baseapp %} +{% import 'images/php.twig' as php %} {{ docker.fromOfficial("php", "8.2-fpm-alpine") }} @@ -6,7 +14,8 @@ {{ environment.general() }} {{ environment.base() }} {{ environment.baseApp() }} -{{ environment.phpOfficialSendmailWorkaround() }} {# Check if needed #} +{# Check if needed #} +{{ environment.phpOfficialSendmailWorkaround() }} {{ environment.phpAlpineIconvWorkaround() }} {{ environment.phpComposerVersion() }} diff --git a/docker/php-official/8.2/Dockerfile.jinja2 b/docker/php-official/8.2/Dockerfile.twig similarity index 72% rename from docker/php-official/8.2/Dockerfile.jinja2 rename to docker/php-official/8.2/Dockerfile.twig index 0f34b7a2d..ff10dc522 100644 --- a/docker/php-official/8.2/Dockerfile.jinja2 +++ b/docker/php-official/8.2/Dockerfile.twig @@ -1,3 +1,11 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} +{% import 'images/base.twig' as base %} +{% import 'images/baseapp.twig' as baseapp %} +{% import 'images/php.twig' as php %} {{ docker.fromOfficial("php", "8.2-fpm-bookworm") }} diff --git a/docker/php-official/8.3-alpine/Dockerfile b/docker/php-official/8.3-alpine/Dockerfile index 91e766232..e989c59c6 100644 --- a/docker/php-official/8.3-alpine/Dockerfile +++ b/docker/php-official/8.3-alpine/Dockerfile @@ -22,7 +22,7 @@ ENV APPLICATION_USER=application \ APPLICATION_PATH=/app \ APPLICATION_UID=1000 \ APPLICATION_GID=1000 -ENV PHP_SENDMAIL_PATH="/usr/sbin/sendmail -t -i" +ENV PHP_SENDMAIL_PATH="/usr/sbin/sendmail -t -i" ENV LD_PRELOAD="/usr/lib/preloadable_libiconv.so" ENV COMPOSER_VERSION="2" diff --git a/docker/php-official/8.3-alpine/Dockerfile.jinja2 b/docker/php-official/8.3-alpine/Dockerfile.twig similarity index 66% rename from docker/php-official/8.3-alpine/Dockerfile.jinja2 rename to docker/php-official/8.3-alpine/Dockerfile.twig index 6cb7047d8..4ed95eb4d 100644 --- a/docker/php-official/8.3-alpine/Dockerfile.jinja2 +++ b/docker/php-official/8.3-alpine/Dockerfile.twig @@ -1,3 +1,11 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} +{% import 'images/base.twig' as base %} +{% import 'images/baseapp.twig' as baseapp %} +{% import 'images/php.twig' as php %} {{ docker.fromOfficial("php", "8.3-fpm-alpine") }} @@ -6,7 +14,8 @@ {{ environment.general() }} {{ environment.base() }} {{ environment.baseApp() }} -{{ environment.phpOfficialSendmailWorkaround() }} {# Check if needed #} +{# Check if needed #} +{{ environment.phpOfficialSendmailWorkaround() }} {{ environment.phpAlpineIconvWorkaround() }} {{ environment.phpComposerVersion() }} diff --git a/docker/php-official/8.3/Dockerfile.jinja2 b/docker/php-official/8.3/Dockerfile.twig similarity index 72% rename from docker/php-official/8.3/Dockerfile.jinja2 rename to docker/php-official/8.3/Dockerfile.twig index d31cc3db2..5a0d17df8 100644 --- a/docker/php-official/8.3/Dockerfile.jinja2 +++ b/docker/php-official/8.3/Dockerfile.twig @@ -1,3 +1,11 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} +{% import 'images/base.twig' as base %} +{% import 'images/baseapp.twig' as baseapp %} +{% import 'images/php.twig' as php %} {{ docker.fromOfficial("php", "8.3-fpm-bookworm") }} diff --git a/docker/php-official/8.4-alpine/Dockerfile b/docker/php-official/8.4-alpine/Dockerfile index 126a6dfb3..6ec47d926 100644 --- a/docker/php-official/8.4-alpine/Dockerfile +++ b/docker/php-official/8.4-alpine/Dockerfile @@ -22,7 +22,7 @@ ENV APPLICATION_USER=application \ APPLICATION_PATH=/app \ APPLICATION_UID=1000 \ APPLICATION_GID=1000 -ENV PHP_SENDMAIL_PATH="/usr/sbin/sendmail -t -i" +ENV PHP_SENDMAIL_PATH="/usr/sbin/sendmail -t -i" ENV LD_PRELOAD="/usr/lib/preloadable_libiconv.so" ENV COMPOSER_VERSION="2" diff --git a/docker/php-official/8.4-alpine/Dockerfile.jinja2 b/docker/php-official/8.4-alpine/Dockerfile.twig similarity index 66% rename from docker/php-official/8.4-alpine/Dockerfile.jinja2 rename to docker/php-official/8.4-alpine/Dockerfile.twig index 60f1707af..439e31c5b 100644 --- a/docker/php-official/8.4-alpine/Dockerfile.jinja2 +++ b/docker/php-official/8.4-alpine/Dockerfile.twig @@ -1,3 +1,11 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} +{% import 'images/base.twig' as base %} +{% import 'images/baseapp.twig' as baseapp %} +{% import 'images/php.twig' as php %} {{ docker.fromOfficial("php", "8.4-fpm-alpine") }} @@ -6,7 +14,8 @@ {{ environment.general() }} {{ environment.base() }} {{ environment.baseApp() }} -{{ environment.phpOfficialSendmailWorkaround() }} {# Check if needed #} +{# Check if needed #} +{{ environment.phpOfficialSendmailWorkaround() }} {{ environment.phpAlpineIconvWorkaround() }} {{ environment.phpComposerVersion() }} diff --git a/docker/php-official/8.4/Dockerfile.jinja2 b/docker/php-official/8.4/Dockerfile.twig similarity index 72% rename from docker/php-official/8.4/Dockerfile.jinja2 rename to docker/php-official/8.4/Dockerfile.twig index a742419b6..2ac7a5059 100644 --- a/docker/php-official/8.4/Dockerfile.jinja2 +++ b/docker/php-official/8.4/Dockerfile.twig @@ -1,3 +1,11 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} +{% import 'images/base.twig' as base %} +{% import 'images/baseapp.twig' as baseapp %} +{% import 'images/php.twig' as php %} {{ docker.fromOfficial("php", "8.4-fpm-bookworm") }} diff --git a/docker/php-official/8.5-alpine/Dockerfile b/docker/php-official/8.5-alpine/Dockerfile index a888149d4..42b25e8e2 100644 --- a/docker/php-official/8.5-alpine/Dockerfile +++ b/docker/php-official/8.5-alpine/Dockerfile @@ -22,7 +22,7 @@ ENV APPLICATION_USER=application \ APPLICATION_PATH=/app \ APPLICATION_UID=1000 \ APPLICATION_GID=1000 -ENV PHP_SENDMAIL_PATH="/usr/sbin/sendmail -t -i" +ENV PHP_SENDMAIL_PATH="/usr/sbin/sendmail -t -i" ENV LD_PRELOAD="/usr/lib/preloadable_libiconv.so" ENV COMPOSER_VERSION="2" diff --git a/docker/php-official/8.5-alpine/Dockerfile.jinja2 b/docker/php-official/8.5-alpine/Dockerfile.twig similarity index 66% rename from docker/php-official/8.5-alpine/Dockerfile.jinja2 rename to docker/php-official/8.5-alpine/Dockerfile.twig index 1c85a2eb5..e60f1eb4d 100644 --- a/docker/php-official/8.5-alpine/Dockerfile.jinja2 +++ b/docker/php-official/8.5-alpine/Dockerfile.twig @@ -1,3 +1,11 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} +{% import 'images/base.twig' as base %} +{% import 'images/baseapp.twig' as baseapp %} +{% import 'images/php.twig' as php %} {{ docker.fromOfficial("php", "8.5-fpm-alpine") }} @@ -6,7 +14,8 @@ {{ environment.general() }} {{ environment.base() }} {{ environment.baseApp() }} -{{ environment.phpOfficialSendmailWorkaround() }} {# Check if needed #} +{# Check if needed #} +{{ environment.phpOfficialSendmailWorkaround() }} {{ environment.phpAlpineIconvWorkaround() }} {{ environment.phpComposerVersion() }} diff --git a/docker/php-official/8.5/Dockerfile.jinja2 b/docker/php-official/8.5/Dockerfile.twig similarity index 72% rename from docker/php-official/8.5/Dockerfile.jinja2 rename to docker/php-official/8.5/Dockerfile.twig index 17a88b6e7..32c9cc74f 100644 --- a/docker/php-official/8.5/Dockerfile.jinja2 +++ b/docker/php-official/8.5/Dockerfile.twig @@ -1,3 +1,11 @@ +{% import 'docker.twig' as docker %} +{% import 'provision.twig' as provision %} +{% import 'environment.twig' as environment %} +{% import 'baselayout.twig' as baselayout %} +{% import 'images/bootstrap.twig' as bootstrap %} +{% import 'images/base.twig' as base %} +{% import 'images/baseapp.twig' as baseapp %} +{% import 'images/php.twig' as php %} {{ docker.fromOfficial("php", "8.5-fpm-bookworm") }} diff --git a/docker/toolbox/latest/Dockerfile.jinja2 b/docker/toolbox/latest/Dockerfile.twig similarity index 92% rename from docker/toolbox/latest/Dockerfile.jinja2 rename to docker/toolbox/latest/Dockerfile.twig index cd1b0c49f..0b4648f90 100644 --- a/docker/toolbox/latest/Dockerfile.jinja2 +++ b/docker/toolbox/latest/Dockerfile.twig @@ -1,3 +1,5 @@ +{% import 'docker.twig' as docker %} +{% import 'tools.twig' as tools %} {{ docker.fromOfficial("alpine") }} ARG TARGETARCH diff --git a/docs/README.md b/docs/README.md index 2853b8dd6..5aa965623 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,7 +13,7 @@ Documentation for the `webdevops/*` Docker images. It describes the images as th | [`webdevops/apache`](images/apache.md) | Apache without PHP | | [`webdevops/bootstrap`, `webdevops/base`, `webdevops/base-app`](images/base.md) | Base images for your own images | | `webdevops/toolbox` | Build helper, not meant to run. Stores the baselayout scripts (`apt-install`, `apk-install`, `docker-image-cleanup`, `docker-run-bootstrap`, `generate-locales` and more), `go-replace` and `gosu` under `/baselayout`. `bootstrap` and `php` copy them from there with `COPY --from=webdevops/toolbox`. In `toolbox` itself only `go-replace` is on the `PATH`. | -| `webdevops/dockerfile-build-env` | Build environment for this repository, based on `base-app:ubuntu-22.04`. It runs `make provision`, see [Build process](../README.md#build-process). | +| `webdevops/dockerfile-build-env` | Build environment for this repository, based on `base-app:ubuntu-22.04`. The CI runs the serverspec and structure tests in it, see [Build process](../README.md#build-process). | `webdevops/storage`, `webdevops/vsftp` and `webdevops/ssh` are no longer built. [Replacing discontinued images](discontinued-images.md) shows what to use instead. diff --git a/docs/known-issues.md b/docs/known-issues.md index ba5304f67..9b9eb7fbf 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -119,12 +119,6 @@ These options and tools are part of the images but do not work, or work only in - **Images:** `php-*` with PHP 8.4 and 8.5. - **Instead:** Use Composer 2, the default. -## Build tools - -- **Effect:** The Makefile target `provision` calls `bin/console`, which needs Python 2. On a host without Python 2 it fails with `python2: No such file or directory`. Inside `webdevops/dockerfile-build-env` it works. -- **Instead:** Run `make provision` in `webdevops/dockerfile-build-env` as shown in the [README](../README.md#file-generation). Build images with `docker build` or `./build-local.sh`, see [Building and testing locally](../README.md#building-and-testing-locally). The CI runs the serverspec tests for every image except `toolbox` and `dockerfile-build-env`, and the structure tests for `bootstrap`, `base`, `base-app`, `php`, `php-dev` and `php-nginx`. To run the structure tests against the images you built locally, use `docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:/app -w /app webdevops/dockerfile-build-env make structure-test`. It tests the local tags that the CI builds and fails if no such image exists. -- **Issue:** [#549](https://github.com/webdevops/Dockerfile/issues/549) (move away from Python 2). - ## Migration from older images - `XDEBUG_REMOTE_*`, `XDEBUG_IDE_KEY`, `XDEBUG_MAX_NESTING_LEVEL` and `XDEBUG_PROFILER_*` are ignored; they belong to Xdebug 2. Set the Xdebug 3 directive with `php.xdebug.` instead, for example `php.xdebug.max_nesting_level: "500"` or `php.xdebug.idekey: PHPSTORM`. The variables listed under [Development images](environment-variables.md#development-images) still work. diff --git a/template/Dockerfile/baselayout.jinja2 b/template/Dockerfile/baselayout.twig similarity index 80% rename from template/Dockerfile/baselayout.jinja2 rename to template/Dockerfile/baselayout.twig index 5bb4a1303..61b1991a5 100644 --- a/template/Dockerfile/baselayout.jinja2 +++ b/template/Dockerfile/baselayout.twig @@ -1,6 +1,5 @@ -{% import 'Dockerfile/tools.jinja2' as tools %} - {% macro copy() %} + # Baselayout copy (from staged image) COPY --from=webdevops/toolbox /baselayout/sbin/* /sbin/ COPY --from=webdevops/toolbox /baselayout/usr/local/bin/* /usr/local/bin/ diff --git a/template/Dockerfile/docker.jinja2 b/template/Dockerfile/docker.twig similarity index 100% rename from template/Dockerfile/docker.jinja2 rename to template/Dockerfile/docker.twig diff --git a/template/Dockerfile/environment.jinja2 b/template/Dockerfile/environment.twig similarity index 100% rename from template/Dockerfile/environment.jinja2 rename to template/Dockerfile/environment.twig diff --git a/template/Dockerfile/images/apache.jinja2 b/template/Dockerfile/images/apache.twig similarity index 100% rename from template/Dockerfile/images/apache.jinja2 rename to template/Dockerfile/images/apache.twig diff --git a/template/Dockerfile/images/base.jinja2 b/template/Dockerfile/images/base.twig similarity index 87% rename from template/Dockerfile/images/base.jinja2 rename to template/Dockerfile/images/base.twig index f63c8e508..3d622f196 100644 --- a/template/Dockerfile/images/base.jinja2 +++ b/template/Dockerfile/images/base.twig @@ -1,6 +1,3 @@ -{% import 'Dockerfile/docker.jinja2' as docker %} -{% import 'Dockerfile/provision.jinja2' as provision %} - {% macro alpine(role='') -%} # Install services && chmod +x /opt/docker/bin/* \ diff --git a/template/Dockerfile/images/baseapp.jinja2 b/template/Dockerfile/images/baseapp.twig similarity index 89% rename from template/Dockerfile/images/baseapp.jinja2 rename to template/Dockerfile/images/baseapp.twig index 3f2774467..6e47a3243 100644 --- a/template/Dockerfile/images/baseapp.jinja2 +++ b/template/Dockerfile/images/baseapp.twig @@ -1,6 +1,3 @@ -{% import 'Dockerfile/docker.jinja2' as docker %} -{% import 'Dockerfile/provision.jinja2' as provision %} - {% macro alpine(role='') -%} && apk-install shadow \ && apk-install \ diff --git a/template/Dockerfile/images/bootstrap.jinja2 b/template/Dockerfile/images/bootstrap.twig similarity index 96% rename from template/Dockerfile/images/bootstrap.jinja2 rename to template/Dockerfile/images/bootstrap.twig index e18f7c2eb..65aff5ca9 100644 --- a/template/Dockerfile/images/bootstrap.jinja2 +++ b/template/Dockerfile/images/bootstrap.twig @@ -1,5 +1,3 @@ -{% import 'Dockerfile/docker.jinja2' as docker %} - {% macro alpine(version='') -%} # Init bootstrap # Add community diff --git a/template/Dockerfile/images/nginx.jinja2 b/template/Dockerfile/images/nginx.twig similarity index 81% rename from template/Dockerfile/images/nginx.jinja2 rename to template/Dockerfile/images/nginx.twig index 1be1ae268..98401d5a2 100644 --- a/template/Dockerfile/images/nginx.jinja2 +++ b/template/Dockerfile/images/nginx.twig @@ -1,6 +1,3 @@ -{% import 'Dockerfile/docker.jinja2' as docker %} -{% import 'Dockerfile/provision.jinja2' as provision %} - {% macro alpine(role='') -%} # Install nginx && apk-install \ diff --git a/template/Dockerfile/images/php-extensions.twig b/template/Dockerfile/images/php-extensions.twig new file mode 100644 index 000000000..173eb917d --- /dev/null +++ b/template/Dockerfile/images/php-extensions.twig @@ -0,0 +1,36 @@ +{# Extensions that are not part of PHP, built from PIE packages or their sources #} +{% macro external() -%} + && cd / \ + && PIE_WORKING_DIRECTORY=/tmp/pie pie install --no-interaction --no-cache \ + --skip-enable-extension --make-parallel-jobs="$(nproc)" \ + apcu/apcu \ + wikimedia/excimer \ + open-telemetry/ext-opentelemetry \ + php-memcached/php-memcached \ + pecl/yaml \ + phpredis/phpredis \ + mongodb/mongodb-extension \ + && rm -rf /tmp/pie \ + && PROTOBUF_RELEASE_URL="$(curl -fsSLI --proto '=https' --proto-redir '=https' --retry 3 -o /dev/null -w '%{url_effective}' https://github.com/protocolbuffers/protobuf/releases/latest)" \ + && PROTOBUF_VERSION="${PROTOBUF_RELEASE_URL##*/}" \ + && mkdir -p /tmp/protobuf \ + && curl -fsSL --proto '=https' --proto-redir '=https' --retry 3 "https://codeload.github.com/protocolbuffers/protobuf/tar.gz/refs/tags/${PROTOBUF_VERSION}" -o /tmp/protobuf.tar.gz \ + && tar -xzf /tmp/protobuf.tar.gz --strip-components=1 -C /tmp/protobuf \ + && cd /tmp/protobuf/php/ext/google/protobuf \ + && mkdir -p third_party \ + && cp -R /tmp/protobuf/third_party/utf8_range third_party/ \ + && phpize && ./configure --enable-protobuf \ + && make -j"$(nproc)" && make install \ + && cd / && rm -rf /tmp/protobuf /tmp/protobuf.tar.gz \ + # Vips' GitHub tags lag behind PECL releases; use the upstream branch. + && git clone --depth 1 https://github.com/libvips/php-vips-ext.git /tmp/vips \ + && cd /tmp/vips \ + && phpize && ./configure --with-vips \ + && make -j"$(nproc)" && make install \ + && cd / && rm -rf /tmp/vips \ + && git clone --depth 1 --branch xmlrpc-1.0.0RC3 https://github.com/php/pecl-networking-xmlrpc.git /tmp/xmlrpc \ + && cd /tmp/xmlrpc \ + && phpize && ./configure --with-xmlrpc \ + && make -j"$(nproc)" && make install \ + && cd / && rm -rf /tmp/xmlrpc +{%- endmacro %} diff --git a/template/Dockerfile/images/php.jinja2 b/template/Dockerfile/images/php.twig similarity index 79% rename from template/Dockerfile/images/php.jinja2 rename to template/Dockerfile/images/php.twig index 16eab6f41..f8dce0ef6 100644 --- a/template/Dockerfile/images/php.jinja2 +++ b/template/Dockerfile/images/php.twig @@ -1,44 +1,6 @@ -{% import 'Dockerfile/docker.jinja2' as docker %} -{% import 'Dockerfile/provision.jinja2' as provision %} -{% import 'Dockerfile/services.jinja2' as services %} - -{% macro externalExtensions() -%} - && cd / \ - && PIE_WORKING_DIRECTORY=/tmp/pie pie install --no-interaction --no-cache \ - --skip-enable-extension --make-parallel-jobs="$(nproc)" \ - apcu/apcu \ - wikimedia/excimer \ - open-telemetry/ext-opentelemetry \ - php-memcached/php-memcached \ - pecl/yaml \ - phpredis/phpredis \ - mongodb/mongodb-extension \ - && rm -rf /tmp/pie \ - && PROTOBUF_RELEASE_URL="$(curl -fsSLI --proto '=https' --proto-redir '=https' --retry 3 -o /dev/null -w '%{url_effective}' https://github.com/protocolbuffers/protobuf/releases/latest)" \ - && PROTOBUF_VERSION="${PROTOBUF_RELEASE_URL##*/}" \ - && mkdir -p /tmp/protobuf \ - && curl -fsSL --proto '=https' --proto-redir '=https' --retry 3 "https://codeload.github.com/protocolbuffers/protobuf/tar.gz/refs/tags/${PROTOBUF_VERSION}" -o /tmp/protobuf.tar.gz \ - && tar -xzf /tmp/protobuf.tar.gz --strip-components=1 -C /tmp/protobuf \ - && cd /tmp/protobuf/php/ext/google/protobuf \ - && mkdir -p third_party \ - && cp -R /tmp/protobuf/third_party/utf8_range third_party/ \ - && phpize && ./configure --enable-protobuf \ - && make -j"$(nproc)" && make install \ - && cd / && rm -rf /tmp/protobuf /tmp/protobuf.tar.gz \ - # Vips' GitHub tags lag behind PECL releases; use the upstream branch. - && git clone --depth 1 https://github.com/libvips/php-vips-ext.git /tmp/vips \ - && cd /tmp/vips \ - && phpize && ./configure --with-vips \ - && make -j"$(nproc)" && make install \ - && cd / && rm -rf /tmp/vips \ - && git clone --depth 1 --branch xmlrpc-1.0.0RC3 https://github.com/php/pecl-networking-xmlrpc.git /tmp/xmlrpc \ - && cd /tmp/xmlrpc \ - && phpize && ./configure --with-xmlrpc \ - && make -j"$(nproc)" && make install \ - && cd / && rm -rf /tmp/xmlrpc -{%- endmacro %} - {% macro official(role='', version='') -%} +{% import 'services.twig' as services -%} +{% import 'images/php-extensions.twig' as extensions -%} # Install php environment && apt-install \ # Install tools @@ -66,9 +28,9 @@ libmcrypt4 \ libzip4 \ libyaml-0-2 \ -{%- if version == '8.2' or version == '8.3' %} +{% if version == '8.2' or version == '8.3' %} libc-client2007e \ -{%- endif %} +{% endif %} # Dev and headers libavif-dev \ libjpeg62-turbo-dev \ @@ -111,14 +73,14 @@ && cd /usr/src/php/ext/amqp && git submodule update --init \ && git clone --branch master --depth 1 https://github.com/Imagick/imagick.git /usr/src/php/ext/imagick \ && docker-php-ext-configure ldap \ -{%- if version == '8.2' or version == '8.3' %} +{% if version == '8.2' or version == '8.3' %} && docker-php-ext-configure ftp --with-openssl-dir \ -{%- elif version == '8.4' or version == '8.5' %} +{% elseif version == '8.4' or version == '8.5' %} && docker-php-ext-configure ftp --with-ftp-ssl \ -{%- endif %} -{%- if version != '8.4' and version != '8.5' %} +{% endif %} +{% if version != '8.4' and version != '8.5' %} && PHP_OPENSSL=yes docker-php-ext-configure imap --with-kerberos --with-imap-ssl \ -{%- endif %} +{% endif %} && docker-php-ext-install -j"$(nproc)" \ bcmath \ amqp \ @@ -133,9 +95,9 @@ gettext \ ldap \ mysqli \ -{%- if version != '8.4' and version != '8.5' %} +{% if version != '8.4' and version != '8.5' %} imap \ -{%- endif %} +{% endif %} pcntl \ pdo_mysql \ pdo_pgsql \ @@ -149,10 +111,10 @@ xsl \ zip \ gd \ -{%- if version != '8.5' %} +{% if version != '8.5' %} opcache \ -{%- endif %} - {{ externalExtensions() }} \ +{% endif %} + {{ extensions.external() }} \ && echo extension=apcu.so > /usr/local/etc/php/conf.d/apcu.ini \ && echo extension=excimer.so > /usr/local/etc/php/conf.d/excimer.ini \ && echo extension=opentelemetry.so > /usr/local/etc/php/conf.d/opentelemetry.ini \ @@ -195,6 +157,8 @@ {%- endmacro %} {% macro officialAlpine(role='', version='') -%} +{% import 'services.twig' as services -%} +{% import 'images/php-extensions.twig' as extensions -%} # Install php environment && apk-install \ file \ @@ -266,14 +230,14 @@ && git clone --branch latest --depth 1 https://github.com/php-amqp/php-amqp.git /usr/src/php/ext/amqp \ && cd /usr/src/php/ext/amqp && git submodule update --init \ && docker-php-ext-configure ldap \ -{%- if version == '8.2' or version == '8.3' %} +{% if version == '8.2' or version == '8.3' %} && docker-php-ext-configure ftp --with-openssl-dir \ -{%- elif version == '8.4' or version == '8.5' %} +{% elseif version == '8.4' or version == '8.5' %} && docker-php-ext-configure ftp --with-ftp-ssl \ -{%- endif %} -{%- if version != '8.4' and version != '8.5' %} +{% endif %} +{% if version != '8.4' and version != '8.5' %} && PHP_OPENSSL=yes docker-php-ext-configure imap --with-kerberos --with-imap-ssl \ -{%- endif %} +{% endif %} && docker-php-ext-install -j"$(nproc)" \ bcmath \ bz2 \ @@ -288,9 +252,9 @@ gettext \ ldap \ mysqli \ -{%- if version != '8.4' and version != '8.5' %} +{% if version != '8.4' and version != '8.5' %} imap \ -{%- endif %} +{% endif %} pcntl \ pdo_mysql \ pdo_pgsql \ @@ -304,10 +268,10 @@ xsl \ zip \ gd \ -{%- if version != '8.5' %} +{% if version != '8.5' %} opcache \ -{%- endif %} - {{ externalExtensions() }} \ +{% endif %} + {{ extensions.external() }} \ && docker-php-ext-enable \ apcu \ excimer \ @@ -358,6 +322,7 @@ {%- endmacro %} {% macro officialDevelopment(role='', version='') -%} +{% import 'services.twig' as services -%} # Install development environment && BLACKFIRE_INSTALL_METHOD=raw bash -c "$(curl -L https://installer.blackfire.io/installer.sh)" \ && blackfire php:install \ @@ -372,6 +337,7 @@ {%- endmacro %} {% macro officialDevelopmentAlpine(role='', version='') -%} +{% import 'services.twig' as services -%} # Install development environment && bash -c "$(curl -L https://installer.blackfire.io/installer.sh)" \ && blackfire php:install \ diff --git a/template/Dockerfile/layout.jinja2 b/template/Dockerfile/layout.jinja2 deleted file mode 100644 index ac8740948..000000000 --- a/template/Dockerfile/layout.jinja2 +++ /dev/null @@ -1,22 +0,0 @@ -{% import 'Dockerfile/docker.jinja2' as docker %} -{% import 'Dockerfile/provision.jinja2' as provision %} -{% import 'Dockerfile/environment.jinja2' as environment %} -{% import 'Dockerfile/services.jinja2' as services %} -{% import 'Dockerfile/tools.jinja2' as tools %} -{% import 'Dockerfile/baselayout.jinja2' as baselayout %} - -{% import 'Dockerfile/images/bootstrap.jinja2' as bootstrap %} -{% import 'Dockerfile/images/base.jinja2' as base %} -{% import 'Dockerfile/images/baseapp.jinja2' as baseapp %} -{% import 'Dockerfile/images/apache.jinja2' as apache %} -{% import 'Dockerfile/images/nginx.jinja2' as nginx %} -{% import 'Dockerfile/images/php.jinja2' as php %} - -#+++++++++++++++++++++++++++++++++++++++ -# Dockerfile for webdevops/{{ Dockerfile.image }}:{{ Dockerfile.tag }} -# -- automatically generated -- -#+++++++++++++++++++++++++++++++++++++++ - -{% block content %} - -{% endblock %} diff --git a/template/Dockerfile/layout.twig b/template/Dockerfile/layout.twig new file mode 100644 index 000000000..1b1d76a6b --- /dev/null +++ b/template/Dockerfile/layout.twig @@ -0,0 +1,6 @@ +#+++++++++++++++++++++++++++++++++++++++ +# Dockerfile for webdevops/{{ Dockerfile.image }}:{{ Dockerfile.tag }} +# -- automatically generated -- +#+++++++++++++++++++++++++++++++++++++++ + +{{ include(template) -}} diff --git a/template/Dockerfile/provision.jinja2 b/template/Dockerfile/provision.twig similarity index 100% rename from template/Dockerfile/provision.jinja2 rename to template/Dockerfile/provision.twig diff --git a/template/Dockerfile/services.jinja2 b/template/Dockerfile/services.twig similarity index 100% rename from template/Dockerfile/services.jinja2 rename to template/Dockerfile/services.twig diff --git a/template/Dockerfile/tools.jinja2 b/template/Dockerfile/tools.twig similarity index 100% rename from template/Dockerfile/tools.jinja2 rename to template/Dockerfile/tools.twig diff --git a/tests/serverspec/__init__.py b/tests/serverspec/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/structure-test/install.sh b/tests/structure-test/install.sh deleted file mode 100755 index 8ce4eb0bc..000000000 --- a/tests/structure-test/install.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 \ - && chmod +x container-structure-test-linux-amd64 \ - && sudo mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test