diff options
Diffstat (limited to 'tests/docker/Makefile.include')
| -rw-r--r-- | tests/docker/Makefile.include | 214 |
1 files changed, 103 insertions, 111 deletions
diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 7032c68895..8df50a0ca0 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -1,26 +1,23 @@ # Makefile for Docker tests -.PHONY: docker docker-test docker-clean docker-image docker-qemu-src +.PHONY: docker docker-help docker-test docker-clean docker-image docker-qemu-src -DOCKER_SUFFIX := .docker -DOCKER_FILES_DIR := $(SRC_PATH)/tests/docker/dockerfiles -DOCKER_DEPRECATED_IMAGES := debian -# we don't run tests on intermediate images (used as base by another image) -DOCKER_PARTIAL_IMAGES := debian debian8 debian9 debian8-mxe debian-ports debian-sid debian-bootstrap -DOCKER_IMAGES := $(filter-out $(DOCKER_DEPRECATED_IMAGES),$(sort $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.docker))))) -DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES)) -# Use a global constant ccache directory to speed up repetitive builds -DOCKER_CCACHE_DIR := $$HOME/.cache/qemu-docker-ccache - -DOCKER_TESTS := $(notdir $(shell \ - find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f)) +NULL := +SPACE := $(NULL) # +COMMA := , -DOCKER_TOOLS := travis +HOST_ARCH = $(shell uname -m) +USER = $(if $(NOUSER),,$(shell id -un)) +UID = $(if $(NOUSER),,$(shell id -u)) -DOCKER_SCRIPT=$(SRC_PATH)/tests/docker/docker.py +DOCKER_FILES_DIR := $(SRC_PATH)/tests/docker/dockerfiles +ifeq ($(HOST_ARCH),x86_64) +DOCKER_DEFAULT_REGISTRY := registry.gitlab.com/qemu-project/qemu +endif +DOCKER_REGISTRY := $(if $(REGISTRY),$(REGISTRY),$(DOCKER_DEFAULT_REGISTRY)) -TESTS ?= % -IMAGES ?= % +RUNC ?= $(if $(shell command -v docker), docker, podman) +DOCKER_SCRIPT=$(SRC_PATH)/tests/docker/docker.py --engine $(RUNC) CUR_TIME := $(shell date +%Y-%m-%d-%H.%M.%S.$$$$) DOCKER_SRC_COPY := $(BUILD_DIR)/docker-src.$(CUR_TIME) @@ -38,25 +35,19 @@ $(DOCKER_SRC_COPY): docker-qemu-src: $(DOCKER_SRC_COPY) -docker-image: ${DOCKER_TARGETS} - -# General rule for building docker images. If we are a sub-make -# invoked with SKIP_DOCKER_BUILD we still check the image is up to date -# though -ifdef SKIP_DOCKER_BUILD +# General rule for building docker images. docker-image-%: $(DOCKER_FILES_DIR)/%.docker - $(call quiet-command, \ - $(DOCKER_SCRIPT) check --quiet qemu:$* $<, \ - "CHECK", "$*") -else -docker-image-%: $(DOCKER_FILES_DIR)/%.docker - $(call quiet-command,\ - $(DOCKER_SCRIPT) build qemu:$* $< \ - $(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \ - $(if $(NOUSER),,--add-current-user) \ - $(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES))\ - $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\ - "BUILD","$*") + $(call quiet-command, \ + DOCKER_BUILDKIT=1 $(RUNC) build \ + $(if $V,,--quiet) \ + $(if $(NOCACHE),--no-cache, \ + $(if $(DOCKER_REGISTRY),--cache-from $(DOCKER_REGISTRY)/qemu/$*)) \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + $(if $(NOUSER),, \ + --build-arg USER=$(USER) \ + --build-arg UID=$(UID)) \ + -t qemu/$* - < $< $(if $V,,> /dev/null),\ + "BUILD", $*) # Special rule for debootstraped binfmt linux-user images docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker @@ -69,72 +60,79 @@ docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker $(if $(wildcard $(EXECUTABLE)), \ $(call quiet-command, \ DEB_ARCH=$(DEB_ARCH) \ - DEB_TYPE=$(DEB_TYPE) \ - $(DOCKER_SCRIPT) build qemu:debian-$* $< \ + DEB_TYPE=$(DEB_TYPE) \ + $(if $(DEB_URL),DEB_URL=$(DEB_URL),) \ + $(DOCKER_SCRIPT) build -t qemu/debian-$* -f $< \ $(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \ $(if $(NOUSER),,--add-current-user) \ $(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES)) \ $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)), \ "BUILD","binfmt debian-$* (debootstrapped)"), \ $(call quiet-command, \ - $(DOCKER_SCRIPT) check --quiet qemu:debian-$* $< || \ + $(DOCKER_SCRIPT) check --quiet qemu/debian-$* $< || \ { echo "You will need to build $(EXECUTABLE)"; exit 1;},\ "CHECK", "debian-$* exists")) +# Special case cross-compiling x86_64 on non-x86_64 systems. +ifeq ($(HOST_ARCH),x86_64) +DOCKER_PARTIAL_IMAGES += debian-amd64-cross +else +DOCKER_PARTIAL_IMAGES += debian-amd64 +endif + +# For non-x86 hosts not all cross-compilers have been packaged +ifneq ($(HOST_ARCH),x86_64) +DOCKER_PARTIAL_IMAGES += debian-mipsel-cross debian-mips64el-cross +DOCKER_PARTIAL_IMAGES += debian-ppc64el-cross +DOCKER_PARTIAL_IMAGES += debian-s390x-cross +DOCKER_PARTIAL_IMAGES += fedora endif -# Enforce dependencies for composite images -docker-image-debian: docker-image-debian9 -docker-image-debian8-mxe: docker-image-debian8 -docker-image-debian-amd64: docker-image-debian9 -docker-image-debian-armel-cross: docker-image-debian9 -docker-image-debian-armhf-cross: docker-image-debian9 -docker-image-debian-arm64-cross: docker-image-debian9 -docker-image-debian-mips-cross: docker-image-debian9 -docker-image-debian-mipsel-cross: docker-image-debian9 -docker-image-debian-mips64el-cross: docker-image-debian9 -docker-image-debian-ppc64el-cross: docker-image-debian9 -docker-image-debian-s390x-cross: docker-image-debian9 -docker-image-debian-win32-cross: docker-image-debian8-mxe -docker-image-debian-win64-cross: docker-image-debian8-mxe - -docker-image-debian-alpha-cross: docker-image-debian-sid -docker-image-debian-hppa-cross: docker-image-debian-sid -docker-image-debian-m68k-cross: docker-image-debian-sid -docker-image-debian-sh4-cross: docker-image-debian-sid -docker-image-debian-sparc64-cross: docker-image-debian-sid -docker-image-debian-mips64-cross: docker-image-debian-sid -docker-image-debian-riscv64-cross: docker-image-debian-sid -docker-image-debian-powerpc-cross: docker-image-debian-sid -docker-image-travis: NOUSER=1 - -# Specialist build images, sometimes very limited tools -docker-image-tricore-cross: docker-image-debian9 +# alpine has no adduser +docker-image-alpine: NOUSER=1 + +debian-toolchain-run = \ + $(if $(NOCACHE), \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) build -t qemu/$1 -f $< \ + $(if $V,,--quiet) --no-cache \ + --registry $(DOCKER_REGISTRY) --extra-files \ + $(DOCKER_FILES_DIR)/$1.d/build-toolchain.sh, \ + "BUILD", $1), \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) fetch $(if $V,,--quiet) \ + qemu/$1 $(DOCKER_REGISTRY), \ + "FETCH", $1) \ + $(call quiet-command, \ + $(DOCKER_SCRIPT) update $(if $V,,--quiet) \ + qemu/$1 \ + $(if $(NOUSER),,--add-current-user) \ + "PREPARE", $1)) +debian-toolchain = $(call debian-toolchain-run,$(patsubst docker-image-%,%,$1)) + +docker-image-debian-microblaze-cross: $(DOCKER_FILES_DIR)/debian-toolchain.docker \ + $(DOCKER_FILES_DIR)/debian-microblaze-cross.d/build-toolchain.sh + $(call debian-toolchain, $@) # These images may be good enough for building tests but not for test builds -DOCKER_PARTIAL_IMAGES += debian-alpha-cross debian-hppa-cross debian-m68k-cross debian-sh4-cross -DOCKER_PARTIAL_IMAGES += debian-sparc64-cross debian-mips64-cross debian-riscv64-cross -DOCKER_PARTIAL_IMAGES += debian-tricore-cross debian-powerpc-cross fedora-i386-cross +DOCKER_PARTIAL_IMAGES += debian-microblaze-cross +DOCKER_PARTIAL_IMAGES += debian-xtensa-cross +DOCKER_PARTIAL_IMAGES += fedora-cris-cross -# Rules for building linux-user powered images -# -# These are slower than using native cross compiler setups but can -# work around issues with poorly working multi-arch systems and broken -# packages. - -# Jessie is the last supported release for powerpc, but multi-arch is -# broken so we need a qemu-linux-user for this target -docker-binfmt-image-debian-powerpc-user: DEB_ARCH = powerpc -docker-binfmt-image-debian-powerpc-user: DEB_TYPE = jessie -docker-binfmt-image-debian-powerpc-user: EXECUTABLE = ${BUILD_DIR}/ppc-linux-user/qemu-ppc -docker-image-debian-powerpc-user-cross: docker-binfmt-image-debian-powerpc-user -DOCKER_USER_IMAGES += debian-powerpc-user +# images that are only used to build other images +DOCKER_VIRTUAL_IMAGES := debian-bootstrap debian-toolchain + +__IMAGES := $(sort $(filter-out $(DOCKER_VIRTUAL_IMAGES), $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.docker))))) +DOCKER_IMAGES := $(if $(IMAGES), $(filter $(IMAGES), $(__IMAGES)), $(__IMAGES)) + +__TESTS := $(notdir $(shell find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f)) +DOCKER_TESTS := $(if $(TESTS), $(filter $(TESTS), $(__TESTS)), $(__TESTS)) # Expand all the pre-requistes for each docker image and test combination -$(foreach i,$(filter-out $(DOCKER_PARTIAL_IMAGES),$(DOCKER_IMAGES) $(DOCKER_DEPRECATED_IMAGES)), \ - $(foreach t,$(DOCKER_TESTS) $(DOCKER_TOOLS), \ +$(foreach i,$(filter-out $(DOCKER_PARTIAL_IMAGES),$(DOCKER_IMAGES)), \ + $(foreach t,$(DOCKER_TESTS), \ $(eval .PHONY: docker-$t@$i) \ - $(eval docker-$t@$i: docker-image-$i docker-run-$t@$i) \ + $(eval docker-$t@$i: docker-image-$i; @$(MAKE) docker-run TEST=$t IMAGE=qemu/$i) \ ) \ $(foreach t,$(DOCKER_TESTS), \ $(eval docker-all-tests: docker-$t@$i) \ @@ -143,57 +141,56 @@ $(foreach i,$(filter-out $(DOCKER_PARTIAL_IMAGES),$(DOCKER_IMAGES) $(DOCKER_DEPR ) docker: - @echo 'Build QEMU and run tests inside Docker containers' + @echo 'Build QEMU and run tests inside $(RUNC) containers' @echo @echo 'Available targets:' @echo @echo ' docker: Print this help.' @echo ' docker-all-tests: Run all image/test combinations.' - @echo ' docker-TEST: Run TEST on all image combinations.' + @echo ' docker-TEST: Run "TEST" on all image combinations.' @echo ' docker-clean: Kill and remove residual docker testing containers.' @echo ' docker-TEST@IMAGE: Run "TEST" in container "IMAGE".' @echo ' Note: "TEST" is one of the listed test name,' @echo ' or a script name under $$QEMU_SRC/tests/docker/;' - @echo ' "IMAGE" is one of the listed container name."' + @echo ' "IMAGE" is one of the listed container name.' @echo ' docker-image: Build all images.' @echo ' docker-image-IMAGE: Build image "IMAGE".' - @echo ' docker-run: For manually running a "TEST" with "IMAGE"' + @echo ' docker-run: For manually running a "TEST" with "IMAGE".' @echo @echo 'Available container images:' @echo ' $(DOCKER_IMAGES)' -ifneq ($(DOCKER_USER_IMAGES),) - @echo - @echo 'Available linux-user images (docker-binfmt-image-debian-%):' - @echo ' $(DOCKER_USER_IMAGES)' -endif @echo @echo 'Available tests:' @echo ' $(DOCKER_TESTS)' @echo - @echo 'Available tools:' - @echo ' $(DOCKER_TOOLS)' - @echo @echo 'Special variables:' @echo ' TARGET_LIST=a,b,c Override target list in builds.' @echo ' EXTRA_CONFIGURE_OPTS="..."' @echo ' Extra configure options.' - @echo ' IMAGES="a b c ..": Filters which images to build or run.' - @echo ' TESTS="x y z .." Filters which tests to run (for docker-test).' + @echo ' TEST_COMMAND="..." Override the default `make check` target.' + @echo ' IMAGES="a b c ..": Restrict available images to subset.' + @echo ' TESTS="x y z .." Restrict available tests to subset.' @echo ' J=[0..9]* Overrides the -jN parameter for make commands' @echo ' (default is 1)' @echo ' DEBUG=1 Stop and drop to shell in the created container' @echo ' before running the command.' @echo ' NETWORK=1 Enable virtual network interface with default backend.' @echo ' NETWORK=$$BACKEND Enable virtual network interface with $$BACKEND.' - @echo ' NOUSER Define to disable adding current user to containers passwd.' + @echo ' NOUSER=1 Define to disable adding current user to containers passwd.' @echo ' NOCACHE=1 Ignore cache when build images.' @echo ' EXECUTABLE=<path> Include executable in image.' @echo ' EXTRA_FILES="<path> [... <path>]"' @echo ' Include extra files in image.' + @echo ' REGISTRY=url Cache builds from registry (default:$(DOCKER_REGISTRY))' + +docker-help: docker + +# Use a global constant ccache directory to speed up repetitive builds +DOCKER_CCACHE_DIR := $$HOME/.cache/qemu-docker-ccache # This rule if for directly running against an arbitrary docker target. # It is called by the expanded docker targets (e.g. make -# docker-test-foo@bar) which will do additional verification. +# docker-test-foo@bar) which will also ensure the image is up to date. # # For example: make docker-run TEST="test-quick" IMAGE="debian:arm64" EXECUTABLE=./aarch64-linux-user/qemu-aarch64 # @@ -205,17 +202,18 @@ docker-run: docker-qemu-src $(if $(EXECUTABLE), \ $(call quiet-command, \ $(DOCKER_SCRIPT) update \ - $(IMAGE) $(EXECUTABLE), \ + $(IMAGE) --executable $(EXECUTABLE), \ " COPYING $(EXECUTABLE) to $(IMAGE)")) $(call quiet-command, \ - $(DOCKER_SCRIPT) run \ - $(if $(NOUSER),,-u $(shell id -u)) \ + $(RUNC) run \ + --rm \ + $(if $(NOUSER),,-u $(UID)) \ --security-opt seccomp=unconfined \ - $(if $V,,--rm) \ $(if $(DEBUG),-ti,) \ $(if $(NETWORK),$(if $(subst $(NETWORK),,1),--net=$(NETWORK)),--net=none) \ -e TARGET_LIST=$(subst $(SPACE),$(COMMA),$(TARGET_LIST)) \ -e EXTRA_CONFIGURE_OPTS="$(EXTRA_CONFIGURE_OPTS)" \ + -e TEST_COMMAND="$(TEST_COMMAND)" \ -e V=$V -e J=$J -e DEBUG=$(DEBUG) \ -e SHOW_ENV=$(SHOW_ENV) \ $(if $(NOUSER),, \ @@ -229,13 +227,7 @@ docker-run: docker-qemu-src $(call quiet-command, rm -r $(DOCKER_SRC_COPY), \ " CLEANUP $(DOCKER_SRC_COPY)") -# Run targets: -# -# Of the form docker-TEST-FOO@IMAGE-BAR which will then be expanded into a call to "make docker-run" -docker-run-%: CMD = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\1/') -docker-run-%: IMAGE = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\2/') -docker-run-%: - @$(MAKE) docker-run TEST=$(CMD) IMAGE=qemu:$(IMAGE) +docker-image: ${DOCKER_IMAGES:%=docker-image-%} docker-clean: $(call quiet-command, $(DOCKER_SCRIPT) clean) |