aboutsummaryrefslogtreecommitdiff
path: root/tests/Makefile.include
blob: f93e611220215d7bc2a10dcb5284c4e4637ff434 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
# -*- Mode: makefile -*-

.PHONY: check-help
check-help:
	@echo "Regression testing targets:"
	@echo
	@echo " $(MAKE) check                Run block, qapi-schema, unit, softfloat, qtest and decodetree tests"
	@echo
	@echo " $(MAKE) check-qtest-TARGET   Run qtest tests for given target"
	@echo " $(MAKE) check-qtest          Run qtest tests"
	@echo " $(MAKE) check-unit           Run qobject tests"
	@echo " $(MAKE) check-speed          Run qobject speed tests"
	@echo " $(MAKE) check-qapi-schema    Run QAPI schema tests"
	@echo " $(MAKE) check-block          Run block tests"
ifeq ($(CONFIG_TCG),y)
	@echo " $(MAKE) check-tcg            Run TCG tests"
	@echo " $(MAKE) check-softfloat      Run FPU emulation tests"
endif
	@echo " $(MAKE) check-acceptance     Run all acceptance (functional) tests"
	@echo
	@echo " $(MAKE) check-report.tap     Generates an aggregated TAP test report"
	@echo " $(MAKE) check-venv           Creates a Python venv for tests"
	@echo " $(MAKE) check-clean          Clean the tests and related data"
	@echo
	@echo "The following are useful for CI builds"
	@echo " $(MAKE) check-build          Build most test binaris"
	@echo " $(MAKE) get-vm-images        Downloads all images used by acceptance tests, according to configured targets (~350 MB each, 1.5 GB max)"
	@echo
	@echo
	@echo "The variable SPEED can be set to control the gtester speed setting."
	@echo "Default options are -k and (for $(MAKE) V=1) --verbose; they can be"
	@echo "changed with variable GTESTER_OPTIONS."

ifneq ($(wildcard config-host.mak),)
export SRC_PATH

# Get the list of all supported sysemu targets
SYSEMU_TARGET_LIST := $(subst -softmmu.mak,,$(notdir \
   $(wildcard $(SRC_PATH)/default-configs/*-softmmu.mak)))

SPEED = quick

# Per guest TCG tests

BUILD_TCG_TARGET_RULES=$(patsubst %,build-tcg-tests-%, $(TARGET_DIRS))
CLEAN_TCG_TARGET_RULES=$(patsubst %,clean-tcg-tests-%, $(TARGET_DIRS))
RUN_TCG_TARGET_RULES=$(patsubst %,run-tcg-tests-%, $(TARGET_DIRS))

# Probe for the Docker Builds needed for each build
$(foreach PROBE_TARGET,$(TARGET_DIRS), 				\
	$(eval -include $(SRC_PATH)/tests/tcg/Makefile.prereqs))

build-tcg-tests-%: $(if $(CONFIG_PLUGIN),plugins)
	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) \
		-f $(SRC_PATH)/tests/tcg/Makefile.qemu \
		SRC_PATH=$(SRC_PATH) \
	       	V="$(V)" TARGET="$*" guest-tests, \
		"BUILD", "TCG tests for $*")

run-tcg-tests-%: build-tcg-tests-% all
	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) \
		-f $(SRC_PATH)/tests/tcg/Makefile.qemu \
		SRC_PATH=$(SRC_PATH) SPEED="$(SPEED)" \
		V="$(V)" TARGET="$*" run-guest-tests, \
		"RUN", "TCG tests for $*")

clean-tcg-tests-%:
	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) \
		-f $(SRC_PATH)/tests/tcg/Makefile.qemu \
		SRC_PATH=$(SRC_PATH) TARGET="$*" clean-guest-tests, \
		"CLEAN", "TCG tests for $*")

.PHONY: build-tcg
build-tcg: $(BUILD_TCG_TARGET_RULES)

.PHONY: check-tcg
check-tcg: $(RUN_TCG_TARGET_RULES)

.PHONY: clean-tcg
clean-tcg: $(CLEAN_TCG_TARGET_RULES)

# Python venv for running tests

.PHONY: check-venv check-acceptance

TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv
TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt
TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results
# Controls the output generated by Avocado when running tests.
# Any number of command separated loggers are accepted.  For more
# information please refer to "avocado --help".
AVOCADO_SHOW=app
AVOCADO_TAGS=$(patsubst %-softmmu,-t arch:%, $(filter %-softmmu,$(TARGET_DIRS)))

$(TESTS_VENV_DIR): $(TESTS_VENV_REQ)
	$(call quiet-command, \
            $(PYTHON) -m venv --system-site-packages $@, \
            VENV, $@)
	$(call quiet-command, \
            $(TESTS_VENV_DIR)/bin/python -m pip -q install -r $(TESTS_VENV_REQ), \
            PIP, $(TESTS_VENV_REQ))
	$(call quiet-command, touch $@)

$(TESTS_RESULTS_DIR):
	$(call quiet-command, mkdir -p $@, \
            MKDIR, $@)

check-venv: $(TESTS_VENV_DIR)

FEDORA_31_ARCHES_CANDIDATES=$(patsubst ppc64,ppc64le,$(TARGETS))
FEDORA_31_ARCHES := x86_64 aarch64 ppc64le s390x
FEDORA_31_DOWNLOAD=$(filter $(FEDORA_31_ARCHES),$(FEDORA_31_ARCHES_CANDIDATES))

# download one specific Fedora 31 image
get-vm-image-fedora-31-%: check-venv
	$(call quiet-command, \
             $(TESTS_VENV_DIR)/bin/python -m avocado vmimage get \
             --distro=fedora --distro-version=31 --arch=$*, \
	"AVOCADO", "Downloading acceptance tests VM image for $*")

# download all vm images, according to defined targets
get-vm-images: check-venv $(patsubst %,get-vm-image-fedora-31-%, $(FEDORA_31_DOWNLOAD))

check-acceptance: check-venv $(TESTS_RESULTS_DIR) get-vm-images
	$(call quiet-command, \
            $(TESTS_VENV_DIR)/bin/python -m avocado \
            --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \
            --filter-by-tags-include-empty --filter-by-tags-include-empty-key \
            $(AVOCADO_TAGS) \
            $(if $(GITLAB_CI),,--failfast=on) tests/acceptance, \
            "AVOCADO", "tests/acceptance")

# Consolidated targets

.PHONY: check-block check check-clean get-vm-images
check:

ifeq ($(CONFIG_TOOLS)$(CONFIG_POSIX),yy)
QEMU_IOTESTS_HELPERS-$(CONFIG_LINUX) = tests/qemu-iotests/socket_scm_helper$(EXESUF)
check: check-block
check-block: $(SRC_PATH)/tests/check-block.sh qemu-img$(EXESUF) \
		qemu-io$(EXESUF) qemu-nbd$(EXESUF) $(QEMU_IOTESTS_HELPERS-y) \
		$(patsubst %-softmmu,qemu-system-%,$(filter %-softmmu,$(TARGET_DIRS)))
	@$<
endif

check-build: $(QEMU_IOTESTS_HELPERS-y)

check-clean:
	rm -rf $(TESTS_VENV_DIR) $(TESTS_RESULTS_DIR)

clean: check-clean

# For backwards compatibility

check-speed: bench-speed

endif