nginx-server: server side of the http performance tests
The nginx-server.sh script implements the same protocol as the
httperf-client.sh script, namely:
client server
| |
|<----------- num_cores ---------| # server indicates num cores
|---------- client_ready ------->| # server is configured for 1 core
|<-- server_num_cores_1_ready ---| # client runs tests for 1 core
|--- client_num_cores_1_done --->| # client is finished testing
| | # server reconfigures for 2 cores
|<-- server_num_cores_2_ready ---| # client runs tests for 2 cores
|--- client_num_cores_2_done --->| # client is finished testing
| | # repeated until num_cores in
| | # increments of 2 (1, 2, 4, 6...)
...
Currently it supports testing NGiNX on plain linux IP stack, and a
special version of NGiNX compiled for OFP and ODP-DPDK, however the
latter is not too portable at the moment and will require further work.
Whether one or the other is tested is configured by the CONFIG_TYPE
environment variable in the test definition: linux-ip or odp-dpdk,
respectively.
Also the maximum amount of cores for which to test can be limited by
setting the MAX_CORES environment variable (0 meaning all cores).
This relies completely on VLAND and lava-{send,wait} for
synchronization.
Change-Id: I775083f7183895039064141b51172a3280249848
Signed-off-by: Josep Puigdemont <josep.puigdemont@linaro.org>
Signed-off-by: Milosz Wasilewski <milosz.wasilewski@linaro.org>
diff --git a/automated/linux/nginx-server/nginx-server.sh b/automated/linux/nginx-server/nginx-server.sh
new file mode 100755
index 0000000..2002357
--- /dev/null
+++ b/automated/linux/nginx-server/nginx-server.sh
@@ -0,0 +1,254 @@
+#!/bin/bash
+set -o errexit
+
+THIS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+TEST_DEFS_DIR=$(readlink -f "${THIS_DIR}/../../..")
+GET_VLAND_IFACE=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_interface.sh
+GET_VLAND_PCI_DEV=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_pci_dev.sh
+
+# vlnad name to use
+VLAND_NAME=${VLAND_NAME:-vlan_one}
+VLAND_IFACE=$($GET_VLAND_IFACE "$VLAND_NAME")
+VLAND_PCI_DEV=$($GET_VLAND_PCI_DEV "$VLAND_NAME")
+
+# Do not run tests on more than MAX_CORES cores
+# 0 means use all cores
+MAX_CORES=${MAX_CORES:-0}
+
+# IP address of the server
+SERVER_IP=${SERVER_IP:-192.168.1.4}
+
+# What kind of configuration to use:
+# linux-ip: plain Linux IP stack
+# odp-dpdk: NGiNX with OFP+ODP+DPDK
+CONFIG_TYPE=${CONFIG_TYPE:-linux-ip}
+
+# DEB_URL
+DEB_URL=${DEB_URL:-http://people.linaro.org/~josep.puigdemont/debs}
+
+function exit_error {
+ echo "-- SERVER ERROR"
+ journalctl -u nginx
+ lava-test-case server_up --status fail
+}
+trap exit_error ERR
+
+# Use this function to configure Linux IP stack
+function config_linux_ip {
+ ip address add "${SERVER_IP}/24" dev "$VLAND_IFACE"
+ ip link set "$VLAND_IFACE" up
+ lava-test-case server_ifup --result pass
+
+ sysctl -w net.ipv4.ip_local_port_range="1500 65500"
+}
+
+# Use this function to configure a device for DPDK usage
+# FIXME: this should use a repository
+function config_dpdk_dev {
+ local driver=${1:-igb_uio}
+ local DEBS="libofp-odp-dpdk0_1.1+git3+4ec95a1-0linaro1linarojessie1_amd64.deb"
+
+ if ! which dpdk-devbind &>/dev/null; then
+ echo "ERROR: dpdk not installed"
+ exit 1
+ fi
+
+ modprobe "$driver"
+
+ dpdk-devbind -u "$VLAND_PCI_DEV"
+ dpdk-devbind -b "${driver}" "$VLAND_PCI_DEV"
+ dpdk-devbind -s
+
+ apt-get install -y nginx
+ systemctl stop nginx
+
+ # we want our version of libofp, with CONFIG_WEBSERVER set
+ apt-get remove -y libofp-odp-dpdk0 nginx-common
+ mkdir ofp
+ cd ofp
+ for deb in $DEBS; do
+ wget -q "$DEB_URL/ofp-config-webserver/$deb"
+ dpkg -i "$deb"
+ done
+ cd ..
+}
+
+# Callback to call before starting nginx when using OFP-DPDK
+# First parameter of callback is the number of cores
+function odp_dpdk_pre_cb {
+ local cores=$1
+ local DEBS="nginx-common_1.9.10-1~linaro1linarojessie1_all.deb \
+ nginx-full_1.9.10-1~linaro1linarojessie1_amd64.deb \
+ nginx_1.9.10-1~linaro1linarojessie1_all.deb"
+
+ apt-get remove -y nginx-common
+
+ # clean hugepages
+ rm -rf /dev/hugepages/*
+
+ mkdir -p "iter_${cores}"
+ cd "iter_${cores}"
+ for deb in $DEBS; do
+ wget -q "$DEB_URL/nginx-ofp-odp-dpdk/${cores}_queue/$deb"
+ dpkg --force-confold -i "$deb"
+ done
+ cd ..
+}
+
+function odp_dpdk_post_cb {
+ local cores=$1
+
+ systemctl status nginx
+ echo "-- AFFINITY $cores"
+ for pid in $(pgrep nginx); do
+ taskset -p "$pid"
+ done
+}
+
+function linux_ip_pre_cb {
+ local cores=$1
+
+ ethtool -L "$VLAND_IFACE" combined "$cores"
+}
+
+function linux_ip_post_cb {
+ local cores=$1
+
+ systemctl status nginx
+ echo "-- AFFINITY $cores"
+ for pid in $(pgrep nginx); do
+ taskset -p "$pid"
+ done
+ echo "-- INTERRUPTS $cores"
+ grep "$VLAND_IFACE" /proc/interrupts
+ echo "--- MPSTAT $cores"
+ mpstat -P ALL | cat
+}
+
+function configure_ramdisk {
+ local ROOT=/www
+ mkdir "$ROOT"
+ mount -t tmpfs -o size=1M tmpfs "$ROOT"
+ echo "-- Ramdisk created: "
+ df -h "$ROOT"
+ echo "-- END"
+ lava-test-case server_www_ramdisk --result pass
+
+ cat > "$ROOT/index.html" <<-EOF
+ <html>
+ <head><title>NGiNX test</title></head>
+ <body>
+ IT WORKS
+ </body>
+ </html>
+ EOF
+}
+
+function write_config {
+ # Simple configuration file for NGiNX
+ cat > /etc/nginx/nginx.conf <<-EOF
+ user www-data;
+ worker_processes $1;
+ timer_resolution 1s;
+ worker_rlimit_nofile 4096;
+ error_log /dev/null crit;
+ pid /run/nginx.pid;
+ ${WRITE_CONFIG_CORE}
+
+ events {
+ worker_connections 1024;
+ ${WRITE_CONFIG_EVENTS}
+ }
+
+ http {
+ access_log off;
+ sendfile on;
+ tcp_nopush on;
+ tcp_nodelay on;
+ keepalive_timeout 0;
+ open_file_cache max=10;
+ server {
+ # TODO: investigate backlog value
+ listen ${SERVER_IP}:80 default_server ${REUSEPORT} so_keepalive=off;
+ location / {
+ root /www;
+ }
+ }
+ }
+ EOF
+ echo "-- CONFIG FOR $1:"
+ cat /etc/nginx/nginx.conf
+ echo "-- END --"
+}
+
+function get_num_real_cores {
+ local cores_socket
+ local num_sockets
+ local num_cores
+
+ cores_socket=$(lscpu | awk -F : '/^Core\(s\) per/ {print $2;}')
+ num_sockets=$(lscpu | awk -F : '/^Socket\(s\)/ {print $2;}')
+ num_cores=$((cores_socket * num_sockets))
+
+ if [ "${MAX_CORES}" -ne 0 ] && [ "${num_cores}" -gt "${MAX_CORES}" ]; then
+ num_cores=$MAX_CORES
+ fi
+ echo "$num_cores"
+}
+
+configure_ramdisk
+
+case ${CONFIG_TYPE} in
+ linux-ip)
+ echo "-- CONFIGURING Linux Kernel IP stack"
+ config_linux_ip
+ PRE_TEST_CB="linux_ip_pre_cb"
+ POST_TEST_CB="linux_ip_post_cb"
+ WRITE_CONFIG_CORE="worker_cpu_affinity auto;"
+ WRITE_CONFIG_EVENTS=""
+ REUSEPORT="reuseport"
+ ;;
+ odp-dpdk)
+ echo "-- CONFIGURING OFP IP Stack"
+ config_dpdk_dev igb_uio
+ PRE_TEST_CB="odp_dpdk_pre_cb"
+ POST_TEST_CB="odp_dpdk_post_cb"
+ WRITE_CONFIG_CORE=""
+ WRITE_CONFIG_EVENTS="use select;"
+ REUSEPORT=""
+ ;;
+ *)
+ echo "Invalid CONFIG_TYPE: $CONFIG_TYPE"
+ exit 1
+ ;;
+esac
+
+NUM_CORES=$(get_num_real_cores)
+echo ">> SEND num_cores cores=$NUM_CORES"
+lava-send num_cores cores="$NUM_CORES"
+
+echo "<< WAIT client_ready"
+lava-wait client_ready
+
+for num_cores in 1 $(seq 2 2 "$NUM_CORES"); do
+ echo "-- BEGIN $num_cores"
+ echo "-- Stopping NGiNX"
+ systemctl stop nginx
+ echo "-- Writing configuration file for $num_cores"
+ write_config "$num_cores"
+ echo "-- CALLING PRE-TEST CALLBACK $PRE_TEST_CB"
+ $PRE_TEST_CB "$num_cores"
+ echo "-- STARTING NGiNX for test $num_cores"
+ systemctl start nginx
+ echo ">> SEND server_num_cores_${num_cores}_ready"
+ lava-send "server_num_cores_${num_cores}_ready"
+ echo "<< WAIT client_num_cores_${num_cores}_done"
+ lava-wait "client_num_cores_${num_cores}_done"
+ echo "-- CALLING POST-TEST CALLBACK $POST_TEST_CB"
+ $POST_TEST_CB "$num_cores"
+ echo "-- END $num_cores"
+done
+
+echo "<< WAIT client_done"
+lava-wait client_done
+echo "A10"