summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJosep Puigdemont <josep.puigdemont@linaro.org>2016-10-20 14:39:29 +0200
committerMilosz Wasilewski <milosz.wasilewski@linaro.org>2017-01-04 14:54:43 +0000
commit88cf9ea68aec68538a88094869f4bad50eedd8d9 (patch)
treeb1a760e61603ba5ecbcd7f9edd11d4dace8928fe
parent7eeb6135acbc300cc163fe42d178e593891fadb5 (diff)
nginx-server: server side of the http performance tests
The nginx-server.sh script implements the same protocol as the httperf-client.sh script, namely: client server | | |<----------- num_cores ---------| # server indicates num cores |---------- client_ready ------->| # server is configured for 1 core |<-- server_num_cores_1_ready ---| # client runs tests for 1 core |--- client_num_cores_1_done --->| # client is finished testing | | # server reconfigures for 2 cores |<-- server_num_cores_2_ready ---| # client runs tests for 2 cores |--- client_num_cores_2_done --->| # client is finished testing | | # repeated until num_cores in | | # increments of 2 (1, 2, 4, 6...) ... Currently it supports testing NGiNX on plain linux IP stack, and a special version of NGiNX compiled for OFP and ODP-DPDK, however the latter is not too portable at the moment and will require further work. Whether one or the other is tested is configured by the CONFIG_TYPE environment variable in the test definition: linux-ip or odp-dpdk, respectively. Also the maximum amount of cores for which to test can be limited by setting the MAX_CORES environment variable (0 meaning all cores). This relies completely on VLAND and lava-{send,wait} for synchronization. Change-Id: I775083f7183895039064141b51172a3280249848 Signed-off-by: Josep Puigdemont <josep.puigdemont@linaro.org> Signed-off-by: Milosz Wasilewski <milosz.wasilewski@linaro.org>
-rw-r--r--automated/linux/nginx-server/nginx-linux.yaml36
-rw-r--r--automated/linux/nginx-server/nginx-odp-dpdk.yaml42
-rwxr-xr-xautomated/linux/nginx-server/nginx-server.sh254
3 files changed, 332 insertions, 0 deletions
diff --git a/automated/linux/nginx-server/nginx-linux.yaml b/automated/linux/nginx-server/nginx-linux.yaml
new file mode 100644
index 0000000..c438c1c
--- /dev/null
+++ b/automated/linux/nginx-server/nginx-linux.yaml
@@ -0,0 +1,36 @@
+metadata:
+ name: httperf-nginx-linux-ip
+ format: Lava-Test-Shell Test Definition 1.0
+ description: Server running plain NGiNX, no acceleration
+ environment:
+ - lava-test-shell
+ maintainer:
+ - josep.puigdemont@linaro.org
+ os:
+ - debian
+ - ubuntu
+ devices:
+ - x86
+
+install:
+ deps:
+ - bash
+ - ethtool
+ - procps
+ - sysstat
+
+params:
+ CONFIG_TYPE: linux-ip
+ MAX_CORES: 0
+ VLAND_NAME: vlan_one
+
+run:
+ steps:
+ - echo "deb http://ftp.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list
+ - apt-get update
+ - apt-get install -y -t jessie-backports nginx
+ - export CONFIG_TYPE
+ - export MAX_CORES
+ - export VLAND_NAME
+ - echo $CONFIG_TYPE, $MAX_CORES, $VLAND_NAME
+ - lava-test-case httperf-nginx-server --shell ./automated/linux/nginx-server/nginx-server.sh
diff --git a/automated/linux/nginx-server/nginx-odp-dpdk.yaml b/automated/linux/nginx-server/nginx-odp-dpdk.yaml
new file mode 100644
index 0000000..35d1e3d
--- /dev/null
+++ b/automated/linux/nginx-server/nginx-odp-dpdk.yaml
@@ -0,0 +1,42 @@
+metadata:
+ name: httperf-nginx-ofp-odp-dpdk
+ format: Lava-Test-Shell Test Definition 1.0
+ description: Server running OFP+ODP+DPDK NGiNX
+ environment:
+ - lava-test-shell
+ maintainer:
+ - josep.puigdemont@linaro.org
+ os:
+ - debian
+ - ubuntu
+ devices:
+ - x86
+
+install:
+ deps:
+ - bash
+ - ethtool
+ - pciutils
+ - procps
+ - sysstat
+ - wget
+
+run:
+ steps:
+ - wget -O - http://deb.opendataplane.org/odp.key|sudo apt-key add -
+ - echo "deb http://deb.opendataplane.org jessie main" > /etc/apt/sources.list.d/odp.list
+ - echo "deb http://ftp.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list
+ - apt-get update
+ - apt-get -t jessie-backports install -y libssl1.0.0 libssl-dev
+ - wget http://people.linaro.org/~anders.roxell/ubuntu-v4.4/linux-headers-4.4.0-31_4.4.0-31.50_all.deb
+ - dpkg -i linux-headers-4.4.0-31_4.4.0-31.50_all.deb
+ - wget http://people.linaro.org/~anders.roxell/x86_64-ofp-2/overlay.tar.gz
+ - tar xzf overlay.tar.gz -C /
+ - depmod -a
+ - sysctl -w vm.nr_hugepages=1024
+ - modprobe uio
+ - apt-get install -y dpdk-igb-uio-dkms dpdk pciutils
+ - export CONFIG_TYPE=${CONFIG_TYPE:-odp-dpdk}
+ - export MAX_CORES=${MAX_CORES:-0}
+ - export VLAND_NAME=${VLAND_NAME:-vlan_one}
+ - lava-test-case httperf-nginx-server --shell ./automated/linux/nginx-server/nginx-server.sh
diff --git a/automated/linux/nginx-server/nginx-server.sh b/automated/linux/nginx-server/nginx-server.sh
new file mode 100755
index 0000000..2002357
--- /dev/null
+++ b/automated/linux/nginx-server/nginx-server.sh
@@ -0,0 +1,254 @@
+#!/bin/bash
+set -o errexit
+
+THIS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+TEST_DEFS_DIR=$(readlink -f "${THIS_DIR}/../../..")
+GET_VLAND_IFACE=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_interface.sh
+GET_VLAND_PCI_DEV=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_pci_dev.sh
+
+# vlnad name to use
+VLAND_NAME=${VLAND_NAME:-vlan_one}
+VLAND_IFACE=$($GET_VLAND_IFACE "$VLAND_NAME")
+VLAND_PCI_DEV=$($GET_VLAND_PCI_DEV "$VLAND_NAME")
+
+# Do not run tests on more than MAX_CORES cores
+# 0 means use all cores
+MAX_CORES=${MAX_CORES:-0}
+
+# IP address of the server
+SERVER_IP=${SERVER_IP:-192.168.1.4}
+
+# What kind of configuration to use:
+# linux-ip: plain Linux IP stack
+# odp-dpdk: NGiNX with OFP+ODP+DPDK
+CONFIG_TYPE=${CONFIG_TYPE:-linux-ip}
+
+# DEB_URL
+DEB_URL=${DEB_URL:-http://people.linaro.org/~josep.puigdemont/debs}
+
+function exit_error {
+ echo "-- SERVER ERROR"
+ journalctl -u nginx
+ lava-test-case server_up --status fail
+}
+trap exit_error ERR
+
+# Use this function to configure Linux IP stack
+function config_linux_ip {
+ ip address add "${SERVER_IP}/24" dev "$VLAND_IFACE"
+ ip link set "$VLAND_IFACE" up
+ lava-test-case server_ifup --result pass
+
+ sysctl -w net.ipv4.ip_local_port_range="1500 65500"
+}
+
+# Use this function to configure a device for DPDK usage
+# FIXME: this should use a repository
+function config_dpdk_dev {
+ local driver=${1:-igb_uio}
+ local DEBS="libofp-odp-dpdk0_1.1+git3+4ec95a1-0linaro1linarojessie1_amd64.deb"
+
+ if ! which dpdk-devbind &>/dev/null; then
+ echo "ERROR: dpdk not installed"
+ exit 1
+ fi
+
+ modprobe "$driver"
+
+ dpdk-devbind -u "$VLAND_PCI_DEV"
+ dpdk-devbind -b "${driver}" "$VLAND_PCI_DEV"
+ dpdk-devbind -s
+
+ apt-get install -y nginx
+ systemctl stop nginx
+
+ # we want our version of libofp, with CONFIG_WEBSERVER set
+ apt-get remove -y libofp-odp-dpdk0 nginx-common
+ mkdir ofp
+ cd ofp
+ for deb in $DEBS; do
+ wget -q "$DEB_URL/ofp-config-webserver/$deb"
+ dpkg -i "$deb"
+ done
+ cd ..
+}
+
+# Callback to call before starting nginx when using OFP-DPDK
+# First parameter of callback is the number of cores
+function odp_dpdk_pre_cb {
+ local cores=$1
+ local DEBS="nginx-common_1.9.10-1~linaro1linarojessie1_all.deb \
+ nginx-full_1.9.10-1~linaro1linarojessie1_amd64.deb \
+ nginx_1.9.10-1~linaro1linarojessie1_all.deb"
+
+ apt-get remove -y nginx-common
+
+ # clean hugepages
+ rm -rf /dev/hugepages/*
+
+ mkdir -p "iter_${cores}"
+ cd "iter_${cores}"
+ for deb in $DEBS; do
+ wget -q "$DEB_URL/nginx-ofp-odp-dpdk/${cores}_queue/$deb"
+ dpkg --force-confold -i "$deb"
+ done
+ cd ..
+}
+
+function odp_dpdk_post_cb {
+ local cores=$1
+
+ systemctl status nginx
+ echo "-- AFFINITY $cores"
+ for pid in $(pgrep nginx); do
+ taskset -p "$pid"
+ done
+}
+
+function linux_ip_pre_cb {
+ local cores=$1
+
+ ethtool -L "$VLAND_IFACE" combined "$cores"
+}
+
+function linux_ip_post_cb {
+ local cores=$1
+
+ systemctl status nginx
+ echo "-- AFFINITY $cores"
+ for pid in $(pgrep nginx); do
+ taskset -p "$pid"
+ done
+ echo "-- INTERRUPTS $cores"
+ grep "$VLAND_IFACE" /proc/interrupts
+ echo "--- MPSTAT $cores"
+ mpstat -P ALL | cat
+}
+
+function configure_ramdisk {
+ local ROOT=/www
+ mkdir "$ROOT"
+ mount -t tmpfs -o size=1M tmpfs "$ROOT"
+ echo "-- Ramdisk created: "
+ df -h "$ROOT"
+ echo "-- END"
+ lava-test-case server_www_ramdisk --result pass
+
+ cat > "$ROOT/index.html" <<-EOF
+ <html>
+ <head><title>NGiNX test</title></head>
+ <body>
+ IT WORKS
+ </body>
+ </html>
+ EOF
+}
+
+function write_config {
+ # Simple configuration file for NGiNX
+ cat > /etc/nginx/nginx.conf <<-EOF
+ user www-data;
+ worker_processes $1;
+ timer_resolution 1s;
+ worker_rlimit_nofile 4096;
+ error_log /dev/null crit;
+ pid /run/nginx.pid;
+ ${WRITE_CONFIG_CORE}
+
+ events {
+ worker_connections 1024;
+ ${WRITE_CONFIG_EVENTS}
+ }
+
+ http {
+ access_log off;
+ sendfile on;
+ tcp_nopush on;
+ tcp_nodelay on;
+ keepalive_timeout 0;
+ open_file_cache max=10;
+ server {
+ # TODO: investigate backlog value
+ listen ${SERVER_IP}:80 default_server ${REUSEPORT} so_keepalive=off;
+ location / {
+ root /www;
+ }
+ }
+ }
+ EOF
+ echo "-- CONFIG FOR $1:"
+ cat /etc/nginx/nginx.conf
+ echo "-- END --"
+}
+
+function get_num_real_cores {
+ local cores_socket
+ local num_sockets
+ local num_cores
+
+ cores_socket=$(lscpu | awk -F : '/^Core\(s\) per/ {print $2;}')
+ num_sockets=$(lscpu | awk -F : '/^Socket\(s\)/ {print $2;}')
+ num_cores=$((cores_socket * num_sockets))
+
+ if [ "${MAX_CORES}" -ne 0 ] && [ "${num_cores}" -gt "${MAX_CORES}" ]; then
+ num_cores=$MAX_CORES
+ fi
+ echo "$num_cores"
+}
+
+configure_ramdisk
+
+case ${CONFIG_TYPE} in
+ linux-ip)
+ echo "-- CONFIGURING Linux Kernel IP stack"
+ config_linux_ip
+ PRE_TEST_CB="linux_ip_pre_cb"
+ POST_TEST_CB="linux_ip_post_cb"
+ WRITE_CONFIG_CORE="worker_cpu_affinity auto;"
+ WRITE_CONFIG_EVENTS=""
+ REUSEPORT="reuseport"
+ ;;
+ odp-dpdk)
+ echo "-- CONFIGURING OFP IP Stack"
+ config_dpdk_dev igb_uio
+ PRE_TEST_CB="odp_dpdk_pre_cb"
+ POST_TEST_CB="odp_dpdk_post_cb"
+ WRITE_CONFIG_CORE=""
+ WRITE_CONFIG_EVENTS="use select;"
+ REUSEPORT=""
+ ;;
+ *)
+ echo "Invalid CONFIG_TYPE: $CONFIG_TYPE"
+ exit 1
+ ;;
+esac
+
+NUM_CORES=$(get_num_real_cores)
+echo ">> SEND num_cores cores=$NUM_CORES"
+lava-send num_cores cores="$NUM_CORES"
+
+echo "<< WAIT client_ready"
+lava-wait client_ready
+
+for num_cores in 1 $(seq 2 2 "$NUM_CORES"); do
+ echo "-- BEGIN $num_cores"
+ echo "-- Stopping NGiNX"
+ systemctl stop nginx
+ echo "-- Writing configuration file for $num_cores"
+ write_config "$num_cores"
+ echo "-- CALLING PRE-TEST CALLBACK $PRE_TEST_CB"
+ $PRE_TEST_CB "$num_cores"
+ echo "-- STARTING NGiNX for test $num_cores"
+ systemctl start nginx
+ echo ">> SEND server_num_cores_${num_cores}_ready"
+ lava-send "server_num_cores_${num_cores}_ready"
+ echo "<< WAIT client_num_cores_${num_cores}_done"
+ lava-wait "client_num_cores_${num_cores}_done"
+ echo "-- CALLING POST-TEST CALLBACK $POST_TEST_CB"
+ $POST_TEST_CB "$num_cores"
+ echo "-- END $num_cores"
+done
+
+echo "<< WAIT client_done"
+lava-wait client_done
+echo "A10"