From 88cf9ea68aec68538a88094869f4bad50eedd8d9 Mon Sep 17 00:00:00 2001 From: Josep Puigdemont Date: Thu, 20 Oct 2016 14:39:29 +0200 Subject: nginx-server: server side of the http performance tests The nginx-server.sh script implements the same protocol as the httperf-client.sh script, namely: client server | | |<----------- num_cores ---------| # server indicates num cores |---------- client_ready ------->| # server is configured for 1 core |<-- server_num_cores_1_ready ---| # client runs tests for 1 core |--- client_num_cores_1_done --->| # client is finished testing | | # server reconfigures for 2 cores |<-- server_num_cores_2_ready ---| # client runs tests for 2 cores |--- client_num_cores_2_done --->| # client is finished testing | | # repeated until num_cores in | | # increments of 2 (1, 2, 4, 6...) ... Currently it supports testing NGiNX on plain linux IP stack, and a special version of NGiNX compiled for OFP and ODP-DPDK, however the latter is not too portable at the moment and will require further work. Whether one or the other is tested is configured by the CONFIG_TYPE environment variable in the test definition: linux-ip or odp-dpdk, respectively. Also the maximum amount of cores for which to test can be limited by setting the MAX_CORES environment variable (0 meaning all cores). This relies completely on VLAND and lava-{send,wait} for synchronization. Change-Id: I775083f7183895039064141b51172a3280249848 Signed-off-by: Josep Puigdemont Signed-off-by: Milosz Wasilewski --- automated/linux/nginx-server/nginx-linux.yaml | 36 ++++ automated/linux/nginx-server/nginx-odp-dpdk.yaml | 42 ++++ automated/linux/nginx-server/nginx-server.sh | 254 +++++++++++++++++++++++ 3 files changed, 332 insertions(+) create mode 100644 automated/linux/nginx-server/nginx-linux.yaml create mode 100644 automated/linux/nginx-server/nginx-odp-dpdk.yaml create mode 100755 automated/linux/nginx-server/nginx-server.sh diff --git a/automated/linux/nginx-server/nginx-linux.yaml b/automated/linux/nginx-server/nginx-linux.yaml new file mode 100644 index 0000000..c438c1c --- /dev/null +++ b/automated/linux/nginx-server/nginx-linux.yaml @@ -0,0 +1,36 @@ +metadata: + name: httperf-nginx-linux-ip + format: Lava-Test-Shell Test Definition 1.0 + description: Server running plain NGiNX, no acceleration + environment: + - lava-test-shell + maintainer: + - josep.puigdemont@linaro.org + os: + - debian + - ubuntu + devices: + - x86 + +install: + deps: + - bash + - ethtool + - procps + - sysstat + +params: + CONFIG_TYPE: linux-ip + MAX_CORES: 0 + VLAND_NAME: vlan_one + +run: + steps: + - echo "deb http://ftp.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list + - apt-get update + - apt-get install -y -t jessie-backports nginx + - export CONFIG_TYPE + - export MAX_CORES + - export VLAND_NAME + - echo $CONFIG_TYPE, $MAX_CORES, $VLAND_NAME + - lava-test-case httperf-nginx-server --shell ./automated/linux/nginx-server/nginx-server.sh diff --git a/automated/linux/nginx-server/nginx-odp-dpdk.yaml b/automated/linux/nginx-server/nginx-odp-dpdk.yaml new file mode 100644 index 0000000..35d1e3d --- /dev/null +++ b/automated/linux/nginx-server/nginx-odp-dpdk.yaml @@ -0,0 +1,42 @@ +metadata: + name: httperf-nginx-ofp-odp-dpdk + format: Lava-Test-Shell Test Definition 1.0 + description: Server running OFP+ODP+DPDK NGiNX + environment: + - lava-test-shell + maintainer: + - josep.puigdemont@linaro.org + os: + - debian + - ubuntu + devices: + - x86 + +install: + deps: + - bash + - ethtool + - pciutils + - procps + - sysstat + - wget + +run: + steps: + - wget -O - http://deb.opendataplane.org/odp.key|sudo apt-key add - + - echo "deb http://deb.opendataplane.org jessie main" > /etc/apt/sources.list.d/odp.list + - echo "deb http://ftp.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list + - apt-get update + - apt-get -t jessie-backports install -y libssl1.0.0 libssl-dev + - wget http://people.linaro.org/~anders.roxell/ubuntu-v4.4/linux-headers-4.4.0-31_4.4.0-31.50_all.deb + - dpkg -i linux-headers-4.4.0-31_4.4.0-31.50_all.deb + - wget http://people.linaro.org/~anders.roxell/x86_64-ofp-2/overlay.tar.gz + - tar xzf overlay.tar.gz -C / + - depmod -a + - sysctl -w vm.nr_hugepages=1024 + - modprobe uio + - apt-get install -y dpdk-igb-uio-dkms dpdk pciutils + - export CONFIG_TYPE=${CONFIG_TYPE:-odp-dpdk} + - export MAX_CORES=${MAX_CORES:-0} + - export VLAND_NAME=${VLAND_NAME:-vlan_one} + - lava-test-case httperf-nginx-server --shell ./automated/linux/nginx-server/nginx-server.sh diff --git a/automated/linux/nginx-server/nginx-server.sh b/automated/linux/nginx-server/nginx-server.sh new file mode 100755 index 0000000..2002357 --- /dev/null +++ b/automated/linux/nginx-server/nginx-server.sh @@ -0,0 +1,254 @@ +#!/bin/bash +set -o errexit + +THIS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +TEST_DEFS_DIR=$(readlink -f "${THIS_DIR}/../../..") +GET_VLAND_IFACE=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_interface.sh +GET_VLAND_PCI_DEV=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_pci_dev.sh + +# vlnad name to use +VLAND_NAME=${VLAND_NAME:-vlan_one} +VLAND_IFACE=$($GET_VLAND_IFACE "$VLAND_NAME") +VLAND_PCI_DEV=$($GET_VLAND_PCI_DEV "$VLAND_NAME") + +# Do not run tests on more than MAX_CORES cores +# 0 means use all cores +MAX_CORES=${MAX_CORES:-0} + +# IP address of the server +SERVER_IP=${SERVER_IP:-192.168.1.4} + +# What kind of configuration to use: +# linux-ip: plain Linux IP stack +# odp-dpdk: NGiNX with OFP+ODP+DPDK +CONFIG_TYPE=${CONFIG_TYPE:-linux-ip} + +# DEB_URL +DEB_URL=${DEB_URL:-http://people.linaro.org/~josep.puigdemont/debs} + +function exit_error { + echo "-- SERVER ERROR" + journalctl -u nginx + lava-test-case server_up --status fail +} +trap exit_error ERR + +# Use this function to configure Linux IP stack +function config_linux_ip { + ip address add "${SERVER_IP}/24" dev "$VLAND_IFACE" + ip link set "$VLAND_IFACE" up + lava-test-case server_ifup --result pass + + sysctl -w net.ipv4.ip_local_port_range="1500 65500" +} + +# Use this function to configure a device for DPDK usage +# FIXME: this should use a repository +function config_dpdk_dev { + local driver=${1:-igb_uio} + local DEBS="libofp-odp-dpdk0_1.1+git3+4ec95a1-0linaro1linarojessie1_amd64.deb" + + if ! which dpdk-devbind &>/dev/null; then + echo "ERROR: dpdk not installed" + exit 1 + fi + + modprobe "$driver" + + dpdk-devbind -u "$VLAND_PCI_DEV" + dpdk-devbind -b "${driver}" "$VLAND_PCI_DEV" + dpdk-devbind -s + + apt-get install -y nginx + systemctl stop nginx + + # we want our version of libofp, with CONFIG_WEBSERVER set + apt-get remove -y libofp-odp-dpdk0 nginx-common + mkdir ofp + cd ofp + for deb in $DEBS; do + wget -q "$DEB_URL/ofp-config-webserver/$deb" + dpkg -i "$deb" + done + cd .. +} + +# Callback to call before starting nginx when using OFP-DPDK +# First parameter of callback is the number of cores +function odp_dpdk_pre_cb { + local cores=$1 + local DEBS="nginx-common_1.9.10-1~linaro1linarojessie1_all.deb \ + nginx-full_1.9.10-1~linaro1linarojessie1_amd64.deb \ + nginx_1.9.10-1~linaro1linarojessie1_all.deb" + + apt-get remove -y nginx-common + + # clean hugepages + rm -rf /dev/hugepages/* + + mkdir -p "iter_${cores}" + cd "iter_${cores}" + for deb in $DEBS; do + wget -q "$DEB_URL/nginx-ofp-odp-dpdk/${cores}_queue/$deb" + dpkg --force-confold -i "$deb" + done + cd .. +} + +function odp_dpdk_post_cb { + local cores=$1 + + systemctl status nginx + echo "-- AFFINITY $cores" + for pid in $(pgrep nginx); do + taskset -p "$pid" + done +} + +function linux_ip_pre_cb { + local cores=$1 + + ethtool -L "$VLAND_IFACE" combined "$cores" +} + +function linux_ip_post_cb { + local cores=$1 + + systemctl status nginx + echo "-- AFFINITY $cores" + for pid in $(pgrep nginx); do + taskset -p "$pid" + done + echo "-- INTERRUPTS $cores" + grep "$VLAND_IFACE" /proc/interrupts + echo "--- MPSTAT $cores" + mpstat -P ALL | cat +} + +function configure_ramdisk { + local ROOT=/www + mkdir "$ROOT" + mount -t tmpfs -o size=1M tmpfs "$ROOT" + echo "-- Ramdisk created: " + df -h "$ROOT" + echo "-- END" + lava-test-case server_www_ramdisk --result pass + + cat > "$ROOT/index.html" <<-EOF + + NGiNX test + + IT WORKS + + + EOF +} + +function write_config { + # Simple configuration file for NGiNX + cat > /etc/nginx/nginx.conf <<-EOF + user www-data; + worker_processes $1; + timer_resolution 1s; + worker_rlimit_nofile 4096; + error_log /dev/null crit; + pid /run/nginx.pid; + ${WRITE_CONFIG_CORE} + + events { + worker_connections 1024; + ${WRITE_CONFIG_EVENTS} + } + + http { + access_log off; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 0; + open_file_cache max=10; + server { + # TODO: investigate backlog value + listen ${SERVER_IP}:80 default_server ${REUSEPORT} so_keepalive=off; + location / { + root /www; + } + } + } + EOF + echo "-- CONFIG FOR $1:" + cat /etc/nginx/nginx.conf + echo "-- END --" +} + +function get_num_real_cores { + local cores_socket + local num_sockets + local num_cores + + cores_socket=$(lscpu | awk -F : '/^Core\(s\) per/ {print $2;}') + num_sockets=$(lscpu | awk -F : '/^Socket\(s\)/ {print $2;}') + num_cores=$((cores_socket * num_sockets)) + + if [ "${MAX_CORES}" -ne 0 ] && [ "${num_cores}" -gt "${MAX_CORES}" ]; then + num_cores=$MAX_CORES + fi + echo "$num_cores" +} + +configure_ramdisk + +case ${CONFIG_TYPE} in + linux-ip) + echo "-- CONFIGURING Linux Kernel IP stack" + config_linux_ip + PRE_TEST_CB="linux_ip_pre_cb" + POST_TEST_CB="linux_ip_post_cb" + WRITE_CONFIG_CORE="worker_cpu_affinity auto;" + WRITE_CONFIG_EVENTS="" + REUSEPORT="reuseport" + ;; + odp-dpdk) + echo "-- CONFIGURING OFP IP Stack" + config_dpdk_dev igb_uio + PRE_TEST_CB="odp_dpdk_pre_cb" + POST_TEST_CB="odp_dpdk_post_cb" + WRITE_CONFIG_CORE="" + WRITE_CONFIG_EVENTS="use select;" + REUSEPORT="" + ;; + *) + echo "Invalid CONFIG_TYPE: $CONFIG_TYPE" + exit 1 + ;; +esac + +NUM_CORES=$(get_num_real_cores) +echo ">> SEND num_cores cores=$NUM_CORES" +lava-send num_cores cores="$NUM_CORES" + +echo "<< WAIT client_ready" +lava-wait client_ready + +for num_cores in 1 $(seq 2 2 "$NUM_CORES"); do + echo "-- BEGIN $num_cores" + echo "-- Stopping NGiNX" + systemctl stop nginx + echo "-- Writing configuration file for $num_cores" + write_config "$num_cores" + echo "-- CALLING PRE-TEST CALLBACK $PRE_TEST_CB" + $PRE_TEST_CB "$num_cores" + echo "-- STARTING NGiNX for test $num_cores" + systemctl start nginx + echo ">> SEND server_num_cores_${num_cores}_ready" + lava-send "server_num_cores_${num_cores}_ready" + echo "<< WAIT client_num_cores_${num_cores}_done" + lava-wait "client_num_cores_${num_cores}_done" + echo "-- CALLING POST-TEST CALLBACK $POST_TEST_CB" + $POST_TEST_CB "$num_cores" + echo "-- END $num_cores" +done + +echo "<< WAIT client_done" +lava-wait client_done +echo "A10" -- cgit v1.2.3