initial nginx draft changes for ofp-odpHEADmaster

Signed-off-by: Kapil Hali <kapil.hali@linaro.org>

9 files changed, 807 insertions, 15 deletions

diff --git a/auto/make b/auto/make
index dca011c1..8f54e4ce 100644
--- a/auto/make
+++ b/auto/make
@@ -17,6 +17,16 @@ mkdir -p $NGX_OBJS/src/core $NGX_OBJS/src/event $NGX_OBJS/src/event/modules \
 ngx_objs_dir=$NGX_OBJS$ngx_regex_dirsep
 ngx_use_pch=`echo $NGX_USE_PCH | sed -e "s/\//$ngx_regex_dirsep/g"`
 
+if [ $OFP_PATH ] && [ $OFP_ODP ]; then
+    ODP_PATH=$OFP_ODP
+else
+    echo "Please define OFP_PATH OFP_ODP environment variable"
+    exit 1
+fi
+
+
+CORE_LIBS="$CORE_LIBS $OFP_PATH/lib/libofp.a $ODP_PATH/lib/libodp.a $ODP_PATH/lib/libodphelper.a"
+CORE_LIBS="$CORE_LIBS $NGX_LIBDL -lrt"
 
 cat << END                                                     > $NGX_MAKEFILE
 
@@ -25,6 +35,11 @@ CFLAGS = $CFLAGS
 CPP =	$CPP
 LINK =	$LINK
 
+
+CFLAGS += -I$OFP_PATH/include
+CFLAGS += -I$ODP_PATH/include
+
+
 END
 
 
diff --git a/auto/sources b/auto/sources
index 2e44ce18..03042c92 100644
--- a/auto/sources
+++ b/auto/sources
@@ -108,7 +108,7 @@ EVENT_SRCS="src/event/ngx_event.c \
 
 
 SELECT_MODULE=ngx_select_module
-SELECT_SRCS=src/event/modules/ngx_select_module.c
+SELECT_SRCS="src/event/modules/ngx_select_module.c src/event/modules/ngx_ofp_module.c"
 WIN32_SELECT_SRCS=src/event/modules/ngx_win32_select_module.c
 
 POLL_MODULE=ngx_poll_module
diff --git a/auto/unix b/auto/unix
index 7bfca8f3..5d6c2b07 100755
--- a/auto/unix
+++ b/auto/unix
@@ -384,14 +384,14 @@ ngx_feature_test="socklen_t optlen = sizeof(struct tcp_info);
 . auto/feature
 
 
-ngx_feature="accept4()"
-ngx_feature_name="NGX_HAVE_ACCEPT4"
-ngx_feature_run=no
-ngx_feature_incs="#include <sys/socket.h>"
-ngx_feature_path=
-ngx_feature_libs=
-ngx_feature_test="accept4(0, NULL, NULL, SOCK_NONBLOCK)"
-. auto/feature
+#ngx_feature="accept4()"
+#ngx_feature_name="NGX_HAVE_ACCEPT4"
+#ngx_feature_run=no
+#ngx_feature_incs="#include <sys/socket.h>"
+#ngx_feature_path=
+#ngx_feature_libs=
+#ngx_feature_test="accept4(0, NULL, NULL, SOCK_NONBLOCK)"
+#. auto/feature
 
 if [ $NGX_FILE_AIO = YES ]; then
 
diff --git a/conf/ofp.conf b/conf/ofp.conf
new file mode 100644
index 00000000..424e3831
--- /dev/null
+++ b/conf/ofp.conf
@@ -0,0 +1,3 @@
+debug 0
+loglevel set debug
+ifconfig fp0 192.168.1.4/24
diff --git a/configure b/configure
new file mode 100755
index 00000000..617d992b
--- /dev/null
+++ b/configure
@@ -0,0 +1,116 @@
+#!/bin/sh
+
+# Copyright (C) Igor Sysoev
+# Copyright (C) Nginx, Inc.
+
+
+LC_ALL=C
+export LC_ALL
+
+. auto/options
+. auto/init
+. auto/sources
+
+test -d $NGX_OBJS || mkdir $NGX_OBJS
+
+echo > $NGX_AUTO_HEADERS_H
+echo > $NGX_AUTOCONF_ERR
+
+echo "#define NGX_CONFIGURE \"$NGX_CONFIGURE\"" > $NGX_AUTO_CONFIG_H
+
+
+if [ $NGX_DEBUG = YES ]; then
+    have=NGX_DEBUG . auto/have
+fi
+
+
+if test -z "$NGX_PLATFORM"; then
+    echo "checking for OS"
+
+    NGX_SYSTEM=`uname -s 2>/dev/null`
+    NGX_RELEASE=`uname -r 2>/dev/null`
+    NGX_MACHINE=`uname -m 2>/dev/null`
+
+    echo " + $NGX_SYSTEM $NGX_RELEASE $NGX_MACHINE"
+
+    NGX_PLATFORM="$NGX_SYSTEM:$NGX_RELEASE:$NGX_MACHINE";
+
+    case "$NGX_SYSTEM" in
+        MINGW32_*)
+            NGX_PLATFORM=win32
+        ;;
+    esac
+
+else
+    echo "building for $NGX_PLATFORM"
+    NGX_SYSTEM=$NGX_PLATFORM
+fi
+
+. auto/cc/conf
+
+if [ "$NGX_PLATFORM" != win32 ]; then
+    . auto/headers
+fi
+
+. auto/os/conf
+
+if [ "$NGX_PLATFORM" != win32 ]; then
+    . auto/unix
+fi
+
+. auto/threads
+. auto/modules
+. auto/lib/conf
+
+case ".$NGX_PREFIX" in
+    .)
+        NGX_PREFIX=${NGX_PREFIX:-/usr/local/nginx}
+        have=NGX_PREFIX value="\"$NGX_PREFIX/\"" . auto/define
+    ;;
+
+    .!)
+        NGX_PREFIX=
+    ;;
+
+    *)
+        have=NGX_PREFIX value="\"$NGX_PREFIX/\"" . auto/define
+    ;;
+esac
+
+if [ ".$NGX_CONF_PREFIX" != "." ]; then
+    have=NGX_CONF_PREFIX value="\"$NGX_CONF_PREFIX/\"" . auto/define
+fi
+
+have=NGX_SBIN_PATH value="\"$NGX_SBIN_PATH\"" . auto/define
+have=NGX_CONF_PATH value="\"$NGX_CONF_PATH\"" . auto/define
+have=NGX_PID_PATH value="\"$NGX_PID_PATH\"" . auto/define
+have=NGX_LOCK_PATH value="\"$NGX_LOCK_PATH\"" . auto/define
+have=NGX_ERROR_LOG_PATH value="\"$NGX_ERROR_LOG_PATH\"" . auto/define
+
+have=NGX_HTTP_LOG_PATH value="\"$NGX_HTTP_LOG_PATH\"" . auto/define
+have=NGX_HTTP_CLIENT_TEMP_PATH value="\"$NGX_HTTP_CLIENT_TEMP_PATH\""
+. auto/define
+have=NGX_HTTP_PROXY_TEMP_PATH value="\"$NGX_HTTP_PROXY_TEMP_PATH\""
+. auto/define
+have=NGX_HTTP_FASTCGI_TEMP_PATH value="\"$NGX_HTTP_FASTCGI_TEMP_PATH\""
+. auto/define
+have=NGX_HTTP_UWSGI_TEMP_PATH value="\"$NGX_HTTP_UWSGI_TEMP_PATH\""
+. auto/define
+have=NGX_HTTP_SCGI_TEMP_PATH value="\"$NGX_HTTP_SCGI_TEMP_PATH\""
+. auto/define
+
+. auto/make
+. auto/lib/make
+. auto/install
+
+# STUB
+. auto/stubs
+
+have=NGX_USER value="\"$NGX_USER\"" . auto/define
+have=NGX_GROUP value="\"$NGX_GROUP\"" . auto/define
+
+if [ ".$NGX_BUILD" != "." ]; then
+    have=NGX_BUILD value="\"$NGX_BUILD\"" . auto/define
+fi
+
+. auto/summary
diff --git a/src/core/nginx.c b/src/core/nginx.c
index 33355870..7cf08122 100644
--- a/src/core/nginx.c
+++ b/src/core/nginx.c
@@ -24,6 +24,7 @@ static char *ngx_set_cpu_affinity(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
 static char *ngx_set_worker_processes(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+void ngx_ofp_init();
 
 
 static ngx_conf_enum_t  ngx_debug_points[] = {
@@ -204,6 +205,12 @@ main(int argc, char *const *argv)
 
     /* TODO */ ngx_max_sockets = -1;
 
+    /* ofp */
+    printf("signal is :%s\n", ngx_signal);
+    if (!ngx_signal) {
+	ngx_ofp_init();
+    }
+
     ngx_time_init();
 
 #if (NGX_PCRE)
diff --git a/src/event/modules/ngx_ofp_module.c b/src/event/modules/ngx_ofp_module.c
new file mode 100644
index 00000000..3dd9a14d
--- /dev/null
+++ b/src/event/modules/ngx_ofp_module.c
@@ -0,0 +1,619 @@
+#include <stdio.h>
+#include <math.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <netinet/in.h>
+#include <termios.h>
+#include <assert.h>
+#include <sys/select.h>
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+
+#ifndef __linux__
+#ifdef __FreeBSD__
+#include <sys/socket.h>
+#else
+#include <net/socket.h>
+#endif
+#endif
+
+#include <sys/time.h>
+#include "ofp.h"
+#include "odp.h"
+#include "ofp_errno.h"
+
+#define _GNU_SOURCE
+#define __USE_GNU
+#ifdef __USE_GNU
+/* Access macros for `cpu_set'.  */
+#define CPU_SETSIZE __CPU_SETSIZE
+#define CPU_SET(cpu, cpusetp)   __CPU_SET (cpu, cpusetp)
+#define CPU_CLR(cpu, cpusetp)   __CPU_CLR (cpu, cpusetp)
+#define CPU_ISSET(cpu, cpusetp) __CPU_ISSET (cpu, cpusetp)
+#define CPU_ZERO(cpusetp)       __CPU_ZERO (cpusetp)
+#endif
+#include <unistd.h>
+#include <sched.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <sys/syscall.h>
+#include <dlfcn.h>
+
+#define ODP_FD_BITS 30
+
+#include <getopt.h>
+
+#define MAX_WORKERS 32
+
+/**
+ * Parsed command line application arguments
+ */
+typedef struct {
+	int core_count;
+	int if_count;		/**< Number of interfaces to be used */
+	char **if_names;	/**< Array of pointers to interface names */
+	char *conf_file;
+} appl_args_t;
+
+/* helper funcs */
+static void parse_args(int argc, char *argv[], appl_args_t *appl_args);
+static void print_info(char *progname, appl_args_t *appl_args);
+static void usage(char *progname);
+
+ofp_init_global_t app_init_params; /**< global OFP init parms */
+
+/** Get rid of path in filename - only for unix-type paths using '/' */
+#define NO_PATH(file_name) (strrchr((file_name), '/') ? \
+				strrchr((file_name), '/') + 1 : (file_name))
+
+
+/** local hook
+ *
+ * @param pkt odp_packet_t
+ * @param protocol int
+ * @return int
+ *
+ */
+static enum ofp_return_code fastpath_local_hook(odp_packet_t pkt, void *arg)
+{
+	int protocol = *(int *)arg;
+	(void) pkt;
+	(void) protocol;
+	return OFP_PKT_CONTINUE;
+}
+
+/** main() Application entry point
+ *
+ * @param argc int
+ * @param argv[] char*
+ * @return int
+ *
+ */
+#include <sys/time.h>
+#include <sys/resource.h>
+
+int my_webserver(int if_count, char **if_name)
+{
+	odph_linux_pthread_t thread_tbl[MAX_WORKERS];
+	appl_args_t params;
+	int core_count, num_workers;
+	odp_cpumask_t cpumask;
+	char cpumaskstr[64];
+
+	struct rlimit rlp;
+	getrlimit(RLIMIT_CORE, &rlp);
+	rlp.rlim_cur = 200000000;
+
+	/* Parse and store the application arguments */
+	parse_args(if_count, if_name, &params);
+
+	/* Print both system and application information */
+	print_info("webserver", &params);
+
+	if (odp_init_global(NULL, NULL)) {
+		OFP_ERR("Error: ODP global init failed.\n");
+		exit(EXIT_FAILURE);
+	}
+	odp_init_local(ODP_THREAD_CONTROL);
+
+	core_count = odp_cpu_count();
+	num_workers = core_count;
+
+	if (params.core_count)
+		num_workers = params.core_count;
+	if (num_workers > MAX_WORKERS)
+		num_workers = MAX_WORKERS;
+
+	/*
+	 * By default core #0 runs Linux kernel background tasks.
+	 * Start mapping thread from core #1
+	 */
+	memset(&app_init_params, 0, sizeof(app_init_params));
+
+	app_init_params.linux_core_id = 0;
+
+	if (core_count > 1)
+		num_workers--;
+
+	num_workers = odp_cpumask_def_worker(&cpumask, num_workers);
+	odp_cpumask_to_str(&cpumask, cpumaskstr, sizeof(cpumaskstr));
+
+	printf("Num worker threads: %i\n", num_workers);
+	printf("first CPU:          %i\n", odp_cpumask_first(&cpumask));
+	printf("cpu mask:           %s\n", cpumaskstr);
+
+	app_init_params.if_count = params.if_count;
+	app_init_params.if_names = params.if_names;
+	app_init_params.pkt_hook[OFP_HOOK_LOCAL] = fastpath_local_hook;
+	if(ofp_init_global(&app_init_params) < 0) {
+		printf("%s: ofp_init_global failed\n", __func__);
+		exit(0);
+	}
+
+	memset(thread_tbl, 0, sizeof(thread_tbl));
+	/* Start dataplane dispatcher worker threads */
+
+	odph_linux_pthread_create(thread_tbl,
+				  &cpumask,
+				  default_event_dispatcher,
+				  ofp_eth_vlan_processing);
+
+	/* other app code here.*/
+	/* Start CLI */
+	ofp_start_cli_thread(app_init_params.linux_core_id, params.conf_file);
+
+	OFP_INFO("HTTP thread started");
+
+	//odp_init_local(ODP_THREAD_CONTROL);
+	ofp_init_local();
+	sleep (1);
+
+	//odph_linux_pthread_join(thread_tbl, num_workers);
+	printf("End Main()\n");
+
+	return 0;
+}
+
+/**
+ * Parse and store the command line arguments
+ *
+ * @param argc       argument count
+ * @param argv[]     argument vector
+ * @param appl_args  Store application arguments here
+ */
+static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
+{
+	memset(appl_args, 0, sizeof(*appl_args));
+	appl_args->if_count = argc;
+
+	/* allocate storage for the if names */
+	appl_args->if_names =
+		calloc(appl_args->if_count, sizeof(char *));
+	appl_args->if_names = argv;
+
+	char filename[]="./conf/ofp.conf";
+	int len = strlen(filename);
+	printf("len : %x, file : %s\n", len, filename);
+	len += 1;	/* add room for '\0' */
+
+	appl_args->conf_file = malloc(len);
+	if (appl_args->conf_file == NULL) {
+		usage(argv[0]);
+		exit(EXIT_FAILURE);
+	}
+
+	strcpy(appl_args->conf_file, filename);
+}
+
+/**
+ * Prinf usage information
+ */
+static void usage(char *progname)
+{
+	printf("\n"
+		"Usage: %s OPTIONS\n"
+		"  E.g. %s -i eth1,eth2,eth3\n"
+		"\n"
+		"ODPFastpath application.\n"
+		"\n"
+		"Mandatory OPTIONS:\n"
+		" -i, --interface Eth interfaces (comma-separated, no spaces)\n"
+		"\n"
+		"Optional OPTIONS\n"
+		"  -c, --count <number> Core count.\n"
+		"  -h, --help           Display help and exit.\n"
+		"\n", NO_PATH(progname), NO_PATH(progname)
+		);
+}
+
+/**
+ * Print system and application info
+ */
+static void print_info(char *progname, appl_args_t *appl_args)
+{
+	int i;
+
+	printf("\n"
+		"ODP system info\n"
+		"---------------\n"
+		"ODP API version: %s\n"
+		"CPU model:       %s\n"
+		"CPU freq (hz):   %"PRIu64"\n"
+		"Cache line size: %i\n"
+		"Core count:      %i\n"
+		"\n",
+		odp_version_api_str(), odp_sys_cpu_model_str(),
+		odp_sys_cpu_hz(), odp_sys_cache_line_size(),
+		odp_cpu_count());
+
+	printf("Running ODP appl: \"%s\"\n"
+		"-----------------\n"
+		"IF-count:        %i\n"
+		"Using IFs:      ",
+		progname, appl_args->if_count);
+	for (i = 0; i < appl_args->if_count; ++i)
+		printf(" %s", appl_args->if_names[i]);
+
+	printf("\n\n");
+	fflush(NULL);
+}
+
+static int inited;
+static int ofp_fd;
+
+static int (*real_socket)(int, int, int);
+static int (*real_bind)(int, __CONST_SOCKADDR_ARG, socklen_t);
+static int (*real_listen)(int, int);
+static int (*real_setsockopt)(int, int, int, const void *, socklen_t);
+static int (*real_ioctl)(int, int, void *);
+static int (*real_select) (int nfds, fd_set *readfds, fd_set *writefds,
+			   fd_set *exceptfds, struct timeval *timeout);
+static int (*real_accept)(int, struct sockaddr *, socklen_t *);
+static int (*real_close)(int);
+static ssize_t (*real_recv)(int, void *, size_t, int);
+static ssize_t (*real_send)(int, const void *, size_t, int);
+static ssize_t (*real_sendfile64)(int, int, off_t *, size_t);
+static int (*real_printf)(const char *__fmt, ...);
+static ssize_t (*real_writev)(int, const struct iovec *, int);
+
+void ngx_ofp_init()
+{
+	int rc;
+
+#define INIT_FUNCTION(func) \
+        real_##func = dlsym(RTLD_NEXT, #func); \
+        assert(real_##func)
+
+	INIT_FUNCTION(socket);
+	INIT_FUNCTION(bind);
+	INIT_FUNCTION(listen);
+	INIT_FUNCTION(setsockopt);
+
+	INIT_FUNCTION(ioctl);
+	INIT_FUNCTION(select);
+	INIT_FUNCTION(printf);
+
+	INIT_FUNCTION(sendfile64);
+	INIT_FUNCTION(accept);
+	INIT_FUNCTION(close);
+	INIT_FUNCTION(recv);
+	INIT_FUNCTION(send);
+	INIT_FUNCTION(writev);
+#undef INIT_FUNCTION
+
+	char *p=malloc(5);
+	strncpy(p, "eth0", 5);
+	my_webserver(1, &p);
+
+	rc = 0;
+	assert(0 == rc);
+	inited = 1;
+}
+
+int socket(int domain, int type, int protocol)
+{
+	int rc;
+
+	if ((inited != 1) || (AF_INET != domain) ||
+		(SOCK_STREAM != type && SOCK_DGRAM != type)) {
+	     rc = real_socket(domain, type, protocol);
+	return rc;
+	}
+
+	rc = ofp_socket(domain, type, protocol);
+	rc |= 1 << ODP_FD_BITS;
+
+	if (ofp_fd == 0)
+	  ofp_fd = rc;
+	return rc;
+}
+
+int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)
+{
+	if (__fd & (1 << ODP_FD_BITS)) {
+		struct ofp_sockaddr ofp_addr;
+		int ret = -9;
+		memcpy(&ofp_addr, __addr, sizeof(ofp_addr));
+		if (ofp_addr.sa_len != sizeof(struct ofp_sockaddr))
+			ofp_addr.sa_len = sizeof(struct ofp_sockaddr);
+		__fd &= ~(1 << ODP_FD_BITS);
+		ret = ofp_bind(__fd, (const struct ofp_sockaddr *)&ofp_addr,
+			ofp_addr.sa_len);
+
+		return ret;
+	} else {
+	    return real_bind(__fd, __addr, __len);
+	}
+}
+
+
+int select (int nfds, fd_set *readfds, fd_set *writefds,
+                  fd_set *exceptfds, struct timeval *timeout)
+{
+	int rc = 0;
+	if (nfds & (1 << ODP_FD_BITS)) {
+		struct ofp_timeval __timeout;
+		if (timeout == NULL) {
+			__timeout.tv_sec = 0;
+			__timeout.tv_usec = 200000;
+		} else {
+			__timeout.tv_sec = timeout->tv_sec;
+			__timeout.tv_usec = timeout->tv_usec;
+		}
+
+		__timeout.tv_sec = 0;
+		__timeout.tv_usec = 200000;
+
+		ofp_fd_set *__readfds = NULL, *__writefds = NULL;
+		ofp_fd_set *__exceptfds = NULL;
+
+		if (readfds) {
+			__readfds = (ofp_fd_set *)readfds;
+		}
+
+		if (writefds) {
+			__writefds = (ofp_fd_set *)writefds;
+		}
+
+		if (exceptfds) {
+			__exceptfds = (ofp_fd_set *)exceptfds;
+		}
+
+		nfds &= ~(1 << ODP_FD_BITS);
+		while (rc <= 0)
+		{
+			rc = ofp_select(nfds, __readfds, __writefds,
+					__exceptfds, &__timeout);
+			if (rc < 0)
+				printf("ofp_errno : %d", ofp_errno);
+		}
+	} else {
+		rc = real_select(nfds, readfds, writefds, exceptfds, timeout);
+	}
+
+	return rc;
+}
+
+int setsockopt (int __fd, int __level, int __optname,
+               const void *__optval, socklen_t __optlen)
+{
+	if (__fd & (1 << ODP_FD_BITS)) {
+		__fd &= ~(1 << ODP_FD_BITS);
+		return ofp_setsockopt(__fd, __level, __optname, __optval, __optlen);
+	} else {
+		return real_setsockopt(__fd, __level, __optname, __optval, __optlen);
+	}
+}
+
+int listen (int __fd, int __n)
+{
+	if (__fd & (1 << ODP_FD_BITS)) {
+		__fd &= ~(1 << ODP_FD_BITS);
+		return ofp_listen(__fd, __n);
+	} else {
+		return real_listen(__fd, __n);
+	}
+}
+
+
+int ioctl(int fd, int request, void *p)
+{
+	int ret = -99;
+
+	if (fd & (1 << ODP_FD_BITS)) {
+		fd &= ~(1 << ODP_FD_BITS);
+		return 0;
+		ret = ofp_ioctl(fd, request, p);
+		if (ofp_errno == OFP_EOPNOTSUPP)
+			ret = 0;
+		return ret;
+	} else {
+		return real_ioctl(fd, request, p);
+	}
+}
+
+ssize_t send (int __fd, const void *__buf, size_t __n, int __flags)
+{
+	if (__fd & (1 << ODP_FD_BITS)) {
+		__fd &= ~(1 << ODP_FD_BITS);
+		return ofp_send(__fd, __buf, __n, __flags);
+	} else {
+		return real_send(__fd, __buf, __n, __flags);
+	}
+}
+
+ssize_t recv (int __fd, void *__buf, size_t __n, int __flags)
+{
+	ssize_t rc;
+	if (__fd & (1 << ODP_FD_BITS)) {
+		__fd &= ~(1 << ODP_FD_BITS);
+		rc = ofp_recv(__fd, __buf, __n, __flags);
+		if (-1 == rc && OFP_EAGAIN == ofp_errno) {
+			errno = EAGAIN;
+		}
+		return rc;
+	} else {
+		return real_recv(__fd, __buf, __n, __flags);
+	}
+}
+
+int accept (int sockfd, __SOCKADDR_ARG addr,
+               socklen_t *__restrict addrlen)
+{
+	int rc = 0;
+
+	if (sockfd & (1 << ODP_FD_BITS)) {
+		struct ofp_sockaddr ofp_addr;
+		memcpy(&ofp_addr, addr, sizeof(ofp_addr));
+		if (ofp_addr.sa_len != sizeof(struct ofp_sockaddr))
+			ofp_addr.sa_len = sizeof(struct ofp_sockaddr);
+
+		sockfd &= ~(1 << ODP_FD_BITS);
+
+		rc = ofp_accept(sockfd, &ofp_addr, addrlen);
+		addr->sa_family = AF_INET;
+		memcpy(addr->sa_data, ofp_addr.sa_data, sizeof(addr->sa_data));
+
+		rc |= 1 << ODP_FD_BITS;
+		if (-1 == rc && OFP_EAGAIN == ofp_errno) {
+			errno = EAGAIN;
+		}
+	} else {
+		rc = real_accept(sockfd, addr, addrlen);
+	}
+	return rc;
+}
+
+int close(int fd)
+{
+	if (fd & (1 << ODP_FD_BITS)) {
+		fd &= ~(1 << ODP_FD_BITS);
+		return ofp_close(fd);
+	} else {
+		return real_close(fd);
+	}
+}
+
+ssize_t writev(int fd, const struct iovec *iov, int iovcnt)
+{
+	ssize_t rc, i, n;
+	int nwrite = 0, data_size;
+	char *buf;
+
+	if (fd & (1 << ODP_FD_BITS))
+	{
+		fd &= ~(1 << ODP_FD_BITS);
+		rc = 0;
+		for (i = 0; i != iovcnt; ++i)
+		{
+			data_size = iov[i].iov_len;
+			buf = iov[i].iov_base;
+			n = data_size;
+
+			while (n > 0)
+			{
+				nwrite = ofp_send(fd, buf + data_size - n, n, 0);
+
+				if(nwrite<=0)
+				{
+					if(errno==OFP_EAGAIN)
+					{
+						usleep(200);
+						continue;
+					}
+					else
+					{
+						return(nwrite);
+					}
+				}
+
+				if (nwrite < n)
+				{
+					usleep(200);
+				}
+				n -= nwrite;
+
+			}
+
+			if (nwrite <= 0)
+				return nwrite;
+
+			rc += data_size;
+		}
+	}
+	else
+	{
+		rc = real_writev(fd, iov, iovcnt);
+	}
+
+	return rc;
+}
+
+
+
+#define BUF_SIZE 1024
+
+#define min(m,n) ((m) < (n) ? (m) : (n))
+#define max(m,n) ((m) > (n) ? (m) : (n))
+
+ssize_t sendfile64(int out_fd, int in_fd, off_t *offset, size_t count)
+{
+	off_t orig = 0;
+	char buf[BUF_SIZE];
+	size_t to_read,  total_sent;
+	int num_sent, num_read;
+	ssize_t rc = 0;
+
+	if (out_fd & (1 << ODP_FD_BITS))
+	{
+		out_fd &= ~(1 << ODP_FD_BITS);
+
+		if (offset != NULL) {
+			orig = lseek(in_fd, 0, SEEK_CUR);
+			if (orig == -1)
+				return -1;
+			if (lseek(in_fd, *offset, SEEK_SET) == -1)
+				return -1;
+		}
+
+		total_sent = 0;
+
+		while (count > 0) {
+			to_read = min(BUF_SIZE, count);
+			num_read = read(in_fd, buf, to_read);
+			if (num_read == -1)
+				return -1;
+			if (num_read == 0)
+				break;				/* EOF */
+
+			num_sent = ofp_send(out_fd, buf, num_read, 0);
+			if (num_sent == -1)
+				return -1;
+			if (num_sent == 0)
+				printf("sendfile64: transferred 0 bytes\n");
+
+			count -= num_sent;
+			total_sent += num_sent;
+		}
+
+		if (offset != NULL) {
+			*offset = lseek(in_fd, 0, SEEK_CUR);
+			if (*offset == -1)
+				return -1;
+			if (lseek(in_fd, orig, SEEK_SET) == -1)
+				return -1;
+		}
+
+		return total_sent;
+
+	} else {
+		rc = real_sendfile64( out_fd, in_fd, offset, count);
+	}
+	return rc;
+
+}
diff --git a/src/event/modules/ngx_select_module.c b/src/event/modules/ngx_select_module.c
index 5a976bd9..186b88ed 100644
--- a/src/event/modules/ngx_select_module.c
+++ b/src/event/modules/ngx_select_module.c
@@ -8,7 +8,7 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 #include <ngx_event.h>
-
+#include <ofp.h>
 
 static ngx_int_t ngx_select_init(ngx_cycle_t *cycle, ngx_msec_t timer);
 static void ngx_select_done(ngx_cycle_t *cycle);
@@ -70,6 +70,22 @@ ngx_module_t  ngx_select_module = {
     NGX_MODULE_V1_PADDING
 };
 
+ofp_fd_set ofp_readfd;
+#define ODP_FD_BITS 30
+#undef FD_SET
+#define FD_SET(fd, fdsetp)      do { \
+				if (fd & (1 << ODP_FD_BITS)) {  \
+				   OFP_FD_SET ((fd & ~(1 << ODP_FD_BITS)) , (ofp_fd_set *)fdsetp) ; \
+				} else { \
+				   OFP_FD_SET (fd , (ofp_fd_set *)fdsetp) ; \
+				} \
+				} while (0)
+
+#undef FD_ISSET
+#define FD_ISSET(fd, fdsetp)    OFP_FD_ISSET((fd & ~(1 << ODP_FD_BITS)), (ofp_fd_set *)fdsetp)
+
+#undef FD_CLR
+#define FD_CLR(fd, fdsetp)      OFP_FD_CLR ((fd & ~(1 << ODP_FD_BITS)), (ofp_fd_set *)fdsetp)
 
 static ngx_int_t
 ngx_select_init(ngx_cycle_t *cycle, ngx_msec_t timer)
@@ -128,8 +144,8 @@ ngx_select_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
 
     c = ev->data;
 
-    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
-                   "select add event fd:%d ev:%i", c->fd, event);
+    ngx_log_debug(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "%s: select add event fd:%d ev:%i", __func__, c->fd, event);
 
     if (ev->index != NGX_INVALID_INDEX) {
         ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
@@ -146,9 +162,15 @@ ngx_select_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
         return NGX_ERROR;
     }
 
-    if (event == NGX_READ_EVENT) {
-        FD_SET(c->fd, &master_read_fd_set);
 
+        ngx_log_debug(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                      "select %s event fd:%d  event:%d",
+                      ev->write ? "write" : "read", c->fd, event);
+
+    if (event == NGX_READ_EVENT) {
+	if (c->fd & (1<<ODP_FD_BITS)) {
+           FD_SET(c->fd, (ofp_fd_set *)&master_read_fd_set);
+	}
     } else if (event == NGX_WRITE_EVENT) {
         FD_SET(c->fd, &master_write_fd_set);
     }
@@ -157,6 +179,9 @@ ngx_select_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
         max_fd = c->fd;
     }
 
+    ngx_log_debug(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                      "%s: select event fd:%d  max_fd:%d",
+                       __func__, c->fd, max_fd);
     ev->active = 1;
 
     event_index[nevents] = ev;
@@ -222,6 +247,8 @@ ngx_select_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
     if (max_fd == -1) {
         for (i = 0; i < nevents; i++) {
             c = event_index[i]->data;
+        ngx_log_debug(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                       "change max_fd: %i, c->fd : %d", max_fd, c->fd);
             if (max_fd < c->fd) {
                 max_fd = c->fd;
             }
@@ -260,6 +287,11 @@ ngx_select_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
     work_read_fd_set = master_read_fd_set;
     work_write_fd_set = master_write_fd_set;
 
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "select max_fd %d, tp %p ", max_fd, tp);
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   " master %p : work %p", (ofp_fd_set *)&master_read_fd_set,
+		   (ofp_fd_set *)&work_read_fd_set);
     ready = select(max_fd + 1, &work_read_fd_set, &work_write_fd_set, NULL, tp);
 
     err = (ready == -1) ? ngx_errno : 0;
diff --git a/src/os/unix/ngx_process.c b/src/os/unix/ngx_process.c
index 6f3f3855..4d75281a 100644
--- a/src/os/unix/ngx_process.c
+++ b/src/os/unix/ngx_process.c
@@ -183,7 +183,7 @@ ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data,
     ngx_process_slot = s;
 
 
-    pid = fork();
+    pid = vfork();
 
     switch (pid) {