path: root/lib/netdev-odp.c

/*
 * Copyright (c) 2014 Nicira, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <config.h>

#include <stdio.h>
#include <string.h>
#include <signal.h>
#include <stdlib.h>
#include <pthread.h>
#include <config.h>
#include <errno.h>
#include <sched.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <net/ethernet.h>

#include "dpif-netdev.h"
#include "list.h"
#include "netdev-odp.h"
#include "netdev-provider.h"
#include "netdev-vport.h"
#include "odp-util.h"
#include "ofp-print.h"
#include "ovs-thread.h"
#include "ovs-rcu.h"
#include "packets.h"
#include "shash.h"
#include "sset.h"
#include "unaligned.h"
#include "timeval.h"
#include "unixctl.h"
#include "openvswitch/vlog.h"

VLOG_DEFINE_THIS_MODULE(odp);

#define SHM_PKT_POOL_NUM_BUFS  (4096 - 1)
#define SHM_PKT_POOL_BUF_SIZE  1856

#define SHM_DP_PACKET_POOL_BUF_SIZE  sizeof(struct dp_packet)

static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);

static odp_pool_t struct_pool;

int odp_initialized = 0;
const odp_platform_init_t *platform_params = NULL;

struct netdev_odp {
    struct netdev up;
    odp_buffer_t odp_buf; /* odp_buffer_t that holds this struct */
    odp_pktio_t pktio;
    odp_pktin_queue_t pktin_queue;
    odp_pktout_queue_t pktout_queue;
    odp_pool_t pkt_pool;
    size_t frame_offset;
    size_t max_frame_len;

    struct ovs_mutex mutex OVS_ACQ_AFTER(odp_mutex);

    struct eth_addr hwaddr;
    enum netdev_flags flags;

    struct netdev_stats stats;
};

struct netdev_rxq_odp {
    struct netdev_rxq up;
    odp_buffer_t odp_buf; /* odp_buffer_t that holds this struct */
};

/* We need a pool of buffers that hold netdev and rxq structures */
#define STRUCTS_SIZE MAX(sizeof(struct netdev_odp), \
                         sizeof(struct netdev_rxq_odp))
#define SHM_STRUCT_POOL_NUM_BUFS   512
#define SHM_STRUCT_POOL_BUF_SIZE   STRUCTS_SIZE

void
free_odp_buf(struct dp_packet *b)
{
    odp_packet_free(b->odp_pkt);
}

OVS_NO_RETURN static void odp_abort(void)
{
    VLOG_ERR("abort\n");
    abort();
}

void
odp_init()
{
    int result;
    odp_init_t params;

    memset(&params, 0, sizeof(params));
    params.log_fn = &odp_override_log;
    params.abort_fn = &odp_abort;

    result = odp_init_global(&params, platform_params);
    if (result)
        ovs_abort(result, "Error: ODP global init failed\n");

    /* Init this thread */
    result = odp_init_local(ODP_THREAD_CONTROL);
    if (result)
        ovs_abort(result, "Error: ODP local init failed.\n");
}

static int
odp_class_init(void)
{
    odp_pool_param_t params;
    int result = 0;

    /* create pool for structures */

    params.type = ODP_POOL_BUFFER;
    params.buf.size = SHM_STRUCT_POOL_BUF_SIZE;
    params.buf.num = SHM_STRUCT_POOL_NUM_BUFS;
    params.buf.align = 0;

    struct_pool = odp_pool_create("struct_pool", &params);

    if (struct_pool == ODP_POOL_INVALID) {
            VLOG_ERR("Error: struct_pool create failed.\n");
            return -1;
    }
    odp_pool_print(struct_pool);

    return result;
}

static struct netdev *
netdev_odp_alloc(void)
{
    struct netdev_odp *netdev;
    odp_buffer_t buf;
    buf = odp_buffer_alloc(struct_pool);
    netdev = odp_buffer_addr(buf);
    memset(netdev, 0, sizeof(*netdev));
    netdev->odp_buf = buf;
    return &netdev->up;
}

static struct netdev_odp *
netdev_odp_cast(const struct netdev *netdev)
{
   return CONTAINER_OF(netdev, struct netdev_odp, up);
}

static int
netdev_odp_construct(struct netdev *netdev_)
{
    int mtu;
    char *odp_if;
    struct netdev_odp *netdev = netdev_odp_cast(netdev_);
    odp_pktio_param_t pktio_param;
    odp_pktin_queue_param_t pktin_queue_param;
    odp_pktout_queue_param_t pktout_queue_param;
    odp_pool_param_t params;
    char pool_name[ODP_POOL_NAME_LEN];

    odp_if = netdev_->name + 4; /* Names always start with "odp:" */

    if (strncmp(netdev_->name, "odp:", 4))
        return ENODEV;

    /* create packet pool */
    params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE;
    params.pkt.uarea_size = SHM_DP_PACKET_POOL_BUF_SIZE;
    params.pkt.len = 0;
    params.pkt.num = SHM_PKT_POOL_NUM_BUFS;
    params.type = ODP_POOL_PACKET;
    snprintf(pool_name, sizeof(pool_name), "pkt_pool_%s", odp_if);

    netdev->pkt_pool = odp_pool_create(pool_name, &params);

    if (netdev->pkt_pool == ODP_POOL_INVALID) {
            VLOG_ERR("Error: packet pool create failed.\n");
            return ENOMEM;
    }
    odp_pool_print(netdev->pkt_pool);

    pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT;
    pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT;
    netdev->pktio = odp_pktio_open(odp_if, netdev->pkt_pool, &pktio_param);

    if (netdev->pktio == ODP_PKTIO_INVALID) {
        VLOG_ERR("Error: odp_pktio_open() failed\n");
        goto err_open;
    }

    pktin_queue_param.op_mode = ODP_PKTIO_OP_MT;
    pktin_queue_param.hash_enable = 1;
    pktin_queue_param.hash_proto.all_bits = UINT32_MAX;
    pktin_queue_param.num_queues = 1;

    if (odp_pktin_queue_config(netdev->pktio, &pktin_queue_param) != 0) {
        VLOG_ERR("Error: odp_pktin_queue_config() failed\n");
        goto err_start;
    }

    if (odp_pktin_queue(netdev->pktio, &netdev->pktin_queue, 1) != 1) {
        VLOG_ERR("Error: odp_pktin_queue() failed\n");
        goto err_start;
    }

    pktout_queue_param.op_mode = ODP_PKTIO_OP_MT;
    pktout_queue_param.num_queues = 1;

    if (odp_pktout_queue_config(netdev->pktio, &pktout_queue_param) != 0) {
        VLOG_ERR("Error: odp_pktout_queue_config() failed\n");
        goto err_start;
    }

    if (odp_pktout_queue(netdev->pktio, &netdev->pktout_queue, 1) != 1) {
        VLOG_ERR("Error: odp_pktout_queue() failed\n");
        goto err_start;
    }

    if (odp_pktio_start(netdev->pktio) != 0) {
        VLOG_ERR("Error: odp_pktio_start() failed\n");
        goto err_start;
    }

    mtu = odp_pktio_mtu(netdev->pktio);
    if (mtu < 0) {
        VLOG_ERR("Error: odp_pktio_mtu() failed with %d\n", mtu);
        goto err_mtu;
    }
    netdev->max_frame_len = mtu + ETHER_HDR_LEN;

    ovs_mutex_init(&netdev->mutex);

    return 0;

err_mtu:
    odp_pktio_stop(netdev->pktio);
err_start:
    odp_pktio_close(netdev->pktio);
err_open:
    odp_pool_destroy(netdev->pkt_pool);

    return odp_errno() < 0 ? -odp_errno() : odp_errno();
}

static void
netdev_odp_destruct(struct netdev *netdev_)
{
    struct netdev_odp *netdev = netdev_odp_cast(netdev_);

    odp_pktio_stop(netdev->pktio);
    odp_pktio_close(netdev->pktio);
    odp_pool_destroy(netdev->pkt_pool);
}

static void
netdev_odp_dealloc(struct netdev *netdev_)
{
    struct netdev_odp *netdev = netdev_odp_cast(netdev_);
    odp_buffer_free(netdev->odp_buf);
}

static int
netdev_odp_get_config(const struct netdev *netdev_, struct smap *args)
{
    struct netdev_odp *netdev = netdev_odp_cast(netdev_);

    ovs_mutex_lock(&netdev->mutex);

    /* TODO: Allow to configure number of queues. */
    smap_add_format(args, "configured_rx_queues", "%u", netdev_->n_rxq);
    smap_add_format(args, "configured_tx_queues", "%u", netdev_->n_rxq);
    ovs_mutex_unlock(&netdev->mutex);

    return 0;
}

static int
clone_pkts(struct netdev_odp *dev, struct dp_packet **pkts,
          odp_packet_t odp_pkts[], int cnt)
{
    int dropped = 0;
    int newcnt = 0;
    int i;

    for (i = 0; i < cnt; i++) {
        size_t size = dp_packet_size((const struct dp_packet*)pkts[i]);
        odp_packet_t pkt;

        if (OVS_UNLIKELY(size > dev->max_frame_len)) {
            VLOG_WARN_RL(&rl, "Too big size %u max_packet_len %u",
                         (unsigned)size,
                         (unsigned)dev->max_frame_len);
            dropped++;
            continue;
        }
        pkt = odp_packet_alloc(dev->pkt_pool, size);

        if (OVS_UNLIKELY(pkt == ODP_PACKET_INVALID)) {
            VLOG_WARN_RL(&rl, "Could not allocate packet");
            dropped += cnt -i;
            break;
        }

        odp_packet_copydata_in(pkt, 0, size, dp_packet_data((const struct dp_packet*)pkts[i]));
        odp_pkts[newcnt] = pkt;
        newcnt++;
    }

    if (OVS_UNLIKELY(dropped)) {
        ovs_mutex_lock(&dev->mutex);
        dev->stats.tx_dropped += dropped;
        ovs_mutex_unlock(&dev->mutex);
    }

    return newcnt;
}

static int
netdev_odp_get_numa_id(const struct netdev *netdev_ OVS_UNUSED)
{
	return 0;
}

static int
netdev_odp_send(struct netdev *netdev, int qid OVS_UNUSED,
                struct dp_packet **pkts, int cnt, bool may_steal)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);
    odp_packet_t odp_pkts[NETDEV_MAX_BURST];
    int pkts_ok, i, sent;

    /* Normally NETDEV_MAX_BURST should be the limit and VLA ar nasty */
    ovs_assert(cnt <= NETDEV_MAX_BURST);

    if (!may_steal || pkts[0]->source != DPBUF_ODP) {
        pkts_ok = clone_pkts(dev, pkts, odp_pkts, cnt);

        if (may_steal) {
            for (i = 0; i < cnt; i++) {
                dp_packet_delete(pkts[i]);
            }
        }
    } else {
        for (i = 0; i < cnt; i++)
            odp_pkts[i] = pkts[i]->odp_pkt;
        pkts_ok = cnt;
    }

    sent = odp_pktout_send(dev->pktout_queue, odp_pkts, pkts_ok);
    if (OVS_UNLIKELY(sent < pkts_ok)) {
        sent = (sent > 0) ? sent : 0;
        for (i = sent; i < pkts_ok; i++)
            odp_packet_free(odp_pkts[i]);
    }

    ovs_mutex_lock(&dev->mutex);
    dev->stats.tx_dropped += pkts_ok - sent;
    ovs_mutex_unlock(&dev->mutex);

    return 0;
}

static int
netdev_odp_set_etheraddr(struct netdev *netdev,
                         const struct eth_addr mac)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    ovs_mutex_lock(&dev->mutex);
    if (!eth_addr_equals(dev->hwaddr, mac)) {
        dev->hwaddr = mac;
        netdev_change_seq_changed(netdev);
    }
    ovs_mutex_unlock(&dev->mutex);

    return 0;
}

static int
netdev_odp_get_etheraddr(const struct netdev *netdev,
                         struct eth_addr *mac)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    ovs_mutex_lock(&dev->mutex);
    *mac = dev->hwaddr;
    ovs_mutex_unlock(&dev->mutex);

    return 0;
}

static int
netdev_odp_get_mtu(const struct netdev *netdev, int *mtup)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    (void) dev;
    (void) mtup;

    return ENOTSUP;
}

static int
netdev_odp_set_mtu(const struct netdev *netdev, int mtu)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    (void) dev;
    (void) mtu;

    return ENOTSUP;
}

static int
netdev_odp_get_ifindex(const struct netdev *netdev)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    (void) dev;

    return ENOTSUP;
}

static int
netdev_odp_get_carrier(const struct netdev *netdev, bool *carrier)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    (void) dev;
    (void) carrier;

    return ENOTSUP;
}

static long long int
netdev_odp_get_carrier_resets(const struct netdev *netdev)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    (void) dev;

    return ENOTSUP;
}

static int
netdev_odp_set_miimon(struct netdev *netdev_ OVS_UNUSED,
                       long long int interval OVS_UNUSED)
{
    return 0;
}

static int
netdev_odp_get_stats(const struct netdev *netdev, struct netdev_stats *stats)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);
    struct odp_pktio_stats_t odp_stats;

    ovs_mutex_lock(&dev->mutex);
    odp_pktio_stats(dev->pktio, &odp_stats);

    memset(stats, 0, sizeof(*stats));

    stats->rx_packets = odp_stats.in_ucast_pkts;
    stats->tx_packets = odp_stats.out_ucast_pkts;
    stats->rx_bytes = odp_stats.in_octets;
    stats->tx_bytes = odp_stats.out_octets;
    stats->rx_errors = odp_stats.in_errors;
    stats->tx_errors = odp_stats.out_errors;
    stats->multicast = UINT64_MAX;

    stats->tx_dropped = dev->stats.tx_dropped;

    stats->rx_dropped = UINT64_MAX;
    stats->collisions = UINT64_MAX;

    stats->rx_length_errors = UINT64_MAX;
    stats->rx_over_errors = UINT64_MAX;
    stats->rx_crc_errors = UINT64_MAX;
    stats->rx_frame_errors = odp_stats.in_unknown_protos;
    stats->rx_fifo_errors = UINT64_MAX;
    stats->rx_missed_errors = odp_stats.in_discards;

    stats->tx_aborted_errors = odp_stats.out_discards;
    stats->tx_carrier_errors = UINT64_MAX;
    stats->tx_fifo_errors = UINT64_MAX;
    stats->tx_heartbeat_errors = UINT64_MAX;
    stats->tx_window_errors = UINT64_MAX;
    ovs_mutex_unlock(&dev->mutex);

    return 0;
}

static int
netdev_odp_get_features(const struct netdev *netdev,
                         enum netdev_features *current,
                         enum netdev_features *advertised OVS_UNUSED,
                         enum netdev_features *supported OVS_UNUSED,
                         enum netdev_features *peer OVS_UNUSED)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    (void) dev;
    (void) current;

    return ENOTSUP;
}

static int
netdev_odp_get_status(const struct netdev *netdev, struct smap *args)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    (void) dev;
    (void) args;

    return ENOTSUP;
}

static int
netdev_odp_update_flags(struct netdev *netdev,
                         enum netdev_flags off, enum netdev_flags on,
                         enum netdev_flags *old_flagsp)
{
    struct netdev_odp *dev = netdev_odp_cast(netdev);

    (void) dev;
    (void) off;
    (void) on;
    (void) old_flagsp;

    if ((off | on) & ~(NETDEV_UP | NETDEV_PROMISC)) {
        return EINVAL;
    }

    *old_flagsp = dev->flags;
    dev->flags |= on;
    dev->flags &= ~off;

    if (dev->flags == *old_flagsp) {
        return 0;
    }

    return 0;
}

static struct netdev_rxq *
netdev_odp_rxq_alloc(void)
{
    struct netdev_rxq_odp *rx;
    odp_buffer_t buf;
    buf = odp_buffer_alloc(struct_pool);
    rx = odp_buffer_addr(buf);
    memset(rx, 0, sizeof(*rx));
    rx->odp_buf = buf;
    return &rx->up;
}

static struct netdev_rxq_odp *
netdev_rxq_odp_cast(const struct netdev_rxq *rx)
{
    return CONTAINER_OF(rx, struct netdev_rxq_odp, up);
}

static int
netdev_odp_rxq_construct(struct netdev_rxq *rxq_)
{
    struct netdev_rxq_odp *rx = netdev_rxq_odp_cast(rxq_);
    struct netdev_odp *netdev = netdev_odp_cast(rx->up.netdev);

    return 0;
}

static void
netdev_odp_rxq_destruct(struct netdev_rxq *rxq_ OVS_UNUSED)
{
}

static void
netdev_odp_rxq_dealloc(struct netdev_rxq *rxq_)
{
    struct netdev_rxq_odp *rxq = netdev_rxq_odp_cast(rxq_);
    odp_buffer_free(rxq->odp_buf);
}

static int
netdev_odp_rxq_recv(struct netdev_rxq *rxq_, struct dp_packet **packets,
                    int *c)
{
    struct netdev_rxq_odp *rx = netdev_rxq_odp_cast(rxq_);
    struct netdev_odp *netdev = netdev_odp_cast(rx->up.netdev);
    int pkts;
    int i;
    odp_packet_t pkt_tbl[NETDEV_MAX_BURST];

    pkts = odp_pktin_recv(netdev->pktin_queue, pkt_tbl, NETDEV_MAX_BURST);
    if (OVS_UNLIKELY(pkts < 1)) {
        if (!pkts)
            return EAGAIN;
        VLOG_ERR_RL(&rl, "ODP: Packet receive error (%d) errno %s\n", pkts, odp_errno_str(odp_errno()));
        return EINVAL;
    }

    /* Build the array of dp_packet pointers */
    for (i = 0; i < pkts; i++) {
        packets[i] = (struct dp_packet*) odp_packet_user_area(pkt_tbl[i]);
        OVS_PREFETCH(packets[i]);
    }
    for (i = 0; i < pkts; i++) {
        packets[i]->odp_pkt = pkt_tbl[i];
        dp_packet_init_odp(packets[i], SHM_PKT_POOL_BUF_SIZE);
    }

    *c = pkts;

    return 0;
}

int odp_override_log(odp_log_level_t level, const char *fmt, ...)
{
    va_list args;
    enum vlog_level ovs_level;

    switch (level) {
    case ODP_LOG_ERR:
    case ODP_LOG_UNIMPLEMENTED:
    case ODP_LOG_ABORT:
        ovs_level = VLL_ERR;
        break;
    case ODP_LOG_PRINT:
        ovs_level = VLL_INFO;
        break;
    case ODP_LOG_DBG:
    default:
        ovs_level = VLL_DBG;
    }

    va_start(args, fmt);
    vlog_rate_limit_valist(THIS_MODULE, ovs_level, &rl, fmt, args);
    va_end(args);

    return 0;
}

static struct netdev_class netdev_odp_class = {
    "odp",
    odp_class_init,             /* init */
    NULL,                       /* netdev_odp_run */
    NULL,                       /* netdev_odp_wait */

    netdev_odp_alloc,
    netdev_odp_construct,
    netdev_odp_destruct,
    netdev_odp_dealloc,
    netdev_odp_get_config,
    NULL,                       /* netdev_odp_set_config */
    NULL,                       /* get_tunnel_config */
    NULL,                       /* build_header */
    NULL,                       /* push_header */
    NULL,                       /* pop_header */
    netdev_odp_get_numa_id,     /* get_numa_id */
    NULL,                       /* set_multiq */

    netdev_odp_send,            /* send */
    NULL,                       /* send_wait */

    netdev_odp_set_etheraddr,
    netdev_odp_get_etheraddr,
    netdev_odp_get_mtu,
    netdev_odp_set_mtu,
    netdev_odp_get_ifindex,
    netdev_odp_get_carrier,
    netdev_odp_get_carrier_resets,
    netdev_odp_set_miimon,
    netdev_odp_get_stats,
    netdev_odp_get_features,
    NULL,                       /* set_advertisements */

    NULL,                       /* set_policing */
    NULL,                       /* get_qos_types */
    NULL,                       /* get_qos_capabilities */
    NULL,                       /* get_qos */
    NULL,                       /* set_qos */
    NULL,                       /* get_queue */
    NULL,                       /* set_queue */
    NULL,                       /* delete_queue */
    NULL,                       /* get_queue_stats */
    NULL,                       /* queue_dump_start */
    NULL,                       /* queue_dump_next */
    NULL,                       /* queue_dump_done */
    NULL,                       /* dump_queue_stats */

    NULL,                       /* get_in4 */
    NULL,                       /* set_in4 */
    NULL,                       /* get_in6 */
    NULL,                       /* add_router */
    NULL,                       /* get_next_hop */
    netdev_odp_get_status,
    NULL,                       /* arp_lookup */

    netdev_odp_update_flags,

    netdev_odp_rxq_alloc,
    netdev_odp_rxq_construct,
    netdev_odp_rxq_destruct,
    netdev_odp_rxq_dealloc,
    netdev_odp_rxq_recv,
    NULL,                       /* rxq_wait */
    NULL,                       /* rxq_drain */
};

void
netdev_odp_register(void)
{
    static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;

    if (ovsthread_once_start(&once)) {
        odp_init();
        netdev_register_provider(&netdev_odp_class);
        ovsthread_once_done(&once);
    }
}

int
pmd_thread_setaffinity_cpu(int cpu)
{
    cpu_set_t cpuset;
    int err;

    CPU_ZERO(&cpuset);
    CPU_SET(cpu, &cpuset);
    err = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
    if (err) {
        VLOG_ERR("Thread affinity error %d",err);
        return err;
    }
    /* NON_PMD_CORE_ID is reserved for use by non pmd threads. */
    ovs_assert(cpu != NON_PMD_CORE_ID);

    return 0;
}