[v3] example: introducing l3fwd

Message ID 1465290295-26535-1-git-send-email-forrest.shi@linaro.org
State New
Headers show

Commit Message

Forrest Shi June 7, 2016, 9:04 a.m.
multi-thread, multi-queue and bi-directional forwarding.

support (port, queue, core) arguments in cmdline.
it means core will handle rx queue at port.
forwarding logic will decide out port by this specification.
if no this argument in cmdline, default used.

Signed-off-by: Xuelin Shi <forrest.shi@linaro.org>
---
change history:
 v3: address the comments of v2
     add argument (port, queue, core) specification support
 v2: merge v1 patch set into one patch 

 example/Makefile.am          |   2 +-
 example/l3fwd/.gitignore     |   4 +
 example/l3fwd/Makefile.am    |  12 +
 example/l3fwd/jhash.h        |  46 +++
 example/l3fwd/odp_l3fwd.c    | 744 +++++++++++++++++++++++++++++++++++++++++++
 example/l3fwd/odp_l3fwd_db.c | 381 ++++++++++++++++++++++
 example/l3fwd/odp_l3fwd_db.h | 122 +++++++
 example/m4/configure.m4      |   1 +
 8 files changed, 1311 insertions(+), 1 deletion(-)
 create mode 100644 example/l3fwd/.gitignore
 create mode 100644 example/l3fwd/Makefile.am
 create mode 100644 example/l3fwd/jhash.h
 create mode 100644 example/l3fwd/odp_l3fwd.c
 create mode 100644 example/l3fwd/odp_l3fwd_db.c
 create mode 100644 example/l3fwd/odp_l3fwd_db.h

Comments

Christophe Milard June 7, 2016, 10:55 a.m. | #1
On 7 June 2016 at 11:04, Xuelin Shi <forrest.shi@linaro.org> wrote:
> multi-thread, multi-queue and bi-directional forwarding.
>
> support (port, queue, core) arguments in cmdline.
> it means core will handle rx queue at port.
> forwarding logic will decide out port by this specification.
> if no this argument in cmdline, default used.
>
> Signed-off-by: Xuelin Shi <forrest.shi@linaro.org>
> ---
> change history:
>  v3: address the comments of v2
>      add argument (port, queue, core) specification support
>  v2: merge v1 patch set into one patch
>
>  example/Makefile.am          |   2 +-
>  example/l3fwd/.gitignore     |   4 +
>  example/l3fwd/Makefile.am    |  12 +
>  example/l3fwd/jhash.h        |  46 +++
>  example/l3fwd/odp_l3fwd.c    | 744 +++++++++++++++++++++++++++++++++++++++++++
>  example/l3fwd/odp_l3fwd_db.c | 381 ++++++++++++++++++++++
>  example/l3fwd/odp_l3fwd_db.h | 122 +++++++
>  example/m4/configure.m4      |   1 +
>  8 files changed, 1311 insertions(+), 1 deletion(-)
>  create mode 100644 example/l3fwd/.gitignore
>  create mode 100644 example/l3fwd/Makefile.am
>  create mode 100644 example/l3fwd/jhash.h
>  create mode 100644 example/l3fwd/odp_l3fwd.c
>  create mode 100644 example/l3fwd/odp_l3fwd_db.c
>  create mode 100644 example/l3fwd/odp_l3fwd_db.h
>
> diff --git a/example/Makefile.am b/example/Makefile.am
> index 7f82c4d..67e4389 100644
> --- a/example/Makefile.am
> +++ b/example/Makefile.am
> @@ -1 +1 @@
> -SUBDIRS = classifier generator ipsec packet time timer traffic_mgmt l2fwd_simple switch
> +SUBDIRS = classifier generator ipsec packet time timer traffic_mgmt l2fwd_simple l3fwd switch
> diff --git a/example/l3fwd/.gitignore b/example/l3fwd/.gitignore
> new file mode 100644
> index 0000000..4a25ae8
> --- /dev/null
> +++ b/example/l3fwd/.gitignore
> @@ -0,0 +1,4 @@
> +odp_l3fwd
> +Makefile
> +Makefile.in
> +*.o
> diff --git a/example/l3fwd/Makefile.am b/example/l3fwd/Makefile.am
> new file mode 100644
> index 0000000..e0dd266
> --- /dev/null
> +++ b/example/l3fwd/Makefile.am
> @@ -0,0 +1,12 @@
> +include $(top_srcdir)/example/Makefile.inc
> +
> +bin_PROGRAMS = odp_l3fwd$(EXEEXT)
> +odp_l3fwd_LDFLAGS = $(AM_LDFLAGS) -static
> +odp_l3fwd_CFLAGS = $(AM_CFLAGS) -I${top_srcdir}/example
> +
> +noinst_HEADERS = \
> +                 $(top_srcdir)/example/l3fwd/jhash.h \
> +                 $(top_srcdir)/example/l3fwd/odp_l3fwd_db.h \
> +                 $(top_srcdir)/example/example_debug.h
> +
> +dist_odp_l3fwd_SOURCES = odp_l3fwd.c odp_l3fwd_db.c
> diff --git a/example/l3fwd/jhash.h b/example/l3fwd/jhash.h
> new file mode 100644
> index 0000000..d3dbd2a
> --- /dev/null
> +++ b/example/l3fwd/jhash.h
> @@ -0,0 +1,46 @@
> +/** jhash.h: Jenkins hash support.
> +  *
> +  * Copyright (C) 2006 Bob Jenkins (bob_jenkins@burtleburtle.net)
> +  *
> +  * http://burtleburtle.net/bob/hash/
> +  *
> +  * These are the credits from Bob's sources:
> +  *
> +  * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
> +  *
> +  * These are functions for producing 32-bit hashes for hash table lookup.
> +  * hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and final()
> +  * are externally useful functions.  Routines to test the hash are included
> +  * if SELF_TEST is defined.  You can use this free for any purpose.  It's in
> +  * the public domain.  It has no warranty.
> +  *
> +  * $FreeBSD$
> +  */
> +
> +#ifndef JHASH_H_
> +#define JHASH_H_
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +/**
> + * Hash calculation utility
> + */
> +#define JHASH_GOLDEN_RATIO     0x9e3779b9
> +#define rot(x, k) (((x) << (k)) | ((x) >> (32 - (k))))
> +#define FWD_BJ3_MIX(a, b, c) \
> +{ \
> +       a -= c; a ^= rot(c, 4); c += b; \
> +       b -= a; b ^= rot(a, 6); a += c; \
> +       c -= b; c ^= rot(b, 8); b += a; \
> +       a -= c; a ^= rot(c, 16); c += b; \
> +       b -= a; b ^= rot(a, 19); a += c; \
> +       c -= b; c ^= rot(b, 4); b += a; \
> +}
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif
> diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c
> new file mode 100644
> index 0000000..fe715d5
> --- /dev/null
> +++ b/example/l3fwd/odp_l3fwd.c
> @@ -0,0 +1,744 @@
> +/* Copyright (c) 2016, Linaro Limited
> + * All rights reserved.
> + *
> + * SPDX-License-Identifier:     BSD-3-Clause
> + */
> +
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <errno.h>
> +
> +#include <example_debug.h>
> +#include <odp_api.h>
> +#include <odp/helper/linux.h>
> +#include <odp/helper/eth.h>
> +#include <odp/helper/ip.h>
> +#include <odp/helper/udp.h>
> +#include <odp/helper/tcp.h>
> +
> +#include <getopt.h>
> +
> +#include "odp_l3fwd_db.h"
> +
> +#define POOL_NUM_PKT   8192
> +#define POOL_SEG_LEN   1856
> +#define MAX_PKT_BURST  32
> +
> +#define MAX_NB_WORKER  8
> +#define MAX_NB_PKTIO   4
> +#define MAX_NB_ROUTE   32
> +#define MAX_NB_RX_QUEUE        8
> +#define MAX_NB_TX_QUEUE        8
> +#define MAX_NB_QCONFS  1024
> +#define MAX_NB_QCONF_PER_CORE  8
> +#define CPU_ID_INVALID (-1)
> +
> +/** Get rid of path in filename - only for unix-type paths using '/' */
> +#define NO_PATH(file_name) (strrchr((file_name), '/') ? \
> +                           strrchr((file_name), '/') + 1 : (file_name))
> +
> +struct l3fwd_pktio_s {
> +       odp_pktio_t pktio;
> +       odp_pktin_queue_t ifin[MAX_NB_RX_QUEUE];
> +       odp_pktout_queue_t ifout[MAX_NB_TX_QUEUE];
> +       uint32_t nb_rxq;
> +       uint32_t nb_txq;
> +};
> +
> +struct l3fwd_qconf_s {
> +       uint8_t if_idx;         /* port index */
> +       uint8_t rxq_idx;        /* recv queue index in a port */
> +       uint8_t core_idx;       /* this core should handle traffic */
> +};
> +
> +struct thread_arg_s {
> +       int cpu;                /* initialized as -1 */
> +       struct l3fwd_qconf_s *qconf_args[MAX_NB_QCONF_PER_CORE];
> +       int count;
> +};
> +
> +static struct l3fwd_qconf_s qconf_config_default[] = {
> +       {0, 0, 1},
> +       {1, 0, 2},
> +       {2, 0, 3},
> +       {3, 0, 4},
> +       {0, 1, 5},
> +       {1, 1, 6},
> +       {2, 1, 7},
> +       {3, 1, 0},
> +};
> +
> +typedef struct {
> +       char *if_names[MAX_NB_PKTIO];
> +       int if_count;
> +       char *route_str[MAX_NB_ROUTE];
> +       int worker_count;
> +       struct l3fwd_qconf_s qconf_config[MAX_NB_QCONFS];
> +       int qconf_count;
> +} app_args_t;
> +
> +struct {
> +       app_args_t              cmd_args;
> +       struct l3fwd_pktio_s    l3fwd_pktios[MAX_NB_PKTIO];
> +       odph_linux_pthread_t    l3fwd_workers[MAX_NB_WORKER];

please use odph_odpthread_t and the set of odph_odpthread* function instead

> +       struct thread_arg_s     worker_args[MAX_NB_WORKER];
> +       uint32_t                nb_pktio;  /* effective pktios */
> +       uint32_t                nb_worker; /* effective workers */
> +} global;
> +
> +static void print_usage(char *progname);
> +static void print_info(char *progname, app_args_t *args);
> +static void parse_cmdline_args(int argc, char *argv[], app_args_t *args);
> +static int parse_config(char *cfg_str, app_args_t *args);
> +static inline void setup_worker_qconf(app_args_t *args);
> +static int split_string(char *str, int stringlen,
> +                       char **tokens, int maxtokens, char delim);
> +
> +static odp_pktio_t create_pktio(const char *name, odp_pool_t pool,
> +                               struct l3fwd_pktio_s *fwd_pktio)
> +{
> +       odp_pktio_param_t pktio_param;
> +       odp_pktin_queue_param_t in_queue_param;
> +       odp_pktout_queue_param_t out_queue_param;
> +       odp_pktio_t pktio;
> +       odp_pktio_capability_t capa;
> +       struct odp_pktin_queue_t *inq;
> +       struct odp_pktout_queue_t *outq;
> +       int rc, nb_rxq, nb_txq;
> +
> +       odp_pktio_param_init(&pktio_param);
> +
> +       pktio = odp_pktio_open(name, pool, &pktio_param);
> +       if (pktio == ODP_PKTIO_INVALID) {
> +               printf("Failed to open %s\n", name);
> +               exit(1);
> +       }
> +
> +       odp_pktin_queue_param_init(&in_queue_param);
> +       odp_pktout_queue_param_init(&out_queue_param);
> +
> +       /** set rx and tx queue number
> +        *  0 < rxq <= capability: set to rxq, else to rx capa
> +        *  0 < txq <= capability: set to txq, else to tx capa
> +        */
> +       rc = odp_pktio_capability(pktio, &capa);
> +       if (rc) {
> +               printf("pktio %s: unable to read capabilities, "
> +                      "set to 1 rx and 1 tx queue\n", name);
> +               fwd_pktio->nb_rxq = 1;
> +               fwd_pktio->nb_txq = 1;
> +       }
> +
> +       nb_rxq = fwd_pktio->nb_rxq;
> +       nb_txq = fwd_pktio->nb_txq;
> +       if (nb_rxq == 0) {
> +               fwd_pktio->nb_rxq = 1;
> +       } else if (nb_rxq > (int)capa.max_input_queues) {
> +               printf("Error queue config: max queue %d for %s.\n",
> +                      (int)capa.max_input_queues - 1, name);
> +               exit(1);
> +       }
> +
> +       if (nb_txq == 0)
> +               fwd_pktio->nb_txq = 1;
> +       else if (nb_txq > (int)capa.max_output_queues)
> +               fwd_pktio->nb_txq = capa.max_output_queues;
> +
> +       in_queue_param.num_queues = fwd_pktio->nb_rxq;
> +       out_queue_param.num_queues = fwd_pktio->nb_txq;
> +
> +       in_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE;
> +       if (odp_pktin_queue_config(pktio, &in_queue_param)) {
> +               printf("Failed to config input queue for %s\n", name);
> +               exit(1);
> +       }
> +
> +       out_queue_param.op_mode = ODP_PKTIO_OP_MT;
> +       if (odp_pktout_queue_config(pktio, &out_queue_param)) {
> +               printf("Failed to config output queue for %s\n", name);
> +               exit(1);
> +       }
> +
> +       nb_rxq = in_queue_param.num_queues;
> +       nb_txq = out_queue_param.num_queues;
> +       inq = &fwd_pktio->ifin[0];
> +       if (odp_pktin_queue(pktio, inq, nb_rxq) != nb_rxq) {
> +               printf("pktin queue query failed for %s\n", name);
> +               exit(1);
> +       }
> +
> +       outq = &fwd_pktio->ifout[0];
> +       if (odp_pktout_queue(pktio, outq, nb_txq) != nb_txq) {
> +               printf("pktout queue query failed for %s\n", name);
> +               exit(1);
> +       }
> +
> +       fwd_pktio->pktio = pktio;
> +
> +       return pktio;
> +}
> +
> +static inline void l3fwd_one_queue(odp_pktin_queue_t inq,
> +                                  odp_pktout_queue_t def_outq)
> +{
> +       odp_packet_t pkt_tbl[MAX_PKT_BURST];
> +       odp_packet_t pkt_tbl_drop[MAX_PKT_BURST];
> +       uint32_t pkts, i;
> +       int need_to_drop = 0;
> +       int ret;
> +
> +       pkts = odp_pktin_recv(inq, pkt_tbl, MAX_PKT_BURST);
> +       if (odp_unlikely(pkts <= 0))
> +               return;
> +
> +       for (i = 0; i < pkts; i++) {
> +               odp_packet_t pkt = pkt_tbl[i];
> +               odph_ethhdr_t *eth;
> +               odph_ipv4hdr_t *ip;
> +               odph_udphdr_t  *udp;
> +               fwd_db_entry_t *entry;
> +               ipv4_tuple5_t key;
> +               uint32_t len;
> +
> +               if (odp_unlikely(!odp_packet_has_ipv4(pkt) ||
> +                                !odp_packet_has_eth(pkt))) {
> +                       printf("warning: invalid ipv4 or eth header\n");
> +                       pkt_tbl_drop[need_to_drop] = pkt;
> +                       need_to_drop++;
> +                       continue;
> +               }
> +
> +               ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &len);
> +               key.dst_ip = odp_be_to_cpu_32(ip->dst_addr);
> +               key.src_ip = odp_be_to_cpu_32(ip->src_addr);
> +               key.proto = ip->proto;
> +               if (odp_packet_has_udp(pkt) ||
> +                   odp_packet_has_tcp(pkt)) {
> +                       /* UDP or TCP*/
> +                       void *ptr = odp_packet_l4_ptr(pkt, NULL);
> +
> +                       udp = (odph_udphdr_t *)ptr;
> +                       key.src_port = odp_be_to_cpu_16(udp->src_port);
> +                       key.dst_port = odp_be_to_cpu_16(udp->dst_port);
> +               } else {
> +                       key.src_port = 0;
> +                       key.dst_port = 0;
> +               }
> +
> +               entry = find_fwd_db_entry(&key);
> +               ip->ttl--;
> +               ip->chksum = odph_ipv4_csum_update(pkt);
> +               eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL);
> +               if (entry) {
> +                       memcpy(eth->src.addr, entry->src_mac, ODPH_ETHADDR_LEN);
> +                       memcpy(eth->dst.addr, entry->dst_mac, ODPH_ETHADDR_LEN);
> +                       ret = odp_pktout_send(entry->outq, &pkt, 1);
> +               } else {
> +                       /* on route, reflect the packet */
> +                       memcpy(eth->dst.addr, eth->src.addr, ODPH_ETHADDR_LEN);
> +                       ret = odp_pktout_send(def_outq, &pkt, 1);
> +               }
> +               if (ret < 1) {
> +                       pkt_tbl_drop[need_to_drop] = pkt;
> +                       need_to_drop++;
> +               }
> +       }
> +
> +       odp_packet_free_multi(pkt_tbl_drop, need_to_drop);
> +}
> +
> +static void *run_worker(void *arg)
> +{
> +       struct thread_arg_s *args = arg;
> +
> +       for (;;) {
> +               int i;
> +               struct l3fwd_pktio_s *port;
> +               struct l3fwd_qconf_s *qconf;
> +
> +               for (i = 0; i < args->count; i++) {
> +                       qconf = args->qconf_args[i];
> +                       if (args->cpu != qconf->core_idx)
> +                               continue;
> +
> +                       port = &global.l3fwd_pktios[qconf->if_idx];
> +                       l3fwd_one_queue(port->ifin[qconf->rxq_idx],
> +                                       port->ifout[0]);
> +               }
> +       }
> +
> +       return NULL;
> +}
> +
> +int main(int argc, char **argv)
> +{
> +       odp_pool_t pool;
> +       odp_pool_param_t params;
> +       odp_instance_t instance;
> +       odph_linux_thr_params_t thr_params;
> +       int cpu;
> +       uint32_t i, nb_worker;
> +       uint8_t mac[ODPH_ETHADDR_LEN];
> +       app_args_t *args;
> +
> +       if (odp_init_global(&instance, NULL, NULL)) {
> +               printf("Error: ODP global init failed.\n");
> +               exit(1);
> +       }
> +
> +       if (odp_init_local(instance, ODP_THREAD_CONTROL)) {
> +               printf("Error: ODP local init failed.\n");
> +               exit(1);
> +       }
> +
> +       /* Clear global argument */
> +       memset(&global, 0, sizeof(global));
> +
> +       /* Parse cmdline arguments */
> +       args = &global.cmd_args;
> +       parse_cmdline_args(argc, argv, args);
> +
> +       /* Distribute the receive queues by core */
> +       setup_worker_qconf(args);
> +
> +       /* Init l3fwd tale */
> +       init_fwd_db();
> +
> +       /* Add route into table */
> +       for (i = 0; i < MAX_NB_ROUTE; i++) {
> +               if (args->route_str[i])
> +                       create_fwd_db_entry(args->route_str[i]);
> +       }
> +
> +       print_info(NO_PATH(argv[0]), args);
> +
> +       /* Create packet pool */
> +       odp_pool_param_init(&params);
> +       params.pkt.seg_len = POOL_SEG_LEN;
> +       params.pkt.len     = POOL_SEG_LEN;
> +       params.pkt.num     = POOL_NUM_PKT;
> +       params.type        = ODP_POOL_PACKET;
> +
> +       pool = odp_pool_create("packet pool", &params);
> +
> +       if (pool == ODP_POOL_INVALID) {
> +               printf("Error: packet pool create failed.\n");
> +               exit(1);
> +       }
> +
> +       global.nb_pktio = args->if_count;
> +       for (i = 0; i < global.nb_pktio; i++) {
> +               struct l3fwd_pktio_s *port;
> +               char *if_name;
> +               char buf[16];
> +               int j;
> +
> +               if_name = args->if_names[i];
> +               port = &global.l3fwd_pktios[i];
> +               port->pktio = create_pktio(if_name, pool, port);
> +               odp_pktio_mac_addr(port->pktio, mac, ODPH_ETHADDR_LEN);
> +               resolve_fwd_db(if_name, port->ifout[0], mac);
> +
> +               /* print mac string, could be used to config pktgen */
> +               sprintf(buf, "%02X:%02X:%02X:%02X:%02X:%02X",
> +                       mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
> +               printf("create pktio %s, mac %s\n", if_name, buf);
> +
> +               /* start pktio */
> +               if (odp_pktio_start(port->pktio)) {
> +                       printf("unable to start pktio: %s\n", if_name);
> +                       exit(1);
> +               }
> +
> +               printf("start pktio: %s, rxq: ", if_name);
> +               for (j = 0; j < args->qconf_count; j++) {
> +                       struct l3fwd_qconf_s *q;
> +
> +                       q = &args->qconf_config[j];
> +                       if (q->if_idx == i)
> +                               printf("(%d, %d, %d) ", q->if_idx, q->rxq_idx,
> +                                      q->core_idx);
> +               }
> +               printf("\n");
> +       }
> +
> +       dump_fwd_db();
> +
> +       /* one worker thread per core.
> +        * if no thread count in cmdline, all cpus are used,
> +        * else create threads as the cmdline.
> +        */
> +       cpu = odp_cpu_count();
> +       if (args->worker_count == 0 || args->worker_count > cpu)
> +               args->worker_count = cpu;
> +
> +       global.nb_worker = args->worker_count;
> +       memset(&thr_params, 0, sizeof(thr_params));
> +       thr_params.start    = run_worker;
> +       thr_params.thr_type = ODP_THREAD_WORKER;
> +       thr_params.instance = instance;
> +
> +       nb_worker = 0;
> +       cpu = odp_cpu_id();
> +       for (i = 0; i < MAX_NB_WORKER; i++) {
> +               struct thread_arg_s *arg;
> +               odp_cpumask_t thr_mask;
> +
> +               /* not create thread for current and invalid cpu */
> +               arg = &global.worker_args[i];
> +               if (arg->cpu == cpu || arg->cpu == CPU_ID_INVALID)
> +                       continue;
> +
> +               odp_cpumask_zero(&thr_mask);
> +               odp_cpumask_set(&thr_mask, arg->cpu);
> +               thr_params.arg = arg;
> +               odph_linux_pthread_create(&global.l3fwd_workers[i],

please use odph_odpthreads_create()

&thr_mask,
> +                                         &thr_params);
> +               nb_worker++;
> +               if (nb_worker == global.nb_worker)
> +                       break;
> +       }
> +
> +       /* if other cores could run the tasks, not run on this cpu */
> +       if (nb_worker < global.nb_worker)
> +               run_worker(&global.l3fwd_workers[cpu]);
> +
> +       /* wait for other threads to join */
> +       for (i = 0; i < MAX_NB_WORKER; i++) {
> +               odph_linux_pthread_t *arg;
> +
> +               arg = &global.l3fwd_workers[i];
> +               if (arg->cpu == cpu || arg->cpu == CPU_ID_INVALID)
> +                       continue;
> +
> +               odph_linux_pthread_join(arg, 1);

please use odph_odpthreads_join()


Christophe.

> +       }
> +
> +       return 0;
> +}
> +
> +static void print_usage(char *progname)
> +{
> +       printf("\n"
> +              "ODP L3 forwarding application.\n"
> +              "\n"
> +              "Usage: %s OPTIONS\n"
> +              "  E.g. %s -i eth0,eth1 -r 1.1.1.0/24:eth0 -r 2.2.2.0/24:eth1\n"
> +              " In the above example,\n"
> +              " eth0 will send pkts to eth1 and vice versa\n"
> +              "\n"
> +              "Mandatory OPTIONS:\n"
> +              "  -i, --interface eth interfaces (comma-separated, no spaces)\n"
> +              "  -r, --route SubNet:Intf[:NextHopMAC]\n"
> +              "        NextHopMAC can be optional, in this case, zeroed mac\n"
> +              "\n"
> +              "Optional OPTIONS:\n"
> +              "  -t, --thread Number of threads to do forwarding\n"
> +              "        optional, default as cpu count\n"
> +              "  -q, --queue  Configure rx queue(s) for port\n"
> +              "        optional, format: [(port, queue, core),...]\n"
> +              "        for example: -q '(0, 0, 1),(1,0,2)'\n"
> +              "  -h, --help   Display help and exit.\n\n"
> +              "\n", NO_PATH(progname), NO_PATH(progname)
> +           );
> +}
> +
> +static void parse_cmdline_args(int argc, char *argv[], app_args_t *args)
> +{
> +       int opt;
> +       int long_index;
> +       char *token, *local;
> +       size_t len, route_index = 0;
> +       int i, mem_failure = 0;
> +
> +       static struct option longopts[] = {
> +               {"interface", required_argument, NULL, 'i'},    /* return 'i' */
> +               {"route", required_argument, NULL, 'r'},        /* return 'r' */
> +               {"thread", optional_argument, NULL, 't'},       /* return 't' */
> +               {"queue", optional_argument, NULL, 'q'},        /* return 'q' */
> +               {"help", no_argument, NULL, 'h'},               /* return 'h' */
> +               {NULL, 0, NULL, 0}
> +       };
> +
> +       while (1) {
> +               opt = getopt_long(argc, argv, "+t:i:r:q:h",
> +                                 longopts, &long_index);
> +
> +               if (opt == -1)
> +                       break;  /* No more options */
> +
> +               switch (opt) {
> +               /* parse number of worker threads to be run*/
> +               case 't':
> +                       i = odp_cpu_count();
> +                       args->worker_count = atoi(optarg);
> +                       if (args->worker_count > i) {
> +                               printf("Too many threads,"
> +                                      "truncate to cpu count: %d\n", i);
> +                               args->worker_count = i;
> +                       }
> +
> +                       break;
> +               /* parse packet-io interface names */
> +               case 'i':
> +                       len = strlen(optarg);
> +                       if (len == 0) {
> +                               print_usage(argv[0]);
> +                               exit(EXIT_FAILURE);
> +                       }
> +                       len += 1;       /* add room for '\0' */
> +
> +                       local = malloc(len);
> +                       if (!local) {
> +                               print_usage(argv[0]);
> +                               exit(EXIT_FAILURE);
> +                       }
> +
> +                       /* count the number of tokens separated by ',' */
> +                       strcpy(local, optarg);
> +                       for (token = strtok(local, ","), i = 0;
> +                            token != NULL;
> +                            token = strtok(NULL, ","), i++)
> +                               ;
> +
> +                       if (i == 0) {
> +                               print_usage(argv[0]);
> +                               exit(EXIT_FAILURE);
> +                       } else if (i > MAX_NB_PKTIO) {
> +                               printf("too many ports specified, "
> +                                      "truncated to %d", MAX_NB_PKTIO);
> +                       }
> +                       args->if_count = i;
> +
> +                       /* store the if names (reset names string) */
> +                       strcpy(local, optarg);
> +                       for (token = strtok(local, ","), i = 0;
> +                            token != NULL; token = strtok(NULL, ","), i++) {
> +                               args->if_names[i] = token;
> +                       }
> +                       break;
> +
> +               /*Configure Route in forwarding database*/
> +               case 'r':
> +                       if (route_index >= MAX_NB_ROUTE) {
> +                               printf("No more routes can be added\n");
> +                               break;
> +                       }
> +                       local = calloc(1, strlen(optarg) + 1);
> +                       if (!local) {
> +                               mem_failure = 1;
> +                               break;
> +                       }
> +                       memcpy(local, optarg, strlen(optarg));
> +                       local[strlen(optarg)] = '\0';
> +                       args->route_str[route_index++] = local;
> +                       break;
> +
> +               case 'h':
> +                       print_usage(argv[0]);
> +                       exit(EXIT_SUCCESS);
> +                       break;
> +
> +               case 'q':
> +                       parse_config(optarg, args);
> +                       break;
> +
> +               default:
> +                       break;
> +               }
> +       }
> +
> +       /* checking arguments */
> +       if (args->if_count == 0) {
> +               printf("\nNo option -i specified.\n");
> +               goto out;
> +       }
> +
> +       if (args->route_str[0] == NULL) {
> +               printf("\nNo option -r specified.\n");
> +               goto out;
> +       }
> +
> +       if (mem_failure == 1) {
> +               printf("\nAllocate memory failure.\n");
> +               goto out;
> +       }
> +       optind = 1;             /* reset 'extern optind' from the getopt lib */
> +       return;
> +
> +out:
> +       print_usage(argv[0]);
> +       exit(EXIT_FAILURE);
> +}
> +
> +/* split string into tokens */
> +int split_string(char *str, int stringlen,
> +                char **tokens, int maxtokens, char delim)
> +{
> +       int i, tok = 0;
> +       int tokstart = 1; /* first token is right at start of string */
> +
> +       if (str == NULL || tokens == NULL)
> +               goto einval_error;
> +
> +       for (i = 0; i < stringlen; i++) {
> +               if (str[i] == '\0' || tok >= maxtokens)
> +                       break;
> +               if (tokstart) {
> +                       tokstart = 0;
> +                       tokens[tok++] = &str[i];
> +               }
> +               if (str[i] == delim) {
> +                       str[i] = '\0';
> +                       tokstart = 1;
> +               }
> +       }
> +       return tok;
> +
> +einval_error:
> +       errno = EINVAL;
> +       return -1;
> +}
> +
> +static int parse_config(char *cfg_str, app_args_t *args)
> +{
> +       char s[256];
> +       const char *p, *p0 = cfg_str;
> +       char *end;
> +       enum fieldnames {
> +               FLD_PORT = 0,
> +               FLD_QUEUE,
> +               FLD_LCORE,
> +               FLD_LAST
> +       };
> +       unsigned long int_fld[FLD_LAST];
> +       char *str_fld[FLD_LAST];
> +       int i;
> +       unsigned size;
> +       int nb_qconfs = 0;
> +       struct l3fwd_qconf_s *qconf_array = &args->qconf_config[0];
> +
> +       p = strchr(p0, '(');
> +       while (p != NULL) {
> +               ++p;
> +               p0 = strchr(p, ')');
> +               if (p0 == NULL)
> +                       return -1;
> +
> +               size = p0 - p;
> +               if (size >= sizeof(s))
> +                       return -1;
> +
> +               snprintf(s, sizeof(s), "%.*s", size, p);
> +               i = split_string(s, sizeof(s), str_fld, FLD_LAST, ',');
> +               if (i != FLD_LAST)
> +                       return -1;
> +               for (i = 0; i < FLD_LAST; i++) {
> +                       errno = 0;
> +                       int_fld[i] = strtoul(str_fld[i], &end, 0);
> +                       if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
> +                               return -1;
> +               }
> +               if (nb_qconfs >= MAX_NB_QCONFS) {
> +                       printf("exceeded max number of queue params: %hu\n",
> +                              nb_qconfs);
> +                       return -1;
> +               }
> +               qconf_array[nb_qconfs].if_idx = (uint8_t)int_fld[FLD_PORT];
> +               qconf_array[nb_qconfs].rxq_idx = (uint8_t)int_fld[FLD_QUEUE];
> +               qconf_array[nb_qconfs].core_idx = (uint8_t)int_fld[FLD_LCORE];
> +               ++nb_qconfs;
> +       }
> +
> +       args->qconf_count = nb_qconfs;
> +
> +       return 0;
> +}
> +
> +static void print_info(char *progname, app_args_t *args)
> +{
> +       int i;
> +
> +       printf("\n"
> +              "ODP system info\n"
> +              "---------------\n"
> +              "ODP API version: %s\n"
> +              "ODP impl name:   %s\n"
> +              "CPU model:       %s\n"
> +              "CPU freq (hz):   %" PRIu64 "\n"
> +              "Cache line size: %i\n"
> +              "CPU count:       %i\n"
> +              "\n",
> +              odp_version_api_str(), odp_version_impl_name(),
> +              odp_cpu_model_str(), odp_cpu_hz_max(),
> +              odp_sys_cache_line_size(), odp_cpu_count());
> +
> +       printf("Running ODP appl: \"%s\"\n"
> +              "-----------------\n"
> +              "IF-count:        %i\n"
> +              "Using IFs:      ",
> +              progname, args->if_count);
> +
> +       for (i = 0; i < args->if_count; ++i)
> +               printf(" %s", args->if_names[i]);
> +
> +       printf("\n\n");
> +       fflush(NULL);
> +}
> +
> +static void setup_worker_qconf(app_args_t *args)
> +{
> +       int i;
> +       int cpu_count;
> +       struct l3fwd_qconf_s *q;
> +       struct thread_arg_s *arg;
> +       struct l3fwd_pktio_s *port;
> +       uint8_t queue_mask[MAX_NB_PKTIO][MAX_NB_RX_QUEUE];
> +
> +       cpu_count = odp_cpu_count();
> +
> +       /* initialise the worker cpu as -1 */
> +       for (i = 0; i < MAX_NB_WORKER; i++) {
> +               arg = &global.worker_args[i];
> +               arg->cpu = CPU_ID_INVALID;
> +       }
> +       memset(&queue_mask[0][0], 0, sizeof(queue_mask));
> +
> +       /* if no queue config specified, use default */
> +       if (args->qconf_count == 0) {
> +               int len, tmp;
> +
> +               tmp = cpu_count < args->if_count ? cpu_count : args->if_count;
> +               len = tmp * sizeof(qconf_config_default[0]);
> +               memcpy(&args->qconf_config[0], &qconf_config_default[0], len);
> +               args->qconf_count = tmp;
> +       }
> +
> +       for (i = 0; i < args->qconf_count; i++) {
> +               q = &args->qconf_config[i];
> +               if (q->core_idx >= cpu_count || q->if_idx >= args->if_count) {
> +                       printf("Error queue config: max port: %d, "
> +                              "max core: %d\n", args->if_count - 1,
> +                              cpu_count - 1);
> +                       exit(1);
> +               }
> +
> +               /* check if one queue is configured twice or more */
> +               if (queue_mask[q->if_idx][q->rxq_idx]) {
> +                       printf("Error queue config: re-config port: %d, "
> +                              "queue: %d\n", q->if_idx, q->rxq_idx);
> +                       exit(1);
> +               }
> +               queue_mask[q->if_idx][q->rxq_idx] = 1;
> +
> +               port = &global.l3fwd_pktios[q->if_idx];
> +               if (port->nb_rxq <= q->rxq_idx)
> +                       port->nb_rxq = q->rxq_idx + 1;
> +
> +               /* put the queue into worker_args */
> +               arg = &global.worker_args[q->core_idx];
> +               arg->qconf_args[arg->count] = q;
> +               arg->count++;
> +               arg->cpu = q->core_idx;
> +       }
> +}
> diff --git a/example/l3fwd/odp_l3fwd_db.c b/example/l3fwd/odp_l3fwd_db.c
> new file mode 100644
> index 0000000..4767a9b
> --- /dev/null
> +++ b/example/l3fwd/odp_l3fwd_db.c
> @@ -0,0 +1,381 @@
> +/* Copyright (c) 2016, Linaro Limited
> + * All rights reserved.
> + *
> + * SPDX-License-Identifier:     BSD-3-Clause
> + */
> +
> +/* enable strtok */
> +#ifndef _GNU_SOURCE
> +#define _GNU_SOURCE
> +#endif
> +
> +#include <stdlib.h>
> +#include <string.h>
> +
> +#include <example_debug.h>
> +
> +#include <odp_api.h>
> +
> +#include <odp_l3fwd_db.h>
> +#include <jhash.h>
> +
> +/**
> + * Compute hash value from a flow
> + */
> +static inline
> +uint64_t l3fwd_calc_hash(ipv4_tuple5_t *key)
> +{
> +       uint64_t l4_ports = 0;
> +       uint32_t dst_ip, src_ip;
> +
> +       src_ip = key->src_ip;
> +       dst_ip = key->dst_ip + JHASH_GOLDEN_RATIO;
> +       FWD_BJ3_MIX(src_ip, dst_ip, l4_ports);
> +
> +       return l4_ports;
> +}
> +
> +/**
> + * Parse text string representing an IPv4 address or subnet
> + *
> + * String is of the format "XXX.XXX.XXX.XXX(/W)" where
> + * "XXX" is decimal value and "/W" is optional subnet length
> + *
> + * @param ipaddress  Pointer to IP address/subnet string to convert
> + * @param addr       Pointer to return IPv4 address
> + * @param mask       Pointer (optional) to return IPv4 mask
> + *
> + * @return 0 if successful else -1
> + */
> +static inline
> +int parse_ipv4_string(char *ipaddress, uint32_t *addr, uint32_t *mask)
> +{
> +       int b[4];
> +       int qualifier = 32;
> +       int converted;
> +
> +       if (strchr(ipaddress, '/')) {
> +               converted = sscanf(ipaddress, "%d.%d.%d.%d/%d",
> +                                  &b[3], &b[2], &b[1], &b[0],
> +                                  &qualifier);
> +               if (5 != converted)
> +                       return -1;
> +       } else {
> +               converted = sscanf(ipaddress, "%d.%d.%d.%d",
> +                                  &b[3], &b[2], &b[1], &b[0]);
> +               if (4 != converted)
> +                       return -1;
> +       }
> +
> +       if ((b[0] > 255) || (b[1] > 255) || (b[2] > 255) || (b[3] > 255))
> +               return -1;
> +       if (!qualifier || (qualifier > 32))
> +               return -1;
> +
> +       *addr = b[0] | b[1] << 8 | b[2] << 16 | b[3] << 24;
> +       if (mask)
> +               *mask = ~(0xFFFFFFFF & ((1ULL << (32 - qualifier)) - 1));
> +
> +       return 0;
> +}
> +
> +/**
> + * Generate text string representing IPv4 range/subnet, output
> + * in "XXX.XXX.XXX.XXX/W" format
> + *
> + * @param b     Pointer to buffer to store string
> + * @param range Pointer to IPv4 address range
> + *
> + * @return Pointer to supplied buffer
> + */
> +static inline
> +char *ipv4_subnet_str(char *b, ip_addr_range_t *range)
> +{
> +       int idx;
> +       int len;
> +
> +       for (idx = 0; idx < 32; idx++)
> +               if (range->mask & (1 << idx))
> +                       break;
> +       len = 32 - idx;
> +
> +       sprintf(b, "%03d.%03d.%03d.%03d/%d",
> +               0xFF & ((range->addr) >> 24),
> +               0xFF & ((range->addr) >> 16),
> +               0xFF & ((range->addr) >>  8),
> +               0xFF & ((range->addr) >>  0),
> +               len);
> +       return b;
> +}
> +
> +/**
> + * Generate text string representing MAC address
> + *
> + * @param b     Pointer to buffer to store string
> + * @param mac   Pointer to MAC address
> + *
> + * @return Pointer to supplied buffer
> + */
> +static inline
> +char *mac_addr_str(char *b, uint8_t *mac)
> +{
> +       sprintf(b, "%02X.%02X.%02X.%02X.%02X.%02X",
> +               mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
> +       return b;
> +}
> +
> +/**
> + * Flow cache table entry
> + */
> +typedef struct flow_entry_s {
> +       ipv4_tuple5_t key;              /**< match key */
> +       struct flow_entry_s *next;      /**< next entry on the list */
> +       fwd_db_entry_t *fwd_entry;      /**< entry info in db */
> +} flow_entry_t;
> +
> +/**
> + * Flow cache table bucket
> + */
> +typedef struct flow_bucket_s {
> +       odp_spinlock_t          lock;   /**< Bucket lock*/
> +       flow_entry_t            *next;  /**< Pointer to first flow entry in bucket*/
> +} flow_bucket_t;
> +
> +/**
> + * Flow hash table, fast lookup cache
> + */
> +typedef struct flow_table_s {
> +       flow_bucket_t *bucket;
> +       uint32_t count;
> +} flow_table_t;
> +
> +static flow_table_t fwd_lookup_cache;
> +
> +static inline
> +void init_fwd_cache(void)
> +{
> +       odp_shm_t               hash_shm;
> +       flow_bucket_t           *bucket;
> +       uint32_t                bucket_count;
> +       uint32_t                i;
> +
> +       bucket_count = FWD_DEF_BUCKET_COUNT;
> +
> +       /*Reserve memory for Routing hash table*/
> +       hash_shm = odp_shm_reserve("route_table",
> +                                  sizeof(flow_bucket_t) * bucket_count,
> +                                  ODP_CACHE_LINE_SIZE, 0);
> +
> +       bucket = odp_shm_addr(hash_shm);
> +       if (!bucket) {
> +               EXAMPLE_ERR("Error: shared mem alloc failed.\n");
> +               exit(-1);
> +       }
> +
> +       fwd_lookup_cache.bucket = bucket;
> +       fwd_lookup_cache.count = bucket_count;
> +
> +       /*Initialize Locks*/
> +       for (i = 0; i < bucket_count; i++) {
> +               bucket = &fwd_lookup_cache.bucket[i];
> +               odp_spinlock_init(&bucket->lock);
> +               bucket->next = NULL;
> +       }
> +}
> +
> +static inline
> +int match_key_flow(ipv4_tuple5_t *key, flow_entry_t *flow)
> +{
> +       if (key->src_ip == flow->key.src_ip &&
> +           key->dst_ip == flow->key.dst_ip &&
> +           key->src_port == flow->key.src_port &&
> +           key->dst_port == flow->key.dst_port &&
> +           key->proto == flow->key.proto)
> +               return 1;
> +
> +       return 0;
> +}
> +
> +static inline
> +flow_entry_t *lookup_fwd_cache(ipv4_tuple5_t *key, flow_bucket_t *bucket)
> +{
> +       flow_entry_t *rst;
> +
> +       for (rst = bucket->next; rst != NULL; rst = rst->next) {
> +               if (match_key_flow(key, rst))
> +                       break;
> +       }
> +
> +       return rst;
> +}
> +
> +static inline
> +flow_entry_t *insert_fwd_cache(ipv4_tuple5_t *key,
> +                              flow_bucket_t *bucket,
> +                              fwd_db_entry_t *entry)
> +{
> +       flow_entry_t *flow;
> +
> +       flow = lookup_fwd_cache(key, bucket);
> +       if (flow)
> +               return flow;
> +
> +       flow = malloc(sizeof(flow_entry_t));
> +       flow->key = *key;
> +       flow->fwd_entry = entry;
> +
> +       odp_spinlock_lock(&bucket->lock);
> +       if (!bucket->next) {
> +               bucket->next = flow;
> +       } else {
> +               flow->next = bucket->next;
> +               bucket->next = flow;
> +       }
> +       odp_spinlock_unlock(&bucket->lock);
> +
> +       return flow;
> +}
> +
> +/** Global pointer to fwd db */
> +fwd_db_t *fwd_db;
> +
> +void init_fwd_db(void)
> +{
> +       odp_shm_t shm;
> +
> +       shm = odp_shm_reserve("shm_fwd_db",
> +                             sizeof(fwd_db_t),
> +                             ODP_CACHE_LINE_SIZE,
> +                             0);
> +
> +       fwd_db = odp_shm_addr(shm);
> +
> +       if (fwd_db == NULL) {
> +               EXAMPLE_ERR("Error: shared mem alloc failed.\n");
> +               exit(EXIT_FAILURE);
> +       }
> +       memset(fwd_db, 0, sizeof(*fwd_db));
> +
> +       init_fwd_cache();
> +}
> +
> +int create_fwd_db_entry(char *input)
> +{
> +       int pos = 0;
> +       char *local;
> +       char *str;
> +       char *save;
> +       char *token;
> +       fwd_db_entry_t *entry = &fwd_db->array[fwd_db->index];
> +
> +       /* Verify we haven't run out of space */
> +       if (MAX_DB <= fwd_db->index)
> +               return -1;
> +
> +       /* Make a local copy */
> +       local = malloc(strlen(input) + 1);
> +       if (NULL == local)
> +               return -1;
> +       strcpy(local, input);
> +
> +       /* Setup for using "strtok_r" to search input string */
> +       str = local;
> +       save = NULL;
> +
> +       /* Parse tokens separated by ':' */
> +       while (NULL != (token = strtok_r(str, ":", &save))) {
> +               str = NULL;  /* reset str for subsequent strtok_r calls */
> +
> +               /* Parse token based on its position */
> +               switch (pos) {
> +               case 0:
> +                       parse_ipv4_string(token,
> +                                         &entry->subnet.addr,
> +                                         &entry->subnet.mask);
> +                       break;
> +               case 1:
> +                       strncpy(entry->oif, token, OIF_LEN - 1);
> +                       entry->oif[OIF_LEN - 1] = 0;
> +                       break;
> +               case 2:
> +                       odph_eth_addr_parse((odph_ethaddr_t *)entry->dst_mac,
> +                                           token);
> +                       break;
> +
> +               default:
> +                       printf("ERROR: extra token \"%s\" at position %d\n",
> +                              token, pos);
> +                       break;
> +               }
> +
> +               /* Advance to next position */
> +               pos++;
> +       }
> +
> +       /* Add route to the list */
> +       fwd_db->index++;
> +       entry->next = fwd_db->list;
> +       fwd_db->list = entry;
> +
> +       free(local);
> +       return 0;
> +}
> +
> +void resolve_fwd_db(char *intf, odp_pktout_queue_t outq, uint8_t *mac)
> +{
> +       fwd_db_entry_t *entry;
> +
> +       /* Walk the list and attempt to set output and MAC */
> +       for (entry = fwd_db->list; NULL != entry; entry = entry->next) {
> +               if (strcmp(intf, entry->oif))
> +                       continue;
> +
> +               entry->outq = outq;
> +               memcpy(entry->src_mac, mac, ODPH_ETHADDR_LEN);
> +       }
> +}
> +
> +void dump_fwd_db_entry(fwd_db_entry_t *entry)
> +{
> +       char subnet_str[MAX_STRING];
> +       char mac_str[MAX_STRING];
> +
> +       printf(" %s %s %s\n",
> +              ipv4_subnet_str(subnet_str, &entry->subnet),
> +              entry->oif,
> +              mac_addr_str(mac_str, entry->dst_mac));
> +}
> +
> +void dump_fwd_db(void)
> +{
> +       fwd_db_entry_t *entry;
> +
> +       printf("\n"
> +              "Routing table\n"
> +              "-------------\n");
> +
> +       for (entry = fwd_db->list; NULL != entry; entry = entry->next)
> +               dump_fwd_db_entry(entry);
> +}
> +
> +fwd_db_entry_t *find_fwd_db_entry(ipv4_tuple5_t *key)
> +{
> +       fwd_db_entry_t *entry;
> +       flow_entry_t *flow;
> +       flow_bucket_t *bucket;
> +       uint64_t hash;
> +
> +       /* first find in cache */
> +       hash = l3fwd_calc_hash(key);
> +       hash &= fwd_lookup_cache.count - 1;
> +       bucket = &fwd_lookup_cache.bucket[hash];
> +       flow = lookup_fwd_cache(key, bucket);
> +       if (flow)
> +               return flow->fwd_entry;
> +
> +       for (entry = fwd_db->list; NULL != entry; entry = entry->next)
> +               if (entry->subnet.addr == (key->dst_ip & entry->subnet.mask))
> +                       break;
> +
> +       return entry;
> +}
> diff --git a/example/l3fwd/odp_l3fwd_db.h b/example/l3fwd/odp_l3fwd_db.h
> new file mode 100644
> index 0000000..eac142e
> --- /dev/null
> +++ b/example/l3fwd/odp_l3fwd_db.h
> @@ -0,0 +1,122 @@
> +/* Copyright (c) 2016, Linaro Limited
> + * All rights reserved.
> + *
> + * SPDX-License-Identifier:     BSD-3-Clause
> + */
> +
> +#ifndef ODP_L3FWD_DB_H_
> +#define ODP_L3FWD_DB_H_
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <odp_api.h>
> +#include <odp/helper/eth.h>
> +
> +#define OIF_LEN 32
> +#define MAX_DB  32
> +#define MAX_STRING  32
> +
> +/**
> + * Default number of flows
> + */
> +#define FWD_DEF_FLOW_COUNT             100000
> +
> +/**
> + * Default Hash bucket number
> + */
> +#define FWD_DEF_BUCKET_COUNT   (FWD_DEF_FLOW_COUNT / 8)
> +
> +/**
> + * IP address range (subnet)
> + */
> +typedef struct ip_addr_range_s {
> +       uint32_t  addr;     /**< IP address */
> +       uint32_t  mask;     /**< mask, 1 indicates bits are valid */
> +} ip_addr_range_t;
> +
> +/**
> + * TCP/UDP flow
> + */
> +typedef struct ipv4_tuple5_s {
> +       uint32_t src_ip;
> +       uint32_t dst_ip;
> +       uint16_t src_port;
> +       uint16_t dst_port;
> +       uint8_t  proto;
> +} ipv4_tuple5_t;
> +
> +/**
> + * Forwarding data base entry
> + */
> +typedef struct fwd_db_entry_s {
> +       struct fwd_db_entry_s *next;          /**< Next entry on list */
> +       char                    oif[OIF_LEN]; /**< Output interface name */
> +       odp_pktout_queue_t      outq;         /**< Output transmit queue */
> +       uint8_t   src_mac[ODPH_ETHADDR_LEN];  /**< Output source MAC */
> +       uint8_t   dst_mac[ODPH_ETHADDR_LEN];  /**< Output destination MAC */
> +       ip_addr_range_t        subnet;        /**< Subnet for this router */
> +} fwd_db_entry_t;
> +
> +/**
> + * Forwarding data base
> + */
> +typedef struct fwd_db_s {
> +       uint32_t          index;          /**< Next available entry */
> +       fwd_db_entry_t   *list;           /**< List of active routes */
> +       fwd_db_entry_t    array[MAX_DB];  /**< Entry storage */
> +} fwd_db_t;
> +
> +/** Global pointer to fwd db */
> +extern fwd_db_t *fwd_db;
> +
> +/** Initialize FWD DB */
> +void init_fwd_db(void);
> +
> +/**
> + * Create a forwarding database entry
> + *
> + * String is of the format "SubNet:Intf:NextHopMAC"
> + *
> + * @param input  Pointer to string describing route
> + *
> + * @return 0 if successful else -1
> + */
> +int create_fwd_db_entry(char *input);
> +
> +/**
> + * Scan FWD DB entries and resolve output queue and source MAC address
> + *
> + * @param intf   Interface name string
> + * @param outq   Output queue for packet transmit
> + * @param mac    MAC address of this interface
> + */
> +void resolve_fwd_db(char *intf, odp_pktout_queue_t outq, uint8_t *mac);
> +
> +/**
> + * Display one forwarding database entry
> + *
> + * @param entry  Pointer to entry to display
> + */
> +void dump_fwd_db_entry(fwd_db_entry_t *entry);
> +
> +/**
> + * Display the forwarding database
> + */
> +void dump_fwd_db(void);
> +
> +/**
> + * Find a matching forwarding database entry
> + *
> + * @param key  ipv4 tuple
> + *
> + * @return pointer to forwarding DB entry else NULL
> + */
> +fwd_db_entry_t *find_fwd_db_entry(ipv4_tuple5_t *key);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif
> diff --git a/example/m4/configure.m4 b/example/m4/configure.m4
> index 9731d81..7868574 100644
> --- a/example/m4/configure.m4
> +++ b/example/m4/configure.m4
> @@ -19,4 +19,5 @@ AC_CONFIG_FILES([example/classifier/Makefile
>                  example/timer/Makefile
>                  example/traffic_mgmt/Makefile
>                  example/l2fwd_simple/Makefile
> +                example/l3fwd/Makefile
>                  example/switch/Makefile])
> --
> 2.1.0.27.g96db324
>
> _______________________________________________
> lng-odp mailing list
> lng-odp@lists.linaro.org
> https://lists.linaro.org/mailman/listinfo/lng-odp
Elo, Matias (Nokia - FI/Espoo) June 7, 2016, 11:17 a.m. | #2
> +	if (nb_txq == 0)
> +		fwd_pktio->nb_txq = 1;
> +	else if (nb_txq > (int)capa.max_output_queues)
> +		fwd_pktio->nb_txq = capa.max_output_queues;
> +
> +	in_queue_param.num_queues = fwd_pktio->nb_rxq;
> +	out_queue_param.num_queues = fwd_pktio->nb_txq;
> +
> +	in_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE;
> +	if (odp_pktin_queue_config(pktio, &in_queue_param)) {
> +		printf("Failed to config input queue for %s\n", name);
> +		exit(1);
> +	}
> +
> +	out_queue_param.op_mode = ODP_PKTIO_OP_MT;
> +	if (odp_pktout_queue_config(pktio, &out_queue_param)) {
> +		printf("Failed to config output queue for %s\n", name);
> +		exit(1);
> +	}

You could use ODP_PKTIO_OP_MT_UNSAFE mode also with the pktout queues if you allocate a private txq per worker for each interface.

I should have time to review the rest of the patch tomorrow morning.

-Matias
Forrest Shi June 12, 2016, 6:35 a.m. | #3
Hi Matias,

Very appreciate your time and review.  See my comments inline.

Thanks,
Forrest

> -----Original Message-----
> From: Elo, Matias (Nokia - FI/Espoo) [mailto:matias.elo@nokia-bell-labs.com]
> Sent: Thursday, June 09, 2016 21:57
> To: Xuelin Shi <forrest.shi@linaro.org>
> Cc: lng-odp@lists.linaro.org
> Subject: RE: [lng-odp][PATCH v3] example: introducing l3fwd
> 
> Hi Forrest,
> 
> On my system the application crashes on start:
> 
> $ sudo ./example/l3fwd/odp_l3fwd -i eth0,eth1 -r 1.1.1.0/24:eth0 -r
> 2.2.2.0/24:eth1 ODP system info
> ---------------
> ODP API version: 1.10.0
> ODP impl name:	 odp-linux
> CPU model:       Intel(R) Xeon(R) CPU E5-2697 v3
> CPU freq (hz):   2600000000
> Cache line size: 64
> CPU count:       28
> 
> Running ODP appl: "odp_l3fwd"
> -----------------
> IF-count:        2
> Using IFs:       eth0 eth1
> 
> pktio/dpdk.c:556:dpdk_open():Invalid dpdk netdev: eth0
> *** buffer overflow detected ***: ./example/l3fwd/odp_l3fwd terminated
> ======= Backtrace: ========= /lib/x86_64-linux-
> gnu/libc.so.6(+0x7338f)[0x7ffff788038f]
> /lib/x86_64-linux-gnu/libc.so.6(__fortify_fail+0x5c)[0x7ffff7917c9c]
> /lib/x86_64-linux-gnu/libc.so.6(+0x109b60)[0x7ffff7916b60]
> /lib/x86_64-linux-gnu/libc.so.6(+0x109069)[0x7ffff7916069]
> /lib/x86_64-linux-gnu/libc.so.6(_IO_default_xsputn+0xbc)[0x7ffff788870c]
> /lib/x86_64-linux-gnu/libc.so.6(_IO_padn+0xa0)[0x7ffff787c6d0]
> /lib/x86_64-linux-gnu/libc.so.6(_IO_vfprintf+0xa8c)[0x7ffff785777c]
> /lib/x86_64-linux-gnu/libc.so.6(__vsprintf_chk+0x84)[0x7ffff79160f4]
> /lib/x86_64-linux-gnu/libc.so.6(__sprintf_chk+0x7d)[0x7ffff791604d]
> ./example/l3fwd/odp_l3fwd[0x40b056]
> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5)[0x7ffff782eec5]
> ./example/l3fwd/odp_l3fwd[0x40bd6b]
> 
> 
> More comments below.
> 
> -Matias
> 
> 
> > diff --git a/example/l3fwd/jhash.h b/example/l3fwd/jhash.h new file
> > mode 100644 index 0000000..d3dbd2a
> > --- /dev/null
> > +++ b/example/l3fwd/jhash.h
> > @@ -0,0 +1,46 @@
> > +/** jhash.h: Jenkins hash support.
> > +  *
> > +  * Copyright (C) 2006 Bob Jenkins (bob_jenkins@burtleburtle.net)
> > +  *
> > +  * http://burtleburtle.net/bob/hash/
> > +  *
> > +  * These are the credits from Bob's sources:
> > +  *
> > +  * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
> > +  *
> > +  * These are functions for producing 32-bit hashes for hash table
lookup.
> > +  * hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and
> > +final()
> > +  * are externally useful functions.  Routines to test the hash are
> > +included
> > +  * if SELF_TEST is defined.  You can use this free for any purpose.
> > +It's in
> > +  * the public domain.  It has no warranty.
> > +  *
> > +  * $FreeBSD$
> > +  */
> > +
> > +#ifndef JHASH_H_
> > +#define JHASH_H_
> > +
> > +#ifdef __cplusplus
> > +extern "C" {
> > +#endif
> > +
> > +/**
> > + * Hash calculation utility
> > + */
> > +#define JHASH_GOLDEN_RATIO	0x9e3779b9
> > +#define rot(x, k) (((x) << (k)) | ((x) >> (32 - (k)))) #define
> > +FWD_BJ3_MIX(a, b, c) \ { \
> > +	a -= c; a ^= rot(c, 4); c += b; \
> > +	b -= a; b ^= rot(a, 6); a += c; \
> > +	c -= b; c ^= rot(b, 8); b += a; \
> > +	a -= c; a ^= rot(c, 16); c += b; \
> > +	b -= a; b ^= rot(a, 19); a += c; \
> > +	c -= b; c ^= rot(b, 4); b += a; \
> > +}
> > +
> > +#ifdef __cplusplus
> > +}
> > +#endif
> > +
> > +#endif
> 
> 
> These macros are only used by the l3fwd_calc_hash() function , so they could
> simply be defined at the top of odp_l3fwd_db.c file.
> 

OK, I have thought it with different copyright and should be in separate file.


> > +
> > +	in_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE;
> > +	if (odp_pktin_queue_config(pktio, &in_queue_param)) {
> > +		printf("Failed to config input queue for %s\n", name);
> > +		exit(1);
> > +	}
> > +
> > +	out_queue_param.op_mode = ODP_PKTIO_OP_MT;
> > +	if (odp_pktout_queue_config(pktio, &out_queue_param)) {
> > +		printf("Failed to config output queue for %s\n", name);
> > +		exit(1);
> > +	}
> 
> As mentioned in the earlier email pktout queues could also be lockless if
each
> worker has a private tx queue.

Could we assume always the same number of rx queues and tx queues? If yes,
it's OK.

> 
> > +		entry = find_fwd_db_entry(&key);
> > +		ip->ttl--;
> > +		ip->chksum = odph_ipv4_csum_update(pkt);
> > +		eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL);
> > +		if (entry) {
> > +			memcpy(eth->src.addr, entry->src_mac,
> > ODPH_ETHADDR_LEN);
> > +			memcpy(eth->dst.addr, entry->dst_mac,
> > ODPH_ETHADDR_LEN);
> > +			ret = odp_pktout_send(entry->outq, &pkt, 1);
> > +		} else {
> > +			/* on route, reflect the packet */
> > +			memcpy(eth->dst.addr, eth->src.addr,
> > ODPH_ETHADDR_LEN);
> > +			ret = odp_pktout_send(def_outq, &pkt, 1);
> > +		}
> 
> Buffering packets and sending them in bursts would probably improve
> performance quite a lot.
> 
OK,  will  be addressed in next version

> > +/**
> > + * Parse text string representing an IPv4 address or subnet
> > + *
> > + * String is of the format "XXX.XXX.XXX.XXX(/W)" where
> > + * "XXX" is decimal value and "/W" is optional subnet length
> > + *
> > + * @param ipaddress  Pointer to IP address/subnet string to convert
> > + * @param addr       Pointer to return IPv4 address
> > + * @param mask       Pointer (optional) to return IPv4 mask
> > + *
> > + * @return 0 if successful else -1
> > + */
> > +static inline
> > +int parse_ipv4_string(char *ipaddress, uint32_t *addr, uint32_t
> > +*mask)
> 
> This should be moved to helper/ip.c, so it can be reused by other
applications.
> 

I want it to inline, but the helper eth.c/ip.c seems  not.


> > +/**
> > + * Generate text string representing MAC address
> > + *
> > + * @param b     Pointer to buffer to store string
> > + * @param mac   Pointer to MAC address
> > + *
> > + * @return Pointer to supplied buffer  */ static inline char
> > +*mac_addr_str(char *b, uint8_t *mac)
> 
> Could also be useful helper function. Instead of uint8_t  should use
> odph_ethaddr_t.
> 
> > + * Forwarding data base entry
> > + */
> > +typedef struct fwd_db_entry_s {
> > +	struct fwd_db_entry_s *next;          /**< Next entry on list */
> > +	char                    oif[OIF_LEN]; /**< Output interface name */
> > +	odp_pktout_queue_t	outq;	      /**< Output transmit queue */
> > +	uint8_t   src_mac[ODPH_ETHADDR_LEN];  /**< Output source MAC */
> > +	uint8_t   dst_mac[ODPH_ETHADDR_LEN];  /**< Output destination
> MAC
> > */
> > +	ip_addr_range_t        subnet;        /**< Subnet for this router */
> > +} fwd_db_entry_t;
> 
> 
> odph_ethaddr_t could be used here.

OK,  will  be addressed in next version
Forrest Shi June 12, 2016, 8:46 a.m. | #4
Hi Matias,

The dpdk_open failed because they just use digit to specify eth interface,
like -i 0,1
For the kernel based driver, the device name is eth0/eth1.  

Thanks,
Forrest

> -----Original Message-----
> From: Elo, Matias (Nokia - FI/Espoo) [mailto:matias.elo@nokia-bell-labs.com]
> Sent: Thursday, June 09, 2016 21:57
> To: Xuelin Shi <forrest.shi@linaro.org>
> Cc: lng-odp@lists.linaro.org
> Subject: RE: [lng-odp][PATCH v3] example: introducing l3fwd
> 
> Hi Forrest,
> 
> On my system the application crashes on start:
> 
> $ sudo ./example/l3fwd/odp_l3fwd -i eth0,eth1 -r 1.1.1.0/24:eth0 -r
> 2.2.2.0/24:eth1 ODP system info
> ---------------
> ODP API version: 1.10.0
> ODP impl name:	 odp-linux
> CPU model:       Intel(R) Xeon(R) CPU E5-2697 v3
> CPU freq (hz):   2600000000
> Cache line size: 64
> CPU count:       28
> 
> Running ODP appl: "odp_l3fwd"
> -----------------
> IF-count:        2
> Using IFs:       eth0 eth1
> 
> pktio/dpdk.c:556:dpdk_open():Invalid dpdk netdev: eth0
> *** buffer overflow detected ***: ./example/l3fwd/odp_l3fwd terminated
> ======= Backtrace: ========= /lib/x86_64-linux-
> gnu/libc.so.6(+0x7338f)[0x7ffff788038f]
> /lib/x86_64-linux-gnu/libc.so.6(__fortify_fail+0x5c)[0x7ffff7917c9c]
> /lib/x86_64-linux-gnu/libc.so.6(+0x109b60)[0x7ffff7916b60]
> /lib/x86_64-linux-gnu/libc.so.6(+0x109069)[0x7ffff7916069]
> /lib/x86_64-linux-gnu/libc.so.6(_IO_default_xsputn+0xbc)[0x7ffff788870c]
> /lib/x86_64-linux-gnu/libc.so.6(_IO_padn+0xa0)[0x7ffff787c6d0]
> /lib/x86_64-linux-gnu/libc.so.6(_IO_vfprintf+0xa8c)[0x7ffff785777c]
> /lib/x86_64-linux-gnu/libc.so.6(__vsprintf_chk+0x84)[0x7ffff79160f4]
> /lib/x86_64-linux-gnu/libc.so.6(__sprintf_chk+0x7d)[0x7ffff791604d]
> ./example/l3fwd/odp_l3fwd[0x40b056]
> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5)[0x7ffff782eec5]
> ./example/l3fwd/odp_l3fwd[0x40bd6b]
> 
> 
> More comments below.
> 
> -Matias
> 
> 
> > diff --git a/example/l3fwd/jhash.h b/example/l3fwd/jhash.h new file
> > mode 100644 index 0000000..d3dbd2a
> > --- /dev/null
> > +++ b/example/l3fwd/jhash.h
> > @@ -0,0 +1,46 @@
> > +/** jhash.h: Jenkins hash support.
> > +  *
> > +  * Copyright (C) 2006 Bob Jenkins (bob_jenkins@burtleburtle.net)
> > +  *
> > +  * http://burtleburtle.net/bob/hash/
> > +  *
> > +  * These are the credits from Bob's sources:
> > +  *
> > +  * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
> > +  *
> > +  * These are functions for producing 32-bit hashes for hash table
lookup.
> > +  * hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and
> > +final()
> > +  * are externally useful functions.  Routines to test the hash are
> > +included
> > +  * if SELF_TEST is defined.  You can use this free for any purpose.
> > +It's in
> > +  * the public domain.  It has no warranty.
> > +  *
> > +  * $FreeBSD$
> > +  */
> > +
> > +#ifndef JHASH_H_
> > +#define JHASH_H_
> > +
> > +#ifdef __cplusplus
> > +extern "C" {
> > +#endif
> > +
> > +/**
> > + * Hash calculation utility
> > + */
> > +#define JHASH_GOLDEN_RATIO	0x9e3779b9
> > +#define rot(x, k) (((x) << (k)) | ((x) >> (32 - (k)))) #define
> > +FWD_BJ3_MIX(a, b, c) \ { \
> > +	a -= c; a ^= rot(c, 4); c += b; \
> > +	b -= a; b ^= rot(a, 6); a += c; \
> > +	c -= b; c ^= rot(b, 8); b += a; \
> > +	a -= c; a ^= rot(c, 16); c += b; \
> > +	b -= a; b ^= rot(a, 19); a += c; \
> > +	c -= b; c ^= rot(b, 4); b += a; \
> > +}
> > +
> > +#ifdef __cplusplus
> > +}
> > +#endif
> > +
> > +#endif
> 
> 
> These macros are only used by the l3fwd_calc_hash() function , so they could
> simply be defined at the top of odp_l3fwd_db.c file.
> 
> 
> > +
> > +	in_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE;
> > +	if (odp_pktin_queue_config(pktio, &in_queue_param)) {
> > +		printf("Failed to config input queue for %s\n", name);
> > +		exit(1);
> > +	}
> > +
> > +	out_queue_param.op_mode = ODP_PKTIO_OP_MT;
> > +	if (odp_pktout_queue_config(pktio, &out_queue_param)) {
> > +		printf("Failed to config output queue for %s\n", name);
> > +		exit(1);
> > +	}
> 
> As mentioned in the earlier email pktout queues could also be lockless if
each
> worker has a private tx queue.
> 
> > +		entry = find_fwd_db_entry(&key);
> > +		ip->ttl--;
> > +		ip->chksum = odph_ipv4_csum_update(pkt);
> > +		eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL);
> > +		if (entry) {
> > +			memcpy(eth->src.addr, entry->src_mac,
> > ODPH_ETHADDR_LEN);
> > +			memcpy(eth->dst.addr, entry->dst_mac,
> > ODPH_ETHADDR_LEN);
> > +			ret = odp_pktout_send(entry->outq, &pkt, 1);
> > +		} else {
> > +			/* on route, reflect the packet */
> > +			memcpy(eth->dst.addr, eth->src.addr,
> > ODPH_ETHADDR_LEN);
> > +			ret = odp_pktout_send(def_outq, &pkt, 1);
> > +		}
> 
> Buffering packets and sending them in bursts would probably improve
> performance quite a lot.
> 
> > +/**
> > + * Parse text string representing an IPv4 address or subnet
> > + *
> > + * String is of the format "XXX.XXX.XXX.XXX(/W)" where
> > + * "XXX" is decimal value and "/W" is optional subnet length
> > + *
> > + * @param ipaddress  Pointer to IP address/subnet string to convert
> > + * @param addr       Pointer to return IPv4 address
> > + * @param mask       Pointer (optional) to return IPv4 mask
> > + *
> > + * @return 0 if successful else -1
> > + */
> > +static inline
> > +int parse_ipv4_string(char *ipaddress, uint32_t *addr, uint32_t
> > +*mask)
> 
> This should be moved to helper/ip.c, so it can be reused by other
applications.
> 
> > +/**
> > + * Generate text string representing MAC address
> > + *
> > + * @param b     Pointer to buffer to store string
> > + * @param mac   Pointer to MAC address
> > + *
> > + * @return Pointer to supplied buffer  */ static inline char
> > +*mac_addr_str(char *b, uint8_t *mac)
> 
> Could also be useful helper function. Instead of uint8_t  should use
> odph_ethaddr_t.
> 
> > + * Forwarding data base entry
> > + */
> > +typedef struct fwd_db_entry_s {
> > +	struct fwd_db_entry_s *next;          /**< Next entry on list */
> > +	char                    oif[OIF_LEN]; /**< Output interface name */
> > +	odp_pktout_queue_t	outq;	      /**< Output transmit queue */
> > +	uint8_t   src_mac[ODPH_ETHADDR_LEN];  /**< Output source MAC */
> > +	uint8_t   dst_mac[ODPH_ETHADDR_LEN];  /**< Output destination
> MAC
> > */
> > +	ip_addr_range_t        subnet;        /**< Subnet for this router */
> > +} fwd_db_entry_t;
> 
> 
> odph_ethaddr_t could be used here.
Mike Holmes June 13, 2016, 10:32 a.m. | #5
On 12 June 2016 at 02:35, forrest.shi <forrest.shi@linaro.org> wrote:

> Hi Matias,
>
> Very appreciate your time and review.  See my comments inline.
>
> Thanks,
> Forrest
>
> > -----Original Message-----
> > From: Elo, Matias (Nokia - FI/Espoo) [mailto:
> matias.elo@nokia-bell-labs.com]
> > Sent: Thursday, June 09, 2016 21:57
> > To: Xuelin Shi <forrest.shi@linaro.org>
> > Cc: lng-odp@lists.linaro.org
> > Subject: RE: [lng-odp][PATCH v3] example: introducing l3fwd
> >
> > Hi Forrest,
> >
> > On my system the application crashes on start:
> >
> > $ sudo ./example/l3fwd/odp_l3fwd -i eth0,eth1 -r 1.1.1.0/24:eth0 -r
> > 2.2.2.0/24:eth1 ODP system info
> > ---------------
> > ODP API version: 1.10.0
> > ODP impl name:         odp-linux
> > CPU model:       Intel(R) Xeon(R) CPU E5-2697 v3
> > CPU freq (hz):   2600000000
> > Cache line size: 64
> > CPU count:       28
> >
> > Running ODP appl: "odp_l3fwd"
> > -----------------
> > IF-count:        2
> > Using IFs:       eth0 eth1
> >
> > pktio/dpdk.c:556:dpdk_open():Invalid dpdk netdev: eth0
> > *** buffer overflow detected ***: ./example/l3fwd/odp_l3fwd terminated
> > ======= Backtrace: ========= /lib/x86_64-linux-
> > gnu/libc.so.6(+0x7338f)[0x7ffff788038f]
> > /lib/x86_64-linux-gnu/libc.so.6(__fortify_fail+0x5c)[0x7ffff7917c9c]
> > /lib/x86_64-linux-gnu/libc.so.6(+0x109b60)[0x7ffff7916b60]
> > /lib/x86_64-linux-gnu/libc.so.6(+0x109069)[0x7ffff7916069]
> > /lib/x86_64-linux-gnu/libc.so.6(_IO_default_xsputn+0xbc)[0x7ffff788870c]
> > /lib/x86_64-linux-gnu/libc.so.6(_IO_padn+0xa0)[0x7ffff787c6d0]
> > /lib/x86_64-linux-gnu/libc.so.6(_IO_vfprintf+0xa8c)[0x7ffff785777c]
> > /lib/x86_64-linux-gnu/libc.so.6(__vsprintf_chk+0x84)[0x7ffff79160f4]
> > /lib/x86_64-linux-gnu/libc.so.6(__sprintf_chk+0x7d)[0x7ffff791604d]
> > ./example/l3fwd/odp_l3fwd[0x40b056]
> > /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5)[0x7ffff782eec5]
> > ./example/l3fwd/odp_l3fwd[0x40bd6b]
> >
> >
> > More comments below.
> >
> > -Matias
> >
> >
> > > diff --git a/example/l3fwd/jhash.h b/example/l3fwd/jhash.h new file
> > > mode 100644 index 0000000..d3dbd2a
> > > --- /dev/null
> > > +++ b/example/l3fwd/jhash.h
> > > @@ -0,0 +1,46 @@
> > > +/** jhash.h: Jenkins hash support.
> > > +  *
> > > +  * Copyright (C) 2006 Bob Jenkins (bob_jenkins@burtleburtle.net)
> > > +  *
> > > +  * http://burtleburtle.net/bob/hash/
> > > +  *
> > > +  * These are the credits from Bob's sources:
> > > +  *
> > > +  * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
> > > +  *
> > > +  * These are functions for producing 32-bit hashes for hash table
> lookup.
> > > +  * hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and
> > > +final()
> > > +  * are externally useful functions.  Routines to test the hash are
> > > +included
> > > +  * if SELF_TEST is defined.  You can use this free for any purpose.
> > > +It's in
> > > +  * the public domain.  It has no warranty.
> > > +  *
> > > +  * $FreeBSD$
> > > +  */
> > > +
> > > +#ifndef JHASH_H_
> > > +#define JHASH_H_
> > > +
> > > +#ifdef __cplusplus
> > > +extern "C" {
> > > +#endif
> > > +
> > > +/**
> > > + * Hash calculation utility
> > > + */
> > > +#define JHASH_GOLDEN_RATIO 0x9e3779b9
> > > +#define rot(x, k) (((x) << (k)) | ((x) >> (32 - (k)))) #define
> > > +FWD_BJ3_MIX(a, b, c) \ { \
> > > +   a -= c; a ^= rot(c, 4); c += b; \
> > > +   b -= a; b ^= rot(a, 6); a += c; \
> > > +   c -= b; c ^= rot(b, 8); b += a; \
> > > +   a -= c; a ^= rot(c, 16); c += b; \
> > > +   b -= a; b ^= rot(a, 19); a += c; \
> > > +   c -= b; c ^= rot(b, 4); b += a; \
> > > +}
> > > +
> > > +#ifdef __cplusplus
> > > +}
> > > +#endif
> > > +
> > > +#endif
> >
> >
> > These macros are only used by the l3fwd_calc_hash() function , so they
> could
> > simply be defined at the top of odp_l3fwd_db.c file.
> >
>
> OK, I have thought it with different copyright and should be in separate
> file.
>
>
> > > +
> > > +   in_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE;
> > > +   if (odp_pktin_queue_config(pktio, &in_queue_param)) {
> > > +           printf("Failed to config input queue for %s\n", name);
> > > +           exit(1);
> > > +   }
> > > +
> > > +   out_queue_param.op_mode = ODP_PKTIO_OP_MT;
> > > +   if (odp_pktout_queue_config(pktio, &out_queue_param)) {
> > > +           printf("Failed to config output queue for %s\n", name);
> > > +           exit(1);
> > > +   }
> >
> > As mentioned in the earlier email pktout queues could also be lockless if
> each
> > worker has a private tx queue.
>
> Could we assume always the same number of rx queues and tx queues? If yes,
> it's OK.
>
> >
> > > +           entry = find_fwd_db_entry(&key);
> > > +           ip->ttl--;
> > > +           ip->chksum = odph_ipv4_csum_update(pkt);
> > > +           eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL);
> > > +           if (entry) {
> > > +                   memcpy(eth->src.addr, entry->src_mac,
> > > ODPH_ETHADDR_LEN);
> > > +                   memcpy(eth->dst.addr, entry->dst_mac,
> > > ODPH_ETHADDR_LEN);
> > > +                   ret = odp_pktout_send(entry->outq, &pkt, 1);
> > > +           } else {
> > > +                   /* on route, reflect the packet */
> > > +                   memcpy(eth->dst.addr, eth->src.addr,
> > > ODPH_ETHADDR_LEN);
> > > +                   ret = odp_pktout_send(def_outq, &pkt, 1);
> > > +           }
> >
> > Buffering packets and sending them in bursts would probably improve
> > performance quite a lot.
> >
> OK,  will  be addressed in next version
>
> > > +/**
> > > + * Parse text string representing an IPv4 address or subnet
> > > + *
> > > + * String is of the format "XXX.XXX.XXX.XXX(/W)" where
> > > + * "XXX" is decimal value and "/W" is optional subnet length
> > > + *
> > > + * @param ipaddress  Pointer to IP address/subnet string to convert
> > > + * @param addr       Pointer to return IPv4 address
> > > + * @param mask       Pointer (optional) to return IPv4 mask
> > > + *
> > > + * @return 0 if successful else -1
> > > + */
> > > +static inline
> > > +int parse_ipv4_string(char *ipaddress, uint32_t *addr, uint32_t
> > > +*mask)
> >
> > This should be moved to helper/ip.c, so it can be reused by other
> applications.
> >
>
> I want it to inline, but the helper eth.c/ip.c seems  not.
>

Yi is currently working on breaking the cyclic dependency on the helpers
and the ip, eth etc headers with the implimentation so that might affect
things.

Mike


>
> > > +/**
> > > + * Generate text string representing MAC address
> > > + *
> > > + * @param b     Pointer to buffer to store string
> > > + * @param mac   Pointer to MAC address
> > > + *
> > > + * @return Pointer to supplied buffer  */ static inline char
> > > +*mac_addr_str(char *b, uint8_t *mac)
> >
> > Could also be useful helper function. Instead of uint8_t  should use
> > odph_ethaddr_t.
> >
> > > + * Forwarding data base entry
> > > + */
> > > +typedef struct fwd_db_entry_s {
> > > +   struct fwd_db_entry_s *next;          /**< Next entry on list */
> > > +   char                    oif[OIF_LEN]; /**< Output interface name */
> > > +   odp_pktout_queue_t      outq;         /**< Output transmit queue */
> > > +   uint8_t   src_mac[ODPH_ETHADDR_LEN];  /**< Output source MAC */
> > > +   uint8_t   dst_mac[ODPH_ETHADDR_LEN];  /**< Output destination
> > MAC
> > > */
> > > +   ip_addr_range_t        subnet;        /**< Subnet for this router
> */
> > > +} fwd_db_entry_t;
> >
> >
> > odph_ethaddr_t could be used here.
>
> OK,  will  be addressed in next version
>
>
> _______________________________________________
> lng-odp mailing list
> lng-odp@lists.linaro.org
> https://lists.linaro.org/mailman/listinfo/lng-odp
>

Patch

diff --git a/example/Makefile.am b/example/Makefile.am
index 7f82c4d..67e4389 100644
--- a/example/Makefile.am
+++ b/example/Makefile.am
@@ -1 +1 @@ 
-SUBDIRS = classifier generator ipsec packet time timer traffic_mgmt l2fwd_simple switch
+SUBDIRS = classifier generator ipsec packet time timer traffic_mgmt l2fwd_simple l3fwd switch
diff --git a/example/l3fwd/.gitignore b/example/l3fwd/.gitignore
new file mode 100644
index 0000000..4a25ae8
--- /dev/null
+++ b/example/l3fwd/.gitignore
@@ -0,0 +1,4 @@ 
+odp_l3fwd
+Makefile
+Makefile.in
+*.o
diff --git a/example/l3fwd/Makefile.am b/example/l3fwd/Makefile.am
new file mode 100644
index 0000000..e0dd266
--- /dev/null
+++ b/example/l3fwd/Makefile.am
@@ -0,0 +1,12 @@ 
+include $(top_srcdir)/example/Makefile.inc
+
+bin_PROGRAMS = odp_l3fwd$(EXEEXT)
+odp_l3fwd_LDFLAGS = $(AM_LDFLAGS) -static
+odp_l3fwd_CFLAGS = $(AM_CFLAGS) -I${top_srcdir}/example
+
+noinst_HEADERS = \
+		  $(top_srcdir)/example/l3fwd/jhash.h \
+		  $(top_srcdir)/example/l3fwd/odp_l3fwd_db.h \
+		  $(top_srcdir)/example/example_debug.h
+
+dist_odp_l3fwd_SOURCES = odp_l3fwd.c odp_l3fwd_db.c
diff --git a/example/l3fwd/jhash.h b/example/l3fwd/jhash.h
new file mode 100644
index 0000000..d3dbd2a
--- /dev/null
+++ b/example/l3fwd/jhash.h
@@ -0,0 +1,46 @@ 
+/** jhash.h: Jenkins hash support.
+  *
+  * Copyright (C) 2006 Bob Jenkins (bob_jenkins@burtleburtle.net)
+  *
+  * http://burtleburtle.net/bob/hash/
+  *
+  * These are the credits from Bob's sources:
+  *
+  * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
+  *
+  * These are functions for producing 32-bit hashes for hash table lookup.
+  * hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and final()
+  * are externally useful functions.  Routines to test the hash are included
+  * if SELF_TEST is defined.  You can use this free for any purpose.  It's in
+  * the public domain.  It has no warranty.
+  *
+  * $FreeBSD$
+  */
+
+#ifndef JHASH_H_
+#define JHASH_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Hash calculation utility
+ */
+#define JHASH_GOLDEN_RATIO	0x9e3779b9
+#define rot(x, k) (((x) << (k)) | ((x) >> (32 - (k))))
+#define FWD_BJ3_MIX(a, b, c) \
+{ \
+	a -= c; a ^= rot(c, 4); c += b; \
+	b -= a; b ^= rot(a, 6); a += c; \
+	c -= b; c ^= rot(b, 8); b += a; \
+	a -= c; a ^= rot(c, 16); c += b; \
+	b -= a; b ^= rot(a, 19); a += c; \
+	c -= b; c ^= rot(b, 4); b += a; \
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c
new file mode 100644
index 0000000..fe715d5
--- /dev/null
+++ b/example/l3fwd/odp_l3fwd.c
@@ -0,0 +1,744 @@ 
+/* Copyright (c) 2016, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+
+#include <example_debug.h>
+#include <odp_api.h>
+#include <odp/helper/linux.h>
+#include <odp/helper/eth.h>
+#include <odp/helper/ip.h>
+#include <odp/helper/udp.h>
+#include <odp/helper/tcp.h>
+
+#include <getopt.h>
+
+#include "odp_l3fwd_db.h"
+
+#define POOL_NUM_PKT	8192
+#define POOL_SEG_LEN	1856
+#define MAX_PKT_BURST	32
+
+#define MAX_NB_WORKER	8
+#define MAX_NB_PKTIO	4
+#define MAX_NB_ROUTE	32
+#define MAX_NB_RX_QUEUE	8
+#define MAX_NB_TX_QUEUE	8
+#define MAX_NB_QCONFS	1024
+#define MAX_NB_QCONF_PER_CORE	8
+#define CPU_ID_INVALID	(-1)
+
+/** Get rid of path in filename - only for unix-type paths using '/' */
+#define NO_PATH(file_name) (strrchr((file_name), '/') ? \
+			    strrchr((file_name), '/') + 1 : (file_name))
+
+struct l3fwd_pktio_s {
+	odp_pktio_t pktio;
+	odp_pktin_queue_t ifin[MAX_NB_RX_QUEUE];
+	odp_pktout_queue_t ifout[MAX_NB_TX_QUEUE];
+	uint32_t nb_rxq;
+	uint32_t nb_txq;
+};
+
+struct l3fwd_qconf_s {
+	uint8_t if_idx;		/* port index */
+	uint8_t rxq_idx;	/* recv queue index in a port */
+	uint8_t core_idx;	/* this core should handle traffic */
+};
+
+struct thread_arg_s {
+	int cpu;		/* initialized as -1 */
+	struct l3fwd_qconf_s *qconf_args[MAX_NB_QCONF_PER_CORE];
+	int count;
+};
+
+static struct l3fwd_qconf_s qconf_config_default[] = {
+	{0, 0, 1},
+	{1, 0, 2},
+	{2, 0, 3},
+	{3, 0, 4},
+	{0, 1, 5},
+	{1, 1, 6},
+	{2, 1, 7},
+	{3, 1, 0},
+};
+
+typedef struct {
+	char *if_names[MAX_NB_PKTIO];
+	int if_count;
+	char *route_str[MAX_NB_ROUTE];
+	int worker_count;
+	struct l3fwd_qconf_s qconf_config[MAX_NB_QCONFS];
+	int qconf_count;
+} app_args_t;
+
+struct {
+	app_args_t		cmd_args;
+	struct l3fwd_pktio_s	l3fwd_pktios[MAX_NB_PKTIO];
+	odph_linux_pthread_t	l3fwd_workers[MAX_NB_WORKER];
+	struct thread_arg_s	worker_args[MAX_NB_WORKER];
+	uint32_t		nb_pktio;  /* effective pktios */
+	uint32_t		nb_worker; /* effective workers */
+} global;
+
+static void print_usage(char *progname);
+static void print_info(char *progname, app_args_t *args);
+static void parse_cmdline_args(int argc, char *argv[], app_args_t *args);
+static int parse_config(char *cfg_str, app_args_t *args);
+static inline void setup_worker_qconf(app_args_t *args);
+static int split_string(char *str, int stringlen,
+			char **tokens, int maxtokens, char delim);
+
+static odp_pktio_t create_pktio(const char *name, odp_pool_t pool,
+				struct l3fwd_pktio_s *fwd_pktio)
+{
+	odp_pktio_param_t pktio_param;
+	odp_pktin_queue_param_t in_queue_param;
+	odp_pktout_queue_param_t out_queue_param;
+	odp_pktio_t pktio;
+	odp_pktio_capability_t capa;
+	struct odp_pktin_queue_t *inq;
+	struct odp_pktout_queue_t *outq;
+	int rc, nb_rxq, nb_txq;
+
+	odp_pktio_param_init(&pktio_param);
+
+	pktio = odp_pktio_open(name, pool, &pktio_param);
+	if (pktio == ODP_PKTIO_INVALID) {
+		printf("Failed to open %s\n", name);
+		exit(1);
+	}
+
+	odp_pktin_queue_param_init(&in_queue_param);
+	odp_pktout_queue_param_init(&out_queue_param);
+
+	/** set rx and tx queue number
+	 *  0 < rxq <= capability: set to rxq, else to rx capa
+	 *  0 < txq <= capability: set to txq, else to tx capa
+	 */
+	rc = odp_pktio_capability(pktio, &capa);
+	if (rc) {
+		printf("pktio %s: unable to read capabilities, "
+		       "set to 1 rx and 1 tx queue\n", name);
+		fwd_pktio->nb_rxq = 1;
+		fwd_pktio->nb_txq = 1;
+	}
+
+	nb_rxq = fwd_pktio->nb_rxq;
+	nb_txq = fwd_pktio->nb_txq;
+	if (nb_rxq == 0) {
+		fwd_pktio->nb_rxq = 1;
+	} else if (nb_rxq > (int)capa.max_input_queues) {
+		printf("Error queue config: max queue %d for %s.\n",
+		       (int)capa.max_input_queues - 1, name);
+		exit(1);
+	}
+
+	if (nb_txq == 0)
+		fwd_pktio->nb_txq = 1;
+	else if (nb_txq > (int)capa.max_output_queues)
+		fwd_pktio->nb_txq = capa.max_output_queues;
+
+	in_queue_param.num_queues = fwd_pktio->nb_rxq;
+	out_queue_param.num_queues = fwd_pktio->nb_txq;
+
+	in_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE;
+	if (odp_pktin_queue_config(pktio, &in_queue_param)) {
+		printf("Failed to config input queue for %s\n", name);
+		exit(1);
+	}
+
+	out_queue_param.op_mode = ODP_PKTIO_OP_MT;
+	if (odp_pktout_queue_config(pktio, &out_queue_param)) {
+		printf("Failed to config output queue for %s\n", name);
+		exit(1);
+	}
+
+	nb_rxq = in_queue_param.num_queues;
+	nb_txq = out_queue_param.num_queues;
+	inq = &fwd_pktio->ifin[0];
+	if (odp_pktin_queue(pktio, inq, nb_rxq) != nb_rxq) {
+		printf("pktin queue query failed for %s\n", name);
+		exit(1);
+	}
+
+	outq = &fwd_pktio->ifout[0];
+	if (odp_pktout_queue(pktio, outq, nb_txq) != nb_txq) {
+		printf("pktout queue query failed for %s\n", name);
+		exit(1);
+	}
+
+	fwd_pktio->pktio = pktio;
+
+	return pktio;
+}
+
+static inline void l3fwd_one_queue(odp_pktin_queue_t inq,
+				   odp_pktout_queue_t def_outq)
+{
+	odp_packet_t pkt_tbl[MAX_PKT_BURST];
+	odp_packet_t pkt_tbl_drop[MAX_PKT_BURST];
+	uint32_t pkts, i;
+	int need_to_drop = 0;
+	int ret;
+
+	pkts = odp_pktin_recv(inq, pkt_tbl, MAX_PKT_BURST);
+	if (odp_unlikely(pkts <= 0))
+		return;
+
+	for (i = 0; i < pkts; i++) {
+		odp_packet_t pkt = pkt_tbl[i];
+		odph_ethhdr_t *eth;
+		odph_ipv4hdr_t *ip;
+		odph_udphdr_t  *udp;
+		fwd_db_entry_t *entry;
+		ipv4_tuple5_t key;
+		uint32_t len;
+
+		if (odp_unlikely(!odp_packet_has_ipv4(pkt) ||
+				 !odp_packet_has_eth(pkt))) {
+			printf("warning: invalid ipv4 or eth header\n");
+			pkt_tbl_drop[need_to_drop] = pkt;
+			need_to_drop++;
+			continue;
+		}
+
+		ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &len);
+		key.dst_ip = odp_be_to_cpu_32(ip->dst_addr);
+		key.src_ip = odp_be_to_cpu_32(ip->src_addr);
+		key.proto = ip->proto;
+		if (odp_packet_has_udp(pkt) ||
+		    odp_packet_has_tcp(pkt)) {
+			/* UDP or TCP*/
+			void *ptr = odp_packet_l4_ptr(pkt, NULL);
+
+			udp = (odph_udphdr_t *)ptr;
+			key.src_port = odp_be_to_cpu_16(udp->src_port);
+			key.dst_port = odp_be_to_cpu_16(udp->dst_port);
+		} else {
+			key.src_port = 0;
+			key.dst_port = 0;
+		}
+
+		entry = find_fwd_db_entry(&key);
+		ip->ttl--;
+		ip->chksum = odph_ipv4_csum_update(pkt);
+		eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL);
+		if (entry) {
+			memcpy(eth->src.addr, entry->src_mac, ODPH_ETHADDR_LEN);
+			memcpy(eth->dst.addr, entry->dst_mac, ODPH_ETHADDR_LEN);
+			ret = odp_pktout_send(entry->outq, &pkt, 1);
+		} else {
+			/* on route, reflect the packet */
+			memcpy(eth->dst.addr, eth->src.addr, ODPH_ETHADDR_LEN);
+			ret = odp_pktout_send(def_outq, &pkt, 1);
+		}
+		if (ret < 1) {
+			pkt_tbl_drop[need_to_drop] = pkt;
+			need_to_drop++;
+		}
+	}
+
+	odp_packet_free_multi(pkt_tbl_drop, need_to_drop);
+}
+
+static void *run_worker(void *arg)
+{
+	struct thread_arg_s *args = arg;
+
+	for (;;) {
+		int i;
+		struct l3fwd_pktio_s *port;
+		struct l3fwd_qconf_s *qconf;
+
+		for (i = 0; i < args->count; i++) {
+			qconf = args->qconf_args[i];
+			if (args->cpu != qconf->core_idx)
+				continue;
+
+			port = &global.l3fwd_pktios[qconf->if_idx];
+			l3fwd_one_queue(port->ifin[qconf->rxq_idx],
+					port->ifout[0]);
+		}
+	}
+
+	return NULL;
+}
+
+int main(int argc, char **argv)
+{
+	odp_pool_t pool;
+	odp_pool_param_t params;
+	odp_instance_t instance;
+	odph_linux_thr_params_t thr_params;
+	int cpu;
+	uint32_t i, nb_worker;
+	uint8_t mac[ODPH_ETHADDR_LEN];
+	app_args_t *args;
+
+	if (odp_init_global(&instance, NULL, NULL)) {
+		printf("Error: ODP global init failed.\n");
+		exit(1);
+	}
+
+	if (odp_init_local(instance, ODP_THREAD_CONTROL)) {
+		printf("Error: ODP local init failed.\n");
+		exit(1);
+	}
+
+	/* Clear global argument */
+	memset(&global, 0, sizeof(global));
+
+	/* Parse cmdline arguments */
+	args = &global.cmd_args;
+	parse_cmdline_args(argc, argv, args);
+
+	/* Distribute the receive queues by core */
+	setup_worker_qconf(args);
+
+	/* Init l3fwd tale */
+	init_fwd_db();
+
+	/* Add route into table */
+	for (i = 0; i < MAX_NB_ROUTE; i++) {
+		if (args->route_str[i])
+			create_fwd_db_entry(args->route_str[i]);
+	}
+
+	print_info(NO_PATH(argv[0]), args);
+
+	/* Create packet pool */
+	odp_pool_param_init(&params);
+	params.pkt.seg_len = POOL_SEG_LEN;
+	params.pkt.len     = POOL_SEG_LEN;
+	params.pkt.num     = POOL_NUM_PKT;
+	params.type        = ODP_POOL_PACKET;
+
+	pool = odp_pool_create("packet pool", &params);
+
+	if (pool == ODP_POOL_INVALID) {
+		printf("Error: packet pool create failed.\n");
+		exit(1);
+	}
+
+	global.nb_pktio = args->if_count;
+	for (i = 0; i < global.nb_pktio; i++) {
+		struct l3fwd_pktio_s *port;
+		char *if_name;
+		char buf[16];
+		int j;
+
+		if_name = args->if_names[i];
+		port = &global.l3fwd_pktios[i];
+		port->pktio = create_pktio(if_name, pool, port);
+		odp_pktio_mac_addr(port->pktio, mac, ODPH_ETHADDR_LEN);
+		resolve_fwd_db(if_name, port->ifout[0], mac);
+
+		/* print mac string, could be used to config pktgen */
+		sprintf(buf, "%02X:%02X:%02X:%02X:%02X:%02X",
+			mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
+		printf("create pktio %s, mac %s\n", if_name, buf);
+
+		/* start pktio */
+		if (odp_pktio_start(port->pktio)) {
+			printf("unable to start pktio: %s\n", if_name);
+			exit(1);
+		}
+
+		printf("start pktio: %s, rxq: ", if_name);
+		for (j = 0; j < args->qconf_count; j++) {
+			struct l3fwd_qconf_s *q;
+
+			q = &args->qconf_config[j];
+			if (q->if_idx == i)
+				printf("(%d, %d, %d) ", q->if_idx, q->rxq_idx,
+				       q->core_idx);
+		}
+		printf("\n");
+	}
+
+	dump_fwd_db();
+
+	/* one worker thread per core.
+	 * if no thread count in cmdline, all cpus are used,
+	 * else create threads as the cmdline.
+	 */
+	cpu = odp_cpu_count();
+	if (args->worker_count == 0 || args->worker_count > cpu)
+		args->worker_count = cpu;
+
+	global.nb_worker = args->worker_count;
+	memset(&thr_params, 0, sizeof(thr_params));
+	thr_params.start    = run_worker;
+	thr_params.thr_type = ODP_THREAD_WORKER;
+	thr_params.instance = instance;
+
+	nb_worker = 0;
+	cpu = odp_cpu_id();
+	for (i = 0; i < MAX_NB_WORKER; i++) {
+		struct thread_arg_s *arg;
+		odp_cpumask_t thr_mask;
+
+		/* not create thread for current and invalid cpu */
+		arg = &global.worker_args[i];
+		if (arg->cpu == cpu || arg->cpu == CPU_ID_INVALID)
+			continue;
+
+		odp_cpumask_zero(&thr_mask);
+		odp_cpumask_set(&thr_mask, arg->cpu);
+		thr_params.arg = arg;
+		odph_linux_pthread_create(&global.l3fwd_workers[i], &thr_mask,
+					  &thr_params);
+		nb_worker++;
+		if (nb_worker == global.nb_worker)
+			break;
+	}
+
+	/* if other cores could run the tasks, not run on this cpu */
+	if (nb_worker < global.nb_worker)
+		run_worker(&global.l3fwd_workers[cpu]);
+
+	/* wait for other threads to join */
+	for (i = 0; i < MAX_NB_WORKER; i++) {
+		odph_linux_pthread_t *arg;
+
+		arg = &global.l3fwd_workers[i];
+		if (arg->cpu == cpu || arg->cpu == CPU_ID_INVALID)
+			continue;
+
+		odph_linux_pthread_join(arg, 1);
+	}
+
+	return 0;
+}
+
+static void print_usage(char *progname)
+{
+	printf("\n"
+	       "ODP L3 forwarding application.\n"
+	       "\n"
+	       "Usage: %s OPTIONS\n"
+	       "  E.g. %s -i eth0,eth1 -r 1.1.1.0/24:eth0 -r 2.2.2.0/24:eth1\n"
+	       " In the above example,\n"
+	       " eth0 will send pkts to eth1 and vice versa\n"
+	       "\n"
+	       "Mandatory OPTIONS:\n"
+	       "  -i, --interface eth interfaces (comma-separated, no spaces)\n"
+	       "  -r, --route SubNet:Intf[:NextHopMAC]\n"
+	       "	NextHopMAC can be optional, in this case, zeroed mac\n"
+	       "\n"
+	       "Optional OPTIONS:\n"
+	       "  -t, --thread Number of threads to do forwarding\n"
+	       "	optional, default as cpu count\n"
+	       "  -q, --queue  Configure rx queue(s) for port\n"
+	       "	optional, format: [(port, queue, core),...]\n"
+	       "	for example: -q '(0, 0, 1),(1,0,2)'\n"
+	       "  -h, --help   Display help and exit.\n\n"
+	       "\n", NO_PATH(progname), NO_PATH(progname)
+	    );
+}
+
+static void parse_cmdline_args(int argc, char *argv[], app_args_t *args)
+{
+	int opt;
+	int long_index;
+	char *token, *local;
+	size_t len, route_index = 0;
+	int i, mem_failure = 0;
+
+	static struct option longopts[] = {
+		{"interface", required_argument, NULL, 'i'},	/* return 'i' */
+		{"route", required_argument, NULL, 'r'},	/* return 'r' */
+		{"thread", optional_argument, NULL, 't'},	/* return 't' */
+		{"queue", optional_argument, NULL, 'q'},	/* return 'q' */
+		{"help", no_argument, NULL, 'h'},		/* return 'h' */
+		{NULL, 0, NULL, 0}
+	};
+
+	while (1) {
+		opt = getopt_long(argc, argv, "+t:i:r:q:h",
+				  longopts, &long_index);
+
+		if (opt == -1)
+			break;	/* No more options */
+
+		switch (opt) {
+		/* parse number of worker threads to be run*/
+		case 't':
+			i = odp_cpu_count();
+			args->worker_count = atoi(optarg);
+			if (args->worker_count > i) {
+				printf("Too many threads,"
+				       "truncate to cpu count: %d\n", i);
+				args->worker_count = i;
+			}
+
+			break;
+		/* parse packet-io interface names */
+		case 'i':
+			len = strlen(optarg);
+			if (len == 0) {
+				print_usage(argv[0]);
+				exit(EXIT_FAILURE);
+			}
+			len += 1;	/* add room for '\0' */
+
+			local = malloc(len);
+			if (!local) {
+				print_usage(argv[0]);
+				exit(EXIT_FAILURE);
+			}
+
+			/* count the number of tokens separated by ',' */
+			strcpy(local, optarg);
+			for (token = strtok(local, ","), i = 0;
+			     token != NULL;
+			     token = strtok(NULL, ","), i++)
+				;
+
+			if (i == 0) {
+				print_usage(argv[0]);
+				exit(EXIT_FAILURE);
+			} else if (i > MAX_NB_PKTIO) {
+				printf("too many ports specified, "
+				       "truncated to %d", MAX_NB_PKTIO);
+			}
+			args->if_count = i;
+
+			/* store the if names (reset names string) */
+			strcpy(local, optarg);
+			for (token = strtok(local, ","), i = 0;
+			     token != NULL; token = strtok(NULL, ","), i++) {
+				args->if_names[i] = token;
+			}
+			break;
+
+		/*Configure Route in forwarding database*/
+		case 'r':
+			if (route_index >= MAX_NB_ROUTE) {
+				printf("No more routes can be added\n");
+				break;
+			}
+			local = calloc(1, strlen(optarg) + 1);
+			if (!local) {
+				mem_failure = 1;
+				break;
+			}
+			memcpy(local, optarg, strlen(optarg));
+			local[strlen(optarg)] = '\0';
+			args->route_str[route_index++] = local;
+			break;
+
+		case 'h':
+			print_usage(argv[0]);
+			exit(EXIT_SUCCESS);
+			break;
+
+		case 'q':
+			parse_config(optarg, args);
+			break;
+
+		default:
+			break;
+		}
+	}
+
+	/* checking arguments */
+	if (args->if_count == 0) {
+		printf("\nNo option -i specified.\n");
+		goto out;
+	}
+
+	if (args->route_str[0] == NULL) {
+		printf("\nNo option -r specified.\n");
+		goto out;
+	}
+
+	if (mem_failure == 1) {
+		printf("\nAllocate memory failure.\n");
+		goto out;
+	}
+	optind = 1;		/* reset 'extern optind' from the getopt lib */
+	return;
+
+out:
+	print_usage(argv[0]);
+	exit(EXIT_FAILURE);
+}
+
+/* split string into tokens */
+int split_string(char *str, int stringlen,
+		 char **tokens, int maxtokens, char delim)
+{
+	int i, tok = 0;
+	int tokstart = 1; /* first token is right at start of string */
+
+	if (str == NULL || tokens == NULL)
+		goto einval_error;
+
+	for (i = 0; i < stringlen; i++) {
+		if (str[i] == '\0' || tok >= maxtokens)
+			break;
+		if (tokstart) {
+			tokstart = 0;
+			tokens[tok++] = &str[i];
+		}
+		if (str[i] == delim) {
+			str[i] = '\0';
+			tokstart = 1;
+		}
+	}
+	return tok;
+
+einval_error:
+	errno = EINVAL;
+	return -1;
+}
+
+static int parse_config(char *cfg_str, app_args_t *args)
+{
+	char s[256];
+	const char *p, *p0 = cfg_str;
+	char *end;
+	enum fieldnames {
+		FLD_PORT = 0,
+		FLD_QUEUE,
+		FLD_LCORE,
+		FLD_LAST
+	};
+	unsigned long int_fld[FLD_LAST];
+	char *str_fld[FLD_LAST];
+	int i;
+	unsigned size;
+	int nb_qconfs = 0;
+	struct l3fwd_qconf_s *qconf_array = &args->qconf_config[0];
+
+	p = strchr(p0, '(');
+	while (p != NULL) {
+		++p;
+		p0 = strchr(p, ')');
+		if (p0 == NULL)
+			return -1;
+
+		size = p0 - p;
+		if (size >= sizeof(s))
+			return -1;
+
+		snprintf(s, sizeof(s), "%.*s", size, p);
+		i = split_string(s, sizeof(s), str_fld, FLD_LAST, ',');
+		if (i != FLD_LAST)
+			return -1;
+		for (i = 0; i < FLD_LAST; i++) {
+			errno = 0;
+			int_fld[i] = strtoul(str_fld[i], &end, 0);
+			if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
+				return -1;
+		}
+		if (nb_qconfs >= MAX_NB_QCONFS) {
+			printf("exceeded max number of queue params: %hu\n",
+			       nb_qconfs);
+			return -1;
+		}
+		qconf_array[nb_qconfs].if_idx = (uint8_t)int_fld[FLD_PORT];
+		qconf_array[nb_qconfs].rxq_idx = (uint8_t)int_fld[FLD_QUEUE];
+		qconf_array[nb_qconfs].core_idx = (uint8_t)int_fld[FLD_LCORE];
+		++nb_qconfs;
+	}
+
+	args->qconf_count = nb_qconfs;
+
+	return 0;
+}
+
+static void print_info(char *progname, app_args_t *args)
+{
+	int i;
+
+	printf("\n"
+	       "ODP system info\n"
+	       "---------------\n"
+	       "ODP API version: %s\n"
+	       "ODP impl name:	 %s\n"
+	       "CPU model:       %s\n"
+	       "CPU freq (hz):   %" PRIu64 "\n"
+	       "Cache line size: %i\n"
+	       "CPU count:       %i\n"
+	       "\n",
+	       odp_version_api_str(), odp_version_impl_name(),
+	       odp_cpu_model_str(), odp_cpu_hz_max(),
+	       odp_sys_cache_line_size(), odp_cpu_count());
+
+	printf("Running ODP appl: \"%s\"\n"
+	       "-----------------\n"
+	       "IF-count:        %i\n"
+	       "Using IFs:      ",
+	       progname, args->if_count);
+
+	for (i = 0; i < args->if_count; ++i)
+		printf(" %s", args->if_names[i]);
+
+	printf("\n\n");
+	fflush(NULL);
+}
+
+static void setup_worker_qconf(app_args_t *args)
+{
+	int i;
+	int cpu_count;
+	struct l3fwd_qconf_s *q;
+	struct thread_arg_s *arg;
+	struct l3fwd_pktio_s *port;
+	uint8_t queue_mask[MAX_NB_PKTIO][MAX_NB_RX_QUEUE];
+
+	cpu_count = odp_cpu_count();
+
+	/* initialise the worker cpu as -1 */
+	for (i = 0; i < MAX_NB_WORKER; i++) {
+		arg = &global.worker_args[i];
+		arg->cpu = CPU_ID_INVALID;
+	}
+	memset(&queue_mask[0][0], 0, sizeof(queue_mask));
+
+	/* if no queue config specified, use default */
+	if (args->qconf_count == 0) {
+		int len, tmp;
+
+		tmp = cpu_count < args->if_count ? cpu_count : args->if_count;
+		len = tmp * sizeof(qconf_config_default[0]);
+		memcpy(&args->qconf_config[0], &qconf_config_default[0], len);
+		args->qconf_count = tmp;
+	}
+
+	for (i = 0; i < args->qconf_count; i++) {
+		q = &args->qconf_config[i];
+		if (q->core_idx >= cpu_count || q->if_idx >= args->if_count) {
+			printf("Error queue config: max port: %d, "
+			       "max core: %d\n", args->if_count - 1,
+			       cpu_count - 1);
+			exit(1);
+		}
+
+		/* check if one queue is configured twice or more */
+		if (queue_mask[q->if_idx][q->rxq_idx]) {
+			printf("Error queue config: re-config port: %d, "
+			       "queue: %d\n", q->if_idx, q->rxq_idx);
+			exit(1);
+		}
+		queue_mask[q->if_idx][q->rxq_idx] = 1;
+
+		port = &global.l3fwd_pktios[q->if_idx];
+		if (port->nb_rxq <= q->rxq_idx)
+			port->nb_rxq = q->rxq_idx + 1;
+
+		/* put the queue into worker_args */
+		arg = &global.worker_args[q->core_idx];
+		arg->qconf_args[arg->count] = q;
+		arg->count++;
+		arg->cpu = q->core_idx;
+	}
+}
diff --git a/example/l3fwd/odp_l3fwd_db.c b/example/l3fwd/odp_l3fwd_db.c
new file mode 100644
index 0000000..4767a9b
--- /dev/null
+++ b/example/l3fwd/odp_l3fwd_db.c
@@ -0,0 +1,381 @@ 
+/* Copyright (c) 2016, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+/* enable strtok */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <example_debug.h>
+
+#include <odp_api.h>
+
+#include <odp_l3fwd_db.h>
+#include <jhash.h>
+
+/**
+ * Compute hash value from a flow
+ */
+static inline
+uint64_t l3fwd_calc_hash(ipv4_tuple5_t *key)
+{
+	uint64_t l4_ports = 0;
+	uint32_t dst_ip, src_ip;
+
+	src_ip = key->src_ip;
+	dst_ip = key->dst_ip + JHASH_GOLDEN_RATIO;
+	FWD_BJ3_MIX(src_ip, dst_ip, l4_ports);
+
+	return l4_ports;
+}
+
+/**
+ * Parse text string representing an IPv4 address or subnet
+ *
+ * String is of the format "XXX.XXX.XXX.XXX(/W)" where
+ * "XXX" is decimal value and "/W" is optional subnet length
+ *
+ * @param ipaddress  Pointer to IP address/subnet string to convert
+ * @param addr       Pointer to return IPv4 address
+ * @param mask       Pointer (optional) to return IPv4 mask
+ *
+ * @return 0 if successful else -1
+ */
+static inline
+int parse_ipv4_string(char *ipaddress, uint32_t *addr, uint32_t *mask)
+{
+	int b[4];
+	int qualifier = 32;
+	int converted;
+
+	if (strchr(ipaddress, '/')) {
+		converted = sscanf(ipaddress, "%d.%d.%d.%d/%d",
+				   &b[3], &b[2], &b[1], &b[0],
+				   &qualifier);
+		if (5 != converted)
+			return -1;
+	} else {
+		converted = sscanf(ipaddress, "%d.%d.%d.%d",
+				   &b[3], &b[2], &b[1], &b[0]);
+		if (4 != converted)
+			return -1;
+	}
+
+	if ((b[0] > 255) || (b[1] > 255) || (b[2] > 255) || (b[3] > 255))
+		return -1;
+	if (!qualifier || (qualifier > 32))
+		return -1;
+
+	*addr = b[0] | b[1] << 8 | b[2] << 16 | b[3] << 24;
+	if (mask)
+		*mask = ~(0xFFFFFFFF & ((1ULL << (32 - qualifier)) - 1));
+
+	return 0;
+}
+
+/**
+ * Generate text string representing IPv4 range/subnet, output
+ * in "XXX.XXX.XXX.XXX/W" format
+ *
+ * @param b     Pointer to buffer to store string
+ * @param range Pointer to IPv4 address range
+ *
+ * @return Pointer to supplied buffer
+ */
+static inline
+char *ipv4_subnet_str(char *b, ip_addr_range_t *range)
+{
+	int idx;
+	int len;
+
+	for (idx = 0; idx < 32; idx++)
+		if (range->mask & (1 << idx))
+			break;
+	len = 32 - idx;
+
+	sprintf(b, "%03d.%03d.%03d.%03d/%d",
+		0xFF & ((range->addr) >> 24),
+		0xFF & ((range->addr) >> 16),
+		0xFF & ((range->addr) >>  8),
+		0xFF & ((range->addr) >>  0),
+		len);
+	return b;
+}
+
+/**
+ * Generate text string representing MAC address
+ *
+ * @param b     Pointer to buffer to store string
+ * @param mac   Pointer to MAC address
+ *
+ * @return Pointer to supplied buffer
+ */
+static inline
+char *mac_addr_str(char *b, uint8_t *mac)
+{
+	sprintf(b, "%02X.%02X.%02X.%02X.%02X.%02X",
+		mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
+	return b;
+}
+
+/**
+ * Flow cache table entry
+ */
+typedef struct flow_entry_s {
+	ipv4_tuple5_t key;		/**< match key */
+	struct flow_entry_s *next;      /**< next entry on the list */
+	fwd_db_entry_t *fwd_entry;	/**< entry info in db */
+} flow_entry_t;
+
+/**
+ * Flow cache table bucket
+ */
+typedef struct flow_bucket_s {
+	odp_spinlock_t		lock;	/**< Bucket lock*/
+	flow_entry_t		*next;	/**< Pointer to first flow entry in bucket*/
+} flow_bucket_t;
+
+/**
+ * Flow hash table, fast lookup cache
+ */
+typedef struct flow_table_s {
+	flow_bucket_t *bucket;
+	uint32_t count;
+} flow_table_t;
+
+static flow_table_t fwd_lookup_cache;
+
+static inline
+void init_fwd_cache(void)
+{
+	odp_shm_t		hash_shm;
+	flow_bucket_t		*bucket;
+	uint32_t		bucket_count;
+	uint32_t		i;
+
+	bucket_count = FWD_DEF_BUCKET_COUNT;
+
+	/*Reserve memory for Routing hash table*/
+	hash_shm = odp_shm_reserve("route_table",
+				   sizeof(flow_bucket_t) * bucket_count,
+				   ODP_CACHE_LINE_SIZE, 0);
+
+	bucket = odp_shm_addr(hash_shm);
+	if (!bucket) {
+		EXAMPLE_ERR("Error: shared mem alloc failed.\n");
+		exit(-1);
+	}
+
+	fwd_lookup_cache.bucket = bucket;
+	fwd_lookup_cache.count = bucket_count;
+
+	/*Initialize Locks*/
+	for (i = 0; i < bucket_count; i++) {
+		bucket = &fwd_lookup_cache.bucket[i];
+		odp_spinlock_init(&bucket->lock);
+		bucket->next = NULL;
+	}
+}
+
+static inline
+int match_key_flow(ipv4_tuple5_t *key, flow_entry_t *flow)
+{
+	if (key->src_ip == flow->key.src_ip &&
+	    key->dst_ip == flow->key.dst_ip &&
+	    key->src_port == flow->key.src_port &&
+	    key->dst_port == flow->key.dst_port &&
+	    key->proto == flow->key.proto)
+		return 1;
+
+	return 0;
+}
+
+static inline
+flow_entry_t *lookup_fwd_cache(ipv4_tuple5_t *key, flow_bucket_t *bucket)
+{
+	flow_entry_t *rst;
+
+	for (rst = bucket->next; rst != NULL; rst = rst->next) {
+		if (match_key_flow(key, rst))
+			break;
+	}
+
+	return rst;
+}
+
+static inline
+flow_entry_t *insert_fwd_cache(ipv4_tuple5_t *key,
+			       flow_bucket_t *bucket,
+			       fwd_db_entry_t *entry)
+{
+	flow_entry_t *flow;
+
+	flow = lookup_fwd_cache(key, bucket);
+	if (flow)
+		return flow;
+
+	flow = malloc(sizeof(flow_entry_t));
+	flow->key = *key;
+	flow->fwd_entry = entry;
+
+	odp_spinlock_lock(&bucket->lock);
+	if (!bucket->next) {
+		bucket->next = flow;
+	} else {
+		flow->next = bucket->next;
+		bucket->next = flow;
+	}
+	odp_spinlock_unlock(&bucket->lock);
+
+	return flow;
+}
+
+/** Global pointer to fwd db */
+fwd_db_t *fwd_db;
+
+void init_fwd_db(void)
+{
+	odp_shm_t shm;
+
+	shm = odp_shm_reserve("shm_fwd_db",
+			      sizeof(fwd_db_t),
+			      ODP_CACHE_LINE_SIZE,
+			      0);
+
+	fwd_db = odp_shm_addr(shm);
+
+	if (fwd_db == NULL) {
+		EXAMPLE_ERR("Error: shared mem alloc failed.\n");
+		exit(EXIT_FAILURE);
+	}
+	memset(fwd_db, 0, sizeof(*fwd_db));
+
+	init_fwd_cache();
+}
+
+int create_fwd_db_entry(char *input)
+{
+	int pos = 0;
+	char *local;
+	char *str;
+	char *save;
+	char *token;
+	fwd_db_entry_t *entry = &fwd_db->array[fwd_db->index];
+
+	/* Verify we haven't run out of space */
+	if (MAX_DB <= fwd_db->index)
+		return -1;
+
+	/* Make a local copy */
+	local = malloc(strlen(input) + 1);
+	if (NULL == local)
+		return -1;
+	strcpy(local, input);
+
+	/* Setup for using "strtok_r" to search input string */
+	str = local;
+	save = NULL;
+
+	/* Parse tokens separated by ':' */
+	while (NULL != (token = strtok_r(str, ":", &save))) {
+		str = NULL;  /* reset str for subsequent strtok_r calls */
+
+		/* Parse token based on its position */
+		switch (pos) {
+		case 0:
+			parse_ipv4_string(token,
+					  &entry->subnet.addr,
+					  &entry->subnet.mask);
+			break;
+		case 1:
+			strncpy(entry->oif, token, OIF_LEN - 1);
+			entry->oif[OIF_LEN - 1] = 0;
+			break;
+		case 2:
+			odph_eth_addr_parse((odph_ethaddr_t *)entry->dst_mac,
+					    token);
+			break;
+
+		default:
+			printf("ERROR: extra token \"%s\" at position %d\n",
+			       token, pos);
+			break;
+		}
+
+		/* Advance to next position */
+		pos++;
+	}
+
+	/* Add route to the list */
+	fwd_db->index++;
+	entry->next = fwd_db->list;
+	fwd_db->list = entry;
+
+	free(local);
+	return 0;
+}
+
+void resolve_fwd_db(char *intf, odp_pktout_queue_t outq, uint8_t *mac)
+{
+	fwd_db_entry_t *entry;
+
+	/* Walk the list and attempt to set output and MAC */
+	for (entry = fwd_db->list; NULL != entry; entry = entry->next) {
+		if (strcmp(intf, entry->oif))
+			continue;
+
+		entry->outq = outq;
+		memcpy(entry->src_mac, mac, ODPH_ETHADDR_LEN);
+	}
+}
+
+void dump_fwd_db_entry(fwd_db_entry_t *entry)
+{
+	char subnet_str[MAX_STRING];
+	char mac_str[MAX_STRING];
+
+	printf(" %s %s %s\n",
+	       ipv4_subnet_str(subnet_str, &entry->subnet),
+	       entry->oif,
+	       mac_addr_str(mac_str, entry->dst_mac));
+}
+
+void dump_fwd_db(void)
+{
+	fwd_db_entry_t *entry;
+
+	printf("\n"
+	       "Routing table\n"
+	       "-------------\n");
+
+	for (entry = fwd_db->list; NULL != entry; entry = entry->next)
+		dump_fwd_db_entry(entry);
+}
+
+fwd_db_entry_t *find_fwd_db_entry(ipv4_tuple5_t *key)
+{
+	fwd_db_entry_t *entry;
+	flow_entry_t *flow;
+	flow_bucket_t *bucket;
+	uint64_t hash;
+
+	/* first find in cache */
+	hash = l3fwd_calc_hash(key);
+	hash &= fwd_lookup_cache.count - 1;
+	bucket = &fwd_lookup_cache.bucket[hash];
+	flow = lookup_fwd_cache(key, bucket);
+	if (flow)
+		return flow->fwd_entry;
+
+	for (entry = fwd_db->list; NULL != entry; entry = entry->next)
+		if (entry->subnet.addr == (key->dst_ip & entry->subnet.mask))
+			break;
+
+	return entry;
+}
diff --git a/example/l3fwd/odp_l3fwd_db.h b/example/l3fwd/odp_l3fwd_db.h
new file mode 100644
index 0000000..eac142e
--- /dev/null
+++ b/example/l3fwd/odp_l3fwd_db.h
@@ -0,0 +1,122 @@ 
+/* Copyright (c) 2016, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef ODP_L3FWD_DB_H_
+#define ODP_L3FWD_DB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_api.h>
+#include <odp/helper/eth.h>
+
+#define OIF_LEN 32
+#define MAX_DB  32
+#define MAX_STRING  32
+
+/**
+ * Default number of flows
+ */
+#define FWD_DEF_FLOW_COUNT		100000
+
+/**
+ * Default Hash bucket number
+ */
+#define FWD_DEF_BUCKET_COUNT	(FWD_DEF_FLOW_COUNT / 8)
+
+/**
+ * IP address range (subnet)
+ */
+typedef struct ip_addr_range_s {
+	uint32_t  addr;     /**< IP address */
+	uint32_t  mask;     /**< mask, 1 indicates bits are valid */
+} ip_addr_range_t;
+
+/**
+ * TCP/UDP flow
+ */
+typedef struct ipv4_tuple5_s {
+	uint32_t src_ip;
+	uint32_t dst_ip;
+	uint16_t src_port;
+	uint16_t dst_port;
+	uint8_t  proto;
+} ipv4_tuple5_t;
+
+/**
+ * Forwarding data base entry
+ */
+typedef struct fwd_db_entry_s {
+	struct fwd_db_entry_s *next;          /**< Next entry on list */
+	char                    oif[OIF_LEN]; /**< Output interface name */
+	odp_pktout_queue_t	outq;	      /**< Output transmit queue */
+	uint8_t   src_mac[ODPH_ETHADDR_LEN];  /**< Output source MAC */
+	uint8_t   dst_mac[ODPH_ETHADDR_LEN];  /**< Output destination MAC */
+	ip_addr_range_t        subnet;        /**< Subnet for this router */
+} fwd_db_entry_t;
+
+/**
+ * Forwarding data base
+ */
+typedef struct fwd_db_s {
+	uint32_t          index;          /**< Next available entry */
+	fwd_db_entry_t   *list;           /**< List of active routes */
+	fwd_db_entry_t    array[MAX_DB];  /**< Entry storage */
+} fwd_db_t;
+
+/** Global pointer to fwd db */
+extern fwd_db_t *fwd_db;
+
+/** Initialize FWD DB */
+void init_fwd_db(void);
+
+/**
+ * Create a forwarding database entry
+ *
+ * String is of the format "SubNet:Intf:NextHopMAC"
+ *
+ * @param input  Pointer to string describing route
+ *
+ * @return 0 if successful else -1
+ */
+int create_fwd_db_entry(char *input);
+
+/**
+ * Scan FWD DB entries and resolve output queue and source MAC address
+ *
+ * @param intf   Interface name string
+ * @param outq   Output queue for packet transmit
+ * @param mac    MAC address of this interface
+ */
+void resolve_fwd_db(char *intf, odp_pktout_queue_t outq, uint8_t *mac);
+
+/**
+ * Display one forwarding database entry
+ *
+ * @param entry  Pointer to entry to display
+ */
+void dump_fwd_db_entry(fwd_db_entry_t *entry);
+
+/**
+ * Display the forwarding database
+ */
+void dump_fwd_db(void);
+
+/**
+ * Find a matching forwarding database entry
+ *
+ * @param key  ipv4 tuple
+ *
+ * @return pointer to forwarding DB entry else NULL
+ */
+fwd_db_entry_t *find_fwd_db_entry(ipv4_tuple5_t *key);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/example/m4/configure.m4 b/example/m4/configure.m4
index 9731d81..7868574 100644
--- a/example/m4/configure.m4
+++ b/example/m4/configure.m4
@@ -19,4 +19,5 @@  AC_CONFIG_FILES([example/classifier/Makefile
 		 example/timer/Makefile
 		 example/traffic_mgmt/Makefile
 		 example/l2fwd_simple/Makefile
+		 example/l3fwd/Makefile
 		 example/switch/Makefile])