[1/2,v1] examples: ipsec: tunnel mode support

From: Alexandru Badicioiu <alexandru.badicioiu@linaro.org>

v1 - added comment for tunnel DB entry use

Tunnel mode is enabled from the command line using -t argument with
the following format: SrcIP:DstIP:TunnelSrcIP:TunnelDstIP.
SrcIP - cleartext packet source IP
DstIP - cleartext packet destination IP
TunnelSrcIP - tunnel source IP
TunnelDstIP - tunnel destination IP

The outbound packets matching SrcIP:DstIP will be encapsulated
in a TunnelSrcIP:TunnelDstIP IPSec tunnel (AH/ESP/AH+ESP)
if a matching outbound SA is determined (as for transport mode).
For inbound packets each entry in the IPSec cache is matched
for the cleartext addresses, as in the transport mode (SrcIP:DstIP)
and then for the tunnel addresses (TunnelSrcIP:TunnelDstIP)
in case cleartext addresses didn't match. After authentication and
decryption tunneled packets are verified against the tunnel entry
(packets came in from the expected tunnel).

Signed-off-by: Alexandru Badicioiu <alexandru.badicioiu@linaro.org>
---
 example/ipsec/odp_ipsec.c        |  105 +++++++++++++++++++++++++++---
 example/ipsec/odp_ipsec_cache.c  |   31 +++++++++-
 example/ipsec/odp_ipsec_cache.h  |    6 ++
 example/ipsec/odp_ipsec_sa_db.c  |  133 +++++++++++++++++++++++++++++++++++++-
 example/ipsec/odp_ipsec_sa_db.h  |   57 ++++++++++++++++
 example/ipsec/odp_ipsec_stream.c |  101 ++++++++++++++++++++++++----
 6 files changed, 403 insertions(+), 30 deletions(-)

CC Steve, it he is not in mailing list.

Maxim.

On 05/19/2015 11:38, alexandru.badicioiu@linaro.org wrote:
> From: Alexandru Badicioiu <alexandru.badicioiu@linaro.org>
>
> v1 - added comment for tunnel DB entry use
>
> Tunnel mode is enabled from the command line using -t argument with
> the following format: SrcIP:DstIP:TunnelSrcIP:TunnelDstIP.
> SrcIP - cleartext packet source IP
> DstIP - cleartext packet destination IP
> TunnelSrcIP - tunnel source IP
> TunnelDstIP - tunnel destination IP
>
> The outbound packets matching SrcIP:DstIP will be encapsulated
> in a TunnelSrcIP:TunnelDstIP IPSec tunnel (AH/ESP/AH+ESP)
> if a matching outbound SA is determined (as for transport mode).
> For inbound packets each entry in the IPSec cache is matched
> for the cleartext addresses, as in the transport mode (SrcIP:DstIP)
> and then for the tunnel addresses (TunnelSrcIP:TunnelDstIP)
> in case cleartext addresses didn't match. After authentication and
> decryption tunneled packets are verified against the tunnel entry
> (packets came in from the expected tunnel).
>
> Signed-off-by: Alexandru Badicioiu <alexandru.badicioiu@linaro.org>
> ---
>   example/ipsec/odp_ipsec.c        |  105 +++++++++++++++++++++++++++---
>   example/ipsec/odp_ipsec_cache.c  |   31 +++++++++-
>   example/ipsec/odp_ipsec_cache.h  |    6 ++
>   example/ipsec/odp_ipsec_sa_db.c  |  133 +++++++++++++++++++++++++++++++++++++-
>   example/ipsec/odp_ipsec_sa_db.h  |   57 ++++++++++++++++
>   example/ipsec/odp_ipsec_stream.c |  101 ++++++++++++++++++++++++----
>   6 files changed, 403 insertions(+), 30 deletions(-)
>
> diff --git a/example/ipsec/odp_ipsec.c b/example/ipsec/odp_ipsec.c
> index 82ed0cb..3931fef 100644
> --- a/example/ipsec/odp_ipsec.c
> +++ b/example/ipsec/odp_ipsec.c
> @@ -135,13 +135,20 @@ typedef struct {
>   	uint8_t  ip_ttl;         /**< Saved IP TTL value */
>   	int      hdr_len;        /**< Length of IPsec headers */
>   	int      trl_len;        /**< Length of IPsec trailers */
> +	uint16_t tun_hdr_offset; /**< Offset of tunnel header from
> +				      buffer start */
>   	uint16_t ah_offset;      /**< Offset of AH header from buffer start */
>   	uint16_t esp_offset;     /**< Offset of ESP header from buffer start */
>   
> +	/* Input only */
> +	uint32_t src_ip;         /**< SA source IP address */
> +	uint32_t dst_ip;         /**< SA dest IP address */
> +
>   	/* Output only */
>   	odp_crypto_op_params_t params;  /**< Parameters for crypto call */
>   	uint32_t *ah_seq;               /**< AH sequence number location */
>   	uint32_t *esp_seq;              /**< ESP sequence number location */
> +	uint16_t *tun_hdr_id;           /**< Tunnel header ID > */
>   } ipsec_ctx_t;
>   
>   /**
> @@ -368,6 +375,7 @@ void ipsec_init_pre(void)
>   	/* Initialize our data bases */
>   	init_sp_db();
>   	init_sa_db();
> +	init_tun_db();
>   	init_ipsec_cache();
>   }
>   
> @@ -387,19 +395,27 @@ void ipsec_init_post(crypto_api_mode_e api_mode)
>   	for (entry = sp_db->list; NULL != entry; entry = entry->next) {
>   		sa_db_entry_t *cipher_sa = NULL;
>   		sa_db_entry_t *auth_sa = NULL;
> +		tun_db_entry_t *tun;
>   
> -		if (entry->esp)
> +		if (entry->esp) {
>   			cipher_sa = find_sa_db_entry(&entry->src_subnet,
>   						     &entry->dst_subnet,
>   						     1);
> -		if (entry->ah)
> +			tun = find_tun_db_entry(cipher_sa->src_ip,
> +						      cipher_sa->dst_ip);
> +		}
> +		if (entry->ah) {
>   			auth_sa = find_sa_db_entry(&entry->src_subnet,
>   						   &entry->dst_subnet,
>   						   0);
> +			tun = find_tun_db_entry(auth_sa->src_ip,
> +						      auth_sa->dst_ip);
> +		}
>   
>   		if (cipher_sa || auth_sa) {
>   			if (create_ipsec_cache_entry(cipher_sa,
>   						     auth_sa,
> +						     tun,
>   						     api_mode,
>   						     entry->input,
>   						     completionq,
> @@ -672,6 +688,8 @@ pkt_disposition_e do_ipsec_in_classify(odp_packet_t pkt,
>   	ctx->ipsec.esp_offset = esp ? ((uint8_t *)esp) - buf : 0;
>   	ctx->ipsec.hdr_len = hdr_len;
>   	ctx->ipsec.trl_len = 0;
> +	ctx->ipsec.src_ip = entry->src_ip;
> +	ctx->ipsec.dst_ip = entry->dst_ip;
>   
>   	/*If authenticating, zero the mutable fields build the request */
>   	if (ah) {
> @@ -752,6 +770,23 @@ pkt_disposition_e do_ipsec_in_finish(odp_packet_t pkt,
>   		trl_len += esp_t->pad_len + sizeof(*esp_t);
>   	}
>   
> +	/* We have a tunneled IPv4 packet */
> +	if (ip->proto == ODPH_IPV4) {
> +		odp_packet_pull_head(pkt, sizeof(*ip) + hdr_len);
> +		odp_packet_pull_tail(pkt, trl_len);
> +		odph_ethhdr_t *eth;
> +		eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL);
> +		eth->type = ODPH_ETHTYPE_IPV4;
> +		ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, NULL);
> +
> +		/* Check inbound policy */
> +		if ((ip->src_addr != ctx->ipsec.src_ip ||
> +		     ip->dst_addr != ctx->ipsec.dst_ip))
> +			return PKT_DROP;
> +
> +		return PKT_CONTINUE;
> +	}
> +
>   	/* Finalize the IPv4 header */
>   	ipv4_adjust_len(ip, -(hdr_len + trl_len));
>   	ip->ttl = ctx->ipsec.ip_ttl;
> @@ -823,9 +858,13 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt,
>   	params.pkt = pkt;
>   	params.out_pkt = entry->in_place ? pkt : ODP_PACKET_INVALID;
>   
> +	if (entry->mode == IPSEC_SA_MODE_TUNNEL) {
> +		hdr_len += sizeof(odph_ipv4hdr_t);
> +		ip_data = (uint8_t *)ip;
> +	}
>   	/* Compute ah and esp, determine length of headers, move the data */
>   	if (entry->ah.alg) {
> -		ah = (odph_ahhdr_t *)(ip_data);
> +		ah = (odph_ahhdr_t *)(ip_data + hdr_len);
>   		hdr_len += sizeof(odph_ahhdr_t);
>   		hdr_len += entry->ah.icv_len;
>   	}
> @@ -837,21 +876,39 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt,
>   	memmove(ip_data + hdr_len, ip_data, ip_data_len);
>   	ip_data += hdr_len;
>   
> +	/* update outer header in tunnel mode */
> +	if (entry->mode == IPSEC_SA_MODE_TUNNEL) {
> +		/* tunnel addresses */
> +		ip->src_addr = odp_cpu_to_be_32(entry->tun_src_ip);
> +		ip->dst_addr = odp_cpu_to_be_32(entry->tun_dst_ip);
> +	}
> +
>   	/* For cipher, compute encrypt length, build headers and request */
>   	if (esp) {
>   		uint32_t encrypt_len;
>   		odph_esptrl_t *esp_t;
>   
> -		encrypt_len = ESP_ENCODE_LEN(ip_data_len + sizeof(*esp_t),
> -					     entry->esp.block_len);
> -		trl_len = encrypt_len - ip_data_len;
> +		if (entry->mode == IPSEC_SA_MODE_TUNNEL) {
> +			encrypt_len = ESP_ENCODE_LEN(ip->tot_len +
> +						     sizeof(*esp_t),
> +						     entry->esp.block_len);
> +			trl_len = encrypt_len - ip->tot_len;
> +		} else {
> +			encrypt_len = ESP_ENCODE_LEN(ip_data_len +
> +						     sizeof(*esp_t),
> +						     entry->esp.block_len);
> +			trl_len = encrypt_len - ip_data_len;
> +		}
>   
>   		esp->spi = odp_cpu_to_be_32(entry->esp.spi);
>   		memcpy(esp + 1, entry->state.iv, entry->esp.iv_len);
>   
>   		esp_t = (odph_esptrl_t *)(ip_data + encrypt_len) - 1;
>   		esp_t->pad_len     = trl_len - sizeof(*esp_t);
> -		esp_t->next_header = ip->proto;
> +		if (entry->mode == IPSEC_SA_MODE_TUNNEL)
> +			esp_t->next_header = ODPH_IPV4;
> +		else
> +			esp_t->next_header = ip->proto;
>   		ip->proto = ODPH_IPPROTO_ESP;
>   
>   		params.cipher_range.offset = ip_data - buf;
> @@ -863,7 +920,10 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt,
>   		memset(ah, 0, sizeof(*ah) + entry->ah.icv_len);
>   		ah->spi = odp_cpu_to_be_32(entry->ah.spi);
>   		ah->ah_len = 1 + (entry->ah.icv_len / 4);
> -		ah->next_header = ip->proto;
> +		if (entry->mode == IPSEC_SA_MODE_TUNNEL && !esp)
> +			ah->next_header = ODPH_IPV4;
> +		else
> +			ah->next_header = ip->proto;
>   		ip->proto = ODPH_IPPROTO_AH;
>   
>   		ip->chksum = 0;
> @@ -886,8 +946,11 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt,
>   	ctx->ipsec.trl_len = trl_len;
>   	ctx->ipsec.ah_offset = ah ? ((uint8_t *)ah) - buf : 0;
>   	ctx->ipsec.esp_offset = esp ? ((uint8_t *)esp) - buf : 0;
> +	ctx->ipsec.tun_hdr_offset = (entry->mode == IPSEC_SA_MODE_TUNNEL) ?
> +				       ((uint8_t *)ip - buf) : 0;
>   	ctx->ipsec.ah_seq = &entry->state.ah_seq;
>   	ctx->ipsec.esp_seq = &entry->state.esp_seq;
> +	ctx->ipsec.tun_hdr_id = &entry->state.tun_hdr_id;
>   	memcpy(&ctx->ipsec.params, &params, sizeof(params));
>   
>   	*skip = FALSE;
> @@ -926,6 +989,20 @@ pkt_disposition_e do_ipsec_out_seq(odp_packet_t pkt,
>   		esp = (odph_esphdr_t *)(ctx->ipsec.esp_offset + buf);
>   		esp->seq_no = odp_cpu_to_be_32((*ctx->ipsec.esp_seq)++);
>   	}
> +	if (ctx->ipsec.tun_hdr_offset) {
> +		odph_ipv4hdr_t *ip;
> +		int ret;
> +		ip = (odph_ipv4hdr_t *)(ctx->ipsec.tun_hdr_offset + buf);
> +		ip->id = odp_cpu_to_be_16((*ctx->ipsec.tun_hdr_id)++);
> +		if (!ip->id) {
> +			/* re-init tunnel hdr id */
> +			ret = odp_random_data((uint8_t *)ctx->ipsec.tun_hdr_id,
> +					      sizeof(*ctx->ipsec.tun_hdr_id),
> +					      1);
> +			if (ret != sizeof(*ctx->ipsec.tun_hdr_id))
> +				abort();
> +		}
> +	}
>   
>   	/* Issue crypto request */
>   	if (odp_crypto_operation(&ctx->ipsec.params,
> @@ -1304,8 +1381,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
>   		{"mode", required_argument, NULL, 'm'},		/* return 'm' */
>   		{"route", required_argument, NULL, 'r'},	/* return 'r' */
>   		{"policy", required_argument, NULL, 'p'},	/* return 'p' */
> -		{"ah", required_argument, NULL, 'a'},	        /* return 'a' */
> -		{"esp", required_argument, NULL, 'e'},	        /* return 'e' */
> +		{"ah", required_argument, NULL, 'a'},		/* return 'a' */
> +		{"esp", required_argument, NULL, 'e'},		/* return 'e' */
> +		{"tunnel", required_argument, NULL, 't'},       /* return 't' */
>   		{"stream", required_argument, NULL, 's'},	/* return 's' */
>   		{"help", no_argument, NULL, 'h'},		/* return 'h' */
>   		{NULL, 0, NULL, 0}
> @@ -1316,7 +1394,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
>   	appl_args->mode = 0;  /* turn off async crypto API by default */
>   
>   	while (!rc) {
> -		opt = getopt_long(argc, argv, "+c:i:m:h:r:p:a:e:s:",
> +		opt = getopt_long(argc, argv, "+c:i:m:h:r:p:a:e:t:s:",
>   				  longopts, &long_index);
>   
>   		if (-1 == opt)
> @@ -1389,6 +1467,10 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
>   			rc = create_sa_db_entry(optarg, TRUE);
>   			break;
>   
> +		case 't':
> +			rc = create_tun_db_entry(optarg);
> +			break;
> +
>   		case 's':
>   			rc = create_stream_db_entry(optarg);
>   			break;
> @@ -1449,6 +1531,7 @@ static void print_info(char *progname, appl_args_t *appl_args)
>   	dump_fwd_db();
>   	dump_sp_db();
>   	dump_sa_db();
> +	dump_tun_db();
>   	printf("\n\n");
>   	fflush(NULL);
>   }
> diff --git a/example/ipsec/odp_ipsec_cache.c b/example/ipsec/odp_ipsec_cache.c
> index 12b960d..046e43c 100644
> --- a/example/ipsec/odp_ipsec_cache.c
> +++ b/example/ipsec/odp_ipsec_cache.c
> @@ -38,6 +38,7 @@ void init_ipsec_cache(void)
>   
>   int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   			     sa_db_entry_t *auth_sa,
> +			     tun_db_entry_t *tun,
>   			     crypto_api_mode_e api_mode,
>   			     odp_bool_t in,
>   			     odp_queue_t completionq,
> @@ -47,12 +48,18 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   	ipsec_cache_entry_t *entry;
>   	enum odp_crypto_ses_create_err ses_create_rc;
>   	odp_crypto_session_t session;
> +	sa_mode_t mode = IPSEC_SA_MODE_TRANSPORT;
>   
>   	/* Verify we have a good entry */
>   	entry = &ipsec_cache->array[ipsec_cache->index];
>   	if (MAX_DB <= ipsec_cache->index)
>   		return -1;
>   
> +	/* Verify SA mode match in case of cipher&auth */
> +	if (cipher_sa && auth_sa &&
> +	    (cipher_sa->mode != auth_sa->mode))
> +		return -1;
> +
>   	/* Setup parameters and call crypto library to create session */
>   	params.op = (in) ? ODP_CRYPTO_OP_DECODE : ODP_CRYPTO_OP_ENCODE;
>   	params.auth_cipher_text = TRUE;
> @@ -79,6 +86,7 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   		params.cipher_key.length  = cipher_sa->key.length;
>   		params.iv.data = entry->state.iv;
>   		params.iv.length = cipher_sa->iv_len;
> +		mode = cipher_sa->mode;
>   	} else {
>   		params.cipher_alg = ODP_CIPHER_ALG_NULL;
>   		params.iv.data = NULL;
> @@ -90,6 +98,7 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   		params.auth_alg = auth_sa->alg.u.auth;
>   		params.auth_key.data = auth_sa->key.data;
>   		params.auth_key.length = auth_sa->key.length;
> +		mode = auth_sa->mode;
>   	} else {
>   		params.auth_alg = ODP_AUTH_ALG_NULL;
>   	}
> @@ -128,6 +137,24 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   		memcpy(&entry->ah.key, &auth_sa->key, sizeof(ipsec_key_t));
>   	}
>   
> +	if (tun) {
> +		entry->tun_src_ip = tun->tun_src_ip;
> +		entry->tun_dst_ip = tun->tun_dst_ip;
> +		mode = IPSEC_SA_MODE_TUNNEL;
> +
> +		int ret;
> +		if (!in) {
> +			/* init tun hdr id */
> +			ret = odp_random_data((uint8_t *)
> +					      &entry->state.tun_hdr_id,
> +					      sizeof(entry->state.tun_hdr_id),
> +					      1);
> +			if (ret != sizeof(entry->state.tun_hdr_id))
> +				return -1;
> +		}
> +	}
> +	entry->mode = mode;
> +
>   	/* Initialize state */
>   	entry->state.esp_seq = 0;
>   	entry->state.ah_seq = 0;
> @@ -156,7 +183,9 @@ ipsec_cache_entry_t *find_ipsec_cache_entry_in(uint32_t src_ip,
>   	/* Look for a hit */
>   	for (; NULL != entry; entry = entry->next) {
>   		if ((entry->src_ip != src_ip) || (entry->dst_ip != dst_ip))
> -			continue;
> +			if ((entry->tun_src_ip != src_ip) ||
> +			    (entry->tun_dst_ip != dst_ip))
> +				continue;
>   		if (ah &&
>   		    ((!entry->ah.alg) ||
>   		     (entry->ah.spi != odp_be_to_cpu_32(ah->spi))))
> diff --git a/example/ipsec/odp_ipsec_cache.h b/example/ipsec/odp_ipsec_cache.h
> index 714cae8..5706007 100644
> --- a/example/ipsec/odp_ipsec_cache.h
> +++ b/example/ipsec/odp_ipsec_cache.h
> @@ -34,6 +34,9 @@ typedef struct ipsec_cache_entry_s {
>   	odp_bool_t                   in_place;    /**< Crypto API mode */
>   	uint32_t                     src_ip;      /**< Source v4 address */
>   	uint32_t                     dst_ip;      /**< Destination v4 address */
> +	sa_mode_t		     mode;        /**< SA mode - transport/tun */
> +	uint32_t                     tun_src_ip;  /**< Tunnel src IPv4 addr */
> +	uint32_t                     tun_dst_ip;  /**< Tunnel dst IPv4 addr */
>   	struct {
>   		enum  odp_cipher_alg alg;         /**< Cipher algorithm */
>   		uint32_t             spi;         /**< Cipher SPI */
> @@ -54,6 +57,7 @@ typedef struct ipsec_cache_entry_s {
>   		uint32_t      esp_seq;         /**< ESP TX sequence number */
>   		uint32_t      ah_seq;          /**< AH TX sequence number */
>   		uint8_t       iv[MAX_IV_LEN];  /**< ESP IV storage */
> +		uint16be_t    tun_hdr_id;      /**< Tunnel header IP ID */
>   	} state;
>   } ipsec_cache_entry_t;
>   
> @@ -78,6 +82,7 @@ void init_ipsec_cache(void);
>    *
>    * @param cipher_sa   Cipher SA DB entry pointer
>    * @param auth_sa     Auth SA DB entry pointer
> + * @param tun         Tunnel DB entry pointer
>    * @param api_mode    Crypto API mode for testing
>    * @param in          Direction (input versus output)
>    * @param completionq Completion queue
> @@ -87,6 +92,7 @@ void init_ipsec_cache(void);
>    */
>   int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   			     sa_db_entry_t *auth_sa,
> +			     tun_db_entry_t *tun,
>   			     crypto_api_mode_e api_mode,
>   			     odp_bool_t in,
>   			     odp_queue_t completionq,
> diff --git a/example/ipsec/odp_ipsec_sa_db.c b/example/ipsec/odp_ipsec_sa_db.c
> index 5837cb6..78f43da 100644
> --- a/example/ipsec/odp_ipsec_sa_db.c
> +++ b/example/ipsec/odp_ipsec_sa_db.c
> @@ -1,7 +1,7 @@
>   /* Copyright (c) 2014, Linaro Limited
>    * All rights reserved.
>    *
> - * SPDX-License-Identifier:     BSD-3-Clause
> + * SPDX-License-Identifier:	BSD-3-Clause
>    */
>   
>   /* enable strtok */
> @@ -19,6 +19,9 @@
>   /** Global pointer to sa db */
>   static sa_db_t *sa_db;
>   
> +/** Global pointer to tun db */
> +static tun_db_t *tun_db;
> +
>   void init_sa_db(void)
>   {
>   	odp_shm_t shm;
> @@ -37,6 +40,22 @@ void init_sa_db(void)
>   	memset(sa_db, 0, sizeof(*sa_db));
>   }
>   
> +void init_tun_db(void)
> +{
> +	odp_shm_t shm;
> +	shm = odp_shm_reserve("shm_tun_db",
> +			      sizeof(tun_db_t),
> +			      ODP_CACHE_LINE_SIZE,
> +			      0);
> +	tun_db = odp_shm_addr(shm);
> +
> +	if (tun_db == NULL) {
> +		EXAMPLE_ERR("Error: shared mem alloc failed.\n");
> +		exit(EXIT_FAILURE);
> +	}
> +	memset(tun_db, 0, sizeof(*tun_db));
> +}
> +
>   int create_sa_db_entry(char *input, odp_bool_t cipher)
>   {
>   	int pos = 0;
> @@ -81,7 +100,7 @@ int create_sa_db_entry(char *input, odp_bool_t cipher)
>   					entry->alg.u.cipher =
>   						ODP_CIPHER_ALG_3DES_CBC;
>   					entry->block_len  = 8;
> -					entry->iv_len     = 8;
> +					entry->iv_len	  = 8;
>   				} else {
>   					entry->alg.u.cipher =
>   						ODP_CIPHER_ALG_NULL;
> @@ -90,7 +109,7 @@ int create_sa_db_entry(char *input, odp_bool_t cipher)
>   				if (0 == strcmp(token, "md5")) {
>   					entry->alg.u.auth =
>   						ODP_AUTH_ALG_MD5_96;
> -					entry->icv_len    = 12;
> +					entry->icv_len	  = 12;
>   				} else {
>   					entry->alg.u.auth = ODP_AUTH_ALG_NULL;
>   				}
> @@ -132,6 +151,89 @@ int create_sa_db_entry(char *input, odp_bool_t cipher)
>   	return 0;
>   }
>   
> +int create_tun_db_entry(char *input)
> +{
> +	int pos = 0;
> +	char *local;
> +	char *str;
> +	char *save;
> +	char *token;
> +	tun_db_entry_t *entry = &tun_db->array[tun_db->index];
> +
> +	/* Verify we have a good entry */
> +	if (MAX_DB <= tun_db->index)
> +		return -1;
> +
> +	/* Make a local copy */
> +	local = malloc(strlen(input) + 1);
> +	if (NULL == local)
> +		return -1;
> +	strcpy(local, input);
> +
> +	/* Setup for using "strtok_r" to search input string */
> +	str = local;
> +	save = NULL;
> +
> +	/* Parse tokens separated by ':' */
> +	while (NULL != (token = strtok_r(str, ":", &save))) {
> +		str = NULL;  /* reset str for subsequent strtok_r calls */
> +
> +		/* Parse token based on its position */
> +		switch (pos) {
> +		case 0:
> +			parse_ipv4_string(token, &entry->src_ip, NULL);
> +			break;
> +		case 1:
> +			parse_ipv4_string(token, &entry->dst_ip, NULL);
> +			break;
> +		case 2:
> +			parse_ipv4_string(token, &entry->tun_src_ip, NULL);
> +			break;
> +		case 3:
> +			parse_ipv4_string(token, &entry->tun_dst_ip, NULL);
> +			break;
> +		default:
> +			printf("ERROR: extra token \"%s\" at position %d\n",
> +			       token, pos);
> +			break;
> +		}
> +		pos++;
> +	}
> +
> +	/* Verify we parsed exactly the number of tokens we expected */
> +	if (4 != pos) {
> +		printf("ERROR: \"%s\" contains %d tokens, expected 4\n",
> +		       input,
> +		       pos);
> +		free(local);
> +		return -1;
> +	}
> +
> +	/* Add route to the list */
> +	tun_db->index++;
> +	entry->next = tun_db->list;
> +	tun_db->list = entry;
> +
> +	free(local);
> +	return 0;
> +}
> +
> +tun_db_entry_t *find_tun_db_entry(uint32_t ip_src,
> +					uint32_t ip_dst)
> +{
> +	tun_db_entry_t *entry = NULL;
> +
> +	/* Scan all entries and return first match */
> +	for (entry = tun_db->list; NULL != entry; entry = entry->next) {
> +		if (entry->src_ip != ip_src)
> +			continue;
> +		if (entry->dst_ip != ip_dst)
> +			continue;
> +		break;
> +	}
> +	return entry;
> +}
> +
>   void dump_sa_db(void)
>   {
>   	sa_db_entry_t *entry;
> @@ -182,3 +284,28 @@ sa_db_entry_t *find_sa_db_entry(ip_addr_range_t *src,
>   	}
>   	return entry;
>   }
> +
> +void dump_tun_db(void)
> +{
> +	tun_db_entry_t *entry;
> +
> +	printf("\n"
> +	       "Tunnel table\n"
> +	       "--------------------------\n");
> +
> +	for (entry = tun_db->list; NULL != entry; entry = entry->next) {
> +		char src_ip_str[MAX_STRING];
> +		char dst_ip_str[MAX_STRING];
> +		char tun_src_ip_str[MAX_STRING];
> +		char tun_dst_ip_str[MAX_STRING];
> +
> +		printf(" %s:%s %s:%s ",
> +		       ipv4_addr_str(src_ip_str, entry->src_ip),
> +		       ipv4_addr_str(dst_ip_str, entry->dst_ip),
> +		       ipv4_addr_str(tun_src_ip_str, entry->tun_src_ip),
> +		       ipv4_addr_str(tun_dst_ip_str, entry->tun_dst_ip)
> +		      );
> +
> +		printf("\n");
> +	}
> +}
> diff --git a/example/ipsec/odp_ipsec_sa_db.h b/example/ipsec/odp_ipsec_sa_db.h
> index c30cbdb..79bfc78 100644
> --- a/example/ipsec/odp_ipsec_sa_db.h
> +++ b/example/ipsec/odp_ipsec_sa_db.h
> @@ -13,6 +13,10 @@ extern "C" {
>   
>   #include <odp_ipsec_misc.h>
>   
> +typedef enum sa_mode_s {
> +	IPSEC_SA_MODE_TRANSPORT,
> +	IPSEC_SA_MODE_TUNNEL
> +} sa_mode_t;
>   /**
>    * Security Assocation (SA) data base entry
>    */
> @@ -26,6 +30,7 @@ typedef struct sa_db_entry_s {
>   	uint32_t              block_len; /**< Cipher block length */
>   	uint32_t              iv_len;    /**< Initialization Vector length */
>   	uint32_t              icv_len;   /**< Integrity Check Value length */
> +	sa_mode_t             mode;      /**< SA mode - transport/tun */
>   } sa_db_entry_t;
>   
>   /**
> @@ -37,6 +42,7 @@ typedef struct sa_db_s {
>   	sa_db_entry_t    array[MAX_DB];  /**< Entry storage */
>   } sa_db_t;
>   
> +
>   /** Initialize SA database global control structure */
>   void init_sa_db(void);
>   
> @@ -69,6 +75,57 @@ sa_db_entry_t *find_sa_db_entry(ip_addr_range_t *src,
>   				ip_addr_range_t *dst,
>   				odp_bool_t cipher);
>   
> +/**
> + * Tunnel entry
> + */
> +typedef struct tun_db_entry_s {
> +	struct tun_db_entry_s *next;
> +	uint32_t        src_ip;        /**< Inner Source IPv4 address */
> +	uint32_t        dst_ip;        /**< Inner Destination IPv4 address */
> +	uint32_t        tun_src_ip; /**< Tunnel Source IPv4 address */
> +	uint32_t        tun_dst_ip; /**< Tunnel Source IPv4 address */
> +} tun_db_entry_t;
> +
> +/**
> + * Tunnel database
> + */
> +typedef struct tun_db_s {
> +	uint32_t         index;          /**< Index of next available entry */
> +	tun_db_entry_t *list;	 /**< List of active entries */
> +	tun_db_entry_t array[MAX_DB]; /**< Entry storage */
> +} tun_db_t;
> +
> +/** Initialize tun database global control structure */
> +void init_tun_db(void);
> +
> +/**
> + * Create an tunnel DB entry
> + *
> + * String is of the format "SrcIP:DstIP:TunSrcIp:TunDstIp"
> + *
> + * @param input  Pointer to string describing tun
> + *
> + * @return 0 if successful else -1
> + */
> +int create_tun_db_entry(char *input);
> +
> +/**
> + * Display the tun DB
> + */
> +void dump_tun_db(void);
> +
> +/**
> + * Find a matching tun DB entry
> + *
> + * @param ip_src    Inner source IP address
> + * @param ip_dst    Inner destination IP address
> + *
> + * @return pointer to tun DB entry else NULL
> + */
> +tun_db_entry_t *find_tun_db_entry(uint32_t ip_src,
> +					uint32_t ip_dst);
> +
> +
>   #ifdef __cplusplus
>   }
>   #endif
> diff --git a/example/ipsec/odp_ipsec_stream.c b/example/ipsec/odp_ipsec_stream.c
> index ed07355..91b57fb 100644
> --- a/example/ipsec/odp_ipsec_stream.c
> +++ b/example/ipsec/odp_ipsec_stream.c
> @@ -177,18 +177,24 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   				uint8_t *dmac,
>   				odp_pool_t pkt_pool)
>   {
> -	ipsec_cache_entry_t *entry = stream->input.entry;
> +	ipsec_cache_entry_t *entry = NULL;
>   	odp_packet_t pkt;
>   	uint8_t *base;
>   	uint8_t *data;
>   	odph_ethhdr_t *eth;
>   	odph_ipv4hdr_t *ip;
> +	odph_ipv4hdr_t *inner_ip = NULL;
>   	odph_ahhdr_t *ah = NULL;
>   	odph_esphdr_t *esp = NULL;
>   	odph_icmphdr_t *icmp;
>   	stream_pkt_hdr_t *test;
>   	unsigned i;
>   
> +	if (stream->input.entry)
> +		entry = stream->input.entry;
> +	else if (stream->output.entry)
> +		entry = stream->output.entry;
> +
>   	/* Get packet */
>   	pkt = odp_packet_alloc(pkt_pool, 0);
>   	if (ODP_PACKET_INVALID == pkt)
> @@ -213,13 +219,22 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   	/* Wait until almost finished to fill in mutable fields */
>   	memset((char *)ip, 0, sizeof(*ip));
>   	ip->ver_ihl = 0x45;
> -	ip->proto = ODPH_IPPROTO_ICMP;
>   	ip->id = odp_cpu_to_be_16(stream->id);
> -	ip->src_addr = odp_cpu_to_be_32(stream->src_ip);
> -	ip->dst_addr = odp_cpu_to_be_32(stream->dst_ip);
> +	/* Outer IP header in tunnel mode */
> +	if (entry && entry->mode == IPSEC_SA_MODE_TUNNEL &&
> +	    (entry == stream->input.entry)) {
> +		ip->proto = ODPH_IPV4;
> +		ip->src_addr = odp_cpu_to_be_32(entry->tun_src_ip);
> +		ip->dst_addr = odp_cpu_to_be_32(entry->tun_dst_ip);
> +	} else {
> +		ip->proto = ODPH_IPPROTO_ICMP;
> +		ip->src_addr = odp_cpu_to_be_32(stream->src_ip);
> +		ip->dst_addr = odp_cpu_to_be_32(stream->dst_ip);
> +	}
>   
>   	/* AH (if specified) */
> -	if (entry && (ODP_AUTH_ALG_NULL != entry->ah.alg)) {
> +	if (entry && (entry == stream->input.entry) &&
> +	    (ODP_AUTH_ALG_NULL != entry->ah.alg)) {
>   		if (ODP_AUTH_ALG_MD5_96 != entry->ah.alg)
>   			abort();
>   
> @@ -234,7 +249,8 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   	}
>   
>   	/* ESP (if specified) */
> -	if (entry && (ODP_CIPHER_ALG_NULL != entry->esp.alg)) {
> +	if (entry && (entry == stream->input.entry) &&
> +	    (ODP_CIPHER_ALG_NULL != entry->esp.alg)) {
>   		if (ODP_CIPHER_ALG_3DES_CBC != entry->esp.alg)
>   			abort();
>   
> @@ -247,6 +263,23 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   		RAND_bytes(esp->iv, 8);
>   	}
>   
> +	/* Inner IP header in tunnel mode */
> +	if (entry && (entry == stream->input.entry) &&
> +	    (entry->mode == IPSEC_SA_MODE_TUNNEL)) {
> +		inner_ip = (odph_ipv4hdr_t *)data;
> +		memset((char *)inner_ip, 0, sizeof(*inner_ip));
> +		inner_ip->ver_ihl = 0x45;
> +		inner_ip->proto = ODPH_IPPROTO_ICMP;
> +		inner_ip->id = odp_cpu_to_be_16(stream->id);
> +		inner_ip->ttl = 64;
> +		inner_ip->tos = 0;
> +		inner_ip->frag_offset = 0;
> +		inner_ip->src_addr = odp_cpu_to_be_32(stream->src_ip);
> +		inner_ip->dst_addr = odp_cpu_to_be_32(stream->dst_ip);
> +		inner_ip->chksum = odp_chksum(inner_ip, sizeof(inner_ip));
> +		data += sizeof(*inner_ip);
> +	}
> +
>   	/* ICMP header so we can see it on wireshark */
>   	icmp = (odph_icmphdr_t *)data;
>   	data += sizeof(*icmp);
> @@ -269,6 +302,13 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   	/* Close ESP if specified */
>   	if (esp) {
>   		int payload_len = data - (uint8_t *)icmp;
> +		uint8_t *encrypt_start = (uint8_t *)icmp;
> +
> +		if (entry->mode == IPSEC_SA_MODE_TUNNEL) {
> +			payload_len = data - (uint8_t *)inner_ip;
> +			encrypt_start = (uint8_t *)inner_ip;
> +		}
> +
>   		int encrypt_len;
>   		odph_esptrl_t *esp_t;
>   		DES_key_schedule ks1, ks2, ks3;
> @@ -290,8 +330,8 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   		DES_set_key((DES_cblock *)&entry->esp.key.data[8], &ks2);
>   		DES_set_key((DES_cblock *)&entry->esp.key.data[16], &ks3);
>   
> -		DES_ede3_cbc_encrypt((uint8_t *)icmp,
> -				     (uint8_t *)icmp,
> +		DES_ede3_cbc_encrypt(encrypt_start,
> +				     encrypt_start,
>   				     encrypt_len,
>   				     &ks1,
>   				     &ks2,
> @@ -340,7 +380,7 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream,
>   			      odp_packet_t pkt)
>   {
> -	ipsec_cache_entry_t *entry = stream->output.entry;
> +	ipsec_cache_entry_t *entry = NULL;
>   	uint8_t *data;
>   	odph_ipv4hdr_t *ip;
>   	odph_ahhdr_t *ah = NULL;
> @@ -348,6 +388,12 @@ odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream,
>   	int hdr_len;
>   	odph_icmphdr_t *icmp;
>   	stream_pkt_hdr_t *test;
> +	uint32_t src_ip, dst_ip;
> +
> +	if (stream->input.entry)
> +		entry = stream->input.entry;
> +	else if (stream->output.entry)
> +		entry = stream->output.entry;
>   
>   	/* Basic IPv4 verify (add checksum verification) */
>   	data = odp_packet_l3_ptr(pkt, NULL);
> @@ -355,13 +401,29 @@ odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream,
>   	data += sizeof(*ip);
>   	if (0x45 != ip->ver_ihl)
>   		return FALSE;
> -	if (stream->src_ip != odp_be_to_cpu_32(ip->src_addr))
> +
> +	src_ip = odp_be_to_cpu_32(ip->src_addr);
> +	dst_ip = odp_be_to_cpu_32(ip->dst_addr);
> +	if ((stream->src_ip != src_ip) && stream->output.entry &&
> +	    (stream->output.entry->tun_src_ip != src_ip))
> +		return FALSE;
> +	if ((stream->dst_ip != dst_ip) && stream->output.entry &&
> +	    (stream->output.entry->tun_dst_ip != dst_ip))
> +		return FALSE;
> +
> +	if ((stream->src_ip != src_ip) && stream->input.entry &&
> +	    (stream->input.entry->tun_src_ip != src_ip))
>   		return FALSE;
> -	if (stream->dst_ip != odp_be_to_cpu_32(ip->dst_addr))
> +	if ((stream->dst_ip != dst_ip) && stream->input.entry &&
> +	    (stream->input.entry->tun_dst_ip != dst_ip))
>   		return FALSE;
>   
>   	/* Find IPsec headers if any and compare against entry */
>   	hdr_len = locate_ipsec_headers(ip, &ah, &esp);
> +
> +	/* Cleartext packet */
> +	if (!ah && !esp)
> +		goto clear_packet;
>   	if (ah) {
>   		if (!entry)
>   			return FALSE;
> @@ -454,12 +516,21 @@ odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream,
>   		ip->proto = esp_t->next_header;
>   	}
>   
> -	/* Verify ICMP packet */
> -	if (ODPH_IPPROTO_ICMP != ip->proto)
> -		return FALSE;
> +clear_packet:
> +	/* Verify IP/ICMP packet */
> +	if (entry && (entry->mode == IPSEC_SA_MODE_TUNNEL) && (ah || esp)) {
> +		if (ODPH_IPV4 != ip->proto)
> +			return FALSE;
> +		odph_ipv4hdr_t *inner_ip = (odph_ipv4hdr_t *)data;
> +		icmp = (odph_icmphdr_t *)(inner_ip + 1);
> +		data = (uint8_t *)icmp;
> +	} else {
> +		if (ODPH_IPPROTO_ICMP != ip->proto)
> +			return FALSE;
> +		icmp = (odph_icmphdr_t *)data;
> +	}
>   
>   	/* Verify ICMP header */
> -	icmp = (odph_icmphdr_t *)data;
>   	data += sizeof(*icmp);
>   	if (ICMP_ECHO != icmp->type)
>   		return FALSE;

Alex you need to update patch to match new checkpatch.

Feel free to include Steve review-by to the patch.

also one comment about tun bellow.

Thank you,
Maxim.

On 05/19/15 11:38, alexandru.badicioiu@linaro.org wrote:
> From: Alexandru Badicioiu <alexandru.badicioiu@linaro.org>
>
> v1 - added comment for tunnel DB entry use
>
> Tunnel mode is enabled from the command line using -t argument with
> the following format: SrcIP:DstIP:TunnelSrcIP:TunnelDstIP.
> SrcIP - cleartext packet source IP
> DstIP - cleartext packet destination IP
> TunnelSrcIP - tunnel source IP
> TunnelDstIP - tunnel destination IP
>
> The outbound packets matching SrcIP:DstIP will be encapsulated
> in a TunnelSrcIP:TunnelDstIP IPSec tunnel (AH/ESP/AH+ESP)
> if a matching outbound SA is determined (as for transport mode).
> For inbound packets each entry in the IPSec cache is matched
> for the cleartext addresses, as in the transport mode (SrcIP:DstIP)
> and then for the tunnel addresses (TunnelSrcIP:TunnelDstIP)
> in case cleartext addresses didn't match. After authentication and
> decryption tunneled packets are verified against the tunnel entry
> (packets came in from the expected tunnel).
>
> Signed-off-by: Alexandru Badicioiu <alexandru.badicioiu@linaro.org>
> ---
>   example/ipsec/odp_ipsec.c        |  105 +++++++++++++++++++++++++++---
>   example/ipsec/odp_ipsec_cache.c  |   31 +++++++++-
>   example/ipsec/odp_ipsec_cache.h  |    6 ++
>   example/ipsec/odp_ipsec_sa_db.c  |  133 +++++++++++++++++++++++++++++++++++++-
>   example/ipsec/odp_ipsec_sa_db.h  |   57 ++++++++++++++++
>   example/ipsec/odp_ipsec_stream.c |  101 ++++++++++++++++++++++++----
>   6 files changed, 403 insertions(+), 30 deletions(-)
>
> diff --git a/example/ipsec/odp_ipsec.c b/example/ipsec/odp_ipsec.c
> index 82ed0cb..3931fef 100644
> --- a/example/ipsec/odp_ipsec.c
> +++ b/example/ipsec/odp_ipsec.c
> @@ -135,13 +135,20 @@ typedef struct {
>   	uint8_t  ip_ttl;         /**< Saved IP TTL value */
>   	int      hdr_len;        /**< Length of IPsec headers */
>   	int      trl_len;        /**< Length of IPsec trailers */
> +	uint16_t tun_hdr_offset; /**< Offset of tunnel header from
> +				      buffer start */
>   	uint16_t ah_offset;      /**< Offset of AH header from buffer start */
>   	uint16_t esp_offset;     /**< Offset of ESP header from buffer start */
>   
> +	/* Input only */
> +	uint32_t src_ip;         /**< SA source IP address */
> +	uint32_t dst_ip;         /**< SA dest IP address */
> +
>   	/* Output only */
>   	odp_crypto_op_params_t params;  /**< Parameters for crypto call */
>   	uint32_t *ah_seq;               /**< AH sequence number location */
>   	uint32_t *esp_seq;              /**< ESP sequence number location */
> +	uint16_t *tun_hdr_id;           /**< Tunnel header ID > */
>   } ipsec_ctx_t;
>   
>   /**
> @@ -368,6 +375,7 @@ void ipsec_init_pre(void)
>   	/* Initialize our data bases */
>   	init_sp_db();
>   	init_sa_db();
> +	init_tun_db();
>   	init_ipsec_cache();
>   }
>   
> @@ -387,19 +395,27 @@ void ipsec_init_post(crypto_api_mode_e api_mode)
>   	for (entry = sp_db->list; NULL != entry; entry = entry->next) {
>   		sa_db_entry_t *cipher_sa = NULL;
>   		sa_db_entry_t *auth_sa = NULL;
> +		tun_db_entry_t *tun;
>   

you also need to set tun to NULL. Arm gcc complains about it:

odp_ipsec.c: In function ‘main’:
odp_ipsec.c:405:32: error: ‘tun’ may be used uninitialized in this 
function [-Werror=maybe-uninitialized]
     if (create_ipsec_cache_entry(cipher_sa,
                                 ^
odp_ipsec.c:387:19: note: ‘tun’ was declared here
    tun_db_entry_t *tun;
                    ^

> -		if (entry->esp)
> +		if (entry->esp) {
>   			cipher_sa = find_sa_db_entry(&entry->src_subnet,
>   						     &entry->dst_subnet,
>   						     1);
> -		if (entry->ah)
> +			tun = find_tun_db_entry(cipher_sa->src_ip,
> +						      cipher_sa->dst_ip);
> +		}
> +		if (entry->ah) {
>   			auth_sa = find_sa_db_entry(&entry->src_subnet,
>   						   &entry->dst_subnet,
>   						   0);
> +			tun = find_tun_db_entry(auth_sa->src_ip,
> +						      auth_sa->dst_ip);
> +		}
>   
>   		if (cipher_sa || auth_sa) {
>   			if (create_ipsec_cache_entry(cipher_sa,
>   						     auth_sa,
> +						     tun,
>   						     api_mode,
>   						     entry->input,
>   						     completionq,
> @@ -672,6 +688,8 @@ pkt_disposition_e do_ipsec_in_classify(odp_packet_t pkt,
>   	ctx->ipsec.esp_offset = esp ? ((uint8_t *)esp) - buf : 0;
>   	ctx->ipsec.hdr_len = hdr_len;
>   	ctx->ipsec.trl_len = 0;
> +	ctx->ipsec.src_ip = entry->src_ip;
> +	ctx->ipsec.dst_ip = entry->dst_ip;
>   
>   	/*If authenticating, zero the mutable fields build the request */
>   	if (ah) {
> @@ -752,6 +770,23 @@ pkt_disposition_e do_ipsec_in_finish(odp_packet_t pkt,
>   		trl_len += esp_t->pad_len + sizeof(*esp_t);
>   	}
>   
> +	/* We have a tunneled IPv4 packet */
> +	if (ip->proto == ODPH_IPV4) {
> +		odp_packet_pull_head(pkt, sizeof(*ip) + hdr_len);
> +		odp_packet_pull_tail(pkt, trl_len);
> +		odph_ethhdr_t *eth;
> +		eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL);
> +		eth->type = ODPH_ETHTYPE_IPV4;
> +		ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, NULL);
> +
> +		/* Check inbound policy */
> +		if ((ip->src_addr != ctx->ipsec.src_ip ||
> +		     ip->dst_addr != ctx->ipsec.dst_ip))
> +			return PKT_DROP;
> +
> +		return PKT_CONTINUE;
> +	}
> +
>   	/* Finalize the IPv4 header */
>   	ipv4_adjust_len(ip, -(hdr_len + trl_len));
>   	ip->ttl = ctx->ipsec.ip_ttl;
> @@ -823,9 +858,13 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt,
>   	params.pkt = pkt;
>   	params.out_pkt = entry->in_place ? pkt : ODP_PACKET_INVALID;
>   
> +	if (entry->mode == IPSEC_SA_MODE_TUNNEL) {
> +		hdr_len += sizeof(odph_ipv4hdr_t);
> +		ip_data = (uint8_t *)ip;
> +	}
>   	/* Compute ah and esp, determine length of headers, move the data */
>   	if (entry->ah.alg) {
> -		ah = (odph_ahhdr_t *)(ip_data);
> +		ah = (odph_ahhdr_t *)(ip_data + hdr_len);
>   		hdr_len += sizeof(odph_ahhdr_t);
>   		hdr_len += entry->ah.icv_len;
>   	}
> @@ -837,21 +876,39 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt,
>   	memmove(ip_data + hdr_len, ip_data, ip_data_len);
>   	ip_data += hdr_len;
>   
> +	/* update outer header in tunnel mode */
> +	if (entry->mode == IPSEC_SA_MODE_TUNNEL) {
> +		/* tunnel addresses */
> +		ip->src_addr = odp_cpu_to_be_32(entry->tun_src_ip);
> +		ip->dst_addr = odp_cpu_to_be_32(entry->tun_dst_ip);
> +	}
> +
>   	/* For cipher, compute encrypt length, build headers and request */
>   	if (esp) {
>   		uint32_t encrypt_len;
>   		odph_esptrl_t *esp_t;
>   
> -		encrypt_len = ESP_ENCODE_LEN(ip_data_len + sizeof(*esp_t),
> -					     entry->esp.block_len);
> -		trl_len = encrypt_len - ip_data_len;
> +		if (entry->mode == IPSEC_SA_MODE_TUNNEL) {
> +			encrypt_len = ESP_ENCODE_LEN(ip->tot_len +
> +						     sizeof(*esp_t),
> +						     entry->esp.block_len);
> +			trl_len = encrypt_len - ip->tot_len;
> +		} else {
> +			encrypt_len = ESP_ENCODE_LEN(ip_data_len +
> +						     sizeof(*esp_t),
> +						     entry->esp.block_len);
> +			trl_len = encrypt_len - ip_data_len;
> +		}
>   
>   		esp->spi = odp_cpu_to_be_32(entry->esp.spi);
>   		memcpy(esp + 1, entry->state.iv, entry->esp.iv_len);
>   
>   		esp_t = (odph_esptrl_t *)(ip_data + encrypt_len) - 1;
>   		esp_t->pad_len     = trl_len - sizeof(*esp_t);
> -		esp_t->next_header = ip->proto;
> +		if (entry->mode == IPSEC_SA_MODE_TUNNEL)
> +			esp_t->next_header = ODPH_IPV4;
> +		else
> +			esp_t->next_header = ip->proto;
>   		ip->proto = ODPH_IPPROTO_ESP;
>   
>   		params.cipher_range.offset = ip_data - buf;
> @@ -863,7 +920,10 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt,
>   		memset(ah, 0, sizeof(*ah) + entry->ah.icv_len);
>   		ah->spi = odp_cpu_to_be_32(entry->ah.spi);
>   		ah->ah_len = 1 + (entry->ah.icv_len / 4);
> -		ah->next_header = ip->proto;
> +		if (entry->mode == IPSEC_SA_MODE_TUNNEL && !esp)
> +			ah->next_header = ODPH_IPV4;
> +		else
> +			ah->next_header = ip->proto;
>   		ip->proto = ODPH_IPPROTO_AH;
>   
>   		ip->chksum = 0;
> @@ -886,8 +946,11 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt,
>   	ctx->ipsec.trl_len = trl_len;
>   	ctx->ipsec.ah_offset = ah ? ((uint8_t *)ah) - buf : 0;
>   	ctx->ipsec.esp_offset = esp ? ((uint8_t *)esp) - buf : 0;
> +	ctx->ipsec.tun_hdr_offset = (entry->mode == IPSEC_SA_MODE_TUNNEL) ?
> +				       ((uint8_t *)ip - buf) : 0;
>   	ctx->ipsec.ah_seq = &entry->state.ah_seq;
>   	ctx->ipsec.esp_seq = &entry->state.esp_seq;
> +	ctx->ipsec.tun_hdr_id = &entry->state.tun_hdr_id;
>   	memcpy(&ctx->ipsec.params, &params, sizeof(params));
>   
>   	*skip = FALSE;
> @@ -926,6 +989,20 @@ pkt_disposition_e do_ipsec_out_seq(odp_packet_t pkt,
>   		esp = (odph_esphdr_t *)(ctx->ipsec.esp_offset + buf);
>   		esp->seq_no = odp_cpu_to_be_32((*ctx->ipsec.esp_seq)++);
>   	}
> +	if (ctx->ipsec.tun_hdr_offset) {
> +		odph_ipv4hdr_t *ip;
> +		int ret;
> +		ip = (odph_ipv4hdr_t *)(ctx->ipsec.tun_hdr_offset + buf);
> +		ip->id = odp_cpu_to_be_16((*ctx->ipsec.tun_hdr_id)++);
> +		if (!ip->id) {
> +			/* re-init tunnel hdr id */
> +			ret = odp_random_data((uint8_t *)ctx->ipsec.tun_hdr_id,
> +					      sizeof(*ctx->ipsec.tun_hdr_id),
> +					      1);
> +			if (ret != sizeof(*ctx->ipsec.tun_hdr_id))
> +				abort();
> +		}
> +	}
>   
>   	/* Issue crypto request */
>   	if (odp_crypto_operation(&ctx->ipsec.params,
> @@ -1304,8 +1381,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
>   		{"mode", required_argument, NULL, 'm'},		/* return 'm' */
>   		{"route", required_argument, NULL, 'r'},	/* return 'r' */
>   		{"policy", required_argument, NULL, 'p'},	/* return 'p' */
> -		{"ah", required_argument, NULL, 'a'},	        /* return 'a' */
> -		{"esp", required_argument, NULL, 'e'},	        /* return 'e' */
> +		{"ah", required_argument, NULL, 'a'},		/* return 'a' */
> +		{"esp", required_argument, NULL, 'e'},		/* return 'e' */
> +		{"tunnel", required_argument, NULL, 't'},       /* return 't' */
>   		{"stream", required_argument, NULL, 's'},	/* return 's' */
>   		{"help", no_argument, NULL, 'h'},		/* return 'h' */
>   		{NULL, 0, NULL, 0}
> @@ -1316,7 +1394,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
>   	appl_args->mode = 0;  /* turn off async crypto API by default */
>   
>   	while (!rc) {
> -		opt = getopt_long(argc, argv, "+c:i:m:h:r:p:a:e:s:",
> +		opt = getopt_long(argc, argv, "+c:i:m:h:r:p:a:e:t:s:",
>   				  longopts, &long_index);
>   
>   		if (-1 == opt)
> @@ -1389,6 +1467,10 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
>   			rc = create_sa_db_entry(optarg, TRUE);
>   			break;
>   
> +		case 't':
> +			rc = create_tun_db_entry(optarg);
> +			break;
> +
>   		case 's':
>   			rc = create_stream_db_entry(optarg);
>   			break;
> @@ -1449,6 +1531,7 @@ static void print_info(char *progname, appl_args_t *appl_args)
>   	dump_fwd_db();
>   	dump_sp_db();
>   	dump_sa_db();
> +	dump_tun_db();
>   	printf("\n\n");
>   	fflush(NULL);
>   }
> diff --git a/example/ipsec/odp_ipsec_cache.c b/example/ipsec/odp_ipsec_cache.c
> index 12b960d..046e43c 100644
> --- a/example/ipsec/odp_ipsec_cache.c
> +++ b/example/ipsec/odp_ipsec_cache.c
> @@ -38,6 +38,7 @@ void init_ipsec_cache(void)
>   
>   int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   			     sa_db_entry_t *auth_sa,
> +			     tun_db_entry_t *tun,
>   			     crypto_api_mode_e api_mode,
>   			     odp_bool_t in,
>   			     odp_queue_t completionq,
> @@ -47,12 +48,18 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   	ipsec_cache_entry_t *entry;
>   	enum odp_crypto_ses_create_err ses_create_rc;
>   	odp_crypto_session_t session;
> +	sa_mode_t mode = IPSEC_SA_MODE_TRANSPORT;
>   
>   	/* Verify we have a good entry */
>   	entry = &ipsec_cache->array[ipsec_cache->index];
>   	if (MAX_DB <= ipsec_cache->index)
>   		return -1;
>   
> +	/* Verify SA mode match in case of cipher&auth */
> +	if (cipher_sa && auth_sa &&
> +	    (cipher_sa->mode != auth_sa->mode))
> +		return -1;
> +
>   	/* Setup parameters and call crypto library to create session */
>   	params.op = (in) ? ODP_CRYPTO_OP_DECODE : ODP_CRYPTO_OP_ENCODE;
>   	params.auth_cipher_text = TRUE;
> @@ -79,6 +86,7 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   		params.cipher_key.length  = cipher_sa->key.length;
>   		params.iv.data = entry->state.iv;
>   		params.iv.length = cipher_sa->iv_len;
> +		mode = cipher_sa->mode;
>   	} else {
>   		params.cipher_alg = ODP_CIPHER_ALG_NULL;
>   		params.iv.data = NULL;
> @@ -90,6 +98,7 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   		params.auth_alg = auth_sa->alg.u.auth;
>   		params.auth_key.data = auth_sa->key.data;
>   		params.auth_key.length = auth_sa->key.length;
> +		mode = auth_sa->mode;
>   	} else {
>   		params.auth_alg = ODP_AUTH_ALG_NULL;
>   	}
> @@ -128,6 +137,24 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   		memcpy(&entry->ah.key, &auth_sa->key, sizeof(ipsec_key_t));
>   	}
>   
> +	if (tun) {
> +		entry->tun_src_ip = tun->tun_src_ip;
> +		entry->tun_dst_ip = tun->tun_dst_ip;
> +		mode = IPSEC_SA_MODE_TUNNEL;
> +
> +		int ret;
> +		if (!in) {
> +			/* init tun hdr id */
> +			ret = odp_random_data((uint8_t *)
> +					      &entry->state.tun_hdr_id,
> +					      sizeof(entry->state.tun_hdr_id),
> +					      1);
> +			if (ret != sizeof(entry->state.tun_hdr_id))
> +				return -1;
> +		}
> +	}
> +	entry->mode = mode;
> +
>   	/* Initialize state */
>   	entry->state.esp_seq = 0;
>   	entry->state.ah_seq = 0;
> @@ -156,7 +183,9 @@ ipsec_cache_entry_t *find_ipsec_cache_entry_in(uint32_t src_ip,
>   	/* Look for a hit */
>   	for (; NULL != entry; entry = entry->next) {
>   		if ((entry->src_ip != src_ip) || (entry->dst_ip != dst_ip))
> -			continue;
> +			if ((entry->tun_src_ip != src_ip) ||
> +			    (entry->tun_dst_ip != dst_ip))
> +				continue;
>   		if (ah &&
>   		    ((!entry->ah.alg) ||
>   		     (entry->ah.spi != odp_be_to_cpu_32(ah->spi))))
> diff --git a/example/ipsec/odp_ipsec_cache.h b/example/ipsec/odp_ipsec_cache.h
> index 714cae8..5706007 100644
> --- a/example/ipsec/odp_ipsec_cache.h
> +++ b/example/ipsec/odp_ipsec_cache.h
> @@ -34,6 +34,9 @@ typedef struct ipsec_cache_entry_s {
>   	odp_bool_t                   in_place;    /**< Crypto API mode */
>   	uint32_t                     src_ip;      /**< Source v4 address */
>   	uint32_t                     dst_ip;      /**< Destination v4 address */
> +	sa_mode_t		     mode;        /**< SA mode - transport/tun */
> +	uint32_t                     tun_src_ip;  /**< Tunnel src IPv4 addr */
> +	uint32_t                     tun_dst_ip;  /**< Tunnel dst IPv4 addr */
>   	struct {
>   		enum  odp_cipher_alg alg;         /**< Cipher algorithm */
>   		uint32_t             spi;         /**< Cipher SPI */
> @@ -54,6 +57,7 @@ typedef struct ipsec_cache_entry_s {
>   		uint32_t      esp_seq;         /**< ESP TX sequence number */
>   		uint32_t      ah_seq;          /**< AH TX sequence number */
>   		uint8_t       iv[MAX_IV_LEN];  /**< ESP IV storage */
> +		uint16be_t    tun_hdr_id;      /**< Tunnel header IP ID */
>   	} state;
>   } ipsec_cache_entry_t;
>   
> @@ -78,6 +82,7 @@ void init_ipsec_cache(void);
>    *
>    * @param cipher_sa   Cipher SA DB entry pointer
>    * @param auth_sa     Auth SA DB entry pointer
> + * @param tun         Tunnel DB entry pointer
>    * @param api_mode    Crypto API mode for testing
>    * @param in          Direction (input versus output)
>    * @param completionq Completion queue
> @@ -87,6 +92,7 @@ void init_ipsec_cache(void);
>    */
>   int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa,
>   			     sa_db_entry_t *auth_sa,
> +			     tun_db_entry_t *tun,
>   			     crypto_api_mode_e api_mode,
>   			     odp_bool_t in,
>   			     odp_queue_t completionq,
> diff --git a/example/ipsec/odp_ipsec_sa_db.c b/example/ipsec/odp_ipsec_sa_db.c
> index 5837cb6..78f43da 100644
> --- a/example/ipsec/odp_ipsec_sa_db.c
> +++ b/example/ipsec/odp_ipsec_sa_db.c
> @@ -1,7 +1,7 @@
>   /* Copyright (c) 2014, Linaro Limited
>    * All rights reserved.
>    *
> - * SPDX-License-Identifier:     BSD-3-Clause
> + * SPDX-License-Identifier:	BSD-3-Clause
>    */
>   
>   /* enable strtok */
> @@ -19,6 +19,9 @@
>   /** Global pointer to sa db */
>   static sa_db_t *sa_db;
>   
> +/** Global pointer to tun db */
> +static tun_db_t *tun_db;
> +
>   void init_sa_db(void)
>   {
>   	odp_shm_t shm;
> @@ -37,6 +40,22 @@ void init_sa_db(void)
>   	memset(sa_db, 0, sizeof(*sa_db));
>   }
>   
> +void init_tun_db(void)
> +{
> +	odp_shm_t shm;
> +	shm = odp_shm_reserve("shm_tun_db",
> +			      sizeof(tun_db_t),
> +			      ODP_CACHE_LINE_SIZE,
> +			      0);
> +	tun_db = odp_shm_addr(shm);
> +
> +	if (tun_db == NULL) {
> +		EXAMPLE_ERR("Error: shared mem alloc failed.\n");
> +		exit(EXIT_FAILURE);
> +	}
> +	memset(tun_db, 0, sizeof(*tun_db));
> +}
> +
>   int create_sa_db_entry(char *input, odp_bool_t cipher)
>   {
>   	int pos = 0;
> @@ -81,7 +100,7 @@ int create_sa_db_entry(char *input, odp_bool_t cipher)
>   					entry->alg.u.cipher =
>   						ODP_CIPHER_ALG_3DES_CBC;
>   					entry->block_len  = 8;
> -					entry->iv_len     = 8;
> +					entry->iv_len	  = 8;
>   				} else {
>   					entry->alg.u.cipher =
>   						ODP_CIPHER_ALG_NULL;
> @@ -90,7 +109,7 @@ int create_sa_db_entry(char *input, odp_bool_t cipher)
>   				if (0 == strcmp(token, "md5")) {
>   					entry->alg.u.auth =
>   						ODP_AUTH_ALG_MD5_96;
> -					entry->icv_len    = 12;
> +					entry->icv_len	  = 12;
>   				} else {
>   					entry->alg.u.auth = ODP_AUTH_ALG_NULL;
>   				}
> @@ -132,6 +151,89 @@ int create_sa_db_entry(char *input, odp_bool_t cipher)
>   	return 0;
>   }
>   
> +int create_tun_db_entry(char *input)
> +{
> +	int pos = 0;
> +	char *local;
> +	char *str;
> +	char *save;
> +	char *token;
> +	tun_db_entry_t *entry = &tun_db->array[tun_db->index];
> +
> +	/* Verify we have a good entry */
> +	if (MAX_DB <= tun_db->index)
> +		return -1;
> +
> +	/* Make a local copy */
> +	local = malloc(strlen(input) + 1);
> +	if (NULL == local)
> +		return -1;
> +	strcpy(local, input);
> +
> +	/* Setup for using "strtok_r" to search input string */
> +	str = local;
> +	save = NULL;
> +
> +	/* Parse tokens separated by ':' */
> +	while (NULL != (token = strtok_r(str, ":", &save))) {
> +		str = NULL;  /* reset str for subsequent strtok_r calls */
> +
> +		/* Parse token based on its position */
> +		switch (pos) {
> +		case 0:
> +			parse_ipv4_string(token, &entry->src_ip, NULL);
> +			break;
> +		case 1:
> +			parse_ipv4_string(token, &entry->dst_ip, NULL);
> +			break;
> +		case 2:
> +			parse_ipv4_string(token, &entry->tun_src_ip, NULL);
> +			break;
> +		case 3:
> +			parse_ipv4_string(token, &entry->tun_dst_ip, NULL);
> +			break;
> +		default:
> +			printf("ERROR: extra token \"%s\" at position %d\n",
> +			       token, pos);
> +			break;
> +		}
> +		pos++;
> +	}
> +
> +	/* Verify we parsed exactly the number of tokens we expected */
> +	if (4 != pos) {
> +		printf("ERROR: \"%s\" contains %d tokens, expected 4\n",
> +		       input,
> +		       pos);
> +		free(local);
> +		return -1;
> +	}
> +
> +	/* Add route to the list */
> +	tun_db->index++;
> +	entry->next = tun_db->list;
> +	tun_db->list = entry;
> +
> +	free(local);
> +	return 0;
> +}
> +
> +tun_db_entry_t *find_tun_db_entry(uint32_t ip_src,
> +					uint32_t ip_dst)
> +{
> +	tun_db_entry_t *entry = NULL;
> +
> +	/* Scan all entries and return first match */
> +	for (entry = tun_db->list; NULL != entry; entry = entry->next) {
> +		if (entry->src_ip != ip_src)
> +			continue;
> +		if (entry->dst_ip != ip_dst)
> +			continue;
> +		break;
> +	}
> +	return entry;
> +}
> +
>   void dump_sa_db(void)
>   {
>   	sa_db_entry_t *entry;
> @@ -182,3 +284,28 @@ sa_db_entry_t *find_sa_db_entry(ip_addr_range_t *src,
>   	}
>   	return entry;
>   }
> +
> +void dump_tun_db(void)
> +{
> +	tun_db_entry_t *entry;
> +
> +	printf("\n"
> +	       "Tunnel table\n"
> +	       "--------------------------\n");
> +
> +	for (entry = tun_db->list; NULL != entry; entry = entry->next) {
> +		char src_ip_str[MAX_STRING];
> +		char dst_ip_str[MAX_STRING];
> +		char tun_src_ip_str[MAX_STRING];
> +		char tun_dst_ip_str[MAX_STRING];
> +
> +		printf(" %s:%s %s:%s ",
> +		       ipv4_addr_str(src_ip_str, entry->src_ip),
> +		       ipv4_addr_str(dst_ip_str, entry->dst_ip),
> +		       ipv4_addr_str(tun_src_ip_str, entry->tun_src_ip),
> +		       ipv4_addr_str(tun_dst_ip_str, entry->tun_dst_ip)
> +		      );
> +
> +		printf("\n");
> +	}
> +}
> diff --git a/example/ipsec/odp_ipsec_sa_db.h b/example/ipsec/odp_ipsec_sa_db.h
> index c30cbdb..79bfc78 100644
> --- a/example/ipsec/odp_ipsec_sa_db.h
> +++ b/example/ipsec/odp_ipsec_sa_db.h
> @@ -13,6 +13,10 @@ extern "C" {
>   
>   #include <odp_ipsec_misc.h>
>   
> +typedef enum sa_mode_s {
> +	IPSEC_SA_MODE_TRANSPORT,
> +	IPSEC_SA_MODE_TUNNEL
> +} sa_mode_t;
>   /**
>    * Security Assocation (SA) data base entry
>    */
> @@ -26,6 +30,7 @@ typedef struct sa_db_entry_s {
>   	uint32_t              block_len; /**< Cipher block length */
>   	uint32_t              iv_len;    /**< Initialization Vector length */
>   	uint32_t              icv_len;   /**< Integrity Check Value length */
> +	sa_mode_t             mode;      /**< SA mode - transport/tun */
>   } sa_db_entry_t;
>   
>   /**
> @@ -37,6 +42,7 @@ typedef struct sa_db_s {
>   	sa_db_entry_t    array[MAX_DB];  /**< Entry storage */
>   } sa_db_t;
>   
> +
>   /** Initialize SA database global control structure */
>   void init_sa_db(void);
>   
> @@ -69,6 +75,57 @@ sa_db_entry_t *find_sa_db_entry(ip_addr_range_t *src,
>   				ip_addr_range_t *dst,
>   				odp_bool_t cipher);
>   
> +/**
> + * Tunnel entry
> + */
> +typedef struct tun_db_entry_s {
> +	struct tun_db_entry_s *next;
> +	uint32_t        src_ip;        /**< Inner Source IPv4 address */
> +	uint32_t        dst_ip;        /**< Inner Destination IPv4 address */
> +	uint32_t        tun_src_ip; /**< Tunnel Source IPv4 address */
> +	uint32_t        tun_dst_ip; /**< Tunnel Source IPv4 address */
> +} tun_db_entry_t;
> +
> +/**
> + * Tunnel database
> + */
> +typedef struct tun_db_s {
> +	uint32_t         index;          /**< Index of next available entry */
> +	tun_db_entry_t *list;	 /**< List of active entries */
> +	tun_db_entry_t array[MAX_DB]; /**< Entry storage */
> +} tun_db_t;
> +
> +/** Initialize tun database global control structure */
> +void init_tun_db(void);
> +
> +/**
> + * Create an tunnel DB entry
> + *
> + * String is of the format "SrcIP:DstIP:TunSrcIp:TunDstIp"
> + *
> + * @param input  Pointer to string describing tun
> + *
> + * @return 0 if successful else -1
> + */
> +int create_tun_db_entry(char *input);
> +
> +/**
> + * Display the tun DB
> + */
> +void dump_tun_db(void);
> +
> +/**
> + * Find a matching tun DB entry
> + *
> + * @param ip_src    Inner source IP address
> + * @param ip_dst    Inner destination IP address
> + *
> + * @return pointer to tun DB entry else NULL
> + */
> +tun_db_entry_t *find_tun_db_entry(uint32_t ip_src,
> +					uint32_t ip_dst);
> +
> +
>   #ifdef __cplusplus
>   }
>   #endif
> diff --git a/example/ipsec/odp_ipsec_stream.c b/example/ipsec/odp_ipsec_stream.c
> index ed07355..91b57fb 100644
> --- a/example/ipsec/odp_ipsec_stream.c
> +++ b/example/ipsec/odp_ipsec_stream.c
> @@ -177,18 +177,24 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   				uint8_t *dmac,
>   				odp_pool_t pkt_pool)
>   {
> -	ipsec_cache_entry_t *entry = stream->input.entry;
> +	ipsec_cache_entry_t *entry = NULL;
>   	odp_packet_t pkt;
>   	uint8_t *base;
>   	uint8_t *data;
>   	odph_ethhdr_t *eth;
>   	odph_ipv4hdr_t *ip;
> +	odph_ipv4hdr_t *inner_ip = NULL;
>   	odph_ahhdr_t *ah = NULL;
>   	odph_esphdr_t *esp = NULL;
>   	odph_icmphdr_t *icmp;
>   	stream_pkt_hdr_t *test;
>   	unsigned i;
>   
> +	if (stream->input.entry)
> +		entry = stream->input.entry;
> +	else if (stream->output.entry)
> +		entry = stream->output.entry;
> +
>   	/* Get packet */
>   	pkt = odp_packet_alloc(pkt_pool, 0);
>   	if (ODP_PACKET_INVALID == pkt)
> @@ -213,13 +219,22 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   	/* Wait until almost finished to fill in mutable fields */
>   	memset((char *)ip, 0, sizeof(*ip));
>   	ip->ver_ihl = 0x45;
> -	ip->proto = ODPH_IPPROTO_ICMP;
>   	ip->id = odp_cpu_to_be_16(stream->id);
> -	ip->src_addr = odp_cpu_to_be_32(stream->src_ip);
> -	ip->dst_addr = odp_cpu_to_be_32(stream->dst_ip);
> +	/* Outer IP header in tunnel mode */
> +	if (entry && entry->mode == IPSEC_SA_MODE_TUNNEL &&
> +	    (entry == stream->input.entry)) {
> +		ip->proto = ODPH_IPV4;
> +		ip->src_addr = odp_cpu_to_be_32(entry->tun_src_ip);
> +		ip->dst_addr = odp_cpu_to_be_32(entry->tun_dst_ip);
> +	} else {
> +		ip->proto = ODPH_IPPROTO_ICMP;
> +		ip->src_addr = odp_cpu_to_be_32(stream->src_ip);
> +		ip->dst_addr = odp_cpu_to_be_32(stream->dst_ip);
> +	}
>   
>   	/* AH (if specified) */
> -	if (entry && (ODP_AUTH_ALG_NULL != entry->ah.alg)) {
> +	if (entry && (entry == stream->input.entry) &&
> +	    (ODP_AUTH_ALG_NULL != entry->ah.alg)) {
>   		if (ODP_AUTH_ALG_MD5_96 != entry->ah.alg)
>   			abort();
>   
> @@ -234,7 +249,8 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   	}
>   
>   	/* ESP (if specified) */
> -	if (entry && (ODP_CIPHER_ALG_NULL != entry->esp.alg)) {
> +	if (entry && (entry == stream->input.entry) &&
> +	    (ODP_CIPHER_ALG_NULL != entry->esp.alg)) {
>   		if (ODP_CIPHER_ALG_3DES_CBC != entry->esp.alg)
>   			abort();
>   
> @@ -247,6 +263,23 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   		RAND_bytes(esp->iv, 8);
>   	}
>   
> +	/* Inner IP header in tunnel mode */
> +	if (entry && (entry == stream->input.entry) &&
> +	    (entry->mode == IPSEC_SA_MODE_TUNNEL)) {
> +		inner_ip = (odph_ipv4hdr_t *)data;
> +		memset((char *)inner_ip, 0, sizeof(*inner_ip));
> +		inner_ip->ver_ihl = 0x45;
> +		inner_ip->proto = ODPH_IPPROTO_ICMP;
> +		inner_ip->id = odp_cpu_to_be_16(stream->id);
> +		inner_ip->ttl = 64;
> +		inner_ip->tos = 0;
> +		inner_ip->frag_offset = 0;
> +		inner_ip->src_addr = odp_cpu_to_be_32(stream->src_ip);
> +		inner_ip->dst_addr = odp_cpu_to_be_32(stream->dst_ip);
> +		inner_ip->chksum = odp_chksum(inner_ip, sizeof(inner_ip));
> +		data += sizeof(*inner_ip);
> +	}
> +
>   	/* ICMP header so we can see it on wireshark */
>   	icmp = (odph_icmphdr_t *)data;
>   	data += sizeof(*icmp);
> @@ -269,6 +302,13 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   	/* Close ESP if specified */
>   	if (esp) {
>   		int payload_len = data - (uint8_t *)icmp;
> +		uint8_t *encrypt_start = (uint8_t *)icmp;
> +
> +		if (entry->mode == IPSEC_SA_MODE_TUNNEL) {
> +			payload_len = data - (uint8_t *)inner_ip;
> +			encrypt_start = (uint8_t *)inner_ip;
> +		}
> +
>   		int encrypt_len;
>   		odph_esptrl_t *esp_t;
>   		DES_key_schedule ks1, ks2, ks3;
> @@ -290,8 +330,8 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   		DES_set_key((DES_cblock *)&entry->esp.key.data[8], &ks2);
>   		DES_set_key((DES_cblock *)&entry->esp.key.data[16], &ks3);
>   
> -		DES_ede3_cbc_encrypt((uint8_t *)icmp,
> -				     (uint8_t *)icmp,
> +		DES_ede3_cbc_encrypt(encrypt_start,
> +				     encrypt_start,
>   				     encrypt_len,
>   				     &ks1,
>   				     &ks2,
> @@ -340,7 +380,7 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream,
>   odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream,
>   			      odp_packet_t pkt)
>   {
> -	ipsec_cache_entry_t *entry = stream->output.entry;
> +	ipsec_cache_entry_t *entry = NULL;
>   	uint8_t *data;
>   	odph_ipv4hdr_t *ip;
>   	odph_ahhdr_t *ah = NULL;
> @@ -348,6 +388,12 @@ odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream,
>   	int hdr_len;
>   	odph_icmphdr_t *icmp;
>   	stream_pkt_hdr_t *test;
> +	uint32_t src_ip, dst_ip;
> +
> +	if (stream->input.entry)
> +		entry = stream->input.entry;
> +	else if (stream->output.entry)
> +		entry = stream->output.entry;
>   
>   	/* Basic IPv4 verify (add checksum verification) */
>   	data = odp_packet_l3_ptr(pkt, NULL);
> @@ -355,13 +401,29 @@ odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream,
>   	data += sizeof(*ip);
>   	if (0x45 != ip->ver_ihl)
>   		return FALSE;
> -	if (stream->src_ip != odp_be_to_cpu_32(ip->src_addr))
> +
> +	src_ip = odp_be_to_cpu_32(ip->src_addr);
> +	dst_ip = odp_be_to_cpu_32(ip->dst_addr);
> +	if ((stream->src_ip != src_ip) && stream->output.entry &&
> +	    (stream->output.entry->tun_src_ip != src_ip))
> +		return FALSE;
> +	if ((stream->dst_ip != dst_ip) && stream->output.entry &&
> +	    (stream->output.entry->tun_dst_ip != dst_ip))
> +		return FALSE;
> +
> +	if ((stream->src_ip != src_ip) && stream->input.entry &&
> +	    (stream->input.entry->tun_src_ip != src_ip))
>   		return FALSE;
> -	if (stream->dst_ip != odp_be_to_cpu_32(ip->dst_addr))
> +	if ((stream->dst_ip != dst_ip) && stream->input.entry &&
> +	    (stream->input.entry->tun_dst_ip != dst_ip))
>   		return FALSE;
>   
>   	/* Find IPsec headers if any and compare against entry */
>   	hdr_len = locate_ipsec_headers(ip, &ah, &esp);
> +
> +	/* Cleartext packet */
> +	if (!ah && !esp)
> +		goto clear_packet;
>   	if (ah) {
>   		if (!entry)
>   			return FALSE;
> @@ -454,12 +516,21 @@ odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream,
>   		ip->proto = esp_t->next_header;
>   	}
>   
> -	/* Verify ICMP packet */
> -	if (ODPH_IPPROTO_ICMP != ip->proto)
> -		return FALSE;
> +clear_packet:
> +	/* Verify IP/ICMP packet */
> +	if (entry && (entry->mode == IPSEC_SA_MODE_TUNNEL) && (ah || esp)) {
> +		if (ODPH_IPV4 != ip->proto)
> +			return FALSE;
> +		odph_ipv4hdr_t *inner_ip = (odph_ipv4hdr_t *)data;
> +		icmp = (odph_icmphdr_t *)(inner_ip + 1);
> +		data = (uint8_t *)icmp;
> +	} else {
> +		if (ODPH_IPPROTO_ICMP != ip->proto)
> +			return FALSE;
> +		icmp = (odph_icmphdr_t *)data;
> +	}
>   
>   	/* Verify ICMP header */
> -	icmp = (odph_icmphdr_t *)data;
>   	data += sizeof(*icmp);
>   	if (ICMP_ECHO != icmp->type)
>   		return FALSE;