@@ -72,6 +72,7 @@ enum init_stage {
NAME_TABLE_INIT,
IPSEC_EVENTS_INIT,
IPSEC_SAD_INIT,
+ IPSEC_INIT,
MODULES_INIT,
ALL_INIT /* All init stages completed */
};
@@ -131,6 +132,9 @@ int _odp_ishm_init_local(void);
int _odp_ishm_term_global(void);
int _odp_ishm_term_local(void);
+int odp_ipsec_init_global(void);
+int odp_ipsec_term_global(void);
+
int odp_ipsec_events_init_global(void);
int odp_ipsec_events_term_global(void);
@@ -278,6 +278,12 @@ int odp_init_global(odp_instance_t *instance,
}
stage = IPSEC_SAD_INIT;
+ if (odp_ipsec_init_global()) {
+ ODP_ERR("ODP IPsec init failed.\n");
+ goto init_failed;
+ }
+ stage = IPSEC_INIT;
+
if (_odp_modules_init_global()) {
ODP_ERR("ODP modules init failed\n");
goto init_failed;
@@ -308,6 +314,13 @@ int _odp_term_global(enum init_stage stage)
switch (stage) {
case ALL_INIT:
case MODULES_INIT:
+ case IPSEC_INIT:
+ if (odp_ipsec_term_global()) {
+ ODP_ERR("ODP IPsec term failed.\n");
+ rc = -1;
+ }
+ /* Fall through */
+
case IPSEC_SAD_INIT:
if (odp_ipsec_sad_term_global()) {
ODP_ERR("ODP IPsec SAD term failed.\n");
@@ -4,101 +4,1057 @@
* SPDX-License-Identifier: BSD-3-Clause
*/
+#include <odp/api/atomic.h>
#include <odp/api/ipsec.h>
+#include <odp/api/packet.h>
+#include <odp/api/shared_memory.h>
+#include <odp_buffer_internal.h>
+#include <odp_buffer_inlines.h>
+#include <odp_debug_internal.h>
#include <odp_ipsec_internal.h>
+#include <odp_pool_internal.h>
-#include <string.h>
+#include <protocols/ip.h>
+#include <protocols/ipsec.h>
+
+typedef void (*ipsec_postprocess_t)(ipsec_ctx_t *ctx);
+
+/**
+ * Per packet IPsec processing context
+ */
+struct ipsec_ctx_s {
+ odp_buffer_t buffer; /**< Buffer for context */
+ ipsec_ctx_t *next; /**< Next context in event */
+
+ ipsec_postprocess_t postprocess;
+ ipsec_sa_t *ipsec_sa;
+ odp_crypto_op_result_t crypto;
+ odp_ipsec_op_status_t status;
+ odp_packet_t pkt;
+
+ uint8_t ip_tos; /**< Saved IP TOS value */
+ uint8_t ip_ttl; /**< Saved IP TTL value */
+ uint16_t ip_frag_offset; /**< Saved IP flags value */
+ unsigned hdr_len; /**< Length of IPsec headers */
+ unsigned trl_len; /**< Length of IPsec trailers */
+
+ uint32_t src_ip; /**< SA source IP address */
+ uint32_t dst_ip; /**< SA dest IP address */
+ uint16_t ipsec_offset; /**< Offset of IPsec header from
+ buffer start */
+ uint8_t iv[MAX_IV_LEN]; /**< ESP IV storage */
+
+ unsigned pkt_out : 1; /**< Packet was output to application */
+};
+
+static odp_pool_t ipsec_ctx_pool = ODP_POOL_INVALID;
+
+#define IPSEC_CTX_POOL_BUF_COUNT 1024
+
+int odp_ipsec_init_global(void)
+{
+ odp_pool_param_t param;
+
+ /* Create context buffer pool */
+ param.buf.size = sizeof(ipsec_ctx_t);
+ param.buf.align = 0;
+ param.buf.num = IPSEC_CTX_POOL_BUF_COUNT;
+ param.type = ODP_POOL_BUFFER;
+
+ ipsec_ctx_pool = odp_pool_create("ipsec_ctx_pool", ¶m);
+ if (ODP_POOL_INVALID == ipsec_ctx_pool) {
+ ODP_ERR("Error: context pool create failed.\n");
+ goto err_ctx;
+ }
+
+ return 0;
+
+err_ctx:
+ return -1;
+}
+
+int odp_ipsec_term_global(void)
+{
+ int ret = 0;
+ int rc = 0;
+
+ ret = odp_pool_destroy(ipsec_ctx_pool);
+ if (ret < 0) {
+ ODP_ERR("ctx pool destroy failed");
+ rc = -1;
+ }
+
+ return rc;
+}
int odp_ipsec_capability(odp_ipsec_capability_t *capa)
{
+ int rc;
+ odp_crypto_capability_t crypto_capa;
+
memset(capa, 0, sizeof(odp_ipsec_capability_t));
+ capa->op_mode_sync = ODP_SUPPORT_PREFERRED;
+ capa->op_mode_async = ODP_SUPPORT_PREFERRED;
+ capa->op_mode_inline_out = ODP_SUPPORT_YES;
+
+ capa->proto_ah = ODP_SUPPORT_YES;
+
+ capa->max_num_sa = ODP_CONFIG_IPSEC_SAS;
+
+ rc = odp_crypto_capability(&crypto_capa);
+ if (rc < 0)
+ return rc;
+
+ capa->ciphers = crypto_capa.ciphers;
+ capa->auths = crypto_capa.auths;
+
return 0;
}
int odp_ipsec_cipher_capability(odp_cipher_alg_t cipher,
odp_crypto_cipher_capability_t capa[], int num)
{
- (void)cipher;
- (void)capa;
- (void)num;
-
- return -1;
+ return odp_crypto_cipher_capability(cipher, capa, num);
}
int odp_ipsec_auth_capability(odp_auth_alg_t auth,
odp_crypto_auth_capability_t capa[], int num)
{
- (void)auth;
- (void)capa;
- (void)num;
-
- return -1;
+ return odp_crypto_auth_capability(auth, capa, num);
}
void odp_ipsec_config_init(odp_ipsec_config_t *config)
{
memset(config, 0, sizeof(odp_ipsec_config_t));
+ config->inbound_mode = ODP_IPSEC_OP_MODE_SYNC;
+ config->outbound_mode = ODP_IPSEC_OP_MODE_SYNC;
+ config->max_num_sa = ODP_CONFIG_IPSEC_SAS;
+ config->inbound.default_queue = ODP_QUEUE_INVALID;
+ config->inbound.lookup.min_spi = 0;
+ config->inbound.lookup.max_spi = UINT32_MAX;
}
+static odp_ipsec_config_t ipsec_config;
+
int odp_ipsec_config(const odp_ipsec_config_t *config)
{
- (void)config;
+ /* FIXME: unsupported for now */
+ if (ODP_IPSEC_OP_MODE_INLINE == config->inbound_mode)
+ return -1;
- return -1;
+ if (ODP_CONFIG_IPSEC_SAS > config->max_num_sa)
+ return -1;
+
+ ipsec_config = *config;
+
+ return 0;
+}
+
+static
+void ipsec_ctx_init(ipsec_ctx_t *ctx, odp_buffer_t buf)
+{
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->buffer = buf;
+
+ ctx->pkt = ODP_PACKET_INVALID;
+ ctx->crypto.pkt = ODP_PACKET_INVALID;
+ ctx->crypto.ok = true;
+}
+
+/**
+ * Allocate per packet processing context.
+ *
+ * @return pointer to context area
+ */
+static
+ipsec_ctx_t *ipsec_ctx_alloc(void)
+{
+ odp_buffer_t ctx_buf = odp_buffer_alloc(ipsec_ctx_pool);
+ ipsec_ctx_t *ctx;
+
+ if (odp_unlikely(ODP_BUFFER_INVALID == ctx_buf))
+ return NULL;
+
+ ctx = odp_buffer_addr(ctx_buf);
+ ipsec_ctx_init(ctx, ctx_buf);
+
+ return ctx;
}
void _odp_ipsec_ctx_free(ipsec_ctx_t *ctx)
{
- (void)ctx;
+ while (NULL != ctx) {
+ ipsec_ctx_t *next = ctx->next;
+
+ if (ODP_PACKET_INVALID != ctx->crypto.pkt)
+ odp_packet_free(ctx->crypto.pkt);
+
+ if (!ctx->pkt_out && ODP_PACKET_INVALID != ctx->pkt)
+ odp_packet_free(ctx->pkt);
+
+ odp_buffer_free(ctx->buffer);
+
+ ctx = next;
+ }
+}
+
+/**
+ * Checksum
+ *
+ * @param buffer calculate chksum for buffer
+ * @param len buffer length
+ *
+ * @return checksum value in host cpu order
+ */
+static inline
+odp_u16sum_t _odp_chksum(void *buffer, int len)
+{
+ uint16_t *buf = (uint16_t *)buffer;
+ uint32_t sum = 0;
+ uint16_t result;
+
+ for (sum = 0; len > 1; len -= 2)
+ sum += *buf++;
+
+ if (len == 1)
+ sum += *(unsigned char *)buf;
+
+ sum = (sum >> 16) + (sum & 0xFFFF);
+ sum += (sum >> 16);
+ result = ~sum;
+
+ return (__odp_force odp_u16sum_t) result;
+}
+
+/**
+ * Calculate and fill in IPv4 checksum
+ *
+ * @note when using this api to populate data destined for the wire
+ * odp_cpu_to_be_16() can be used to remove sparse warnings
+ *
+ * @param pkt ODP packet
+ *
+ * @return IPv4 checksum in host cpu order, or 0 on failure
+ */
+static inline odp_u16sum_t
+_odp_ipv4_csum_update(odp_packet_t pkt)
+{
+ uint16_t *w;
+ _odp_ipv4hdr_t *ip;
+ int nleft = sizeof(_odp_ipv4hdr_t);
+
+ ip = (_odp_ipv4hdr_t *)odp_packet_l3_ptr(pkt, NULL);
+ if (ip == NULL)
+ return 0;
+
+ ip->chksum = 0;
+ w = (uint16_t *)(void *)ip;
+ ip->chksum = _odp_chksum(w, nleft);
+ return ip->chksum;
+}
+
+#define ipv4_hdr_len(ip) (_ODP_IPV4HDR_IHL(ip->ver_ihl) * 4)
+static inline
+void ipv4_adjust_len(_odp_ipv4hdr_t *ip, int adj)
+{
+ ip->tot_len = odp_cpu_to_be_16(odp_be_to_cpu_16(ip->tot_len) + adj);
+}
+
+static
+void ipsec_finish(ipsec_ctx_t *ctx,
+ odp_ipsec_packet_result_t *res,
+ odp_packet_t *pkt)
+{
+ odp_crypto_op_result_t *result = &ctx->crypto;
+
+ res->status = ctx->status;
+
+ if (ODP_PACKET_INVALID != result->pkt) {
+ ctx->pkt = result->pkt;
+ result->pkt = ODP_PACKET_INVALID;
+ }
+
+ /* Check crypto result */
+ if (!result->ok) {
+ if (result->cipher_status.alg_err != ODP_CRYPTO_ALG_ERR_NONE ||
+ result->cipher_status.hw_err != ODP_CRYPTO_HW_ERR_NONE)
+ res->status.error.alg = 1;
+
+ if (result->auth_status.alg_err != ODP_CRYPTO_ALG_ERR_NONE ||
+ result->auth_status.hw_err != ODP_CRYPTO_HW_ERR_NONE)
+ res->status.error.auth = 1;
+ } else {
+ if (ctx->postprocess)
+ ctx->postprocess(ctx);
+ }
+
+ *pkt = ctx->pkt;
+ ctx->pkt_out = 1;
+
+ if (NULL != ctx->ipsec_sa) {
+ res->sa = ctx->ipsec_sa->ipsec_sa_hdl;
+ _odp_ipsec_sa_unuse(ctx->ipsec_sa);
+ } else {
+ res->sa = ODP_IPSEC_SA_INVALID;
+ }
+}
+
+static void ipsec_in_postprocess(ipsec_ctx_t *ctx);
+
+static
+void ipsec_in_single(ipsec_ctx_t *ctx)
+{
+ odp_packet_t pkt = ctx->pkt;
+ uint32_t ip_offset = odp_packet_l3_offset(pkt);
+ _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL);
+ uint16_t ip_hdr_len = ipv4_hdr_len(ip);
+ odp_crypto_op_param_t param;
+ odp_bool_t posted = 0;
+ int rc = -1;
+
+ ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset);
+ ODP_ASSERT(NULL != ip);
+
+ /* Initialize parameters block */
+ memset(¶m, 0, sizeof(param));
+ param.ctx = ctx;
+
+ /* Save everything to context */
+ ctx->ip_tos = ip->tos;
+ ctx->ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset);
+ ctx->ip_ttl = ip->ttl;
+
+ ctx->postprocess = ipsec_in_postprocess;
+ ctx->ipsec_offset = ip_offset + ip_hdr_len;
+
+ /* Check IP header for IPSec protocols and look it up */
+ if (_ODP_IPPROTO_AH == ip->proto) {
+ _odp_ahhdr_t ah;
+
+ if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset, sizeof(ah), &ah) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ if (NULL == ctx->ipsec_sa) {
+ ipsec_sa_lookup_t lookup;
+
+ lookup.proto = ODP_IPSEC_AH;
+ lookup.spi = odp_be_to_cpu_32(ah.spi);
+ lookup.dst_addr = &ip->dst_addr;
+ ctx->ipsec_sa = _odp_ipsec_sa_lookup(&lookup);
+ if (NULL == ctx->ipsec_sa) {
+ ctx->status.error.sa_lookup = 1;
+ goto out;
+ }
+ }
+
+ if (ODP_IPSEC_AH != ctx->ipsec_sa->proto) {
+ ctx->status.error.proto = 1;
+ goto out;
+ }
+
+ ctx->hdr_len = (ah.ah_len + 2) * 4;
+ ctx->trl_len = 0;
+
+ /* If authenticating, zero the mutable fields build the request */
+ ip->chksum = 0;
+ ip->tos = 0;
+ ip->frag_offset = 0;
+ ip->ttl = 0;
+
+ param.auth_range.offset = ip_offset;
+ param.auth_range.length = odp_be_to_cpu_16(ip->tot_len);
+ param.hash_result_offset = ctx->ipsec_offset + _ODP_AHHDR_LEN;
+ } else if (_ODP_IPPROTO_ESP == ip->proto) {
+ _odp_esphdr_t esp;
+
+ if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset, sizeof(esp), &esp) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ if (NULL == ctx->ipsec_sa) {
+ ipsec_sa_lookup_t lookup;
+
+ lookup.proto = ODP_IPSEC_ESP;
+ lookup.spi = odp_be_to_cpu_32(esp.spi);
+ lookup.dst_addr = &ip->dst_addr;
+ ctx->ipsec_sa = _odp_ipsec_sa_lookup(&lookup);
+ if (NULL == ctx->ipsec_sa) {
+ ctx->status.error.sa_lookup = 1;
+ goto out;
+ }
+ }
+
+ if (ODP_IPSEC_ESP != ctx->ipsec_sa->proto) {
+ ctx->status.error.proto = 1;
+ goto out;
+ }
+
+ if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset + _ODP_ESPHDR_LEN, ctx->ipsec_sa->esp_iv_len, ctx->iv) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ ctx->hdr_len = _ODP_ESPHDR_LEN + ctx->ipsec_sa->esp_iv_len;
+ ctx->trl_len = _ODP_ESPTRL_LEN + ctx->ipsec_sa->icv_len;
+
+ param.cipher_range.offset = ctx->ipsec_offset + ctx->hdr_len;
+ param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->hdr_len - ctx->ipsec_sa->icv_len;
+ param.override_iv_ptr = ctx->iv;
+
+ param.auth_range.offset = ctx->ipsec_offset;
+ param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->ipsec_sa->icv_len;
+ param.hash_result_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->ipsec_sa->icv_len;
+ } else {
+ ctx->status.error.proto = 1;
+ goto out;
+ }
+
+ if (_odp_ipsec_sa_update_stats(ctx->ipsec_sa, odp_packet_len(pkt), &ctx->status) < 0)
+ goto out;
+
+ param.session = ctx->ipsec_sa->session;
+ param.pkt = pkt;
+ /* Create new packet after all length extensions */
+ if (ctx->ipsec_sa->in_place) {
+ param.out_pkt = pkt;
+ } else {
+ param.out_pkt = odp_packet_alloc(odp_packet_pool(pkt),
+ odp_packet_len(pkt));
+ /* uarea will be copied by odp_crypto_operation */
+ odp_packet_user_ptr_set(param.out_pkt,
+ odp_packet_user_ptr(param.pkt));
+ }
+ pkt = ODP_PACKET_INVALID;
+
+ rc = odp_crypto_operation(¶m, &posted, &ctx->crypto);
+ if (rc < 0) {
+ ODP_DBG("Crypto failed\n");
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ ODP_ASSERT(!posted);
+
+out:
+ ctx->pkt = pkt;
}
+static
+void ipsec_in_postprocess(ipsec_ctx_t *ctx)
+{
+ odp_packet_t pkt = ctx->pkt;
+ uint32_t ip_offset = odp_packet_l3_offset(pkt);
+ _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL);
+ uint16_t ip_hdr_len = ipv4_hdr_len(ip);
+
+ if (_ODP_IPPROTO_AH == ip->proto) {
+ /*
+ * Finish auth
+ */
+ _odp_ahhdr_t ah;
+
+ if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset, sizeof(ah), &ah) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ ip->proto = ah.next_header;
+
+ /* Restore mutable fields */
+ ip->ttl = ctx->ip_ttl;
+ ip->tos = ctx->ip_tos;
+ ip->frag_offset = odp_cpu_to_be_16(ctx->ip_frag_offset);
+ } else if (_ODP_IPPROTO_ESP == ip->proto) {
+ /*
+ * Finish cipher by finding ESP trailer and processing
+ */
+ _odp_esptrl_t esptrl;
+ uint32_t esptrl_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->trl_len;
+
+ if (odp_packet_copy_to_mem(pkt, esptrl_offset, sizeof(esptrl), &esptrl) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ ip->proto = esptrl.next_header;
+ ctx->trl_len += esptrl.pad_len;
+ } else {
+ ctx->status.error.proto = 1;
+ goto out;
+ }
+
+ if (ip->proto == _ODP_IPV4) {
+ ip->ttl -= ctx->ipsec_sa->dec_ttl;
+ _odp_ipv4_csum_update(pkt);
+
+ /* We have a tunneled IPv4 packet, strip outer and IPsec headers */
+ odp_packet_move_data(pkt, ip_hdr_len + ctx->hdr_len, 0, ip_offset);
+ if (odp_packet_trunc_head(&pkt, ip_hdr_len + ctx->hdr_len, NULL, NULL) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ } else {
+ /* Finalize the IPv4 header */
+ ipv4_adjust_len(ip, -(ctx->hdr_len + ctx->trl_len));
+
+ _odp_ipv4_csum_update(pkt);
+
+ odp_packet_move_data(pkt, ctx->hdr_len, 0, ip_offset + ip_hdr_len);
+ if (odp_packet_trunc_head(&pkt, ctx->hdr_len, NULL, NULL) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+ }
+
+ if (odp_packet_trunc_tail(&pkt, ctx->trl_len, NULL, NULL) < 0)
+ ctx->status.error.alg = 1;
+
+out:
+ ctx->pkt = pkt;
+}
+
+/** Helper for calculating encode length using data length and block size */
+#define ESP_ENCODE_LEN(x, b) ((((x) + ((b) - 1)) / (b)) * (b))
+
+static void ipsec_out_postprocess(ipsec_ctx_t *ctx);
+
+static
+void ipsec_out_single(ipsec_ctx_t *ctx)
+{
+ odp_packet_t pkt = ctx->pkt;
+ uint32_t ip_offset = odp_packet_l3_offset(pkt);
+ _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL);
+ uint16_t ip_hdr_len = ipv4_hdr_len(ip);
+ odp_crypto_op_param_t param;
+ odp_bool_t posted = 0;
+ int rc = -1;
+
+ ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset);
+ ODP_ASSERT(NULL != ip);
+ ODP_ASSERT(NULL != ctx->ipsec_sa);
+
+ /* Initialize parameters block */
+ memset(¶m, 0, sizeof(param));
+ param.ctx = ctx;
+
+ if (ctx->ipsec_sa->mode == ODP_IPSEC_MODE_TUNNEL) {
+ _odp_ipv4hdr_t out_ip;
+ _odp_ipv4hdr_t *inner_ip;
+ uint16_t tun_hdr_offset = ip_offset + ip_hdr_len;
+
+ ip->ttl -= ctx->ipsec_sa->dec_ttl;
+
+ if (odp_packet_extend_head(&pkt, _ODP_IPV4HDR_LEN, NULL, NULL) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ odp_packet_move_data(pkt, 0, _ODP_IPV4HDR_LEN, ip_offset);
+
+ inner_ip = odp_packet_offset(pkt, tun_hdr_offset, NULL, NULL);
+
+ out_ip.ver_ihl = 0x45;
+ out_ip.tos = inner_ip->tos; /* FIXME */
+ out_ip.tot_len = odp_cpu_to_be_16(odp_be_to_cpu_16(inner_ip->tot_len) + _ODP_IPV4HDR_LEN);
+ /* No need to convert to BE: ID just should not be duplicated */
+ out_ip.id = (odp_atomic_fetch_add_u32(&ctx->ipsec_sa->tun_hdr_id, 1) + 1) & 0xffff;
+ out_ip.frag_offset = 0;
+ out_ip.ttl = ctx->ipsec_sa->tun_ttl;
+ out_ip.proto = _ODP_IPV4;
+ out_ip.src_addr = ctx->ipsec_sa->tun_src_ip;
+ out_ip.dst_addr = ctx->ipsec_sa->tun_dst_ip;
+
+ odp_packet_copy_from_mem(pkt, ip_offset, _ODP_IPV4HDR_LEN, &out_ip);
+
+ odp_packet_l4_offset_set(pkt, ip_offset + _ODP_IPV4HDR_LEN);
+
+ ip = odp_packet_l3_ptr(pkt, NULL);
+ ip_hdr_len = _ODP_IPV4HDR_LEN;
+ }
+
+ /* Save IPv4 stuff */
+ ctx->ip_tos = ip->tos;
+ ctx->ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset);
+ ctx->ip_ttl = ip->ttl;
+
+ ctx->postprocess = ipsec_out_postprocess;
+
+ ctx->ipsec_offset = ip_offset + ip_hdr_len;
+
+ if (ctx->ipsec_sa->proto == ODP_IPSEC_AH) {
+ ctx->hdr_len = _ODP_AHHDR_LEN + ctx->ipsec_sa->icv_len;
+ ctx->trl_len = 0;
+ } else if (ctx->ipsec_sa->proto == ODP_IPSEC_ESP) {
+ uint32_t encrypt_len;
+ uint16_t ip_next_len = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len;
+
+ ctx->hdr_len += _ODP_ESPHDR_LEN + ctx->ipsec_sa->esp_iv_len;
+
+ encrypt_len = ESP_ENCODE_LEN(ip_next_len + _ODP_ESPTRL_LEN,
+ ctx->ipsec_sa->esp_block_len);
+ ctx->trl_len = encrypt_len - ip_next_len + ctx->ipsec_sa->icv_len;
+ } else {
+ ctx->status.error.proto = 1;
+ goto out;
+ }
+
+ if (odp_packet_extend_tail(&pkt, ctx->trl_len, NULL, NULL) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ if (odp_packet_extend_head(&pkt, ctx->hdr_len, NULL, NULL) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ odp_packet_move_data(pkt, 0, ctx->hdr_len, ctx->ipsec_offset);
+
+ ip = odp_packet_l3_ptr(pkt, NULL);
+
+ /* Set IPv4 length before authentication */
+ ipv4_adjust_len(ip, ctx->hdr_len + ctx->trl_len);
+
+ /* For authentication, build header clear mutables and build request */
+ if (ctx->ipsec_sa->proto == ODP_IPSEC_AH) {
+ _odp_ahhdr_t ah;
+ uint8_t icv[ctx->ipsec_sa->icv_len];
+
+ memset(&ah, 0, sizeof(ah));
+ ah.spi = odp_cpu_to_be_32(ctx->ipsec_sa->spi);
+ ah.ah_len = 1 + (ctx->ipsec_sa->icv_len / 4);
+ ah.seq_no = odp_cpu_to_be_32(odp_atomic_fetch_add_u32(&ctx->ipsec_sa->seq, 1) + 1);
+ ah.next_header = ip->proto;
+ ip->proto = _ODP_IPPROTO_AH;
+
+ odp_packet_copy_from_mem(pkt, ctx->ipsec_offset, _ODP_AHHDR_LEN, &ah);
+ memset(icv, 0, ctx->ipsec_sa->icv_len);
+ odp_packet_copy_from_mem(pkt, ctx->ipsec_offset + _ODP_AHHDR_LEN, ctx->ipsec_sa->icv_len, icv);
+
+ ip->chksum = 0;
+ ip->tos = 0;
+ ip->frag_offset = 0;
+ ip->ttl = 0;
+
+ param.auth_range.offset = ip_offset;
+ param.auth_range.length = odp_be_to_cpu_16(ip->tot_len);
+ param.hash_result_offset = ctx->ipsec_offset + _ODP_AHHDR_LEN;
+ }
+
+ if (ctx->ipsec_sa->proto == ODP_IPSEC_ESP) {
+ _odp_esphdr_t esp;
+ _odp_esptrl_t esptrl;
+ uint32_t esptrl_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->ipsec_sa->icv_len - _ODP_ESPTRL_LEN;
+
+ memset(&esp, 0, sizeof(esp));
+ memset(&esptrl, 0, sizeof(esptrl));
+ esp.spi = odp_cpu_to_be_32(ctx->ipsec_sa->spi);
+ esp.seq_no = odp_cpu_to_be_32(odp_atomic_fetch_add_u32(&ctx->ipsec_sa->seq, 1) + 1);
+
+ esptrl.pad_len = ctx->trl_len - _ODP_ESPTRL_LEN - ctx->ipsec_sa->icv_len;
+ esptrl.next_header = ip->proto;
+ ip->proto = _ODP_IPPROTO_ESP;
+
+ odp_packet_copy_from_mem(pkt, ctx->ipsec_offset, _ODP_ESPHDR_LEN, &esp);
+ odp_packet_copy_from_mem(pkt, ctx->ipsec_offset + _ODP_ESPHDR_LEN, ctx->ipsec_sa->esp_iv_len, ctx->ipsec_sa->iv);
+ odp_packet_copy_from_mem(pkt, esptrl_offset, _ODP_ESPTRL_LEN, &esptrl);
+
+ param.cipher_range.offset = ctx->ipsec_offset + ctx->hdr_len;
+ param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->hdr_len - ctx->ipsec_sa->icv_len;
+
+ param.auth_range.offset = ctx->ipsec_offset;
+ param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->ipsec_sa->icv_len;
+ param.hash_result_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->ipsec_sa->icv_len;
+ }
+
+ if (_odp_ipsec_sa_update_stats(ctx->ipsec_sa, odp_packet_len(pkt), &ctx->status) < 0)
+ goto out;
+
+ param.session = ctx->ipsec_sa->session;
+ param.pkt = pkt;
+ /* Create new packet after all length extensions */
+ if (ctx->ipsec_sa->in_place) {
+ param.out_pkt = pkt;
+ } else {
+ param.out_pkt = odp_packet_alloc(odp_packet_pool(pkt),
+ odp_packet_len(pkt));
+ odp_packet_user_ptr_set(param.out_pkt,
+ odp_packet_user_ptr(param.pkt));
+ }
+ pkt = ODP_PACKET_INVALID;
+
+ rc = odp_crypto_operation(¶m, &posted, &ctx->crypto);
+ if (rc < 0) {
+ ODP_DBG("Crypto failed\n");
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ ODP_ASSERT(!posted);
+
+out:
+ ctx->pkt = pkt;
+}
+
+static
+void ipsec_out_postprocess(ipsec_ctx_t *ctx)
+{
+ odp_packet_t pkt = ctx->pkt;
+ _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL);
+
+ /* Finalize the IPv4 header */
+ if (ip->proto == _ODP_IPPROTO_AH) {
+ ip->ttl = ctx->ip_ttl;
+ ip->tos = ctx->ip_tos;
+ ip->frag_offset = odp_cpu_to_be_16(ctx->ip_frag_offset);
+ }
+
+ _odp_ipv4_csum_update(pkt);
+}
+
+#if 0
+static odp_ipsec_op_opt_t default_opt = {
+ .mode = ODP_IPSEC_FRAG_DISABLED,
+};
+#endif
+
int odp_ipsec_in(const odp_ipsec_op_param_t *input,
odp_ipsec_op_result_t *output)
{
- (void)input;
- (void)output;
+ int in_pkt = 0;
+ int out_pkt = 0;
+ unsigned sa_idx = 0;
+ unsigned opt_idx = 0;
+ unsigned sa_inc = (input->num_sa > 1) ? 1 : 0;
+ unsigned opt_inc = (input->num_opt > 1) ? 1 : 0;
- return -1;
+ while (in_pkt < input->num_pkt && out_pkt < output->num_pkt) {
+ ipsec_ctx_t ctx;
+
+ ipsec_ctx_init(&ctx, ODP_BUFFER_INVALID);
+
+#if 0
+ odp_ipsec_op_opt_t *opt;
+
+ if (0 == input->num_opt)
+ opt = &default_opt;
+ else
+ opt = &input->opt[opt_idx];
+#endif
+
+ ctx.pkt = input->pkt[in_pkt];
+
+ if (0 == input->num_sa) {
+ ctx.ipsec_sa = NULL;
+ } else {
+ ctx.ipsec_sa = _odp_ipsec_sa_use(input->sa[sa_idx]);
+ ODP_ASSERT(NULL != ctx.ipsec_sa);
+ }
+
+ ipsec_in_single(&ctx);
+
+ ipsec_finish(&ctx, &output->res[out_pkt], &output->pkt[out_pkt]);
+
+ in_pkt++;
+ out_pkt++;
+ sa_idx += sa_inc;
+ opt_idx += opt_inc;
+ }
+
+ return in_pkt;
}
int odp_ipsec_out(const odp_ipsec_op_param_t *input,
- odp_ipsec_op_result_t *output)
+ odp_ipsec_op_result_t *output)
{
- (void)input;
- (void)output;
+ int in_pkt = 0;
+ int out_pkt = 0;
+ unsigned sa_idx = 0;
+ unsigned opt_idx = 0;
+ unsigned sa_inc = (input->num_sa > 1) ? 1 : 0;
+ unsigned opt_inc = (input->num_opt > 1) ? 1 : 0;
- return -1;
+ ODP_ASSERT(input->num_sa != 0);
+
+ while (in_pkt < input->num_pkt && out_pkt < output->num_pkt) {
+ odp_ipsec_sa_t sa;
+ ipsec_ctx_t ctx;
+
+ ipsec_ctx_init(&ctx, ODP_BUFFER_INVALID);
+
+ sa = input->sa[sa_idx];
+
+ ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa);
+
+#if 0
+ odp_ipsec_op_opt_t *opt;
+
+ if (0 == input->num_opt)
+ opt = &default_opt;
+ else
+ opt = &input->opt[opt_idx];
+#endif
+
+ ctx.pkt = input->pkt[in_pkt];
+ ctx.ipsec_sa = _odp_ipsec_sa_use(sa);
+
+ ipsec_out_single(&ctx);
+
+ ipsec_finish(&ctx, &output->res[out_pkt], &output->pkt[out_pkt]);
+
+ in_pkt++;
+ out_pkt++;
+ sa_idx += sa_inc;
+ opt_idx += opt_inc;
+ }
+
+ return in_pkt;
}
int odp_ipsec_in_enq(const odp_ipsec_op_param_t *input)
{
- (void)input;
+ int in_pkt = 0;
+ unsigned sa_idx = 0;
+ unsigned opt_idx = 0;
+ unsigned sa_inc = (input->num_sa > 1) ? 1 : 0;
+ unsigned opt_inc = (input->num_opt > 1) ? 1 : 0;
- return -1;
+ while (in_pkt < input->num_pkt) {
+ ipsec_ctx_t *ctx;
+ odp_queue_t queue;
+
+ ctx = ipsec_ctx_alloc();
+ if (NULL == ctx)
+ break;
+
+#if 0
+ odp_ipsec_op_opt_t *opt;
+
+ if (0 == input->num_opt)
+ opt = &default_opt;
+ else
+ opt = &input->opt[opt_idx];
+#endif
+
+ ctx->pkt = input->pkt[in_pkt];
+
+ if (0 == input->num_sa) {
+ ctx->ipsec_sa = NULL;
+ } else {
+ ctx->ipsec_sa = _odp_ipsec_sa_use(input->sa[sa_idx]);
+ ODP_ASSERT(NULL != ctx->ipsec_sa);
+ }
+
+ ipsec_in_single(ctx);
+
+ in_pkt++;
+ sa_idx += sa_inc;
+ opt_idx += opt_inc;
+
+ /* IN might have looked up SA for the packet */
+ if (NULL == ctx->ipsec_sa)
+ queue = ipsec_config.inbound.default_queue;
+ else
+ queue = ctx->ipsec_sa->queue;
+ if (odp_unlikely(_odp_ipsec_result_send(queue, ctx) < 0)) {
+ _odp_ipsec_ctx_free(ctx);
+ break;
+ }
+ }
+
+ return in_pkt;
}
int odp_ipsec_out_enq(const odp_ipsec_op_param_t *input)
{
- (void)input;
+ int in_pkt = 0;
+ unsigned sa_idx = 0;
+ unsigned opt_idx = 0;
+ unsigned sa_inc = (input->num_sa > 1) ? 1 : 0;
+ unsigned opt_inc = (input->num_opt > 1) ? 1 : 0;
- return -1;
+ ODP_ASSERT(input->num_sa != 0);
+
+ while (in_pkt < input->num_pkt) {
+ odp_ipsec_sa_t sa;
+ ipsec_ctx_t *ctx;
+
+ ctx = ipsec_ctx_alloc();
+ if (NULL == ctx)
+ break;
+
+ sa = input->sa[sa_idx];
+
+ ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa);
+
+#if 0
+ odp_ipsec_op_opt_t *opt;
+
+ if (0 == input->num_opt)
+ opt = &default_opt;
+ else
+ opt = &input->opt[opt_idx];
+#endif
+
+ ctx->pkt = input->pkt[in_pkt];
+ ctx->ipsec_sa = _odp_ipsec_sa_use(sa);
+
+ ipsec_out_single(ctx);
+
+ in_pkt++;
+ sa_idx += sa_inc;
+ opt_idx += opt_inc;
+
+ if (odp_unlikely(_odp_ipsec_result_send(ctx->ipsec_sa->queue, ctx) < 0)) {
+ _odp_ipsec_ctx_free(ctx);
+ break;
+ }
+ }
+
+ return in_pkt;
}
-int odp_ipsec_out_inline(const odp_ipsec_op_param_t *op_param,
+static
+odp_bool_t _odp_ipsec_out_inline_send(ipsec_ctx_t *ctx,
+ const odp_ipsec_inline_op_param_t *inline_param)
+{
+ if (ctx->status.all_error || !ctx->crypto.ok)
+ return false;
+
+ while (ctx) {
+ ipsec_ctx_t *next = ctx->next;
+ odp_ipsec_packet_result_t dummy;
+ odp_packet_t pkt;
+ uint32_t offset;
+ odp_pktout_queue_t queue;
+ uint32_t hdr_len = inline_param->outer_hdr.len;
+
+ ctx->next = NULL;
+
+ ipsec_finish(ctx, &dummy, &pkt);
+ offset = odp_packet_l3_offset(pkt);
+
+ if (offset >= hdr_len) {
+ offset = offset - hdr_len;
+ } else {
+ if (odp_packet_extend_head(&pkt, hdr_len - offset, NULL, NULL) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ odp_packet_l3_offset_set(pkt, hdr_len);
+
+ offset = 0;
+ }
+
+ if (odp_packet_copy_from_mem(pkt, offset, hdr_len, inline_param->outer_hdr.ptr) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ if (odp_pktout_queue(inline_param->pktio, &queue, 1) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+ if (odp_pktout_send(queue, &pkt, 1) < 0) {
+ ctx->status.error.alg = 1;
+ goto out;
+ }
+
+out:
+ if (ctx->status.all_error) {
+ if (odp_unlikely(_odp_ipsec_result_send(ctx->ipsec_sa->queue, ctx) < 0))
+ _odp_ipsec_ctx_free(ctx);
+ } else {
+ _odp_ipsec_ctx_free(ctx);
+ }
+ ctx = next;
+ }
+
+ return true;
+}
+
+int odp_ipsec_out_inline(const odp_ipsec_op_param_t *input,
const odp_ipsec_inline_op_param_t *inline_param)
{
- (void)op_param;
- (void)inline_param;
+ int in_pkt = 0;
+ unsigned sa_idx = 0;
+ unsigned opt_idx = 0;
+ unsigned sa_inc = (input->num_sa > 1) ? 1 : 0;
+ unsigned opt_inc = (input->num_opt > 1) ? 1 : 0;
- return -1;
+ ODP_ASSERT(input->num_sa != 0);
+
+ while (in_pkt < input->num_pkt) {
+ odp_ipsec_sa_t sa;
+ ipsec_ctx_t *ctx;
+
+ ctx = ipsec_ctx_alloc();
+ if (NULL == ctx)
+ break;
+
+ sa = input->sa[sa_idx];
+
+ ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa);
+
+#if 0
+ odp_ipsec_op_opt_t *opt;
+
+ if (0 == input->num_opt)
+ opt = &default_opt;
+ else
+ opt = &input->opt[opt_idx];
+#endif
+
+ ctx->pkt = input->pkt[in_pkt];
+ ctx->ipsec_sa = _odp_ipsec_sa_use(sa);
+
+ ipsec_out_single(ctx);
+
+ in_pkt++;
+ sa_idx += sa_inc;
+ opt_idx += opt_inc;
+
+ /* FIXME: inline_param should have been put into context */
+ if (!_odp_ipsec_out_inline_send(ctx, &inline_param[in_pkt - 1])) {
+ /* In case of an error, submit result event */
+ if (odp_unlikely(_odp_ipsec_result_send(ctx->ipsec_sa->queue, ctx) < 0)) {
+ _odp_ipsec_ctx_free(ctx);
+ break;
+ }
+ }
+ }
+
+ return in_pkt;
}
int _odp_ipsec_ctx_result(ipsec_ctx_t *ctx, odp_ipsec_op_result_t *result)
{
- (void)ctx;
- (void)result;
+ int out_pkt = 0;
- return -1;
+ if (NULL == result)
+ goto count;
+
+ while (NULL != ctx && out_pkt < result->num_pkt) {
+ ipsec_finish(ctx, &result->res[out_pkt], &result->pkt[out_pkt]);
+ out_pkt++;
+ ctx = ctx->next;
+ }
+
+ result->num_pkt = out_pkt;
+
+count:
+ while (NULL != ctx) {
+ out_pkt++;
+ ctx = ctx->next;
+ }
+
+ return out_pkt;
}