[API-NEXT,v9,7/10] linux-gen: crypto: provide packet API implementation

diff --git a/platform/linux-generic/include/odp_crypto_internal.h b/platform/linux-generic/include/odp_crypto_internal.h
index 52b94d07..12d1720b 100644
--- a/platform/linux-generic/include/odp_crypto_internal.h
+++ b/platform/linux-generic/include/odp_crypto_internal.h
@@ -23,7 +23,8 @@  typedef struct odp_crypto_generic_session odp_crypto_generic_session_t;
  * Algorithm handler function prototype
  */
 typedef
-odp_crypto_alg_err_t (*crypto_func_t)(odp_crypto_op_param_t *param,
+odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt,
+				      const odp_crypto_packet_op_param_t *param,
 				      odp_crypto_generic_session_t *session);
 
 /**
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 9f28fc05..a90ec477 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -128,6 +128,9 @@  typedef struct {
 	/* Classifier destination queue */
 	odp_queue_t dst_queue;
 
+	/* Result for crypto packet op */
+	odp_crypto_packet_result_t crypto_op_result;
+
 	/* Packet data storage */
 	uint8_t data[0];
 } odp_packet_hdr_t;
diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c
index 075bb02b..792c2eca 100644
--- a/platform/linux-generic/odp_crypto.c
+++ b/platform/linux-generic/odp_crypto.c
@@ -113,7 +113,8 @@  void free_session(odp_crypto_generic_session_t *session)
 }
 
 static odp_crypto_alg_err_t
-null_crypto_routine(odp_crypto_op_param_t *param ODP_UNUSED,
+null_crypto_routine(odp_packet_t pkt ODP_UNUSED,
+		    const odp_crypto_packet_op_param_t *param ODP_UNUSED,
 		    odp_crypto_generic_session_t *session ODP_UNUSED)
 {
 	return ODP_CRYPTO_ALG_ERR_NONE;
@@ -121,11 +122,11 @@  null_crypto_routine(odp_crypto_op_param_t *param ODP_UNUSED,
 
 static
 void packet_hmac_calculate(HMAC_CTX *ctx,
-			   odp_crypto_op_param_t *param,
+			   odp_packet_t pkt,
+			   const odp_crypto_packet_op_param_t *param,
 			   odp_crypto_generic_session_t *session,
 			   uint8_t *hash)
 {
-	odp_packet_t pkt = param->out_pkt;
 	uint32_t offset = param->auth_range.offset;
 	uint32_t len   = param->auth_range.length;
 
@@ -152,7 +153,8 @@  void packet_hmac_calculate(HMAC_CTX *ctx,
 
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
 static
-void packet_hmac(odp_crypto_op_param_t *param,
+void packet_hmac(odp_packet_t pkt,
+		 const odp_crypto_packet_op_param_t *param,
 		 odp_crypto_generic_session_t *session,
 		 uint8_t *hash)
 {
@@ -160,12 +162,13 @@  void packet_hmac(odp_crypto_op_param_t *param,
 
 	/* Hash it */
 	HMAC_CTX_init(&ctx);
-	packet_hmac_calculate(&ctx, param, session, hash);
+	packet_hmac_calculate(&ctx, pkt, param, session, hash);
 	HMAC_CTX_cleanup(&ctx);
 }
 #else
 static
-void packet_hmac(odp_crypto_op_param_t *param,
+void packet_hmac(odp_packet_t pkt,
+		 const odp_crypto_packet_op_param_t *param,
 		 odp_crypto_generic_session_t *session,
 		 uint8_t *hash)
 {
@@ -173,22 +176,23 @@  void packet_hmac(odp_crypto_op_param_t *param,
 
 	/* Hash it */
 	ctx = HMAC_CTX_new();
-	packet_hmac_calculate(ctx, param, session, hash);
+	packet_hmac_calculate(ctx, pkt, param, session, hash);
 	HMAC_CTX_free(ctx);
 }
 #endif
 
 static
-odp_crypto_alg_err_t auth_gen(odp_crypto_op_param_t *param,
+odp_crypto_alg_err_t auth_gen(odp_packet_t pkt,
+			      const odp_crypto_packet_op_param_t *param,
 			      odp_crypto_generic_session_t *session)
 {
 	uint8_t  hash[EVP_MAX_MD_SIZE];
 
 	/* Hash it */
-	packet_hmac(param, session, hash);
+	packet_hmac(pkt, param, session, hash);
 
 	/* Copy to the output location */
-	odp_packet_copy_from_mem(param->out_pkt,
+	odp_packet_copy_from_mem(pkt,
 				 param->hash_result_offset,
 				 session->auth.bytes,
 				 hash);
@@ -197,7 +201,8 @@  odp_crypto_alg_err_t auth_gen(odp_crypto_op_param_t *param,
 }
 
 static
-odp_crypto_alg_err_t auth_check(odp_crypto_op_param_t *param,
+odp_crypto_alg_err_t auth_check(odp_packet_t pkt,
+				const odp_crypto_packet_op_param_t *param,
 				odp_crypto_generic_session_t *session)
 {
 	uint32_t bytes = session->auth.bytes;
@@ -205,14 +210,14 @@  odp_crypto_alg_err_t auth_check(odp_crypto_op_param_t *param,
 	uint8_t  hash_out[EVP_MAX_MD_SIZE];
 
 	/* Copy current value out and clear it before authentication */
-	odp_packet_copy_to_mem(param->out_pkt, param->hash_result_offset,
+	odp_packet_copy_to_mem(pkt, param->hash_result_offset,
 			       bytes, hash_in);
 
-	_odp_packet_set_data(param->out_pkt, param->hash_result_offset,
+	_odp_packet_set_data(pkt, param->hash_result_offset,
 			     0, bytes);
 
 	/* Hash it */
-	packet_hmac(param, session, hash_out);
+	packet_hmac(pkt, param, session, hash_out);
 
 	/* Verify match */
 	if (0 != memcmp(hash_in, hash_out, bytes))
@@ -223,9 +228,10 @@  odp_crypto_alg_err_t auth_check(odp_crypto_op_param_t *param,
 }
 
 static
-int internal_encrypt(EVP_CIPHER_CTX *ctx, odp_crypto_op_param_t *param)
+int internal_encrypt(EVP_CIPHER_CTX *ctx,
+		     odp_packet_t pkt,
+		     const odp_crypto_packet_op_param_t *param)
 {
-	odp_packet_t pkt = param->out_pkt;
 	unsigned in_pos = param->cipher_range.offset;
 	unsigned out_pos = param->cipher_range.offset;
 	unsigned in_len = param->cipher_range.length;
@@ -278,9 +284,10 @@  int internal_encrypt(EVP_CIPHER_CTX *ctx, odp_crypto_op_param_t *param)
 }
 
 static
-int internal_decrypt(EVP_CIPHER_CTX *ctx, odp_crypto_op_param_t *param)
+int internal_decrypt(EVP_CIPHER_CTX *ctx,
+		     odp_packet_t pkt,
+		     const odp_crypto_packet_op_param_t *param)
 {
-	odp_packet_t pkt = param->out_pkt;
 	unsigned in_pos = param->cipher_range.offset;
 	unsigned out_pos = param->cipher_range.offset;
 	unsigned in_len = param->cipher_range.length;
@@ -333,7 +340,8 @@  int internal_decrypt(EVP_CIPHER_CTX *ctx, odp_crypto_op_param_t *param)
 }
 
 static
-odp_crypto_alg_err_t cipher_encrypt(odp_crypto_op_param_t *param,
+odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt,
+				    const odp_crypto_packet_op_param_t *param,
 				    odp_crypto_generic_session_t *session)
 {
 	EVP_CIPHER_CTX *ctx;
@@ -354,7 +362,7 @@  odp_crypto_alg_err_t cipher_encrypt(odp_crypto_op_param_t *param,
 	EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
 	EVP_CIPHER_CTX_set_padding(ctx, 0);
 
-	ret = internal_encrypt(ctx, param);
+	ret = internal_encrypt(ctx, pkt, param);
 
 	EVP_CIPHER_CTX_free(ctx);
 
@@ -363,7 +371,8 @@  odp_crypto_alg_err_t cipher_encrypt(odp_crypto_op_param_t *param,
 }
 
 static
-odp_crypto_alg_err_t cipher_decrypt(odp_crypto_op_param_t *param,
+odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt,
+				    const odp_crypto_packet_op_param_t *param,
 				    odp_crypto_generic_session_t *session)
 {
 	EVP_CIPHER_CTX *ctx;
@@ -384,7 +393,7 @@  odp_crypto_alg_err_t cipher_decrypt(odp_crypto_op_param_t *param,
 	EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
 	EVP_CIPHER_CTX_set_padding(ctx, 0);
 
-	ret = internal_decrypt(ctx, param);
+	ret = internal_decrypt(ctx, pkt, param);
 
 	EVP_CIPHER_CTX_free(ctx);
 
@@ -420,7 +429,8 @@  static int process_cipher_param(odp_crypto_generic_session_t *session,
 }
 
 static
-odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param,
+odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
+				     const odp_crypto_packet_op_param_t *param,
 				     odp_crypto_generic_session_t *session)
 {
 	EVP_CIPHER_CTX *ctx;
@@ -452,11 +462,11 @@  odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param,
 		EVP_EncryptUpdate(ctx, NULL, &dummy_len,
 				  aad_head, aad_len);
 
-	ret = internal_encrypt(ctx, param);
+	ret = internal_encrypt(ctx, pkt, param);
 
 	EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG,
 			    session->p.auth_digest_len, block);
-	odp_packet_copy_from_mem(param->out_pkt, param->hash_result_offset,
+	odp_packet_copy_from_mem(pkt, param->hash_result_offset,
 				 session->p.auth_digest_len, block);
 
 	EVP_CIPHER_CTX_free(ctx);
@@ -466,7 +476,8 @@  odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param,
 }
 
 static
-odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param,
+odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
+				     const odp_crypto_packet_op_param_t *param,
 				     odp_crypto_generic_session_t *session)
 {
 	EVP_CIPHER_CTX *ctx;
@@ -493,7 +504,7 @@  odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param,
 	EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
 	EVP_CIPHER_CTX_set_padding(ctx, 0);
 
-	odp_packet_copy_to_mem(param->out_pkt, param->hash_result_offset,
+	odp_packet_copy_to_mem(pkt, param->hash_result_offset,
 			       session->p.auth_digest_len, block);
 	EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG,
 			    session->p.auth_digest_len, block);
@@ -503,7 +514,7 @@  odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param,
 		EVP_DecryptUpdate(ctx, NULL, &dummy_len,
 				  aad_head, aad_len);
 
-	ret = internal_decrypt(ctx, param);
+	ret = internal_decrypt(ctx, pkt, param);
 
 	EVP_CIPHER_CTX_free(ctx);
 
@@ -833,87 +844,59 @@  int odp_crypto_session_destroy(odp_crypto_session_t session)
 	return 0;
 }
 
+/*
+ * Shim function around packet operation, can be used by other implementations.
+ */
 int
 odp_crypto_operation(odp_crypto_op_param_t *param,
 		     odp_bool_t *posted,
 		     odp_crypto_op_result_t *result)
 {
-	odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE;
-	odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE;
-	odp_crypto_generic_session_t *session;
+	odp_crypto_packet_op_param_t packet_param;
+	odp_packet_t out_pkt = param->out_pkt;
+	odp_crypto_packet_result_t packet_result;
 	odp_crypto_op_result_t local_result;
-	odp_bool_t allocated = false;
-
-	session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
+	int rc;
 
-	/* Resolve output buffer */
-	if (ODP_PACKET_INVALID == param->out_pkt &&
-	    ODP_POOL_INVALID != session->p.output_pool) {
-		param->out_pkt = odp_packet_alloc(session->p.output_pool,
-				odp_packet_len(param->pkt));
-		allocated = true;
-	}
+	packet_param.session = param->session;
+	packet_param.override_iv_ptr = param->override_iv_ptr;
+	packet_param.hash_result_offset = param->hash_result_offset;
+	packet_param.aad.ptr = param->aad.ptr;
+	packet_param.aad.length = param->aad.length;
+	packet_param.cipher_range = param->cipher_range;
+	packet_param.auth_range = param->auth_range;
 
-	if (odp_unlikely(ODP_PACKET_INVALID == param->out_pkt)) {
-		ODP_DBG("Alloc failed.\n");
-		return -1;
-	}
+	rc = odp_crypto_packet_op(&param->pkt, &out_pkt, &packet_param, 1);
+	if (rc < 0)
+		return rc;
 
-	if (param->pkt != param->out_pkt) {
-		int ret;
+	rc = odp_crypto_packet_result(&packet_result, out_pkt);
+	if (rc < 0)
+		return rc;
 
-		ret = odp_packet_copy_from_pkt(param->out_pkt,
-					       0,
-					       param->pkt,
-					       0,
-					       odp_packet_len(param->pkt));
-		if (odp_unlikely(ret < 0))
-			goto err;
+	/* Indicate to caller operation was sync */
+	*posted = 0;
 
-		_odp_packet_copy_md_to_packet(param->pkt, param->out_pkt);
-		odp_packet_free(param->pkt);
-		param->pkt = ODP_PACKET_INVALID;
-	}
-
-	/* Invoke the functions */
-	if (session->do_cipher_first) {
-		rc_cipher = session->cipher.func(param, session);
-		rc_auth = session->auth.func(param, session);
-	} else {
-		rc_auth = session->auth.func(param, session);
-		rc_cipher = session->cipher.func(param, session);
-	}
+	_odp_buffer_event_subtype_set(_odp_packet_to_buffer(out_pkt),
+				      ODP_EVENT_PACKET_BASIC);
 
 	/* Fill in result */
 #if ODP_DEPRECATED_API
 	local_result.ctx = param->ctx;
 #endif
-	local_result.pkt = param->out_pkt;
-	local_result.cipher_status.alg_err = rc_cipher;
-	local_result.cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
-	local_result.auth_status.alg_err = rc_auth;
-	local_result.auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
-	local_result.ok =
-		(rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) &&
-		(rc_auth == ODP_CRYPTO_ALG_ERR_NONE);
+	local_result.pkt = out_pkt;
+	local_result.cipher_status = packet_result.cipher_status;
+	local_result.auth_status = packet_result.auth_status;
+	local_result.ok = packet_result.ok;
 
-	/* Synchronous, simply return results */
-	if (!result)
-		goto err;
-	*result = local_result;
+	/*
+	 * Be bug-to-bug compatible. Return output packet also through params.
+	 */
+	param->out_pkt = out_pkt;
 
-	/* Indicate to caller operation was sync */
-	*posted = 0;
+	*result = local_result;
 
 	return 0;
-
-err:
-	if (allocated) {
-		odp_packet_free(param->out_pkt);
-		param->out_pkt = ODP_PACKET_INVALID;
-	}
-
-	return -1;
 }
 
 static void ODP_UNUSED openssl_thread_id(CRYPTO_THREADID ODP_UNUSED *id)
@@ -1095,3 +1078,171 @@  uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl)
 {
 	return (uint64_t)hdl;
 }
+
+odp_packet_t odp_crypto_packet_from_event(odp_event_t ev)
+{
+	/* This check not mandated by the API specification */
+	ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET);
+	ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO);
+
+	return odp_packet_from_event(ev);
+}
+
+odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
+{
+	return odp_packet_to_event(pkt);
+}
+
+static
+odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
+{
+	odp_packet_hdr_t *hdr = odp_packet_hdr(pkt);
+
+	return &hdr->crypto_op_result;
+}
+
+int odp_crypto_packet_result(odp_crypto_packet_result_t *result,
+			     odp_packet_t packet)
+{
+	odp_crypto_packet_result_t *op_result;
+
+	ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) ==
+		   ODP_EVENT_PACKET_CRYPTO);
+
+	op_result = get_op_result_from_packet(packet);
+
+	memcpy(result, op_result, sizeof(*result));
+
+	return 0;
+}
+
+static
+int odp_crypto_op(odp_packet_t pkt_in,
+		  odp_packet_t *pkt_out,
+		  const odp_crypto_packet_op_param_t *param)
+{
+	odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE;
+	odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE;
+	odp_crypto_generic_session_t *session;
+	odp_crypto_packet_result_t local_result;
+	odp_bool_t allocated = false;
+	odp_packet_t out_pkt = *pkt_out;
+	odp_crypto_packet_result_t *op_result;
+
+	session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
+
+	/* Resolve output buffer */
+	if (ODP_PACKET_INVALID == out_pkt &&
+	    ODP_POOL_INVALID != session->p.output_pool) {
+		out_pkt = odp_packet_alloc(session->p.output_pool,
+					   odp_packet_len(pkt_in));
+		allocated = true;
+	}
+
+	if (odp_unlikely(ODP_PACKET_INVALID == out_pkt)) {
+		ODP_DBG("Alloc failed.\n");
+		return -1;
+	}
+
+	if (pkt_in != out_pkt) {
+		int ret;
+
+		ret = odp_packet_copy_from_pkt(out_pkt,
+					       0,
+					       pkt_in,
+					       0,
+					       odp_packet_len(pkt_in));
+		if (odp_unlikely(ret < 0))
+			goto err;
+
+		_odp_packet_copy_md_to_packet(pkt_in, out_pkt);
+		odp_packet_free(pkt_in);
+		pkt_in = ODP_PACKET_INVALID;
+	}
+
+	/* Invoke the functions */
+	if (session->do_cipher_first) {
+		rc_cipher = session->cipher.func(out_pkt, param, session);
+		rc_auth = session->auth.func(out_pkt, param, session);
+	} else {
+		rc_auth = session->auth.func(out_pkt, param, session);
+		rc_cipher = session->cipher.func(out_pkt, param, session);
+	}
+
+	/* Fill in result */
+	local_result.cipher_status.alg_err = rc_cipher;
+	local_result.cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+	local_result.auth_status.alg_err = rc_auth;
+	local_result.auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+	local_result.ok =
+		(rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) &&
+		(rc_auth == ODP_CRYPTO_ALG_ERR_NONE);
+
+	_odp_buffer_event_subtype_set(_odp_packet_to_buffer(out_pkt),
+				      ODP_EVENT_PACKET_CRYPTO);
+	op_result = get_op_result_from_packet(out_pkt);
+	*op_result = local_result;
+
+	/* Synchronous, simply return results */
+	*pkt_out = out_pkt;
+
+	return 0;
+
+err:
+	if (allocated) {
+		odp_packet_free(out_pkt);
+		out_pkt = ODP_PACKET_INVALID;
+	}
+
+	return -1;
+}
+
+int odp_crypto_packet_op(const odp_packet_t pkt_in[],
+			 odp_packet_t pkt_out[],
+			 const odp_crypto_packet_op_param_t param[],
+			 int num_pkt)
+{
+	int i, rc;
+	odp_crypto_generic_session_t *session;
+
+	session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
+	ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.packet_op_mode);
+
+	for (i = 0; i < num_pkt; i++) {
+		rc = odp_crypto_op(pkt_in[i], &pkt_out[i], &param[i]);
+		if (rc < 0)
+			break;
+	}
+
+	return i;
+}
+
+int odp_crypto_packet_op_enq(const odp_packet_t pkt_in[],
+			     const odp_packet_t pkt_out[],
+			     const odp_crypto_packet_op_param_t param[],
+			     int num_pkt)
+{
+	odp_packet_t pkt;
+	odp_event_t event;
+	odp_crypto_generic_session_t *session;
+	int i, rc;
+
+	session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
+	ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.packet_op_mode);
+	ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue);
+
+	for (i = 0; i < num_pkt; i++) {
+		pkt = pkt_out[i];
+		rc = odp_crypto_op(pkt_in[i], &pkt, &param[i]);
+		if (rc < 0)
+			break;
+
+		event = odp_packet_to_event(pkt);
+		if (odp_queue_enq(session->p.compl_queue, event)) {
+			odp_event_free(event);
+			break;
+		}
+	}
+
+	return i;
+}