@@ -450,13 +450,11 @@ caam_jr_prep_cdb(struct caam_jr_session *ses)
&alginfo_c, &alginfo_a);
}
} else {
- /* Auth_only_len is set as 0 here and it will be
- * overwritten in fd for each packet.
- */
+ /* Auth_only_len is overwritten in fd for each job */
shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
true, swap, SHR_SERIAL,
&alginfo_c, &alginfo_a,
- ses->iv.length, 0,
+ ses->iv.length,
ses->digest_length, ses->dir);
}
}
@@ -1066,10 +1064,11 @@ build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
ses->iv.offset);
struct sec_job_descriptor_t *jobdescr;
- uint32_t auth_only_len;
-
- auth_only_len = op->sym->auth.data.length -
- op->sym->cipher.data.length;
+ uint16_t auth_hdr_len = sym->cipher.data.offset -
+ sym->auth.data.offset;
+ uint16_t auth_tail_len = sym->auth.data.length -
+ sym->cipher.data.length - auth_hdr_len;
+ uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
if (sym->m_dst) {
mbuf = sym->m_dst;
@@ -1208,10 +1207,11 @@ build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses)
uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
ses->iv.offset);
struct sec_job_descriptor_t *jobdescr;
- uint32_t auth_only_len;
-
- auth_only_len = op->sym->auth.data.length -
- op->sym->cipher.data.length;
+ uint16_t auth_hdr_len = sym->cipher.data.offset -
+ sym->auth.data.offset;
+ uint16_t auth_tail_len = sym->auth.data.length -
+ sym->cipher.data.length - auth_hdr_len;
+ uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
src_start_addr = rte_pktmbuf_iova(sym->m_src);
if (sym->m_dst)
@@ -583,8 +583,11 @@ build_authenc_sg_fd(dpaa2_sec_session *sess,
struct ctxt_priv *priv = sess->ctxt;
struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
struct sec_flow_context *flc;
- uint32_t auth_only_len = sym_op->auth.data.length -
- sym_op->cipher.data.length;
+ uint16_t auth_hdr_len = sym_op->cipher.data.offset -
+ sym_op->auth.data.offset;
+ uint16_t auth_tail_len = sym_op->auth.data.length -
+ sym_op->cipher.data.length - auth_hdr_len;
+ uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
int icv_len = sess->digest_length;
uint8_t *old_icv;
struct rte_mbuf *mbuf;
@@ -727,8 +730,12 @@ build_authenc_fd(dpaa2_sec_session *sess,
struct ctxt_priv *priv = sess->ctxt;
struct qbman_fle *fle, *sge;
struct sec_flow_context *flc;
- uint32_t auth_only_len = sym_op->auth.data.length -
- sym_op->cipher.data.length;
+ uint16_t auth_hdr_len = sym_op->cipher.data.offset -
+ sym_op->auth.data.offset;
+ uint16_t auth_tail_len = sym_op->auth.data.length -
+ sym_op->cipher.data.length - auth_hdr_len;
+ uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
+
int icv_len = sess->digest_length, retval;
uint8_t *old_icv;
uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
@@ -2217,7 +2224,6 @@ dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev,
struct rte_crypto_sym_xform *xform,
dpaa2_sec_session *session)
{
- struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt;
struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
struct alginfo authdata, cipherdata;
int bufsize;
@@ -2411,7 +2417,6 @@ dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev,
0, SHR_SERIAL,
&cipherdata, &authdata,
session->iv.length,
- ctxt->auth_only_len,
session->digest_length,
session->dir);
if (bufsize < 0) {
@@ -1412,9 +1412,6 @@ cnstr_shdsc_ipsec_new_decap(uint32_t *descbuf, bool ps,
*
* @ivlen: length of the IV to be read from the input frame, before any data
* to be processed
- * @auth_only_len: length of the data to be authenticated-only (commonly IP
- * header, IV, Sequence number and SPI)
- * Note: Extended Sequence Number processing is NOT supported
*
* @trunc_len: the length of the ICV to be written to the output frame. If 0,
* then the corresponding length of the digest, according to the
@@ -1425,30 +1422,30 @@ cnstr_shdsc_ipsec_new_decap(uint32_t *descbuf, bool ps,
* will be done correctly:
* For encapsulation:
* Input:
- * +----+----------------+---------------------------------------------+
- * | IV | Auth-only data | Padded data to be authenticated & Encrypted |
- * +----+----------------+---------------------------------------------+
+ * +----+----------------+-----------------------------------------------+
+ * | IV | Auth-only head | Padded data to be auth & Enc | Auth-only tail |
+ * +----+----------------+-----------------------------------------------+
* Output:
* +--------------------------------------+
* | Authenticated & Encrypted data | ICV |
* +--------------------------------+-----+
-
+ *
* For decapsulation:
* Input:
- * +----+----------------+--------------------------------+-----+
- * | IV | Auth-only data | Authenticated & Encrypted data | ICV |
- * +----+----------------+--------------------------------+-----+
+ * +----+----------------+-----------------+----------------------+
+ * | IV | Auth-only head | Auth & Enc data | Auth-only tail | ICV |
+ * +----+----------------+-----------------+----------------------+
* Output:
- * +----+--------------------------+
+ * +----+---------------------------+
* | Decrypted & authenticated data |
- * +----+--------------------------+
+ * +----+---------------------------+
*
* Note: This descriptor can use per-packet commands, encoded as below in the
* DPOVRD register:
- * 32 24 16 0
- * +------+---------------------+
- * | 0x80 | 0x00| auth_only_len |
- * +------+---------------------+
+ * 32 28 16 1
+ * +------+------------------------------+
+ * | 0x8 | auth_tail_len | auth_hdr_len |
+ * +------+------------------------------+
*
* This mechanism is available only for SoCs having SEC ERA >= 3. In other
* words, this will not work for P4080TO2
@@ -1465,7 +1462,7 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap,
enum rta_share_type share,
struct alginfo *cipherdata,
struct alginfo *authdata,
- uint16_t ivlen, uint16_t auth_only_len,
+ uint16_t ivlen,
uint8_t trunc_len, uint8_t dir)
{
struct program prg;
@@ -1473,16 +1470,16 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap,
const bool need_dk = (dir == DIR_DEC) &&
(cipherdata->algtype == OP_ALG_ALGSEL_AES) &&
(cipherdata->algmode == OP_ALG_AAI_CBC);
+ int data_type;
- LABEL(skip_patch_len);
LABEL(keyjmp);
LABEL(skipkeys);
- LABEL(aonly_len_offset);
- REFERENCE(pskip_patch_len);
+ LABEL(proc_icv);
+ LABEL(no_auth_tail);
REFERENCE(pkeyjmp);
REFERENCE(pskipkeys);
- REFERENCE(read_len);
- REFERENCE(write_len);
+ REFERENCE(p_proc_icv);
+ REFERENCE(p_no_auth_tail);
PROGRAM_CNTXT_INIT(p, descbuf, 0);
@@ -1500,48 +1497,15 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap,
SHR_HDR(p, share, 1, SC);
- /*
- * M0 will contain the value provided by the user when creating
- * the shared descriptor. If the user provided an override in
- * DPOVRD, then M0 will contain that value
- */
- MATHB(p, MATH0, ADD, auth_only_len, MATH0, 4, IMMED2);
-
- if (rta_sec_era >= RTA_SEC_ERA_3) {
- /*
- * Check if the user wants to override the auth-only len
- */
- MATHB(p, DPOVRD, ADD, 0x80000000, MATH2, 4, IMMED2);
-
- /*
- * No need to patch the length of the auth-only data read if
- * the user did not override it
- */
- pskip_patch_len = JUMP(p, skip_patch_len, LOCAL_JUMP, ALL_TRUE,
- MATH_N);
-
- /* Get auth-only len in M0 */
- MATHB(p, MATH2, AND, 0xFFFF, MATH0, 4, IMMED2);
-
- /*
- * Since M0 is used in calculations, don't mangle it, copy
- * its content to M1 and use this for patching.
- */
- MATHB(p, MATH0, ADD, MATH1, MATH1, 4, 0);
-
- read_len = MOVE(p, DESCBUF, 0, MATH1, 0, 6, WAITCOMP | IMMED);
- write_len = MOVE(p, MATH1, 0, DESCBUF, 0, 8, WAITCOMP | IMMED);
-
- SET_LABEL(p, skip_patch_len);
- }
- /*
- * MATH0 contains the value in DPOVRD w/o the MSB, or the initial
- * value, as provided by the user at descriptor creation time
- */
- if (dir == DIR_ENC)
- MATHB(p, MATH0, ADD, ivlen, MATH0, 4, IMMED2);
- else
- MATHB(p, MATH0, ADD, ivlen + trunc_len, MATH0, 4, IMMED2);
+ /* Collect the (auth_tail || auth_hdr) len from DPOVRD */
+ MATHB(p, DPOVRD, ADD, 0x80000000, MATH2, 4, IMMED2);
+
+ /* Get auth_hdr len in MATH0 */
+ MATHB(p, MATH2, AND, 0xFFFF, MATH0, 4, IMMED2);
+
+ /* Get auth_tail len in MATH2 */
+ MATHB(p, MATH2, AND, 0xFFF0000, MATH2, 4, IMMED2);
+ MATHI(p, MATH2, RSHIFT, 16, MATH2, 4, IMMED2);
pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SHRD);
@@ -1581,61 +1545,70 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap,
OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir);
}
+ /* Read IV */
+ if (cipherdata->algmode == OP_ALG_AAI_CTR)
+ SEQLOAD(p, CONTEXT1, 16, ivlen, 0);
+ else
+ SEQLOAD(p, CONTEXT1, 0, ivlen, 0);
+
+ /*
+ * authenticate auth_hdr data
+ */
+ MATHB(p, MATH0, ADD, ZERO, VSEQINSZ, 4, 0);
+ SEQFIFOLOAD(p, MSG2, 0, VLF);
+
/*
* Prepare the length of the data to be both encrypted/decrypted
* and authenticated/checked
*/
- MATHB(p, SEQINSZ, SUB, MATH0, VSEQINSZ, 4, 0);
+ MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0);
+ if (dir == DIR_DEC) {
+ MATHB(p, VSEQINSZ, SUB, trunc_len, VSEQINSZ, 4, IMMED2);
+ data_type = MSGINSNOOP;
+ } else {
+ data_type = MSGOUTSNOOP;
+ }
- MATHB(p, VSEQINSZ, SUB, MATH3, VSEQOUTSZ, 4, 0);
+ MATHB(p, VSEQINSZ, ADD, ZERO, VSEQOUTSZ, 4, 0);
/* Prepare for writing the output frame */
SEQFIFOSTORE(p, MSG, 0, 0, VLF);
- SET_LABEL(p, aonly_len_offset);
- /* Read IV */
- if (cipherdata->algmode == OP_ALG_AAI_CTR)
- SEQLOAD(p, CONTEXT1, 16, ivlen, 0);
- else
- SEQLOAD(p, CONTEXT1, 0, ivlen, 0);
+ /* Check if there is no auth-tail */
+ MATHB(p, MATH2, ADD, ZERO, MATH2, 4, 0);
+ p_no_auth_tail = JUMP(p, no_auth_tail, LOCAL_JUMP, ALL_TRUE, MATH_Z);
/*
- * Read data needed only for authentication. This is overwritten above
- * if the user requested it.
+ * Read input plain/cipher text, encrypt/decrypt & auth & write
+ * to output
*/
- SEQFIFOLOAD(p, MSG2, auth_only_len, 0);
+ SEQFIFOLOAD(p, data_type, 0, VLF | LAST1 | FLUSH1);
+
+ /* Authenticate auth tail */
+ MATHB(p, MATH2, ADD, ZERO, VSEQINSZ, 4, 0);
+ SEQFIFOLOAD(p, MSG2, 0, VLF | LAST2);
+
+ /* Jump to process icv */
+ p_proc_icv = JUMP(p, proc_icv, LOCAL_JUMP, ALL_FALSE, MATH_Z);
+
+ SET_LABEL(p, no_auth_tail);
- if (dir == DIR_ENC) {
- /*
- * Read input plaintext, encrypt and authenticate & write to
- * output
- */
- SEQFIFOLOAD(p, MSGOUTSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1);
+ SEQFIFOLOAD(p, data_type, 0, VLF | LAST1 | LAST2 | FLUSH1);
+ SET_LABEL(p, proc_icv);
+
+ if (dir == DIR_ENC)
/* Finally, write the ICV */
SEQSTORE(p, CONTEXT2, 0, trunc_len, 0);
- } else {
- /*
- * Read input ciphertext, decrypt and authenticate & write to
- * output
- */
- SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1);
-
+ else
/* Read the ICV to check */
SEQFIFOLOAD(p, ICV2, trunc_len, LAST2);
- }
PATCH_JUMP(p, pkeyjmp, keyjmp);
PATCH_JUMP(p, pskipkeys, skipkeys);
- PATCH_JUMP(p, pskipkeys, skipkeys);
-
- if (rta_sec_era >= RTA_SEC_ERA_3) {
- PATCH_JUMP(p, pskip_patch_len, skip_patch_len);
- PATCH_MOVE(p, read_len, aonly_len_offset);
- PATCH_MOVE(p, write_len, aonly_len_offset);
- }
-
+ PATCH_JUMP(p, p_no_auth_tail, no_auth_tail);
+ PATCH_JUMP(p, p_proc_icv, proc_icv);
return PROGRAM_FINALIZE(p);
}
@@ -742,7 +742,7 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses)
*/
shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
true, swap, SHR_SERIAL, &alginfo_c, &alginfo_a,
- ses->iv.length, 0,
+ ses->iv.length,
ses->digest_length, ses->dir);
}
@@ -1753,7 +1753,8 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
struct rte_crypto_op *op;
struct dpaa_sec_job *cf;
dpaa_sec_session *ses;
- uint32_t auth_only_len, index, flags[DPAA_SEC_BURST] = {0};
+ uint16_t auth_hdr_len, auth_tail_len;
+ uint32_t index, flags[DPAA_SEC_BURST] = {0};
struct qman_fq *inq[DPAA_SEC_BURST];
while (nb_ops) {
@@ -1809,8 +1810,10 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
goto send_pkts;
}
- auth_only_len = op->sym->auth.data.length -
+ auth_hdr_len = op->sym->auth.data.length -
op->sym->cipher.data.length;
+ auth_tail_len = 0;
+
if (rte_pktmbuf_is_contiguous(op->sym->m_src) &&
((op->sym->m_dst == NULL) ||
rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
@@ -1824,8 +1827,15 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
cf = build_cipher_only(op, ses);
} else if (is_aead(ses)) {
cf = build_cipher_auth_gcm(op, ses);
- auth_only_len = ses->auth_only_len;
+ auth_hdr_len = ses->auth_only_len;
} else if (is_auth_cipher(ses)) {
+ auth_hdr_len =
+ op->sym->cipher.data.offset
+ - op->sym->auth.data.offset;
+ auth_tail_len =
+ op->sym->auth.data.length
+ - op->sym->cipher.data.length
+ - auth_hdr_len;
cf = build_cipher_auth(op, ses);
} else {
DPAA_SEC_DP_ERR("not supported ops");
@@ -1842,8 +1852,15 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
cf = build_cipher_only_sg(op, ses);
} else if (is_aead(ses)) {
cf = build_cipher_auth_gcm_sg(op, ses);
- auth_only_len = ses->auth_only_len;
+ auth_hdr_len = ses->auth_only_len;
} else if (is_auth_cipher(ses)) {
+ auth_hdr_len =
+ op->sym->cipher.data.offset
+ - op->sym->auth.data.offset;
+ auth_tail_len =
+ op->sym->auth.data.length
+ - op->sym->cipher.data.length
+ - auth_hdr_len;
cf = build_cipher_auth_sg(op, ses);
} else {
DPAA_SEC_DP_ERR("not supported ops");
@@ -1865,12 +1882,16 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg));
fd->_format1 = qm_fd_compound;
fd->length29 = 2 * sizeof(struct qm_sg_entry);
+
/* Auth_only_len is set as 0 in descriptor and it is
* overwritten here in the fd.cmd which will update
* the DPOVRD reg.
*/
- if (auth_only_len)
- fd->cmd = 0x80000000 | auth_only_len;
+ if (auth_hdr_len || auth_tail_len) {
+ fd->cmd = 0x80000000;
+ fd->cmd |=
+ ((auth_tail_len << 16) | auth_hdr_len);
+ }
/* In case of PDCP, per packet HFN is stored in
* mbuf priv after sym_op.