Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright(c) 2017-2022 Intel Corporation
3 : : */
4 : :
5 : : #include <rte_cryptodev.h>
6 : : #include <cryptodev_pmd.h>
7 : : #include "qat_sym_session.h"
8 : : #include "qat_sym.h"
9 : : #include "qat_asym.h"
10 : : #include "qat_crypto.h"
11 : : #include "qat_crypto_pmd_gens.h"
12 : :
13 : : #define MIXED_CRYPTO_MIN_FW_VER 0x04090000
14 : :
15 : : static struct rte_cryptodev_capabilities qat_sym_crypto_legacy_caps_gen2[] = {
16 : : QAT_SYM_CIPHER_CAP(DES_CBC,
17 : : CAP_SET(block_size, 8),
18 : : CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)),
19 : : QAT_SYM_CIPHER_CAP(3DES_CBC,
20 : : CAP_SET(block_size, 8),
21 : : CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)),
22 : : QAT_SYM_CIPHER_CAP(3DES_CTR,
23 : : CAP_SET(block_size, 8),
24 : : CAP_RNG(key_size, 16, 24, 8), CAP_RNG(iv_size, 8, 8, 0)),
25 : : QAT_SYM_PLAIN_AUTH_CAP(SHA1,
26 : : CAP_SET(block_size, 64),
27 : : CAP_RNG(digest_size, 1, 20, 1)),
28 : : QAT_SYM_AUTH_CAP(SHA224,
29 : : CAP_SET(block_size, 64),
30 : : CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 28, 1),
31 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
32 : : QAT_SYM_AUTH_CAP(SHA224_HMAC,
33 : : CAP_SET(block_size, 64),
34 : : CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 28, 1),
35 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
36 : : QAT_SYM_AUTH_CAP(SHA1_HMAC,
37 : : CAP_SET(block_size, 64),
38 : : CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 20, 1),
39 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
40 : : QAT_SYM_AUTH_CAP(MD5_HMAC,
41 : : CAP_SET(block_size, 64),
42 : : CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 16, 1),
43 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
44 : : QAT_SYM_CIPHER_CAP(DES_DOCSISBPI,
45 : : CAP_SET(block_size, 8),
46 : : CAP_RNG(key_size, 8, 8, 0), CAP_RNG(iv_size, 8, 8, 0)),
47 : : };
48 : :
49 : : static struct rte_cryptodev_capabilities qat_sym_crypto_caps_gen2[] = {
50 : : QAT_SYM_AEAD_CAP(AES_GCM,
51 : : CAP_SET(block_size, 16),
52 : : CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4),
53 : : CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 0, 12, 12)),
54 : : QAT_SYM_AEAD_CAP(AES_CCM,
55 : : CAP_SET(block_size, 16),
56 : : CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 2),
57 : : CAP_RNG(aad_size, 0, 224, 1), CAP_RNG(iv_size, 7, 13, 1)),
58 : : QAT_SYM_AUTH_CAP(AES_GMAC,
59 : : CAP_SET(block_size, 16),
60 : : CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4),
61 : : CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 0, 12, 12)),
62 : : QAT_SYM_AUTH_CAP(AES_CMAC,
63 : : CAP_SET(block_size, 16),
64 : : CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 4),
65 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
66 : : QAT_SYM_AUTH_CAP(SHA256,
67 : : CAP_SET(block_size, 64),
68 : : CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 32, 1),
69 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
70 : : QAT_SYM_AUTH_CAP(SHA384,
71 : : CAP_SET(block_size, 128),
72 : : CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 48, 1),
73 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
74 : : QAT_SYM_AUTH_CAP(SHA512,
75 : : CAP_SET(block_size, 128),
76 : : CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 64, 1),
77 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
78 : : QAT_SYM_PLAIN_AUTH_CAP(SHA3_256,
79 : : CAP_SET(block_size, 136),
80 : : CAP_RNG(digest_size, 32, 32, 0)),
81 : : QAT_SYM_AUTH_CAP(SHA256_HMAC,
82 : : CAP_SET(block_size, 64),
83 : : CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 32, 1),
84 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
85 : : QAT_SYM_AUTH_CAP(SHA384_HMAC,
86 : : CAP_SET(block_size, 128),
87 : : CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 48, 1),
88 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
89 : : QAT_SYM_AUTH_CAP(SHA512_HMAC,
90 : : CAP_SET(block_size, 128),
91 : : CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 64, 1),
92 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
93 : : QAT_SYM_AUTH_CAP(AES_XCBC_MAC,
94 : : CAP_SET(block_size, 16),
95 : : CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 12, 12, 0),
96 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
97 : : QAT_SYM_AUTH_CAP(SNOW3G_UIA2,
98 : : CAP_SET(block_size, 16),
99 : : CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0),
100 : : CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)),
101 : : QAT_SYM_AUTH_CAP(KASUMI_F9,
102 : : CAP_SET(block_size, 8),
103 : : CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0),
104 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
105 : : QAT_SYM_AUTH_CAP(NULL,
106 : : CAP_SET(block_size, 1),
107 : : CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(digest_size),
108 : : CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
109 : : QAT_SYM_CIPHER_CAP(AES_CBC,
110 : : CAP_SET(block_size, 16),
111 : : CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)),
112 : : QAT_SYM_CIPHER_CAP(AES_CTR,
113 : : CAP_SET(block_size, 16),
114 : : CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)),
115 : : QAT_SYM_CIPHER_CAP(AES_XTS,
116 : : CAP_SET(block_size, 16),
117 : : CAP_RNG(key_size, 32, 64, 32), CAP_RNG(iv_size, 16, 16, 0)),
118 : : QAT_SYM_CIPHER_CAP(AES_DOCSISBPI,
119 : : CAP_SET(block_size, 16),
120 : : CAP_RNG(key_size, 16, 32, 16), CAP_RNG(iv_size, 16, 16, 0)),
121 : : QAT_SYM_CIPHER_CAP(SNOW3G_UEA2,
122 : : CAP_SET(block_size, 16),
123 : : CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)),
124 : : QAT_SYM_CIPHER_CAP(KASUMI_F8,
125 : : CAP_SET(block_size, 8),
126 : : CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 8, 8, 0)),
127 : : QAT_SYM_CIPHER_CAP(NULL,
128 : : CAP_SET(block_size, 1),
129 : : CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(iv_size)),
130 : : QAT_SYM_CIPHER_CAP(ZUC_EEA3,
131 : : CAP_SET(block_size, 16),
132 : : CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)),
133 : : QAT_SYM_AUTH_CAP(ZUC_EIA3,
134 : : CAP_SET(block_size, 16),
135 : : CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0),
136 : : CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)),
137 : : RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
138 : : };
139 : :
140 : : static int
141 : 0 : qat_sym_crypto_qp_setup_gen2(struct rte_cryptodev *dev, uint16_t qp_id,
142 : : const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
143 : : {
144 : 0 : struct qat_cryptodev_private *qat_sym_private = dev->data->dev_private;
145 : : struct qat_qp *qp;
146 : : int ret;
147 : :
148 [ # # ]: 0 : if (qat_cryptodev_qp_setup(dev, qp_id, qp_conf, socket_id)) {
149 : 0 : QAT_LOG(DEBUG, "QAT qp setup failed");
150 : 0 : return -1;
151 : : }
152 : :
153 : 0 : qp = qat_sym_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][qp_id];
154 : 0 : ret = qat_cq_get_fw_version(qp);
155 [ # # ]: 0 : if (ret < 0) {
156 : 0 : qat_cryptodev_qp_release(dev, qp_id);
157 : 0 : return ret;
158 : : }
159 : :
160 [ # # ]: 0 : if (ret != 0)
161 : 0 : QAT_LOG(DEBUG, "QAT firmware version: %d.%d.%d",
162 : : (ret >> 24) & 0xff,
163 : : (ret >> 16) & 0xff,
164 : : (ret >> 8) & 0xff);
165 : : else
166 : 0 : QAT_LOG(DEBUG, "unknown QAT firmware version");
167 : :
168 : : /* set capabilities based on the fw version */
169 [ # # ]: 0 : qat_sym_private->internal_capabilities |= QAT_SYM_CAP_VALID |
170 : : ((ret >= MIXED_CRYPTO_MIN_FW_VER) ?
171 : : QAT_SYM_CAP_MIXED_CRYPTO : 0);
172 : 0 : return 0;
173 : : }
174 : :
175 : : void
176 : 0 : qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
177 : : uint8_t hash_flag)
178 : : {
179 : : struct icp_qat_fw_comn_req_hdr *header = &session->fw_req.comn_hdr;
180 : : struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *cd_ctrl =
181 : : (struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *)
182 : : session->fw_req.cd_ctrl.content_desc_ctrl_lw;
183 : :
184 : : /* Set the Use Extended Protocol Flags bit in LW 1 */
185 : 0 : QAT_FIELD_SET(header->comn_req_flags,
186 : : QAT_COMN_EXT_FLAGS_USED,
187 : : QAT_COMN_EXT_FLAGS_BITPOS,
188 : : QAT_COMN_EXT_FLAGS_MASK);
189 : :
190 : : /* Set Hash Flags in LW 28 */
191 : 0 : cd_ctrl->hash_flags |= hash_flag;
192 : :
193 : : /* Set proto flags in LW 1 */
194 [ # # # ]: 0 : switch (session->qat_cipher_alg) {
195 : 0 : case ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2:
196 : 0 : ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
197 : : ICP_QAT_FW_LA_SNOW_3G_PROTO);
198 : 0 : ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
199 : : header->serv_specif_flags, 0);
200 : 0 : break;
201 : 0 : case ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3:
202 : 0 : ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
203 : : ICP_QAT_FW_LA_NO_PROTO);
204 : 0 : ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
205 : : header->serv_specif_flags,
206 : : ICP_QAT_FW_LA_ZUC_3G_PROTO);
207 : 0 : break;
208 : 0 : default:
209 : 0 : ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
210 : : ICP_QAT_FW_LA_NO_PROTO);
211 : 0 : ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
212 : : header->serv_specif_flags, 0);
213 : 0 : break;
214 : : }
215 : 0 : }
216 : :
217 : : static int
218 : 0 : qat_sym_crypto_set_session_gen2(void *cdev, void *session)
219 : : {
220 : : struct rte_cryptodev *dev = cdev;
221 : : struct qat_sym_session *ctx = session;
222 : 0 : const struct qat_cryptodev_private *qat_private =
223 : 0 : dev->data->dev_private;
224 : : int ret;
225 : :
226 : 0 : ret = qat_sym_crypto_set_session_gen1(cdev, session);
227 [ # # ]: 0 : if (ret == -ENOTSUP) {
228 : : /* GEN1 returning -ENOTSUP as it cannot handle some mixed algo,
229 : : * but some are not supported by GEN2, so checking here
230 : : */
231 [ # # ]: 0 : if ((qat_private->internal_capabilities &
232 : : QAT_SYM_CAP_MIXED_CRYPTO) == 0)
233 : : return -ENOTSUP;
234 : :
235 [ # # ]: 0 : if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 &&
236 [ # # ]: 0 : ctx->qat_cipher_alg !=
237 : : ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
238 : 0 : qat_sym_session_set_ext_hash_flags_gen2(ctx,
239 : : 1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS);
240 [ # # ]: 0 : } else if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 &&
241 [ # # ]: 0 : ctx->qat_cipher_alg !=
242 : : ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
243 : 0 : qat_sym_session_set_ext_hash_flags_gen2(ctx,
244 : : 1 << ICP_QAT_FW_AUTH_HDR_FLAG_SNOW3G_UIA2_BITPOS);
245 [ # # # # ]: 0 : } else if ((ctx->aes_cmac ||
246 : 0 : ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) &&
247 : 0 : (ctx->qat_cipher_alg ==
248 [ # # ]: 0 : ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
249 : : ctx->qat_cipher_alg ==
250 : : ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)) {
251 : 0 : qat_sym_session_set_ext_hash_flags_gen2(ctx, 0);
252 : : }
253 : :
254 : : ret = 0;
255 : : }
256 : :
257 : : return ret;
258 : : }
259 : :
260 : : struct rte_cryptodev_ops qat_sym_crypto_ops_gen2 = {
261 : :
262 : : /* Device related operations */
263 : : .dev_configure = qat_cryptodev_config,
264 : : .dev_start = qat_cryptodev_start,
265 : : .dev_stop = qat_cryptodev_stop,
266 : : .dev_close = qat_cryptodev_close,
267 : : .dev_infos_get = qat_cryptodev_info_get,
268 : :
269 : : .stats_get = qat_cryptodev_stats_get,
270 : : .stats_reset = qat_cryptodev_stats_reset,
271 : : .queue_pair_setup = qat_sym_crypto_qp_setup_gen2,
272 : : .queue_pair_release = qat_cryptodev_qp_release,
273 : :
274 : : /* Crypto related operations */
275 : : .sym_session_get_size = qat_sym_session_get_private_size,
276 : : .sym_session_configure = qat_sym_session_configure,
277 : : .sym_session_clear = qat_sym_session_clear,
278 : :
279 : : /* Raw data-path API related operations */
280 : : .sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,
281 : : .sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,
282 : : };
283 : :
284 : : static int
285 : 0 : qat_sym_crypto_cap_get_gen2(struct qat_cryptodev_private *internals,
286 : : const char *capa_memz_name,
287 : : const uint16_t __rte_unused slice_map)
288 : : {
289 : : uint32_t legacy_capa_num;
290 : : uint32_t size = sizeof(qat_sym_crypto_caps_gen2);
291 : : uint32_t legacy_size = sizeof(qat_sym_crypto_legacy_caps_gen2);
292 : : legacy_capa_num = legacy_size/sizeof(struct rte_cryptodev_capabilities);
293 : :
294 [ # # ]: 0 : if (unlikely(qat_legacy_capa))
295 : : size = size + legacy_size;
296 : :
297 : 0 : internals->capa_mz = rte_memzone_lookup(capa_memz_name);
298 [ # # ]: 0 : if (internals->capa_mz == NULL) {
299 : 0 : internals->capa_mz = rte_memzone_reserve(capa_memz_name,
300 : 0 : size, rte_socket_id(), 0);
301 [ # # ]: 0 : if (internals->capa_mz == NULL) {
302 : 0 : QAT_LOG(DEBUG,
303 : : "Error allocating memzone for capabilities");
304 : 0 : return -1;
305 : : }
306 : : }
307 : :
308 : 0 : struct rte_cryptodev_capabilities *addr =
309 : : (struct rte_cryptodev_capabilities *)
310 : 0 : internals->capa_mz->addr;
311 : : struct rte_cryptodev_capabilities *capabilities;
312 : :
313 [ # # ]: 0 : if (unlikely(qat_legacy_capa)) {
314 : : capabilities = qat_sym_crypto_legacy_caps_gen2;
315 : : memcpy(addr, capabilities, legacy_size);
316 : 0 : addr += legacy_capa_num;
317 : : }
318 : : capabilities = qat_sym_crypto_caps_gen2;
319 : : memcpy(addr, capabilities, sizeof(qat_sym_crypto_caps_gen2));
320 : 0 : internals->qat_dev_capabilities = internals->capa_mz->addr;
321 : :
322 : 0 : return 0;
323 : : }
324 : :
325 : 235 : RTE_INIT(qat_sym_crypto_gen2_init)
326 : : {
327 : 235 : qat_sym_gen_dev_ops[QAT_GEN2].cryptodev_ops = &qat_sym_crypto_ops_gen2;
328 : 235 : qat_sym_gen_dev_ops[QAT_GEN2].get_capabilities =
329 : : qat_sym_crypto_cap_get_gen2;
330 : 235 : qat_sym_gen_dev_ops[QAT_GEN2].set_session =
331 : : qat_sym_crypto_set_session_gen2;
332 : 235 : qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
333 : : qat_sym_configure_raw_dp_ctx_gen1;
334 : 235 : qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
335 : : qat_sym_crypto_feature_flags_get_gen1;
336 : 235 : qat_sym_gen_dev_ops[QAT_GEN2].create_security_ctx =
337 : : qat_sym_create_security_gen1;
338 : 235 : }
339 : :
340 : 235 : RTE_INIT(qat_asym_crypto_gen2_init)
341 : : {
342 : 235 : qat_asym_gen_dev_ops[QAT_GEN2].cryptodev_ops =
343 : : &qat_asym_crypto_ops_gen1;
344 : 235 : qat_asym_gen_dev_ops[QAT_GEN2].get_capabilities =
345 : : qat_asym_crypto_cap_get_gen1;
346 : 235 : qat_asym_gen_dev_ops[QAT_GEN2].get_feature_flags =
347 : : qat_asym_crypto_feature_flags_get_gen1;
348 : 235 : qat_asym_gen_dev_ops[QAT_GEN2].set_session =
349 : : qat_asym_crypto_set_session_gen1;
350 : 235 : }
|