path: root/platform/linux-keystone2/odp_crypto.c

/*
 * Copyright (c) 2014, Linaro Limited
 * Copyright (c) 2014, Texas Instruments Incorporated
 * All rights reserved.
 *
 * SPDX-License-Identifier:     BSD-3-Clause
 */

#include <odp/crypto.h>
#include <odp/event.h>
#include <odp/random.h>
#include <odp/atomic.h>
#include <odp/spinlock.h>
#include <odp/sync.h>
#include <odp/align.h>
#include <odp/hints.h>
#include <odp/shared_memory.h>
#include <odp/queue.h>
#include <odp/byteorder.h>

#include <odp/plat/debug.h>

#include <odp_packet_internal.h>
#include <odp_queue_internal.h>
#include <odp_pool_internal.h>
#include <odp_internal.h>

#include <string.h>
#include <odp/plat/ti_mcsdk.h>

#define ODP_CRYPTO_MAX_IV_LENGTH 32
static const char SHM_DEFAULT_NAME[] = "crypto_pool";

#define MAX_SESSIONS 32
static shres_table_t *session_tbl;
static int session_free_cb(void *data);

struct iv_full {
	uint8_t data[ODP_CRYPTO_MAX_IV_LENGTH];
	size_t  length;
};

/**
 * Per crypto session data structure
 */
struct odp_crypto_session_s {
	nwal_Handle         dm_handle;
	nwalTxDmPSCmdInfo_t dm_ps_cmdinfo;
	odp_pool_t          out_pool;
	cppi_flow_entry_t  *cppi_flow;
	odp_queue_t         compl_queue;
	odp_crypto_op_mode_t pref_mode;

	struct {
		odp_cipher_alg_t alg;
		struct iv_full        iv;
	} cipher;

	struct {
		odp_auth_alg_t alg;
		struct iv_full      iv;
		uint32_t            tag_len;
	} auth;

	uint32_t            index;
	odp_crypto_op_t  op;
	shres_head_t entry_head;       /**< shres head */
};

static const char odp_session_table_name[] = "odp_session_entries";

static struct odp_crypto_session_s *alloc_session(void)
{
	struct odp_crypto_session_s *entry;

	entry = shres_alloc(session_tbl, typeof(*entry));
	if (!entry) {
		ODP_ERR("Couldn't allocate sesssion entry\n");
		return ODP_CRYPTO_SESSION_INVALID;
	}

	return entry;
}

static void _print_nwalCreateDmSAParams(nwalCreateDmSAParams_t *dmSaParam)
{
	odp_pr_dbg("dmSaParam.dmChnType = %u\n",
		   dmSaParam->dmSaParam.dmChnType);
	odp_pr_dbg("dmSaParam.authMode = %u\n", dmSaParam->dmSaParam.authMode);
	odp_pr_dbg("dmSaParam.cipherMode = %u\n",
		   dmSaParam->dmSaParam.cipherMode);
	odp_pr_dbg("dmSaParam.enc1st = %u\n", dmSaParam->dmSaParam.enc1st);
	odp_pr_dbg("dmSaParam.macSize = %u\n", dmSaParam->dmSaParam.macSize);
	odp_pr_dbg("dmSaParam.aadSize = %u\n", dmSaParam->dmSaParam.aadSize);
	odp_pr_dbg("dmSaParam.replayWindow = %u\n",
		   dmSaParam->dmSaParam.replayWindow);

	if (dmSaParam->dmSaParam.cipherMode != NWAL_SA_EALG_NULL)
		odp_pr_dbg_mem(dmSaParam->keyParam.pEncKey,
			       dmSaParam->keyParam.encKeySize,
			       "keyParam.pEncKey");
	if (dmSaParam->dmSaParam.authMode != NWAL_SA_AALG_NULL)
		odp_pr_dbg_mem(dmSaParam->keyParam.pAuthKey,
			       dmSaParam->keyParam.macKeySize,
			       "keyParam.pAuthKey");
}

static inline int decode_auth_mode(odp_auth_alg_t auth_mode, uint32_t *tag_len)
{
	switch (auth_mode) {
	case ODP_AUTH_ALG_NULL:
		*tag_len = 0;
		return NWAL_SA_AALG_NULL;
	case ODP_AUTH_ALG_MD5_96:
		*tag_len = 96 / 8;
		return NWAL_SA_AALG_HMAC_MD5;
	case ODP_AUTH_ALG_SHA256_128:
		*tag_len = 128 / 8;
		return NWAL_SA_AALG_HMAC_SHA2_256_RFC4868;
	default:
		break;
	}

	return NWAL_SA_AALG_HMAC_SHA2_256_RFC4868 + 10;
}

static inline int decode_cipher_mode(odp_cipher_alg_t cipher_mode)
{
	switch (cipher_mode) {
	case ODP_CIPHER_ALG_NULL:
		return NWAL_SA_EALG_NULL;
	case ODP_CIPHER_ALG_DES:
		return NWAL_SA_EALG_DES_CBC;
	case ODP_CIPHER_ALG_3DES_CBC:
		return NWAL_SA_EALG_3DES_CBC;
	case ODP_CIPHER_ALG_AES128_CBC:
		return NWAL_SA_EALG_AES_CBC;
	case ODP_CIPHER_ALG_AES_CTR:
		return NWAL_SA_EALG_AES_CTR;
	case ODP_CIPHER_ALG_AES_CCM:
		return NWAL_SA_EALG_AES_CCM;
	case ODP_CIPHER_ALG_AES_GCM:
		return NWAL_SA_EALG_AES_GCM;
	default:
		break;
	}

	return NWAL_SA_EALG_AES_GCM + 5;
}

int odp_crypto_session_create(odp_crypto_session_params_t *params,
			      odp_crypto_session_t *session_out,
			      odp_crypto_ses_create_err_t *status)
{
	nwal_AppId app_id;
	nwal_RetValue nwal_ret;
	nwalCreateDmSAParams_t sa_params;
	struct odp_crypto_session_s *session;

	ODP_AS_STR((params->cipher_alg != ODP_CIPHER_ALG_NULL ||
		    params->auth_alg != ODP_AUTH_ALG_NULL),
		   "Both algorithms are NULL");

	if (params->cipher_alg == ODP_CIPHER_ALG_NULL) {
		params->cipher_key.data   = NULL;
		params->cipher_key.length = 0;
	}

	if (params->auth_alg == ODP_AUTH_ALG_NULL &&
	    params->cipher_alg != ODP_CIPHER_ALG_AES_GCM &&
	    params->cipher_alg != ODP_CIPHER_ALG_AES_CCM) {
		params->auth_key.data   = NULL;
		params->auth_key.length = 0;
	}

	/* Default to failure result */
	*status  = ODP_CRYPTO_SES_CREATE_ERR_NONE;
	*session_out = ODP_CRYPTO_SESSION_INVALID;

	/* Allocate memory for this session */
	session = alloc_session();
	if (!session) {
		*status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM;
		return -1;
	}

	/* Copy stuff over */
	session->op = params->op;
	session->cipher.alg = params->cipher_alg;
	session->auth.alg = params->auth_alg;
	session->pref_mode = params->pref_mode;
	if (sizeof(session->cipher.iv.data) < params->iv.length) {
		*status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER;
		return -1;
	}
	if (params->iv.data) {
		memcpy(session->cipher.iv.data, params->iv.data,
		       params->iv.length);
		/** @todo: need separate IV for Auth */
		memcpy(session->auth.iv.data, params->iv.data,
		       params->iv.length);
	}

	session->compl_queue = params->compl_queue;
	session->out_pool = params->output_pool;
	session->cppi_flow = _odp_pool_cppi_flow_create(session->out_pool,
							      0);
	if (!session->cppi_flow) {
		ODP_ERR("couldn't create pool's flow\n");
		return -1;
	}

	memset(&sa_params, 0, sizeof(nwalCreateDmSAParams_t));
	sa_params.dmSaParam.dmChnType =
			(params->op == ODP_CRYPTO_OP_DECODE) ?
					NWAL_DM_CHAN_DECRYPT :
					NWAL_DM_CHAN_ENCRYPT;
	sa_params.dmSaParam.replayWindow = 64; /** @todo: always 64? */
	sa_params.dmSaParam.authMode = decode_auth_mode(params->auth_alg,
							&session->auth.tag_len);
	sa_params.dmSaParam.cipherMode = decode_cipher_mode(params->cipher_alg);

	sa_params.dmSaParam.enc1st = (params->op == ODP_CRYPTO_OP_ENCODE) ?
			params->auth_cipher_text : !params->auth_cipher_text;

	if ((sa_params.dmSaParam.cipherMode == NWAL_SA_EALG_AES_GCM) ||
	    (sa_params.dmSaParam.cipherMode	== NWAL_SA_EALG_AES_CCM) ||
	    (sa_params.dmSaParam.authMode == NWAL_SA_AALG_GMAC)) {
		sa_params.dmSaParam.macSize = 16;
		sa_params.dmSaParam.aadSize = 8;
		/* Enc1st needs to always be true for combined algorithms */
		sa_params.dmSaParam.enc1st = nwal_TRUE;
	} else if (sa_params.dmSaParam.authMode != NWAL_SA_AALG_NULL) {
		sa_params.dmSaParam.macSize = 12;
		sa_params.dmSaParam.aadSize = 0;
	} else {
		sa_params.dmSaParam.enc1st = nwal_TRUE;
		sa_params.dmSaParam.macSize = 0;
	}

	sa_params.keyParam.pEncKey = params->cipher_key.data;
	sa_params.keyParam.encKeySize = params->cipher_key.length;
	sa_params.keyParam.pAuthKey = params->auth_key.data;
	sa_params.keyParam.macKeySize = params->auth_key.length;

	ODP_AS_STR(session->auth.tag_len <=
		   ODP_FIELD_SIZEOF(struct odp_pkthdr, crypto.dec.hash_tag),
		   "Auth tag length is bigger than hash_tag array");

	_print_nwalCreateDmSAParams(&sa_params);
	odp_pr_dbg("Session addr: %p\n", session);

	if (ODP_QUEUE_INVALID != session->compl_queue)
		app_id = (nwal_AppId)_odp_queue_sched_tag(session->compl_queue);
	else
		app_id = 0;

	nwal_ret = nwal_setDMSecAssoc(odp_global->nwal.handle,
			app_id,
			&sa_params,
			&session->dm_handle);
	if (nwal_ret != nwal_OK) {
		odp_pr_err("nwal_setDMSecAssoc() returned Error Code %d\n",
			   nwal_ret);
		*status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM;
		return -1;
	}

	nwal_ret = nwal_initDMPSCmdInfo(odp_global->nwal.handle,
			session->dm_handle,
			&session->dm_ps_cmdinfo);

	if (ODP_QUEUE_INVALID != session->compl_queue)
		session->dm_ps_cmdinfo.rxSbSaQ =
				_odp_queue_to_qmss_queue(session->compl_queue);
	session->dm_ps_cmdinfo.rxPktFlowId = cppi_flow_id(session->cppi_flow);

	*status = ODP_CRYPTO_SES_CREATE_ERR_NONE;
	*session_out = session;

	return 0;
}

int odp_crypto_session_destroy(odp_crypto_session_t ses)
{
	struct odp_crypto_session_s *entry = ses;

	if (shres_free(session_tbl, entry) < 0)
		return -1;

	return 0;
}

#define ODP_CRYPTO_BUFFER_PROCESSED_OFFSET (-1)

static inline void hash_copy_be32(uint8_t *dest, const uint32be_t *sa, size_t n)
{
	union hash_u {
		uint32_t hash32;
		uint8_t hash[4];
	} hash;

	n /= 4;
	while (n--) {
		unsigned int i;
		hash.hash32 = odp_be_to_cpu_32(*sa++);
		for (i = 0; i < sizeof(hash.hash); i++)
			*dest++ = hash.hash[i];
	};
}

static inline int hash_compare_be32(const uint32_t *orig, const uint32be_t *sa,
				    size_t n)
{
	n /= 4;
	while (n--) {
		if (*orig++ != odp_be_to_cpu_32(*sa++))
			return 1;
	};
	return 0;
}

/**
 *  Set bufPtr to origBuffPtr to pass buffer header via SA
 *
 *  @return offset value
 */
static inline int16_t odp_crypto_packet_preprocess(odp_packet_t pkt)
{
	struct odp_pkthdr *hdr;
	int16_t offset;
	Cppi_HostDesc *desc;

	desc = _odp_pkt_to_cppi_desc(pkt);
	hdr = _odp_pkt_hdr(pkt);
	offset = _cppi_desc_buf_vptr(desc) - _cppi_desc_orig_vptr(desc);
	hdr->crypto.saved_buf_offset = offset;
	odp_pr_dbg("buffPtr: 0x%08x, buffLen: 0x%x, offset: 0x%x\n",
		   desc->buffPtr, desc->buffLen, offset);

	_cppi_desc_shift_vptr(desc, -offset);
	odp_pr_vdbg_packet(pkt);
	return offset;
}

extern int packet_data_copy(Cppi_HostDesc *desc_in,  uint32_t offset_in,
		     Cppi_HostDesc *desc_out, uint32_t offset_out,
		     uint32_t len);

int odp_crypto_operation(odp_crypto_op_params_t *params,
			 odp_bool_t *posted,
			 odp_crypto_op_result_t *result)
{
	nwalTxDmPSCmdInfo_t *dm_cmd_info;
	Cppi_HostDesc       *desc;
	struct odp_crypto_session_s *session;
	odp_packet_t pkt = params->pkt;
	struct odp_pkthdr *hdr = _odp_pkt_hdr(pkt);
	uint32_t offset;
	uint8_t *data;

	session = (struct odp_crypto_session_s *)(intptr_t)params->session;

	dm_cmd_info = &session->dm_ps_cmdinfo;

	/* Save hash tag for decode operation and fill hash result with 0's*/
	data  = odp_packet_data(pkt);
	data += params->hash_result_offset;
	hdr->crypto.hash_offset = params->hash_result_offset;
	hdr->crypto.op_context = params->ctx;
	hdr->crypto.session = session;
	if (session->op == ODP_CRYPTO_OP_DECODE) {
		memcpy(hdr->crypto.dec.hash_tag, data, session->auth.tag_len);
		memset(data, 0, session->auth.tag_len);
	}

	offset = odp_crypto_packet_preprocess(pkt);
	desc = _odp_pkt_to_cppi_desc(pkt);

	nwal_mCmdDMUpdate(Pktlib_getPacketFromDesc(desc),
			  dm_cmd_info,
			  nwal_HANDLE_INVALID,
			  params->cipher_range.offset + offset,
			  params->cipher_range.length,
			  (params->override_iv_ptr) ?
					params->override_iv_ptr :
					session->cipher.iv.data,
			  params->auth_range.offset + offset,
			  params->auth_range.length,
			  NULL,
			  0, /** @todo: Should be aadSize from session? */
			  NULL);

	desc = Osal_qmssConvertDescVirtToPhy(0, desc);
	Qmss_queuePushDescSizeRaw(dm_cmd_info->txQueue,
				  desc,
				  NWAL_DESC_SIZE);

	odp_event_t event;
	Cppi_HostDesc *rx_desc;
	if (params->out_pkt != ODP_PACKET_INVALID ||
	    session->pref_mode == ODP_CRYPTO_SYNC) {
		/* Poll completion queue for results */
		do {
			rx_desc = (void *)QMSS_DESC_PTR(Qmss_queuePop(session->dm_ps_cmdinfo.rxSbSaQ));
			event = _cppi_desc_to_odp_ev(rx_desc);
		} while (event == ODP_EVENT_INVALID);
	} else {
		*posted = 1;
		return 0;
	}

	/* if out packet is specified copy event then*/
	if (params->out_pkt != ODP_PACKET_INVALID) {
		Cppi_HostDesc *out_desc = _odp_pkt_to_cppi_desc(params->out_pkt);
		uint32_t pkt_len = _cppi_desc_pkt_len(out_desc);

		/* copy packet data */
		out_desc->buffPtr = (uint32_t)_odp_mem_phys_to_virt((odp_pa_t )out_desc->buffPtr);
		packet_data_copy(out_desc, 0, rx_desc, 0, pkt_len);

		/* copy packet header */
		out_desc->origBuffPtr = (uint32_t)_odp_mem_phys_to_virt((odp_pa_t )out_desc->origBuffPtr);
		packet_header_copy(_odp_pkt_hdr(params->out_pkt),
				   _odp_pkt_hdr(_odp_ev_to_pkt(event)));
		/* copy PS data */
		Cppi_copyPSData((Cppi_Desc *)out_desc, (Cppi_Desc *)rx_desc);

		odp_packet_free(_odp_ev_to_pkt(event));
		event = odp_packet_to_event(params->out_pkt);
	}

	if (session->pref_mode == ODP_CRYPTO_SYNC ||
	    params->out_pkt != ODP_PACKET_INVALID) {
		odp_crypto_compl_t compl_event;
		compl_event = odp_crypto_compl_from_event(event);
		odp_crypto_compl_result(compl_event, result);
		odp_crypto_compl_free(compl_event);
		*posted = 0;
	}

	if (params->out_pkt != ODP_PACKET_INVALID) {
		odp_queue_enq(session->compl_queue, event);
		*posted = 1;
	}

	return 0;
}

int odp_crypto_init_global(void)
{
	ODP_DBG("\nCrypto init global\n");
	session_tbl = shres_table_create(odp_session_table_name,
					  struct odp_crypto_session_s,
					  MAX_SESSIONS, session_free_cb);
	if (session_tbl == SHRES_TABLE_INVALID)
		return -1;

	return 0;
}

static int session_free_cb(void *data)
{
	struct odp_crypto_session_s *entry =
		(struct odp_crypto_session_s *)data;

	nwal_RetValue nwal_ret;
	cppi_flow_destroy(entry->cppi_flow);
	nwal_ret = nwal_delDMSecAssoc(odp_global->nwal.handle,
				      entry->dm_handle);
	return (nwal_ret == nwal_OK) ? 0 : -1;
}

int odp_crypto_term_global(void)
{
	if (shres_table_destroy(odp_session_table_name) < 0)
		return -1;

	return 0;
}

ssize_t odp_random_data(uint8_t *buf, int32_t size,
			odp_bool_t use_entropy ODP_UNUSED)
{
	nwal_RetValue ret;
	Sa_RngData_t random;
	int32_t length, rest, add, chank;

	length = 0;
	chank = sizeof(Sa_RngData_t);
	while (length < size) {
		ret = nwal_getSARandomNum(odp_global->nwal.handle, &random);
		if (ret != nwal_OK)
			break;

		rest = size - length;
		add = rest > chank ? chank : rest;

		memcpy(buf, (uint8_t *)&random, add);
		length += add;
	}

	return length;
}

void odp_crypto_compl_result(odp_crypto_compl_t compl,
			     odp_crypto_op_result_t *result)
{
	result->ctx = NULL;
	Cppi_HostDesc *desc;
	int16_t offset;
	uint8_t *auth_tag = NULL;
	uint32_t auth_tag_len = 0;
	struct odp_pkthdr *hdr;
	struct odp_crypto_session_s *session;
	Ti_Pkt *ti_pkt;
	odp_packet_t pkt = _odp_ev_to_pkt(odp_crypto_compl_to_event(compl));

	odp_pr_vdbg_packet(pkt);
	hdr  = _odp_pkt_hdr(pkt);
	offset = hdr->crypto.saved_buf_offset;
	if (odp_unlikely(offset == ODP_CRYPTO_BUFFER_PROCESSED_OFFSET)) {
		/* Buffer already post-processed */
		return;
	}
	ODP_AS_STR(offset >= 0, "Wrong saved buffer offset\n");

	hdr->crypto.saved_buf_offset = ODP_CRYPTO_BUFFER_PROCESSED_OFFSET;
	desc = _odp_pkt_to_cppi_desc(pkt);
	ti_pkt = Pktlib_getPacketFromDesc(desc);

	odp_pr_dbg("buffPtr: 0x%08x, buffLen: 0x%x, offset: 0x%x\n",
		   desc->buffPtr, desc->buffLen, offset);
	_cppi_desc_shift_vptr(desc, offset);

	session = hdr->crypto.session;
	odp_pr_dbg("Session addr: %p\n", session);

	nwal_mmGetDmAuthTag(ti_pkt, &auth_tag, &auth_tag_len);

	ODP_AS_STR(!((uintptr_t)auth_tag & 0x3),
		   "Auth tag is not 4 bytes aligned");

	if (session->op == ODP_CRYPTO_OP_ENCODE) {
		/* Copy hash to packet */
		uint8_t *data = odp_packet_data(pkt);
		data += hdr->crypto.hash_offset;
		hash_copy_be32(data, (uint32be_t *)(void *)auth_tag,
			       session->auth.tag_len);
	} else if (hash_compare_be32(hdr->crypto.dec.hash_tag,
				     (uint32be_t *)(void *)auth_tag,
				     session->auth.tag_len)) {
		odp_pr_dbg("ICV is wrong\n");
		odp_pr_dbg_mem(hdr->crypto.dec.hash_tag, session->auth.tag_len,
			       "Saved auth tag");
		odp_pr_dbg_mem(auth_tag, session->auth.tag_len,
			       "Decoded auth tag");
		result->auth_status.alg_err = ODP_CRYPTO_ALG_ERR_ICV_CHECK;
		result->ok = 0;
		return;
	}

	result->ok = 1;
	result->ctx = hdr->crypto.op_context;
	result->pkt = pkt;

	return;
}