#include <stdio.h>

#include <inttypes.h>

#include <math.h>

#include <rte_eal.h>

#include <rte_common.h>

#include <rte_dev.h>

#include <rte_launch.h>

#include <rte_bbdev.h>

#include <rte_cycles.h>

#include <rte_lcore.h>

#include <rte_malloc.h>

#include <rte_random.h>

#include <rte_hexdump.h>

#include <rte_interrupts.h>

#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC

#include <fpga_lte_fec.h>

#include "main.h"

#include "test_bbdev_vector.h"

#define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))

#define MAX_QUEUES RTE_MAX_LCORE

#define TEST_REPETITIONS 1000

#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC

#define FPGA_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")

#define FPGA_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")

#define VF_UL_QUEUE_VALUE 4

#define VF_DL_QUEUE_VALUE 4

#define UL_BANDWIDTH 3

#define DL_BANDWIDTH 3

#define UL_LOAD_BALANCE 128

#define DL_LOAD_BALANCE 128

#define FLR_TIMEOUT 610

#define OPS_CACHE_SIZE 256U

#define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */

#define SYNC_WAIT 0

#define SYNC_START 1

#define INVALID_QUEUE_ID -1

static struct test_bbdev_vector test_vector;

static bool intr_enabled;

static struct active_device {

} active_devs[RTE_BBDEV_MAX_DEVS];

static uint8_t nb_active_devs;

struct test_buffers {

};

struct test_op_params {

};

struct thread_params {

};

#ifdef RTE_BBDEV_OFFLOAD_COST

struct test_time_stats {

};

typedef int (test_case_function)(struct active_device *ad,

struct test_op_params *op_params);

static inline void

mbuf_reset(struct rte_mbuf *m)

{

m->pkt_len = 0;

do {

m->data_len = 0;

m = m->next;

} while (m != NULL);

}

static inline bool

check_bit(uint32_t bitmap, uint32_t bitmask)

{

return bitmap & bitmask;

}

static inline void

set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)

{

ad->supported_ops |= (1 << op_type);

}

static inline bool

is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)

{

return ad->supported_ops & (1 << op_type);

}

static inline bool

flags_match(uint32_t flags_req, uint32_t flags_present)

{

return (flags_req & flags_present) == flags_req;

}

static void

clear_soft_out_cap(uint32_t *op_flags)

{

*op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;

*op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;

*op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;

}

static int

check_dev_cap(const struct rte_bbdev_info *dev_info)

{

unsigned int i;

unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs,

nb_harq_inputs, nb_harq_outputs;

const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;

nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;

nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;

nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;

nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments;

nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments;

for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {

if (op_cap->type != test_vector.op_type)

continue;

if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {

const struct rte_bbdev_op_cap_turbo_dec *cap =

&op_cap->cap.turbo_dec;

/* Ignore lack of soft output capability, just skip

if ((test_vector.turbo_dec.op_flags &

RTE_BBDEV_TURBO_SOFT_OUTPUT) &&

!(cap->capability_flags &

RTE_BBDEV_TURBO_SOFT_OUTPUT)) {

printf(

"INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",

dev_info->dev_name);

clear_soft_out_cap(

&test_vector.turbo_dec.op_flags);

}

if (!flags_match(test_vector.turbo_dec.op_flags,

cap->capability_flags))

return TEST_FAILED;

if (nb_inputs > cap->num_buffers_src) {

printf("Too many inputs defined: %u, max: %u\n",

nb_inputs, cap->num_buffers_src);

return TEST_FAILED;

}

if (nb_soft_outputs > cap->num_buffers_soft_out &&

(test_vector.turbo_dec.op_flags &

RTE_BBDEV_TURBO_SOFT_OUTPUT)) {

printf(

"Too many soft outputs defined: %u, max: %u\n",

nb_soft_outputs,

cap->num_buffers_soft_out);

return TEST_FAILED;

}

if (nb_hard_outputs > cap->num_buffers_hard_out) {

printf(

"Too many hard outputs defined: %u, max: %u\n",

nb_hard_outputs,

cap->num_buffers_hard_out);

return TEST_FAILED;

}

if (intr_enabled && !(cap->capability_flags &

RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {

printf(

"Dequeue interrupts are not supported!\n");

return TEST_FAILED;

}

return TEST_SUCCESS;

} else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {

const struct rte_bbdev_op_cap_turbo_enc *cap =

&op_cap->cap.turbo_enc;

if (!flags_match(test_vector.turbo_enc.op_flags,

cap->capability_flags))

return TEST_FAILED;

if (nb_inputs > cap->num_buffers_src) {

printf("Too many inputs defined: %u, max: %u\n",

nb_inputs, cap->num_buffers_src);

return TEST_FAILED;

}

if (nb_hard_outputs > cap->num_buffers_dst) {

printf(

"Too many hard outputs defined: %u, max: %u\n",

nb_hard_outputs, cap->num_buffers_dst);

return TEST_FAILED;

}

if (intr_enabled && !(cap->capability_flags &

RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {

printf(

"Dequeue interrupts are not supported!\n");

return TEST_FAILED;

}

return TEST_SUCCESS;

} else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) {

const struct rte_bbdev_op_cap_ldpc_enc *cap =

&op_cap->cap.ldpc_enc;

if (!flags_match(test_vector.ldpc_enc.op_flags,

cap->capability_flags)){

printf("Flag Mismatch\n");

return TEST_FAILED;

}

if (nb_inputs > cap->num_buffers_src) {

printf("Too many inputs defined: %u, max: %u\n",

nb_inputs, cap->num_buffers_src);

return TEST_FAILED;

}

if (nb_hard_outputs > cap->num_buffers_dst) {

printf(

"Too many hard outputs defined: %u, max: %u\n",

nb_hard_outputs, cap->num_buffers_dst);

return TEST_FAILED;

}

if (intr_enabled && !(cap->capability_flags &

RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {

printf(

"Dequeue interrupts are not supported!\n");

return TEST_FAILED;

}

return TEST_SUCCESS;

} else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) {

const struct rte_bbdev_op_cap_ldpc_dec *cap =

&op_cap->cap.ldpc_dec;

if (!flags_match(test_vector.ldpc_dec.op_flags,

cap->capability_flags)){

printf("Flag Mismatch\n");

return TEST_FAILED;

}

if (nb_inputs > cap->num_buffers_src) {

printf("Too many inputs defined: %u, max: %u\n",

nb_inputs, cap->num_buffers_src);

return TEST_FAILED;

}

if (nb_hard_outputs > cap->num_buffers_hard_out) {

printf(

"Too many hard outputs defined: %u, max: %u\n",

nb_hard_outputs,

cap->num_buffers_hard_out);

return TEST_FAILED;

}

if (nb_harq_inputs > cap->num_buffers_hard_out) {

printf(

"Too many HARQ inputs defined: %u, max: %u\n",

nb_hard_outputs,

cap->num_buffers_hard_out);

return TEST_FAILED;

}

if (nb_harq_outputs > cap->num_buffers_hard_out) {

printf(

"Too many HARQ outputs defined: %u, max: %u\n",

nb_hard_outputs,

cap->num_buffers_hard_out);

return TEST_FAILED;

}

if (intr_enabled && !(cap->capability_flags &

RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {

printf(

"Dequeue interrupts are not supported!\n");

return TEST_FAILED;

}

return TEST_SUCCESS;

}

}

if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))

return TEST_SUCCESS; /* Special case for NULL device */

return TEST_FAILED;

}

static unsigned int

optimal_mempool_size(unsigned int val)

{

return rte_align32pow2(val + 1) - 1;

}

static struct rte_mempool *

create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,

int socket_id, unsigned int mbuf_pool_size,

const char *op_type_str)

{

unsigned int i;

uint32_t max_seg_sz = 0;

char pool_name[RTE_MEMPOOL_NAMESIZE];

/* find max input segment size */

for (i = 0; i < entries->nb_segments; ++i)

if (entries->segments[i].length > max_seg_sz)

max_seg_sz = entries->segments[i].length;

snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,

dev_id);

return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,

RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,

(unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);

}

static int

create_mempools(struct active_device *ad, int socket_id,

enum rte_bbdev_op_type org_op_type, uint16_t num_ops)

{

struct rte_mempool *mp;

unsigned int ops_pool_size, mbuf_pool_size = 0;

char pool_name[RTE_MEMPOOL_NAMESIZE];

const char *op_type_str;

enum rte_bbdev_op_type op_type = org_op_type;

struct op_data_entries *in = &test_vector.entries[DATA_INPUT];

struct op_data_entries *hard_out =

&test_vector.entries[DATA_HARD_OUTPUT];

struct op_data_entries *soft_out =

&test_vector.entries[DATA_SOFT_OUTPUT];

struct op_data_entries *harq_in =

&test_vector.entries[DATA_HARQ_INPUT];

struct op_data_entries *harq_out =

&test_vector.entries[DATA_HARQ_OUTPUT];

/* allocate ops mempool */

ops_pool_size = optimal_mempool_size(RTE_MAX(

/* Ops used plus 1 reference op */

RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),

/* Minimal cache size plus 1 reference op */

(unsigned int)(1.5 * rte_lcore_count() *

OPS_CACHE_SIZE + 1)),

OPS_POOL_SIZE_MIN));

if (org_op_type == RTE_BBDEV_OP_NONE)

op_type = RTE_BBDEV_OP_TURBO_ENC;

op_type_str = rte_bbdev_op_type_str(op_type);

TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);

snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,

ad->dev_id);

mp = rte_bbdev_op_pool_create(pool_name, op_type,

ops_pool_size, OPS_CACHE_SIZE, socket_id);

TEST_ASSERT_NOT_NULL(mp,

"ERROR Failed to create %u items ops pool for dev %u on socket %u.",

ops_pool_size,

ad->dev_id,

socket_id);

ad->ops_mempool = mp;

/* Do not create inputs and outputs mbufs for BaseBand Null Device */

if (org_op_type == RTE_BBDEV_OP_NONE)

return TEST_SUCCESS;

/* Inputs */

mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);

mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");

TEST_ASSERT_NOT_NULL(mp,

"ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",

mbuf_pool_size,

ad->dev_id,

socket_id);

ad->in_mbuf_pool = mp;

/* Hard outputs */

mbuf_pool_size = optimal_mempool_size(ops_pool_size *

hard_out->nb_segments);

mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,

"hard_out");

TEST_ASSERT_NOT_NULL(mp,

"ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",

mbuf_pool_size,

ad->dev_id,

socket_id);

ad->hard_out_mbuf_pool = mp;

/* Soft outputs */

if (soft_out->nb_segments > 0) {

mbuf_pool_size = optimal_mempool_size(ops_pool_size *

soft_out->nb_segments);

mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id,

mbuf_pool_size,

"soft_out");

TEST_ASSERT_NOT_NULL(mp,

"ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",

mbuf_pool_size,

ad->dev_id,

socket_id);

ad->soft_out_mbuf_pool = mp;

}

/* HARQ inputs */

if (harq_in->nb_segments > 0) {

mbuf_pool_size = optimal_mempool_size(ops_pool_size *

harq_in->nb_segments);

mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id,

mbuf_pool_size,

"harq_in");

TEST_ASSERT_NOT_NULL(mp,

"ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.",

mbuf_pool_size,

ad->dev_id,

socket_id);

ad->harq_in_mbuf_pool = mp;

}

/* HARQ outputs */

if (harq_out->nb_segments > 0) {

mbuf_pool_size = optimal_mempool_size(ops_pool_size *

harq_out->nb_segments);

mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id,

mbuf_pool_size,

"harq_out");

TEST_ASSERT_NOT_NULL(mp,

"ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.",

mbuf_pool_size,

ad->dev_id,

socket_id);

ad->harq_out_mbuf_pool = mp;

}

return TEST_SUCCESS;

}

static int

add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,

struct test_bbdev_vector *vector)

{

int ret;

unsigned int queue_id;

struct rte_bbdev_queue_conf qconf;

struct active_device *ad = &active_devs[nb_active_devs];

unsigned int nb_queues;

enum rte_bbdev_op_type op_type = vector->op_type;

#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC

if ((get_init_device() == true) &&

(!strcmp(info->drv.driver_name, FPGA_PF_DRIVER_NAME))) {

struct fpga_lte_fec_conf conf;

unsigned int i;

printf("Configure FPGA FEC Driver %s with default values\n",

info->drv.driver_name);

/* clear default configuration before initialization */

memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));

/* Set PF mode :

conf.pf_mode_en = true;

for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {

/* Number of UL queues per VF (fpga supports 8 VFs) */

conf.vf_ul_queues_number[i] = VF_UL_QUEUE_VALUE;

/* Number of DL queues per VF (fpga supports 8 VFs) */

conf.vf_dl_queues_number[i] = VF_DL_QUEUE_VALUE;

}

/* UL bandwidth. Needed for schedule algorithm */

conf.ul_bandwidth = UL_BANDWIDTH;

/* DL bandwidth */

conf.dl_bandwidth = DL_BANDWIDTH;

/* UL & DL load Balance Factor to 64 */

conf.ul_load_balance = UL_LOAD_BALANCE;

conf.dl_load_balance = DL_LOAD_BALANCE;

/**< FLR timeout value */

conf.flr_time_out = FLR_TIMEOUT;

/* setup FPGA PF with configuration information */

ret = fpga_lte_fec_configure(info->dev_name, &conf);

TEST_ASSERT_SUCCESS(ret,

"Failed to configure 4G FPGA PF for bbdev %s",

info->dev_name);

}

nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);

nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);

/* setup device */

ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);

if (ret < 0) {

printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",

dev_id, nb_queues, info->socket_id, ret);

return TEST_FAILED;

}

/* configure interrupts if needed */

if (intr_enabled) {

ret = rte_bbdev_intr_enable(dev_id);

if (ret < 0) {

printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,

ret);

return TEST_FAILED;

}

}

/* setup device queues */

qconf.socket = info->socket_id;

qconf.queue_size = info->drv.default_queue_conf.queue_size;

qconf.priority = 0;

qconf.deferred_start = 0;

qconf.op_type = op_type;

for (queue_id = 0; queue_id < nb_queues; ++queue_id) {

ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);

if (ret != 0) {

printf(

"Allocated all queues (id=%u) at prio%u on dev%u\n",

queue_id, qconf.priority, dev_id);

qconf.priority++;

ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,

&qconf);

}

if (ret != 0) {

printf("All queues on dev %u allocated: %u\n",

dev_id, queue_id);

break;

}

ad->queue_ids[queue_id] = queue_id;

}

TEST_ASSERT(queue_id != 0,

"ERROR Failed to configure any queues on dev %u",

dev_id);

ad->nb_queues = queue_id;

set_avail_op(ad, op_type);

return TEST_SUCCESS;

}

static int

add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,

struct test_bbdev_vector *vector)

{

int ret;

active_devs[nb_active_devs].driver_name = info->drv.driver_name;

active_devs[nb_active_devs].dev_id = dev_id;

ret = add_bbdev_dev(dev_id, info, vector);

if (ret == TEST_SUCCESS)

++nb_active_devs;

return ret;

}

static uint8_t

populate_active_devices(void)

{

int ret;

uint8_t dev_id;

uint8_t nb_devs_added = 0;

struct rte_bbdev_info info;

RTE_BBDEV_FOREACH(dev_id) {

rte_bbdev_info_get(dev_id, &info);

if (check_dev_cap(&info)) {

printf(

"Device %d (%s) does not support specified capabilities\n",

dev_id, info.dev_name);

continue;

}

ret = add_active_device(dev_id, &info, &test_vector);

if (ret != 0) {

printf("Adding active bbdev %s skipped\n",

info.dev_name);

continue;

}

nb_devs_added++;

}

return nb_devs_added;

}

static int

read_test_vector(void)

{

int ret;

memset(&test_vector, 0, sizeof(test_vector));

printf("Test vector file = %s\n", get_vector_filename());

ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);

TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",

get_vector_filename());

return TEST_SUCCESS;

}

static int

testsuite_setup(void)

{

TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");

if (populate_active_devices() == 0) {

printf("No suitable devices found!\n");

return TEST_SKIPPED;

}

return TEST_SUCCESS;

}

static int

interrupt_testsuite_setup(void)

{

TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");

/* Enable interrupts */

intr_enabled = true;

/* Special case for NULL device (RTE_BBDEV_OP_NONE) */

if (populate_active_devices() == 0 ||

test_vector.op_type == RTE_BBDEV_OP_NONE) {

intr_enabled = false;

printf("No suitable devices found!\n");

return TEST_SKIPPED;

}

return TEST_SUCCESS;

}

static void

testsuite_teardown(void)

{

uint8_t dev_id;

/* Unconfigure devices */

RTE_BBDEV_FOREACH(dev_id)

rte_bbdev_close(dev_id);

/* Clear active devices structs. */

memset(active_devs, 0, sizeof(active_devs));

nb_active_devs = 0;

}

static int

ut_setup(void)

{

uint8_t i, dev_id;

for (i = 0; i < nb_active_devs; i++) {

dev_id = active_devs[i].dev_id;

/* reset bbdev stats */

TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),

"Failed to reset stats of bbdev %u", dev_id);

/* start the device */

TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),

"Failed to start bbdev %u", dev_id);

}

return TEST_SUCCESS;

}

static void

ut_teardown(void)

{

uint8_t i, dev_id;

struct rte_bbdev_stats stats;

for (i = 0; i < nb_active_devs; i++) {

dev_id = active_devs[i].dev_id;

/* read stats and print */

rte_bbdev_stats_get(dev_id, &stats);

/* Stop the device */

rte_bbdev_stop(dev_id);

}

}

static int

init_op_data_objs(struct rte_bbdev_op_data *bufs,

struct op_data_entries *ref_entries,

struct rte_mempool *mbuf_pool, const uint16_t n,

enum op_data_type op_type, uint16_t min_alignment)

{

int ret;

unsigned int i, j;

for (i = 0; i < n; ++i) {

char *data;

struct op_data_buf *seg = &ref_entries->segments[0];

struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);

TEST_ASSERT_NOT_NULL(m_head,

"Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",

op_type, n * ref_entries->nb_segments,

mbuf_pool->size);

TEST_ASSERT_SUCCESS(((seg->length + RTE_PKTMBUF_HEADROOM) >

(uint32_t)UINT16_MAX),

"Given data is bigger than allowed mbuf segment size");

bufs[i].data = m_head;

bufs[i].offset = 0;

bufs[i].length = 0;

if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) {

data = rte_pktmbuf_append(m_head, seg->length);

TEST_ASSERT_NOT_NULL(data,

"Couldn't append %u bytes to mbuf from %d data type mbuf pool",

seg->length, op_type);

TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),

"Data addr in mbuf (%p) is not aligned to device min alignment (%u)",

data, min_alignment);

rte_memcpy(data, seg->addr, seg->length);

bufs[i].length += seg->length;

for (j = 1; j < ref_entries->nb_segments; ++j) {

struct rte_mbuf *m_tail =

rte_pktmbuf_alloc(mbuf_pool);

TEST_ASSERT_NOT_NULL(m_tail,

"Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",

op_type,

n * ref_entries->nb_segments,

mbuf_pool->size);

seg += 1;

data = rte_pktmbuf_append(m_tail, seg->length);

TEST_ASSERT_NOT_NULL(data,

"Couldn't append %u bytes to mbuf from %d data type mbuf pool",

seg->length, op_type);

TEST_ASSERT(data == RTE_PTR_ALIGN(data,

min_alignment),

"Data addr in mbuf (%p) is not aligned to device min alignment (%u)",

data, min_alignment);

rte_memcpy(data, seg->addr, seg->length);

bufs[i].length += seg->length;

ret = rte_pktmbuf_chain(m_head, m_tail);

TEST_ASSERT_SUCCESS(ret,

"Couldn't chain mbufs from %d data type mbuf pool",

op_type);

}

} else {

/* allocate chained-mbuf for output buffer */

for (j = 1; j < ref_entries->nb_segments; ++j) {

struct rte_mbuf *m_tail =

rte_pktmbuf_alloc(mbuf_pool);

TEST_ASSERT_NOT_NULL(m_tail,

"Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",

op_type,

n * ref_entries->nb_segments,

mbuf_pool->size);

ret = rte_pktmbuf_chain(m_head, m_tail);

TEST_ASSERT_SUCCESS(ret,

"Couldn't chain mbufs from %d data type mbuf pool",

op_type);

}

}

}

return 0;

}

static int

allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,

const int socket)

{

int i;

*buffers = rte_zmalloc_socket(NULL, len, 0, socket);

if (*buffers == NULL) {

printf("WARNING: Failed to allocate op_data on socket %d\n",

socket);

/* try to allocate memory on other detected sockets */

for (i = 0; i < socket; i++) {

*buffers = rte_zmalloc_socket(NULL, len, 0, i);

if (*buffers != NULL)

break;

}

}

return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;

}

static void

limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops,

const uint16_t n, const int8_t max_llr_modulus)

{

uint16_t i, byte_idx;

for (i = 0; i < n; ++i) {

struct rte_mbuf *m = input_ops[i].data;

while (m != NULL) {

int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,

input_ops[i].offset);

for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);

++byte_idx)

llr[byte_idx] = round((double)max_llr_modulus *

llr[byte_idx] / INT8_MAX);

m = m->next;

}

}

}

static void

ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops,

const uint16_t n, const int8_t llr_size,

const int8_t llr_decimals)

{

if (input_ops == NULL)

return;

uint16_t i, byte_idx;

int16_t llr_max, llr_min, llr_tmp;

llr_max = (1 << (llr_size - 1)) - 1;

llr_min = -llr_max;

for (i = 0; i < n; ++i) {

struct rte_mbuf *m = input_ops[i].data;

while (m != NULL) {

int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *,

input_ops[i].offset);

for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m);

++byte_idx) {

llr_tmp = llr[byte_idx];

if (llr_decimals == 2)

llr_tmp *= 2;

else if (llr_decimals == 0)

llr_tmp /= 2;

llr_tmp = RTE_MIN(llr_max,

RTE_MAX(llr_min, llr_tmp));

llr[byte_idx] = (int8_t) llr_tmp;

}

m = m->next;

}

}

}

static int

fill_queue_buffers(struct test_op_params *op_params,

uint16_t min_alignment, const int socket_id)

{

int ret;

enum op_data_type type;

const uint16_t n = op_params->num_to_process;

struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {

in_mp,

soft_out_mp,

hard_out_mp,

harq_in_mp,

harq_out_mp,

};

struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {

&op_params->q_bufs[socket_id][queue_id].inputs,

&op_params->q_bufs[socket_id][queue_id].soft_outputs,

&op_params->q_bufs[socket_id][queue_id].hard_outputs,

&op_params->q_bufs[socket_id][queue_id].harq_inputs,

&op_params->q_bufs[socket_id][queue_id].harq_outputs,

};

for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {

struct op_data_entries *ref_entries =

&test_vector.entries[type];

if (ref_entries->nb_segments == 0)

continue;

ret = allocate_buffers_on_socket(queue_ops[type],

n * sizeof(struct rte_bbdev_op_data),

socket_id);

TEST_ASSERT_SUCCESS(ret,

"Couldn't allocate memory for rte_bbdev_op_data structs");

ret = init_op_data_objs(*queue_ops[type], ref_entries,

mbuf_pools[type], n, type, min_alignment);

TEST_ASSERT_SUCCESS(ret,

"Couldn't init rte_bbdev_op_data structs");

}

if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)

limit_input_llr_val_range(*queue_ops[DATA_INPUT], n,

capabilities->cap.turbo_dec.max_llr_modulus);

if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) {

ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n,

capabilities->cap.ldpc_dec.llr_size,

capabilities->cap.ldpc_dec.llr_decimals);

ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n,

capabilities->cap.ldpc_dec.llr_size,

capabilities->cap.ldpc_dec.llr_decimals);

}

return 0;

}

static void

free_buffers(struct active_device *ad, struct test_op_params *op_params)

{

unsigned int i, j;

rte_mempool_free(ad->ops_mempool);

rte_mempool_free(ad->in_mbuf_pool);

rte_mempool_free(ad->hard_out_mbuf_pool);

rte_mempool_free(ad->soft_out_mbuf_pool);

rte_mempool_free(ad->harq_in_mbuf_pool);

rte_mempool_free(ad->harq_out_mbuf_pool);