/* Copyright 2015 EZchip Semiconductor Ltd. All Rights Reserved. * * Copyright (c) 2015, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Local vars */ static const _odp_int_name_kind_t PROFILE_TO_HANDLE_KIND[ODP_TM_NUM_PROFILES] = { [TM_SHAPER_PROFILE] = ODP_TM_SHAPER_PROFILE_HANDLE, [TM_SCHED_PROFILE] = ODP_TM_SCHED_PROFILE_HANDLE, [TM_THRESHOLD_PROFILE] = ODP_TM_THRESHOLD_PROFILE_HANDLE, [TM_WRED_PROFILE] = ODP_TM_WRED_PROFILE_HANDLE }; static const pkt_desc_t EMPTY_PKT_DESC = { .word = 0 }; #define MAX_PRIORITIES ODP_TM_MAX_PRIORITIES #define NUM_SHAPER_COLORS ODP_NUM_SHAPER_COLORS static tm_prop_t basic_prop_tbl[MAX_PRIORITIES][NUM_SHAPER_COLORS] = { [0] = { [ODP_TM_SHAPER_GREEN] = { 0, DECR_BOTH }, [ODP_TM_SHAPER_YELLOW] = { 0, DECR_BOTH }, [ODP_TM_SHAPER_RED] = { 0, DELAY_PKT } }, [1] = { [ODP_TM_SHAPER_GREEN] = { 1, DECR_BOTH }, [ODP_TM_SHAPER_YELLOW] = { 1, DECR_BOTH }, [ODP_TM_SHAPER_RED] = { 1, DELAY_PKT } }, [2] = { [ODP_TM_SHAPER_GREEN] = { 2, DECR_BOTH }, [ODP_TM_SHAPER_YELLOW] = { 2, DECR_BOTH }, [ODP_TM_SHAPER_RED] = { 2, DELAY_PKT } }, [3] = { [ODP_TM_SHAPER_GREEN] = { 3, DECR_BOTH }, [ODP_TM_SHAPER_YELLOW] = { 3, DECR_BOTH }, [ODP_TM_SHAPER_RED] = { 3, DELAY_PKT } }, [4] = { [ODP_TM_SHAPER_GREEN] = { 4, DECR_BOTH }, [ODP_TM_SHAPER_YELLOW] = { 4, DECR_BOTH }, [ODP_TM_SHAPER_RED] = { 4, DELAY_PKT } }, [5] = { [ODP_TM_SHAPER_GREEN] = { 5, DECR_BOTH }, [ODP_TM_SHAPER_YELLOW] = { 5, DECR_BOTH }, [ODP_TM_SHAPER_RED] = { 5, DELAY_PKT } }, [6] = { [ODP_TM_SHAPER_GREEN] = { 6, DECR_BOTH }, [ODP_TM_SHAPER_YELLOW] = { 6, DECR_BOTH }, [ODP_TM_SHAPER_RED] = { 6, DELAY_PKT } }, [7] = { [ODP_TM_SHAPER_GREEN] = { 7, DECR_BOTH }, [ODP_TM_SHAPER_YELLOW] = { 7, DECR_BOTH }, [ODP_TM_SHAPER_RED] = { 7, DELAY_PKT } } }; /* Profile tables. */ static dynamic_tbl_t odp_tm_profile_tbls[ODP_TM_NUM_PROFILES]; /* TM systems table. */ static tm_system_t *odp_tm_systems[ODP_TM_MAX_NUM_SYSTEMS]; static odp_ticketlock_t tm_create_lock; static odp_ticketlock_t tm_profile_lock; static odp_barrier_t tm_first_enq; static int g_main_thread_cpu = -1; /* Forward function declarations. */ static void tm_queue_cnts_decrement(tm_system_t *tm_system, tm_wred_node_t *tm_wred_node, uint32_t priority, uint32_t frame_len); static odp_bool_t tm_demote_pkt_desc(tm_system_t *tm_system, tm_node_obj_t *tm_node_obj, tm_schedulers_obj_t *blocked_scheduler, tm_shaper_obj_t *timer_shaper, pkt_desc_t *demoted_pkt_desc); static tm_queue_obj_t *get_tm_queue_obj(tm_system_t *tm_system, pkt_desc_t *pkt_desc) { tm_queue_obj_t *tm_queue_obj; uint32_t queue_num; if ((!pkt_desc) || (pkt_desc->queue_num == 0)) return NULL; queue_num = pkt_desc->queue_num; tm_queue_obj = tm_system->queue_num_tbl[queue_num]; return tm_queue_obj; } static odp_bool_t pkt_descs_equal(pkt_desc_t *pkt_desc1, pkt_desc_t *pkt_desc2) { if ((!pkt_desc1) || (pkt_desc1->queue_num == 0)) return 0; else if ((!pkt_desc2) || (pkt_desc2->queue_num == 0)) return 0; else if ((pkt_desc1->queue_num == pkt_desc2->queue_num) && (pkt_desc1->epoch == pkt_desc2->epoch) && (pkt_desc1->pkt_len == pkt_desc2->pkt_len)) return 1; else return 0; } static odp_bool_t pkt_descs_not_equal(pkt_desc_t *pkt_desc1, pkt_desc_t *pkt_desc2) { if ((!pkt_desc1) || (pkt_desc1->queue_num == 0)) return 1; else if ((!pkt_desc2) || (pkt_desc2->queue_num == 0)) return 1; else if ((pkt_desc1->queue_num == pkt_desc2->queue_num) && (pkt_desc1->epoch == pkt_desc2->epoch) && (pkt_desc1->pkt_len == pkt_desc2->pkt_len)) return 0; else return 1; } static void tm_init_random_data(tm_random_data_t *tm_random_data) { uint32_t byte_cnt; /* Keep calling odp_random_data until we have filled the entire random * data buffer. Hopefully we don't need to call odp_random_data too * many times to get 256 bytes worth of random data. */ byte_cnt = 0; while (byte_cnt < 256) byte_cnt += odp_random_data(&tm_random_data->buf[byte_cnt], 256 - byte_cnt, 1); tm_random_data->next_random_byte = 0; } static uint16_t tm_random16(tm_random_data_t *tm_random_data) { uint32_t buf_idx; uint16_t rand8a, rand8b; if (255 <= tm_random_data->next_random_byte) tm_init_random_data(tm_random_data); /* Collect 2 random bytes and return them. Endianness does NOT matter * here. This code does assume that next_random_byte is a multiple of * 2. */ buf_idx = tm_random_data->next_random_byte; rand8a = tm_random_data->buf[buf_idx++]; rand8b = tm_random_data->buf[buf_idx++]; tm_random_data->next_random_byte = buf_idx; return (rand8b << 8) | rand8a; } static odp_bool_t tm_random_drop(tm_random_data_t *tm_random_data, odp_tm_percent_t drop_prob) { odp_bool_t drop; uint32_t random_int, scaled_random_int; /* Pick a random integer between 0 and 10000. */ random_int = tm_random16(tm_random_data); scaled_random_int = (random_int * 10000) >> 16; drop = scaled_random_int < (uint32_t)drop_prob; return drop; } static void *alloc_entry_in_dynamic_tbl(dynamic_tbl_t *dynamic_tbl, uint32_t record_size, uint32_t *dynamic_idx_ptr) { uint32_t num_allocd, new_num_allocd, idx; void **new_array_ptrs, *new_record; num_allocd = dynamic_tbl->num_allocd; if (num_allocd <= dynamic_tbl->num_used) { /* Need to alloc or realloc the array of ptrs. */ if (num_allocd <= 32) new_num_allocd = 64; else new_num_allocd = 4 * num_allocd; new_array_ptrs = malloc(new_num_allocd * sizeof(void *)); memset(new_array_ptrs, 0, new_num_allocd * sizeof(void *)); if (dynamic_tbl->num_used != 0) memcpy(new_array_ptrs, dynamic_tbl->array_ptrs, dynamic_tbl->num_used * sizeof(void *)); if (dynamic_tbl->array_ptrs) free(dynamic_tbl->array_ptrs); dynamic_tbl->num_allocd = new_num_allocd; dynamic_tbl->array_ptrs = new_array_ptrs; } idx = dynamic_tbl->num_used; new_record = malloc(record_size); memset(new_record, 0, record_size); dynamic_tbl->array_ptrs[idx] = new_record; dynamic_tbl->num_used++; if (dynamic_idx_ptr) *dynamic_idx_ptr = idx; return new_record; } static void free_dynamic_tbl_entry(dynamic_tbl_t *dynamic_tbl, uint32_t record_size ODP_UNUSED, uint32_t dynamic_idx) { void *record; record = dynamic_tbl->array_ptrs[dynamic_idx]; if (record) { free(record); dynamic_tbl->array_ptrs[dynamic_idx] = NULL; dynamic_tbl->num_freed++; if (dynamic_tbl->num_freed == dynamic_tbl->num_used) { free(dynamic_tbl->array_ptrs); memset(dynamic_tbl, 0, sizeof(dynamic_tbl_t)); } } } static input_work_queue_t *input_work_queue_create(void) { input_work_queue_t *input_work_queue; input_work_queue = malloc(sizeof(input_work_queue_t)); memset(input_work_queue, 0, sizeof(input_work_queue_t)); odp_atomic_init_u64(&input_work_queue->queue_cnt, 0); odp_ticketlock_init(&input_work_queue->lock); return input_work_queue; } static void input_work_queue_destroy(input_work_queue_t *input_work_queue) { /* We first need to "flush/drain" this input_work_queue before * freeing it. Of course, elsewhere it is essential to have first * stopped new tm_enq() (et al) calls from succeeding. */ odp_ticketlock_lock(&input_work_queue->lock); free(input_work_queue); } static int input_work_queue_append(tm_system_t *tm_system, input_work_item_t *work_item) { input_work_queue_t *input_work_queue; input_work_item_t *entry_ptr; uint32_t queue_cnt, tail_idx; input_work_queue = tm_system->input_work_queue; queue_cnt = odp_atomic_load_u64(&input_work_queue->queue_cnt); if (INPUT_WORK_RING_SIZE <= queue_cnt) { input_work_queue->enqueue_fail_cnt++; return -1; } odp_ticketlock_lock(&input_work_queue->lock); tail_idx = input_work_queue->tail_idx; entry_ptr = &input_work_queue->work_ring[tail_idx]; *entry_ptr = *work_item; tail_idx++; if (INPUT_WORK_RING_SIZE <= tail_idx) tail_idx = 0; input_work_queue->total_enqueues++; input_work_queue->tail_idx = tail_idx; odp_ticketlock_unlock(&input_work_queue->lock); odp_atomic_inc_u64(&input_work_queue->queue_cnt); if (input_work_queue->peak_cnt <= queue_cnt) input_work_queue->peak_cnt = queue_cnt + 1; return 0; } static int input_work_queue_remove(input_work_queue_t *input_work_queue, input_work_item_t *work_item) { input_work_item_t *entry_ptr; uint32_t queue_cnt, head_idx; queue_cnt = odp_atomic_load_u64(&input_work_queue->queue_cnt); if (queue_cnt == 0) return -1; odp_ticketlock_lock(&input_work_queue->lock); head_idx = input_work_queue->head_idx; entry_ptr = &input_work_queue->work_ring[head_idx]; *work_item = *entry_ptr; head_idx++; if (INPUT_WORK_RING_SIZE <= head_idx) head_idx = 0; input_work_queue->total_dequeues++; input_work_queue->head_idx = head_idx; odp_ticketlock_unlock(&input_work_queue->lock); odp_atomic_dec_u64(&input_work_queue->queue_cnt); return 0; } static tm_system_t *tm_system_alloc(void) { tm_system_t *tm_system; uint32_t tm_idx; /* Find an open slot in the odp_tm_systems array. */ for (tm_idx = 0; tm_idx < ODP_TM_MAX_NUM_SYSTEMS; tm_idx++) { if (!odp_tm_systems[tm_idx]) { tm_system = malloc(sizeof(tm_system_t)); memset(tm_system, 0, sizeof(tm_system_t)); odp_tm_systems[tm_idx] = tm_system; tm_system->tm_idx = tm_idx; return tm_system; } } return NULL; } static void tm_system_free(tm_system_t *tm_system) { if (tm_system->root_node) free(tm_system->root_node); if (tm_system->queue_num_tbl) free(tm_system->queue_num_tbl); odp_tm_systems[tm_system->tm_idx] = NULL; free(tm_system); } static void *tm_common_profile_create(const char *name, profile_kind_t profile_kind, uint32_t object_size, tm_handle_t *profile_handle_ptr, _odp_int_name_t *name_tbl_id_ptr) { _odp_int_name_kind_t handle_kind; _odp_int_name_t name_tbl_id; dynamic_tbl_t *dynamic_tbl; tm_handle_t profile_handle; uint32_t dynamic_tbl_idx; void *object_ptr; /* Note that alloc_entry_in_dynamic_tbl will zero out all of the memory * that it allocates, so an additional memset here is unnnecessary. */ dynamic_tbl = &odp_tm_profile_tbls[profile_kind]; object_ptr = alloc_entry_in_dynamic_tbl(dynamic_tbl, object_size, &dynamic_tbl_idx); if (!object_ptr) return NULL; handle_kind = PROFILE_TO_HANDLE_KIND[profile_kind]; profile_handle = MAKE_PROFILE_HANDLE(profile_kind, dynamic_tbl_idx); name_tbl_id = ODP_INVALID_NAME; if ((name != NULL) && (name[0] != '\0')) { name_tbl_id = _odp_int_name_tbl_add(name, handle_kind, profile_handle); if (name_tbl_id == ODP_INVALID_NAME) { free_dynamic_tbl_entry(dynamic_tbl, object_size, dynamic_tbl_idx); return NULL; } } *profile_handle_ptr = profile_handle; if (name_tbl_id_ptr) *name_tbl_id_ptr = name_tbl_id; return object_ptr; } static int tm_common_profile_destroy(tm_handle_t profile_handle, uint32_t object_size, _odp_int_name_t name_tbl_id) { profile_kind_t profile_kind; dynamic_tbl_t *dynamic_tbl; uint32_t dynamic_tbl_idx; if (name_tbl_id != ODP_INVALID_NAME) _odp_int_name_tbl_delete(name_tbl_id); profile_kind = GET_PROFILE_KIND(profile_handle); dynamic_tbl = &odp_tm_profile_tbls[profile_kind]; dynamic_tbl_idx = GET_TBL_IDX(profile_handle); free_dynamic_tbl_entry(dynamic_tbl, object_size, dynamic_tbl_idx); return 0; } static void *tm_get_profile_params(tm_handle_t profile_handle, profile_kind_t expected_profile_kind) { profile_kind_t profile_kind; dynamic_tbl_t *dynamic_tbl; uint32_t dynamic_tbl_idx; profile_kind = GET_PROFILE_KIND(profile_handle); if (profile_kind != expected_profile_kind) return NULL; dynamic_tbl = &odp_tm_profile_tbls[profile_kind]; dynamic_tbl_idx = GET_TBL_IDX(profile_handle); return dynamic_tbl->array_ptrs[dynamic_tbl_idx]; } static uint64_t tm_bps_to_rate(uint64_t bps) { /* This code assumes that bps is in the range 1 kbps .. 1 tbps. */ if ((bps >> 32) == 0) return (bps << 23) / ODP_TIME_SEC_IN_NS; else return ((bps << 15) / ODP_TIME_SEC_IN_NS) << 8; } static uint64_t tm_rate_to_bps(uint64_t rate) { /* This code assumes that bps is in the range 1 kbps .. 1 tbps. */ return (rate * ODP_TIME_SEC_IN_NS) >> 23; } static uint64_t tm_max_time_delta(uint64_t rate) { if (rate == 0) return 0; else return (1ULL << (26 + 30)) / rate; } static void tm_shaper_params_cvt_to(odp_tm_shaper_params_t *odp_shaper_params, tm_shaper_params_t *tm_shaper_params) { uint64_t commit_rate, peak_rate, max_commit_time_delta, highest_rate; uint64_t max_peak_time_delta; uint32_t min_time_delta; int64_t commit_burst, peak_burst; commit_rate = tm_bps_to_rate(odp_shaper_params->commit_bps); if ((odp_shaper_params->commit_bps == 0) || (commit_rate == 0)) { tm_shaper_params->max_commit_time_delta = 0; tm_shaper_params->max_peak_time_delta = 0; tm_shaper_params->commit_rate = 0; tm_shaper_params->peak_rate = 0; tm_shaper_params->max_commit = 0; tm_shaper_params->max_peak = 0; tm_shaper_params->min_time_delta = 0; tm_shaper_params->len_adjust = 0; tm_shaper_params->dual_rate = 0; tm_shaper_params->enabled = 0; return; } max_commit_time_delta = tm_max_time_delta(commit_rate); commit_burst = (int64_t)odp_shaper_params->commit_burst; peak_rate = tm_bps_to_rate(odp_shaper_params->peak_bps); if ((odp_shaper_params->peak_bps == 0) || (peak_rate == 0)) { peak_rate = 0; max_peak_time_delta = 0; peak_burst = 0; min_time_delta = (uint32_t)((1 << 26) / commit_rate); } else { max_peak_time_delta = tm_max_time_delta(peak_rate); peak_burst = (int64_t)odp_shaper_params->peak_burst; highest_rate = MAX(commit_rate, peak_rate); min_time_delta = (uint32_t)((1 << 26) / highest_rate); } tm_shaper_params->max_commit_time_delta = max_commit_time_delta; tm_shaper_params->max_peak_time_delta = max_peak_time_delta; tm_shaper_params->commit_rate = commit_rate; tm_shaper_params->peak_rate = peak_rate; tm_shaper_params->max_commit = commit_burst << (26 - 3); tm_shaper_params->max_peak = peak_burst << (26 - 3); tm_shaper_params->min_time_delta = min_time_delta; tm_shaper_params->len_adjust = odp_shaper_params->shaper_len_adjust; tm_shaper_params->dual_rate = odp_shaper_params->dual_rate; tm_shaper_params->enabled = 1; } static void tm_shaper_params_cvt_from(tm_shaper_params_t *tm_shaper_params, odp_tm_shaper_params_t *odp_shaper_params) { uint64_t commit_bps, peak_bps, commit_burst, peak_burst; if (tm_shaper_params->enabled == 0) { memset(odp_shaper_params, 0, sizeof(odp_tm_shaper_params_t)); return; } commit_bps = tm_rate_to_bps(tm_shaper_params->commit_rate); peak_bps = tm_rate_to_bps(tm_shaper_params->peak_rate); commit_burst = tm_shaper_params->max_commit >> (26 - 3); peak_burst = tm_shaper_params->max_peak >> (26 - 3); odp_shaper_params->commit_bps = commit_bps; odp_shaper_params->peak_bps = peak_bps; odp_shaper_params->commit_burst = (uint32_t)commit_burst; odp_shaper_params->peak_burst = (uint32_t)peak_burst; odp_shaper_params->shaper_len_adjust = tm_shaper_params->len_adjust; odp_shaper_params->dual_rate = tm_shaper_params->dual_rate; } static void tm_shaper_obj_init(tm_system_t *tm_system, tm_shaper_params_t *shaper_params, tm_shaper_obj_t *shaper_obj) { shaper_obj->shaper_params = shaper_params; shaper_obj->last_update_time = tm_system->current_time; shaper_obj->callback_time = 0; shaper_obj->commit_cnt = shaper_params->max_commit; shaper_obj->peak_cnt = shaper_params->max_peak; shaper_obj->callback_reason = NO_CALLBACK; shaper_obj->initialized = 1; } /* Any locking required and validity checks must be done by the caller! */ static void tm_shaper_config_set(tm_system_t *tm_system, odp_tm_shaper_t shaper_profile, tm_shaper_obj_t *shaper_obj) { tm_shaper_params_t *shaper_params; /* First remove any old shaper_params. */ if (shaper_obj->shaper_params != NULL) { shaper_obj->shaper_params->ref_cnt--; shaper_obj->shaper_params = NULL; } if (shaper_profile == ODP_TM_INVALID) return; shaper_params = tm_get_profile_params(shaper_profile, TM_SHAPER_PROFILE); if (shaper_params == NULL) return; shaper_params->ref_cnt++; if (shaper_obj->initialized == 0) tm_shaper_obj_init(tm_system, shaper_params, shaper_obj); else shaper_obj->shaper_params = shaper_params; } /* Any locking required and validity checks must be done by the caller! */ static void tm_sched_config_set(tm_shaper_obj_t *shaper_obj, odp_tm_sched_t sched_profile) { tm_sched_params_t *sched_params; if (shaper_obj->sched_params != NULL) { shaper_obj->sched_params->ref_cnt--; shaper_obj->sched_params = NULL; } if (sched_profile == ODP_TM_INVALID) return; sched_params = tm_get_profile_params(sched_profile, TM_SCHED_PROFILE); if (sched_params == NULL) return; sched_params->ref_cnt++; shaper_obj->sched_params = sched_params; } /* Any locking required and validity checks must be done by the caller! */ static void tm_threshold_config_set(tm_wred_node_t *wred_node, odp_tm_threshold_t thresholds_profile) { tm_queue_thresholds_t *threshold_params; if (wred_node->threshold_params != NULL) { wred_node->threshold_params->ref_cnt--; wred_node->threshold_params = NULL; } if (thresholds_profile == ODP_TM_INVALID) return; threshold_params = tm_get_profile_params(thresholds_profile, TM_THRESHOLD_PROFILE); if (threshold_params == NULL) return; threshold_params->ref_cnt++; wred_node->threshold_params = threshold_params; } /* Any locking required and validity checks must be done by the caller! */ static void tm_wred_config_set(tm_wred_node_t *wred_node, uint32_t color, odp_tm_wred_t wred_profile) { tm_wred_params_t *wred_params; if (wred_node->wred_params[color] != NULL) { wred_node->wred_params[color]->ref_cnt--; wred_node->wred_params[color] = NULL; } if (wred_profile == ODP_TM_INVALID) return; wred_params = tm_get_profile_params(wred_profile, TM_WRED_PROFILE); if (wred_params == NULL) return; wred_params->ref_cnt++; wred_node->wred_params[color] = wred_params; } static void update_shaper_elapsed_time(tm_system_t *tm_system, tm_shaper_params_t *shaper_params, tm_shaper_obj_t *shaper_obj) { uint64_t time_delta; int64_t commit, peak, commit_inc, peak_inc, max_commit, max_peak; if (shaper_params->enabled == 0) { shaper_obj->commit_cnt = shaper_params->max_commit; shaper_obj->peak_cnt = shaper_params->max_peak; shaper_obj->last_update_time = tm_system->current_time; return; } /* If the time_delta is "too small" then we just exit without making * any changes. Too small is defined such that * time_delta/ODP_TIME_SEC_IN_NS * MAX(commit_rate, peak_rate) is less * than a byte. */ time_delta = tm_system->current_time - shaper_obj->last_update_time; if (time_delta < (uint64_t)shaper_params->min_time_delta) return; commit = shaper_obj->commit_cnt; max_commit = shaper_params->max_commit; /* If the time_delta is "too large" then we need to prevent overflow * of the multiplications of time_delta and either commit_rate or * peak_rate. */ if (shaper_params->max_commit_time_delta <= time_delta) commit_inc = max_commit; else commit_inc = time_delta * shaper_params->commit_rate; shaper_obj->commit_cnt = (int64_t)MIN(max_commit, commit + commit_inc); if (shaper_params->peak_rate != 0) { peak = shaper_obj->peak_cnt; max_peak = shaper_params->max_peak; if (shaper_params->max_peak_time_delta <= time_delta) peak_inc = max_peak; else peak_inc = time_delta * shaper_params->peak_rate; shaper_obj->peak_cnt = (int64_t)MIN(max_peak, peak + peak_inc); } shaper_obj->last_update_time = tm_system->current_time; } static uint64_t time_till_not_red(tm_shaper_params_t *shaper_params, tm_shaper_obj_t *shaper_obj) { uint64_t min_time_delay, commit_delay, peak_delay; /* Normal case requires that peak_cnt be <= commit_cnt and that * peak_rate be >= commit_rate, but just in case this code handles other * weird cases that might actually be invalid. */ commit_delay = 0; if (shaper_obj->commit_cnt < 0) commit_delay = (-shaper_obj->commit_cnt) / shaper_params->commit_rate; min_time_delay = MAX(shaper_obj->shaper_params->min_time_delta, 256); commit_delay = MAX(commit_delay, min_time_delay); if (shaper_params->peak_rate == 0) return commit_delay; peak_delay = 0; if (shaper_obj->peak_cnt < 0) peak_delay = (-shaper_obj->peak_cnt) / shaper_params->peak_rate; peak_delay = MAX(peak_delay, min_time_delay); if (0 < shaper_obj->commit_cnt) return peak_delay; else if (0 < shaper_obj->peak_cnt) return commit_delay; else return MIN(commit_delay, peak_delay); } static int delete_timer(tm_system_t *tm_system ODP_UNUSED, tm_queue_obj_t *tm_queue_obj, uint8_t cancel_timer) { tm_shaper_obj_t *shaper_obj; shaper_obj = tm_queue_obj->timer_shaper; if (cancel_timer) tm_queue_obj->timer_cancels_outstanding++; if ((tm_queue_obj->timer_reason == NO_CALLBACK) || (!shaper_obj)) { tm_queue_obj->timer_reason = NO_CALLBACK; tm_queue_obj->timer_shaper = NULL; return -1; } tm_queue_obj->timer_reason = NO_CALLBACK; tm_queue_obj->timer_seq++; tm_queue_obj->timer_shaper = NULL; if (tm_queue_obj->delayed_cnt != 0) tm_queue_obj->delayed_cnt--; if (shaper_obj->timer_outstanding != 0) shaper_obj->timer_outstanding--; shaper_obj->timer_tm_queue = NULL; shaper_obj->callback_reason = NO_CALLBACK; return 0; } static void tm_unblock_pkt(tm_system_t *tm_system, pkt_desc_t *pkt_desc) { tm_queue_obj_t *tm_queue_obj; tm_queue_obj = get_tm_queue_obj(tm_system, pkt_desc); if (!tm_queue_obj) return; tm_queue_obj->blocked_scheduler = NULL; if (tm_queue_obj->blocked_cnt != 0) tm_queue_obj->blocked_cnt--; } static void tm_block_pkt(tm_system_t *tm_system, tm_node_obj_t *tm_node_obj, tm_schedulers_obj_t *schedulers_obj, pkt_desc_t *pkt_desc, uint8_t priority) { tm_queue_obj_t *tm_queue_obj; /* Remove the blocked pkt from all downstream entities. * This only happens if the propagation of another pkt caused this pkt * to become blocked, since if the propagation of a pkt causes itself * to be blocked then there can't be any downstream copies to remove. * The caller signals us which case it is by whether the tm_node_obj * is NULL or not. */ tm_queue_obj = get_tm_queue_obj(tm_system, pkt_desc); if (!tm_queue_obj) return; if (tm_node_obj) tm_demote_pkt_desc(tm_system, tm_node_obj, tm_queue_obj->blocked_scheduler, tm_queue_obj->timer_shaper, pkt_desc); else if (tm_queue_obj->timer_reason != NO_CALLBACK) ODP_DBG("%s timer_reason != NO_CALLBACK\n", __func__); tm_queue_obj->blocked_cnt = 1; tm_queue_obj->blocked_scheduler = schedulers_obj; tm_queue_obj->blocked_priority = priority; } static odp_bool_t delay_pkt(tm_system_t *tm_system, tm_shaper_obj_t *shaper_obj, pkt_desc_t *pkt_desc) { tm_queue_obj_t *tm_queue_obj; uint64_t delay_time, wakeup_time, timer_context; int rc; /* Calculate elapsed time before this pkt will be * green or yellow. */ delay_time = time_till_not_red(shaper_obj->shaper_params, shaper_obj); wakeup_time = tm_system->current_time + delay_time; tm_queue_obj = get_tm_queue_obj(tm_system, pkt_desc); if (!tm_queue_obj) return false; /* Insert into timer wheel. */ timer_context = (((uint64_t)tm_queue_obj->timer_seq + 1) << 32) | (((uint64_t)tm_queue_obj->queue_num) << 4); rc = _odp_timer_wheel_insert(tm_system->_odp_int_timer_wheel, wakeup_time, timer_context); if (rc < 0) { ODP_DBG("%s odp_timer_wheel_insert() failed rc=%d\n", __func__, rc); return false; } tm_queue_obj->delayed_cnt++; tm_queue_obj->timer_seq++; tm_queue_obj->timer_reason = UNDELAY_PKT; tm_queue_obj->timer_shaper = shaper_obj; shaper_obj->timer_outstanding++; shaper_obj->callback_reason = UNDELAY_PKT; shaper_obj->callback_time = wakeup_time; shaper_obj->timer_tm_queue = tm_queue_obj; shaper_obj->in_pkt_desc = *pkt_desc; shaper_obj->out_pkt_desc = EMPTY_PKT_DESC; shaper_obj->valid_finish_time = 0; return true; } /* We call rm_pkt_from_shaper for pkts sent AND for pkts demoted. This function * returns true iff the shaper has a change in its output (e.g. empty to * non-empty, non-empty to empty or non-empty to a different non-empty pkt). */ static odp_bool_t rm_pkt_from_shaper(tm_system_t *tm_system, tm_shaper_obj_t *shaper_obj, pkt_desc_t *pkt_desc_to_remove, uint8_t is_sent_pkt) { tm_shaper_params_t *shaper_params; tm_shaper_action_t shaper_action; tm_queue_obj_t *tm_queue_obj; odp_bool_t was_empty; uint32_t frame_len; int64_t tkn_count; tm_queue_obj = get_tm_queue_obj(tm_system, pkt_desc_to_remove); if (!tm_queue_obj) return false; if (pkt_descs_not_equal(&shaper_obj->in_pkt_desc, pkt_desc_to_remove)) return false; was_empty = shaper_obj->out_pkt_desc.queue_num == 0; shaper_action = shaper_obj->propagation_result.action; if ((tm_queue_obj->timer_shaper == shaper_obj) || (shaper_obj->callback_reason == UNDELAY_PKT)) { /* Need to delete the timer - which is either a cancel or just * a delete of a sent_pkt. */ if (tm_queue_obj->timer_reason == NO_CALLBACK) return false; if (!is_sent_pkt) tm_queue_obj->timer_cancels_outstanding++; tm_queue_obj->timer_reason = NO_CALLBACK; tm_queue_obj->timer_seq++; tm_queue_obj->timer_shaper = NULL; if (tm_queue_obj->delayed_cnt != 0) tm_queue_obj->delayed_cnt--; if (shaper_obj->timer_outstanding != 0) shaper_obj->timer_outstanding--; shaper_obj->callback_time = 0; shaper_obj->callback_reason = NO_CALLBACK; shaper_obj->timer_tm_queue = NULL; } shaper_obj->propagation_result.action = DECR_NOTHING; shaper_obj->in_pkt_desc = EMPTY_PKT_DESC; shaper_obj->out_pkt_desc = EMPTY_PKT_DESC; shaper_obj->out_priority = 0; if ((!is_sent_pkt) || (shaper_action == DECR_NOTHING) || (shaper_action == DELAY_PKT)) return !was_empty; shaper_params = shaper_obj->shaper_params; if (shaper_params->enabled) { frame_len = pkt_desc_to_remove->pkt_len + pkt_desc_to_remove->shaper_len_adjust + shaper_params->len_adjust; tkn_count = ((int64_t)frame_len) << 26; if ((shaper_action == DECR_BOTH) || (shaper_action == DECR_COMMIT)) shaper_obj->commit_cnt -= tkn_count; if (shaper_params->peak_rate != 0) if ((shaper_action == DECR_BOTH) || (shaper_action == DECR_PEAK)) shaper_obj->peak_cnt -= tkn_count; } return !was_empty; } /* The run_shaper function returns true iff the shaper has a change in its * output (e.g. empty to non-empty, non-empty to empty or non-empty to a * different non-empty pkt). */ static odp_bool_t run_shaper(tm_system_t *tm_system, tm_shaper_obj_t *shaper_obj, pkt_desc_t *pkt_desc, uint8_t priority) { odp_tm_shaper_color_t shaper_color; tm_shaper_params_t *shaper_params; odp_bool_t output_change; tm_prop_t propagation; shaper_params = shaper_obj->shaper_params; shaper_color = ODP_TM_SHAPER_GREEN; if (shaper_params) { update_shaper_elapsed_time(tm_system, shaper_params, shaper_obj); if (shaper_params->enabled) { if (0 < shaper_obj->commit_cnt) shaper_color = ODP_TM_SHAPER_GREEN; else if (shaper_params->peak_rate == 0) shaper_color = ODP_TM_SHAPER_RED; else if (shaper_obj->peak_cnt <= 0) shaper_color = ODP_TM_SHAPER_RED; else shaper_color = ODP_TM_SHAPER_YELLOW; if (shaper_color == ODP_TM_SHAPER_GREEN) tm_system->shaper_green_cnt++; else if (shaper_color == ODP_TM_SHAPER_YELLOW) tm_system->shaper_yellow_cnt++; else tm_system->shaper_red_cnt++; } /* Run through propagation tbl to get shaper_action and * out_priority */ propagation = basic_prop_tbl[priority][shaper_color]; priority = propagation.output_priority; /* See if this shaper had a previous timer associated with it. * If so we need to cancel it. */ if ((shaper_obj->timer_outstanding != 0) && (shaper_obj->in_pkt_desc.queue_num != 0)) (void)rm_pkt_from_shaper(tm_system, shaper_obj, &shaper_obj->in_pkt_desc, 0); shaper_obj->propagation_result = propagation; if (propagation.action == DELAY_PKT) return delay_pkt(tm_system, shaper_obj, pkt_desc); } shaper_obj->callback_time = 0; shaper_obj->callback_reason = NO_CALLBACK; /* Propagate pkt_desc to the next level */ output_change = pkt_descs_not_equal(&shaper_obj->out_pkt_desc, pkt_desc) || (shaper_obj->out_priority != priority); shaper_obj->in_pkt_desc = *pkt_desc; shaper_obj->input_priority = priority; shaper_obj->out_pkt_desc = *pkt_desc; shaper_obj->out_priority = priority; if (output_change) shaper_obj->valid_finish_time = 0; return output_change; } static int tm_set_finish_time(tm_schedulers_obj_t *schedulers_obj, tm_shaper_obj_t *prod_shaper_obj, pkt_desc_t *new_pkt_desc, uint8_t new_priority) { odp_tm_sched_mode_t mode; tm_sched_params_t *sched_params; tm_sched_state_t *sched_state; uint64_t base_virtual_time, virtual_finish_time; uint32_t inverted_weight, frame_len, frame_weight; /* Check to see if this pkt_desc is "new" or not */ if (prod_shaper_obj->valid_finish_time) { /* return 0 or 1 depending on whether this pkt_desc is the * current chosen one */ return 0; } /* First determine the virtual finish_time of this new pkt. */ sched_params = prod_shaper_obj->sched_params; if (!sched_params) { mode = ODP_TM_BYTE_BASED_WEIGHTS; inverted_weight = 4096; /* Same as a weight of 16. */ } else { mode = sched_params->sched_modes[new_priority]; inverted_weight = sched_params->inverted_weights[new_priority]; } frame_len = new_pkt_desc->pkt_len; if (mode == ODP_TM_FRAME_BASED_WEIGHTS) frame_len = 256; frame_weight = ((inverted_weight * frame_len) + (1 << 15)) >> 16; sched_state = &schedulers_obj->sched_states[new_priority]; base_virtual_time = MAX(prod_shaper_obj->virtual_finish_time, sched_state->base_virtual_time); virtual_finish_time = base_virtual_time + frame_weight; prod_shaper_obj->virtual_finish_time = virtual_finish_time; prod_shaper_obj->valid_finish_time = 1; return 1; } /* The run_sched function returns true iff the scheduler has a change in its * output (e.g. empty to non-empty, non-empty to empty or non-empty to a * different non-empty pkt). */ static odp_bool_t run_sched(tm_system_t *tm_system, tm_shaper_obj_t *prod_shaper_obj, tm_schedulers_obj_t *schedulers_obj, pkt_desc_t *new_pkt_desc, uint8_t new_priority) { tm_sched_state_t *new_sched_state; tm_node_obj_t *tm_node_obj; pkt_desc_t prev_best_pkt_desc; uint64_t new_finish_time, prev_best_time; uint32_t new_priority_mask, num_priorities, priority; int rc; /* First make sure that this "new_pkt_desc" isn't already in this * scheduler */ num_priorities = schedulers_obj->num_priorities; for (priority = 0; priority < num_priorities; priority++) { new_sched_state = &schedulers_obj->sched_states[priority]; prev_best_pkt_desc = new_sched_state->smallest_pkt_desc; if (pkt_descs_equal(new_pkt_desc, &prev_best_pkt_desc)) { ODP_DBG("%s spurious execution ****\n", __func__); return false; } } /* Determine the virtual finish_time of this new pkt. */ tm_set_finish_time(schedulers_obj, prod_shaper_obj, new_pkt_desc, new_priority); new_finish_time = prod_shaper_obj->virtual_finish_time; new_sched_state = &schedulers_obj->sched_states[new_priority]; prev_best_pkt_desc = new_sched_state->smallest_pkt_desc; prev_best_time = new_sched_state->smallest_finish_time; if (prev_best_pkt_desc.queue_num) { /* See if this new pkt has the smallest virtual finish time * for all fanin at the given input priority level, as * represented by the new_sched_state. */ if (prev_best_time <= new_finish_time) { /* Since this new pkt doesn't have the smallest * virtual finish time, just insert it into this * sched_state's list sorted by virtual finish times. */ rc = _odp_sorted_list_insert( tm_system->_odp_int_sorted_pool, new_sched_state->sorted_list, new_finish_time, new_pkt_desc->word); if (0 <= rc) { new_sched_state->sorted_list_cnt++; tm_block_pkt(tm_system, NULL, schedulers_obj, new_pkt_desc, new_priority); } return false; } /* Since this new pkt does have the smallest virtual finish * time, insert the previous best into this sched_state's list * sorted by virtual finish times (though it should always be * inserted at the front), and continue processing. */ rc = _odp_sorted_list_insert (tm_system->_odp_int_sorted_pool, new_sched_state->sorted_list, prev_best_time, prev_best_pkt_desc.word); if (rc < 0) return false; new_sched_state->sorted_list_cnt++; tm_node_obj = schedulers_obj->enclosing_entity; tm_block_pkt(tm_system, tm_node_obj, schedulers_obj, &prev_best_pkt_desc, new_priority); } /* Record the fact that this new pkt is now the best choice of this * sched_state. */ new_sched_state->smallest_finish_time = new_finish_time; new_sched_state->smallest_pkt_desc = *new_pkt_desc; schedulers_obj->priority_bit_mask |= 1ULL << new_priority; /* Next see if this new pkt is the highest priority pkt of all of the * schedulers within this tm_node (recalling that the highest priority * has the lowest priority value). */ new_priority_mask = (1ULL << new_priority) - 1; if ((schedulers_obj->priority_bit_mask & new_priority_mask) != 0) { tm_block_pkt(tm_system, NULL, schedulers_obj, new_pkt_desc, new_priority); return false; } else if (schedulers_obj->out_pkt_desc.queue_num != 0) { tm_node_obj = schedulers_obj->enclosing_entity; tm_block_pkt(tm_system, tm_node_obj, schedulers_obj, &schedulers_obj->out_pkt_desc, schedulers_obj->highest_priority); } schedulers_obj->highest_priority = new_priority; schedulers_obj->out_pkt_desc = *new_pkt_desc; return true; } /* We call rm_pkt_from_scheduler both for pkts sent AND for pkts demoted. * This function returns true iff the shaper has a change in its output * (e.g. empty to non-empty, non-empty to empty or non-empty to a different * non-empty pkt). */ static odp_bool_t rm_pkt_from_sched(tm_system_t *tm_system, tm_schedulers_obj_t *schedulers_obj, pkt_desc_t *pkt_desc_to_remove, uint8_t pkt_desc_priority, uint8_t is_sent_pkt) { _odp_int_sorted_pool_t sorted_pool; _odp_int_sorted_list_t sorted_list; tm_sched_state_t *sched_state, *best_sched_state; pkt_desc_t best_pkt_desc, pkt_desc; uint64_t finish_time; uint32_t priority, best_priority; uint8_t found; int rc; if (is_sent_pkt) priority = schedulers_obj->highest_priority; else priority = pkt_desc_priority; sched_state = &schedulers_obj->sched_states[priority]; sorted_pool = tm_system->_odp_int_sorted_pool; sorted_list = sched_state->sorted_list; found = 0; if (pkt_descs_equal(&sched_state->smallest_pkt_desc, pkt_desc_to_remove)) found = 1; else if (!is_sent_pkt) { rc = _odp_sorted_list_delete(sorted_pool, sorted_list, pkt_desc_to_remove->word); if (0 <= rc) { sched_state->sorted_list_cnt--; return false; } } if (!found) return false; /* This is the case where the pkt_desc_to_remove is NOT in a sorted * list but is instead the best/smallest pkt_desc for "some" priority * level. If is_sent_pkt is TRUE, then this priority level MUST be * the highest_priority level, but if FALSE then this priority level * can be at any priority. */ if (is_sent_pkt) { finish_time = sched_state->smallest_finish_time; sched_state->base_virtual_time = finish_time; } sched_state->smallest_finish_time = 0; sched_state->smallest_pkt_desc = EMPTY_PKT_DESC; schedulers_obj->priority_bit_mask &= ~(1ULL << priority); if (pkt_descs_equal(&schedulers_obj->out_pkt_desc, pkt_desc_to_remove)) { schedulers_obj->out_pkt_desc = EMPTY_PKT_DESC; schedulers_obj->highest_priority = 0; } /* Now that we have removed the pkt_desc_to_remove, we need to see if * there is a new pkt_desc available to become the best pkt_desc for * this priority level/sched_state. */ if (sched_state->sorted_list_cnt != 0) { rc = _odp_sorted_list_remove(sorted_pool, sorted_list, &finish_time, &pkt_desc.word); if (rc <= 0) return false; sched_state->sorted_list_cnt--; sched_state->smallest_finish_time = finish_time; sched_state->smallest_pkt_desc = pkt_desc; schedulers_obj->priority_bit_mask |= 1ULL << priority; } /* Finally we must see if there is a new_pkt desc that is now the * best/smallest for this entire schedulers_obj - which could even be * the pkt_desc we just removed from the sorted_list, or not. */ if (schedulers_obj->priority_bit_mask == 0) { schedulers_obj->out_pkt_desc = EMPTY_PKT_DESC; schedulers_obj->highest_priority = 0; } else { /* Set the highest_priority and out_pkt_desc of the * schedulers_obj. Such a pkt_desc must exist because * otherwise priority_bit_mask would be 0. */ best_priority = __builtin_ctz(schedulers_obj->priority_bit_mask); best_sched_state = &schedulers_obj->sched_states[best_priority]; best_pkt_desc = best_sched_state->smallest_pkt_desc; schedulers_obj->highest_priority = best_priority; schedulers_obj->out_pkt_desc = best_pkt_desc; tm_unblock_pkt(tm_system, &best_pkt_desc); } return true; } /* The propagate_pkt_desc function returns true iff there is a new pkt at the * egress (i.e tm_system->egress_pkt_desc was set). */ static odp_bool_t tm_propagate_pkt_desc(tm_system_t *tm_system, tm_shaper_obj_t *shaper_obj, pkt_desc_t *new_pkt_desc, uint8_t new_priority) { tm_schedulers_obj_t *schedulers_obj; tm_node_obj_t *tm_node_obj; pkt_desc_t prev_shaper_pkt, new_shaper_pkt, prev_sched_pkt; pkt_desc_t new_sched_pkt; odp_bool_t shaper_change, shaper_was_empty, shaper_is_empty; odp_bool_t sched_change, sched_was_empty, sched_is_empty, ret_code; uint8_t prev_shaper_prio, new_shaper_prio, new_sched_prio; /* Run shaper. */ prev_shaper_pkt = shaper_obj->out_pkt_desc; prev_shaper_prio = shaper_obj->out_priority; shaper_was_empty = prev_shaper_pkt.queue_num == 0; shaper_change = run_shaper(tm_system, shaper_obj, new_pkt_desc, new_priority); new_shaper_pkt = shaper_obj->out_pkt_desc; new_shaper_prio = shaper_obj->out_priority; shaper_is_empty = new_shaper_pkt.queue_num == 0; tm_node_obj = shaper_obj->next_tm_node; while (!tm_node_obj->is_root_node) { /* not at egress */ if (!shaper_change) return false; schedulers_obj = tm_node_obj->schedulers_obj; prev_sched_pkt = schedulers_obj->out_pkt_desc; sched_was_empty = prev_sched_pkt.queue_num == 0; sched_change = false; /* First remove any old pkt from the scheduler. */ if (!shaper_was_empty) sched_change = rm_pkt_from_sched(tm_system, schedulers_obj, &prev_shaper_pkt, prev_shaper_prio, false); /* Run scheduler, including priority multiplexor. */ if (!shaper_is_empty) sched_change |= run_sched(tm_system, shaper_obj, schedulers_obj, &new_shaper_pkt, new_shaper_prio); if (!sched_change) return false; new_sched_pkt = schedulers_obj->out_pkt_desc; new_sched_prio = schedulers_obj->highest_priority; sched_is_empty = new_sched_pkt.queue_num == 0; shaper_obj = &tm_node_obj->shaper_obj; prev_shaper_pkt = shaper_obj->out_pkt_desc; prev_shaper_prio = shaper_obj->out_priority; shaper_was_empty = prev_shaper_pkt.queue_num == 0; shaper_change = false; /* First remove any old pkt from the shaper. */ if (!sched_was_empty) shaper_change = rm_pkt_from_shaper(tm_system, shaper_obj, &prev_sched_pkt, false); /* Run shaper. */ if (!sched_is_empty) shaper_change |= run_shaper(tm_system, shaper_obj, &new_sched_pkt, new_sched_prio); new_shaper_pkt = shaper_obj->out_pkt_desc; new_shaper_prio = shaper_obj->out_priority; shaper_is_empty = new_shaper_pkt.queue_num == 0; tm_node_obj = shaper_obj->next_tm_node; } ret_code = false; if (!shaper_is_empty) { tm_system->egress_pkt_desc = new_shaper_pkt; ret_code = true; } return ret_code; } static void tm_pkt_desc_init(pkt_desc_t *pkt_desc, odp_packet_t pkt, tm_queue_obj_t *tm_queue_obj) { tm_queue_obj->epoch++; pkt_desc->queue_num = tm_queue_obj->queue_num; pkt_desc->pkt_len = odp_packet_len(pkt); pkt_desc->shaper_len_adjust = odp_packet_shaper_len_adjust(pkt); pkt_desc->drop_eligible = odp_packet_drop_eligible(pkt); pkt_desc->pkt_color = odp_packet_color(pkt); pkt_desc->epoch = tm_queue_obj->epoch & 0x0F; } /* The demote_pkt_desc function returns true iff there is a new pkt at the * egress (i.e tm_system->egress_pkt_desc was set). */ static odp_bool_t tm_demote_pkt_desc(tm_system_t *tm_system, tm_node_obj_t *tm_node_obj, tm_schedulers_obj_t *blocked_scheduler, tm_shaper_obj_t *timer_shaper, pkt_desc_t *demoted_pkt_desc) { tm_schedulers_obj_t *schedulers_obj; tm_shaper_obj_t *shaper_obj; pkt_desc_t prev_shaper_pkt, new_shaper_pkt, prev_sched_pkt; pkt_desc_t new_sched_pkt; odp_bool_t shaper_change, shaper_was_empty, shaper_is_empty; odp_bool_t sched_change, sched_was_empty, sched_is_empty, ret_code; uint8_t prev_shaper_prio, new_shaper_prio, new_sched_prio; uint8_t demoted_priority; shaper_obj = &tm_node_obj->shaper_obj; if ((!blocked_scheduler) && (!timer_shaper)) return false; if (tm_node_obj->schedulers_obj == blocked_scheduler) return false; /* See if this first shaper_obj is delaying the demoted_pkt_desc */ prev_shaper_pkt = shaper_obj->out_pkt_desc; prev_shaper_prio = shaper_obj->out_priority; shaper_was_empty = prev_shaper_pkt.queue_num == 0; demoted_priority = 3; if (pkt_descs_equal(&shaper_obj->out_pkt_desc, demoted_pkt_desc)) demoted_priority = shaper_obj->out_priority; shaper_change = rm_pkt_from_shaper(tm_system, shaper_obj, demoted_pkt_desc, 0); if (shaper_obj == timer_shaper) return false; new_shaper_pkt = shaper_obj->out_pkt_desc; new_shaper_prio = shaper_obj->out_priority; shaper_is_empty = new_shaper_pkt.queue_num == 0; tm_node_obj = shaper_obj->next_tm_node; while (!tm_node_obj->is_root_node) { /* not at egress */ if ((!demoted_pkt_desc) && (!shaper_change)) return false; schedulers_obj = tm_node_obj->schedulers_obj; prev_sched_pkt = schedulers_obj->out_pkt_desc; sched_was_empty = prev_sched_pkt.queue_num == 0; sched_change = false; /* First remove the demoted pkt or any old pkt from the * scheduler. */ if (demoted_pkt_desc) { sched_change = rm_pkt_from_sched(tm_system, schedulers_obj, demoted_pkt_desc, demoted_priority, 0); if (schedulers_obj == blocked_scheduler) demoted_pkt_desc = NULL; } else if (!shaper_was_empty) sched_change = rm_pkt_from_sched(tm_system, schedulers_obj, &prev_shaper_pkt, prev_shaper_prio, false); /* Run scheduler, including priority multiplexor. */ if (!shaper_is_empty) sched_change |= run_sched(tm_system, shaper_obj, schedulers_obj, &new_shaper_pkt, new_shaper_prio); if ((!demoted_pkt_desc) && (!sched_change)) return false; new_sched_pkt = schedulers_obj->out_pkt_desc; new_sched_prio = schedulers_obj->highest_priority; sched_is_empty = new_sched_pkt.queue_num == 0; shaper_obj = &tm_node_obj->shaper_obj; prev_shaper_pkt = shaper_obj->out_pkt_desc; prev_shaper_prio = shaper_obj->out_priority; shaper_was_empty = prev_shaper_pkt.queue_num == 0; shaper_change = false; /* First remove the demoted pkt or any old pkt from the * scheduler. */ if (demoted_pkt_desc) { if (pkt_descs_equal(&shaper_obj->out_pkt_desc, demoted_pkt_desc)) demoted_priority = shaper_obj->out_priority; shaper_change = rm_pkt_from_shaper(tm_system, shaper_obj, demoted_pkt_desc, 0); if (shaper_obj == timer_shaper) demoted_pkt_desc = NULL; } else if (!sched_was_empty) shaper_change = rm_pkt_from_shaper(tm_system, shaper_obj, &prev_sched_pkt, false); /* Run shaper. */ if (!sched_is_empty) shaper_change |= run_shaper(tm_system, shaper_obj, &new_sched_pkt, new_sched_prio); new_shaper_pkt = shaper_obj->out_pkt_desc; new_shaper_prio = shaper_obj->out_priority; shaper_is_empty = new_shaper_pkt.queue_num == 0; tm_node_obj = shaper_obj->next_tm_node; } ret_code = false; if (!shaper_is_empty) { tm_system->egress_pkt_desc = new_shaper_pkt; ret_code = true; } return ret_code; } /* The consume_pkt_desc function returns true iff there is a new pkt at the * egress (i.e tm_system->egress_pkt_desc was set). */ static odp_bool_t tm_consume_pkt_desc(tm_system_t *tm_system, tm_shaper_obj_t *shaper_obj, pkt_desc_t *new_pkt_desc, uint8_t new_priority, pkt_desc_t *sent_pkt_desc) { tm_schedulers_obj_t *schedulers_obj; tm_node_obj_t *tm_node_obj; pkt_desc_t prev_shaper_pkt, new_shaper_pkt, prev_sched_pkt; pkt_desc_t new_sched_pkt; odp_bool_t shaper_is_empty, sched_is_empty, ret_code; uint8_t sent_priority, new_shaper_prio, new_sched_prio; /* Verify that the shaper output is the sent_pkt_desc. */ prev_shaper_pkt = shaper_obj->out_pkt_desc; if (pkt_descs_not_equal(&prev_shaper_pkt, sent_pkt_desc)) return false; /* Remove the sent_pkt_desc from the shaper. */ rm_pkt_from_shaper(tm_system, shaper_obj, sent_pkt_desc, true); /* If there is a new pkt then run that through the shaper. */ if (new_pkt_desc && (new_pkt_desc->queue_num != 0)) run_shaper(tm_system, shaper_obj, new_pkt_desc, new_priority); new_shaper_pkt = shaper_obj->out_pkt_desc; new_shaper_prio = shaper_obj->out_priority; shaper_is_empty = new_shaper_pkt.queue_num == 0; if (pkt_descs_equal(&new_shaper_pkt, sent_pkt_desc)) ODP_DBG("%s shaper has old pkt_desc\n", __func__); tm_node_obj = shaper_obj->next_tm_node; while (!tm_node_obj->is_root_node) { /* not at egress */ schedulers_obj = tm_node_obj->schedulers_obj; prev_sched_pkt = schedulers_obj->out_pkt_desc; sent_priority = schedulers_obj->highest_priority; /* Verify that the scheduler output is the sent_pkt_desc. */ if (pkt_descs_not_equal(&prev_sched_pkt, sent_pkt_desc)) { ODP_DBG("%s sched has bad out pkt_desc\n", __func__); return false; } /* Remove the sent_pkt_desc from the scheduler. */ rm_pkt_from_sched(tm_system, schedulers_obj, sent_pkt_desc, sent_priority, true); /* If there is a new pkt then run that through the scheduler. */ if (!shaper_is_empty) run_sched(tm_system, shaper_obj, schedulers_obj, &new_shaper_pkt, new_shaper_prio); new_sched_pkt = schedulers_obj->out_pkt_desc; new_sched_prio = schedulers_obj->highest_priority; sched_is_empty = new_sched_pkt.queue_num == 0; if (pkt_descs_equal(&new_sched_pkt, sent_pkt_desc)) ODP_DBG("%s sched has old pkt_desc\n", __func__); if (pkt_descs_equal(&new_sched_pkt, sent_pkt_desc)) ODP_DBG("%s scheduler has old pkt_desc\n", __func__); shaper_obj = &tm_node_obj->shaper_obj; prev_shaper_pkt = shaper_obj->out_pkt_desc; /* Verify that the shaper output is the sent_pkt_desc. */ if (pkt_descs_not_equal(&prev_shaper_pkt, sent_pkt_desc)) { ODP_DBG("%s shaper has bad out pkt_desc\n", __func__); return false; } /* Remove the sent_pkt_desc from the shaper. */ rm_pkt_from_shaper(tm_system, shaper_obj, sent_pkt_desc, true); /* If there is a new pkt then run that through the shaper. */ if (!sched_is_empty) run_shaper(tm_system, shaper_obj, &new_sched_pkt, new_sched_prio); new_shaper_pkt = shaper_obj->out_pkt_desc; new_shaper_prio = shaper_obj->out_priority; shaper_is_empty = new_shaper_pkt.queue_num == 0; if (pkt_descs_equal(&new_shaper_pkt, sent_pkt_desc)) ODP_DBG("%s shaper has old pkt_desc\n", __func__); tm_node_obj = shaper_obj->next_tm_node; } ret_code = false; if (!shaper_is_empty) { tm_system->egress_pkt_desc = new_shaper_pkt; ret_code = true; } return ret_code; } /* The consume_sent_pkt function returns true iff there is a new pkt at the * egress (i.e tm_system->egress_pkt_desc was set). */ static odp_bool_t tm_consume_sent_pkt(tm_system_t *tm_system, pkt_desc_t *sent_pkt_desc) { _odp_int_pkt_queue_t _odp_int_pkt_queue; tm_queue_obj_t *tm_queue_obj; odp_packet_t pkt; pkt_desc_t *new_pkt_desc; uint32_t queue_num, pkt_len; int rc; queue_num = sent_pkt_desc->queue_num; tm_queue_obj = tm_system->queue_num_tbl[queue_num]; if (!tm_queue_obj) return false; pkt_len = sent_pkt_desc->pkt_len; tm_queue_obj->pkts_consumed_cnt++; tm_queue_cnts_decrement(tm_system, tm_queue_obj->tm_wred_node, tm_queue_obj->priority, pkt_len); /* Get the next pkt in the tm_queue, if there is one. */ _odp_int_pkt_queue = tm_queue_obj->_odp_int_pkt_queue; rc = _odp_pkt_queue_remove(tm_system->_odp_int_queue_pool, _odp_int_pkt_queue, &pkt); if (rc < 0) return false; new_pkt_desc = NULL; if (0 < rc) { tm_queue_obj->pkt = pkt; tm_pkt_desc_init(&tm_queue_obj->in_pkt_desc, pkt, tm_queue_obj); new_pkt_desc = &tm_queue_obj->in_pkt_desc; tm_queue_obj->pkts_dequeued_cnt++; } return tm_consume_pkt_desc(tm_system, &tm_queue_obj->shaper_obj, new_pkt_desc, tm_queue_obj->priority, sent_pkt_desc); } static odp_tm_percent_t tm_queue_fullness(tm_wred_params_t *wred_params, tm_queue_thresholds_t *thresholds, tm_queue_cnts_t *queue_cnts) { uint64_t current_cnt, max_cnt, fullness; if (wred_params->use_byte_fullness) { current_cnt = odp_atomic_load_u64(&queue_cnts->byte_cnt); max_cnt = thresholds->max_bytes; } else { current_cnt = odp_atomic_load_u64(&queue_cnts->pkt_cnt); max_cnt = thresholds->max_pkts; } if (max_cnt == 0) return 0; fullness = (10000 * current_cnt) / max_cnt; return (odp_tm_percent_t)MIN(fullness, 50000); } static odp_bool_t tm_local_random_drop(tm_system_t *tm_system, tm_wred_params_t *wred_params, odp_tm_percent_t queue_fullness) { odp_tm_percent_t min_threshold, med_threshold, first_threshold; odp_tm_percent_t drop_prob; odp_tm_percent_t med_drop_prob, max_drop_prob; uint32_t denom, numer; if (wred_params->enable_wred == 0) return 0; min_threshold = wred_params->min_threshold; med_threshold = wred_params->med_threshold; first_threshold = (min_threshold != 0) ? min_threshold : med_threshold; if (10000 <= queue_fullness) return 1; else if (queue_fullness <= first_threshold) return 0; /* Determine if we have two active thresholds, min_threshold and * med_threshold or just med_threshold. */ med_drop_prob = wred_params->med_drop_prob; max_drop_prob = wred_params->max_drop_prob; if (min_threshold == 0) { denom = (uint32_t)(10000 - med_threshold); numer = (uint32_t)max_drop_prob; drop_prob = (numer * (uint32_t)(queue_fullness - med_threshold)) / denom; } else if ((min_threshold < queue_fullness) && (queue_fullness < med_threshold)) { denom = (uint32_t)(med_threshold - min_threshold); numer = (uint32_t)med_drop_prob; drop_prob = (numer * (uint32_t)(queue_fullness - min_threshold)) / denom; } else { /* med_threshold <= queue_fullness. */ denom = (uint32_t)(10000 - med_threshold); numer = (uint32_t)(max_drop_prob - med_drop_prob); drop_prob = max_drop_prob - ((numer * (10000 - queue_fullness)) / denom); } if (drop_prob == 0) return 0; else if (10000 <= drop_prob) return 1; else return tm_random_drop(&tm_system->tm_random_data, drop_prob); } static odp_bool_t tm_queue_is_full(tm_queue_thresholds_t *thresholds, tm_queue_cnts_t *queue_cnts) { odp_bool_t queue_is_full; uint64_t max_bytes; uint32_t max_pkts; max_bytes = thresholds->max_bytes; max_pkts = thresholds->max_pkts; queue_is_full = 0; if (max_pkts != 0) queue_is_full = max_pkts <= odp_atomic_load_u64(&queue_cnts->pkt_cnt); if (max_bytes != 0) queue_is_full |= max_bytes <= odp_atomic_load_u64(&queue_cnts->byte_cnt); return queue_is_full; } static odp_bool_t random_early_discard(tm_system_t *tm_system, tm_queue_obj_t *tm_queue_obj ODP_UNUSED, tm_wred_node_t *tm_wred_node, odp_packet_color_t pkt_color) { tm_queue_thresholds_t *thresholds; tm_wred_params_t *wred_params; odp_tm_percent_t fullness; tm_queue_cnts_t *queue_cnts; thresholds = tm_wred_node->threshold_params; if (thresholds) { wred_params = tm_wred_node->wred_params[pkt_color]; queue_cnts = &tm_wred_node->queue_cnts; if ((!wred_params) || (wred_params->enable_wred == 0)) { if (tm_queue_is_full(thresholds, queue_cnts)) return 1; } else { fullness = tm_queue_fullness(wred_params, thresholds, queue_cnts); if (tm_local_random_drop(tm_system, wred_params, fullness)) return 1; } } /* For each tm_wred_node between the initial and a TM egress, do a * Random Early Discard check, if enabled. */ tm_wred_node = tm_wred_node->next_tm_wred_node; while (tm_wred_node) { thresholds = tm_wred_node->threshold_params; if (thresholds) { wred_params = tm_wred_node->wred_params[pkt_color]; queue_cnts = &tm_wred_node->queue_cnts; if ((wred_params) && wred_params->enable_wred) { fullness = tm_queue_fullness(wred_params, thresholds, queue_cnts); if (tm_local_random_drop(tm_system, wred_params, fullness)) return 1; } } tm_wred_node = tm_wred_node->next_tm_wred_node; } return 0; } /* Returns the current queue pkt_cnt. */ static uint32_t tm_queue_cnts_increment(tm_system_t *tm_system, tm_wred_node_t *tm_wred_node, uint32_t priority, uint32_t frame_len) { tm_queue_cnts_t *queue_cnts; uint32_t tm_queue_pkt_cnt; odp_ticketlock_lock(&tm_wred_node->tm_wred_node_lock); queue_cnts = &tm_wred_node->queue_cnts; odp_atomic_inc_u64(&queue_cnts->pkt_cnt); odp_atomic_add_u64(&queue_cnts->byte_cnt, frame_len); tm_queue_pkt_cnt = (uint32_t)odp_atomic_load_u64(&queue_cnts->pkt_cnt); odp_ticketlock_unlock(&tm_wred_node->tm_wred_node_lock); /* For each tm_wred_node between the initial one and a TM egress, * increment its queue_cnts, if enabled. */ tm_wred_node = tm_wred_node->next_tm_wred_node; while (tm_wred_node) { odp_ticketlock_lock(&tm_wred_node->tm_wred_node_lock); queue_cnts = &tm_wred_node->queue_cnts; odp_atomic_inc_u64(&queue_cnts->pkt_cnt); odp_atomic_add_u64(&queue_cnts->byte_cnt, frame_len); odp_ticketlock_unlock(&tm_wred_node->tm_wred_node_lock); tm_wred_node = tm_wred_node->next_tm_wred_node; } queue_cnts = &tm_system->total_info.queue_cnts; odp_atomic_inc_u64(&queue_cnts->pkt_cnt); odp_atomic_add_u64(&queue_cnts->byte_cnt, frame_len); queue_cnts = &tm_system->priority_info[priority].queue_cnts; odp_atomic_inc_u64(&queue_cnts->pkt_cnt); odp_atomic_add_u64(&queue_cnts->byte_cnt, frame_len); return tm_queue_pkt_cnt; } static void tm_queue_cnts_decrement(tm_system_t *tm_system, tm_wred_node_t *tm_wred_node, uint32_t priority, uint32_t frame_len) { tm_queue_cnts_t *queue_cnts; odp_ticketlock_lock(&tm_wred_node->tm_wred_node_lock); queue_cnts = &tm_wred_node->queue_cnts; odp_atomic_dec_u64(&queue_cnts->pkt_cnt); odp_atomic_sub_u64(&queue_cnts->byte_cnt, frame_len); odp_ticketlock_unlock(&tm_wred_node->tm_wred_node_lock); /* For each tm_wred_node between the initial one and a TM egress, * decrement its queue_cnts, if enabled. */ tm_wred_node = tm_wred_node->next_tm_wred_node; while (tm_wred_node) { odp_ticketlock_lock(&tm_wred_node->tm_wred_node_lock); queue_cnts = &tm_wred_node->queue_cnts; odp_atomic_dec_u64(&queue_cnts->pkt_cnt); odp_atomic_sub_u64(&queue_cnts->byte_cnt, frame_len); odp_ticketlock_unlock(&tm_wred_node->tm_wred_node_lock); tm_wred_node = tm_wred_node->next_tm_wred_node; } queue_cnts = &tm_system->total_info.queue_cnts; odp_atomic_dec_u64(&queue_cnts->pkt_cnt); odp_atomic_sub_u64(&queue_cnts->byte_cnt, frame_len); queue_cnts = &tm_system->priority_info[priority].queue_cnts; odp_atomic_dec_u64(&queue_cnts->pkt_cnt); odp_atomic_sub_u64(&queue_cnts->byte_cnt, frame_len); } static int tm_enqueue(tm_system_t *tm_system, tm_queue_obj_t *tm_queue_obj, odp_packet_t pkt) { input_work_item_t work_item; odp_packet_color_t pkt_color; tm_wred_node_t *initial_tm_wred_node; odp_bool_t drop_eligible, drop; uint32_t frame_len, pkt_depth; int rc; odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); /* If we're from an ordered queue and not in order * record the event and wait until order is resolved */ if (queue_tm_reorder(&tm_queue_obj->tm_qentry, &pkt_hdr->buf_hdr)) return 0; if (tm_system->first_enq == 0) { odp_barrier_wait(&tm_system->tm_system_barrier); tm_system->first_enq = 1; } pkt_color = odp_packet_color(pkt); drop_eligible = odp_packet_drop_eligible(pkt); initial_tm_wred_node = tm_queue_obj->tm_wred_node; if (drop_eligible) { drop = random_early_discard(tm_system, tm_queue_obj, initial_tm_wred_node, pkt_color); if (drop) return -1; } work_item.queue_num = tm_queue_obj->queue_num; work_item.pkt = pkt; rc = input_work_queue_append(tm_system, &work_item); if (rc < 0) { ODP_DBG("%s work queue full\n", __func__); return rc; } frame_len = odp_packet_len(pkt); pkt_depth = tm_queue_cnts_increment(tm_system, initial_tm_wred_node, tm_queue_obj->priority, frame_len); return pkt_depth; } static void egress_vlan_marking(tm_vlan_marking_t *vlan_marking, odp_packet_t odp_pkt) { _odp_vlanhdr_t vlan_hdr, *vlan_hdr_ptr; _odp_ethhdr_t *ether_hdr_ptr; odp_bool_t split_hdr; uint32_t hdr_len; uint16_t old_tci, new_tci; ether_hdr_ptr = odp_packet_l2_ptr(odp_pkt, &hdr_len); vlan_hdr_ptr = (_odp_vlanhdr_t *)(ether_hdr_ptr + 1); /* If the split_hdr variable below is TRUE, then this indicates that * for this odp (output) packet the VLAN header is not all in the same * segment and so cannot be accessed using a traditional header record * overlay. This should be a very rare occurrence, but still need to * handle this case for correctness, but because of the rarity the * code handling this is more optimized for ease of understanding and * correctness rather then performance. */ split_hdr = hdr_len < (_ODP_ETHHDR_LEN + _ODP_VLANHDR_LEN); if (split_hdr) { odp_packet_copy_to_mem(odp_pkt, _ODP_ETHHDR_LEN, _ODP_VLANHDR_LEN, &vlan_hdr); vlan_hdr_ptr = &vlan_hdr; } old_tci = odp_be_to_cpu_16(vlan_hdr_ptr->tci); new_tci = old_tci; if (vlan_marking->drop_eligible_enabled) new_tci |= _ODP_VLANHDR_DEI_MASK; if (new_tci == old_tci) return; vlan_hdr_ptr->tci = odp_cpu_to_be_16(new_tci); if (split_hdr) odp_packet_copy_from_mem(odp_pkt, _ODP_ETHHDR_LEN, _ODP_VLANHDR_LEN, &vlan_hdr); } static void egress_ipv4_tos_marking(tm_tos_marking_t *tos_marking, odp_packet_t odp_pkt) { _odp_ipv4hdr_t ipv4_hdr, *ipv4_hdr_ptr; odp_bool_t split_hdr; uint32_t hdr_len, l3_offset, old_chksum, ones_compl_sum, tos_diff; uint8_t old_tos, new_tos, ecn; l3_offset = odp_packet_l3_offset(odp_pkt); ipv4_hdr_ptr = odp_packet_l3_ptr(odp_pkt, &hdr_len); /* If the split_hdr variable below is TRUE, then this indicates that * for this odp (output) packet the IPv4 header is not all in the same * segment and so cannot be accessed using a traditional header record * overlay. This should be a very rare occurrence, but still need to * handle this case for correctness, but because of the rarity the * code handling this is more optimized for ease of understanding and * correctness rather then performance. */ split_hdr = hdr_len < 12; if (split_hdr) { odp_packet_copy_to_mem(odp_pkt, l3_offset, _ODP_IPV4HDR_LEN, &ipv4_hdr); ipv4_hdr_ptr = &ipv4_hdr; } old_tos = ipv4_hdr_ptr->tos; new_tos = old_tos; if (tos_marking->drop_prec_enabled) new_tos = (new_tos & tos_marking->inverted_dscp_mask) | tos_marking->shifted_dscp; if (tos_marking->ecn_ce_enabled && odp_packet_has_tcp(odp_pkt)) { ecn = old_tos & _ODP_IP_TOS_ECN_MASK; if ((ecn == _ODP_IP_ECN_ECT0) || (ecn == _ODP_IP_ECN_ECT1)) new_tos = (new_tos & ~_ODP_IP_TOS_ECN_MASK) | (_ODP_IP_ECN_CE << _ODP_IP_TOS_ECN_SHIFT); } if (new_tos == old_tos) return; /* Now do an incremental IPv4 header checksum (see RFC 1624). Note * that the underlying one's complement arithmetic must be done using * 32-bit arithmetic in order to capture carry bits, but all "~" * operations must be anded with 0xFFFF to constrain the "~" result to * fit in 16 bits. Also note the last step which wraps any carry * bits back into the sum - this being the key difference between * one's complement summation and two's complement. Finally note that * in this specific case the carry out check does NOT need to be * repeated since it can be proven that the carry in sum cannot * cause another carry out. */ old_chksum = (uint32_t)odp_be_to_cpu_16(ipv4_hdr_ptr->chksum); ones_compl_sum = (~old_chksum) & 0xFFFF; tos_diff = ((uint32_t)new_tos) + ((~(uint32_t)old_tos) & 0xFFFF); ones_compl_sum += tos_diff; if (ones_compl_sum >= 0x10000) ones_compl_sum = (ones_compl_sum >> 16) + (ones_compl_sum & 0xFFFF); ipv4_hdr_ptr->tos = new_tos; ipv4_hdr_ptr->chksum = odp_cpu_to_be_16((~ones_compl_sum) & 0xFFFF); if (split_hdr) odp_packet_copy_from_mem(odp_pkt, l3_offset, _ODP_IPV4HDR_LEN, &ipv4_hdr); } static void egress_ipv6_tc_marking(tm_tos_marking_t *tos_marking, odp_packet_t odp_pkt) { _odp_ipv6hdr_t ipv6_hdr, *ipv6_hdr_ptr; odp_bool_t split_hdr; uint32_t hdr_len, old_ver_tc_flow, new_ver_tc_flow, l3_offset; uint8_t old_tc, new_tc, ecn; l3_offset = odp_packet_l3_offset(odp_pkt); ipv6_hdr_ptr = odp_packet_l3_ptr(odp_pkt, &hdr_len); /* If the split_hdr variable below is TRUE, then this indicates that * for this odp (output) packet the IPv6 header is not all in the same * segment and so cannot be accessed using a traditional header record * overlay. This should be a very rare occurrence, but still need to * handle this case for correctness, but because of the rarity the * code handling this is more optimized for ease of understanding and * correctness rather then performance. */ split_hdr = hdr_len < 4; if (split_hdr) { odp_packet_copy_to_mem(odp_pkt, l3_offset, _ODP_IPV6HDR_LEN, &ipv6_hdr); ipv6_hdr_ptr = &ipv6_hdr; } old_ver_tc_flow = odp_be_to_cpu_32(ipv6_hdr_ptr->ver_tc_flow); old_tc = (old_ver_tc_flow & _ODP_IPV6HDR_TC_MASK) >> _ODP_IPV6HDR_TC_SHIFT; new_tc = old_tc; if (tos_marking->drop_prec_enabled) new_tc = (new_tc & tos_marking->inverted_dscp_mask) | tos_marking->shifted_dscp; if (tos_marking->ecn_ce_enabled && odp_packet_has_tcp(odp_pkt)) { ecn = old_tc & _ODP_IP_TOS_ECN_MASK; if ((ecn == _ODP_IP_ECN_ECT0) || (ecn == _ODP_IP_ECN_ECT1)) new_tc = (new_tc & ~_ODP_IP_TOS_ECN_MASK) | (_ODP_IP_ECN_CE << _ODP_IP_TOS_ECN_SHIFT); } if (new_tc == old_tc) return; new_ver_tc_flow = (old_ver_tc_flow & ~_ODP_IPV6HDR_TC_MASK) | (new_tc << _ODP_IPV6HDR_TC_SHIFT); ipv6_hdr_ptr->ver_tc_flow = odp_cpu_to_be_32(new_ver_tc_flow); if (split_hdr) odp_packet_copy_from_mem(odp_pkt, l3_offset, _ODP_IPV6HDR_LEN, &ipv6_hdr); } static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt) { odp_packet_color_t color; tm_vlan_marking_t *vlan_marking; tm_tos_marking_t *ip_marking; color = odp_packet_color(odp_pkt); if (odp_packet_has_vlan(odp_pkt)) { vlan_marking = &tm_system->marking.vlan_marking[color]; if (vlan_marking->marking_enabled) egress_vlan_marking(vlan_marking, odp_pkt); } if (odp_packet_has_ipv4(odp_pkt)) { ip_marking = &tm_system->marking.ip_tos_marking[color]; if (ip_marking->marking_enabled) egress_ipv4_tos_marking(ip_marking, odp_pkt); } else if (odp_packet_has_ipv6(odp_pkt)) { ip_marking = &tm_system->marking.ip_tos_marking[color]; if (ip_marking->marking_enabled) egress_ipv6_tc_marking(ip_marking, odp_pkt); } } static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends) { tm_queue_obj_t *tm_queue_obj; odp_packet_t odp_pkt; pkt_desc_t *pkt_desc; uint32_t cnt, queue_num; for (cnt = 1; cnt <= max_sends; cnt++) { pkt_desc = &tm_system->egress_pkt_desc; queue_num = pkt_desc->queue_num; if (queue_num == 0) return; tm_queue_obj = tm_system->queue_num_tbl[queue_num]; if (!tm_queue_obj) return; odp_pkt = tm_queue_obj->pkt; if (odp_pkt == ODP_PACKET_INVALID) { tm_system->egress_pkt_desc = EMPTY_PKT_DESC; return; } if (tm_system->marking_enabled) tm_egress_marking(tm_system, odp_pkt); tm_system->egress_pkt_desc = EMPTY_PKT_DESC; if (tm_system->egress.egress_kind == ODP_TM_EGRESS_PKT_IO) odp_pktout_send(tm_system->egress.pktout, &odp_pkt, 1); else if (tm_system->egress.egress_kind == ODP_TM_EGRESS_FN) tm_system->egress.egress_fcn(odp_pkt); else return; tm_queue_obj->sent_pkt = tm_queue_obj->pkt; tm_queue_obj->sent_pkt_desc = tm_queue_obj->in_pkt_desc; tm_queue_obj->pkt = ODP_PACKET_INVALID; tm_queue_obj->in_pkt_desc = EMPTY_PKT_DESC; tm_consume_sent_pkt(tm_system, &tm_queue_obj->sent_pkt_desc); tm_queue_obj->sent_pkt = ODP_PACKET_INVALID; tm_queue_obj->sent_pkt_desc = EMPTY_PKT_DESC; if (tm_system->egress_pkt_desc.queue_num == 0) return; } } static int tm_process_input_work_queue(tm_system_t *tm_system, input_work_queue_t *input_work_queue, uint32_t pkts_to_process) { input_work_item_t work_item; tm_queue_obj_t *tm_queue_obj; tm_shaper_obj_t *shaper_obj; odp_packet_t pkt; pkt_desc_t *pkt_desc; uint32_t cnt; int rc; for (cnt = 1; cnt <= pkts_to_process; cnt++) { rc = input_work_queue_remove(input_work_queue, &work_item); if (rc < 0) { ODP_DBG("%s input_work_queue_remove() failed\n", __func__); return rc; } tm_queue_obj = tm_system->queue_num_tbl[work_item.queue_num]; pkt = work_item.pkt; if (!tm_queue_obj) { odp_packet_free(pkt); return 0; } tm_queue_obj->pkts_rcvd_cnt++; if (tm_queue_obj->pkt != ODP_PACKET_INVALID) { /* If the tm_queue_obj already has a pkt to work with, * then just add this new pkt to the associated * _odp_int_pkt_queue. */ (void)_odp_pkt_queue_append( tm_system->_odp_int_queue_pool, tm_queue_obj->_odp_int_pkt_queue, pkt); tm_queue_obj->pkts_enqueued_cnt++; } else { /* If the tm_queue_obj doesn't have a pkt to work * with, then make this one the head pkt. */ tm_queue_obj->pkt = pkt; tm_pkt_desc_init(&tm_queue_obj->in_pkt_desc, pkt, tm_queue_obj); pkt_desc = &tm_queue_obj->in_pkt_desc; shaper_obj = &tm_queue_obj->shaper_obj; rc = tm_propagate_pkt_desc(tm_system, shaper_obj, pkt_desc, tm_queue_obj->priority); if (0 < rc) return 1; /* Send through spigot */ } } return 0; } static int tm_process_expired_timers(tm_system_t *tm_system, _odp_timer_wheel_t _odp_int_timer_wheel, uint64_t current_time ODP_UNUSED) { tm_shaper_obj_t *shaper_obj; tm_queue_obj_t *tm_queue_obj; pkt_desc_t *pkt_desc; uint64_t timer_context; uint32_t work_done, cnt, queue_num, timer_seq; uint8_t priority; work_done = 0; for (cnt = 1; cnt <= 2; cnt++) { timer_context = _odp_timer_wheel_next_expired(_odp_int_timer_wheel); if (!timer_context) return work_done; queue_num = (timer_context & 0xFFFFFFFF) >> 4; timer_seq = timer_context >> 32; tm_queue_obj = tm_system->queue_num_tbl[queue_num]; if (!tm_queue_obj) return work_done; if ((tm_queue_obj->timer_reason == NO_CALLBACK) || (!tm_queue_obj->timer_shaper) || (tm_queue_obj->timer_seq != timer_seq)) { if (tm_queue_obj->timer_cancels_outstanding != 0) tm_queue_obj->timer_cancels_outstanding--; else ODP_DBG("%s bad timer return\n", __func__); return work_done; } shaper_obj = tm_queue_obj->timer_shaper; pkt_desc = &shaper_obj->in_pkt_desc; priority = shaper_obj->input_priority; delete_timer(tm_system, tm_queue_obj, 0); tm_propagate_pkt_desc(tm_system, shaper_obj, pkt_desc, priority); work_done++; if (tm_system->egress_pkt_desc.queue_num != 0) tm_send_pkt(tm_system, 2); } return work_done; } static volatile uint64_t busy_wait_counter; static odp_bool_t main_loop_running; static odp_atomic_u64_t atomic_request_cnt; static odp_atomic_u64_t currently_serving_cnt; static odp_atomic_u64_t atomic_done_cnt; static void busy_wait(uint32_t iterations) { uint32_t cnt; for (cnt = 1; cnt <= iterations; cnt++) busy_wait_counter++; } static void signal_request(void) { uint64_t my_request_num, serving_cnt; my_request_num = odp_atomic_fetch_inc_u64(&atomic_request_cnt) + 1; serving_cnt = odp_atomic_load_u64(¤tly_serving_cnt); while (serving_cnt != my_request_num) { busy_wait(100); serving_cnt = odp_atomic_load_u64(¤tly_serving_cnt); } } static void check_for_request(void) { uint64_t request_num, serving_cnt, done_cnt; request_num = odp_atomic_load_u64(&atomic_request_cnt); serving_cnt = odp_atomic_load_u64(¤tly_serving_cnt); if (serving_cnt == request_num) return; /* Signal the other requesting thread to proceed and then * wait for their done indication */ odp_atomic_inc_u64(¤tly_serving_cnt); busy_wait(100); done_cnt = odp_atomic_load_u64(&atomic_done_cnt); while (done_cnt != request_num) { busy_wait(100); done_cnt = odp_atomic_load_u64(&atomic_done_cnt); } } static void signal_request_done(void) { odp_atomic_inc_u64(&atomic_done_cnt); } static int thread_affinity_get(odp_cpumask_t *odp_cpu_mask) { cpu_set_t linux_cpu_set; uint32_t cpu_num; int rc; CPU_ZERO(&linux_cpu_set); rc = sched_getaffinity(0, sizeof(cpu_set_t), &linux_cpu_set); if (rc != 0) { ODP_DBG("%s sched_getaffinity failed with rc=%d\n", __func__, rc); return -1; } odp_cpumask_zero(odp_cpu_mask); for (cpu_num = 0; cpu_num < sizeof(cpu_set_t); cpu_num++) if (CPU_ISSET(cpu_num, &linux_cpu_set)) odp_cpumask_set(odp_cpu_mask, cpu_num); return 0; } static void *tm_system_thread(void *arg) { _odp_timer_wheel_t _odp_int_timer_wheel; input_work_queue_t *input_work_queue; tm_system_t *tm_system; uint64_t current_ns; uint32_t destroying, work_queue_cnt, timer_cnt; int rc; rc = odp_init_local(INSTANCE_ID, ODP_THREAD_WORKER); ODP_ASSERT(rc == 0); tm_system = arg; _odp_int_timer_wheel = tm_system->_odp_int_timer_wheel; input_work_queue = tm_system->input_work_queue; /* Wait here until we have seen the first enqueue operation. */ odp_barrier_wait(&tm_system->tm_system_barrier); main_loop_running = true; destroying = odp_atomic_load_u64(&tm_system->destroying); current_ns = odp_time_to_ns(odp_time_local()); _odp_timer_wheel_start(_odp_int_timer_wheel, current_ns); while (destroying == 0) { /* See if another thread wants to make a configuration * change. */ check_for_request(); current_ns = odp_time_to_ns(odp_time_local()); tm_system->current_time = current_ns; rc = _odp_timer_wheel_curr_time_update(_odp_int_timer_wheel, current_ns); if (0 < rc) { /* Process a batch of expired timers - each of which * could cause a pkt to egress the tm system. */ timer_cnt = 1; (void)tm_process_expired_timers(tm_system, _odp_int_timer_wheel, current_ns); } else { timer_cnt = _odp_timer_wheel_count(_odp_int_timer_wheel); } current_ns = odp_time_to_ns(odp_time_local()); tm_system->current_time = current_ns; work_queue_cnt = odp_atomic_load_u64(&input_work_queue->queue_cnt); if (work_queue_cnt != 0) { tm_process_input_work_queue(tm_system, input_work_queue, 1); } if (tm_system->egress_pkt_desc.queue_num != 0) tm_send_pkt(tm_system, 1); current_ns = odp_time_to_ns(odp_time_local()); tm_system->current_time = current_ns; tm_system->is_idle = (timer_cnt == 0) && (work_queue_cnt == 0); destroying = odp_atomic_load_u64(&tm_system->destroying); } odp_barrier_wait(&tm_system->tm_system_destroy_barrier); odp_term_local(); return NULL; } odp_bool_t odp_tm_is_idle(odp_tm_t odp_tm) { tm_system_t *tm_system; tm_system = GET_TM_SYSTEM(odp_tm); return tm_system->is_idle; } void odp_tm_requirements_init(odp_tm_requirements_t *requirements) { memset(requirements, 0, sizeof(odp_tm_requirements_t)); } void odp_tm_egress_init(odp_tm_egress_t *egress) { memset(egress, 0, sizeof(odp_tm_egress_t)); } static tm_node_obj_t *create_dummy_root_node(void) { tm_node_obj_t *tm_node_obj; tm_node_obj = malloc(sizeof(tm_node_obj_t)); if (!tm_node_obj) return NULL; memset(tm_node_obj, 0, sizeof(tm_node_obj_t)); tm_node_obj->is_root_node = true; return tm_node_obj; } int odp_tm_capabilities(odp_tm_capabilities_t capabilities[] ODP_UNUSED, uint32_t capabilities_size) { odp_tm_level_capabilities_t *per_level_cap; odp_tm_capabilities_t *cap_ptr; odp_packet_color_t color; uint32_t level_idx; if (capabilities_size == 0) return -1; cap_ptr = &capabilities[0]; memset(cap_ptr, 0, sizeof(odp_tm_capabilities_t)); cap_ptr->max_tm_queues = ODP_TM_MAX_TM_QUEUES; cap_ptr->max_levels = ODP_TM_MAX_LEVELS; cap_ptr->tm_queue_shaper_supported = true; cap_ptr->tm_queue_wred_supported = true; cap_ptr->tm_queue_dual_slope_supported = true; cap_ptr->vlan_marking_supported = true; cap_ptr->ecn_marking_supported = true; cap_ptr->drop_prec_marking_supported = true; for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) cap_ptr->marking_colors_supported[color] = true; for (level_idx = 0; level_idx < cap_ptr->max_levels; level_idx++) { per_level_cap = &cap_ptr->per_level[level_idx]; per_level_cap->max_num_tm_nodes = ODP_TM_MAX_NUM_TM_NODES; per_level_cap->max_fanin_per_node = ODP_TM_MAX_TM_NODE_FANIN; per_level_cap->max_priority = ODP_TM_MAX_PRIORITIES - 1; per_level_cap->min_weight = ODP_TM_MIN_SCHED_WEIGHT; per_level_cap->max_weight = ODP_TM_MAX_SCHED_WEIGHT; per_level_cap->tm_node_shaper_supported = true; per_level_cap->tm_node_wred_supported = true; per_level_cap->tm_node_dual_slope_supported = true; per_level_cap->fair_queuing_supported = true; per_level_cap->weights_supported = true; } return 1; } static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr, odp_tm_requirements_t *req_ptr) { odp_tm_level_requirements_t *per_level_req; odp_tm_level_capabilities_t *per_level_cap; odp_packet_color_t color; odp_bool_t shaper_supported, wred_supported; odp_bool_t dual_slope; uint32_t num_levels, level_idx, max_nodes; uint32_t max_queues, max_fanin; uint8_t max_priority, min_weight, max_weight; num_levels = MAX(MIN(req_ptr->num_levels, ODP_TM_MAX_LEVELS), 1); memset(cap_ptr, 0, sizeof(odp_tm_capabilities_t)); max_queues = MIN(req_ptr->max_tm_queues, ODP_TM_MAX_NUM_TM_NODES); shaper_supported = req_ptr->tm_queue_shaper_needed; wred_supported = req_ptr->tm_queue_wred_needed; dual_slope = req_ptr->tm_queue_dual_slope_needed; cap_ptr->max_tm_queues = max_queues; cap_ptr->max_levels = num_levels; cap_ptr->tm_queue_shaper_supported = shaper_supported; cap_ptr->tm_queue_wred_supported = wred_supported; cap_ptr->tm_queue_dual_slope_supported = dual_slope; cap_ptr->vlan_marking_supported = req_ptr->vlan_marking_needed; cap_ptr->ecn_marking_supported = req_ptr->ecn_marking_needed; cap_ptr->drop_prec_marking_supported = req_ptr->drop_prec_marking_needed; for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) cap_ptr->marking_colors_supported[color] = req_ptr->marking_colors_needed[color]; for (level_idx = 0; level_idx < num_levels; level_idx++) { per_level_cap = &cap_ptr->per_level[level_idx]; per_level_req = &req_ptr->per_level[level_idx]; max_nodes = MIN(per_level_req->max_num_tm_nodes, ODP_TM_MAX_NUM_TM_NODES); max_fanin = MIN(per_level_req->max_fanin_per_node, 1024); max_priority = MIN(per_level_req->max_priority, ODP_TM_MAX_PRIORITIES - 1); min_weight = MAX(per_level_req->min_weight, ODP_TM_MIN_SCHED_WEIGHT); max_weight = MIN(per_level_req->max_weight, ODP_TM_MAX_SCHED_WEIGHT); shaper_supported = per_level_req->tm_node_shaper_needed; wred_supported = per_level_req->tm_node_wred_needed; dual_slope = per_level_req->tm_node_dual_slope_needed; per_level_cap->max_num_tm_nodes = max_nodes; per_level_cap->max_fanin_per_node = max_fanin; per_level_cap->max_priority = max_priority; per_level_cap->min_weight = min_weight; per_level_cap->max_weight = max_weight; per_level_cap->tm_node_shaper_supported = shaper_supported; per_level_cap->tm_node_wred_supported = wred_supported; per_level_cap->tm_node_dual_slope_supported = dual_slope; per_level_cap->fair_queuing_supported = true; per_level_cap->weights_supported = true; } } static int affinitize_main_thread(void) { odp_cpumask_t odp_cpu_mask; cpu_set_t linux_cpu_set; uint32_t cpu_count, cpu_num; int rc; rc = thread_affinity_get(&odp_cpu_mask); if (rc != 0) return rc; /* If the affinity cpu set returned above has exactly one cpu, then * just record this value and return. */ cpu_count = odp_cpumask_count(&odp_cpu_mask); if (cpu_count == 1) { g_main_thread_cpu = odp_cpumask_first(&odp_cpu_mask); return 0; } else if (cpu_count == 0) { return -1; } cpu_num = odp_cpumask_first(&odp_cpu_mask); CPU_ZERO(&linux_cpu_set); CPU_SET(cpu_num, &linux_cpu_set); rc = sched_setaffinity(0, sizeof(cpu_set_t), &linux_cpu_set); if (rc == 0) g_main_thread_cpu = cpu_num; else ODP_DBG("%s sched_setaffinity failed with rc=%d\n", __func__, rc); return rc; } static uint32_t tm_thread_cpu_select(void) { odp_cpumask_t odp_cpu_mask; int cpu_count; odp_cpumask_default_worker(&odp_cpu_mask, 0); if ((g_main_thread_cpu != -1) && odp_cpumask_isset(&odp_cpu_mask, g_main_thread_cpu)) odp_cpumask_clr(&odp_cpu_mask, g_main_thread_cpu); cpu_count = odp_cpumask_count(&odp_cpu_mask); if (cpu_count < 1) { odp_cpumask_all_available(&odp_cpu_mask); if ((g_main_thread_cpu != -1) && odp_cpumask_isset(&odp_cpu_mask, g_main_thread_cpu)) cpu_count = odp_cpumask_count(&odp_cpu_mask); if (cpu_count < 1) odp_cpumask_all_available(&odp_cpu_mask); } return odp_cpumask_first(&odp_cpu_mask); } static int tm_thread_create(tm_system_t *tm_system) { cpu_set_t cpu_set; uint32_t cpu_num; int rc; pthread_attr_init(&tm_system->attr); cpu_num = tm_thread_cpu_select(); CPU_ZERO(&cpu_set); CPU_SET(cpu_num, &cpu_set); pthread_attr_setaffinity_np(&tm_system->attr, sizeof(cpu_set_t), &cpu_set); rc = pthread_create(&tm_system->thread, &tm_system->attr, tm_system_thread, tm_system); if (rc != 0) ODP_DBG("Failed to start thread on cpu num=%u\n", cpu_num); return rc; } odp_tm_t odp_tm_create(const char *name, odp_tm_requirements_t *requirements, odp_tm_egress_t *egress) { _odp_int_name_t name_tbl_id; tm_system_t *tm_system; odp_bool_t create_fail; odp_tm_t odp_tm; uint32_t malloc_len, max_num_queues, max_queued_pkts, max_timers; uint32_t max_tm_queues, max_sorted_lists; int rc; /* Allocate tm_system_t record. */ odp_ticketlock_lock(&tm_create_lock); tm_system = tm_system_alloc(); if (!tm_system) { odp_ticketlock_unlock(&tm_create_lock); return ODP_TM_INVALID; } odp_tm = MAKE_ODP_TM_HANDLE(tm_system); name_tbl_id = _odp_int_name_tbl_add(name, ODP_TM_HANDLE, odp_tm); if (name_tbl_id == ODP_INVALID_NAME) { tm_system_free(tm_system); odp_ticketlock_unlock(&tm_create_lock); return ODP_TM_INVALID; } tm_system->name_tbl_id = name_tbl_id; max_tm_queues = requirements->max_tm_queues; memcpy(&tm_system->egress, egress, sizeof(odp_tm_egress_t)); memcpy(&tm_system->requirements, requirements, sizeof(odp_tm_requirements_t)); tm_system_capabilities_set(&tm_system->capabilities, &tm_system->requirements); malloc_len = max_tm_queues * sizeof(tm_queue_obj_t *); tm_system->queue_num_tbl = malloc(malloc_len); memset(tm_system->queue_num_tbl, 0, malloc_len); tm_system->next_queue_num = 1; tm_init_random_data(&tm_system->tm_random_data); max_sorted_lists = 2 * max_tm_queues; max_num_queues = max_tm_queues; max_queued_pkts = 16 * max_tm_queues; max_timers = 2 * max_tm_queues; create_fail = 0; tm_system->_odp_int_sorted_pool = _ODP_INT_SORTED_POOL_INVALID; tm_system->_odp_int_queue_pool = _ODP_INT_QUEUE_POOL_INVALID; tm_system->_odp_int_timer_wheel = _ODP_INT_TIMER_WHEEL_INVALID; odp_ticketlock_init(&tm_system->tm_system_lock); odp_barrier_init(&tm_system->tm_system_barrier, 2); odp_atomic_init_u64(&tm_system->destroying, 0); tm_system->_odp_int_sorted_pool = _odp_sorted_pool_create( max_sorted_lists); create_fail |= tm_system->_odp_int_sorted_pool == _ODP_INT_SORTED_POOL_INVALID; if (create_fail == 0) { tm_system->_odp_int_queue_pool = _odp_queue_pool_create( max_num_queues, max_queued_pkts); create_fail |= tm_system->_odp_int_queue_pool == _ODP_INT_QUEUE_POOL_INVALID; } if (create_fail == 0) { tm_system->_odp_int_timer_wheel = _odp_timer_wheel_create( max_timers, tm_system); create_fail |= tm_system->_odp_int_timer_wheel == _ODP_INT_TIMER_WHEEL_INVALID; } if (create_fail == 0) { tm_system->root_node = create_dummy_root_node(); create_fail |= tm_system->root_node == NULL; } if (create_fail == 0) { tm_system->input_work_queue = input_work_queue_create(); create_fail |= !tm_system->input_work_queue; } if (create_fail == 0) { affinitize_main_thread(); rc = tm_thread_create(tm_system); create_fail |= rc < 0; } if (create_fail) { _odp_int_name_tbl_delete(name_tbl_id); if (tm_system->input_work_queue) input_work_queue_destroy(tm_system->input_work_queue); if (tm_system->_odp_int_sorted_pool != _ODP_INT_SORTED_POOL_INVALID) _odp_sorted_pool_destroy( tm_system->_odp_int_sorted_pool); if (tm_system->_odp_int_queue_pool != _ODP_INT_QUEUE_POOL_INVALID) _odp_queue_pool_destroy( tm_system->_odp_int_queue_pool); if (tm_system->_odp_int_timer_wheel != _ODP_INT_TIMER_WHEEL_INVALID) _odp_timer_wheel_destroy( tm_system->_odp_int_timer_wheel); tm_system_free(tm_system); odp_ticketlock_unlock(&tm_create_lock); return ODP_TM_INVALID; } odp_ticketlock_unlock(&tm_create_lock); return odp_tm; } odp_tm_t odp_tm_find(const char *name, odp_tm_requirements_t *requirements ODP_UNUSED, odp_tm_egress_t *egress ODP_UNUSED) { _odp_int_name_t odp_name; uint64_t user_data; /* Currently does not consider the requirements in the lookup - * just the name */ odp_name = _odp_int_name_tbl_lookup(name, ODP_TM_HANDLE); user_data = _odp_int_name_tbl_user_data(odp_name); return (odp_tm_t)user_data; } int odp_tm_capability(odp_tm_t odp_tm, odp_tm_capabilities_t *capabilities) { tm_system_t *tm_system; tm_system = GET_TM_SYSTEM(odp_tm); memcpy(capabilities, &tm_system->capabilities, sizeof(odp_tm_capabilities_t)); return 0; } int odp_tm_destroy(odp_tm_t odp_tm) { tm_system_t *tm_system; int rc; tm_system = GET_TM_SYSTEM(odp_tm); /* First mark the tm_system as being in the destroying state so that * all new pkts are prevented from coming in. */ odp_barrier_init(&tm_system->tm_system_destroy_barrier, 2); odp_atomic_inc_u64(&tm_system->destroying); odp_barrier_wait(&tm_system->tm_system_destroy_barrier); /* Next wait for the thread to exit. */ rc = pthread_join(tm_system->thread, NULL); ODP_ASSERT(rc == 0); pthread_attr_destroy(&tm_system->attr); input_work_queue_destroy(tm_system->input_work_queue); _odp_sorted_pool_destroy(tm_system->_odp_int_sorted_pool); _odp_queue_pool_destroy(tm_system->_odp_int_queue_pool); _odp_timer_wheel_destroy(tm_system->_odp_int_timer_wheel); tm_system_free(tm_system); return 0; } /* Return true if any of the marking enables is set, otherwise return FALSE */ static odp_bool_t tm_marking_enabled(tm_system_t *tm_system) { odp_packet_color_t color; tm_marking_t *tm_marking; tm_marking = &tm_system->marking; for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) { if ((tm_marking->vlan_marking[color].marking_enabled) || (tm_marking->ip_tos_marking[color].marking_enabled)) return true; } return false; } int odp_tm_vlan_marking(odp_tm_t odp_tm, odp_packet_color_t color, odp_bool_t drop_eligible_enabled) { tm_vlan_marking_t *vlan_marking; tm_system_t *tm_system; tm_system = GET_TM_SYSTEM(odp_tm); if ((tm_system == NULL) || (ODP_NUM_PACKET_COLORS <= color)) return -1; if (drop_eligible_enabled) if ((!tm_system->requirements.vlan_marking_needed) || (!tm_system->requirements.marking_colors_needed[color])) return -1; vlan_marking = &tm_system->marking.vlan_marking[color]; vlan_marking->marking_enabled = drop_eligible_enabled; vlan_marking->drop_eligible_enabled = drop_eligible_enabled; if (vlan_marking->marking_enabled) tm_system->marking_enabled = true; else tm_system->marking_enabled = tm_marking_enabled(tm_system); return 0; } int odp_tm_ecn_marking(odp_tm_t odp_tm, odp_packet_color_t color, odp_bool_t ecn_ce_enabled) { tm_tos_marking_t *tos_marking; tm_system_t *tm_system; tm_system = GET_TM_SYSTEM(odp_tm); if ((tm_system == NULL) || (ODP_NUM_PACKET_COLORS <= color)) return -1; if (ecn_ce_enabled) if ((!tm_system->requirements.ecn_marking_needed) || (!tm_system->requirements.marking_colors_needed[color])) return -1; tos_marking = &tm_system->marking.ip_tos_marking[color]; tos_marking->marking_enabled = tos_marking->drop_prec_enabled | ecn_ce_enabled; tos_marking->ecn_ce_enabled = ecn_ce_enabled; if (ecn_ce_enabled) tm_system->marking_enabled = true; else tm_system->marking_enabled = tm_marking_enabled(tm_system); return 0; } int odp_tm_drop_prec_marking(odp_tm_t odp_tm, odp_packet_color_t color, odp_bool_t drop_prec_enabled) { tm_tos_marking_t *tos_marking; tm_system_t *tm_system; uint8_t dscp_mask, new_dscp, inverted_mask, tos_mask; uint8_t shifted_dscp; tm_system = GET_TM_SYSTEM(odp_tm); if ((tm_system == NULL) || (ODP_NUM_PACKET_COLORS <= color)) return -1; if (drop_prec_enabled) if ((!tm_system->requirements.drop_prec_marking_needed) || (!tm_system->requirements.marking_colors_needed[color])) return -1; dscp_mask = DROP_PRECEDENCE_MASK; if (color == ODP_PACKET_YELLOW) new_dscp = MEDIUM_DROP_PRECEDENCE; else if (color == ODP_PACKET_RED) new_dscp = HIGH_DROP_PRECEDENCE; else new_dscp = LOW_DROP_PRECEDENCE; if (drop_prec_enabled) { new_dscp = new_dscp & dscp_mask; inverted_mask = (uint8_t)~dscp_mask; tos_mask = (inverted_mask << _ODP_IP_TOS_DSCP_SHIFT) | _ODP_IP_TOS_ECN_MASK; shifted_dscp = new_dscp << _ODP_IP_TOS_DSCP_SHIFT; } else { tos_mask = 0xFF; /* Note that this is an inverted mask */ shifted_dscp = 0; } tos_marking = &tm_system->marking.ip_tos_marking[color]; tos_marking->marking_enabled = drop_prec_enabled | tos_marking->ecn_ce_enabled; tos_marking->drop_prec_enabled = drop_prec_enabled; tos_marking->shifted_dscp = shifted_dscp; tos_marking->inverted_dscp_mask = tos_mask; if (drop_prec_enabled) tm_system->marking_enabled = true; else tm_system->marking_enabled = tm_marking_enabled(tm_system); return 0; } void odp_tm_shaper_params_init(odp_tm_shaper_params_t *params) { memset(params, 0, sizeof(odp_tm_shaper_params_t)); } odp_tm_shaper_t odp_tm_shaper_create(const char *name, odp_tm_shaper_params_t *params) { tm_shaper_params_t *profile_obj; odp_tm_shaper_t shaper_handle; _odp_int_name_t name_tbl_id; profile_obj = tm_common_profile_create(name, TM_SHAPER_PROFILE, sizeof(tm_shaper_params_t), &shaper_handle, &name_tbl_id); if (!profile_obj) return ODP_TM_INVALID; tm_shaper_params_cvt_to(params, profile_obj); profile_obj->name_tbl_id = name_tbl_id; profile_obj->shaper_profile = shaper_handle; return shaper_handle; } int odp_tm_shaper_destroy(odp_tm_shaper_t shaper_profile) { tm_shaper_params_t *profile_obj; if (shaper_profile == ODP_TM_INVALID) return -1; profile_obj = tm_get_profile_params(shaper_profile, TM_SHAPER_PROFILE); if (!profile_obj) return -1; if (profile_obj->ref_cnt != 0) return -1; return tm_common_profile_destroy(shaper_profile, sizeof(tm_shaper_params_t), profile_obj->name_tbl_id); } int odp_tm_shaper_params_read(odp_tm_shaper_t shaper_profile, odp_tm_shaper_params_t *params) { tm_shaper_params_t *profile_obj; if (shaper_profile == ODP_TM_INVALID) return -1; profile_obj = tm_get_profile_params(shaper_profile, TM_SHAPER_PROFILE); if (!profile_obj) return -1; tm_shaper_params_cvt_from(profile_obj, params); return 0; } int odp_tm_shaper_params_update(odp_tm_shaper_t shaper_profile, odp_tm_shaper_params_t *params) { tm_shaper_params_t *profile_obj; if (shaper_profile == ODP_TM_INVALID) return -1; profile_obj = tm_get_profile_params(shaper_profile, TM_SHAPER_PROFILE); if (!profile_obj) return -1; if (!main_loop_running) { tm_shaper_params_cvt_to(params, profile_obj); return 0; } signal_request(); tm_shaper_params_cvt_to(params, profile_obj); signal_request_done(); return 0; } odp_tm_shaper_t odp_tm_shaper_lookup(const char *name) { _odp_int_name_t odp_name; uint64_t user_data; odp_name = _odp_int_name_tbl_lookup(name, ODP_TM_SHAPER_PROFILE_HANDLE); user_data = _odp_int_name_tbl_user_data(odp_name); return (odp_tm_shaper_t)user_data; } void odp_tm_sched_params_init(odp_tm_sched_params_t *params) { memset(params, 0, sizeof(odp_tm_sched_params_t)); } static void tm_sched_params_cvt_to(odp_tm_sched_params_t *odp_sched_params, tm_sched_params_t *tm_sched_params) { odp_tm_sched_mode_t sched_mode; uint32_t priority, weight, inv_weight; for (priority = 0; priority < ODP_TM_MAX_PRIORITIES; priority++) { sched_mode = odp_sched_params->sched_modes[priority]; weight = odp_sched_params->sched_weights[priority]; if (weight == 0) inv_weight = 0; else inv_weight = 0x10000 / weight; tm_sched_params->sched_modes[priority] = sched_mode; tm_sched_params->inverted_weights[priority] = inv_weight; } } static void tm_sched_params_cvt_from(tm_sched_params_t *tm_sched_params, odp_tm_sched_params_t *odp_sched_params) { odp_tm_sched_mode_t sched_mode; uint32_t priority, weight, inv_weight; for (priority = 0; priority < ODP_TM_MAX_PRIORITIES; priority++) { sched_mode = tm_sched_params->sched_modes[priority]; inv_weight = tm_sched_params->inverted_weights[priority]; weight = 0x10000 / inv_weight; odp_sched_params->sched_modes[priority] = sched_mode; odp_sched_params->sched_weights[priority] = weight; } } odp_tm_sched_t odp_tm_sched_create(const char *name, odp_tm_sched_params_t *params) { tm_sched_params_t *profile_obj; _odp_int_name_t name_tbl_id; odp_tm_sched_t sched_handle; profile_obj = tm_common_profile_create(name, TM_SCHED_PROFILE, sizeof(tm_sched_params_t), &sched_handle, &name_tbl_id); if (!profile_obj) return ODP_TM_INVALID; tm_sched_params_cvt_to(params, profile_obj); profile_obj->name_tbl_id = name_tbl_id; profile_obj->sched_profile = sched_handle; return sched_handle; } int odp_tm_sched_destroy(odp_tm_sched_t sched_profile) { tm_sched_params_t *profile_obj; if (sched_profile == ODP_TM_INVALID) return -1; profile_obj = tm_get_profile_params(sched_profile, TM_SCHED_PROFILE); if (!profile_obj) return -1; if (profile_obj->ref_cnt != 0) return -1; return tm_common_profile_destroy(sched_profile, sizeof(tm_sched_params_t), profile_obj->name_tbl_id); } int odp_tm_sched_params_read(odp_tm_sched_t sched_profile, odp_tm_sched_params_t *params) { tm_sched_params_t *profile_obj; if (sched_profile == ODP_TM_INVALID) return -1; profile_obj = tm_get_profile_params(sched_profile, TM_SCHED_PROFILE); if (!profile_obj) return -1; tm_sched_params_cvt_from(profile_obj, params); return 0; } int odp_tm_sched_params_update(odp_tm_sched_t sched_profile, odp_tm_sched_params_t *params) { tm_sched_params_t *profile_obj; if (sched_profile == ODP_TM_INVALID) return -1; profile_obj = tm_get_profile_params(sched_profile, TM_SCHED_PROFILE); if (!profile_obj) return -1; if (!main_loop_running) { tm_sched_params_cvt_to(params, profile_obj); return 0; } signal_request(); tm_sched_params_cvt_to(params, profile_obj); signal_request_done(); return 0; } odp_tm_sched_t odp_tm_sched_lookup(const char *name) { _odp_int_name_t odp_name; uint64_t user_data; odp_name = _odp_int_name_tbl_lookup(name, ODP_TM_SCHED_PROFILE_HANDLE); user_data = _odp_int_name_tbl_user_data(odp_name); return (odp_tm_sched_t)user_data; } void odp_tm_threshold_params_init(odp_tm_threshold_params_t *params) { memset(params, 0, sizeof(odp_tm_threshold_params_t)); } odp_tm_threshold_t odp_tm_threshold_create(const char *name, odp_tm_threshold_params_t *params) { tm_queue_thresholds_t *profile_obj; odp_tm_threshold_t threshold_handle; _odp_int_name_t name_tbl_id; profile_obj = tm_common_profile_create(name, TM_THRESHOLD_PROFILE, sizeof(tm_queue_thresholds_t), &threshold_handle, &name_tbl_id); if (!profile_obj) return ODP_TM_INVALID; profile_obj->max_pkts = params->enable_max_pkts ? params->max_pkts : 0; profile_obj->max_bytes = params->enable_max_bytes ? params->max_bytes : 0; profile_obj->name_tbl_id = name_tbl_id; profile_obj->thresholds_profile = threshold_handle; return threshold_handle; } int odp_tm_threshold_destroy(odp_tm_threshold_t threshold_profile) { tm_queue_thresholds_t *threshold_params; if (threshold_profile == ODP_TM_INVALID) return -1; threshold_params = tm_get_profile_params(threshold_profile, TM_THRESHOLD_PROFILE); if (!threshold_params) return -1; if (threshold_params->ref_cnt != 0) return -1; return tm_common_profile_destroy(threshold_profile, sizeof(odp_tm_threshold_params_t), threshold_params->name_tbl_id); } int odp_tm_thresholds_params_read(odp_tm_threshold_t threshold_profile, odp_tm_threshold_params_t *params) { tm_queue_thresholds_t *threshold_params; if (threshold_profile == ODP_TM_INVALID) return -1; threshold_params = tm_get_profile_params(threshold_profile, TM_THRESHOLD_PROFILE); if (!threshold_params) return -1; params->max_pkts = threshold_params->max_pkts; params->max_bytes = threshold_params->max_bytes; params->enable_max_pkts = threshold_params->max_pkts != 0; params->enable_max_bytes = threshold_params->max_bytes != 0; return 0; } int odp_tm_thresholds_params_update(odp_tm_threshold_t threshold_profile, odp_tm_threshold_params_t *params) { tm_queue_thresholds_t *profile_obj; if (threshold_profile == ODP_TM_INVALID) return -1; profile_obj = tm_get_profile_params(threshold_profile, TM_THRESHOLD_PROFILE); if (!profile_obj) return -1; if (!main_loop_running) { profile_obj->max_pkts = params->enable_max_pkts ? params->max_pkts : 0; profile_obj->max_bytes = params->enable_max_bytes ? params->max_bytes : 0; return 0; } signal_request(); profile_obj->max_pkts = params->enable_max_pkts ? params->max_pkts : 0; profile_obj->max_bytes = params->enable_max_bytes ? params->max_bytes : 0; signal_request_done(); return 0; } odp_tm_threshold_t odp_tm_thresholds_lookup(const char *name) { _odp_int_name_t odp_name; uint64_t user_data; odp_name = _odp_int_name_tbl_lookup(name, ODP_TM_THRESHOLD_PROFILE_HANDLE); user_data = _odp_int_name_tbl_user_data(odp_name); return (odp_tm_threshold_t)user_data; } void odp_tm_wred_params_init(odp_tm_wred_params_t *params) { memset(params, 0, sizeof(odp_tm_wred_params_t)); } static void tm_wred_params_cvt_to(odp_tm_wred_params_t *odp_tm_wred_params, tm_wred_params_t *wred_params) { wred_params->min_threshold = odp_tm_wred_params->min_threshold; wred_params->med_threshold = odp_tm_wred_params->med_threshold; wred_params->med_drop_prob = odp_tm_wred_params->med_drop_prob; wred_params->max_drop_prob = odp_tm_wred_params->max_drop_prob; wred_params->enable_wred = odp_tm_wred_params->enable_wred; wred_params->use_byte_fullness = odp_tm_wred_params->use_byte_fullness; } static void tm_wred_params_cvt_from(tm_wred_params_t *wred_params, odp_tm_wred_params_t *odp_tm_wred_params) { odp_tm_wred_params->min_threshold = wred_params->min_threshold; odp_tm_wred_params->med_threshold = wred_params->med_threshold; odp_tm_wred_params->med_drop_prob = wred_params->med_drop_prob; odp_tm_wred_params->max_drop_prob = wred_params->max_drop_prob; odp_tm_wred_params->enable_wred = wred_params->enable_wred; odp_tm_wred_params->use_byte_fullness = wred_params->use_byte_fullness; } odp_tm_wred_t odp_tm_wred_create(const char *name, odp_tm_wred_params_t *params) { tm_wred_params_t *profile_obj; odp_tm_wred_t wred_handle; _odp_int_name_t name_tbl_id; profile_obj = tm_common_profile_create(name, TM_WRED_PROFILE, sizeof(tm_wred_params_t), &wred_handle, &name_tbl_id); if (!profile_obj) return ODP_TM_INVALID; tm_wred_params_cvt_to(params, profile_obj); profile_obj->name_tbl_id = name_tbl_id; profile_obj->wred_profile = wred_handle; return wred_handle; } int odp_tm_wred_destroy(odp_tm_wred_t wred_profile) { tm_wred_params_t *wred_params; if (wred_profile == ODP_TM_INVALID) return -1; wred_params = tm_get_profile_params(wred_profile, TM_WRED_PROFILE); if (!wred_params) return -1; if (wred_params->ref_cnt != 0) return -1; return tm_common_profile_destroy(wred_profile, sizeof(tm_wred_params_t), ODP_INVALID_NAME); } int odp_tm_wred_params_read(odp_tm_wred_t wred_profile, odp_tm_wred_params_t *params) { tm_wred_params_t *wred_params; if (wred_profile == ODP_TM_INVALID) return -1; wred_params = tm_get_profile_params(wred_profile, TM_WRED_PROFILE); if (!wred_params) return -1; tm_wred_params_cvt_from(wred_params, params); return 0; } int odp_tm_wred_params_update(odp_tm_wred_t wred_profile, odp_tm_wred_params_t *params) { tm_wred_params_t *wred_params; if (wred_profile == ODP_TM_INVALID) return -1; wred_params = tm_get_profile_params(wred_profile, TM_WRED_PROFILE); if (!wred_params) return -1; if (!main_loop_running) { tm_wred_params_cvt_to(params, wred_params); return 0; } signal_request(); tm_wred_params_cvt_to(params, wred_params); signal_request_done(); return 0; } odp_tm_wred_t odp_tm_wred_lookup(const char *name) { _odp_int_name_t odp_name; uint64_t user_data; odp_name = _odp_int_name_tbl_lookup(name, ODP_TM_WRED_PROFILE_HANDLE); user_data = _odp_int_name_tbl_user_data(odp_name); return (odp_tm_wred_t)user_data; } void odp_tm_node_params_init(odp_tm_node_params_t *params) { memset(params, 0, sizeof(odp_tm_node_params_t)); } odp_tm_node_t odp_tm_node_create(odp_tm_t odp_tm, const char *name, odp_tm_node_params_t *params) { odp_tm_level_requirements_t *requirements; _odp_int_sorted_list_t sorted_list; tm_schedulers_obj_t *schedulers_obj; _odp_int_name_t name_tbl_id; tm_wred_node_t *tm_wred_node; tm_node_obj_t *tm_node_obj; odp_tm_node_t odp_tm_node; odp_tm_wred_t wred_profile; tm_system_t *tm_system; uint32_t level, num_priorities, priority, schedulers_obj_len, color; /* Allocate a tm_node_obj_t record. */ tm_system = GET_TM_SYSTEM(odp_tm); tm_node_obj = malloc(sizeof(tm_node_obj_t)); if (!tm_node_obj) return ODP_TM_INVALID; tm_wred_node = malloc(sizeof(tm_wred_node_t)); if (!tm_wred_node) { free(tm_node_obj); return ODP_TM_INVALID; } level = params->level; requirements = &tm_system->requirements.per_level[level]; num_priorities = requirements->max_priority + 1; schedulers_obj_len = sizeof(tm_schedulers_obj_t) + (sizeof(tm_sched_state_t) * num_priorities); schedulers_obj = malloc(schedulers_obj_len); if (!schedulers_obj) { free(tm_wred_node); free(tm_node_obj); return ODP_TM_INVALID; } memset(schedulers_obj, 0, schedulers_obj_len); odp_tm_node = MAKE_ODP_TM_NODE(tm_node_obj); name_tbl_id = ODP_INVALID_NAME; if ((name) && (name[0] != '\0')) { name_tbl_id = _odp_int_name_tbl_add(name, ODP_TM_NODE_HANDLE, odp_tm_node); if (name_tbl_id == ODP_INVALID_NAME) { free(schedulers_obj); free(tm_wred_node); free(tm_node_obj); return ODP_TM_INVALID; } } memset(tm_node_obj, 0, sizeof(tm_node_obj_t)); memset(tm_wred_node, 0, sizeof(tm_wred_node_t)); memset(schedulers_obj, 0, schedulers_obj_len); tm_node_obj->user_context = params->user_context; tm_node_obj->name_tbl_id = name_tbl_id; tm_node_obj->max_fanin = params->max_fanin; tm_node_obj->is_root_node = false; tm_node_obj->level = params->level; tm_node_obj->tm_idx = tm_system->tm_idx; tm_node_obj->tm_wred_node = tm_wred_node; tm_node_obj->schedulers_obj = schedulers_obj; odp_ticketlock_init(&tm_wred_node->tm_wred_node_lock); schedulers_obj->num_priorities = num_priorities; for (priority = 0; priority < num_priorities; priority++) { sorted_list = _odp_sorted_list_create( tm_system->_odp_int_sorted_pool, params->max_fanin); schedulers_obj->sched_states[priority].sorted_list = sorted_list; } odp_ticketlock_lock(&tm_system->tm_system_lock); if (params->shaper_profile != ODP_TM_INVALID) tm_shaper_config_set(tm_system, params->shaper_profile, &tm_node_obj->shaper_obj); if (params->threshold_profile != ODP_TM_INVALID) tm_threshold_config_set(tm_wred_node, params->threshold_profile); for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) { wred_profile = params->wred_profile[color]; if (wred_profile != ODP_TM_INVALID) tm_wred_config_set(tm_wred_node, color, wred_profile); } tm_node_obj->magic_num = TM_NODE_MAGIC_NUM; tm_node_obj->shaper_obj.enclosing_entity = tm_node_obj; tm_node_obj->shaper_obj.in_tm_node_obj = 1; tm_node_obj->schedulers_obj->enclosing_entity = tm_node_obj; odp_ticketlock_unlock(&tm_system->tm_system_lock); return odp_tm_node; } int odp_tm_node_destroy(odp_tm_node_t tm_node) { _odp_int_sorted_list_t sorted_list; _odp_int_sorted_pool_t sorted_pool; tm_schedulers_obj_t *schedulers_obj; tm_sched_state_t *sched_state; tm_wred_params_t *wred_params; tm_shaper_obj_t *shaper_obj; tm_wred_node_t *tm_wred_node; tm_node_obj_t *tm_node_obj; tm_system_t *tm_system; uint32_t color, num_priorities, priority; int rc; /* First lookup tm_node. */ tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (!tm_node_obj) return -1; tm_system = odp_tm_systems[tm_node_obj->tm_idx]; if (!tm_system) return -1; /* Next make sure node_obj is disconnected and has no fanin. */ shaper_obj = &tm_node_obj->shaper_obj; if (shaper_obj->next_tm_node != NULL) return -1; if ((tm_node_obj->current_tm_queue_fanin != 0) || (tm_node_obj->current_tm_node_fanin != 0)) return -1; /* Check that there is no shaper profile, threshold profile or wred * profile currently associated with this tm_node. */ if (shaper_obj->shaper_params != NULL) return -1; tm_wred_node = tm_node_obj->tm_wred_node; if (tm_wred_node != NULL) { if (tm_wred_node->threshold_params != NULL) return -1; for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) { wred_params = tm_wred_node->wred_params[color]; if (wred_params != NULL) return -1; } } /* Now that all of the checks are done, time to so some freeing. */ odp_ticketlock_lock(&tm_system->tm_system_lock); if (tm_node_obj->name_tbl_id != ODP_INVALID_NAME) _odp_int_name_tbl_delete(tm_node_obj->name_tbl_id); if (tm_node_obj->tm_wred_node != NULL) free(tm_node_obj->tm_wred_node); schedulers_obj = tm_node_obj->schedulers_obj; if (schedulers_obj != NULL) { num_priorities = schedulers_obj->num_priorities; for (priority = 0; priority < num_priorities; priority++) { sched_state = &schedulers_obj->sched_states[priority]; sorted_list = sched_state->sorted_list; sorted_pool = tm_system->_odp_int_sorted_pool; rc = _odp_sorted_list_destroy(sorted_pool, sorted_list); if (rc != 0) return rc; } } free(schedulers_obj); free(tm_node_obj); odp_ticketlock_unlock(&tm_system->tm_system_lock); return 0; } int odp_tm_node_shaper_config(odp_tm_node_t tm_node, odp_tm_shaper_t shaper_profile) { tm_node_obj_t *tm_node_obj; tm_system_t *tm_system; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (!tm_node_obj) return -1; tm_system = odp_tm_systems[tm_node_obj->tm_idx]; if (!tm_system) return -1; odp_ticketlock_lock(&tm_profile_lock); tm_shaper_config_set(tm_system, shaper_profile, &tm_node_obj->shaper_obj); odp_ticketlock_unlock(&tm_profile_lock); return 0; } int odp_tm_node_sched_config(odp_tm_node_t tm_node, odp_tm_node_t tm_fan_in_node, odp_tm_sched_t sched_profile) { tm_shaper_obj_t *child_shaper_obj; tm_node_obj_t *tm_node_obj, *child_tm_node_obj; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (!tm_node_obj) return -1; child_tm_node_obj = GET_TM_NODE_OBJ(tm_fan_in_node); if (!child_tm_node_obj) return -1; odp_ticketlock_lock(&tm_profile_lock); child_shaper_obj = &child_tm_node_obj->shaper_obj; tm_sched_config_set(child_shaper_obj, sched_profile); odp_ticketlock_unlock(&tm_profile_lock); return 0; } int odp_tm_node_threshold_config(odp_tm_node_t tm_node, odp_tm_threshold_t thresholds_profile) { tm_node_obj_t *tm_node_obj; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if ((!tm_node_obj) || (!tm_node_obj->tm_wred_node)) return -1; odp_ticketlock_lock(&tm_profile_lock); tm_threshold_config_set(tm_node_obj->tm_wred_node, thresholds_profile); odp_ticketlock_unlock(&tm_profile_lock); return 0; } int odp_tm_node_wred_config(odp_tm_node_t tm_node, odp_packet_color_t pkt_color, odp_tm_wred_t wred_profile) { tm_wred_node_t *wred_node; tm_node_obj_t *tm_node_obj; uint32_t color; int rc; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (!tm_node_obj) return -1; wred_node = tm_node_obj->tm_wred_node; odp_ticketlock_lock(&tm_profile_lock); rc = 0; if (pkt_color == ODP_PACKET_ALL_COLORS) { for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) tm_wred_config_set(wred_node, color, wred_profile); } else if (pkt_color < ODP_NUM_PACKET_COLORS) { tm_wred_config_set(wred_node, pkt_color, wred_profile); } else { rc = -1; } odp_ticketlock_unlock(&tm_profile_lock); return rc; } odp_tm_node_t odp_tm_node_lookup(odp_tm_t odp_tm ODP_UNUSED, const char *name) { _odp_int_name_t odp_name; uint64_t user_data; odp_name = _odp_int_name_tbl_lookup(name, ODP_TM_NODE_HANDLE); user_data = _odp_int_name_tbl_user_data(odp_name); return (odp_tm_node_t)user_data; } void *odp_tm_node_context(odp_tm_node_t tm_node) { tm_node_obj_t *tm_node_obj; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (!tm_node_obj) return NULL; return tm_node_obj->user_context; } int odp_tm_node_context_set(odp_tm_node_t tm_node, void *user_context) { tm_node_obj_t *tm_node_obj; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (!tm_node_obj) return -1; tm_node_obj->user_context = user_context; return 0; } void odp_tm_queue_params_init(odp_tm_queue_params_t *params) { memset(params, 0, sizeof(odp_tm_queue_params_t)); } odp_tm_queue_t odp_tm_queue_create(odp_tm_t odp_tm, odp_tm_queue_params_t *params) { _odp_int_pkt_queue_t _odp_int_pkt_queue; tm_queue_obj_t *tm_queue_obj; tm_wred_node_t *tm_wred_node; odp_tm_queue_t odp_tm_queue; odp_tm_wred_t wred_profile; tm_system_t *tm_system; uint32_t color; /* Allocate a tm_queue_obj_t record. */ tm_system = GET_TM_SYSTEM(odp_tm); tm_queue_obj = malloc(sizeof(tm_queue_obj_t)); if (!tm_queue_obj) return ODP_TM_INVALID; tm_wred_node = malloc(sizeof(tm_wred_node_t)); if (!tm_wred_node) { free(tm_queue_obj); return ODP_TM_INVALID; } _odp_int_pkt_queue = _odp_pkt_queue_create( tm_system->_odp_int_queue_pool); if (_odp_int_pkt_queue == _ODP_INT_PKT_QUEUE_INVALID) { free(tm_wred_node); free(tm_queue_obj); return ODP_TM_INVALID; } odp_tm_queue = MAKE_ODP_TM_QUEUE(tm_queue_obj); memset(tm_queue_obj, 0, sizeof(tm_queue_obj_t)); memset(tm_wred_node, 0, sizeof(tm_wred_node_t)); tm_queue_obj->user_context = params->user_context; tm_queue_obj->priority = params->priority; tm_queue_obj->tm_idx = tm_system->tm_idx; tm_queue_obj->queue_num = tm_system->next_queue_num++; tm_queue_obj->tm_wred_node = tm_wred_node; tm_queue_obj->_odp_int_pkt_queue = _odp_int_pkt_queue; tm_queue_obj->pkt = ODP_PACKET_INVALID; odp_ticketlock_init(&tm_wred_node->tm_wred_node_lock); tm_queue_obj->tm_qentry.s.type = ODP_QUEUE_TYPE_TM; tm_queue_obj->tm_qentry.s.enqueue = queue_tm_reenq; tm_queue_obj->tm_qentry.s.enqueue_multi = queue_tm_reenq_multi; tm_system->queue_num_tbl[tm_queue_obj->queue_num] = tm_queue_obj; odp_ticketlock_lock(&tm_system->tm_system_lock); if (params->shaper_profile != ODP_TM_INVALID) tm_shaper_config_set(tm_system, params->shaper_profile, &tm_queue_obj->shaper_obj); if (params->threshold_profile != ODP_TM_INVALID) tm_threshold_config_set(tm_wred_node, params->threshold_profile); for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) { wred_profile = params->wred_profile[color]; if (wred_profile != ODP_TM_INVALID) tm_wred_config_set(tm_wred_node, color, wred_profile); } tm_queue_obj->magic_num = TM_QUEUE_MAGIC_NUM; tm_queue_obj->shaper_obj.enclosing_entity = tm_queue_obj; tm_queue_obj->shaper_obj.in_tm_node_obj = 0; odp_ticketlock_unlock(&tm_system->tm_system_lock); return odp_tm_queue; } int odp_tm_queue_destroy(odp_tm_queue_t tm_queue) { tm_wred_params_t *wred_params; tm_shaper_obj_t *shaper_obj; tm_queue_obj_t *tm_queue_obj; tm_wred_node_t *tm_wred_node; tm_system_t *tm_system; uint32_t color; /* First lookup tm_queue. */ tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) return -1; tm_system = odp_tm_systems[tm_queue_obj->tm_idx]; if (!tm_system) return -1; /* Check to see if the tm_queue_obj is disconnected AND that it has no * current pkt, otherwise the destroy fails. */ shaper_obj = &tm_queue_obj->shaper_obj; if ((shaper_obj->next_tm_node != NULL) || (tm_queue_obj->pkt != ODP_PACKET_INVALID)) return -1; /* Check that there is no shaper profile, threshold profile or wred * profile currently associated with this tm_queue. */ if (shaper_obj->shaper_params != NULL) return -1; tm_wred_node = tm_queue_obj->tm_wred_node; if (tm_wred_node != NULL) { if (tm_wred_node->threshold_params != NULL) return -1; for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) { wred_params = tm_wred_node->wred_params[color]; if (wred_params != NULL) return -1; } } /* Now that all of the checks are done, time to so some freeing. */ odp_ticketlock_lock(&tm_system->tm_system_lock); tm_system->queue_num_tbl[tm_queue_obj->queue_num] = NULL; /* First delete any associated tm_wred_node and then the tm_queue_obj * itself */ free(tm_queue_obj->tm_wred_node); free(tm_queue_obj); odp_ticketlock_unlock(&tm_system->tm_system_lock); return 0; } void *odp_tm_queue_context(odp_tm_queue_t tm_queue) { tm_queue_obj_t *tm_queue_obj; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) return NULL; return tm_queue_obj->user_context; } int odp_tm_queue_context_set(odp_tm_queue_t tm_queue, void *user_context) { tm_queue_obj_t *tm_queue_obj; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) return -1; tm_queue_obj->user_context = user_context; return 0; } int odp_tm_queue_shaper_config(odp_tm_queue_t tm_queue, odp_tm_shaper_t shaper_profile) { tm_queue_obj_t *tm_queue_obj; tm_system_t *tm_system; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) return -1; tm_system = odp_tm_systems[tm_queue_obj->tm_idx]; if (!tm_system) return -1; odp_ticketlock_lock(&tm_profile_lock); tm_shaper_config_set(tm_system, shaper_profile, &tm_queue_obj->shaper_obj); odp_ticketlock_unlock(&tm_profile_lock); return 0; } int odp_tm_queue_sched_config(odp_tm_node_t tm_node, odp_tm_queue_t tm_fan_in_queue, odp_tm_sched_t sched_profile) { tm_shaper_obj_t *child_shaper_obj; tm_queue_obj_t *child_tm_queue_obj; tm_node_obj_t *tm_node_obj; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (!tm_node_obj) return -1; child_tm_queue_obj = GET_TM_QUEUE_OBJ(tm_fan_in_queue); if (!child_tm_queue_obj) return -1; odp_ticketlock_lock(&tm_profile_lock); child_shaper_obj = &child_tm_queue_obj->shaper_obj; tm_sched_config_set(child_shaper_obj, sched_profile); odp_ticketlock_unlock(&tm_profile_lock); return 0; } int odp_tm_queue_threshold_config(odp_tm_queue_t tm_queue, odp_tm_threshold_t thresholds_profile) { tm_queue_obj_t *tm_queue_obj; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) return -1; odp_ticketlock_lock(&tm_profile_lock); tm_threshold_config_set(tm_queue_obj->tm_wred_node, thresholds_profile); odp_ticketlock_unlock(&tm_profile_lock); return 0; } int odp_tm_queue_wred_config(odp_tm_queue_t tm_queue, odp_packet_color_t pkt_color, odp_tm_wred_t wred_profile) { tm_wred_node_t *wred_node; tm_queue_obj_t *tm_queue_obj; uint32_t color; int rc; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) return -1; wred_node = tm_queue_obj->tm_wred_node; odp_ticketlock_lock(&tm_profile_lock); rc = 0; if (pkt_color == ODP_PACKET_ALL_COLORS) { for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) tm_wred_config_set(wred_node, color, wred_profile); } else if (pkt_color < ODP_NUM_PACKET_COLORS) { tm_wred_config_set(wred_node, pkt_color, wred_profile); } else { rc = -1; } odp_ticketlock_unlock(&tm_profile_lock); return rc; } static int tm_append_to_fanin_list(tm_node_obj_t *tm_node_obj, tm_shaper_obj_t *shaper_obj) { tm_shaper_obj_t *head_ptr, *tail_ptr; head_ptr = tm_node_obj->fanin_list_head; tail_ptr = tm_node_obj->fanin_list_tail; if ((head_ptr == NULL) && (tail_ptr != NULL)) return -9; else if ((head_ptr != NULL) && (tail_ptr == NULL)) return -10; if (tail_ptr == NULL) { tm_node_obj->fanin_list_head = shaper_obj; shaper_obj->fanin_list_prev = NULL; } else { tail_ptr->fanin_list_next = shaper_obj; shaper_obj->fanin_list_prev = tail_ptr; } tm_node_obj->fanin_list_tail = shaper_obj; shaper_obj->fanin_list_next = NULL; return 0; } static int tm_remove_from_fanin_list(tm_node_obj_t *tm_node_obj, tm_shaper_obj_t *shaper_obj) { tm_shaper_obj_t *next_ptr, *prev_ptr; next_ptr = shaper_obj->fanin_list_next; prev_ptr = shaper_obj->fanin_list_prev; if (next_ptr != NULL) next_ptr->fanin_list_prev = prev_ptr; else tm_node_obj->fanin_list_tail = prev_ptr; if (prev_ptr != NULL) prev_ptr->fanin_list_next = next_ptr; else tm_node_obj->fanin_list_head = next_ptr; return 0; } int odp_tm_node_connect(odp_tm_node_t src_tm_node, odp_tm_node_t dst_tm_node) { tm_wred_node_t *src_tm_wred_node, *dst_tm_wred_node; tm_node_obj_t *src_tm_node_obj, *dst_tm_node_obj; tm_system_t *tm_system; if ((src_tm_node == ODP_TM_INVALID) || (src_tm_node == ODP_TM_ROOT) || (dst_tm_node == ODP_TM_INVALID)) return -1; src_tm_node_obj = GET_TM_NODE_OBJ(src_tm_node); if ((!src_tm_node_obj) || src_tm_node_obj->is_root_node) return -1; tm_system = odp_tm_systems[src_tm_node_obj->tm_idx]; if (!tm_system) return -1; src_tm_wred_node = src_tm_node_obj->tm_wred_node; if (dst_tm_node == ODP_TM_ROOT) { src_tm_node_obj->shaper_obj.next_tm_node = tm_system->root_node; src_tm_wred_node->next_tm_wred_node = NULL; return 0; } dst_tm_node_obj = GET_TM_NODE_OBJ(dst_tm_node); if ((!dst_tm_node_obj) || dst_tm_node_obj->is_root_node) return -1; dst_tm_wred_node = dst_tm_node_obj->tm_wred_node; if (src_tm_node_obj->tm_idx != dst_tm_node_obj->tm_idx) return -1; src_tm_wred_node->next_tm_wred_node = dst_tm_wred_node; src_tm_node_obj->shaper_obj.next_tm_node = dst_tm_node_obj; dst_tm_node_obj->current_tm_node_fanin++; /* Finally add this src_tm_node_obj to the dst_tm_node_obj's fanin * list. */ return tm_append_to_fanin_list(dst_tm_node_obj, &src_tm_node_obj->shaper_obj); } int odp_tm_node_disconnect(odp_tm_node_t src_tm_node) { tm_wred_node_t *src_tm_wred_node; tm_node_obj_t *src_tm_node_obj, *dst_tm_node_obj; src_tm_node_obj = GET_TM_NODE_OBJ(src_tm_node); if (!src_tm_node_obj) return -1; dst_tm_node_obj = src_tm_node_obj->shaper_obj.next_tm_node; if ((dst_tm_node_obj != NULL) && (!dst_tm_node_obj->is_root_node)) { tm_remove_from_fanin_list(dst_tm_node_obj, &src_tm_node_obj->shaper_obj); if (dst_tm_node_obj->current_tm_node_fanin != 0) dst_tm_node_obj->current_tm_node_fanin--; } src_tm_wred_node = src_tm_node_obj->tm_wred_node; if (src_tm_wred_node != NULL) src_tm_wred_node->next_tm_wred_node = NULL; src_tm_node_obj->shaper_obj.next_tm_node = NULL; return 0; } int odp_tm_queue_connect(odp_tm_queue_t tm_queue, odp_tm_node_t dst_tm_node) { tm_wred_node_t *src_tm_wred_node, *dst_tm_wred_node; tm_queue_obj_t *src_tm_queue_obj; tm_node_obj_t *dst_tm_node_obj, *root_node; tm_system_t *tm_system; if ((tm_queue == ODP_TM_INVALID) || (tm_queue == ODP_TM_ROOT) || (dst_tm_node == ODP_TM_INVALID)) return -1; src_tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!src_tm_queue_obj) return -1; tm_system = odp_tm_systems[src_tm_queue_obj->tm_idx]; if (!tm_system) return -1; src_tm_wred_node = src_tm_queue_obj->tm_wred_node; if (dst_tm_node == ODP_TM_ROOT) { root_node = tm_system->root_node; src_tm_queue_obj->shaper_obj.next_tm_node = root_node; src_tm_wred_node->next_tm_wred_node = NULL; return 0; } dst_tm_node_obj = GET_TM_NODE_OBJ(dst_tm_node); if ((!dst_tm_node_obj) || dst_tm_node_obj->is_root_node) return -1; dst_tm_wred_node = dst_tm_node_obj->tm_wred_node; if (src_tm_queue_obj->tm_idx != dst_tm_node_obj->tm_idx) return -1; src_tm_wred_node->next_tm_wred_node = dst_tm_wred_node; src_tm_queue_obj->shaper_obj.next_tm_node = dst_tm_node_obj; dst_tm_node_obj->current_tm_queue_fanin++; /* Finally add this src_tm_queue_obj to the dst_tm_node_obj's fanin * list. */ return tm_append_to_fanin_list(dst_tm_node_obj, &src_tm_queue_obj->shaper_obj); } int odp_tm_queue_disconnect(odp_tm_queue_t tm_queue) { tm_wred_node_t *src_tm_wred_node; tm_queue_obj_t *src_tm_queue_obj; tm_node_obj_t *dst_tm_node_obj; src_tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!src_tm_queue_obj) return -1; dst_tm_node_obj = src_tm_queue_obj->shaper_obj.next_tm_node; if ((dst_tm_node_obj != NULL) && (!dst_tm_node_obj->is_root_node)) { tm_remove_from_fanin_list(dst_tm_node_obj, &src_tm_queue_obj->shaper_obj); if (dst_tm_node_obj->current_tm_queue_fanin != 0) dst_tm_node_obj->current_tm_queue_fanin--; } src_tm_wred_node = src_tm_queue_obj->tm_wred_node; if (src_tm_wred_node != NULL) src_tm_wred_node->next_tm_wred_node = NULL; src_tm_queue_obj->shaper_obj.next_tm_node = NULL; return 0; } int odp_tm_enq(odp_tm_queue_t tm_queue, odp_packet_t pkt) { tm_queue_obj_t *tm_queue_obj; tm_system_t *tm_system; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) return -1; tm_system = odp_tm_systems[tm_queue_obj->tm_idx]; if (!tm_system) return -1; if (odp_atomic_load_u64(&tm_system->destroying)) return -1; return tm_enqueue(tm_system, tm_queue_obj, pkt); } int odp_tm_enq_with_cnt(odp_tm_queue_t tm_queue, odp_packet_t pkt) { tm_queue_obj_t *tm_queue_obj; tm_system_t *tm_system; uint32_t pkt_cnt; int rc; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) return -1; tm_system = odp_tm_systems[tm_queue_obj->tm_idx]; if (!tm_system) return -1; if (odp_atomic_load_u64(&tm_system->destroying)) return -1; rc = tm_enqueue(tm_system, tm_queue_obj, pkt); if (rc < 0) return rc; pkt_cnt = rc; return pkt_cnt; } int odp_tm_node_info(odp_tm_node_t tm_node, odp_tm_node_info_t *info) { tm_queue_thresholds_t *threshold_params; tm_shaper_params_t *shaper_params; tm_wred_params_t *wred_params; tm_shaper_obj_t *shaper_obj; tm_wred_node_t *tm_wred_node; tm_node_obj_t *tm_node_obj, *next_tm_node; uint32_t color; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (tm_node_obj == NULL) return -1; info->shaper_profile = ODP_TM_INVALID; info->threshold_profile = ODP_TM_INVALID; for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) info->wred_profile[color] = ODP_TM_INVALID; info->level = tm_node_obj->level; info->tm_queue_fanin = tm_node_obj->current_tm_queue_fanin; info->tm_node_fanin = tm_node_obj->current_tm_node_fanin; shaper_obj = &tm_node_obj->shaper_obj; next_tm_node = shaper_obj->next_tm_node; if (next_tm_node == NULL) info->next_tm_node = ODP_TM_INVALID; else if (next_tm_node->is_root_node) info->next_tm_node = ODP_TM_ROOT; else info->next_tm_node = MAKE_ODP_TM_NODE(next_tm_node); shaper_params = shaper_obj->shaper_params; if (shaper_params != NULL) info->shaper_profile = shaper_params->shaper_profile; tm_wred_node = tm_node_obj->tm_wred_node; if (tm_wred_node != NULL) { threshold_params = tm_wred_node->threshold_params; if (threshold_params != NULL) info->threshold_profile = threshold_params->thresholds_profile; for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) { wred_params = tm_wred_node->wred_params[color]; if (wred_params != NULL) info->wred_profile[color] = wred_params->wred_profile; } } return 0; } int odp_tm_node_fanin_info(odp_tm_node_t tm_node, odp_tm_node_fanin_info_t *info) { tm_sched_params_t *sched_params; tm_shaper_obj_t *shaper_obj, *next_shaper_obj; tm_queue_obj_t *fanin_tm_queue_obj; tm_node_obj_t *tm_node_obj, *fanin_tm_node_obj; tm_node_obj = GET_TM_NODE_OBJ(tm_node); if (tm_node_obj == NULL) return -1; else if ((info->tm_queue != ODP_TM_INVALID) && (info->tm_node != ODP_TM_INVALID)) return -1; else if (info->is_last) return -1; if (info->tm_queue != ODP_TM_INVALID) { fanin_tm_queue_obj = GET_TM_QUEUE_OBJ(info->tm_queue); if (fanin_tm_queue_obj == NULL) return -1; shaper_obj = &fanin_tm_queue_obj->shaper_obj; } else if (info->tm_node != ODP_TM_INVALID) { fanin_tm_node_obj = GET_TM_NODE_OBJ(info->tm_node); if (fanin_tm_node_obj == NULL) return -1; shaper_obj = &fanin_tm_node_obj->shaper_obj; } else { shaper_obj = NULL; } if ((shaper_obj != NULL) && (shaper_obj->next_tm_node != tm_node_obj)) return -1; if (shaper_obj == NULL) next_shaper_obj = tm_node_obj->fanin_list_head; else next_shaper_obj = shaper_obj->fanin_list_next; if ((next_shaper_obj == NULL) || (next_shaper_obj->next_tm_node != tm_node_obj)) return -1; info->is_last = next_shaper_obj->fanin_list_next == NULL; if (next_shaper_obj->in_tm_node_obj) { fanin_tm_node_obj = next_shaper_obj->enclosing_entity; info->tm_node = MAKE_ODP_TM_NODE(fanin_tm_node_obj); info->tm_queue = ODP_TM_INVALID; } else { fanin_tm_queue_obj = next_shaper_obj->enclosing_entity; info->tm_queue = MAKE_ODP_TM_QUEUE(fanin_tm_queue_obj); info->tm_node = ODP_TM_INVALID; } info->sched_profile = ODP_TM_INVALID; sched_params = next_shaper_obj->sched_params; if (sched_params != NULL) info->sched_profile = sched_params->sched_profile; return 0; } int odp_tm_queue_info(odp_tm_queue_t tm_queue, odp_tm_queue_info_t *info) { tm_queue_thresholds_t *threshold_params; tm_shaper_params_t *shaper_params; tm_wred_params_t *wred_params; tm_shaper_obj_t *shaper_obj; tm_wred_node_t *tm_wred_node; tm_queue_obj_t *tm_queue_obj; tm_node_obj_t *next_tm_node; uint32_t color; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (tm_queue_obj == NULL) return -1; info->shaper_profile = ODP_TM_INVALID; info->threshold_profile = ODP_TM_INVALID; for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) info->wred_profile[color] = ODP_TM_INVALID; info->active_pkt = tm_queue_obj->pkt; shaper_obj = &tm_queue_obj->shaper_obj; next_tm_node = shaper_obj->next_tm_node; if (next_tm_node == NULL) info->next_tm_node = ODP_TM_INVALID; else if (next_tm_node->is_root_node) info->next_tm_node = ODP_TM_ROOT; else info->next_tm_node = MAKE_ODP_TM_NODE(next_tm_node); shaper_params = shaper_obj->shaper_params; if (shaper_params != NULL) info->shaper_profile = shaper_params->shaper_profile; tm_wred_node = tm_queue_obj->tm_wred_node; if (tm_wred_node != NULL) { threshold_params = tm_wred_node->threshold_params; if (threshold_params != NULL) info->threshold_profile = threshold_params->thresholds_profile; for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) { wred_params = tm_wred_node->wred_params[color]; if (wred_params != NULL) info->wred_profile[color] = wred_params->wred_profile; } } return 0; } static int tm_query_info_copy(tm_queue_info_t *queue_info, uint32_t query_flags, odp_tm_query_info_t *info) { tm_queue_thresholds_t *threshold_params; memset(info, 0, sizeof(odp_tm_query_info_t)); info->total_pkt_cnt = odp_atomic_load_u64(&queue_info->queue_cnts.pkt_cnt); info->total_byte_cnt = odp_atomic_load_u64(&queue_info->queue_cnts.byte_cnt); info->total_pkt_cnt_valid = 1; info->total_byte_cnt_valid = 1; info->approx_byte_cnt = 0; if (query_flags & ODP_TM_QUERY_THRESHOLDS) { threshold_params = queue_info->threshold_params; if (!threshold_params) return -1; info->max_pkt_cnt = threshold_params->max_pkts; info->max_byte_cnt = threshold_params->max_bytes; info->max_pkt_cnt_valid = info->max_pkt_cnt != 0; info->max_byte_cnt_valid = info->max_byte_cnt != 0; } return 0; } int odp_tm_queue_query(odp_tm_queue_t tm_queue, uint32_t query_flags, odp_tm_query_info_t *info) { tm_queue_info_t queue_info; tm_queue_obj_t *tm_queue_obj; tm_wred_node_t *tm_wred_node; tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue); if (!tm_queue_obj) return -1; tm_wred_node = tm_queue_obj->tm_wred_node; if (!tm_wred_node) return -1; /* **TBD** Where do we get the queue_info from. */ queue_info.threshold_params = tm_wred_node->threshold_params; queue_info.queue_cnts = tm_wred_node->queue_cnts; return tm_query_info_copy(&queue_info, query_flags, info); } int odp_tm_priority_query(odp_tm_t odp_tm, uint8_t priority, uint32_t query_flags, odp_tm_query_info_t *info) { tm_queue_info_t queue_info; tm_system_t *tm_system; tm_system = GET_TM_SYSTEM(odp_tm); queue_info = tm_system->priority_info[priority]; return tm_query_info_copy(&queue_info, query_flags, info); } int odp_tm_total_query(odp_tm_t odp_tm, uint32_t query_flags, odp_tm_query_info_t *info) { tm_queue_info_t queue_info; tm_system_t *tm_system; tm_system = GET_TM_SYSTEM(odp_tm); queue_info = tm_system->total_info; return tm_query_info_copy(&queue_info, query_flags, info); } int odp_tm_priority_threshold_config(odp_tm_t odp_tm, uint8_t priority, odp_tm_threshold_t thresholds_profile) { tm_system_t *tm_system; tm_system = GET_TM_SYSTEM(odp_tm); if (thresholds_profile == ODP_TM_INVALID) return -1; odp_ticketlock_lock(&tm_profile_lock); tm_system->priority_info[priority].threshold_params = tm_get_profile_params(thresholds_profile, TM_THRESHOLD_PROFILE); odp_ticketlock_unlock(&tm_profile_lock); return 0; } int odp_tm_total_threshold_config(odp_tm_t odp_tm, odp_tm_threshold_t thresholds_profile) { tm_system_t *tm_system; tm_system = GET_TM_SYSTEM(odp_tm); if (thresholds_profile == ODP_TM_INVALID) return -1; odp_ticketlock_lock(&tm_profile_lock); tm_system->total_info.threshold_params = tm_get_profile_params( thresholds_profile, TM_THRESHOLD_PROFILE); odp_ticketlock_unlock(&tm_profile_lock); return 0; } void odp_tm_stats_print(odp_tm_t odp_tm) { input_work_queue_t *input_work_queue; tm_queue_obj_t *tm_queue_obj; tm_system_t *tm_system; uint32_t queue_num, max_queue_num; tm_system = GET_TM_SYSTEM(odp_tm); input_work_queue = tm_system->input_work_queue; ODP_DBG("odp_tm_stats_print - tm_system=0x%" PRIX64 " tm_idx=%u\n", odp_tm, tm_system->tm_idx); ODP_DBG(" input_work_queue size=%u current cnt=%u peak cnt=%u\n", INPUT_WORK_RING_SIZE, input_work_queue->queue_cnt, input_work_queue->peak_cnt); ODP_DBG(" input_work_queue enqueues=%" PRIu64 " dequeues=% " PRIu64 " fail_cnt=%" PRIu64 "\n", input_work_queue->total_enqueues, input_work_queue->total_dequeues, input_work_queue->enqueue_fail_cnt); ODP_DBG(" green_cnt=%" PRIu64 " yellow_cnt=%" PRIu64 " red_cnt=%" PRIu64 "\n", tm_system->shaper_green_cnt, tm_system->shaper_yellow_cnt, tm_system->shaper_red_cnt); _odp_pkt_queue_stats_print(tm_system->_odp_int_queue_pool); _odp_timer_wheel_stats_print(tm_system->_odp_int_timer_wheel); _odp_sorted_list_stats_print(tm_system->_odp_int_sorted_pool); max_queue_num = tm_system->next_queue_num; for (queue_num = 1; queue_num < max_queue_num; queue_num++) { tm_queue_obj = tm_system->queue_num_tbl[queue_num]; if (tm_queue_obj && tm_queue_obj->pkts_rcvd_cnt != 0) ODP_DBG("queue_num=%u priority=%u rcvd=%u enqueued=%u " "dequeued=%u consumed=%u\n", queue_num, tm_queue_obj->priority, tm_queue_obj->pkts_rcvd_cnt, tm_queue_obj->pkts_enqueued_cnt, tm_queue_obj->pkts_dequeued_cnt, tm_queue_obj->pkts_consumed_cnt); } } int odp_tm_init_global(void) { odp_ticketlock_init(&tm_create_lock); odp_ticketlock_init(&tm_profile_lock); odp_barrier_init(&tm_first_enq, 2); odp_atomic_init_u64(&atomic_request_cnt, 0); odp_atomic_init_u64(¤tly_serving_cnt, 0); odp_atomic_init_u64(&atomic_done_cnt, 0); return 0; } int odp_tm_term_global(void) { return 0; }