rte_graph_feature_arc_worker.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(C) 2025 Marvell International Ltd.
 */

#ifndef _RTE_GRAPH_FEATURE_ARC_WORKER_H_
#define _RTE_GRAPH_FEATURE_ARC_WORKER_H_

#include <stddef.h>
#include <rte_graph_feature_arc.h>
#include <rte_bitops.h>
#include <rte_mbuf.h>
#include <rte_mbuf_dyn.h>

#ifdef __cplusplus
extern "C" {
#endif

struct rte_graph_feature_node_list {
    STAILQ_ENTRY(rte_graph_feature_node_list) next_feature;

    char feature_name[RTE_GRAPH_FEATURE_ARC_NAMELEN];

    rte_node_t feature_node_id;

    int32_t ref_count;

    rte_node_process_t feature_node_process_fn;

    rte_graph_feature_change_notifier_cb_t notifier_cb;

    uint32_t finfo_index;

    void *feature_arc;

    rte_edge_t edge_to_this_feature;

    rte_edge_t edge_to_last_feature;
};

struct __rte_cache_aligned rte_graph_feature_arc {
    RTE_MARKER slow_path_variables;

    STAILQ_HEAD(, rte_graph_feature_node_list) all_features;

    char feature_arc_name[RTE_GRAPH_FEATURE_ARC_NAMELEN];

    uint32_t runtime_enabled_features;

    uint16_t max_features;

    rte_graph_feature_arc_t feature_arc_index;

    void *feature_arc_main;

    struct rte_node_register *start_node;
    struct rte_graph_feature_register end_feature;

    rte_node_process_t arc_start_process;

    size_t arc_size;

    rte_graph_feature_data_t *feature_data_by_index;

    uint32_t feature_size;

    uint64_t *feature_bit_mask_by_index;

    alignas(RTE_CACHE_LINE_SIZE) RTE_MARKER fast_path_variables;

    RTE_ATOMIC(uint64_t) fp_feature_enable_bitmask;

    uint16_t num_added_features;
    uint16_t max_indexes;

    uint16_t fp_first_feature_offset;

    uint16_t fp_feature_data_offset;

    int mbuf_dyn_offset;

    /*
     * Arc specific fast path data
     * It accommodates:
     *
     *  1. first enabled feature data for every index (rte_graph_feature_data_t or fdata)
     *  +--------------------------------------------------------------+ <- cache_aligned
     *  |  0th Index    | 1st Index   |  ... | max_index - 1           |
     *  +--------------------------------------------------------------+
     *  |  Startfdata0  | Startfdata1 |  ... | Startfdata(max_index-1) |
     *  +--------------------------------------------------------------+
     *
     *  2. struct rte_graph_feature_data per index per feature
     *  +----------------------------------------+ ^ <- Start (Reserved, cache aligned)
     *  |  struct rte_graph_feature_data[Index0] | |
     *  +----------------------------------------+ | feature_size
     *  |  struct rte_graph_feature_data[Index1] | |
     *  +----------------------------------------+ ^ <- Feature-0 (cache_aligned)
     *  |  struct rte_graph_feature_data[Index0] | |
     *  +----------------------------------------+ | feature_size
     *  |  struct rte_graph_feature_data[Index1] | |
     *  +----------------------------------------+ v <- Feature-1 (cache aligned)
     *  |  struct rte_graph_feature_data[Index0] | ^
     *  +----------------------------------------+ | feature_size
     *  |  struct rte_graph_feature_data[Index1] | |
     *  +----------------------------------------+ v
     *  |         ...            ....            |
     *  |         ...            ....            |
     *  |         ...            ....            |
     *  +----------------------------------------+ v <- Feature Index-1 (cache aligned)
     *  |  struct rte_graph_feature_data[Index0] | ^
     *  +----------------------------------------+ | feature_size
     *  |  struct rte_graph_feature_data[Index1] | |
     *  +----------------------------------------+ v <- Extra (Reserved, cache aligned)
     *  |  struct rte_graph_feature_data[Index0] | ^
     *  +----------------------------------------+ | feature_size
     *  |  struct rte_graph_feature_data[Index1] | |
     *  +----------------------------------------+ v
     */
    RTE_MARKER8 fp_arc_data;
};

typedef struct rte_feature_arc_main {
    uint32_t num_feature_arcs;

    uint32_t max_feature_arcs;

    int arc_mbuf_dyn_offset;

    uintptr_t feature_arcs[];
} rte_graph_feature_arc_main_t;

struct rte_graph_feature_data {
    RTE_ATOMIC(rte_edge_t) next_edge;

    RTE_ATOMIC(uint16_t) app_cookie;

    RTE_ATOMIC(rte_graph_feature_data_t) next_feature_data;
};

struct rte_graph_feature_arc_mbuf_dynfields {
    rte_graph_feature_data_t feature_data;
};

#define RTE_GRAPH_FEATURE_ARC_DYNFIELD_NAME    "__rte_graph_feature_arc_mbuf_dynfield"

#define RTE_GRAPH_FEATURE_DATA_SIZE_LOG2    3

#define RTE_GRAPH_FEATURE_DATA_NUM_PER_FEATURE(arc)             \
    (arc->feature_size >> RTE_GRAPH_FEATURE_DATA_SIZE_LOG2)

#define RTE_GRAPH_FEATURE_TO_FEATURE_DATA(arc, feature, index)          \
        ((rte_graph_feature_data_t)                 \
         ((RTE_GRAPH_FEATURE_DATA_NUM_PER_FEATURE(arc) * (feature)) + (index)))

#define GRAPH_FEATURE_ARC_PTR_INITIALIZER  ((uintptr_t)UINTPTR_MAX)

extern rte_graph_feature_arc_main_t *__rte_graph_feature_arc_main;

__rte_experimental
static __rte_always_inline struct rte_graph_feature_arc_mbuf_dynfields *
rte_graph_feature_arc_mbuf_dynfields_get(struct rte_mbuf *mbuf,
                     const int dyn_offset)
{
    return RTE_MBUF_DYNFIELD(mbuf, dyn_offset,
                 struct rte_graph_feature_arc_mbuf_dynfields *);
}

__rte_experimental
static __rte_always_inline int
rte_graph_feature_is_valid(rte_graph_feature_t feature)
{
    return (feature != RTE_GRAPH_FEATURE_INVALID);
}

__rte_experimental
static __rte_always_inline int
rte_graph_feature_data_is_valid(rte_graph_feature_data_t feature_data)
{
    return (feature_data != RTE_GRAPH_FEATURE_DATA_INVALID);
}

__rte_experimental
static __rte_always_inline struct rte_graph_feature_arc *
rte_graph_feature_arc_get(rte_graph_feature_arc_t arc)
{
    uintptr_t fa = GRAPH_FEATURE_ARC_PTR_INITIALIZER;
    rte_graph_feature_arc_main_t *fm = NULL;

    fm = __rte_graph_feature_arc_main;

    if (likely((fm != NULL) && (arc < fm->max_feature_arcs)))
        fa = fm->feature_arcs[arc];

    return (fa == GRAPH_FEATURE_ARC_PTR_INITIALIZER) ?
        NULL : (struct rte_graph_feature_arc *)fa;
}

__rte_experimental
static __rte_always_inline struct rte_graph_feature_data*
__rte_graph_feature_data_get(struct rte_graph_feature_arc *arc,
                 rte_graph_feature_data_t fdata)
{
    return ((struct rte_graph_feature_data *) ((uint8_t *)arc + arc->fp_feature_data_offset +
                           (fdata << RTE_GRAPH_FEATURE_DATA_SIZE_LOG2)));
}

__rte_experimental
static __rte_always_inline rte_edge_t
__rte_graph_feature_data_edge_get(struct rte_graph_feature_data *fdata)
{
    return rte_atomic_load_explicit(&fdata->next_edge, rte_memory_order_relaxed);
}

__rte_experimental
static __rte_always_inline uint16_t
__rte_graph_feature_data_app_cookie_get(struct rte_graph_feature_data *fdata)
{
    return rte_atomic_load_explicit(&fdata->app_cookie, rte_memory_order_relaxed);
}

__rte_experimental
static __rte_always_inline rte_graph_feature_data_t
__rte_graph_feature_data_next_feature_get(struct rte_graph_feature_data *fdata)
{
    return rte_atomic_load_explicit(&fdata->next_feature_data, rte_memory_order_relaxed);
}


__rte_experimental
static __rte_always_inline uint16_t
rte_graph_feature_data_app_cookie_get(struct rte_graph_feature_arc *arc,
                      rte_graph_feature_data_t fdata)
{
    struct rte_graph_feature_data *fdata_obj = ((struct rte_graph_feature_data *)
                            ((uint8_t *)arc + arc->fp_feature_data_offset +
                            (fdata << RTE_GRAPH_FEATURE_DATA_SIZE_LOG2)));


    return rte_atomic_load_explicit(&fdata_obj->app_cookie, rte_memory_order_relaxed);
}

__rte_experimental
static __rte_always_inline int
rte_graph_feature_data_next_feature_get(struct rte_graph_feature_arc *arc,
                    rte_graph_feature_data_t *fdata,
                    rte_edge_t *next_edge)
{
    struct rte_graph_feature_data *fdptr = ((struct rte_graph_feature_data *)
                        ((uint8_t *)arc + arc->fp_feature_data_offset +
                        ((*fdata) << RTE_GRAPH_FEATURE_DATA_SIZE_LOG2)));
    *fdata = rte_atomic_load_explicit(&fdptr->next_feature_data, rte_memory_order_relaxed);
    *next_edge = rte_atomic_load_explicit(&fdptr->next_edge, rte_memory_order_relaxed);


    return ((*fdata) != RTE_GRAPH_FEATURE_DATA_INVALID);
}

__rte_experimental
static __rte_always_inline struct rte_graph_feature_data*
rte_graph_feature_data_get(struct rte_graph_feature_arc *arc,
               rte_graph_feature_data_t fdata)
{
    if (fdata != RTE_GRAPH_FEATURE_DATA_INVALID)
        return ((struct rte_graph_feature_data *)
            ((uint8_t *)arc + arc->fp_feature_data_offset +
            (fdata << RTE_GRAPH_FEATURE_DATA_SIZE_LOG2)));
    else
        return NULL;
}

__rte_experimental
static __rte_always_inline int
rte_graph_feature_data_first_feature_get(struct rte_graph_feature_arc *arc,
                     uint32_t index,
                     rte_graph_feature_data_t *fdata,
                     rte_edge_t *edge)
{
    struct rte_graph_feature_data *fdata_obj = NULL;
    rte_graph_feature_data_t *fd;

    fd = (rte_graph_feature_data_t *)((uint8_t *)arc + arc->fp_first_feature_offset +
                      (sizeof(rte_graph_feature_data_t) * index));

    if ((*fd) != RTE_GRAPH_FEATURE_DATA_INVALID) {
        fdata_obj = ((struct rte_graph_feature_data *)
                 ((uint8_t *)arc + arc->fp_feature_data_offset +
                 ((*fd) << RTE_GRAPH_FEATURE_DATA_SIZE_LOG2)));

        *edge = rte_atomic_load_explicit(&fdata_obj->next_edge,
                         rte_memory_order_relaxed);

        *fdata = rte_atomic_load_explicit(&fdata_obj->next_feature_data,
                          rte_memory_order_relaxed);
        return 1;
    }

    return 0;
}

__rte_experimental
static __rte_always_inline uint64_t
rte_graph_feature_arc_is_any_feature_enabled(struct rte_graph_feature_arc *arc)
{
    if (unlikely(arc == NULL))
        return 0;

    return (rte_atomic_load_explicit(&arc->fp_feature_enable_bitmask,
                     rte_memory_order_relaxed));
}

__rte_experimental
static __rte_always_inline void
rte_graph_feature_arc_prefetch(struct rte_graph_feature_arc *arc)
{
    rte_prefetch0((void *)arc->fast_path_variables);
}

__rte_experimental
static __rte_always_inline void
rte_graph_feature_arc_feature_data_prefetch(struct rte_graph_feature_arc *arc,
                        rte_graph_feature_data_t fdata)
{
    struct rte_graph_feature_data *fdata_obj = ((struct rte_graph_feature_data *)
                            ((uint8_t *)arc + arc->fp_feature_data_offset +
                            (fdata << RTE_GRAPH_FEATURE_DATA_SIZE_LOG2)));
    rte_prefetch0((void *)fdata_obj);
}

#ifdef __cplusplus
}
#endif

#endif