LCOV - code coverage report
Current view: top level - lib/ring - rte_ring_rts_elem_pvt.h (source / functions) Hit Total Coverage
Test: Code coverage Lines: 33 37 89.2 %
Date: 2024-01-22 16:13:49 Functions: 0 0 -
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 80 538 14.9 %

           Branch data     Line data    Source code
       1                 :            : /* SPDX-License-Identifier: BSD-3-Clause
       2                 :            :  *
       3                 :            :  * Copyright (c) 2010-2020 Intel Corporation
       4                 :            :  * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org
       5                 :            :  * All rights reserved.
       6                 :            :  * Derived from FreeBSD's bufring.h
       7                 :            :  * Used as BSD-3 Licensed with permission from Kip Macy.
       8                 :            :  */
       9                 :            : 
      10                 :            : #ifndef _RTE_RING_RTS_ELEM_PVT_H_
      11                 :            : #define _RTE_RING_RTS_ELEM_PVT_H_
      12                 :            : 
      13                 :            : /**
      14                 :            :  * @file rte_ring_rts_elem_pvt.h
      15                 :            :  * It is not recommended to include this file directly,
      16                 :            :  * include <rte_ring.h> instead.
      17                 :            :  * Contains internal helper functions for Relaxed Tail Sync (RTS) ring mode.
      18                 :            :  * For more information please refer to <rte_ring_rts.h>.
      19                 :            :  */
      20                 :            : 
      21                 :            : /**
      22                 :            :  * @internal This function updates tail values.
      23                 :            :  */
      24                 :            : static __rte_always_inline void
      25                 :            : __rte_ring_rts_update_tail(struct rte_ring_rts_headtail *ht)
      26                 :            : {
      27                 :            :         union __rte_ring_rts_poscnt h, ot, nt;
      28                 :            : 
      29                 :            :         /*
      30                 :            :          * If there are other enqueues/dequeues in progress that
      31                 :            :          * might preceded us, then don't update tail with new value.
      32                 :            :          */
      33                 :            : 
      34                 :       6567 :         ot.raw = rte_atomic_load_explicit(&ht->tail.raw, rte_memory_order_acquire);
      35                 :            : 
      36                 :            :         do {
      37                 :            :                 /* on 32-bit systems we have to do atomic read here */
      38                 :       6567 :                 h.raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_relaxed);
      39                 :            : 
      40                 :       6567 :                 nt.raw = ot.raw;
      41   [ +  -  -  -  :       6567 :                 if (++nt.val.cnt == h.val.cnt)
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  -  +  -  +  
          -  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
      42                 :       6567 :                         nt.val.pos = h.val.pos;
      43                 :            : 
      44                 :       6567 :         } while (rte_atomic_compare_exchange_strong_explicit(&ht->tail.raw,
      45                 :            :                         (uint64_t *)(uintptr_t)&ot.raw, nt.raw,
      46   [ -  +  -  -  :       6567 :                         rte_memory_order_release, rte_memory_order_acquire) == 0);
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
      47                 :       6567 : }
      48                 :            : 
      49                 :            : /**
      50                 :            :  * @internal This function waits till head/tail distance wouldn't
      51                 :            :  * exceed pre-defined max value.
      52                 :            :  */
      53                 :            : static __rte_always_inline void
      54                 :            : __rte_ring_rts_head_wait(const struct rte_ring_rts_headtail *ht,
      55                 :            :         union __rte_ring_rts_poscnt *h)
      56                 :            : {
      57                 :            :         uint32_t max;
      58                 :            : 
      59                 :       6581 :         max = ht->htd_max;
      60                 :            : 
      61   [ -  +  -  -  :       6581 :         while (h->val.pos - ht->tail.val.pos > max) {
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  +  -  +  -  
          +  -  +  -  +  
          -  +  -  +  -  
          +  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
      62                 :          0 :                 rte_pause();
      63                 :          0 :                 h->raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_acquire);
      64                 :            :         }
      65                 :            : }
      66                 :            : 
      67                 :            : /**
      68                 :            :  * @internal This function updates the producer head for enqueue.
      69                 :            :  */
      70                 :            : static __rte_always_inline uint32_t
      71                 :            : __rte_ring_rts_move_prod_head(struct rte_ring *r, uint32_t num,
      72                 :            :         enum rte_ring_queue_behavior behavior, uint32_t *old_head,
      73                 :            :         uint32_t *free_entries)
      74                 :            : {
      75                 :            :         uint32_t n;
      76                 :            :         union __rte_ring_rts_poscnt nh, oh;
      77                 :            : 
      78                 :       3821 :         const uint32_t capacity = r->capacity;
      79                 :            : 
      80                 :       3821 :         oh.raw = rte_atomic_load_explicit(&r->rts_prod.head.raw, rte_memory_order_acquire);
      81                 :            : 
      82                 :            :         do {
      83                 :            :                 /* Reset n to the initial burst count */
      84                 :            :                 n = num;
      85                 :            : 
      86                 :            :                 /*
      87                 :            :                  * wait for prod head/tail distance,
      88                 :            :                  * make sure that we read prod head *before*
      89                 :            :                  * reading cons tail.
      90                 :            :                  */
      91                 :            :                 __rte_ring_rts_head_wait(&r->rts_prod, &oh);
      92                 :            : 
      93                 :            :                 /*
      94                 :            :                  *  The subtraction is done between two unsigned 32bits value
      95                 :            :                  * (the result is always modulo 32 bits even if we have
      96                 :            :                  * *old_head > cons_tail). So 'free_entries' is always between 0
      97                 :            :                  * and capacity (which is < size).
      98                 :            :                  */
      99                 :       3821 :                 *free_entries = capacity + r->cons.tail - oh.val.pos;
     100                 :            : 
     101                 :            :                 /* check that we have enough room in ring */
     102   [ +  +  -  -  :       3821 :                 if (unlikely(n > *free_entries))
          -  -  -  -  -  
          -  -  -  +  +  
          +  +  +  +  +  
          +  #  #  #  #  
                   #  # ]
     103                 :            :                         n = (behavior == RTE_RING_QUEUE_FIXED) ?
     104                 :            :                                         0 : *free_entries;
     105                 :            : 
     106   [ -  -  -  -  :       2765 :                 if (n == 0)
          -  -  -  -  +  
          +  +  -  +  +  
                   +  - ]
     107                 :            :                         break;
     108                 :            : 
     109                 :       3807 :                 nh.val.pos = oh.val.pos + n;
     110                 :       3807 :                 nh.val.cnt = oh.val.cnt + 1;
     111                 :            : 
     112                 :            :         /*
     113                 :            :          * this CAS(ACQUIRE, ACQUIRE) serves as a hoist barrier to prevent:
     114                 :            :          *  - OOO reads of cons tail value
     115                 :            :          *  - OOO copy of elems to the ring
     116                 :            :          */
     117                 :       3807 :         } while (rte_atomic_compare_exchange_strong_explicit(&r->rts_prod.head.raw,
     118                 :            :                         (uint64_t *)(uintptr_t)&oh.raw, nh.raw,
     119   [ -  +  -  -  :       3807 :                         rte_memory_order_acquire, rte_memory_order_acquire) == 0);
          -  -  -  -  -  
          -  -  -  -  +  
          -  +  -  +  -  
          +  #  #  #  #  
                   #  # ]
     120                 :            : 
     121                 :       3821 :         *old_head = oh.val.pos;
     122                 :            :         return n;
     123                 :            : }
     124                 :            : 
     125                 :            : /**
     126                 :            :  * @internal This function updates the consumer head for dequeue
     127                 :            :  */
     128                 :            : static __rte_always_inline unsigned int
     129                 :            : __rte_ring_rts_move_cons_head(struct rte_ring *r, uint32_t num,
     130                 :            :         enum rte_ring_queue_behavior behavior, uint32_t *old_head,
     131                 :            :         uint32_t *entries)
     132                 :            : {
     133                 :            :         uint32_t n;
     134                 :            :         union __rte_ring_rts_poscnt nh, oh;
     135                 :            : 
     136                 :       2760 :         oh.raw = rte_atomic_load_explicit(&r->rts_cons.head.raw, rte_memory_order_acquire);
     137                 :            : 
     138                 :            :         /* move cons.head atomically */
     139                 :            :         do {
     140                 :            :                 /* Restore n as it may change every loop */
     141                 :            :                 n = num;
     142                 :            : 
     143                 :            :                 /*
     144                 :            :                  * wait for cons head/tail distance,
     145                 :            :                  * make sure that we read cons head *before*
     146                 :            :                  * reading prod tail.
     147                 :            :                  */
     148                 :            :                 __rte_ring_rts_head_wait(&r->rts_cons, &oh);
     149                 :            : 
     150                 :            :                 /* The subtraction is done between two unsigned 32bits value
     151                 :            :                  * (the result is always modulo 32 bits even if we have
     152                 :            :                  * cons_head > prod_tail). So 'entries' is always between 0
     153                 :            :                  * and size(ring)-1.
     154                 :            :                  */
     155                 :       2760 :                 *entries = r->prod.tail - oh.val.pos;
     156                 :            : 
     157                 :            :                 /* Set the actual entries for dequeue */
     158   [ -  -  -  -  :       1380 :                 if (n > *entries)
          -  -  -  -  -  
          +  -  +  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     159                 :            :                         n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
     160                 :            : 
     161   [ -  -  -  -  :       2760 :                 if (unlikely(n == 0))
          -  -  -  -  -  
          -  -  -  +  -  
          +  -  +  -  +  
          -  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     162                 :            :                         break;
     163                 :            : 
     164                 :       2760 :                 nh.val.pos = oh.val.pos + n;
     165                 :       2760 :                 nh.val.cnt = oh.val.cnt + 1;
     166                 :            : 
     167                 :            :         /*
     168                 :            :          * this CAS(ACQUIRE, ACQUIRE) serves as a hoist barrier to prevent:
     169                 :            :          *  - OOO reads of prod tail value
     170                 :            :          *  - OOO copy of elems from the ring
     171                 :            :          */
     172                 :       2760 :         } while (rte_atomic_compare_exchange_strong_explicit(&r->rts_cons.head.raw,
     173                 :            :                         (uint64_t *)(uintptr_t)&oh.raw, nh.raw,
     174   [ -  -  -  -  :       2760 :                         rte_memory_order_acquire, rte_memory_order_acquire) == 0);
          -  -  -  -  -  
          -  -  -  -  +  
          -  +  -  +  -  
          +  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     175                 :            : 
     176                 :       2760 :         *old_head = oh.val.pos;
     177                 :            :         return n;
     178                 :            : }
     179                 :            : 
     180                 :            : /**
     181                 :            :  * @internal Enqueue several objects on the RTS ring.
     182                 :            :  *
     183                 :            :  * @param r
     184                 :            :  *   A pointer to the ring structure.
     185                 :            :  * @param obj_table
     186                 :            :  *   A pointer to a table of objects.
     187                 :            :  * @param esize
     188                 :            :  *   The size of ring element, in bytes. It must be a multiple of 4.
     189                 :            :  *   This must be the same value used while creating the ring. Otherwise
     190                 :            :  *   the results are undefined.
     191                 :            :  * @param n
     192                 :            :  *   The number of objects to add in the ring from the obj_table.
     193                 :            :  * @param behavior
     194                 :            :  *   RTE_RING_QUEUE_FIXED:    Enqueue a fixed number of items from a ring
     195                 :            :  *   RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
     196                 :            :  * @param free_space
     197                 :            :  *   returns the amount of space after the enqueue operation has finished
     198                 :            :  * @return
     199                 :            :  *   Actual number of objects enqueued.
     200                 :            :  *   If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
     201                 :            :  */
     202                 :            : static __rte_always_inline unsigned int
     203                 :            : __rte_ring_do_rts_enqueue_elem(struct rte_ring *r, const void *obj_table,
     204                 :            :         uint32_t esize, uint32_t n, enum rte_ring_queue_behavior behavior,
     205                 :            :         uint32_t *free_space)
     206                 :            : {
     207                 :            :         uint32_t free, head;
     208                 :            : 
     209                 :            :         n =  __rte_ring_rts_move_prod_head(r, n, behavior, &head, &free);
     210                 :            : 
     211   [ +  +  -  -  :       3821 :         if (n != 0) {
          -  -  -  -  -  
          -  -  -  +  +  
          +  +  +  +  +  
          +  #  #  #  #  
                   #  # ]
     212                 :            :                 __rte_ring_enqueue_elems(r, head, obj_table, esize, n);
     213                 :            :                 __rte_ring_rts_update_tail(&r->rts_prod);
     214                 :            :         }
     215                 :            : 
     216   [ -  +  -  +  :       2770 :         if (free_space != NULL)
             -  +  -  + ]
     217                 :          0 :                 *free_space = free - n;
     218                 :            :         return n;
     219                 :            : }
     220                 :            : 
     221                 :            : /**
     222                 :            :  * @internal Dequeue several objects from the RTS ring.
     223                 :            :  *
     224                 :            :  * @param r
     225                 :            :  *   A pointer to the ring structure.
     226                 :            :  * @param obj_table
     227                 :            :  *   A pointer to a table of objects.
     228                 :            :  * @param esize
     229                 :            :  *   The size of ring element, in bytes. It must be a multiple of 4.
     230                 :            :  *   This must be the same value used while creating the ring. Otherwise
     231                 :            :  *   the results are undefined.
     232                 :            :  * @param n
     233                 :            :  *   The number of objects to pull from the ring.
     234                 :            :  * @param behavior
     235                 :            :  *   RTE_RING_QUEUE_FIXED:    Dequeue a fixed number of items from a ring
     236                 :            :  *   RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
     237                 :            :  * @param available
     238                 :            :  *   returns the number of remaining ring entries after the dequeue has finished
     239                 :            :  * @return
     240                 :            :  *   - Actual number of objects dequeued.
     241                 :            :  *     If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
     242                 :            :  */
     243                 :            : static __rte_always_inline unsigned int
     244                 :            : __rte_ring_do_rts_dequeue_elem(struct rte_ring *r, void *obj_table,
     245                 :            :         uint32_t esize, uint32_t n, enum rte_ring_queue_behavior behavior,
     246                 :            :         uint32_t *available)
     247                 :            : {
     248                 :            :         uint32_t entries, head;
     249                 :            : 
     250                 :            :         n = __rte_ring_rts_move_cons_head(r, n, behavior, &head, &entries);
     251                 :            : 
     252   [ -  -  -  -  :       2760 :         if (n != 0) {
          -  -  -  -  -  
          -  -  -  +  -  
          +  -  +  -  +  
          -  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                      # ]
     253                 :            :                 __rte_ring_dequeue_elems(r, head, obj_table, esize, n);
     254                 :            :                 __rte_ring_rts_update_tail(&r->rts_cons);
     255                 :            :         }
     256                 :            : 
     257   [ -  +  -  +  :       2760 :         if (available != NULL)
             -  +  -  + ]
     258                 :          0 :                 *available = entries - n;
     259                 :            :         return n;
     260                 :            : }
     261                 :            : 
     262                 :            : #endif /* _RTE_RING_RTS_ELEM_PVT_H_ */

Generated by: LCOV version 1.14