[doc][dash-p4] Update metering bucket design (#611)

dash-pipeline/SAI/specs/dash_meter.yaml

@@ -4,42 +4,59 @@ description: DASH meter
 api_type: overlay
 sai_apis:
 - !!python/object:utils.sai_spec.sai_api.SaiApi
-  name: meter_bucket
-  description: meter bucket
-  is_object: true
-  enums: []
-  structs: []
+  name: meter_bucket_entry
+  description: meter bucket entry
+  is_object: false
+  enums:
+  - !!python/object:utils.sai_spec.sai_enum.SaiEnum
+    name: sai_meter_bucket_entry_action_t
+    description: 'Attribute data for #SAI_METER_BUCKET_ENTRY_ATTR_ACTION'
+    members:
+    - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+      name: SAI_METER_BUCKET_ENTRY_ACTION_UPDATE_METER_BUCKET
+      description: ''
+      value: '0'
+  structs:
+  - !!python/object:utils.sai_spec.sai_struct.SaiStruct
+    name: sai_meter_bucket_entry_t
+    description: Entry for meter_bucket_entry
+    members:
+    - !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry
+      name: switch_id
+      description: Switch ID
+      type: sai_object_id_t
+      objects: SAI_OBJECT_TYPE_SWITCH
+      valid_only: null
+    - !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry
+      name: eni_id
+      description: Exact matched key eni_id
+      type: sai_object_id_t
+      objects: SAI_OBJECT_TYPE_ENI
+      valid_only: null
+    - !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry
+      name: meter_class
+      description: Exact matched key meter_class
+      type: sai_uint32_t
+      objects: null
+      valid_only: null
   attributes:
   - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
-    name: SAI_METER_BUCKET_ATTR_ENI_ID
-    description: Exact matched key eni_id
-    type: sai_object_id_t
-    attr_value_field: u16
-    default: null
-    isresourcetype: false
-    flags: MANDATORY_ON_CREATE | CREATE_ONLY
-    object_name: SAI_OBJECT_TYPE_ENI
-    allow_null: false
-    valid_only: null
-    is_vlan: false
-    deprecated: false
-  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
-    name: SAI_METER_BUCKET_ATTR_METER_CLASS
-    description: Exact matched key meter_class
-    type: sai_uint32_t
-    attr_value_field: u32
-    default: null
+    name: SAI_METER_BUCKET_ENTRY_ATTR_ACTION
+    description: Action
+    type: sai_meter_bucket_entry_action_t
+    attr_value_field: null
+    default: SAI_METER_BUCKET_ENTRY_ACTION_UPDATE_METER_BUCKET
     isresourcetype: false
-    flags: MANDATORY_ON_CREATE | CREATE_ONLY
+    flags: CREATE_AND_SET
     object_name: null
     allow_null: false
     valid_only: null
     is_vlan: false
     deprecated: false
   stats:
   - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
-    name: SAI_METER_BUCKET_STAT_OUTBOUND_BYTES
-    description: DASH METER_BUCKET OUTBOUND_BYTES stat count
+    name: SAI_METER_BUCKET_ENTRY_STAT_OUTBOUND_BYTES
+    description: DASH METER_BUCKET_ENTRY OUTBOUND_BYTES stat count
     type: sai_uint64_t
     attr_value_field: u64
     default: null
@@ -48,11 +65,11 @@ sai_apis:
     object_name: null
     allow_null: false
     valid_only: null
-    deprecated: false
     is_vlan: false
+    deprecated: false
   - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
-    name: SAI_METER_BUCKET_STAT_INBOUND_BYTES
-    description: DASH METER_BUCKET INBOUND_BYTES stat count
+    name: SAI_METER_BUCKET_ENTRY_STAT_INBOUND_BYTES
+    description: DASH METER_BUCKET_ENTRY INBOUND_BYTES stat count
     type: sai_uint64_t
     attr_value_field: u64
     default: null
@@ -61,16 +78,16 @@ sai_apis:
     object_name: null
     allow_null: false
     valid_only: null
-    deprecated: false
     is_vlan: false
+    deprecated: false
   p4_meta: !!python/object:utils.sai_spec.sai_api_p4_meta.SaiApiP4Meta
     tables:
     - !!python/object:utils.sai_spec.sai_api_p4_meta.SaiApiP4MetaTable
       id: 45482818
       actions:
-        default: !!python/object:utils.sai_spec.sai_api_p4_meta.SaiApiP4MetaAction
-          name: default
-          id: 21630451
+        SAI_METER_BUCKET_ENTRY_ACTION_UPDATE_METER_BUCKET: !!python/object:utils.sai_spec.sai_api_p4_meta.SaiApiP4MetaAction
+          name: SAI_METER_BUCKET_ENTRY_ACTION_UPDATE_METER_BUCKET
+          id: 23515737
           attr_param_id: {}
 - !!python/object:utils.sai_spec.sai_api.SaiApi
   name: meter_policy

dash-pipeline/SAI/specs/sai_spec.yaml

@@ -38,6 +38,7 @@ object_types:
 - SAI_OBJECT_TYPE_FLOW_ENTRY
 - SAI_OBJECT_TYPE_FLOW_ENTRY_BULK_GET_SESSION_FILTER
 - SAI_OBJECT_TYPE_FLOW_ENTRY_BULK_GET_SESSION
+- SAI_OBJECT_TYPE_METER_BUCKET_ENTRY
 object_entries:
 - !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry
   name: direction_lookup_entry
@@ -93,6 +94,12 @@ object_entries:
   type: sai_flow_entry_t
   objects: null
   valid_only: object_type == SAI_OBJECT_TYPE_FLOW_ENTRY,
+- !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry
+  name: meter_bucket_entry
+  description: Object entry for DASH API meter_bucket_entry
+  type: sai_meter_bucket_entry_t
+  objects: null
+  valid_only: object_type == SAI_OBJECT_TYPE_METER_BUCKET_ENTRY,
 enums:
 - !!python/object:utils.sai_spec.sai_enum.SaiEnum
   name: sai_dash_direction_t

dash-pipeline/SAI/templates/headers/sai_stats.j2

@@ -1,5 +1,5 @@
 /**
- * @brief Counter IDs for {{ api.description | upper }}
+ * @brief Counter IDs for {{ api.description }}
  */
 typedef enum _sai_{{ api.name }}_stat_t
 {

dash-pipeline/bmv2/stages/metering_update.p4

@@ -47,18 +47,18 @@ control metering_update_stage(
 
     // MAX_METER_BUCKET = MAX_ENI(64) * NUM_BUCKETS_PER_ENI(4096)
     #define MAX_METER_BUCKETS 262144
-    DEFINE_BYTE_COUNTER(meter_bucket_outbound, MAX_METER_BUCKETS, name="outbound", action_names="meter_bucket_action", attr_type="stats")
-    DEFINE_BYTE_COUNTER(meter_bucket_inbound, MAX_METER_BUCKETS, name="inbound", action_names="meter_bucket_action", attr_type="stats")
-    action meter_bucket_action() {}
+    DEFINE_BYTE_COUNTER(meter_bucket_outbound, MAX_METER_BUCKETS, name="outbound", action_names="update_meter_bucket", attr_type="stats")
+    DEFINE_BYTE_COUNTER(meter_bucket_inbound, MAX_METER_BUCKETS, name="inbound", action_names="update_meter_bucket", attr_type="stats")
+    action update_meter_bucket() {}
 
-    @SaiTable[name = "meter_bucket", api = "dash_meter", order = 0, isobject="true"]
+    @SaiTable[name = "meter_bucket", api = "dash_meter", order = 0]
     table meter_bucket {
         key = {
             meta.eni_id: exact @SaiVal[type="sai_object_id_t"];
             meta.meter_class: exact;
         }
         actions = {
-            meter_bucket_action;
+            update_meter_bucket;
             @defaultonly NoAction;
         }
         const default_action = NoAction();

documentation/metering/metering.md

@@ -23,6 +23,9 @@
       3. [3.3.3. Meter rule](#333-meter-rule)
    4. [3.4. Capability](#34-capability)
 4. [4. Metering bucket selection in DASH pipeline](#4-metering-bucket-selection-in-dash-pipeline)
+5. [5. Metering bucket stats fetch process](#5-metering-bucket-stats-fetch-process)
+   1. [5.1. Metering bucket creation](#51-metering-bucket-creation)
+   2. [5.2. Fetching metering bucket stats](#52-fetching-metering-bucket-stats)
 
 ## 1. Background
 
@@ -61,10 +64,12 @@ The following attributes will be involved in determining the final metering buck
 
 ### 3.1. Meter bucket
 
+The meter bucket is defined as an entry table with each entry defined as below:
+
 | Attribute | Type | Default Value | Description |
 | --- | --- | --- | --- |
-| SAI_METER_BUCKET_ATTR_ENI_ID | `sai_object_id_t` | SAI_NULL_OBJECT_ID | ENI ID of this metering class. |
-| SAI_METER_BUCKET_ATTR_METER_CLASS | `sai_uint32_t` | 0 | Meter class of this meter bucket. |
+| eni_id | `sai_object_id_t` | SAI_NULL_OBJECT_ID | ENI ID of this metering class. |
+| meter_class | `sai_uint32_t` | 0 | Meter class of this meter bucket. |
 
 To fetch the metering data from each meter bucket, we are going to leverage the SAI stats APIs, which provides get, get and clear and other frequently used semantics. It will also reduce the work in SONiC stack, as SONiC already have good support over the stats APIs.
 
@@ -152,3 +157,30 @@ When a packet arrives at an ENI, it will go through the steps below to find its
    1. Meter policy v4 or v6 will be selected based on the IP family of the original overlay packet.
    2. The overlay destination (outbound pipeline) / source (inbound pipeline) IP will be used for ternary match against the meter rules in the meter policy to find the meter class.
 6. **Meter Update**: The final meter class will be used for update the counters in meter bucket. If final meter class is 0, no meter bucket will be updated.
+
+## 5. Metering bucket stats fetch process
+
+### 5.1. Metering bucket creation
+
+Whenever the ENI is created, we will assume the all metering buckets are created together with the ENI and initialized to 0, without the need of any additional SAI API calls, such as `create_meter_bucket`.
+
+Each ENI will maintain the metering bucket from 1 to SAI_SWITCH_ATTR_DASH_CAPS_MAX_METER_BUCKET_COUNT_PER_ENI, inclusive.
+
+### 5.2. Fetching metering bucket stats
+
+Every once a while, the counters of each metering bucket will be pulled. 
+
+To save the cost, it is preferred to use the buck get stats API defined in `saiobject.h` to get all stats back in one call, with all metering bucket entry specified:
+
+```c
+sai_status_t sai_bulk_object_get_stats(
+        _In_ sai_object_id_t switch_id,
+        _In_ sai_object_type_t object_type,
+        _In_ uint32_t object_count,
+        _In_ const sai_object_key_t *object_key,
+        _In_ uint32_t number_of_counters,
+        _In_ const sai_stat_id_t *counter_ids,
+        _In_ sai_stats_mode_t mode,
+        _Inout_ sai_status_t *object_statuses,
+        _Out_ uint64_t *counters);
+```