ENA: Update to v2.8.3

Source: https://github.com/amzn/amzn-drivers/

Change Log:

## r2.8.3 release notes
**New Features**
* PHC module param enablement
* PHC devlink param enablement
* Add hint for interrupt moderation for the device
* Change initial static RX interrupt moderation interval
* Enable DIM by default on all CPU Architectures

**Buf Fixes**
* DMA sync for CPU before accessing buffer
* Fix ena_probe destroy order
* Validate completion descriptors consistency
* Fix TX packets missing completion counter

**Minor Changes**
* Compilation fixes for RHEL 9.0, 9.1 and SLES 15SP4
* PHC info dynamic allocation
* Publish devlink reload for RHEL 9.0 and 9.1
* Add ENA Express documentation

## r2.8.2 release notes
**Buf Fixes**
* Fix devlink large LLQ config not fully applied

Signed-off-by: Andrew Panyakin <apanyaki@amazon.com>

Author: paniakin-aws

drivers/amazon/net/ena/ena_com.c

@@ -1823,8 +1823,8 @@ int ena_com_phc_config(struct ena_com_dev *ena_dev)
 		return ret;
 	}
 
-	phc->enabled = true;
-	netdev_dbg(ena_dev->net_device, "PHC is enabled\n");
+	phc->active = true;
+	netdev_dbg(ena_dev->net_device, "PHC is active in the device\n");
 
 	return ret;
 }
@@ -1833,7 +1833,7 @@ void ena_com_phc_destroy(struct ena_com_dev *ena_dev)
 {
 	struct ena_com_phc_info *phc = &ena_dev->phc;
 
-	phc->enabled = false;
+	phc->active = false;
 
 	/* In case PHC is not supported by the device, silently exiting */
 	if (!phc->virt_addr)
@@ -1855,8 +1855,9 @@ int ena_com_phc_get(struct ena_com_dev *ena_dev, u64 *timestamp)
 	ktime_t block_time;
 	int ret = 0;
 
-	if (!phc->enabled) {
-		netdev_err(ena_dev->net_device, "PHC feature is not enabled\n");
+	if (!phc->active) {
+		netdev_err(ena_dev->net_device,
+			   "PHC feature is not active in the device\n");
 		return -EOPNOTSUPP;
 	}
 

drivers/amazon/net/ena/ena_com.h

@@ -49,7 +49,7 @@
 /* ENA adaptive interrupt moderation settings */
 
 #define ENA_INTR_INITIAL_TX_INTERVAL_USECS 64
-#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 0
+#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 20
 #define ENA_DEFAULT_INTR_DELAY_RESOLUTION 1
 
 #define ENA_HASH_KEY_SIZE 40
@@ -302,8 +302,8 @@ struct ena_com_phc_info {
 	/* Request id sent to the device */
 	u16 req_id;
 
-	/* True if PHC is enabled */
-	bool enabled;
+	/* True if PHC is active in the device */
+	bool active;
 
 	/* PHC shared memory - memory handle */
 
@@ -1146,13 +1146,16 @@ static inline int ena_com_get_customer_metric_count(struct ena_com_dev *ena_dev)
  * @rx_delay_interval: Rx interval in usecs
  * @tx_delay_interval: Tx interval in usecs
  * @unmask: unmask enable/disable
+ * @no_moderation_update: 0 - Indicates that any of the TX/RX intervals was
+ *                        updated, 1 - otherwise
  *
  * Prepare interrupt update register with the supplied parameters.
  */
 static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg,
 					   u32 rx_delay_interval,
 					   u32 tx_delay_interval,
-					   bool unmask)
+					   bool unmask,
+					   bool no_moderation_update)
 {
 	intr_reg->intr_control = 0;
 	intr_reg->intr_control |= rx_delay_interval &
@@ -1164,6 +1167,10 @@ static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg,
 
 	if (unmask)
 		intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK;
+
+	intr_reg->intr_control |=
+		(((u32)no_moderation_update) << ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT) &
+			ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK;
 }
 
 static inline u8 *ena_com_get_next_bounce_buffer(struct ena_com_io_bounce_buffer_control *bounce_buf_ctrl)

drivers/amazon/net/ena/ena_devlink.c

@@ -1,12 +1,19 @@
 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 /*
- * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2015-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include "linux/pci.h"
 
 #include "ena_devlink.h"
 #ifdef ENA_DEVLINK_SUPPORT
+#ifdef ENA_PHC_SUPPORT
+#include "ena_phc.h"
+
+static int ena_devlink_phc_enable_validate(struct devlink *devlink, u32 id,
+					   union devlink_param_value val,
+					   struct netlink_ext_ack *extack);
+#endif /* ENA_PHC_SUPPORT */
 
 static int ena_devlink_llq_header_validate(struct devlink *devlink, u32 id,
 					   union devlink_param_value val,
@@ -15,13 +22,22 @@ static int ena_devlink_llq_header_validate(struct devlink *devlink, u32 id,
 enum ena_devlink_param_id {
 	ENA_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
 	ENA_DEVLINK_PARAM_ID_LLQ_HEADER_SIZE,
+#ifdef ENA_PHC_SUPPORT
+	ENA_DEVLINK_PARAM_ID_PHC_ENABLE,
+#endif /* ENA_PHC_SUPPORT */
 };
 
 static const struct devlink_param ena_devlink_params[] = {
 	DEVLINK_PARAM_DRIVER(ENA_DEVLINK_PARAM_ID_LLQ_HEADER_SIZE,
 			     "large_llq_header", DEVLINK_PARAM_TYPE_BOOL,
 			     BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
 			     NULL, NULL, ena_devlink_llq_header_validate),
+#ifdef ENA_PHC_SUPPORT
+	DEVLINK_PARAM_DRIVER(ENA_DEVLINK_PARAM_ID_PHC_ENABLE,
+			     "phc_enable", DEVLINK_PARAM_TYPE_BOOL,
+			     BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
+			     NULL, NULL, ena_devlink_phc_enable_validate),
+ #endif /* ENA_PHC_SUPPORT */
 };
 
 static int ena_devlink_llq_header_validate(struct devlink *devlink, u32 id,
@@ -47,6 +63,25 @@ static int ena_devlink_llq_header_validate(struct devlink *devlink, u32 id,
 	return 0;
 }
 
+#ifdef ENA_PHC_SUPPORT
+static int ena_devlink_phc_enable_validate(struct devlink *devlink, u32 id,
+					   union devlink_param_value val,
+					   struct netlink_ext_ack *extack)
+{
+	struct ena_adapter *adapter = ENA_DEVLINK_PRIV(devlink);
+
+	if (!val.vbool)
+		return 0;
+
+	if (!ena_com_phc_supported(adapter->ena_dev)) {
+		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support PHC");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+#endif /* ENA_PHC_SUPPORT */
 #ifdef ENA_DEVLINK_CONFIGURE_AFTER_REGISTER
 /* Determines if ena_devlink_register has been called.
  * Prefer to check if the driver enabled reloading capabilities, but fallback
@@ -82,6 +117,16 @@ void ena_devlink_params_get(struct devlink *devlink)
 	}
 
 	adapter->large_llq_header_enabled = val.vbool;
+#ifdef ENA_PHC_SUPPORT
+
+	err = devlink_param_driverinit_value_get(devlink, ENA_DEVLINK_PARAM_ID_PHC_ENABLE, &val);
+	if (err) {
+		netdev_err(adapter->netdev, "Failed to query PHC param\n");
+		return;
+	}
+
+	ena_phc_enable(adapter, val.vbool);
+#endif /* ENA_PHC_SUPPORT */
 }
 
 void ena_devlink_disable_large_llq_header_param(struct devlink *devlink)
@@ -100,6 +145,22 @@ void ena_devlink_disable_large_llq_header_param(struct devlink *devlink)
 					   value);
 }
 
+#ifdef ENA_PHC_SUPPORT
+void ena_devlink_disable_phc_param(struct devlink *devlink)
+{
+	union devlink_param_value value;
+
+#ifdef ENA_DEVLINK_CONFIGURE_AFTER_REGISTER
+	/* If devlink params aren't registered, don't access them */
+	if (!ena_is_devlink_params_registered(devlink))
+		return;
+
+#endif
+	value.vbool = false;
+	devlink_param_driverinit_value_set(devlink, ENA_DEVLINK_PARAM_ID_PHC_ENABLE, value);
+}
+
+#endif /* ENA_PHC_SUPPORT */
 static int ena_devlink_reload_down(struct devlink *devlink,
 #ifdef ENA_DEVLINK_RELOAD_NS_CHANGE_SUPPORT
 				   bool netns_change,
@@ -164,11 +225,11 @@ static int ena_devlink_reload_up(struct devlink *devlink,
 	rtnl_lock();
 	/* Check that no other routine initialized the device (e.g.
 	 * ena_fw_reset_device()). Also we're under devlink_mutex here,
-	 * so devink (and ena_adapter with it) isn't freed under our
-	 * feet.
+	 * so devlink isn't freed under our feet.
 	 */
 	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
 		err = ena_restore_device(adapter);
+
 	rtnl_unlock();
 
 #ifdef ENA_DEVLINK_RELOAD_LIMIT_AND_ACTION_SUPPORT
@@ -220,9 +281,18 @@ static int ena_devlink_configure_params(struct devlink *devlink)
 					   ENA_DEVLINK_PARAM_ID_LLQ_HEADER_SIZE,
 					   value);
 
+#ifdef ENA_PHC_SUPPORT
+	value.vbool = ena_phc_is_enabled(adapter);
+	devlink_param_driverinit_value_set(devlink, ENA_DEVLINK_PARAM_ID_PHC_ENABLE, value);
+
+#endif /* ENA_PHC_SUPPORT */
 #ifdef ENA_DEVLINK_RELOAD_SUPPORT_ADVERTISEMENT_NEEDED
 	devlink_set_features(devlink, DEVLINK_F_RELOAD);
 
+#endif
+#ifdef ENA_DEVLINK_PUBLISH_REQUIRED
+	devlink_params_publish(devlink);
+
 #endif
 #ifdef ENA_DEVLINK_RELOAD_ENABLING_REQUIRED
 	devlink_reload_enable(devlink);
@@ -270,6 +340,10 @@ static void ena_devlink_configure_params_clean(struct devlink *devlink)
 #ifdef ENA_DEVLINK_RELOAD_ENABLING_REQUIRED
 	devlink_reload_disable(devlink);
 
+#endif
+#ifdef ENA_DEVLINK_PUBLISH_REQUIRED
+	devlink_params_unpublish(devlink);
+
 #endif
 	devlink_params_unregister(devlink, ena_devlink_params,
 				  ARRAY_SIZE(ena_devlink_params));

drivers/amazon/net/ena/ena_devlink.h

@@ -1,6 +1,6 @@
-// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
 /*
- * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2015-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef DEVLINK_H
@@ -12,19 +12,18 @@
 #endif
 
 #ifdef ENA_DEVLINK_SUPPORT
-
 #define ENA_DEVLINK_PRIV(devlink) \
-	(*(struct ena_adapter **) devlink_priv(devlink))
+	(*(struct ena_adapter **)devlink_priv(devlink))
 
 struct devlink *ena_devlink_alloc(struct ena_adapter *adapter);
 void ena_devlink_free(struct devlink *devlink);
 void ena_devlink_register(struct devlink *devlink, struct device *dev);
 void ena_devlink_unregister(struct devlink *devlink);
 void ena_devlink_params_get(struct devlink *devlink);
 void ena_devlink_disable_large_llq_header_param(struct devlink *devlink);
+void ena_devlink_disable_phc_param(struct devlink *devlink);
 
 #else /* ENA_DEVLINK_SUPPORT */
-
 #ifdef ENA_NO_DEVLINK_HEADERS
 struct devlink {};
 #endif
@@ -39,7 +38,7 @@ static inline void ena_devlink_register(struct devlink *devlink, struct device *
 static inline void ena_devlink_unregister(struct devlink *devlink) { }
 static inline void ena_devlink_params_get(struct devlink *devlink) { }
 static inline void ena_devlink_disable_large_llq_header_param(struct devlink *devlink) { }
+static inline void ena_devlink_disable_phc_param(struct devlink *devlink) { }
 
 #endif /* ENA_DEVLINK_SUPPORT */
-
 #endif /* DEVLINK_H */

drivers/amazon/net/ena/ena_eth_com.c

@@ -233,43 +233,55 @@ static struct ena_eth_io_rx_cdesc_base *
 		idx * io_cq->cdesc_entry_size_in_bytes);
 }
 
-static u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
-					   u16 *first_cdesc_idx)
+static int ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
+				    u16 *first_cdesc_idx,
+				    u16 *num_descs)
 {
+	u16 count = io_cq->cur_rx_pkt_cdesc_count, head_masked;
 	struct ena_eth_io_rx_cdesc_base *cdesc;
-	u16 count = 0, head_masked;
 	u32 last = 0;
 
 	do {
+		u32 status;
+
 		cdesc = ena_com_get_next_rx_cdesc(io_cq);
 		if (!cdesc)
 			break;
+		status = READ_ONCE(cdesc->status);
 
 		ena_com_cq_inc_head(io_cq);
+		if (unlikely((status & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK) >>
+		    ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT && count != 0)) {
+			struct ena_com_dev *dev = ena_com_io_cq_to_ena_dev(io_cq);
+
+			netdev_err(dev->net_device,
+				   "First bit is on in descriptor #%d on q_id: %d, req_id: %u\n",
+				   count, io_cq->qid, cdesc->req_id);
+			return -EFAULT;
+		}
 		count++;
-		last = (READ_ONCE(cdesc->status) &
-			ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >>
-		       ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT;
+		last = (status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >>
+			ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT;
 	} while (!last);
 
 	if (last) {
 		*first_cdesc_idx = io_cq->cur_rx_pkt_cdesc_start_idx;
-		count += io_cq->cur_rx_pkt_cdesc_count;
 
 		head_masked = io_cq->head & (io_cq->q_depth - 1);
 
+		*num_descs = count;
 		io_cq->cur_rx_pkt_cdesc_count = 0;
 		io_cq->cur_rx_pkt_cdesc_start_idx = head_masked;
 
 		netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device,
 			   "ENA q_id: %d packets were completed. first desc idx %u descs# %d\n",
 			   io_cq->qid, *first_cdesc_idx, count);
 	} else {
-		io_cq->cur_rx_pkt_cdesc_count += count;
-		count = 0;
+		io_cq->cur_rx_pkt_cdesc_count = count;
+		*num_descs = 0;
 	}
 
-	return count;
+	return 0;
 }
 
 static int ena_com_create_meta(struct ena_com_io_sq *io_sq,
@@ -546,10 +558,14 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
 	u16 cdesc_idx = 0;
 	u16 nb_hw_desc;
 	u16 i = 0;
+	int rc;
 
 	WARN(io_cq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX, "wrong Q type");
 
-	nb_hw_desc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx);
+	rc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx, &nb_hw_desc);
+	if (unlikely(rc != 0))
+		return -EFAULT;
+
 	if (nb_hw_desc == 0) {
 		ena_rx_ctx->descs = nb_hw_desc;
 		return 0;

drivers/amazon/net/ena/ena_eth_io_defs.h

@@ -261,7 +261,8 @@ struct ena_eth_io_intr_reg {
 	/* 14:0 : rx_intr_delay
 	 * 29:15 : tx_intr_delay
 	 * 30 : intr_unmask
-	 * 31 : reserved
+	 * 31 : no_moderation_update - 0 - moderation
+	 *    updated, 1 - moderation not updated
 	 */
 	u32 intr_control;
 };
@@ -381,6 +382,8 @@ struct ena_eth_io_numa_node_cfg_reg {
 #define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK              GENMASK(29, 15)
 #define ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT               30
 #define ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK                BIT(30)
+#define ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT      31
+#define ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK       BIT(31)
 
 /* numa_node_cfg_reg */
 #define ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK              GENMASK(7, 0)