From b94901f3ede8fa5ea334f59bf947eba7d0cb24f3 Mon Sep 17 00:00:00 2001 From: Eyal Perry Date: Tue, 22 Jul 2014 15:44:09 +0300 Subject: [PATCH 1/4] net/mlx4_en: current_mac isn't updated in port up When port is down dev_addr is changed (e.g. by bonding) but current_mac is not touched. When port is up again, hash_mac is updated to dev_addr, but current_mac isn't. This leads to inconsistency between current_mac and mac_hash. Because of that, mlx4_en_replace_mac() fails to find current_mac in mac_hash. Fix is to reset current_mac to dev_addr when port is up - as we do for mac_hash. Signed-off-by: Eyal Perry Signed-off-by: Amir Vadai Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx4/en_netdev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index 887cf01d831d..82708bd5c339 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -644,6 +644,7 @@ static int mlx4_en_get_qp(struct mlx4_en_priv *priv) goto alloc_err; } memcpy(entry->mac, priv->dev->dev_addr, sizeof(entry->mac)); + memcpy(priv->current_mac, entry->mac, sizeof(priv->current_mac)); entry->reg_id = reg_id; hlist_add_head_rcu(&entry->hlist, From 0fef9d0308d4c524da716b4b669d8754594450b2 Mon Sep 17 00:00:00 2001 From: Amir Vadai Date: Tue, 22 Jul 2014 15:44:10 +0300 Subject: [PATCH 2/4] net/mlx4_en: Disable blueflame using ethtool private flags Enable the user to turn off the hardware feature called BlueFlame. Since it is something specific to mlx4_en hardware, we control the feature via ethtool private flags. Signed-off-by: Amir Vadai Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlx4/en_ethtool.c | 57 +++++++++++++++++++ .../net/ethernet/mellanox/mlx4/en_netdev.c | 1 + drivers/net/ethernet/mellanox/mlx4/en_tx.c | 13 +++-- drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 5 ++ 4 files changed, 72 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c index 68d763d2d030..50e85cc1d61f 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c @@ -98,6 +98,10 @@ mlx4_en_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) drvinfo->eedump_len = 0; } +static const char mlx4_en_priv_flags[][ETH_GSTRING_LEN] = { + "blueflame", +}; + static const char main_strings[][ETH_GSTRING_LEN] = { "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", @@ -235,6 +239,8 @@ static int mlx4_en_get_sset_count(struct net_device *dev, int sset) case ETH_SS_TEST: return MLX4_EN_NUM_SELF_TEST - !(priv->mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UC_LOOPBACK) * 2; + case ETH_SS_PRIV_FLAGS: + return ARRAY_SIZE(mlx4_en_priv_flags); default: return -EOPNOTSUPP; } @@ -358,6 +364,12 @@ static void mlx4_en_get_strings(struct net_device *dev, #endif } break; + case ETH_SS_PRIV_FLAGS: + for (i = 0; i < ARRAY_SIZE(mlx4_en_priv_flags); i++) + strcpy(data + i * ETH_GSTRING_LEN, + mlx4_en_priv_flags[i]); + break; + } } @@ -1209,6 +1221,49 @@ static int mlx4_en_get_ts_info(struct net_device *dev, return ret; } +int mlx4_en_set_priv_flags(struct net_device *dev, u32 flags) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + bool bf_enabled_new = !!(flags & MLX4_EN_PRIV_FLAGS_BLUEFLAME); + bool bf_enabled_old = !!(priv->pflags & MLX4_EN_PRIV_FLAGS_BLUEFLAME); + int i; + + if (bf_enabled_new == bf_enabled_old) + return 0; /* Nothing to do */ + + if (bf_enabled_new) { + bool bf_supported = true; + + for (i = 0; i < priv->tx_ring_num; i++) + bf_supported &= priv->tx_ring[i]->bf_alloced; + + if (!bf_supported) { + en_err(priv, "BlueFlame is not supported\n"); + return -EINVAL; + } + + priv->pflags |= MLX4_EN_PRIV_FLAGS_BLUEFLAME; + } else { + priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME; + } + + for (i = 0; i < priv->tx_ring_num; i++) + priv->tx_ring[i]->bf_enabled = bf_enabled_new; + + en_info(priv, "BlueFlame %s\n", + bf_enabled_new ? "Enabled" : "Disabled"); + + return 0; +} + +u32 mlx4_en_get_priv_flags(struct net_device *dev) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + + return priv->pflags; +} + + const struct ethtool_ops mlx4_en_ethtool_ops = { .get_drvinfo = mlx4_en_get_drvinfo, .get_settings = mlx4_en_get_settings, @@ -1236,6 +1291,8 @@ const struct ethtool_ops mlx4_en_ethtool_ops = { .get_channels = mlx4_en_get_channels, .set_channels = mlx4_en_set_channels, .get_ts_info = mlx4_en_get_ts_info, + .set_priv_flags = mlx4_en_set_priv_flags, + .get_priv_flags = mlx4_en_get_priv_flags, }; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index 82708bd5c339..bb536aa613f4 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -2465,6 +2465,7 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, priv->port = port; priv->port_up = false; priv->flags = prof->flags; + priv->pflags = MLX4_EN_PRIV_FLAGS_BLUEFLAME; priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | MLX4_WQE_CTRL_SOLICITED); priv->num_tx_rings_p_up = mdev->profile.num_tx_rings_p_up; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 5045bab59633..dae3da6d8dd0 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -126,8 +126,13 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, ring->bf.uar = &mdev->priv_uar; ring->bf.uar->map = mdev->uar_map; ring->bf_enabled = false; - } else - ring->bf_enabled = true; + ring->bf_alloced = false; + priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME; + } else { + ring->bf_alloced = true; + ring->bf_enabled = !!(priv->pflags & + MLX4_EN_PRIV_FLAGS_BLUEFLAME); + } ring->hwtstamp_tx_type = priv->hwtstamp_config.tx_type; ring->queue_index = queue_index; @@ -161,7 +166,7 @@ void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring = *pring; en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); - if (ring->bf_enabled) + if (ring->bf_alloced) mlx4_bf_free(mdev->dev, &ring->bf); mlx4_qp_remove(mdev->dev, &ring->qp); mlx4_qp_free(mdev->dev, &ring->qp); @@ -195,7 +200,7 @@ int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 1, 0, ring->qpn, ring->cqn, user_prio, &ring->context); - if (ring->bf_enabled) + if (ring->bf_alloced) ring->context.usr_page = cpu_to_be32(ring->bf.uar->index); err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, &ring->context, diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index 2b19dd1f2c5d..54c277412e42 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -93,6 +93,8 @@ * OS related constants and tunables */ +#define MLX4_EN_PRIV_FLAGS_BLUEFLAME 1 + #define MLX4_EN_WATCHDOG_TIMEOUT (15 * HZ) /* Use the maximum between 16384 and a single page */ @@ -278,6 +280,7 @@ struct mlx4_en_tx_ring { unsigned long wake_queue; struct mlx4_bf bf; bool bf_enabled; + bool bf_alloced; struct netdev_queue *tx_queue; int hwtstamp_tx_type; int inline_thold; @@ -592,6 +595,8 @@ struct mlx4_en_priv { #endif u64 tunnel_reg_id; __be16 vxlan_port; + + u32 pflags; }; enum mlx4_en_wol { From 2599d8580f93f0794d2fa850501b1068ce1d0aa8 Mon Sep 17 00:00:00 2001 From: Amir Vadai Date: Tue, 22 Jul 2014 15:44:11 +0300 Subject: [PATCH 3/4] net/mlx4_core: Use low memory profile on kdump kernel When running in kdump kernel, reduce number of resources allocated for the hardware. This will enable the NIC to operate in this low memory environment at the expense of performance and some features not related to the basic NIC functionality. Signed-off-by: Amir Vadai Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx4/main.c | 28 ++++++++++++++++++++--- include/linux/mlx4/device.h | 7 ++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index 82ab427290c3..80b8c5f30e4e 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -120,6 +120,16 @@ static struct mlx4_profile default_profile = { .num_mtt = 1 << 20, /* It is really num mtt segements */ }; +static struct mlx4_profile low_mem_profile = { + .num_qp = 1 << 17, + .num_srq = 1 << 6, + .rdmarc_per_qp = 1 << 4, + .num_cq = 1 << 8, + .num_mcg = 1 << 8, + .num_mpt = 1 << 9, + .num_mtt = 1 << 7, +}; + static int log_num_mac = 7; module_param_named(log_num_mac, log_num_mac, int, 0444); MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)"); @@ -129,6 +139,8 @@ module_param_named(log_num_vlan, log_num_vlan, int, 0444); MODULE_PARM_DESC(log_num_vlan, "Log2 max number of VLANs per ETH port (0-7)"); /* Log2 max number of VLANs per ETH port (0-7) */ #define MLX4_LOG_NUM_VLANS 7 +#define MLX4_MIN_LOG_NUM_VLANS 0 +#define MLX4_MIN_LOG_NUM_MAC 1 static bool use_prio; module_param_named(use_prio, use_prio, bool, 0444); @@ -287,8 +299,13 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) if (mlx4_is_mfunc(dev)) dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_SENSE_SUPPORT; - dev->caps.log_num_macs = log_num_mac; - dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS; + if (mlx4_low_memory_profile()) { + dev->caps.log_num_macs = MLX4_MIN_LOG_NUM_MAC; + dev->caps.log_num_vlans = MLX4_MIN_LOG_NUM_VLANS; + } else { + dev->caps.log_num_macs = log_num_mac; + dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS; + } for (i = 1; i <= dev->caps.num_ports; ++i) { dev->caps.port_type[i] = MLX4_PORT_TYPE_NONE; @@ -1587,7 +1604,12 @@ static int mlx4_init_hca(struct mlx4_dev *dev) if (mlx4_is_master(dev)) mlx4_parav_master_pf_caps(dev); - profile = default_profile; + if (mlx4_low_memory_profile()) { + mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n"); + profile = low_mem_profile; + } else { + profile = default_profile; + } if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) profile.num_mcg = MLX4_FS_NUM_MCG; diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h index fa660aedb822..e15b1544ea83 100644 --- a/include/linux/mlx4/device.h +++ b/include/linux/mlx4/device.h @@ -1254,4 +1254,11 @@ int mlx4_vf_smi_enabled(struct mlx4_dev *dev, int slave, int port); int mlx4_vf_get_enable_smi_admin(struct mlx4_dev *dev, int slave, int port); int mlx4_vf_set_enable_smi_admin(struct mlx4_dev *dev, int slave, int port, int enable); + +/* Returns true if running in low memory profile (kdump kernel) */ +static inline bool mlx4_low_memory_profile(void) +{ + return reset_devices; +} + #endif /* MLX4_DEVICE_H */ From ea1c1af1396cac9f8a1160acdac17f80e4b4f2c4 Mon Sep 17 00:00:00 2001 From: Amir Vadai Date: Tue, 22 Jul 2014 15:44:12 +0300 Subject: [PATCH 4/4] net/mlx4_en: Reduce memory consumption on kdump kernel When memory is limited, reduce number of rx and tx rings. Signed-off-by: Amir Vadai Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx4/en_main.c | 6 ++++-- drivers/net/ethernet/mellanox/mlx4/en_rx.c | 5 +++-- drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_main.c b/drivers/net/ethernet/mellanox/mlx4/en_main.c index f953c1d7eae6..3626fdf4cb5d 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_main.c @@ -129,8 +129,10 @@ static int mlx4_en_get_profile(struct mlx4_en_dev *mdev) int i; params->udp_rss = udp_rss; - params->num_tx_rings_p_up = min_t(int, num_online_cpus(), - MLX4_EN_MAX_TX_RING_P_UP); + params->num_tx_rings_p_up = mlx4_low_memory_profile() ? + MLX4_EN_MIN_TX_RING_P_UP : + min_t(int, num_online_cpus(), MLX4_EN_MAX_TX_RING_P_UP); + if (params->udp_rss && !(mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS)) { mlx4_warn(mdev, "UDP RSS is not supported on this device\n"); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 7765a08f9e84..9c909d23f14c 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -335,8 +335,9 @@ void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev) dev->caps.comp_pool/ dev->caps.num_ports) - 1; - num_rx_rings = min_t(int, num_of_eqs, - netif_get_num_default_rss_queues()); + num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS : + min_t(int, num_of_eqs, + netif_get_num_default_rss_queues()); mdev->profile.prof[i].rx_ring_num = rounddown_pow_of_two(num_rx_rings); } diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index 54c277412e42..3de41be49425 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -121,6 +121,7 @@ enum { #define MLX4_EN_MIN_TX_SIZE (4096 / TXBB_SIZE) #define MLX4_EN_SMALL_PKT_SIZE 64 +#define MLX4_EN_MIN_TX_RING_P_UP 1 #define MLX4_EN_MAX_TX_RING_P_UP 32 #define MLX4_EN_NUM_UP 8 #define MLX4_EN_DEF_TX_RING_SIZE 512