diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox')
78 files changed, 3127 insertions, 1641 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx4/Kconfig b/drivers/net/ethernet/mellanox/mlx4/Kconfig index ff8057ed97ee..8491db57b0b0 100644 --- a/drivers/net/ethernet/mellanox/mlx4/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx4/Kconfig @@ -26,6 +26,7 @@ config MLX4_EN_DCB config MLX4_CORE tristate depends on PCI + select NET_DEVLINK default n config MLX4_DEBUG diff --git a/drivers/net/ethernet/mellanox/mlx4/catas.c b/drivers/net/ethernet/mellanox/mlx4/catas.c index c81d15bf259c..87e90b5d4d7d 100644 --- a/drivers/net/ethernet/mellanox/mlx4/catas.c +++ b/drivers/net/ethernet/mellanox/mlx4/catas.c @@ -129,10 +129,6 @@ static int mlx4_reset_slave(struct mlx4_dev *dev) comm_flags = rst_req << COM_CHAN_RST_REQ_OFFSET; __raw_writel((__force u32)cpu_to_be32(comm_flags), (__iomem char *)priv->mfunc.comm + MLX4_COMM_CHAN_FLAGS); - /* Make sure that our comm channel write doesn't - * get mixed in with writes from another CPU. - */ - mmiowb(); end = msecs_to_jiffies(MLX4_COMM_TIME) + jiffies; while (time_before(jiffies, end)) { diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c index a5d5d6fc1da0..c678344d22a2 100644 --- a/drivers/net/ethernet/mellanox/mlx4/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c @@ -281,7 +281,6 @@ static int mlx4_comm_cmd_post(struct mlx4_dev *dev, u8 cmd, u16 param) val = param | (cmd << 16) | (priv->cmd.comm_toggle << 31); __raw_writel((__force u32) cpu_to_be32(val), &priv->mfunc.comm->slave_write); - mmiowb(); mutex_unlock(&dev->persist->device_state_mutex); return 0; } @@ -496,12 +495,6 @@ static int mlx4_cmd_post(struct mlx4_dev *dev, u64 in_param, u64 out_param, (op_modifier << HCR_OPMOD_SHIFT) | op), hcr + 6); - /* - * Make sure that our HCR writes don't get mixed in with - * writes from another CPU starting a FW command. - */ - mmiowb(); - cmd->toggle = cmd->toggle ^ 1; ret = 0; @@ -2206,7 +2199,6 @@ static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd, } __raw_writel((__force u32) cpu_to_be32(reply), &priv->mfunc.comm[slave].slave_read); - mmiowb(); return; @@ -2410,7 +2402,6 @@ int mlx4_multi_func_init(struct mlx4_dev *dev) &priv->mfunc.comm[i].slave_write); __raw_writel((__force u32) 0, &priv->mfunc.comm[i].slave_read); - mmiowb(); for (port = 1; port <= MLX4_MAX_PORTS; port++) { struct mlx4_vport_state *admin_vport; struct mlx4_vport_state *oper_vport; @@ -2576,10 +2567,6 @@ void mlx4_report_internal_err_comm_event(struct mlx4_dev *dev) slave_read |= (u32)COMM_CHAN_EVENT_INTERNAL_ERR; __raw_writel((__force u32)cpu_to_be32(slave_read), &priv->mfunc.comm[slave].slave_read); - /* Make sure that our comm channel write doesn't - * get mixed in with writes from another CPU. - */ - mmiowb(); } } diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 2cbd2bd7c67c..36a92b19e613 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -685,16 +685,15 @@ static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc, } u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev, - select_queue_fallback_t fallback) + struct net_device *sb_dev) { struct mlx4_en_priv *priv = netdev_priv(dev); u16 rings_p_up = priv->num_tx_rings_p_up; if (netdev_get_num_tc(dev)) - return fallback(dev, skb, NULL); + return netdev_pick_tx(dev, skb, NULL); - return fallback(dev, skb, NULL) % rings_p_up; + return netdev_pick_tx(dev, skb, NULL) % rings_p_up; } static void mlx4_bf_copy(void __iomem *dst, const void *src, @@ -1043,7 +1042,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) send_doorbell = __netdev_tx_sent_queue(ring->tx_queue, tx_info->nr_bytes, - skb->xmit_more); + netdev_xmit_more()); real_size = (real_size / 16) & 0x3f; diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index 8137454e2534..630f15977f09 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -698,8 +698,7 @@ void mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); void mlx4_en_tx_irq(struct mlx4_cq *mcq); u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev, - select_queue_fallback_t fallback); + struct net_device *sb_dev); netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev); netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_ring *rx_ring, struct mlx4_en_rx_alloc *frame, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index 6debffb8336b..9aca8086ee01 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -5,6 +5,7 @@ config MLX5_CORE tristate "Mellanox 5th generation network adapters (ConnectX series) core driver" depends on PCI + select NET_DEVLINK imply PTP_1588_CLOCK imply VXLAN default n diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 1a16f6d73cbc..243368dc23db 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -22,7 +22,8 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ # mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \ en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \ - en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o + en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o \ + en/params.o # # Netdev extra @@ -35,7 +36,7 @@ mlx5_core-$(CONFIG_MLX5_ESWITCH) += en_rep.o en_tc.o en/tc_tun.o lib/port_tu # # Core extra # -mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o ecpf.o +mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o ecpf.o rdma.o mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o mlx5_core-$(CONFIG_VXLAN) += lib/vxlan.o mlx5_core-$(CONFIG_PTP_1588_CLOCK) += lib/clock.o @@ -57,5 +58,3 @@ mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \ en_accel/ipsec_stats.o mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/tls.o en_accel/tls_rxtx.o en_accel/tls_stats.o - -CFLAGS_tracepoint.o := -I$(src) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c index 9008e17126db..549f962cd86e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c @@ -57,15 +57,16 @@ static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev, int node) { struct mlx5_priv *priv = &dev->priv; + struct device *device = dev->device; int original_node; void *cpu_handle; mutex_lock(&priv->alloc_mutex); - original_node = dev_to_node(&dev->pdev->dev); - set_dev_node(&dev->pdev->dev, node); - cpu_handle = dma_alloc_coherent(&dev->pdev->dev, size, dma_handle, + original_node = dev_to_node(device); + set_dev_node(device, node); + cpu_handle = dma_alloc_coherent(device, size, dma_handle, GFP_KERNEL); - set_dev_node(&dev->pdev->dev, original_node); + set_dev_node(device, original_node); mutex_unlock(&priv->alloc_mutex); return cpu_handle; } @@ -110,7 +111,7 @@ EXPORT_SYMBOL(mlx5_buf_alloc); void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) { - dma_free_coherent(&dev->pdev->dev, buf->size, buf->frags->buf, + dma_free_coherent(dev->device, buf->size, buf->frags->buf, buf->frags->map); kfree(buf->frags); @@ -139,7 +140,7 @@ int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, if (!frag->buf) goto err_free_buf; if (frag->map & ((1 << buf->page_shift) - 1)) { - dma_free_coherent(&dev->pdev->dev, frag_sz, + dma_free_coherent(dev->device, frag_sz, buf->frags[i].buf, buf->frags[i].map); mlx5_core_warn(dev, "unexpected map alignment: %pad, page_shift=%d\n", &frag->map, buf->page_shift); @@ -152,7 +153,7 @@ int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, err_free_buf: while (i--) - dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf->frags[i].buf, + dma_free_coherent(dev->device, PAGE_SIZE, buf->frags[i].buf, buf->frags[i].map); kfree(buf->frags); err_out: @@ -168,7 +169,7 @@ void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) for (i = 0; i < buf->npages; i++) { int frag_sz = min_t(int, size, PAGE_SIZE); - dma_free_coherent(&dev->pdev->dev, frag_sz, buf->frags[i].buf, + dma_free_coherent(dev->device, frag_sz, buf->frags[i].buf, buf->frags[i].map); size -= frag_sz; } @@ -274,7 +275,7 @@ void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db) __set_bit(db->index, db->u.pgdir->bitmap); if (bitmap_full(db->u.pgdir->bitmap, db_per_page)) { - dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, + dma_free_coherent(dev->device, PAGE_SIZE, db->u.pgdir->db_page, db->u.pgdir->db_dma); list_del(&db->u.pgdir->list); bitmap_free(db->u.pgdir->bitmap); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index be48c6440251..937ba4bcb056 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -917,7 +917,6 @@ static void cmd_work_handler(struct work_struct *work) mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx); wmb(); iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell); - mmiowb(); /* if not in polling don't use ent after this point */ if (cmd_mode == CMD_MODE_POLLING || poll_cmd) { poll_timeout(ent); @@ -1347,7 +1346,7 @@ static void set_wqname(struct mlx5_core_dev *dev) struct mlx5_cmd *cmd = &dev->cmd; snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s", - dev_name(&dev->pdev->dev)); + dev_name(dev->device)); } static void clean_debug_files(struct mlx5_core_dev *dev) @@ -1852,7 +1851,7 @@ static void create_msg_cache(struct mlx5_core_dev *dev) static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) { - struct device *ddev = &dev->pdev->dev; + struct device *ddev = dev->device; cmd->cmd_alloc_buf = dma_alloc_coherent(ddev, MLX5_ADAPTER_PAGE_SIZE, &cmd->alloc_dma, GFP_KERNEL); @@ -1883,7 +1882,7 @@ static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) { - struct device *ddev = &dev->pdev->dev; + struct device *ddev = dev->device; dma_free_coherent(ddev, cmd->alloc_size, cmd->cmd_alloc_buf, cmd->alloc_dma); @@ -1902,14 +1901,13 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) memset(cmd, 0, sizeof(*cmd)); cmd_if_rev = cmdif_rev(dev); if (cmd_if_rev != CMD_IF_REV) { - dev_err(&dev->pdev->dev, - "Driver cmdif rev(%d) differs from firmware's(%d)\n", - CMD_IF_REV, cmd_if_rev); + mlx5_core_err(dev, + "Driver cmdif rev(%d) differs from firmware's(%d)\n", + CMD_IF_REV, cmd_if_rev); return -EINVAL; } - cmd->pool = dma_pool_create("mlx5_cmd", &dev->pdev->dev, size, align, - 0); + cmd->pool = dma_pool_create("mlx5_cmd", dev->device, size, align, 0); if (!cmd->pool) return -ENOMEM; @@ -1921,14 +1919,14 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) cmd->log_sz = cmd_l >> 4 & 0xf; cmd->log_stride = cmd_l & 0xf; if (1 << cmd->log_sz > MLX5_MAX_COMMANDS) { - dev_err(&dev->pdev->dev, "firmware reports too many outstanding commands %d\n", - 1 << cmd->log_sz); + mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n", + 1 << cmd->log_sz); err = -EINVAL; goto err_free_page; } if (cmd->log_sz + cmd->log_stride > MLX5_ADAPTER_PAGE_SHIFT) { - dev_err(&dev->pdev->dev, "command queue size overflow\n"); + mlx5_core_err(dev, "command queue size overflow\n"); err = -EINVAL; goto err_free_page; } @@ -1939,8 +1937,8 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) cmd->cmdif_rev = ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16; if (cmd->cmdif_rev > CMD_IF_REV) { - dev_err(&dev->pdev->dev, "driver does not support command interface version. driver %d, firmware %d\n", - CMD_IF_REV, cmd->cmdif_rev); + mlx5_core_err(dev, "driver does not support command interface version. driver %d, firmware %d\n", + CMD_IF_REV, cmd->cmdif_rev); err = -EOPNOTSUPP; goto err_free_page; } @@ -1956,7 +1954,7 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) cmd_h = (u32)((u64)(cmd->dma) >> 32); cmd_l = (u32)(cmd->dma); if (cmd_l & 0xfff) { - dev_err(&dev->pdev->dev, "invalid command queue address\n"); + mlx5_core_err(dev, "invalid command queue address\n"); err = -ENOMEM; goto err_free_page; } @@ -1976,7 +1974,7 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) set_wqname(dev); cmd->wq = create_singlethread_workqueue(cmd->wq_name); if (!cmd->wq) { - dev_err(&dev->pdev->dev, "failed to create command workqueue\n"); + mlx5_core_err(dev, "failed to create command workqueue\n"); err = -ENOMEM; goto err_cache; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer_tracepoint.h b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer_tracepoint.h index 83f90e9aff45..3038be575923 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer_tracepoint.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer_tracepoint.h @@ -47,7 +47,7 @@ TRACE_EVENT(mlx5_fw, TP_ARGS(tracer, trace_timestamp, lost, event_id, msg), TP_STRUCT__entry( - __string(dev_name, dev_name(&tracer->dev->pdev->dev)) + __string(dev_name, dev_name(tracer->dev->device)) __field(u64, trace_timestamp) __field(bool, lost) __field(u8, event_id) @@ -55,7 +55,8 @@ TRACE_EVENT(mlx5_fw, ), TP_fast_assign( - __assign_str(dev_name, dev_name(&tracer->dev->pdev->dev)); + __assign_str(dev_name, + dev_name(tracer->dev->device)); __entry->trace_timestamp = trace_timestamp; __entry->lost = lost; __entry->event_id = event_id; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 71c65cc17904..3a183d690e23 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -240,8 +240,8 @@ struct mlx5e_params { bool rx_cqe_compress_def; struct net_dim_cq_moder rx_cq_moderation; struct net_dim_cq_moder tx_cq_moderation; + bool tunneled_offload_en; bool lro_en; - u32 lro_wqe_sz; u8 tx_min_inline_mode; bool vlan_strip_disable; bool scatter_fcs_en; @@ -410,14 +410,17 @@ struct mlx5e_xdp_info_fifo { struct mlx5e_xdp_wqe_info { u8 num_wqebbs; - u8 num_ds; + u8 num_pkts; }; struct mlx5e_xdp_mpwqe { /* Current MPWQE session */ struct mlx5e_tx_wqe *wqe; u8 ds_count; + u8 pkt_count; u8 max_ds_count; + u8 complete; + u8 inline_on; }; struct mlx5e_xdpsq; @@ -429,7 +432,6 @@ struct mlx5e_xdpsq { /* dirtied @completion */ u32 xdpi_fifo_cc; u16 cc; - bool redirect_flush; /* dirtied @xmit */ u32 xdpi_fifo_pc ____cacheline_aligned_in_smp; @@ -462,10 +464,10 @@ struct mlx5e_xdpsq { struct mlx5e_icosq { /* data path */ + u16 cc; + u16 pc; - /* dirtied @xmit */ - u16 pc ____cacheline_aligned_in_smp; - + struct mlx5_wqe_ctrl_seg *doorbell_cseg; struct mlx5e_cq cq; /* write@xmit, read@completion */ @@ -532,7 +534,8 @@ typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq); typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16); enum mlx5e_rq_flag { - MLX5E_RQ_FLAG_XDP_XMIT = BIT(0), + MLX5E_RQ_FLAG_XDP_XMIT, + MLX5E_RQ_FLAG_XDP_REDIRECT, }; struct mlx5e_rq_frag_info { @@ -563,8 +566,10 @@ struct mlx5e_rq { struct mlx5e_mpw_info *info; mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq; u16 num_strides; + u16 actual_wq_head; u8 log_stride_sz; - bool umr_in_progress; + u8 umr_in_progress; + u8 umr_last_bulk; } mpwqe; }; struct { @@ -769,12 +774,12 @@ struct mlx5e_profile { void mlx5e_build_ptys2ethtool_map(void); u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev, - select_queue_fallback_t fallback); + struct net_device *sb_dev); netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev); netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, - struct mlx5e_tx_wqe *wqe, u16 pi); + struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more); +void mlx5e_trigger_irq(struct mlx5e_icosq *sq); void mlx5e_completion_event(struct mlx5_core_cq *mcq); void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event); int mlx5e_napi_poll(struct napi_struct *napi, int budget); @@ -858,6 +863,7 @@ void mlx5e_close_channels(struct mlx5e_channels *chs); * switching channels */ typedef int (*mlx5e_fp_hw_modify)(struct mlx5e_priv *priv); +int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv); int mlx5e_safe_switch_channels(struct mlx5e_priv *priv, struct mlx5e_channels *new_chs, mlx5e_fp_hw_modify hw_modify); @@ -885,6 +891,53 @@ static inline bool mlx5e_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev) MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.inner_ip_version)); } +static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_ETH(mdev, swp) && + MLX5_CAP_ETH(mdev, swp_csum) && MLX5_CAP_ETH(mdev, swp_lso); +} + +struct mlx5e_swp_spec { + __be16 l3_proto; + u8 l4_proto; + u8 is_tun; + __be16 tun_l3_proto; + u8 tun_l4_proto; +}; + +static inline void +mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg, + struct mlx5e_swp_spec *swp_spec) +{ + /* SWP offsets are in 2-bytes words */ + eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2; + if (swp_spec->l3_proto == htons(ETH_P_IPV6)) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6; + if (swp_spec->l4_proto) { + eseg->swp_outer_l4_offset = skb_transport_offset(skb) / 2; + if (swp_spec->l4_proto == IPPROTO_UDP) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP; + } + + if (swp_spec->is_tun) { + eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2; + if (swp_spec->tun_l3_proto == htons(ETH_P_IPV6)) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; + } else { /* typically for ipsec when xfrm mode != XFRM_MODE_TUNNEL */ + eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2; + if (swp_spec->l3_proto == htons(ETH_P_IPV6)) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; + } + switch (swp_spec->tun_l4_proto) { + case IPPROTO_UDP: + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; + /* fall through */ + case IPPROTO_TCP: + eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2; + break; + } +} + static inline void mlx5e_sq_fetch_wqe(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe **wqe, u16 *pi) @@ -929,7 +982,7 @@ void mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, */ wmb(); - mlx5_write64((__be32 *)ctrl, uar_map, NULL); + mlx5_write64((__be32 *)ctrl, uar_map); } static inline void mlx5e_cq_arm(struct mlx5e_cq *cq) @@ -1041,6 +1094,7 @@ mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *prof int mlx5e_attach_netdev(struct mlx5e_priv *priv); void mlx5e_detach_netdev(struct mlx5e_priv *priv); void mlx5e_destroy_netdev(struct mlx5e_priv *priv); +void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv); void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, struct mlx5e_rss_params *rss_params, struct mlx5e_params *params, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c new file mode 100644 index 000000000000..d3744bffbae3 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2019 Mellanox Technologies. */ + +#include "en/params.h" + +u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params) +{ + u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); + u16 linear_rq_headroom = params->xdp_prog ? + XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM; + u32 frag_sz; + + linear_rq_headroom += NET_IP_ALIGN; + + frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu); + + if (params->xdp_prog && frag_sz < PAGE_SIZE) + frag_sz = PAGE_SIZE; + + return frag_sz; +} + +u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params) +{ + u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params); + + return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz); +} + +bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params) +{ + u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params); + + return !params->lro_en && frag_sz <= PAGE_SIZE; +} + +#define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \ + MLX5_MPWQE_LOG_STRIDE_SZ_BASE) +bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, + struct mlx5e_params *params) +{ + u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params); + s8 signed_log_num_strides_param; + u8 log_num_strides; + + if (!mlx5e_rx_is_linear_skb(params)) + return false; + + if (order_base_2(frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ) + return false; + + if (MLX5_CAP_GEN(mdev, ext_stride_num_range)) + return true; + + log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz); + signed_log_num_strides_param = + (s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE; + + return signed_log_num_strides_param >= 0; +} + +u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params) +{ + u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params); + + /* Numbers are unsigned, don't subtract to avoid underflow. */ + if (params->log_rq_mtu_frames < + log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW) + return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW; + + return params->log_rq_mtu_frames - log_pkts_per_wqe; +} + +u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, + struct mlx5e_params *params) +{ + if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params)) + return order_base_2(mlx5e_rx_get_linear_frag_sz(params)); + + return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev); +} + +u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, + struct mlx5e_params *params) +{ + return MLX5_MPWRQ_LOG_WQE_SZ - + mlx5e_mpwqe_get_log_stride_size(mdev, params); +} + +u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, + struct mlx5e_params *params) +{ + u16 linear_rq_headroom = params->xdp_prog ? + XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM; + bool is_linear_skb; + + linear_rq_headroom += NET_IP_ALIGN; + + is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ? + mlx5e_rx_is_linear_skb(params) : + mlx5e_rx_mpwqe_is_linear_skb(mdev, params); + + return is_linear_skb ? linear_rq_headroom : 0; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h new file mode 100644 index 000000000000..b106a0236f36 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2019 Mellanox Technologies. */ + +#ifndef __MLX5_EN_PARAMS_H__ +#define __MLX5_EN_PARAMS_H__ + +#include "en.h" + +u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params); +u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params); +bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params); +bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, + struct mlx5e_params *params); +u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params); +u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, + struct mlx5e_params *params); +u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, + struct mlx5e_params *params); +u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, + struct mlx5e_params *params); + +#endif /* __MLX5_EN_PARAMS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c index 4ab0d030b544..633b117eb13e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c @@ -167,23 +167,23 @@ static int update_xoff_threshold(struct mlx5e_port_buffer *port_buffer, } /** - * update_buffer_lossy() - * max_mtu: netdev's max_mtu - * pfc_en: <input> current pfc configuration - * buffer: <input> current prio to buffer mapping - * xoff: <input> xoff value - * port_buffer: <output> port receive buffer configuration - * change: <output> + * update_buffer_lossy - Update buffer configuration based on pfc + * @max_mtu: netdev's max_mtu + * @pfc_en: <input> current pfc configuration + * @buffer: <input> current prio to buffer mapping + * @xoff: <input> xoff value + * @port_buffer: <output> port receive buffer configuration + * @change: <output> * - * Update buffer configuration based on pfc configuraiton and priority - * to buffer mapping. - * Buffer's lossy bit is changed to: - * lossless if there is at least one PFC enabled priority mapped to this buffer - * lossy if all priorities mapped to this buffer are PFC disabled + * Update buffer configuration based on pfc configuraiton and + * priority to buffer mapping. + * Buffer's lossy bit is changed to: + * lossless if there is at least one PFC enabled priority + * mapped to this buffer lossy if all priorities mapped to + * this buffer are PFC disabled * - * Return: - * Return 0 if no error. - * Set change to true if buffer configuration is modified. + * @return: 0 if no error, + * sets change to true if buffer configuration was modified. */ static int update_buffer_lossy(unsigned int max_mtu, u8 pfc_en, u8 *buffer, u32 xoff, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 9d38e62cdf24..476dd97f7f2f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -186,12 +186,17 @@ static int mlx5e_tx_reporter_recover_from_ctx(struct mlx5e_tx_err_ctx *err_ctx) static int mlx5e_tx_reporter_recover_all(struct mlx5e_priv *priv) { - int err; + int err = 0; rtnl_lock(); mutex_lock(&priv->state_lock); - mlx5e_close_locked(priv->netdev); - err = mlx5e_open_locked(priv->netdev); + + if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) + goto out; + + err = mlx5e_safe_reopen_channels(priv); + +out: mutex_unlock(&priv->state_lock); rtnl_unlock(); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c index fa2a3c444cdc..fe5d4d7f15ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c @@ -39,6 +39,10 @@ static int get_route_and_out_devs(struct mlx5e_priv *priv, return -EOPNOTSUPP; } + if (!(mlx5e_eswitch_rep(*out_dev) && + mlx5e_is_uplink_rep(netdev_priv(*out_dev)))) + return -EOPNOTSUPP; + return 0; } @@ -70,7 +74,7 @@ static int mlx5e_route_lookup_ipv4(struct mlx5e_priv *priv, if (ret) return ret; - if (mlx5_lag_is_multipath(mdev) && !rt->rt_gateway) + if (mlx5_lag_is_multipath(mdev) && rt->rt_gw_family != AF_INET) return -ENETUNREACH; #else return -EOPNOTSUPP; @@ -96,7 +100,7 @@ static const char *mlx5e_netdev_kind(struct net_device *dev) if (dev->rtnl_link_ops) return dev->rtnl_link_ops->kind; else - return ""; + return "unknown"; } static int mlx5e_route_lookup_ipv6(struct mlx5e_priv *priv, @@ -636,8 +640,10 @@ int mlx5e_tc_tun_parse(struct net_device *filter_dev, headers_c, headers_v); } else { netdev_warn(priv->netdev, - "decapsulation offload is not supported for %s net device (%d)\n", - mlx5e_netdev_kind(filter_dev), tunnel_type); + "decapsulation offload is not supported for %s (kind: \"%s\")\n", + netdev_name(filter_dev), + mlx5e_netdev_kind(filter_dev)); + return -EOPNOTSUPP; } return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index 03b2a9f9c589..eb8ef78e5626 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -33,6 +33,26 @@ #include <linux/bpf_trace.h> #include "en/xdp.h" +int mlx5e_xdp_max_mtu(struct mlx5e_params *params) +{ + int hr = NET_IP_ALIGN + XDP_PACKET_HEADROOM; + + /* Let S := SKB_DATA_ALIGN(sizeof(struct skb_shared_info)). + * The condition checked in mlx5e_rx_is_linear_skb is: + * SKB_DATA_ALIGN(sw_mtu + hard_mtu + hr) + S <= PAGE_SIZE (1) + * (Note that hw_mtu == sw_mtu + hard_mtu.) + * What is returned from this function is: + * max_mtu = PAGE_SIZE - S - hr - hard_mtu (2) + * After assigning sw_mtu := max_mtu, the left side of (1) turns to + * SKB_DATA_ALIGN(PAGE_SIZE - S) + S, which is equal to PAGE_SIZE, + * because both PAGE_SIZE and S are already aligned. Any number greater + * than max_mtu would make the left side of (1) greater than PAGE_SIZE, + * so max_mtu is the maximum MTU allowed. + */ + + return MLX5E_HW2SW_MTU(params, SKB_MAX_HEAD(hr)); +} + static inline bool mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_dma_info *di, struct xdp_buff *xdp) @@ -85,7 +105,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, if (unlikely(err)) goto xdp_abort; __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); - rq->xdpsq.redirect_flush = true; + __set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags); mlx5e_page_dma_unmap(rq, di); rq->stats->xdp_redirect++; return true; @@ -105,6 +125,7 @@ xdp_abort: static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) { struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; + struct mlx5e_xdpsq_stats *stats = sq->stats; struct mlx5_wq_cyc *wq = &sq->wq; u8 wqebbs; u16 pi; @@ -112,7 +133,9 @@ static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) mlx5e_xdpsq_fetch_wqe(sq, &session->wqe); prefetchw(session->wqe->data); - session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT; + session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT; + session->pkt_count = 0; + session->complete = 0; pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); @@ -131,6 +154,10 @@ static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) MLX5E_XDP_MPW_MAX_WQEBBS); session->max_ds_count = MLX5_SEND_WQEBB_NUM_DS * wqebbs; + + mlx5e_xdp_update_inline_state(sq); + + stats->mpwqe++; } static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq) @@ -147,7 +174,7 @@ static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq) cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count); wi->num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS); - wi->num_ds = ds_count - MLX5E_XDP_TX_EMPTY_DS_COUNT; + wi->num_pkts = session->pkt_count; sq->pc += wi->num_wqebbs; @@ -162,11 +189,9 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; struct mlx5e_xdpsq_stats *stats = sq->stats; - dma_addr_t dma_addr = xdpi->dma_addr; struct xdp_frame *xdpf = xdpi->xdpf; - unsigned int dma_len = xdpf->len; - if (unlikely(sq->hw_mtu < dma_len)) { + if (unlikely(sq->hw_mtu < xdpf->len)) { stats->err++; return false; } @@ -183,9 +208,10 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, mlx5e_xdp_mpwqe_session_start(sq); } - mlx5e_xdp_mpwqe_add_dseg(sq, dma_addr, dma_len); + mlx5e_xdp_mpwqe_add_dseg(sq, xdpi, stats); - if (unlikely(session->ds_count == session->max_ds_count)) + if (unlikely(session->complete || + session->ds_count == session->max_ds_count)) mlx5e_xdp_mpwqe_complete(sq); mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi); @@ -249,12 +275,33 @@ static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info * return true; } +static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq, + struct mlx5e_xdp_wqe_info *wi, + struct mlx5e_rq *rq, + bool recycle) +{ + struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; + u16 i; + + for (i = 0; i < wi->num_pkts; i++) { + struct mlx5e_xdp_info xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo); + + if (rq) { + /* XDP_TX */ + mlx5e_page_release(rq, &xdpi.di, recycle); + } else { + /* XDP_REDIRECT */ + dma_unmap_single(sq->pdev, xdpi.dma_addr, + xdpi.xdpf->len, DMA_TO_DEVICE); + xdp_return_frame(xdpi.xdpf); + } + } +} + bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) { - struct mlx5e_xdp_info_fifo *xdpi_fifo; struct mlx5e_xdpsq *sq; struct mlx5_cqe64 *cqe; - bool is_redirect; u16 sqcc; int i; @@ -267,9 +314,6 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) if (!cqe) return false; - is_redirect = !rq; - xdpi_fifo = &sq->db.xdpi_fifo; - /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), * otherwise a cq overrun may occur */ @@ -291,7 +335,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) do { struct mlx5e_xdp_wqe_info *wi; - u16 ci, j; + u16 ci; last_wqe = (sqcc == wqe_counter); ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); @@ -299,19 +343,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) sqcc += wi->num_wqebbs; - for (j = 0; j < wi->num_ds; j++) { - struct mlx5e_xdp_info xdpi = - mlx5e_xdpi_fifo_pop(xdpi_fifo); - - if (is_redirect) { - xdp_return_frame(xdpi.xdpf); - dma_unmap_single(sq->pdev, xdpi.dma_addr, - xdpi.xdpf->len, DMA_TO_DEVICE); - } else { - /* Recycle RX page */ - mlx5e_page_release(rq, &xdpi.di, true); - } - } + mlx5e_free_xdpsq_desc(sq, wi, rq, true); } while (!last_wqe); } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); @@ -328,31 +360,16 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq) { - struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; - bool is_redirect = !rq; - while (sq->cc != sq->pc) { struct mlx5e_xdp_wqe_info *wi; - u16 ci, i; + u16 ci; ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc); wi = &sq->db.wqe_info[ci]; sq->cc += wi->num_wqebbs; - for (i = 0; i < wi->num_ds; i++) { - struct mlx5e_xdp_info xdpi = - mlx5e_xdpi_fifo_pop(xdpi_fifo); - - if (is_redirect) { - xdp_return_frame(xdpi.xdpf); - dma_unmap_single(sq->pdev, xdpi.dma_addr, - xdpi.xdpf->len, DMA_TO_DEVICE); - } else { - /* Recycle RX page */ - mlx5e_page_release(rq, &xdpi.di, false); - } - } + mlx5e_free_xdpsq_desc(sq, wi, rq, false); } } @@ -419,9 +436,9 @@ void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq) mlx5e_xmit_xdp_doorbell(xdpsq); - if (xdpsq->redirect_flush) { + if (test_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags)) { xdp_do_flush_map(); - xdpsq->redirect_flush = false; + __clear_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h index ee27a7c8cd87..8b537a4b0840 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h @@ -34,13 +34,12 @@ #include "en.h" -#define MLX5E_XDP_MAX_MTU ((int)(PAGE_SIZE - \ - MLX5_SKB_FRAG_SZ(XDP_PACKET_HEADROOM))) #define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN) #define MLX5E_XDP_TX_EMPTY_DS_COUNT \ (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) #define MLX5E_XDP_TX_DS_COUNT (MLX5E_XDP_TX_EMPTY_DS_COUNT + 1 /* SG DS */) +int mlx5e_xdp_max_mtu(struct mlx5e_params *params); bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, void *va, u16 *rx_headroom, u32 *len); bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq); @@ -75,16 +74,68 @@ static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq) } } +/* Enable inline WQEs to shift some load from a congested HCA (HW) to + * a less congested cpu (SW). + */ +static inline void mlx5e_xdp_update_inline_state(struct mlx5e_xdpsq *sq) +{ + u16 outstanding = sq->xdpi_fifo_pc - sq->xdpi_fifo_cc; + struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; + +#define MLX5E_XDP_INLINE_WATERMARK_LOW 10 +#define MLX5E_XDP_INLINE_WATERMARK_HIGH 128 + + if (session->inline_on) { + if (outstanding <= MLX5E_XDP_INLINE_WATERMARK_LOW) + session->inline_on = 0; + return; + } + + /* inline is false */ + if (outstanding >= MLX5E_XDP_INLINE_WATERMARK_HIGH) + session->inline_on = 1; +} + static inline void -mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, dma_addr_t dma_addr, u16 dma_len) +mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi, + struct mlx5e_xdpsq_stats *stats) { struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; + dma_addr_t dma_addr = xdpi->dma_addr; + struct xdp_frame *xdpf = xdpi->xdpf; struct mlx5_wqe_data_seg *dseg = - (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count++; + (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count; + u16 dma_len = xdpf->len; + session->pkt_count++; + +#define MLX5E_XDP_INLINE_WQE_SZ_THRSD (256 - sizeof(struct mlx5_wqe_inline_seg)) + + if (session->inline_on && dma_len <= MLX5E_XDP_INLINE_WQE_SZ_THRSD) { + struct mlx5_wqe_inline_seg *inline_dseg = + (struct mlx5_wqe_inline_seg *)dseg; + u16 ds_len = sizeof(*inline_dseg) + dma_len; + u16 ds_cnt = DIV_ROUND_UP(ds_len, MLX5_SEND_WQE_DS); + + if (unlikely(session->ds_count + ds_cnt > session->max_ds_count)) { + /* Not enough space for inline wqe, send with memory pointer */ + session->complete = true; + goto no_inline; + } + + inline_dseg->byte_count = cpu_to_be32(dma_len | MLX5_INLINE_SEG); + memcpy(inline_dseg->data, xdpf->data, dma_len); + + session->ds_count += ds_cnt; + stats->inlnw++; + return; + } + +no_inline: dseg->addr = cpu_to_be64(dma_addr); dseg->byte_count = cpu_to_be32(dma_len); dseg->lkey = sq->mkey_be; + session->ds_count++; } static inline void mlx5e_xdpsq_fetch_wqe(struct mlx5e_xdpsq *sq, @@ -111,5 +162,4 @@ mlx5e_xdpi_fifo_pop(struct mlx5e_xdp_info_fifo *fifo) { return fifo->xi[(*fifo->cc)++ & fifo->mask]; } - #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index 1dd225380a66..6da7c88742dc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -40,6 +40,57 @@ #include "en_accel/tls_rxtx.h" #include "en.h" +#if IS_ENABLED(CONFIG_GENEVE) +static inline bool mlx5_geneve_tx_allowed(struct mlx5_core_dev *mdev) +{ + return mlx5_tx_swp_supported(mdev); +} + +static inline void +mlx5e_tx_tunnel_accel(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg) +{ + struct mlx5e_swp_spec swp_spec = {}; + unsigned int offset = 0; + __be16 l3_proto; + u8 l4_proto; + + l3_proto = vlan_get_protocol(skb); + switch (l3_proto) { + case htons(ETH_P_IP): + l4_proto = ip_hdr(skb)->protocol; + break; + case htons(ETH_P_IPV6): + l4_proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL); + break; + default: + return; + } + + if (l4_proto != IPPROTO_UDP || + udp_hdr(skb)->dest != cpu_to_be16(GENEVE_UDP_PORT)) + return; + swp_spec.l3_proto = l3_proto; + swp_spec.l4_proto = l4_proto; + swp_spec.is_tun = true; + if (inner_ip_hdr(skb)->version == 6) { + swp_spec.tun_l3_proto = htons(ETH_P_IPV6); + swp_spec.tun_l4_proto = inner_ipv6_hdr(skb)->nexthdr; + } else { + swp_spec.tun_l3_proto = htons(ETH_P_IP); + swp_spec.tun_l4_proto = inner_ip_hdr(skb)->protocol; + } + + mlx5e_set_eseg_swp(skb, eseg, &swp_spec); +} + +#else +static inline bool mlx5_geneve_tx_allowed(struct mlx5_core_dev *mdev) +{ + return false; +} + +#endif /* CONFIG_GENEVE */ + static inline void mlx5e_udp_gso_handle_tx_skb(struct sk_buff *skb) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 53608afd39b6..0dd17514caae 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -136,7 +136,7 @@ static void mlx5e_ipsec_set_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg, u8 mode, struct xfrm_offload *xo) { - u8 proto; + struct mlx5e_swp_spec swp_spec = {}; /* Tunnel Mode: * SWP: OutL3 InL3 InL4 @@ -146,35 +146,23 @@ static void mlx5e_ipsec_set_swp(struct sk_buff *skb, * SWP: OutL3 InL4 * InL3 * Pkt: MAC IP ESP L4 - * - * Offsets are in 2-byte words, counting from start of frame */ - eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2; - if (skb->protocol == htons(ETH_P_IPV6)) - eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6; - - if (mode == XFRM_MODE_TUNNEL) { - eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2; + swp_spec.l3_proto = skb->protocol; + swp_spec.is_tun = mode == XFRM_MODE_TUNNEL; + if (swp_spec.is_tun) { if (xo->proto == IPPROTO_IPV6) { - eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; - proto = inner_ipv6_hdr(skb)->nexthdr; + swp_spec.tun_l3_proto = htons(ETH_P_IPV6); + swp_spec.tun_l4_proto = inner_ipv6_hdr(skb)->nexthdr; } else { - proto = inner_ip_hdr(skb)->protocol; + swp_spec.tun_l3_proto = htons(ETH_P_IP); + swp_spec.tun_l4_proto = inner_ip_hdr(skb)->protocol; } } else { - eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2; - if (skb->protocol == htons(ETH_P_IPV6)) - eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; - proto = xo->proto; - } - switch (proto) { - case IPPROTO_UDP: - eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; - /* Fall through */ - case IPPROTO_TCP: - eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2; - break; + swp_spec.tun_l3_proto = skb->protocol; + swp_spec.tun_l4_proto = xo->proto; } + + mlx5e_set_eseg_swp(skb, eseg, &swp_spec); } void mlx5e_ipsec_set_iv_esn(struct sk_buff *skb, struct xfrm_state *x, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c index be137d4a9169..439bf5953885 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c @@ -181,7 +181,6 @@ static void mlx5e_tls_complete_sync_skb(struct sk_buff *skb, */ nskb->ip_summed = CHECKSUM_PARTIAL; - nskb->xmit_more = 1; nskb->queue_mapping = skb->queue_mapping; } @@ -248,7 +247,7 @@ mlx5e_tls_handle_ooo(struct mlx5e_tls_offload_context_tx *context, sq->stats->tls_resync_bytes += nskb->len; mlx5e_tls_complete_sync_skb(skb, nskb, tcp_seq, headln, cpu_to_be64(info.rcd_sn)); - mlx5e_sq_xmit(sq, nskb, *wqe, *pi); + mlx5e_sq_xmit(sq, nskb, *wqe, *pi, true); mlx5e_sq_fetch_wqe(sq, wqe, pi); return skb; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 5efce4a3ff79..7efaa58ae034 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -1561,7 +1561,7 @@ static int mlx5e_get_module_info(struct net_device *netdev, struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5_core_dev *dev = priv->mdev; int size_read = 0; - u8 data[4]; + u8 data[4] = {0}; size_read = mlx5_query_module_eeprom(dev, 0, 2, data); if (size_read < 2) @@ -1571,22 +1571,22 @@ static int mlx5e_get_module_info(struct net_device *netdev, switch (data[0]) { case MLX5_MODULE_ID_QSFP: modinfo->type = ETH_MODULE_SFF_8436; - modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_MAX_LEN; break; case MLX5_MODULE_ID_QSFP_PLUS: case MLX5_MODULE_ID_QSFP28: /* data[1] = revision id */ if (data[0] == MLX5_MODULE_ID_QSFP28 || data[1] >= 0x3) { modinfo->type = ETH_MODULE_SFF_8636; - modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_MAX_LEN; } else { modinfo->type = ETH_MODULE_SFF_8436; - modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_MAX_LEN; } break; case MLX5_MODULE_ID_SFP: modinfo->type = ETH_MODULE_SFF_8472; - modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN; + modinfo->eeprom_len = MLX5_EEPROM_PAGE_LENGTH; break; default: netdev_err(priv->netdev, "%s: cable type not recognized:0x%x\n", @@ -1768,7 +1768,8 @@ static int set_pflag_rx_no_csum_complete(struct net_device *netdev, bool enable) struct mlx5e_channel *c; int i; - if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) + if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || + priv->channels.params.xdp_prog) return 0; for (i = 0; i < channels->num; i++) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index b5fdbd3190d9..457cc39423f2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -34,6 +34,7 @@ #include <net/pkt_cls.h> #include <linux/mlx5/fs.h> #include <net/vxlan.h> +#include <net/geneve.h> #include <linux/bpf.h> #include <linux/if_bridge.h> #include <net/page_pool.h> @@ -43,6 +44,7 @@ #include "en_rep.h" #include "en_accel/ipsec.h" #include "en_accel/ipsec_rxtx.h" +#include "en_accel/en_accel.h" #include "en_accel/tls.h" #include "accel/ipsec.h" #include "accel/tls.h" @@ -53,6 +55,7 @@ #include "lib/eq.h" #include "en/monitor_stats.h" #include "en/reporter.h" +#include "en/params.h" struct mlx5e_rq_param { u32 rqc[MLX5_ST_SZ_DW(rqc)]; @@ -101,108 +104,9 @@ bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) return true; } -static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params) -{ - u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); - u16 linear_rq_headroom = params->xdp_prog ? - XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM; - u32 frag_sz; - - linear_rq_headroom += NET_IP_ALIGN; - - frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu); - - if (params->xdp_prog && frag_sz < PAGE_SIZE) - frag_sz = PAGE_SIZE; - - return frag_sz; -} - -static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params) -{ - u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params); - - return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz); -} - -static bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, - struct mlx5e_params *params) -{ - u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params); - - return !params->lro_en && frag_sz <= PAGE_SIZE; -} - -#define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \ - MLX5_MPWQE_LOG_STRIDE_SZ_BASE) -static bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, - struct mlx5e_params *params) -{ - u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params); - s8 signed_log_num_strides_param; - u8 log_num_strides; - - if (!mlx5e_rx_is_linear_skb(mdev, params)) - return false; - - if (order_base_2(frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ) - return false; - - if (MLX5_CAP_GEN(mdev, ext_stride_num_range)) - return true; - - log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz); - signed_log_num_strides_param = - (s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE; - - return signed_log_num_strides_param >= 0; -} - -static u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params) -{ - if (params->log_rq_mtu_frames < - mlx5e_mpwqe_log_pkts_per_wqe(params) + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW) - return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW; - - return params->log_rq_mtu_frames - mlx5e_mpwqe_log_pkts_per_wqe(params); -} - -static u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, - struct mlx5e_params *params) -{ - if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params)) - return order_base_2(mlx5e_rx_get_linear_frag_sz(params)); - - return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev); -} - -static u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, - struct mlx5e_params *params) -{ - return MLX5_MPWRQ_LOG_WQE_SZ - - mlx5e_mpwqe_get_log_stride_size(mdev, params); -} - -static u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, - struct mlx5e_params *params) -{ - u16 linear_rq_headroom = params->xdp_prog ? - XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM; - bool is_linear_skb; - - linear_rq_headroom += NET_IP_ALIGN; - - is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ? - mlx5e_rx_is_linear_skb(mdev, params) : - mlx5e_rx_mpwqe_is_linear_skb(mdev, params); - - return is_linear_skb ? linear_rq_headroom : 0; -} - void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params) { - params->lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ; params->log_rq_mtu_frames = is_kdump_kernel() ? MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE : MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE; @@ -469,7 +373,6 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) } static int mlx5e_init_di_list(struct mlx5e_rq *rq, - struct mlx5e_params *params, int wq_sz, int cpu) { int len = wq_sz << rq->wqe.info.log_num_frags; @@ -597,7 +500,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, goto err_free; } - err = mlx5e_init_di_list(rq, params, wq_sz, c->cpu); + err = mlx5e_init_di_list(rq, wq_sz, c->cpu); if (err) goto err_free; rq->post_wqes = mlx5e_post_rx_wqes; @@ -615,7 +518,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, goto err_free; } - rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(mdev, params) ? + rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params) ? mlx5e_skb_from_cqe_linear : mlx5e_skb_from_cqe_nonlinear; rq->mkey_be = c->mkey_be; @@ -902,10 +805,14 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq) if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { struct mlx5_wq_ll *wq = &rq->mpwqe.wq; + u16 head = wq->head; + int i; - /* UMR WQE (if in progress) is always at wq->head */ - if (rq->mpwqe.umr_in_progress) - rq->dealloc_wqe(rq, wq->head); + /* Outstanding UMR WQEs (in progress) start at wq->head */ + for (i = 0; i < rq->mpwqe.umr_in_progress; i++) { + rq->dealloc_wqe(rq, head); + head = mlx5_wq_ll_get_wqe_next_ix(wq, head); + } while (!mlx5_wq_ll_is_empty(wq)) { struct mlx5e_rx_wqe_ll *wqe; @@ -951,7 +858,11 @@ static int mlx5e_open_rq(struct mlx5e_channel *c, if (params->rx_dim_enabled) __set_bit(MLX5E_RQ_STATE_AM, &c->rq.state); - if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE)) + /* We disable csum_complete when XDP is enabled since + * XDP programs might manipulate packets which will render + * skb->checksum incorrect. + */ + if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || c->xdp) __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &c->rq.state); return 0; @@ -966,16 +877,8 @@ err_free_rq: static void mlx5e_activate_rq(struct mlx5e_rq *rq) { - struct mlx5e_icosq *sq = &rq->channel->icosq; - struct mlx5_wq_cyc *wq = &sq->wq; - struct mlx5e_tx_wqe *nopwqe; - - u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); - set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); - sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP; - nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc); - mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl); + mlx5e_trigger_irq(&rq->channel->icosq); } static void mlx5e_deactivate_rq(struct mlx5e_rq *rq) @@ -1087,7 +990,7 @@ static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa) { - u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq); + int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); sq->db.ico_wqe = kvzalloc_node(array_size(wq_sz, sizeof(*sq->db.ico_wqe)), @@ -1523,7 +1426,7 @@ static int mlx5e_open_xdpsq(struct mlx5e_channel *c, dseg->lkey = sq->mkey_be; wi->num_wqebbs = 1; - wi->num_ds = 1; + wi->num_pkts = 1; } } @@ -1891,7 +1794,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, c->tstamp = &priv->tstamp; c->ix = ix; c->cpu = cpu; - c->pdev = &priv->mdev->pdev->dev; + c->pdev = priv->mdev->device; c->netdev = priv->netdev; c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key); c->num_tc = params->num_tc; @@ -2049,7 +1952,7 @@ static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, byte_count += MLX5E_METADATA_ETHER_LEN; #endif - if (mlx5e_rx_is_linear_skb(mdev, params)) { + if (mlx5e_rx_is_linear_skb(params)) { int frag_stride; frag_stride = mlx5e_rx_get_linear_frag_sz(params); @@ -2103,6 +2006,13 @@ static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs) return order_base_2(sz); } +static u8 mlx5e_get_rq_log_wq_sz(void *rqc) +{ + void *wq = MLX5_ADDR_OF(rqc, rqc, wq); + + return MLX5_GET(wq, wq, log_wq_sz); +} + static void mlx5e_build_rq_param(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_rq_param *param) @@ -2137,7 +2047,7 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv, MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); - param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev); + param->wq.buf_numa_node = dev_to_node(mdev->device); } static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv, @@ -2152,7 +2062,7 @@ static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv, mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); MLX5_SET(rqc, rqc, counter_set_id, priv->drop_rq_q_counter); - param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev); + param->wq.buf_numa_node = dev_to_node(mdev->device); } static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv, @@ -2164,7 +2074,7 @@ static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv, MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn); - param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev); + param->wq.buf_numa_node = dev_to_node(priv->mdev->device); } static void mlx5e_build_sq_param(struct mlx5e_priv *priv, @@ -2173,10 +2083,13 @@ static void mlx5e_build_sq_param(struct mlx5e_priv *priv, { void *sqc = param->sqc; void *wq = MLX5_ADDR_OF(sqc, sqc, wq); + bool allow_swp; + allow_swp = mlx5_geneve_tx_allowed(priv->mdev) || + !!MLX5_IPSEC_DEV(priv->mdev); mlx5e_build_sq_param_common(priv, param); MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); - MLX5_SET(sqc, sqc, allow_swp, !!MLX5_IPSEC_DEV(priv->mdev)); + MLX5_SET(sqc, sqc, allow_swp, allow_swp); } static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv, @@ -2266,13 +2179,28 @@ static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE); } +static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params, + struct mlx5e_rq_param *rqp) +{ + switch (params->rq_wq_type) { + case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + return order_base_2(MLX5E_UMR_WQEBBS) + + mlx5e_get_rq_log_wq_sz(rqp->rqc); + default: /* MLX5_WQ_TYPE_CYCLIC */ + return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; + } +} + static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_channel_param *cparam) { - u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; + u8 icosq_log_wq_sz; mlx5e_build_rq_param(priv, params, &cparam->rq); + + icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq); + mlx5e_build_sq_param(priv, params, &cparam->sq); mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq); @@ -2328,14 +2256,18 @@ static void mlx5e_activate_channels(struct mlx5e_channels *chs) mlx5e_activate_channel(chs->c[i]); } +#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ + static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) { int err = 0; int i; - for (i = 0; i < chs->num; i++) - err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, - err ? 0 : 20000); + for (i = 0; i < chs->num; i++) { + int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; + + err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout); + } return err ? -ETIMEDOUT : 0; } @@ -2632,7 +2564,7 @@ static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc) MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO); MLX5_SET(tirc, tirc, lro_max_ip_payload_size, - (params->lro_wqe_sz - ROUGH_MAX_L2_L3_HDR_SZ) >> 8); + (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - ROUGH_MAX_L2_L3_HDR_SZ) >> 8); MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout); } @@ -2742,22 +2674,6 @@ free_in: return err; } -static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv, - enum mlx5e_traffic_types tt, - u32 *tirc) -{ - MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn); - - mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc); - - MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); - MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn); - MLX5_SET(tirc, tirc, tunneled_offload_en, 0x1); - - mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, - &tirc_default_config[tt], tirc, true); -} - static int mlx5e_set_mtu(struct mlx5_core_dev *mdev, struct mlx5e_params *params, u16 mtu) { @@ -2807,6 +2723,21 @@ int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) return 0; } +void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv) +{ + struct mlx5e_params *params = &priv->channels.params; + struct net_device *netdev = priv->netdev; + struct mlx5_core_dev *mdev = priv->mdev; + u16 max_mtu; + + /* MTU range: 68 - hw-specific max */ + netdev->min_mtu = ETH_MIN_MTU; + + mlx5_query_port_max_mtu(mdev, &max_mtu, 1); + netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu), + ETH_MAX_MTU); +} + static void mlx5e_netdev_set_tcs(struct net_device *netdev) { struct mlx5e_priv *priv = netdev_priv(netdev); @@ -2937,6 +2868,14 @@ int mlx5e_safe_switch_channels(struct mlx5e_priv *priv, return 0; } +int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv) +{ + struct mlx5e_channels new_channels = {}; + + new_channels.params = priv->channels.params; + return mlx5e_safe_switch_channels(priv, &new_channels, NULL); +} + void mlx5e_timestamp_init(struct mlx5e_priv *priv) { priv->tstamp.tx_type = HWTSTAMP_TX_OFF; @@ -3046,8 +2985,8 @@ static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev, struct mlx5e_cq *cq, struct mlx5e_cq_param *param) { - param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev); - param->wq.db_numa_node = dev_to_node(&mdev->pdev->dev); + param->wq.buf_numa_node = dev_to_node(mdev->device); + param->wq.db_numa_node = dev_to_node(mdev->device); return mlx5e_alloc_cq_common(mdev, param, cq); } @@ -3155,32 +3094,42 @@ static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) mlx5e_destroy_tis(priv->mdev, priv->tisn[tc]); } -static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, - enum mlx5e_traffic_types tt, - u32 *tirc) +static void mlx5e_build_indir_tir_ctx_common(struct mlx5e_priv *priv, + u32 rqtn, u32 *tirc) { MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn); + MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); + MLX5_SET(tirc, tirc, indirect_table, rqtn); + MLX5_SET(tirc, tirc, tunneled_offload_en, + priv->channels.params.tunneled_offload_en); mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc); +} - MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); - MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn); - +static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, + enum mlx5e_traffic_types tt, + u32 *tirc) +{ + mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc); mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, &tirc_default_config[tt], tirc, false); } static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc) { - MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn); - - mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc); - - MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); - MLX5_SET(tirc, tirc, indirect_table, rqtn); + mlx5e_build_indir_tir_ctx_common(priv, rqtn, tirc); MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8); } +static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv, + enum mlx5e_traffic_types tt, + u32 *tirc) +{ + mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc); + mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, + &tirc_default_config[tt], tirc, true); +} + int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc) { struct mlx5e_tir *tir; @@ -3763,9 +3712,9 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, new_channels.params.sw_mtu = new_mtu; if (params->xdp_prog && - !mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) { + !mlx5e_rx_is_linear_skb(&new_channels.params)) { netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n", - new_mtu, MLX5E_XDP_MAX_MTU); + new_mtu, mlx5e_xdp_max_mtu(params)); err = -EINVAL; goto out; } @@ -4103,6 +4052,12 @@ static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, /* Verify if UDP port is being offloaded by HW */ if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) return features; + +#if IS_ENABLED(CONFIG_GENEVE) + /* Support Geneve offload for default UDP port */ + if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) + return features; +#endif } out: @@ -4161,11 +4116,10 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) if (!report_failed) goto unlock; - mlx5e_close_locked(priv->netdev); - err = mlx5e_open_locked(priv->netdev); + err = mlx5e_safe_reopen_channels(priv); if (err) netdev_err(priv->netdev, - "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n", + "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", err); unlock: @@ -4199,9 +4153,10 @@ static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog) new_channels.params = priv->channels.params; new_channels.params.xdp_prog = prog; - if (!mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) { + if (!mlx5e_rx_is_linear_skb(&new_channels.params)) { netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n", - new_channels.params.sw_mtu, MLX5E_XDP_MAX_MTU); + new_channels.params.sw_mtu, + mlx5e_xdp_max_mtu(&new_channels.params)); return -EINVAL; } @@ -4252,7 +4207,7 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) mlx5e_set_rq_type(priv->mdev, &priv->channels.params); if (was_opened && reset) - mlx5e_open_locked(netdev); + err = mlx5e_open_locked(netdev); if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset) goto unlock; @@ -4542,7 +4497,7 @@ void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, if (!slow_pci_heuristic(mdev) && mlx5e_striding_rq_possible(mdev, params) && (mlx5e_rx_mpwqe_is_linear_skb(mdev, params) || - !mlx5e_rx_is_linear_skb(mdev, params))) + !mlx5e_rx_is_linear_skb(params))) MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true); mlx5e_set_rq_type(mdev, params); mlx5e_init_rq_type_params(mdev, params); @@ -4553,7 +4508,7 @@ void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params, { enum mlx5e_traffic_types tt; - rss_params->hfunc = ETH_RSS_HASH_XOR; + rss_params->hfunc = ETH_RSS_HASH_TOP; netdev_rss_key_fill(rss_params->toeplitz_hash_key, sizeof(rss_params->toeplitz_hash_key)); mlx5e_build_default_indir_rqt(rss_params->indirection_rqt, @@ -4618,6 +4573,8 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, /* RSS */ mlx5e_build_rss_params(rss_params, params->num_channels); + params->tunneled_offload_en = + mlx5e_tunnel_inner_ft_supported(mdev); } static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) @@ -4639,7 +4596,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) bool fcs_supported; bool fcs_enabled; - SET_NETDEV_DEV(netdev, &mdev->pdev->dev); + SET_NETDEV_DEV(netdev, mdev->device); netdev->netdev_ops = &mlx5e_netdev_ops; @@ -4674,7 +4631,8 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; - if (mlx5_vxlan_allowed(mdev->vxlan) || MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) { + if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev) || + MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) { netdev->hw_enc_features |= NETIF_F_IP_CSUM; netdev->hw_enc_features |= NETIF_F_IPV6_CSUM; netdev->hw_enc_features |= NETIF_F_TSO; @@ -4682,7 +4640,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; } - if (mlx5_vxlan_allowed(mdev->vxlan)) { + if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_UDP_TUNNEL_CSUM; netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | @@ -4901,7 +4859,6 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) { struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; - u16 max_mtu; mlx5e_init_l2_addr(priv); @@ -4909,10 +4866,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) if (!netif_running(netdev)) mlx5_set_port_admin_status(mdev, MLX5_PORT_DOWN); - /* MTU range: 68 - hw-specific max */ - netdev->min_mtu = ETH_MIN_MTU; - mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1); - netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu); + mlx5e_set_netdev_mtu_boundaries(priv); mlx5e_set_dev_port_mtu(priv); mlx5_lag_add(mdev, netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index a66b6ed80b30..91e24f1cead8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -795,7 +795,8 @@ static int mlx5e_nic_rep_netdevice_event(struct notifier_block *nb, struct mlx5e_priv *priv = netdev_priv(rpriv->netdev); struct net_device *netdev = netdev_notifier_info_to_dev(ptr); - if (!mlx5e_tc_tun_device_to_offload(priv, netdev)) + if (!mlx5e_tc_tun_device_to_offload(priv, netdev) && + !is_vlan_dev(netdev)) return NOTIFY_OK; switch (event) { @@ -1374,6 +1375,7 @@ static void mlx5e_build_rep_params(struct net_device *netdev) mlx5e_set_rx_cq_mode_params(params, cq_period_mode); params->num_tc = 1; + params->tunneled_offload_en = false; mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); @@ -1389,7 +1391,7 @@ static void mlx5e_build_rep_netdev(struct net_device *netdev) struct mlx5_core_dev *mdev = priv->mdev; if (rep->vport == MLX5_VPORT_UPLINK) { - SET_NETDEV_DEV(netdev, &priv->mdev->pdev->dev); + SET_NETDEV_DEV(netdev, mdev->device); netdev->netdev_ops = &mlx5e_netdev_ops_uplink_rep; /* we want a persistent mac for the uplink rep */ mlx5_query_nic_vport_mac_address(mdev, 0, netdev->dev_addr); @@ -1623,13 +1625,7 @@ static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv) static void mlx5e_vf_rep_enable(struct mlx5e_priv *priv) { - struct net_device *netdev = priv->netdev; - struct mlx5_core_dev *mdev = priv->mdev; - u16 max_mtu; - - netdev->min_mtu = ETH_MIN_MTU; - mlx5_query_port_max_mtu(mdev, &max_mtu, 1); - netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu); + mlx5e_set_netdev_mtu_boundaries(priv); } static int uplink_rep_async_event(struct notifier_block *nb, unsigned long event, void *data) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 3dde5c7e0739..13133e7f088e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -409,14 +409,15 @@ mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle mlx5e_page_release(rq, &dma_info[i], recycle); } -static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq) +static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n) { struct mlx5_wq_ll *wq = &rq->mpwqe.wq; - struct mlx5e_rx_wqe_ll *wqe = mlx5_wq_ll_get_wqe(wq, wq->head); - rq->mpwqe.umr_in_progress = false; + do { + u16 next_wqe_index = mlx5_wq_ll_get_wqe_next_ix(wq, wq->head); - mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index)); + mlx5_wq_ll_push(wq, next_wqe_index); + } while (--n); /* ensure wqes are visible to device before updating doorbell record */ dma_wmb(); @@ -426,7 +427,7 @@ static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq) static inline u16 mlx5e_icosq_wrap_cnt(struct mlx5e_icosq *sq) { - return sq->pc >> MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; + return mlx5_wq_cyc_get_ctr_wrap_cnt(&sq->wq, sq->pc); } static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq *sq, @@ -478,8 +479,6 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE); wi->consumed_strides = 0; - rq->mpwqe.umr_in_progress = true; - umr_wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | MLX5_OPCODE_UMR); @@ -487,7 +486,8 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR; sq->pc += MLX5E_UMR_WQEBBS; - mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &umr_wqe->ctrl); + + sq->doorbell_cseg = &umr_wqe->ctrl; return 0; @@ -542,37 +542,13 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) return !!err; } -static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq, - struct mlx5e_icosq *sq, - struct mlx5e_rq *rq, - struct mlx5_cqe64 *cqe) -{ - struct mlx5_wq_cyc *wq = &sq->wq; - u16 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter)); - struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci]; - - mlx5_cqwq_pop(&cq->wq); - - if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) { - netdev_WARN_ONCE(cq->channel->netdev, - "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe)); - return; - } - - if (likely(icowi->opcode == MLX5_OPCODE_UMR)) { - mlx5e_post_rx_mpwqe(rq); - return; - } - - if (unlikely(icowi->opcode != MLX5_OPCODE_NOP)) - netdev_WARN_ONCE(cq->channel->netdev, - "Bad OPCODE in ICOSQ WQE info: 0x%x\n", icowi->opcode); -} - static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) { struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq); struct mlx5_cqe64 *cqe; + u8 completed_umr = 0; + u16 sqcc; + int i; if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) return; @@ -581,28 +557,96 @@ static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) if (likely(!cqe)) return; - /* by design, there's only a single cqe */ - mlx5e_poll_ico_single_cqe(cq, sq, rq, cqe); + /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), + * otherwise a cq overrun may occur + */ + sqcc = sq->cc; + + i = 0; + do { + u16 wqe_counter; + bool last_wqe; + + mlx5_cqwq_pop(&cq->wq); + + wqe_counter = be16_to_cpu(cqe->wqe_counter); + + if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) { + netdev_WARN_ONCE(cq->channel->netdev, + "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe)); + break; + } + do { + struct mlx5e_sq_wqe_info *wi; + u16 ci; + + last_wqe = (sqcc == wqe_counter); + + ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); + wi = &sq->db.ico_wqe[ci]; + + if (likely(wi->opcode == MLX5_OPCODE_UMR)) { + sqcc += MLX5E_UMR_WQEBBS; + completed_umr++; + } else if (likely(wi->opcode == MLX5_OPCODE_NOP)) { + sqcc++; + } else { + netdev_WARN_ONCE(cq->channel->netdev, + "Bad OPCODE in ICOSQ WQE info: 0x%x\n", + wi->opcode); + } + + } while (!last_wqe); + + } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); + + sq->cc = sqcc; mlx5_cqwq_update_db_record(&cq->wq); + + if (likely(completed_umr)) { + mlx5e_post_rx_mpwqe(rq, completed_umr); + rq->mpwqe.umr_in_progress -= completed_umr; + } } bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq) { + struct mlx5e_icosq *sq = &rq->channel->icosq; struct mlx5_wq_ll *wq = &rq->mpwqe.wq; + u8 missing, i; + u16 head; if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) return false; - mlx5e_poll_ico_cq(&rq->channel->icosq.cq, rq); + mlx5e_poll_ico_cq(&sq->cq, rq); + + missing = mlx5_wq_ll_missing(wq) - rq->mpwqe.umr_in_progress; - if (mlx5_wq_ll_is_full(wq)) + if (unlikely(rq->mpwqe.umr_in_progress > rq->mpwqe.umr_last_bulk)) + rq->stats->congst_umr++; + +#define UMR_WQE_BULK (2) + if (likely(missing < UMR_WQE_BULK)) return false; - if (!rq->mpwqe.umr_in_progress) - mlx5e_alloc_rx_mpwqe(rq, wq->head); - else - rq->stats->congst_umr += mlx5_wq_ll_missing(wq) > 2; + head = rq->mpwqe.actual_wq_head; + i = missing; + do { + if (unlikely(mlx5e_alloc_rx_mpwqe(rq, head))) + break; + head = mlx5_wq_ll_get_wqe_next_ix(wq, head); + } while (--i); + + rq->mpwqe.umr_last_bulk = missing - i; + if (sq->doorbell_cseg) { + mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg); + sq->doorbell_cseg = NULL; + } + + rq->mpwqe.umr_in_progress += rq->mpwqe.umr_last_bulk; + rq->mpwqe.actual_wq_head = head; return false; } @@ -692,7 +736,14 @@ static inline bool is_last_ethertype_ip(struct sk_buff *skb, int *network_depth, { *proto = ((struct ethhdr *)skb->data)->h_proto; *proto = __vlan_get_protocol(skb, *proto, network_depth); - return (*proto == htons(ETH_P_IP) || *proto == htons(ETH_P_IPV6)); + + if (*proto == htons(ETH_P_IP)) + return pskb_may_pull(skb, *network_depth + sizeof(struct iphdr)); + + if (*proto == htons(ETH_P_IPV6)) + return pskb_may_pull(skb, *network_depth + sizeof(struct ipv6hdr)); + + return false; } static inline void mlx5e_enable_ecn(struct mlx5e_rq *rq, struct sk_buff *skb) @@ -712,17 +763,6 @@ static inline void mlx5e_enable_ecn(struct mlx5e_rq *rq, struct sk_buff *skb) rq->stats->ecn_mark += !!rc; } -static u32 mlx5e_get_fcs(const struct sk_buff *skb) -{ - const void *fcs_bytes; - u32 _fcs_bytes; - - fcs_bytes = skb_header_pointer(skb, skb->len - ETH_FCS_LEN, - ETH_FCS_LEN, &_fcs_bytes); - - return __get_unaligned_cpu32(fcs_bytes); -} - static u8 get_ip_proto(struct sk_buff *skb, int network_depth, __be16 proto) { void *ip_p = skb->data + network_depth; @@ -733,6 +773,68 @@ static u8 get_ip_proto(struct sk_buff *skb, int network_depth, __be16 proto) #define short_frame(size) ((size) <= ETH_ZLEN + ETH_FCS_LEN) +#define MAX_PADDING 8 + +static void +tail_padding_csum_slow(struct sk_buff *skb, int offset, int len, + struct mlx5e_rq_stats *stats) +{ + stats->csum_complete_tail_slow++; + skb->csum = csum_block_add(skb->csum, + skb_checksum(skb, offset, len, 0), + offset); +} + +static void +tail_padding_csum(struct sk_buff *skb, int offset, + struct mlx5e_rq_stats *stats) +{ + u8 tail_padding[MAX_PADDING]; + int len = skb->len - offset; + void *tail; + + if (unlikely(len > MAX_PADDING)) { + tail_padding_csum_slow(skb, offset, len, stats); + return; + } + + tail = skb_header_pointer(skb, offset, len, tail_padding); + if (unlikely(!tail)) { + tail_padding_csum_slow(skb, offset, len, stats); + return; + } + + stats->csum_complete_tail++; + skb->csum = csum_block_add(skb->csum, csum_partial(tail, len, 0), offset); +} + +static void +mlx5e_skb_padding_csum(struct sk_buff *skb, int network_depth, __be16 proto, + struct mlx5e_rq_stats *stats) +{ + struct ipv6hdr *ip6; + struct iphdr *ip4; + int pkt_len; + + switch (proto) { + case htons(ETH_P_IP): + ip4 = (struct iphdr *)(skb->data + network_depth); + pkt_len = network_depth + ntohs(ip4->tot_len); + break; + case htons(ETH_P_IPV6): + ip6 = (struct ipv6hdr *)(skb->data + network_depth); + pkt_len = network_depth + sizeof(*ip6) + ntohs(ip6->payload_len); + break; + default: + return; + } + + if (likely(pkt_len >= skb->len)) + return; + + tail_padding_csum(skb, pkt_len, stats); +} + static inline void mlx5e_handle_csum(struct net_device *netdev, struct mlx5_cqe64 *cqe, struct mlx5e_rq *rq, @@ -752,7 +854,8 @@ static inline void mlx5e_handle_csum(struct net_device *netdev, return; } - if (unlikely(test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state))) + /* True when explicitly set via priv flag, or XDP prog is loaded */ + if (test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state)) goto csum_unnecessary; /* CQE csum doesn't cover padding octets in short ethernet @@ -780,18 +883,15 @@ static inline void mlx5e_handle_csum(struct net_device *netdev, skb->csum = csum_partial(skb->data + ETH_HLEN, network_depth - ETH_HLEN, skb->csum); - if (unlikely(netdev->features & NETIF_F_RXFCS)) - skb->csum = csum_block_add(skb->csum, - (__force __wsum)mlx5e_get_fcs(skb), - skb->len - ETH_FCS_LEN); + + mlx5e_skb_padding_csum(skb, network_depth, proto, stats); stats->csum_complete++; return; } csum_unnecessary: if (likely((cqe->hds_ip_ext & CQE_L3_OK) && - ((cqe->hds_ip_ext & CQE_L4_OK) || - (get_cqe_l4_hdr_type(cqe) == CQE_L4_HDR_TYPE_NONE)))) { + (cqe->hds_ip_ext & CQE_L4_OK))) { skb->ip_summed = CHECKSUM_UNNECESSARY; if (cqe_is_tunneled(cqe)) { skb->csum_level = 1; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 1a78e05cbba8..483d321d2151 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -59,10 +59,14 @@ static const struct counter_desc sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_unnecessary) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_none) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_complete) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_complete_tail) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_complete_tail_slow) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_unnecessary_inner) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_redirect) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_xmit) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_mpwqe) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_inlnw) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_err) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) }, @@ -77,6 +81,8 @@ static const struct counter_desc sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_wake) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_cqe_err) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_xmit) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_mpwqe) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_inlnw) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_full) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_err) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_cqes) }, @@ -87,7 +93,6 @@ static const struct counter_desc sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_buff_alloc_err) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_blks) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_pkts) }, - { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_page_reuse) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_reuse) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) }, @@ -151,11 +156,15 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) s->rx_removed_vlan_packets += rq_stats->removed_vlan_packets; s->rx_csum_none += rq_stats->csum_none; s->rx_csum_complete += rq_stats->csum_complete; + s->rx_csum_complete_tail += rq_stats->csum_complete_tail; + s->rx_csum_complete_tail_slow += rq_stats->csum_complete_tail_slow; s->rx_csum_unnecessary += rq_stats->csum_unnecessary; s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner; s->rx_xdp_drop += rq_stats->xdp_drop; s->rx_xdp_redirect += rq_stats->xdp_redirect; s->rx_xdp_tx_xmit += xdpsq_stats->xmit; + s->rx_xdp_tx_mpwqe += xdpsq_stats->mpwqe; + s->rx_xdp_tx_inlnw += xdpsq_stats->inlnw; s->rx_xdp_tx_full += xdpsq_stats->full; s->rx_xdp_tx_err += xdpsq_stats->err; s->rx_xdp_tx_cqe += xdpsq_stats->cqes; @@ -166,7 +175,6 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) s->rx_buff_alloc_err += rq_stats->buff_alloc_err; s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks; s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts; - s->rx_page_reuse += rq_stats->page_reuse; s->rx_cache_reuse += rq_stats->cache_reuse; s->rx_cache_full += rq_stats->cache_full; s->rx_cache_empty += rq_stats->cache_empty; @@ -181,6 +189,8 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) s->ch_eq_rearm += ch_stats->eq_rearm; /* xdp redirect */ s->tx_xdp_xmit += xdpsq_red_stats->xmit; + s->tx_xdp_mpwqe += xdpsq_red_stats->mpwqe; + s->tx_xdp_inlnw += xdpsq_red_stats->inlnw; s->tx_xdp_full += xdpsq_red_stats->full; s->tx_xdp_err += xdpsq_red_stats->err; s->tx_xdp_cqes += xdpsq_red_stats->cqes; @@ -1190,6 +1200,8 @@ static const struct counter_desc rq_stats_desc[] = { { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, packets) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, bytes) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_complete) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_complete_tail) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_complete_tail_slow) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_unnecessary) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_unnecessary_inner) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_none) }, @@ -1206,7 +1218,6 @@ static const struct counter_desc rq_stats_desc[] = { { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, buff_alloc_err) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_blks) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) }, - { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, page_reuse) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_reuse) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) }, @@ -1239,6 +1250,8 @@ static const struct counter_desc sq_stats_desc[] = { static const struct counter_desc rq_xdpsq_stats_desc[] = { { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) }, + { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) }, + { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) }, { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) }, { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) }, { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) }, @@ -1246,6 +1259,8 @@ static const struct counter_desc rq_xdpsq_stats_desc[] = { static const struct counter_desc xdpsq_stats_desc[] = { { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) }, + { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) }, + { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) }, { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) }, { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) }, { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) }, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index 4640d4f986f8..cdddcc46971b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -71,10 +71,14 @@ struct mlx5e_sw_stats { u64 rx_csum_unnecessary; u64 rx_csum_none; u64 rx_csum_complete; + u64 rx_csum_complete_tail; + u64 rx_csum_complete_tail_slow; u64 rx_csum_unnecessary_inner; u64 rx_xdp_drop; u64 rx_xdp_redirect; u64 rx_xdp_tx_xmit; + u64 rx_xdp_tx_mpwqe; + u64 rx_xdp_tx_inlnw; u64 rx_xdp_tx_full; u64 rx_xdp_tx_err; u64 rx_xdp_tx_cqe; @@ -89,6 +93,8 @@ struct mlx5e_sw_stats { u64 tx_queue_wake; u64 tx_cqe_err; u64 tx_xdp_xmit; + u64 tx_xdp_mpwqe; + u64 tx_xdp_inlnw; u64 tx_xdp_full; u64 tx_xdp_err; u64 tx_xdp_cqes; @@ -99,7 +105,6 @@ struct mlx5e_sw_stats { u64 rx_buff_alloc_err; u64 rx_cqe_compress_blks; u64 rx_cqe_compress_pkts; - u64 rx_page_reuse; u64 rx_cache_reuse; u64 rx_cache_full; u64 rx_cache_empty; @@ -181,6 +186,8 @@ struct mlx5e_rq_stats { u64 packets; u64 bytes; u64 csum_complete; + u64 csum_complete_tail; + u64 csum_complete_tail_slow; u64 csum_unnecessary; u64 csum_unnecessary_inner; u64 csum_none; @@ -197,7 +204,6 @@ struct mlx5e_rq_stats { u64 buff_alloc_err; u64 cqe_compress_blks; u64 cqe_compress_pkts; - u64 page_reuse; u64 cache_reuse; u64 cache_full; u64 cache_empty; @@ -237,6 +243,8 @@ struct mlx5e_sq_stats { struct mlx5e_xdpsq_stats { u64 xmit; + u64 mpwqe; + u64 inlnw; u64 full; u64 err; /* dirtied @completion */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index d75dc44eb2ff..122f457091a2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -44,6 +44,7 @@ #include <net/tc_act/tc_pedit.h> #include <net/tc_act/tc_csum.h> #include <net/arp.h> +#include <net/ipv6_stubs.h> #include "en.h" #include "en_rep.h" #include "en_tc.h" @@ -663,7 +664,8 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, } netdev_dbg(priv->netdev, "add hairpin: tirn %x rqn %x peer %s sqn %x prio %d (log) data %d packets %d\n", - hp->tirn, hp->pair->rqn[0], hp->pair->peer_mdev->priv.name, + hp->tirn, hp->pair->rqn[0], + dev_name(hp->pair->peer_mdev->device), hp->pair->sqn[0], match_prio, params.log_data_size, params.log_num_packets); hpe->hp = hp; @@ -700,7 +702,7 @@ static void mlx5e_hairpin_flow_del(struct mlx5e_priv *priv, hpe = list_entry(next, struct mlx5e_hairpin_entry, flows); netdev_dbg(priv->netdev, "del hairpin: peer %s\n", - hpe->hp->pair->peer_mdev->priv.name); + dev_name(hpe->hp->pair->peer_mdev->device)); mlx5e_hairpin_destroy(hpe->hp); hash_del(&hpe->hairpin_hlist); @@ -1437,6 +1439,26 @@ static int parse_tunnel_attr(struct mlx5e_priv *priv, return 0; } +static void *get_match_headers_criteria(u32 flags, + struct mlx5_flow_spec *spec) +{ + return (flags & MLX5_FLOW_CONTEXT_ACTION_DECAP) ? + MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + inner_headers) : + MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers); +} + +static void *get_match_headers_value(u32 flags, + struct mlx5_flow_spec *spec) +{ + return (flags & MLX5_FLOW_CONTEXT_ACTION_DECAP) ? + MLX5_ADDR_OF(fte_match_param, spec->match_value, + inner_headers) : + MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers); +} + static int __parse_cls_flower(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec, struct tc_cls_flower_offload *f, @@ -1502,10 +1524,10 @@ static int __parse_cls_flower(struct mlx5e_priv *priv, /* In decap flow, header pointers should point to the inner * headers, outer header were already set by parse_tunnel_attr */ - headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - inner_headers); - headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, - inner_headers); + headers_c = get_match_headers_criteria(MLX5_FLOW_CONTEXT_ACTION_DECAP, + spec); + headers_v = get_match_headers_value(MLX5_FLOW_CONTEXT_ACTION_DECAP, + spec); } if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { @@ -1520,11 +1542,23 @@ static int __parse_cls_flower(struct mlx5e_priv *priv, if (match.mask->n_proto) *match_level = MLX5_MATCH_L2; } - - if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN) || + is_vlan_dev(filter_dev)) { + struct flow_dissector_key_vlan filter_dev_mask; + struct flow_dissector_key_vlan filter_dev_key; struct flow_match_vlan match; - flow_rule_match_vlan(rule, &match); + if (is_vlan_dev(filter_dev)) { + match.key = &filter_dev_key; + match.key->vlan_id = vlan_dev_vlan_id(filter_dev); + match.key->vlan_tpid = vlan_dev_vlan_proto(filter_dev); + match.key->vlan_priority = 0; + match.mask = &filter_dev_mask; + memset(match.mask, 0xff, sizeof(*match.mask)); + match.mask->vlan_priority = 0; + } else { + flow_rule_match_vlan(rule, &match); + } if (match.mask->vlan_id || match.mask->vlan_priority || match.mask->vlan_tpid) { @@ -1827,6 +1861,7 @@ static int parse_cls_flower(struct mlx5e_priv *priv, struct pedit_headers { struct ethhdr eth; + struct vlan_hdr vlan; struct iphdr ip4; struct ipv6hdr ip6; struct tcphdr tcp; @@ -1873,38 +1908,73 @@ struct mlx5_fields { u8 field; u8 size; u32 offset; + u32 match_offset; }; -#define OFFLOAD(fw_field, size, field, off) \ - {MLX5_ACTION_IN_FIELD_OUT_ ## fw_field, size, offsetof(struct pedit_headers, field) + (off)} +#define OFFLOAD(fw_field, size, field, off, match_field) \ + {MLX5_ACTION_IN_FIELD_OUT_ ## fw_field, size, \ + offsetof(struct pedit_headers, field) + (off), \ + MLX5_BYTE_OFF(fte_match_set_lyr_2_4, match_field)} + +static bool cmp_val_mask(void *valp, void *maskp, void *matchvalp, + void *matchmaskp, int size) +{ + bool same = false; + + switch (size) { + case sizeof(u8): + same = ((*(u8 *)valp) & (*(u8 *)maskp)) == + ((*(u8 *)matchvalp) & (*(u8 *)matchmaskp)); + break; + case sizeof(u16): + same = ((*(u16 *)valp) & (*(u16 *)maskp)) == + ((*(u16 *)matchvalp) & (*(u16 *)matchmaskp)); + break; + case sizeof(u32): + same = ((*(u32 *)valp) & (*(u32 *)maskp)) == + ((*(u32 *)matchvalp) & (*(u32 *)matchmaskp)); + break; + } + + return same; +} static struct mlx5_fields fields[] = { - OFFLOAD(DMAC_47_16, 4, eth.h_dest[0], 0), - OFFLOAD(DMAC_15_0, 2, eth.h_dest[4], 0), - OFFLOAD(SMAC_47_16, 4, eth.h_source[0], 0), - OFFLOAD(SMAC_15_0, 2, eth.h_source[4], 0), - OFFLOAD(ETHERTYPE, 2, eth.h_proto, 0), - - OFFLOAD(IP_TTL, 1, ip4.ttl, 0), - OFFLOAD(SIPV4, 4, ip4.saddr, 0), - OFFLOAD(DIPV4, 4, ip4.daddr, 0), - - OFFLOAD(SIPV6_127_96, 4, ip6.saddr.s6_addr32[0], 0), - OFFLOAD(SIPV6_95_64, 4, ip6.saddr.s6_addr32[1], 0), - OFFLOAD(SIPV6_63_32, 4, ip6.saddr.s6_addr32[2], 0), - OFFLOAD(SIPV6_31_0, 4, ip6.saddr.s6_addr32[3], 0), - OFFLOAD(DIPV6_127_96, 4, ip6.daddr.s6_addr32[0], 0), - OFFLOAD(DIPV6_95_64, 4, ip6.daddr.s6_addr32[1], 0), - OFFLOAD(DIPV6_63_32, 4, ip6.daddr.s6_addr32[2], 0), - OFFLOAD(DIPV6_31_0, 4, ip6.daddr.s6_addr32[3], 0), - OFFLOAD(IPV6_HOPLIMIT, 1, ip6.hop_limit, 0), - - OFFLOAD(TCP_SPORT, 2, tcp.source, 0), - OFFLOAD(TCP_DPORT, 2, tcp.dest, 0), - OFFLOAD(TCP_FLAGS, 1, tcp.ack_seq, 5), - - OFFLOAD(UDP_SPORT, 2, udp.source, 0), - OFFLOAD(UDP_DPORT, 2, udp.dest, 0), + OFFLOAD(DMAC_47_16, 4, eth.h_dest[0], 0, dmac_47_16), + OFFLOAD(DMAC_15_0, 2, eth.h_dest[4], 0, dmac_15_0), + OFFLOAD(SMAC_47_16, 4, eth.h_source[0], 0, smac_47_16), + OFFLOAD(SMAC_15_0, 2, eth.h_source[4], 0, smac_15_0), + OFFLOAD(ETHERTYPE, 2, eth.h_proto, 0, ethertype), + OFFLOAD(FIRST_VID, 2, vlan.h_vlan_TCI, 0, first_vid), + + OFFLOAD(IP_TTL, 1, ip4.ttl, 0, ttl_hoplimit), + OFFLOAD(SIPV4, 4, ip4.saddr, 0, src_ipv4_src_ipv6.ipv4_layout.ipv4), + OFFLOAD(DIPV4, 4, ip4.daddr, 0, dst_ipv4_dst_ipv6.ipv4_layout.ipv4), + + OFFLOAD(SIPV6_127_96, 4, ip6.saddr.s6_addr32[0], 0, + src_ipv4_src_ipv6.ipv6_layout.ipv6[0]), + OFFLOAD(SIPV6_95_64, 4, ip6.saddr.s6_addr32[1], 0, + src_ipv4_src_ipv6.ipv6_layout.ipv6[4]), + OFFLOAD(SIPV6_63_32, 4, ip6.saddr.s6_addr32[2], 0, + src_ipv4_src_ipv6.ipv6_layout.ipv6[8]), + OFFLOAD(SIPV6_31_0, 4, ip6.saddr.s6_addr32[3], 0, + src_ipv4_src_ipv6.ipv6_layout.ipv6[12]), + OFFLOAD(DIPV6_127_96, 4, ip6.daddr.s6_addr32[0], 0, + dst_ipv4_dst_ipv6.ipv6_layout.ipv6[0]), + OFFLOAD(DIPV6_95_64, 4, ip6.daddr.s6_addr32[1], 0, + dst_ipv4_dst_ipv6.ipv6_layout.ipv6[4]), + OFFLOAD(DIPV6_63_32, 4, ip6.daddr.s6_addr32[2], 0, + dst_ipv4_dst_ipv6.ipv6_layout.ipv6[8]), + OFFLOAD(DIPV6_31_0, 4, ip6.daddr.s6_addr32[3], 0, + dst_ipv4_dst_ipv6.ipv6_layout.ipv6[12]), + OFFLOAD(IPV6_HOPLIMIT, 1, ip6.hop_limit, 0, ttl_hoplimit), + + OFFLOAD(TCP_SPORT, 2, tcp.source, 0, tcp_sport), + OFFLOAD(TCP_DPORT, 2, tcp.dest, 0, tcp_dport), + OFFLOAD(TCP_FLAGS, 1, tcp.ack_seq, 5, tcp_flags), + + OFFLOAD(UDP_SPORT, 2, udp.source, 0, udp_sport), + OFFLOAD(UDP_DPORT, 2, udp.dest, 0, udp_dport), }; /* On input attr->max_mod_hdr_actions tells how many HW actions can be parsed at @@ -1913,9 +1983,14 @@ static struct mlx5_fields fields[] = { */ static int offload_pedit_fields(struct pedit_headers_action *hdrs, struct mlx5e_tc_flow_parse_attr *parse_attr, + u32 *action_flags, struct netlink_ext_ack *extack) { struct pedit_headers *set_masks, *add_masks, *set_vals, *add_vals; + void *headers_c = get_match_headers_criteria(*action_flags, + &parse_attr->spec); + void *headers_v = get_match_headers_value(*action_flags, + &parse_attr->spec); int i, action_size, nactions, max_actions, first, last, next_z; void *s_masks_p, *a_masks_p, *vals_p; struct mlx5_fields *f; @@ -1939,6 +2014,8 @@ static int offload_pedit_fields(struct pedit_headers_action *hdrs, nactions = parse_attr->num_mod_hdr_actions; for (i = 0; i < ARRAY_SIZE(fields); i++) { + bool skip; + f = &fields[i]; /* avoid seeing bits set from previous iterations */ s_mask = 0; @@ -1967,19 +2044,34 @@ static int offload_pedit_fields(struct pedit_headers_action *hdrs, return -EOPNOTSUPP; } + skip = false; if (s_mask) { + void *match_mask = headers_c + f->match_offset; + void *match_val = headers_v + f->match_offset; + cmd = MLX5_ACTION_TYPE_SET; mask = s_mask; vals_p = (void *)set_vals + f->offset; + /* don't rewrite if we have a match on the same value */ + if (cmp_val_mask(vals_p, s_masks_p, match_val, + match_mask, f->size)) + skip = true; /* clear to denote we consumed this field */ memset(s_masks_p, 0, f->size); } else { + u32 zero = 0; + cmd = MLX5_ACTION_TYPE_ADD; mask = a_mask; vals_p = (void *)add_vals + f->offset; + /* add 0 is no change */ + if (!memcmp(vals_p, &zero, f->size)) + skip = true; /* clear to denote we consumed this field */ memset(a_masks_p, 0, f->size); } + if (skip) + continue; field_bsize = f->size * BITS_PER_BYTE; @@ -2026,6 +2118,15 @@ static int offload_pedit_fields(struct pedit_headers_action *hdrs, return 0; } +static int mlx5e_flow_namespace_max_modify_action(struct mlx5_core_dev *mdev, + int namespace) +{ + if (namespace == MLX5_FLOW_NAMESPACE_FDB) /* FDB offloading */ + return MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, max_modify_header_actions); + else /* namespace is MLX5_FLOW_NAMESPACE_KERNEL - NIC offloading */ + return MLX5_CAP_FLOWTABLE_NIC_RX(mdev, max_modify_header_actions); +} + static int alloc_mod_hdr_actions(struct mlx5e_priv *priv, struct pedit_headers_action *hdrs, int namespace, @@ -2037,11 +2138,7 @@ static int alloc_mod_hdr_actions(struct mlx5e_priv *priv, hdrs[TCA_PEDIT_KEY_EX_CMD_ADD].pedits; action_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto); - if (namespace == MLX5_FLOW_NAMESPACE_FDB) /* FDB offloading */ - max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev, max_modify_header_actions); - else /* namespace is MLX5_FLOW_NAMESPACE_KERNEL - NIC offloading */ - max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, max_modify_header_actions); - + max_actions = mlx5e_flow_namespace_max_modify_action(priv->mdev, namespace); /* can get up to crazingly 16 HW actions in 32 bits pedit SW key */ max_actions = min(max_actions, nkeys * 16); @@ -2074,6 +2171,12 @@ static int parse_tc_pedit_action(struct mlx5e_priv *priv, goto out_err; } + if (!mlx5e_flow_namespace_max_modify_action(priv->mdev, namespace)) { + NL_SET_ERR_MSG_MOD(extack, + "The pedit offload action is not supported"); + goto out_err; + } + mask = act->mangle.mask; val = act->mangle.val; offset = act->mangle.offset; @@ -2092,6 +2195,7 @@ out_err: static int alloc_tc_pedit_action(struct mlx5e_priv *priv, int namespace, struct mlx5e_tc_flow_parse_attr *parse_attr, struct pedit_headers_action *hdrs, + u32 *action_flags, struct netlink_ext_ack *extack) { struct pedit_headers *cmd_masks; @@ -2104,7 +2208,7 @@ static int alloc_tc_pedit_action(struct mlx5e_priv *priv, int namespace, goto out_err; } - err = offload_pedit_fields(hdrs, parse_attr, extack); + err = offload_pedit_fields(hdrs, parse_attr, action_flags, extack); if (err < 0) goto out_dealloc_parsed_actions; @@ -2216,11 +2320,7 @@ static bool modify_header_match_supported(struct mlx5_flow_spec *spec, u8 ip_proto; int i; - if (actions & MLX5_FLOW_CONTEXT_ACTION_DECAP) - headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, inner_headers); - else - headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); - + headers_v = get_match_headers_value(actions, spec); ethertype = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ethertype); /* for non-IP we only re-write MACs, so we're okay */ @@ -2266,7 +2366,8 @@ static bool actions_match_supported(struct mlx5e_priv *priv, actions = flow->nic_attr->action; if (flow->flags & MLX5E_TC_FLOW_EGRESS && - !(actions & MLX5_FLOW_CONTEXT_ACTION_DECAP)) + !((actions & MLX5_FLOW_CONTEXT_ACTION_DECAP) || + (actions & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP))) return false; if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) @@ -2291,6 +2392,74 @@ static bool same_hw_devs(struct mlx5e_priv *priv, struct mlx5e_priv *peer_priv) return (fsystem_guid == psystem_guid); } +static int add_vlan_rewrite_action(struct mlx5e_priv *priv, int namespace, + const struct flow_action_entry *act, + struct mlx5e_tc_flow_parse_attr *parse_attr, + struct pedit_headers_action *hdrs, + u32 *action, struct netlink_ext_ack *extack) +{ + u16 mask16 = VLAN_VID_MASK; + u16 val16 = act->vlan.vid & VLAN_VID_MASK; + const struct flow_action_entry pedit_act = { + .id = FLOW_ACTION_MANGLE, + .mangle.htype = FLOW_ACT_MANGLE_HDR_TYPE_ETH, + .mangle.offset = offsetof(struct vlan_ethhdr, h_vlan_TCI), + .mangle.mask = ~(u32)be16_to_cpu(*(__be16 *)&mask16), + .mangle.val = (u32)be16_to_cpu(*(__be16 *)&val16), + }; + u8 match_prio_mask, match_prio_val; + void *headers_c, *headers_v; + int err; + + headers_c = get_match_headers_criteria(*action, &parse_attr->spec); + headers_v = get_match_headers_value(*action, &parse_attr->spec); + + if (!(MLX5_GET(fte_match_set_lyr_2_4, headers_c, cvlan_tag) && + MLX5_GET(fte_match_set_lyr_2_4, headers_v, cvlan_tag))) { + NL_SET_ERR_MSG_MOD(extack, + "VLAN rewrite action must have VLAN protocol match"); + return -EOPNOTSUPP; + } + + match_prio_mask = MLX5_GET(fte_match_set_lyr_2_4, headers_c, first_prio); + match_prio_val = MLX5_GET(fte_match_set_lyr_2_4, headers_v, first_prio); + if (act->vlan.prio != (match_prio_val & match_prio_mask)) { + NL_SET_ERR_MSG_MOD(extack, + "Changing VLAN prio is not supported"); + return -EOPNOTSUPP; + } + + err = parse_tc_pedit_action(priv, &pedit_act, namespace, parse_attr, + hdrs, NULL); + *action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + + return err; +} + +static int +add_vlan_prio_tag_rewrite_action(struct mlx5e_priv *priv, + struct mlx5e_tc_flow_parse_attr *parse_attr, + struct pedit_headers_action *hdrs, + u32 *action, struct netlink_ext_ack *extack) +{ + const struct flow_action_entry prio_tag_act = { + .vlan.vid = 0, + .vlan.prio = + MLX5_GET(fte_match_set_lyr_2_4, + get_match_headers_value(*action, + &parse_attr->spec), + first_prio) & + MLX5_GET(fte_match_set_lyr_2_4, + get_match_headers_criteria(*action, + &parse_attr->spec), + first_prio), + }; + + return add_vlan_rewrite_action(priv, MLX5_FLOW_NAMESPACE_FDB, + &prio_tag_act, parse_attr, hdrs, action, + extack); +} + static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct flow_action *flow_action, struct mlx5e_tc_flow_parse_attr *parse_attr, @@ -2326,6 +2495,15 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; break; + case FLOW_ACTION_VLAN_MANGLE: + err = add_vlan_rewrite_action(priv, + MLX5_FLOW_NAMESPACE_KERNEL, + act, parse_attr, hdrs, + &action, extack); + if (err) + return err; + + break; case FLOW_ACTION_CSUM: if (csum_offload_supported(priv, action, act->csum_flags, @@ -2365,16 +2543,22 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, } break; default: - return -EINVAL; + NL_SET_ERR_MSG_MOD(extack, "The offload action is not supported"); + return -EOPNOTSUPP; } } if (hdrs[TCA_PEDIT_KEY_EX_CMD_SET].pedits || hdrs[TCA_PEDIT_KEY_EX_CMD_ADD].pedits) { err = alloc_tc_pedit_action(priv, MLX5_FLOW_NAMESPACE_KERNEL, - parse_attr, hdrs, extack); + parse_attr, hdrs, &action, extack); if (err) return err; + /* in case all pedit actions are skipped, remove the MOD_HDR + * flag. + */ + if (parse_attr->num_mod_hdr_actions == 0) + action &= ~MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; } attr->action = action; @@ -2544,8 +2728,7 @@ static int parse_tc_vlan_action(struct mlx5e_priv *priv, } break; default: - /* action is FLOW_ACT_VLAN_MANGLE */ - return -EOPNOTSUPP; + return -EINVAL; } attr->total_vlan = vlan_idx + 1; @@ -2553,15 +2736,60 @@ static int parse_tc_vlan_action(struct mlx5e_priv *priv, return 0; } +static int add_vlan_push_action(struct mlx5e_priv *priv, + struct mlx5_esw_flow_attr *attr, + struct net_device **out_dev, + u32 *action) +{ + struct net_device *vlan_dev = *out_dev; + struct flow_action_entry vlan_act = { + .id = FLOW_ACTION_VLAN_PUSH, + .vlan.vid = vlan_dev_vlan_id(vlan_dev), + .vlan.proto = vlan_dev_vlan_proto(vlan_dev), + .vlan.prio = 0, + }; + int err; + + err = parse_tc_vlan_action(priv, &vlan_act, attr, action); + if (err) + return err; + + *out_dev = dev_get_by_index_rcu(dev_net(vlan_dev), + dev_get_iflink(vlan_dev)); + if (is_vlan_dev(*out_dev)) + err = add_vlan_push_action(priv, attr, out_dev, action); + + return err; +} + +static int add_vlan_pop_action(struct mlx5e_priv *priv, + struct mlx5_esw_flow_attr *attr, + u32 *action) +{ + int nest_level = vlan_get_encap_level(attr->parse_attr->filter_dev); + struct flow_action_entry vlan_act = { + .id = FLOW_ACTION_VLAN_POP, + }; + int err = 0; + + while (nest_level--) { + err = parse_tc_vlan_action(priv, &vlan_act, attr, action); + if (err) + return err; + } + + return err; +} + static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct flow_action *flow_action, - struct mlx5e_tc_flow_parse_attr *parse_attr, struct mlx5e_tc_flow *flow, struct netlink_ext_ack *extack) { struct pedit_headers_action hdrs[2] = {}; struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct mlx5_esw_flow_attr *attr = flow->esw_attr; + struct mlx5e_tc_flow_parse_attr *parse_attr = attr->parse_attr; struct mlx5e_rep_priv *rpriv = priv->ppriv; const struct ip_tunnel_info *info = NULL; const struct flow_action_entry *act; @@ -2633,6 +2861,20 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, uplink_upper == out_dev) out_dev = uplink_dev; + if (is_vlan_dev(out_dev)) { + err = add_vlan_push_action(priv, attr, + &out_dev, + &action); + if (err) + return err; + } + if (is_vlan_dev(parse_attr->filter_dev)) { + err = add_vlan_pop_action(priv, attr, + &action); + if (err) + return err; + } + if (!mlx5e_eswitch_rep(out_dev)) return -EOPNOTSUPP; @@ -2646,7 +2888,6 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, out_dev->ifindex; parse_attr->tun_info[attr->out_count] = *info; encap = false; - attr->parse_attr = parse_attr; attr->dests[attr->out_count].flags |= MLX5_ESW_DEST_ENCAP; attr->out_count++; @@ -2679,7 +2920,27 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, break; case FLOW_ACTION_VLAN_PUSH: case FLOW_ACTION_VLAN_POP: - err = parse_tc_vlan_action(priv, act, attr, &action); + if (act->id == FLOW_ACTION_VLAN_PUSH && + (action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP)) { + /* Replace vlan pop+push with vlan modify */ + action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_POP; + err = add_vlan_rewrite_action(priv, + MLX5_FLOW_NAMESPACE_FDB, + act, parse_attr, hdrs, + &action, extack); + } else { + err = parse_tc_vlan_action(priv, act, attr, &action); + } + if (err) + return err; + + attr->split_count = attr->out_count; + break; + case FLOW_ACTION_VLAN_MANGLE: + err = add_vlan_rewrite_action(priv, + MLX5_FLOW_NAMESPACE_FDB, + act, parse_attr, hdrs, + &action, extack); if (err) return err; @@ -2705,16 +2966,39 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, break; } default: - return -EINVAL; + NL_SET_ERR_MSG_MOD(extack, "The offload action is not supported"); + return -EOPNOTSUPP; } } + if (MLX5_CAP_GEN(esw->dev, prio_tag_required) && + action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP) { + /* For prio tag mode, replace vlan pop with rewrite vlan prio + * tag rewrite. + */ + action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_POP; + err = add_vlan_prio_tag_rewrite_action(priv, parse_attr, hdrs, + &action, extack); + if (err) + return err; + } + if (hdrs[TCA_PEDIT_KEY_EX_CMD_SET].pedits || hdrs[TCA_PEDIT_KEY_EX_CMD_ADD].pedits) { err = alloc_tc_pedit_action(priv, MLX5_FLOW_NAMESPACE_FDB, - parse_attr, hdrs, extack); + parse_attr, hdrs, &action, extack); if (err) return err; + /* in case all pedit actions are skipped, remove the MOD_HDR + * flag. we might have set split_count either by pedit or + * pop/push. if there is no pop/push either, reset it too. + */ + if (parse_attr->num_mod_hdr_actions == 0) { + action &= ~MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + if (!((action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP) || + (action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH))) + attr->split_count = 0; + } } attr->action = action; @@ -2883,7 +3167,7 @@ __mlx5e_add_fdb_flow(struct mlx5e_priv *priv, if (err) goto err_free; - err = parse_tc_fdb_actions(priv, &rule->action, parse_attr, flow, extack); + err = parse_tc_fdb_actions(priv, &rule->action, flow, extack); if (err) goto err_free; @@ -3080,6 +3364,7 @@ int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv, netdev_warn_once(priv->netdev, "flow cookie %lx already exists, ignoring\n", f->cookie); + err = -EEXIST; goto out; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 25a8f8260c14..7b61126fcec9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -32,6 +32,7 @@ #include <linux/tcp.h> #include <linux/if_vlan.h> +#include <net/geneve.h> #include <net/dsfield.h> #include "en.h" #include "ipoib/ipoib.h" @@ -110,11 +111,10 @@ static inline int mlx5e_get_dscp_up(struct mlx5e_priv *priv, struct sk_buff *skb #endif u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev, - select_queue_fallback_t fallback) + struct net_device *sb_dev) { + int channel_ix = netdev_pick_tx(dev, skb, NULL); struct mlx5e_priv *priv = netdev_priv(dev); - int channel_ix = fallback(dev, skb, NULL); u16 num_channels; int up = 0; @@ -163,7 +163,7 @@ static inline u16 mlx5e_calc_min_inline(enum mlx5_inline_modes mode, case MLX5_INLINE_MODE_NONE: return 0; case MLX5_INLINE_MODE_TCP_UDP: - hlen = eth_get_headlen(skb->data, skb_headlen(skb)); + hlen = eth_get_headlen(skb->dev, skb->data, skb_headlen(skb)); if (hlen == ETH_HLEN && !skb_vlan_tag_present(skb)) hlen += VLAN_HLEN; break; @@ -297,7 +297,8 @@ static inline void mlx5e_fill_sq_frag_edge(struct mlx5e_txqsq *sq, static inline void mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb, u8 opcode, u16 ds_cnt, u8 num_wqebbs, u32 num_bytes, u8 num_dma, - struct mlx5e_tx_wqe_info *wi, struct mlx5_wqe_ctrl_seg *cseg) + struct mlx5e_tx_wqe_info *wi, struct mlx5_wqe_ctrl_seg *cseg, + bool xmit_more) { struct mlx5_wq_cyc *wq = &sq->wq; @@ -320,14 +321,14 @@ mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb, sq->stats->stopped++; } - if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) + if (!xmit_more || netif_xmit_stopped(sq->txq)) mlx5e_notify_hw(wq, sq->pc, sq->uar_map, cseg); } #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start)) netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, - struct mlx5e_tx_wqe *wqe, u16 pi) + struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more) { struct mlx5_wq_cyc *wq = &sq->wq; struct mlx5_wqe_ctrl_seg *cseg; @@ -360,7 +361,7 @@ netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, } stats->bytes += num_bytes; - stats->xmit_more += skb->xmit_more; + stats->xmit_more += netdev_xmit_more(); headlen = skb->len - ihs - skb->data_len; ds_cnt += !!headlen; @@ -392,6 +393,10 @@ netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, eseg = &wqe->eth; dseg = wqe->data; +#if IS_ENABLED(CONFIG_GENEVE) + if (skb->encapsulation) + mlx5e_tx_tunnel_accel(skb, eseg); +#endif mlx5e_txwqe_build_eseg_csum(sq, skb, eseg); eseg->mss = mss; @@ -419,7 +424,7 @@ netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, goto err_drop; mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt, num_wqebbs, num_bytes, - num_dma, wi, cseg); + num_dma, wi, cseg, xmit_more); return NETDEV_TX_OK; @@ -445,7 +450,7 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) if (unlikely(!skb)) return NETDEV_TX_OK; - return mlx5e_sq_xmit(sq, skb, wqe, pi); + return mlx5e_sq_xmit(sq, skb, wqe, pi, netdev_xmit_more()); } static void mlx5e_dump_error_cqe(struct mlx5e_txqsq *sq, @@ -655,7 +660,7 @@ netdev_tx_t mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, } stats->bytes += num_bytes; - stats->xmit_more += skb->xmit_more; + stats->xmit_more += netdev_xmit_more(); headlen = skb->len - ihs - skb->data_len; ds_cnt += !!headlen; @@ -700,7 +705,7 @@ netdev_tx_t mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, goto err_drop; mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt, num_wqebbs, num_bytes, - num_dma, wi, cseg); + num_dma, wi, cseg, false); return NETDEV_TX_OK; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index b4af5e19f6ac..f9862bf75491 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -71,6 +71,17 @@ static void mlx5e_handle_rx_dim(struct mlx5e_rq *rq) net_dim(&rq->dim, dim_sample); } +void mlx5e_trigger_irq(struct mlx5e_icosq *sq) +{ + struct mlx5_wq_cyc *wq = &sq->wq; + struct mlx5e_tx_wqe *nopwqe; + u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); + + sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP; + nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc); + mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl); +} + int mlx5e_napi_poll(struct napi_struct *napi, int budget) { struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index bb6e5b5d9681..23883d1fa22f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -291,6 +291,9 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, const char *name, mlx5_fill_page_array(&eq->buf, pas); MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); + if (!param->mask && MLX5_CAP_GEN(dev, log_max_uctx)) + MLX5_SET(create_eq_in, in, uid, MLX5_SHARED_RESOURCE_UID); + MLX5_SET64(create_eq_in, in, event_bitmask, param->mask); eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); @@ -504,8 +507,7 @@ static u64 gather_async_events_mask(struct mlx5_core_dev *dev) if (MLX5_VPORT_MANAGER(dev)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE); - if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH && - MLX5_CAP_GEN(dev, general_notification_event)) + if (MLX5_CAP_GEN(dev, general_notification_event)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_GENERAL_EVENT); if (MLX5_CAP_GEN(dev, port_module_event)) @@ -707,7 +709,7 @@ void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm) __raw_writel((__force u32)cpu_to_be32(val), addr); /* We still want ordering, just not swabbing, so add a barrier */ - mb(); + wmb(); } EXPORT_SYMBOL(mlx5_eq_update_ci); @@ -900,14 +902,12 @@ mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector) } EXPORT_SYMBOL(mlx5_comp_irq_get_affinity_mask); +#ifdef CONFIG_RFS_ACCEL struct cpu_rmap *mlx5_eq_table_get_rmap(struct mlx5_core_dev *dev) { -#ifdef CONFIG_RFS_ACCEL return dev->priv.eq_table->rmap; -#else - return NULL; -#endif } +#endif struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 8a67fd197b79..9ea0ccfe5ef5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -72,25 +72,22 @@ static void esw_cleanup_vepa_rules(struct mlx5_eswitch *esw); MC_ADDR_CHANGE | \ PROMISC_CHANGE) -/* The vport getter/iterator are only valid after esw->total_vports - * and vport->vport are initialized in mlx5_eswitch_init. - */ -#define mlx5_esw_for_all_vports(esw, i, vport) \ - for ((i) = MLX5_VPORT_PF; \ - (vport) = &(esw)->vports[i], \ - (i) < (esw)->total_vports; (i)++) +struct mlx5_vport *__must_check +mlx5_eswitch_get_vport(struct mlx5_eswitch *esw, u16 vport_num) +{ + u16 idx; -#define mlx5_esw_for_each_vf_vport(esw, i, vport, nvfs) \ - for ((i) = MLX5_VPORT_FIRST_VF; \ - (vport) = &(esw)->vports[i], \ - (i) <= (nvfs); (i)++) + if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager)) + return ERR_PTR(-EPERM); -static struct mlx5_vport *mlx5_eswitch_get_vport(struct mlx5_eswitch *esw, - u16 vport_num) -{ - u16 idx = mlx5_eswitch_vport_num_to_index(esw, vport_num); + idx = mlx5_eswitch_vport_num_to_index(esw, vport_num); + + if (idx > esw->total_vports - 1) { + esw_debug(esw->dev, "vport out of range: num(0x%x), idx(0x%x)\n", + vport_num, idx); + return ERR_PTR(-EINVAL); + } - WARN_ON(idx > esw->total_vports - 1); return &esw->vports[idx]; } @@ -644,9 +641,8 @@ static int esw_del_mc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr) /* Apply vport UC/MC list to HW l2 table and FDB table */ static void esw_apply_vport_addr_list(struct mlx5_eswitch *esw, - u16 vport_num, int list_type) + struct mlx5_vport *vport, int list_type) { - struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); bool is_uc = list_type == MLX5_NVPRT_LIST_TYPE_UC; vport_addr_action vport_addr_add; vport_addr_action vport_addr_del; @@ -679,9 +675,8 @@ static void esw_apply_vport_addr_list(struct mlx5_eswitch *esw, /* Sync vport UC/MC list from vport context */ static void esw_update_vport_addr_list(struct mlx5_eswitch *esw, - u16 vport_num, int list_type) + struct mlx5_vport *vport, int list_type) { - struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); bool is_uc = list_type == MLX5_NVPRT_LIST_TYPE_UC; u8 (*mac_list)[ETH_ALEN]; struct l2addr_node *node; @@ -710,12 +705,12 @@ static void esw_update_vport_addr_list(struct mlx5_eswitch *esw, if (!vport->enabled) goto out; - err = mlx5_query_nic_vport_mac_list(esw->dev, vport_num, list_type, + err = mlx5_query_nic_vport_mac_list(esw->dev, vport->vport, list_type, mac_list, &size); if (err) goto out; esw_debug(esw->dev, "vport[%d] context update %s list size (%d)\n", - vport_num, is_uc ? "UC" : "MC", size); + vport->vport, is_uc ? "UC" : "MC", size); for (i = 0; i < size; i++) { if (is_uc && !is_valid_ether_addr(mac_list[i])) @@ -753,10 +748,10 @@ static void esw_update_vport_addr_list(struct mlx5_eswitch *esw, if (!addr) { esw_warn(esw->dev, "Failed to add MAC(%pM) to vport[%d] DB\n", - mac_list[i], vport_num); + mac_list[i], vport->vport); continue; } - addr->vport = vport_num; + addr->vport = vport->vport; addr->action = MLX5_ACTION_ADD; } out: @@ -766,9 +761,9 @@ out: /* Sync vport UC/MC list from vport context * Must be called after esw_update_vport_addr_list */ -static void esw_update_vport_mc_promisc(struct mlx5_eswitch *esw, u16 vport_num) +static void esw_update_vport_mc_promisc(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { - struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); struct l2addr_node *node; struct vport_addr *addr; struct hlist_head *hash; @@ -791,20 +786,20 @@ static void esw_update_vport_mc_promisc(struct mlx5_eswitch *esw, u16 vport_num) if (!addr) { esw_warn(esw->dev, "Failed to add allmulti MAC(%pM) to vport[%d] DB\n", - mac, vport_num); + mac, vport->vport); continue; } - addr->vport = vport_num; + addr->vport = vport->vport; addr->action = MLX5_ACTION_ADD; addr->mc_promisc = true; } } /* Apply vport rx mode to HW FDB table */ -static void esw_apply_vport_rx_mode(struct mlx5_eswitch *esw, u16 vport_num, +static void esw_apply_vport_rx_mode(struct mlx5_eswitch *esw, + struct mlx5_vport *vport, bool promisc, bool mc_promisc) { - struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); struct esw_mc_addr *allmulti_addr = &esw->mc_promisc; if (IS_ERR_OR_NULL(vport->allmulti_rule) != mc_promisc) @@ -812,7 +807,7 @@ static void esw_apply_vport_rx_mode(struct mlx5_eswitch *esw, u16 vport_num, if (mc_promisc) { vport->allmulti_rule = - esw_fdb_set_vport_allmulti_rule(esw, vport_num); + esw_fdb_set_vport_allmulti_rule(esw, vport->vport); if (!allmulti_addr->uplink_rule) allmulti_addr->uplink_rule = esw_fdb_set_vport_allmulti_rule(esw, @@ -835,8 +830,8 @@ promisc: return; if (promisc) { - vport->promisc_rule = esw_fdb_set_vport_promisc_rule(esw, - vport_num); + vport->promisc_rule = + esw_fdb_set_vport_promisc_rule(esw, vport->vport); } else if (vport->promisc_rule) { mlx5_del_flow_rules(vport->promisc_rule); vport->promisc_rule = NULL; @@ -844,23 +839,23 @@ promisc: } /* Sync vport rx mode from vport context */ -static void esw_update_vport_rx_mode(struct mlx5_eswitch *esw, u16 vport_num) +static void esw_update_vport_rx_mode(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { - struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); int promisc_all = 0; int promisc_uc = 0; int promisc_mc = 0; int err; err = mlx5_query_nic_vport_promisc(esw->dev, - vport_num, + vport->vport, &promisc_uc, &promisc_mc, &promisc_all); if (err) return; esw_debug(esw->dev, "vport[%d] context update rx mode promisc_all=%d, all_multi=%d\n", - vport_num, promisc_all, promisc_mc); + vport->vport, promisc_all, promisc_mc); if (!vport->info.trusted || !vport->enabled) { promisc_uc = 0; @@ -868,7 +863,7 @@ static void esw_update_vport_rx_mode(struct mlx5_eswitch *esw, u16 vport_num) promisc_all = 0; } - esw_apply_vport_rx_mode(esw, vport_num, promisc_all, + esw_apply_vport_rx_mode(esw, vport, promisc_all, (promisc_all || promisc_mc)); } @@ -883,27 +878,21 @@ static void esw_vport_change_handle_locked(struct mlx5_vport *vport) vport->vport, mac); if (vport->enabled_events & UC_ADDR_CHANGE) { - esw_update_vport_addr_list(esw, vport->vport, - MLX5_NVPRT_LIST_TYPE_UC); - esw_apply_vport_addr_list(esw, vport->vport, - MLX5_NVPRT_LIST_TYPE_UC); + esw_update_vport_addr_list(esw, vport, MLX5_NVPRT_LIST_TYPE_UC); + esw_apply_vport_addr_list(esw, vport, MLX5_NVPRT_LIST_TYPE_UC); } - if (vport->enabled_events & MC_ADDR_CHANGE) { - esw_update_vport_addr_list(esw, vport->vport, - MLX5_NVPRT_LIST_TYPE_MC); - } + if (vport->enabled_events & MC_ADDR_CHANGE) + esw_update_vport_addr_list(esw, vport, MLX5_NVPRT_LIST_TYPE_MC); if (vport->enabled_events & PROMISC_CHANGE) { - esw_update_vport_rx_mode(esw, vport->vport); + esw_update_vport_rx_mode(esw, vport); if (!IS_ERR_OR_NULL(vport->allmulti_rule)) - esw_update_vport_mc_promisc(esw, vport->vport); + esw_update_vport_mc_promisc(esw, vport); } - if (vport->enabled_events & (PROMISC_CHANGE | MC_ADDR_CHANGE)) { - esw_apply_vport_addr_list(esw, vport->vport, - MLX5_NVPRT_LIST_TYPE_MC); - } + if (vport->enabled_events & (PROMISC_CHANGE | MC_ADDR_CHANGE)) + esw_apply_vport_addr_list(esw, vport, MLX5_NVPRT_LIST_TYPE_MC); esw_debug(esw->dev, "vport[%d] Context Changed: Done\n", vport->vport); if (vport->enabled) @@ -922,8 +911,8 @@ static void esw_vport_change_handler(struct work_struct *work) mutex_unlock(&esw->state_lock); } -static int esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, - struct mlx5_vport *vport) +int esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_group *vlan_grp = NULL; @@ -1006,8 +995,8 @@ out: return err; } -static void esw_vport_cleanup_egress_rules(struct mlx5_eswitch *esw, - struct mlx5_vport *vport) +void esw_vport_cleanup_egress_rules(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { if (!IS_ERR_OR_NULL(vport->egress.allowed_vlan)) mlx5_del_flow_rules(vport->egress.allowed_vlan); @@ -1019,8 +1008,8 @@ static void esw_vport_cleanup_egress_rules(struct mlx5_eswitch *esw, vport->egress.drop_rule = NULL; } -static void esw_vport_disable_egress_acl(struct mlx5_eswitch *esw, - struct mlx5_vport *vport) +void esw_vport_disable_egress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { if (IS_ERR_OR_NULL(vport->egress.acl)) return; @@ -1036,8 +1025,8 @@ static void esw_vport_disable_egress_acl(struct mlx5_eswitch *esw, vport->egress.acl = NULL; } -static int esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, - struct mlx5_vport *vport) +int esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_core_dev *dev = esw->dev; @@ -1168,8 +1157,8 @@ out: return err; } -static void esw_vport_cleanup_ingress_rules(struct mlx5_eswitch *esw, - struct mlx5_vport *vport) +void esw_vport_cleanup_ingress_rules(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { if (!IS_ERR_OR_NULL(vport->ingress.drop_rule)) mlx5_del_flow_rules(vport->ingress.drop_rule); @@ -1181,8 +1170,8 @@ static void esw_vport_cleanup_ingress_rules(struct mlx5_eswitch *esw, vport->ingress.allow_rule = NULL; } -static void esw_vport_disable_ingress_acl(struct mlx5_eswitch *esw, - struct mlx5_vport *vport) +void esw_vport_disable_ingress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { if (IS_ERR_OR_NULL(vport->ingress.acl)) return; @@ -1420,10 +1409,10 @@ static void esw_destroy_tsar(struct mlx5_eswitch *esw) esw->qos.enabled = false; } -static int esw_vport_enable_qos(struct mlx5_eswitch *esw, int vport_num, +static int esw_vport_enable_qos(struct mlx5_eswitch *esw, + struct mlx5_vport *vport, u32 initial_max_rate, u32 initial_bw_share) { - struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {0}; struct mlx5_core_dev *dev = esw->dev; void *vport_elem; @@ -1440,7 +1429,7 @@ static int esw_vport_enable_qos(struct mlx5_eswitch *esw, int vport_num, SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT); vport_elem = MLX5_ADDR_OF(scheduling_context, sched_ctx, element_attributes); - MLX5_SET(vport_element, vport_elem, vport_number, vport_num); + MLX5_SET(vport_element, vport_elem, vport_number, vport->vport); MLX5_SET(scheduling_context, sched_ctx, parent_element_id, esw->qos.root_tsar_id); MLX5_SET(scheduling_context, sched_ctx, max_average_bw, @@ -1453,7 +1442,7 @@ static int esw_vport_enable_qos(struct mlx5_eswitch *esw, int vport_num, &vport->qos.esw_tsar_ix); if (err) { esw_warn(esw->dev, "E-Switch create TSAR vport element failed (vport=%d,err=%d)\n", - vport_num, err); + vport->vport, err); return err; } @@ -1461,10 +1450,10 @@ static int esw_vport_enable_qos(struct mlx5_eswitch *esw, int vport_num, return 0; } -static void esw_vport_disable_qos(struct mlx5_eswitch *esw, int vport_num) +static void esw_vport_disable_qos(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { - struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); - int err = 0; + int err; if (!vport->qos.enabled) return; @@ -1474,15 +1463,15 @@ static void esw_vport_disable_qos(struct mlx5_eswitch *esw, int vport_num) vport->qos.esw_tsar_ix); if (err) esw_warn(esw->dev, "E-Switch destroy TSAR vport element failed (vport=%d,err=%d)\n", - vport_num, err); + vport->vport, err); vport->qos.enabled = false; } -static int esw_vport_qos_config(struct mlx5_eswitch *esw, int vport_num, +static int esw_vport_qos_config(struct mlx5_eswitch *esw, + struct mlx5_vport *vport, u32 max_rate, u32 bw_share) { - struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {0}; struct mlx5_core_dev *dev = esw->dev; void *vport_elem; @@ -1499,7 +1488,7 @@ static int esw_vport_qos_config(struct mlx5_eswitch *esw, int vport_num, SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT); vport_elem = MLX5_ADDR_OF(scheduling_context, sched_ctx, element_attributes); - MLX5_SET(vport_element, vport_elem, vport_number, vport_num); + MLX5_SET(vport_element, vport_elem, vport_number, vport->vport); MLX5_SET(scheduling_context, sched_ctx, parent_element_id, esw->qos.root_tsar_id); MLX5_SET(scheduling_context, sched_ctx, max_average_bw, @@ -1515,7 +1504,7 @@ static int esw_vport_qos_config(struct mlx5_eswitch *esw, int vport_num, bitmask); if (err) { esw_warn(esw->dev, "E-Switch modify TSAR vport element failed (vport=%d,err=%d)\n", - vport_num, err); + vport->vport, err); return err; } @@ -1618,7 +1607,7 @@ static void esw_enable_vport(struct mlx5_eswitch *esw, struct mlx5_vport *vport, esw_apply_vport_conf(esw, vport); /* Attach vport to the eswitch rate limiter */ - if (esw_vport_enable_qos(esw, vport_num, vport->info.max_rate, + if (esw_vport_enable_qos(esw, vport, vport->info.max_rate, vport->qos.bw_share)) esw_warn(esw->dev, "Failed to attach vport %d to eswitch rate limiter", vport_num); @@ -1663,7 +1652,7 @@ static void esw_disable_vport(struct mlx5_eswitch *esw, */ esw_vport_change_handle_locked(vport); vport->enabled_events = 0; - esw_vport_disable_qos(esw, vport_num); + esw_vport_disable_qos(esw, vport); if (esw->manager_vport != vport_num && esw->mode == SRIOV_LEGACY) { mlx5_modify_vport_admin_state(esw->dev, @@ -1688,6 +1677,9 @@ static int eswitch_vport_event(struct notifier_block *nb, vport_num = be16_to_cpu(eqe->data.vport_change.vport_num); vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) + return NOTIFY_OK; + if (vport->enabled) queue_work(esw->work_queue, &vport->vport_change_handler); @@ -1922,22 +1914,19 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) } /* Vport Administration */ -#define LEGAL_VPORT(esw, vport) (vport >= 0 && vport < esw->total_vports) - int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, int vport, u8 mac[ETH_ALEN]) { - struct mlx5_vport *evport; + struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); u64 node_guid; int err = 0; - if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager)) - return -EPERM; - if (!LEGAL_VPORT(esw, vport) || is_multicast_ether_addr(mac)) + if (IS_ERR(evport)) + return PTR_ERR(evport); + if (is_multicast_ether_addr(mac)) return -EINVAL; mutex_lock(&esw->state_lock); - evport = &esw->vports[vport]; if (evport->info.spoofchk && !is_valid_ether_addr(mac)) mlx5_core_warn(esw->dev, @@ -1972,16 +1961,15 @@ unlock: int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, int vport, int link_state) { - struct mlx5_vport *evport; + struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); int err = 0; if (!ESW_ALLOWED(esw)) return -EPERM; - if (!LEGAL_VPORT(esw, vport)) - return -EINVAL; + if (IS_ERR(evport)) + return PTR_ERR(evport); mutex_lock(&esw->state_lock); - evport = &esw->vports[vport]; err = mlx5_modify_vport_admin_state(esw->dev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, @@ -2003,14 +1991,10 @@ unlock: int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw, int vport, struct ifla_vf_info *ivi) { - struct mlx5_vport *evport; - - if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager)) - return -EPERM; - if (!LEGAL_VPORT(esw, vport)) - return -EINVAL; + struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); - evport = &esw->vports[vport]; + if (IS_ERR(evport)) + return PTR_ERR(evport); memset(ivi, 0, sizeof(*ivi)); ivi->vf = vport - 1; @@ -2032,16 +2016,17 @@ int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw, int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, int vport, u16 vlan, u8 qos, u8 set_flags) { - struct mlx5_vport *evport; + struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); int err = 0; if (!ESW_ALLOWED(esw)) return -EPERM; - if (!LEGAL_VPORT(esw, vport) || (vlan > 4095) || (qos > 7)) + if (IS_ERR(evport)) + return PTR_ERR(evport); + if (vlan > 4095 || qos > 7) return -EINVAL; mutex_lock(&esw->state_lock); - evport = &esw->vports[vport]; err = modify_esw_vport_cvlan(esw->dev, vport, vlan, qos, set_flags); if (err) @@ -2075,17 +2060,16 @@ int mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, int mlx5_eswitch_set_vport_spoofchk(struct mlx5_eswitch *esw, int vport, bool spoofchk) { - struct mlx5_vport *evport; + struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); bool pschk; int err = 0; if (!ESW_ALLOWED(esw)) return -EPERM; - if (!LEGAL_VPORT(esw, vport)) - return -EINVAL; + if (IS_ERR(evport)) + return PTR_ERR(evport); mutex_lock(&esw->state_lock); - evport = &esw->vports[vport]; pschk = evport->info.spoofchk; evport->info.spoofchk = spoofchk; if (pschk && !is_valid_ether_addr(evport->info.mac)) @@ -2226,15 +2210,14 @@ out: int mlx5_eswitch_set_vport_trust(struct mlx5_eswitch *esw, int vport, bool setting) { - struct mlx5_vport *evport; + struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); if (!ESW_ALLOWED(esw)) return -EPERM; - if (!LEGAL_VPORT(esw, vport)) - return -EINVAL; + if (IS_ERR(evport)) + return PTR_ERR(evport); mutex_lock(&esw->state_lock); - evport = &esw->vports[vport]; evport->info.trusted = setting; if (evport->enabled) esw_vport_change_handle_locked(evport); @@ -2284,7 +2267,7 @@ static int normalize_vports_min_rate(struct mlx5_eswitch *esw, u32 divider) if (bw_share == evport->qos.bw_share) continue; - err = esw_vport_qos_config(esw, evport->vport, vport_max_rate, + err = esw_vport_qos_config(esw, evport, vport_max_rate, bw_share); if (!err) evport->qos.bw_share = bw_share; @@ -2298,7 +2281,7 @@ static int normalize_vports_min_rate(struct mlx5_eswitch *esw, u32 divider) int mlx5_eswitch_set_vport_rate(struct mlx5_eswitch *esw, int vport, u32 max_rate, u32 min_rate) { - struct mlx5_vport *evport; + struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); u32 fw_max_bw_share; u32 previous_min_rate; u32 divider; @@ -2308,8 +2291,8 @@ int mlx5_eswitch_set_vport_rate(struct mlx5_eswitch *esw, int vport, if (!ESW_ALLOWED(esw)) return -EPERM; - if (!LEGAL_VPORT(esw, vport)) - return -EINVAL; + if (IS_ERR(evport)) + return PTR_ERR(evport); fw_max_bw_share = MLX5_CAP_QOS(esw->dev, max_tsar_bw_share); min_rate_supported = MLX5_CAP_QOS(esw->dev, esw_bw_share) && @@ -2320,7 +2303,6 @@ int mlx5_eswitch_set_vport_rate(struct mlx5_eswitch *esw, int vport, return -EOPNOTSUPP; mutex_lock(&esw->state_lock); - evport = &esw->vports[vport]; if (min_rate == evport->info.min_rate) goto set_max_rate; @@ -2338,7 +2320,7 @@ set_max_rate: if (max_rate == evport->info.max_rate) goto unlock; - err = esw_vport_qos_config(esw, vport, max_rate, evport->qos.bw_share); + err = esw_vport_qos_config(esw, evport, max_rate, evport->qos.bw_share); if (!err) evport->info.max_rate = max_rate; @@ -2348,11 +2330,10 @@ unlock: } static int mlx5_eswitch_query_vport_drop_stats(struct mlx5_core_dev *dev, - int vport_idx, + struct mlx5_vport *vport, struct mlx5_vport_drop_stats *stats) { struct mlx5_eswitch *esw = dev->priv.eswitch; - struct mlx5_vport *vport = &esw->vports[vport_idx]; u64 rx_discard_vport_down, tx_discard_vport_down; u64 bytes = 0; int err = 0; @@ -2372,7 +2353,7 @@ static int mlx5_eswitch_query_vport_drop_stats(struct mlx5_core_dev *dev, !MLX5_CAP_GEN(dev, transmit_discard_vport_down)) return 0; - err = mlx5_query_vport_down_stats(dev, vport_idx, 1, + err = mlx5_query_vport_down_stats(dev, vport->vport, 1, &rx_discard_vport_down, &tx_discard_vport_down); if (err) @@ -2387,19 +2368,18 @@ static int mlx5_eswitch_query_vport_drop_stats(struct mlx5_core_dev *dev, } int mlx5_eswitch_get_vport_stats(struct mlx5_eswitch *esw, - int vport, + int vport_num, struct ifla_vf_stats *vf_stats) { + struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out); u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0}; struct mlx5_vport_drop_stats stats = {0}; int err = 0; u32 *out; - if (!ESW_ALLOWED(esw)) - return -EPERM; - if (!LEGAL_VPORT(esw, vport)) - return -EINVAL; + if (IS_ERR(vport)) + return PTR_ERR(vport); out = kvzalloc(outlen, GFP_KERNEL); if (!out) @@ -2408,7 +2388,7 @@ int mlx5_eswitch_get_vport_stats(struct mlx5_eswitch *esw, MLX5_SET(query_vport_counter_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_COUNTER); MLX5_SET(query_vport_counter_in, in, op_mod, 0); - MLX5_SET(query_vport_counter_in, in, vport_number, vport); + MLX5_SET(query_vport_counter_in, in, vport_number, vport->vport); MLX5_SET(query_vport_counter_in, in, other_vport, 1); memset(out, 0, outlen); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 3f3cd32ae60a..ed3fad689ec9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -227,6 +227,18 @@ int esw_offloads_init(struct mlx5_eswitch *esw, int vf_nvports, int total_nvports); void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw); int esw_offloads_init_reps(struct mlx5_eswitch *esw); +void esw_vport_cleanup_ingress_rules(struct mlx5_eswitch *esw, + struct mlx5_vport *vport); +int esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport); +void esw_vport_cleanup_egress_rules(struct mlx5_eswitch *esw, + struct mlx5_vport *vport); +int esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport); +void esw_vport_disable_egress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport); +void esw_vport_disable_ingress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport); /* E-Switch API */ int mlx5_eswitch_init(struct mlx5_core_dev *dev); @@ -376,11 +388,11 @@ bool mlx5_esw_multipath_prereq(struct mlx5_core_dev *dev0, #define MLX5_DEBUG_ESWITCH_MASK BIT(3) -#define esw_info(dev, format, ...) \ - pr_info("(%s): E-Switch: " format, (dev)->priv.name, ##__VA_ARGS__) +#define esw_info(__dev, format, ...) \ + dev_info((__dev)->device, "E-Switch: " format, ##__VA_ARGS__) -#define esw_warn(dev, format, ...) \ - pr_warn("(%s): E-Switch: " format, (dev)->priv.name, ##__VA_ARGS__) +#define esw_warn(__dev, format, ...) \ + dev_warn((__dev)->device, "E-Switch: " format, ##__VA_ARGS__) #define esw_debug(dev, format, ...) \ mlx5_core_dbg_mask(dev, MLX5_DEBUG_ESWITCH_MASK, format, ##__VA_ARGS__) @@ -431,6 +443,54 @@ static inline int mlx5_eswitch_index_to_vport_num(struct mlx5_eswitch *esw, return index; } +/* TODO: This mlx5e_tc function shouldn't be called by eswitch */ +void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw); + +/* The vport getter/iterator are only valid after esw->total_vports + * and vport->vport are initialized in mlx5_eswitch_init. + */ +#define mlx5_esw_for_all_vports(esw, i, vport) \ + for ((i) = MLX5_VPORT_PF; \ + (vport) = &(esw)->vports[i], \ + (i) < (esw)->total_vports; (i)++) + +#define mlx5_esw_for_each_vf_vport(esw, i, vport, nvfs) \ + for ((i) = MLX5_VPORT_FIRST_VF; \ + (vport) = &(esw)->vports[(i)], \ + (i) <= (nvfs); (i)++) + +#define mlx5_esw_for_each_vf_vport_reverse(esw, i, vport, nvfs) \ + for ((i) = (nvfs); \ + (vport) = &(esw)->vports[(i)], \ + (i) >= MLX5_VPORT_FIRST_VF; (i)--) + +/* The rep getter/iterator are only valid after esw->total_vports + * and vport->vport are initialized in mlx5_eswitch_init. + */ +#define mlx5_esw_for_all_reps(esw, i, rep) \ + for ((i) = MLX5_VPORT_PF; \ + (rep) = &(esw)->offloads.vport_reps[i], \ + (i) < (esw)->total_vports; (i)++) + +#define mlx5_esw_for_each_vf_rep(esw, i, rep, nvfs) \ + for ((i) = MLX5_VPORT_FIRST_VF; \ + (rep) = &(esw)->offloads.vport_reps[i], \ + (i) <= (nvfs); (i)++) + +#define mlx5_esw_for_each_vf_rep_reverse(esw, i, rep, nvfs) \ + for ((i) = (nvfs); \ + (rep) = &(esw)->offloads.vport_reps[i], \ + (i) >= MLX5_VPORT_FIRST_VF; (i)--) + +#define mlx5_esw_for_each_vf_vport_num(esw, vport, nvfs) \ + for ((vport) = MLX5_VPORT_FIRST_VF; (vport) <= (nvfs); (vport)++) + +#define mlx5_esw_for_each_vf_vport_num_reverse(esw, vport, nvfs) \ + for ((vport) = (nvfs); (vport) >= MLX5_VPORT_FIRST_VF; (vport)--) + +struct mlx5_vport *__must_check +mlx5_eswitch_get_vport(struct mlx5_eswitch *esw, u16 vport_num); + #else /* CONFIG_MLX5_ESWITCH */ /* eswitch API stubs */ static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 9b2d78ee22b8..e09ae27485ee 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -37,17 +37,13 @@ #include <linux/mlx5/fs.h> #include "mlx5_core.h" #include "eswitch.h" +#include "rdma.h" #include "en.h" #include "fs_core.h" #include "lib/devcom.h" #include "ecpf.h" #include "lib/eq.h" -enum { - FDB_FAST_PATH = 0, - FDB_SLOW_PATH -}; - /* There are two match-all miss flows, one for unicast dst mac and * one for multicast. */ @@ -58,32 +54,6 @@ enum { #define UPLINK_REP_INDEX 0 -/* The rep getter/iterator are only valid after esw->total_vports - * and vport->vport are initialized in mlx5_eswitch_init. - */ -#define mlx5_esw_for_all_reps(esw, i, rep) \ - for ((i) = MLX5_VPORT_PF; \ - (rep) = &(esw)->offloads.vport_reps[i], \ - (i) < (esw)->total_vports; (i)++) - -#define mlx5_esw_for_each_vf_rep(esw, i, rep, nvfs) \ - for ((i) = MLX5_VPORT_FIRST_VF; \ - (rep) = &(esw)->offloads.vport_reps[i], \ - (i) <= (nvfs); (i)++) - -#define mlx5_esw_for_each_vf_rep_reverse(esw, i, rep, nvfs) \ - for ((i) = (nvfs); \ - (rep) = &(esw)->offloads.vport_reps[i], \ - (i) >= MLX5_VPORT_FIRST_VF; (i)--) - -#define mlx5_esw_for_each_vf_vport(esw, vport, nvfs) \ - for ((vport) = MLX5_VPORT_FIRST_VF; \ - (vport) <= (nvfs); (vport)++) - -#define mlx5_esw_for_each_vf_vport_reverse(esw, vport, nvfs) \ - for ((vport) = (nvfs); \ - (vport) >= MLX5_VPORT_FIRST_VF; (vport)--) - static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, u16 vport_num) { @@ -363,7 +333,7 @@ static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val) esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none"); for (vf_vport = 1; vf_vport < esw->enabled_vports; vf_vport++) { rep = &esw->offloads.vport_reps[vf_vport]; - if (rep->rep_if[REP_ETH].state != REP_LOADED) + if (atomic_read(&rep->rep_if[REP_ETH].state) != REP_LOADED) continue; err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val); @@ -663,7 +633,7 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, flows[mlx5_eswitch_ecpf_idx(esw)] = flow; } - mlx5_esw_for_each_vf_vport(esw, i, mlx5_core_max_vfs(esw->dev)) { + mlx5_esw_for_each_vf_vport_num(esw, i, mlx5_core_max_vfs(esw->dev)) { MLX5_SET(fte_match_set_misc, misc, source_port, i); flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, spec, &flow_act, &dest, 1); @@ -681,7 +651,7 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, add_vf_flow_err: nvports = --i; - mlx5_esw_for_each_vf_vport_reverse(esw, i, nvports) + mlx5_esw_for_each_vf_vport_num_reverse(esw, i, nvports) mlx5_del_flow_rules(flows[i]); if (mlx5_ecpf_vport_exists(esw->dev)) @@ -704,7 +674,8 @@ static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw) flows = esw->fdb_table.offloads.peer_miss_rules; - mlx5_esw_for_each_vf_vport_reverse(esw, i, mlx5_core_max_vfs(esw->dev)) + mlx5_esw_for_each_vf_vport_num_reverse(esw, i, + mlx5_core_max_vfs(esw->dev)) mlx5_del_flow_rules(flows[i]); if (mlx5_ecpf_vport_exists(esw->dev)) @@ -1287,13 +1258,13 @@ void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) int esw_offloads_init_reps(struct mlx5_eswitch *esw) { - int total_vfs = MLX5_TOTAL_VPORTS(esw->dev); + int total_vports = MLX5_TOTAL_VPORTS(esw->dev); struct mlx5_core_dev *dev = esw->dev; struct mlx5_eswitch_rep *rep; u8 hw_id[ETH_ALEN], rep_type; int vport; - esw->offloads.vport_reps = kcalloc(total_vfs, + esw->offloads.vport_reps = kcalloc(total_vports, sizeof(struct mlx5_eswitch_rep), GFP_KERNEL); if (!esw->offloads.vport_reps) @@ -1306,7 +1277,8 @@ int esw_offloads_init_reps(struct mlx5_eswitch *esw) ether_addr_copy(rep->hw_id, hw_id); for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) - rep->rep_if[rep_type].state = REP_UNREGISTERED; + atomic_set(&rep->rep_if[rep_type].state, + REP_UNREGISTERED); } return 0; @@ -1315,11 +1287,9 @@ int esw_offloads_init_reps(struct mlx5_eswitch *esw) static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep, u8 rep_type) { - if (rep->rep_if[rep_type].state != REP_LOADED) - return; - - rep->rep_if[rep_type].unload(rep); - rep->rep_if[rep_type].state = REP_REGISTERED; + if (atomic_cmpxchg(&rep->rep_if[rep_type].state, + REP_LOADED, REP_REGISTERED) == REP_LOADED) + rep->rep_if[rep_type].unload(rep); } static void __unload_reps_special_vport(struct mlx5_eswitch *esw, u8 rep_type) @@ -1380,16 +1350,15 @@ static int __esw_offloads_load_rep(struct mlx5_eswitch *esw, { int err = 0; - if (rep->rep_if[rep_type].state != REP_REGISTERED) - return 0; - - err = rep->rep_if[rep_type].load(esw->dev, rep); - if (err) - return err; - - rep->rep_if[rep_type].state = REP_LOADED; + if (atomic_cmpxchg(&rep->rep_if[rep_type].state, + REP_REGISTERED, REP_LOADED) == REP_REGISTERED) { + err = rep->rep_if[rep_type].load(esw->dev, rep); + if (err) + atomic_set(&rep->rep_if[rep_type].state, + REP_REGISTERED); + } - return 0; + return err; } static int __load_reps_special_vport(struct mlx5_eswitch *esw, u8 rep_type) @@ -1523,8 +1492,6 @@ static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, return 0; } -void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw); - static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw) { mlx5e_tc_clean_fdb_peer_flows(esw); @@ -1607,6 +1574,169 @@ static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS); } +static int esw_vport_ingress_prio_tag_config(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) +{ + struct mlx5_core_dev *dev = esw->dev; + struct mlx5_flow_act flow_act = {0}; + struct mlx5_flow_spec *spec; + int err = 0; + + /* For prio tag mode, there is only 1 FTEs: + * 1) Untagged packets - push prio tag VLAN, allow + * Unmatched traffic is allowed by default + */ + + if (!MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support)) + return -EOPNOTSUPP; + + esw_vport_cleanup_ingress_rules(esw, vport); + + err = esw_vport_enable_ingress_acl(esw, vport); + if (err) { + mlx5_core_warn(esw->dev, + "failed to enable prio tag ingress acl (%d) on vport[%d]\n", + err, vport->vport); + return err; + } + + esw_debug(esw->dev, + "vport[%d] configure ingress rules\n", vport->vport); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto out_no_mem; + } + + /* Untagged packets - push prio tag VLAN, allow */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.cvlan_tag, 0); + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH | + MLX5_FLOW_CONTEXT_ACTION_ALLOW; + flow_act.vlan[0].ethtype = ETH_P_8021Q; + flow_act.vlan[0].vid = 0; + flow_act.vlan[0].prio = 0; + vport->ingress.allow_rule = + mlx5_add_flow_rules(vport->ingress.acl, spec, + &flow_act, NULL, 0); + if (IS_ERR(vport->ingress.allow_rule)) { + err = PTR_ERR(vport->ingress.allow_rule); + esw_warn(esw->dev, + "vport[%d] configure ingress untagged allow rule, err(%d)\n", + vport->vport, err); + vport->ingress.allow_rule = NULL; + goto out; + } + +out: + kvfree(spec); +out_no_mem: + if (err) + esw_vport_cleanup_ingress_rules(esw, vport); + return err; +} + +static int esw_vport_egress_prio_tag_config(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) +{ + struct mlx5_flow_act flow_act = {0}; + struct mlx5_flow_spec *spec; + int err = 0; + + /* For prio tag mode, there is only 1 FTEs: + * 1) prio tag packets - pop the prio tag VLAN, allow + * Unmatched traffic is allowed by default + */ + + esw_vport_cleanup_egress_rules(esw, vport); + + err = esw_vport_enable_egress_acl(esw, vport); + if (err) { + mlx5_core_warn(esw->dev, + "failed to enable egress acl (%d) on vport[%d]\n", + err, vport->vport); + return err; + } + + esw_debug(esw->dev, + "vport[%d] configure prio tag egress rules\n", vport->vport); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto out_no_mem; + } + + /* prio tag vlan rule - pop it so VF receives untagged packets */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, 0); + + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_VLAN_POP | + MLX5_FLOW_CONTEXT_ACTION_ALLOW; + vport->egress.allowed_vlan = + mlx5_add_flow_rules(vport->egress.acl, spec, + &flow_act, NULL, 0); + if (IS_ERR(vport->egress.allowed_vlan)) { + err = PTR_ERR(vport->egress.allowed_vlan); + esw_warn(esw->dev, + "vport[%d] configure egress pop prio tag vlan rule failed, err(%d)\n", + vport->vport, err); + vport->egress.allowed_vlan = NULL; + goto out; + } + +out: + kvfree(spec); +out_no_mem: + if (err) + esw_vport_cleanup_egress_rules(esw, vport); + return err; +} + +static int esw_prio_tag_acls_config(struct mlx5_eswitch *esw, int nvports) +{ + struct mlx5_vport *vport = NULL; + int i, j; + int err; + + mlx5_esw_for_each_vf_vport(esw, i, vport, nvports) { + err = esw_vport_ingress_prio_tag_config(esw, vport); + if (err) + goto err_ingress; + err = esw_vport_egress_prio_tag_config(esw, vport); + if (err) + goto err_egress; + } + + return 0; + +err_egress: + esw_vport_disable_ingress_acl(esw, vport); +err_ingress: + mlx5_esw_for_each_vf_vport_reverse(esw, j, vport, i - 1) { + esw_vport_disable_egress_acl(esw, vport); + esw_vport_disable_ingress_acl(esw, vport); + } + + return err; +} + +static void esw_prio_tag_acls_cleanup(struct mlx5_eswitch *esw) +{ + struct mlx5_vport *vport; + int i; + + mlx5_esw_for_each_vf_vport(esw, i, vport, esw->nvports) { + esw_vport_disable_egress_acl(esw, vport); + esw_vport_disable_ingress_acl(esw, vport); + } +} + static int esw_offloads_steering_init(struct mlx5_eswitch *esw, int nvports) { int err; @@ -1614,6 +1744,12 @@ static int esw_offloads_steering_init(struct mlx5_eswitch *esw, int nvports) memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); mutex_init(&esw->fdb_table.offloads.fdb_prio_lock); + if (MLX5_CAP_GEN(esw->dev, prio_tag_required)) { + err = esw_prio_tag_acls_config(esw, nvports); + if (err) + return err; + } + err = esw_create_offloads_fdb_tables(esw, nvports); if (err) return err; @@ -1642,6 +1778,8 @@ static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) esw_destroy_vport_rx_group(esw); esw_destroy_offloads_table(esw); esw_destroy_offloads_fdb_tables(esw); + if (MLX5_CAP_GEN(esw->dev, prio_tag_required)) + esw_prio_tag_acls_cleanup(esw); } static void esw_host_params_event_handler(struct work_struct *work) @@ -1700,8 +1838,6 @@ int esw_offloads_init(struct mlx5_eswitch *esw, int vf_nvports, { int err; - mutex_init(&esw->fdb_table.offloads.fdb_prio_lock); - err = esw_offloads_steering_init(esw, total_nvports); if (err) return err; @@ -1719,6 +1855,8 @@ int esw_offloads_init(struct mlx5_eswitch *esw, int vf_nvports, esw->host_info.num_vfs = vf_nvports; } + mlx5_rdma_enable_roce(esw->dev); + return 0; err_reps: @@ -1757,6 +1895,7 @@ void esw_offloads_cleanup(struct mlx5_eswitch *esw) num_vfs = esw->dev->priv.sriov.num_vfs; } + mlx5_rdma_disable_roce(esw->dev); esw_offloads_devcom_cleanup(esw); esw_offloads_unload_all_reps(esw, num_vfs); esw_offloads_steering_cleanup(esw); @@ -2076,7 +2215,7 @@ void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, rep_if->get_proto_dev = __rep_if->get_proto_dev; rep_if->priv = __rep_if->priv; - rep_if->state = REP_REGISTERED; + atomic_set(&rep_if->state, REP_REGISTERED); } } EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); @@ -2091,7 +2230,7 @@ void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) __unload_reps_all_vport(esw, max_vf, rep_type); mlx5_esw_for_all_reps(esw, i, rep) - rep->rep_if[rep_type].state = REP_UNREGISTERED; + atomic_set(&rep->rep_if[rep_type].state, REP_UNREGISTERED); } EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); @@ -2111,7 +2250,7 @@ void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, rep = mlx5_eswitch_get_rep(esw, vport); - if (rep->rep_if[rep_type].state == REP_LOADED && + if (atomic_read(&rep->rep_if[rep_type].state) == REP_LOADED && rep->rep_if[rep_type].get_proto_dev) return rep->rep_if[rep_type].get_proto_dev(rep); return NULL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c index 5d5864e8df3c..a81e8d2168d8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/events.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c @@ -21,6 +21,7 @@ struct mlx5_event_nb { static int any_notifier(struct notifier_block *, unsigned long, void *); static int temp_warn(struct notifier_block *, unsigned long, void *); static int port_module(struct notifier_block *, unsigned long, void *); +static int pcie_core(struct notifier_block *, unsigned long, void *); /* handler which forwards the event to events->nh, driver notifiers */ static int forward_event(struct notifier_block *, unsigned long, void *); @@ -30,6 +31,7 @@ static struct mlx5_nb events_nbs_ref[] = { {.nb.notifier_call = any_notifier, .event_type = MLX5_EVENT_TYPE_NOTIFY_ANY }, {.nb.notifier_call = temp_warn, .event_type = MLX5_EVENT_TYPE_TEMP_WARN_EVENT }, {.nb.notifier_call = port_module, .event_type = MLX5_EVENT_TYPE_PORT_MODULE_EVENT }, + {.nb.notifier_call = pcie_core, .event_type = MLX5_EVENT_TYPE_GENERAL_EVENT }, /* Events to be forwarded (as is) to mlx5 core interfaces (mlx5e/mlx5_ib) */ {.nb.notifier_call = forward_event, .event_type = MLX5_EVENT_TYPE_PORT_CHANGE }, @@ -51,11 +53,14 @@ static struct mlx5_nb events_nbs_ref[] = { struct mlx5_events { struct mlx5_core_dev *dev; + struct workqueue_struct *wq; struct mlx5_event_nb notifiers[ARRAY_SIZE(events_nbs_ref)]; /* driver notifier chain */ struct atomic_notifier_head nh; /* port module events stats */ struct mlx5_pme_stats pme_stats; + /*pcie_core*/ + struct work_struct pcie_core_work; }; static const char *eqe_type_str(u8 type) @@ -249,6 +254,69 @@ static int port_module(struct notifier_block *nb, unsigned long type, void *data return NOTIFY_OK; } +enum { + MLX5_PCI_POWER_COULD_NOT_BE_READ = 0x0, + MLX5_PCI_POWER_SUFFICIENT_REPORTED = 0x1, + MLX5_PCI_POWER_INSUFFICIENT_REPORTED = 0x2, +}; + +static void mlx5_pcie_event(struct work_struct *work) +{ + u32 out[MLX5_ST_SZ_DW(mpein_reg)] = {0}; + u32 in[MLX5_ST_SZ_DW(mpein_reg)] = {0}; + struct mlx5_events *events; + struct mlx5_core_dev *dev; + u8 power_status; + u16 pci_power; + + events = container_of(work, struct mlx5_events, pcie_core_work); + dev = events->dev; + + if (!MLX5_CAP_MCAM_FEATURE(dev, pci_status_and_power)) + return; + + mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), + MLX5_REG_MPEIN, 0, 0); + power_status = MLX5_GET(mpein_reg, out, pwr_status); + pci_power = MLX5_GET(mpein_reg, out, pci_power); + + switch (power_status) { + case MLX5_PCI_POWER_COULD_NOT_BE_READ: + mlx5_core_info_rl(dev, + "PCIe slot power capability was not advertised.\n"); + break; + case MLX5_PCI_POWER_INSUFFICIENT_REPORTED: + mlx5_core_warn_rl(dev, + "Detected insufficient power on the PCIe slot (%uW).\n", + pci_power); + break; + case MLX5_PCI_POWER_SUFFICIENT_REPORTED: + mlx5_core_info_rl(dev, + "PCIe slot advertised sufficient power (%uW).\n", + pci_power); + break; + } +} + +static int pcie_core(struct notifier_block *nb, unsigned long type, void *data) +{ + struct mlx5_event_nb *event_nb = mlx5_nb_cof(nb, + struct mlx5_event_nb, + nb); + struct mlx5_events *events = event_nb->ctx; + struct mlx5_eqe *eqe = data; + + switch (eqe->sub_type) { + case MLX5_GENERAL_SUBTYPE_PCI_POWER_CHANGE_EVENT: + queue_work(events->wq, &events->pcie_core_work); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + void mlx5_get_pme_stats(struct mlx5_core_dev *dev, struct mlx5_pme_stats *stats) { *stats = dev->priv.events->pme_stats; @@ -277,11 +345,17 @@ int mlx5_events_init(struct mlx5_core_dev *dev) ATOMIC_INIT_NOTIFIER_HEAD(&events->nh); events->dev = dev; dev->priv.events = events; + events->wq = create_singlethread_workqueue("mlx5_events"); + if (!events->wq) + return -ENOMEM; + INIT_WORK(&events->pcie_core_work, mlx5_pcie_event); + return 0; } void mlx5_events_cleanup(struct mlx5_core_dev *dev) { + destroy_workqueue(dev->priv.events->wq); kvfree(dev->priv.events); } @@ -304,6 +378,7 @@ void mlx5_events_stop(struct mlx5_core_dev *dev) for (i = ARRAY_SIZE(events_nbs_ref) - 1; i >= 0 ; i--) mlx5_eq_notifier_unregister(dev, &events->notifiers[i].nb); + flush_workqueue(events->wq); } int mlx5_notifier_register(struct mlx5_core_dev *dev, struct notifier_block *nb) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c index 873541ef4c1b..ca2296a2f9ee 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c @@ -135,7 +135,7 @@ static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe) *conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc); /* Make sure that doorbell record is visible before ringing */ wmb(); - mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET, NULL); + mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET); } static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.h b/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.h index 7e2e871dbf83..52c9dee91ea4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.h @@ -37,6 +37,7 @@ #include <linux/mlx5/eq.h> +#include "mlx5_core.h" #include "lib/eq.h" #include "fpga/cmd.h" @@ -62,26 +63,26 @@ struct mlx5_fpga_device { }; #define mlx5_fpga_dbg(__adev, format, ...) \ - dev_dbg(&(__adev)->mdev->pdev->dev, "FPGA: %s:%d:(pid %d): " format, \ - __func__, __LINE__, current->pid, ##__VA_ARGS__) + mlx5_core_dbg((__adev)->mdev, "FPGA: %s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, ##__VA_ARGS__) #define mlx5_fpga_err(__adev, format, ...) \ - dev_err(&(__adev)->mdev->pdev->dev, "FPGA: %s:%d:(pid %d): " format, \ - __func__, __LINE__, current->pid, ##__VA_ARGS__) + mlx5_core_err((__adev)->mdev, "FPGA: %s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, ##__VA_ARGS__) #define mlx5_fpga_warn(__adev, format, ...) \ - dev_warn(&(__adev)->mdev->pdev->dev, "FPGA: %s:%d:(pid %d): " format, \ - __func__, __LINE__, current->pid, ##__VA_ARGS__) + mlx5_core_warn((__adev)->mdev, "FPGA: %s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, ##__VA_ARGS__) #define mlx5_fpga_warn_ratelimited(__adev, format, ...) \ - dev_warn_ratelimited(&(__adev)->mdev->pdev->dev, "FPGA: %s:%d: " \ - format, __func__, __LINE__, ##__VA_ARGS__) + mlx5_core_err_rl((__adev)->mdev, "FPGA: %s:%d: " \ + format, __func__, __LINE__, ##__VA_ARGS__) #define mlx5_fpga_notice(__adev, format, ...) \ - dev_notice(&(__adev)->mdev->pdev->dev, "FPGA: " format, ##__VA_ARGS__) + mlx5_core_info((__adev)->mdev, "FPGA: " format, ##__VA_ARGS__) #define mlx5_fpga_info(__adev, format, ...) \ - dev_info(&(__adev)->mdev->pdev->dev, "FPGA: " format, ##__VA_ARGS__) + mlx5_core_info((__adev)->mdev, "FPGA: " format, ##__VA_ARGS__) int mlx5_fpga_init(struct mlx5_core_dev *mdev); void mlx5_fpga_cleanup(struct mlx5_core_dev *mdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c index 5a22c5874f3b..52c47d3dd5a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c @@ -989,32 +989,33 @@ static enum fs_flow_table_type egress_to_fs_ft(bool egress) return egress ? FS_FT_NIC_TX : FS_FT_NIC_RX; } -static int fpga_ipsec_fs_create_flow_group(struct mlx5_core_dev *dev, +static int fpga_ipsec_fs_create_flow_group(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 *in, - unsigned int *group_id, + struct mlx5_flow_group *fg, bool is_egress) { - int (*create_flow_group)(struct mlx5_core_dev *dev, + int (*create_flow_group)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 *in, - unsigned int *group_id) = + struct mlx5_flow_group *fg) = mlx5_fs_cmd_get_default(egress_to_fs_ft(is_egress))->create_flow_group; char *misc_params_c = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria.misc_parameters); + struct mlx5_core_dev *dev = ns->dev; u32 saved_outer_esp_spi_mask; u8 match_criteria_enable; int ret; if (MLX5_CAP_FLOWTABLE(dev, flow_table_properties_nic_receive.ft_field_support.outer_esp_spi)) - return create_flow_group(dev, ft, in, group_id); + return create_flow_group(ns, ft, in, fg); match_criteria_enable = MLX5_GET(create_flow_group_in, in, match_criteria_enable); saved_outer_esp_spi_mask = MLX5_GET(fte_match_set_misc, misc_params_c, outer_esp_spi); if (!match_criteria_enable || !saved_outer_esp_spi_mask) - return create_flow_group(dev, ft, in, group_id); + return create_flow_group(ns, ft, in, fg); MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, 0); @@ -1023,7 +1024,7 @@ static int fpga_ipsec_fs_create_flow_group(struct mlx5_core_dev *dev, MLX5_SET(create_flow_group_in, in, match_criteria_enable, match_criteria_enable & ~MLX5_MATCH_MISC_PARAMETERS); - ret = create_flow_group(dev, ft, in, group_id); + ret = create_flow_group(ns, ft, in, fg); MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, saved_outer_esp_spi_mask); MLX5_SET(create_flow_group_in, in, match_criteria_enable, match_criteria_enable); @@ -1031,17 +1032,18 @@ static int fpga_ipsec_fs_create_flow_group(struct mlx5_core_dev *dev, return ret; } -static int fpga_ipsec_fs_create_fte(struct mlx5_core_dev *dev, +static int fpga_ipsec_fs_create_fte(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_group *fg, struct fs_fte *fte, bool is_egress) { - int (*create_fte)(struct mlx5_core_dev *dev, + int (*create_fte)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_group *fg, struct fs_fte *fte) = mlx5_fs_cmd_get_default(egress_to_fs_ft(is_egress))->create_fte; + struct mlx5_core_dev *dev = ns->dev; struct mlx5_fpga_device *fdev = dev->fpga; struct mlx5_fpga_ipsec *fipsec = fdev->ipsec; struct mlx5_fpga_ipsec_rule *rule; @@ -1053,7 +1055,7 @@ static int fpga_ipsec_fs_create_fte(struct mlx5_core_dev *dev, !(fte->action.action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | MLX5_FLOW_CONTEXT_ACTION_DECRYPT))) - return create_fte(dev, ft, fg, fte); + return create_fte(ns, ft, fg, fte); rule = kzalloc(sizeof(*rule), GFP_KERNEL); if (!rule) @@ -1070,7 +1072,7 @@ static int fpga_ipsec_fs_create_fte(struct mlx5_core_dev *dev, WARN_ON(rule_insert(fipsec, rule)); modify_spec_mailbox(dev, fte, &mbox_mod); - ret = create_fte(dev, ft, fg, fte); + ret = create_fte(ns, ft, fg, fte); restore_spec_mailbox(fte, &mbox_mod); if (ret) { _rule_delete(fipsec, rule); @@ -1081,19 +1083,20 @@ static int fpga_ipsec_fs_create_fte(struct mlx5_core_dev *dev, return ret; } -static int fpga_ipsec_fs_update_fte(struct mlx5_core_dev *dev, +static int fpga_ipsec_fs_update_fte(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id, + struct mlx5_flow_group *fg, int modify_mask, struct fs_fte *fte, bool is_egress) { - int (*update_fte)(struct mlx5_core_dev *dev, + int (*update_fte)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id, + struct mlx5_flow_group *fg, int modify_mask, struct fs_fte *fte) = mlx5_fs_cmd_get_default(egress_to_fs_ft(is_egress))->update_fte; + struct mlx5_core_dev *dev = ns->dev; bool is_esp = fte->action.esp_id; struct mailbox_mod mbox_mod; int ret; @@ -1102,24 +1105,25 @@ static int fpga_ipsec_fs_update_fte(struct mlx5_core_dev *dev, !(fte->action.action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | MLX5_FLOW_CONTEXT_ACTION_DECRYPT))) - return update_fte(dev, ft, group_id, modify_mask, fte); + return update_fte(ns, ft, fg, modify_mask, fte); modify_spec_mailbox(dev, fte, &mbox_mod); - ret = update_fte(dev, ft, group_id, modify_mask, fte); + ret = update_fte(ns, ft, fg, modify_mask, fte); restore_spec_mailbox(fte, &mbox_mod); return ret; } -static int fpga_ipsec_fs_delete_fte(struct mlx5_core_dev *dev, +static int fpga_ipsec_fs_delete_fte(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct fs_fte *fte, bool is_egress) { - int (*delete_fte)(struct mlx5_core_dev *dev, + int (*delete_fte)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct fs_fte *fte) = mlx5_fs_cmd_get_default(egress_to_fs_ft(is_egress))->delete_fte; + struct mlx5_core_dev *dev = ns->dev; struct mlx5_fpga_device *fdev = dev->fpga; struct mlx5_fpga_ipsec *fipsec = fdev->ipsec; struct mlx5_fpga_ipsec_rule *rule; @@ -1131,7 +1135,7 @@ static int fpga_ipsec_fs_delete_fte(struct mlx5_core_dev *dev, !(fte->action.action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | MLX5_FLOW_CONTEXT_ACTION_DECRYPT))) - return delete_fte(dev, ft, fte); + return delete_fte(ns, ft, fte); rule = rule_search(fipsec, fte); if (!rule) @@ -1141,84 +1145,84 @@ static int fpga_ipsec_fs_delete_fte(struct mlx5_core_dev *dev, rule_delete(fipsec, rule); modify_spec_mailbox(dev, fte, &mbox_mod); - ret = delete_fte(dev, ft, fte); + ret = delete_fte(ns, ft, fte); restore_spec_mailbox(fte, &mbox_mod); return ret; } static int -mlx5_fpga_ipsec_fs_create_flow_group_egress(struct mlx5_core_dev *dev, +mlx5_fpga_ipsec_fs_create_flow_group_egress(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 *in, - unsigned int *group_id) + struct mlx5_flow_group *fg) { - return fpga_ipsec_fs_create_flow_group(dev, ft, in, group_id, true); + return fpga_ipsec_fs_create_flow_group(ns, ft, in, fg, true); } static int -mlx5_fpga_ipsec_fs_create_fte_egress(struct mlx5_core_dev *dev, +mlx5_fpga_ipsec_fs_create_fte_egress(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_group *fg, struct fs_fte *fte) { - return fpga_ipsec_fs_create_fte(dev, ft, fg, fte, true); + return fpga_ipsec_fs_create_fte(ns, ft, fg, fte, true); } static int -mlx5_fpga_ipsec_fs_update_fte_egress(struct mlx5_core_dev *dev, +mlx5_fpga_ipsec_fs_update_fte_egress(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id, + struct mlx5_flow_group *fg, int modify_mask, struct fs_fte *fte) { - return fpga_ipsec_fs_update_fte(dev, ft, group_id, modify_mask, fte, + return fpga_ipsec_fs_update_fte(ns, ft, fg, modify_mask, fte, true); } static int -mlx5_fpga_ipsec_fs_delete_fte_egress(struct mlx5_core_dev *dev, +mlx5_fpga_ipsec_fs_delete_fte_egress(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct fs_fte *fte) { - return fpga_ipsec_fs_delete_fte(dev, ft, fte, true); + return fpga_ipsec_fs_delete_fte(ns, ft, fte, true); } static int -mlx5_fpga_ipsec_fs_create_flow_group_ingress(struct mlx5_core_dev *dev, +mlx5_fpga_ipsec_fs_create_flow_group_ingress(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 *in, - unsigned int *group_id) + struct mlx5_flow_group *fg) { - return fpga_ipsec_fs_create_flow_group(dev, ft, in, group_id, false); + return fpga_ipsec_fs_create_flow_group(ns, ft, in, fg, false); } static int -mlx5_fpga_ipsec_fs_create_fte_ingress(struct mlx5_core_dev *dev, +mlx5_fpga_ipsec_fs_create_fte_ingress(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_group *fg, struct fs_fte *fte) { - return fpga_ipsec_fs_create_fte(dev, ft, fg, fte, false); + return fpga_ipsec_fs_create_fte(ns, ft, fg, fte, false); } static int -mlx5_fpga_ipsec_fs_update_fte_ingress(struct mlx5_core_dev *dev, +mlx5_fpga_ipsec_fs_update_fte_ingress(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id, + struct mlx5_flow_group *fg, int modify_mask, struct fs_fte *fte) { - return fpga_ipsec_fs_update_fte(dev, ft, group_id, modify_mask, fte, + return fpga_ipsec_fs_update_fte(ns, ft, fg, modify_mask, fte, false); } static int -mlx5_fpga_ipsec_fs_delete_fte_ingress(struct mlx5_core_dev *dev, +mlx5_fpga_ipsec_fs_delete_fte_ingress(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct fs_fte *fte) { - return fpga_ipsec_fs_delete_fte(dev, ft, fte, false); + return fpga_ipsec_fs_delete_fte(ns, ft, fte, false); } static struct mlx5_flow_cmds fpga_ipsec_ingress; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c index 8de64e88c670..22a2ef111514 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c @@ -148,14 +148,16 @@ static int mlx5_fpga_tls_alloc_swid(struct idr *idr, spinlock_t *idr_spinlock, return ret; } -static void mlx5_fpga_tls_release_swid(struct idr *idr, - spinlock_t *idr_spinlock, u32 swid) +static void *mlx5_fpga_tls_release_swid(struct idr *idr, + spinlock_t *idr_spinlock, u32 swid) { unsigned long flags; + void *ptr; spin_lock_irqsave(idr_spinlock, flags); - idr_remove(idr, swid); + ptr = idr_remove(idr, swid); spin_unlock_irqrestore(idr_spinlock, flags); + return ptr; } static void mlx_tls_kfree_complete(struct mlx5_fpga_conn *conn, @@ -165,20 +167,12 @@ static void mlx_tls_kfree_complete(struct mlx5_fpga_conn *conn, kfree(buf); } -struct mlx5_teardown_stream_context { - struct mlx5_fpga_tls_command_context cmd; - u32 swid; -}; - static void mlx5_fpga_tls_teardown_completion(struct mlx5_fpga_conn *conn, struct mlx5_fpga_device *fdev, struct mlx5_fpga_tls_command_context *cmd, struct mlx5_fpga_dma_buf *resp) { - struct mlx5_teardown_stream_context *ctx = - container_of(cmd, struct mlx5_teardown_stream_context, cmd); - if (resp) { u32 syndrome = MLX5_GET(tls_resp, resp->sg[0].data, syndrome); @@ -186,14 +180,6 @@ mlx5_fpga_tls_teardown_completion(struct mlx5_fpga_conn *conn, mlx5_fpga_err(fdev, "Teardown stream failed with syndrome = %d", syndrome); - else if (MLX5_GET(tls_cmd, cmd->buf.sg[0].data, direction_sx)) - mlx5_fpga_tls_release_swid(&fdev->tls->tx_idr, - &fdev->tls->tx_idr_spinlock, - ctx->swid); - else - mlx5_fpga_tls_release_swid(&fdev->tls->rx_idr, - &fdev->tls->rx_idr_spinlock, - ctx->swid); } mlx5_fpga_tls_put_command_ctx(cmd); } @@ -217,22 +203,22 @@ int mlx5_fpga_tls_resync_rx(struct mlx5_core_dev *mdev, u32 handle, u32 seq, void *cmd; int ret; - rcu_read_lock(); - flow = idr_find(&mdev->fpga->tls->rx_idr, ntohl(handle)); - rcu_read_unlock(); - - if (!flow) { - WARN_ONCE(1, "Received NULL pointer for handle\n"); - return -EINVAL; - } - buf = kzalloc(size, GFP_ATOMIC); if (!buf) return -ENOMEM; cmd = (buf + 1); + rcu_read_lock(); + flow = idr_find(&mdev->fpga->tls->rx_idr, ntohl(handle)); + if (unlikely(!flow)) { + rcu_read_unlock(); + WARN_ONCE(1, "Received NULL pointer for handle\n"); + kfree(buf); + return -EINVAL; + } mlx5_fpga_tls_flow_to_cmd(flow, cmd); + rcu_read_unlock(); MLX5_SET(tls_cmd, cmd, swid, ntohl(handle)); MLX5_SET64(tls_cmd, cmd, tls_rcd_sn, be64_to_cpu(rcd_sn)); @@ -253,7 +239,7 @@ int mlx5_fpga_tls_resync_rx(struct mlx5_core_dev *mdev, u32 handle, u32 seq, static void mlx5_fpga_tls_send_teardown_cmd(struct mlx5_core_dev *mdev, void *flow, u32 swid, gfp_t flags) { - struct mlx5_teardown_stream_context *ctx; + struct mlx5_fpga_tls_command_context *ctx; struct mlx5_fpga_dma_buf *buf; void *cmd; @@ -261,7 +247,7 @@ static void mlx5_fpga_tls_send_teardown_cmd(struct mlx5_core_dev *mdev, if (!ctx) return; - buf = &ctx->cmd.buf; + buf = &ctx->buf; cmd = (ctx + 1); MLX5_SET(tls_cmd, cmd, command_type, CMD_TEARDOWN_STREAM); MLX5_SET(tls_cmd, cmd, swid, swid); @@ -272,8 +258,7 @@ static void mlx5_fpga_tls_send_teardown_cmd(struct mlx5_core_dev *mdev, buf->sg[0].data = cmd; buf->sg[0].size = MLX5_TLS_COMMAND_SIZE; - ctx->swid = swid; - mlx5_fpga_tls_cmd_send(mdev->fpga, &ctx->cmd, + mlx5_fpga_tls_cmd_send(mdev->fpga, ctx, mlx5_fpga_tls_teardown_completion); } @@ -283,13 +268,14 @@ void mlx5_fpga_tls_del_flow(struct mlx5_core_dev *mdev, u32 swid, struct mlx5_fpga_tls *tls = mdev->fpga->tls; void *flow; - rcu_read_lock(); if (direction_sx) - flow = idr_find(&tls->tx_idr, swid); + flow = mlx5_fpga_tls_release_swid(&tls->tx_idr, + &tls->tx_idr_spinlock, + swid); else - flow = idr_find(&tls->rx_idr, swid); - - rcu_read_unlock(); + flow = mlx5_fpga_tls_release_swid(&tls->rx_idr, + &tls->rx_idr_spinlock, + swid); if (!flow) { mlx5_fpga_err(mdev->fpga, "No flow information for swid %u\n", @@ -297,6 +283,7 @@ void mlx5_fpga_tls_del_flow(struct mlx5_core_dev *mdev, u32 swid, return; } + synchronize_rcu(); /* before kfree(flow) */ mlx5_fpga_tls_send_teardown_cmd(mdev, flow, swid, flags); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index c44ccb67c4a3..013b1ca4a791 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -39,7 +39,7 @@ #include "mlx5_core.h" #include "eswitch.h" -static int mlx5_cmd_stub_update_root_ft(struct mlx5_core_dev *dev, +static int mlx5_cmd_stub_update_root_ft(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 underlay_qpn, bool disconnect) @@ -47,47 +47,43 @@ static int mlx5_cmd_stub_update_root_ft(struct mlx5_core_dev *dev, return 0; } -static int mlx5_cmd_stub_create_flow_table(struct mlx5_core_dev *dev, - u16 vport, - enum fs_flow_table_op_mod op_mod, - enum fs_flow_table_type type, - unsigned int level, +static int mlx5_cmd_stub_create_flow_table(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, unsigned int log_size, - struct mlx5_flow_table *next_ft, - unsigned int *table_id, u32 flags) + struct mlx5_flow_table *next_ft) { return 0; } -static int mlx5_cmd_stub_destroy_flow_table(struct mlx5_core_dev *dev, +static int mlx5_cmd_stub_destroy_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft) { return 0; } -static int mlx5_cmd_stub_modify_flow_table(struct mlx5_core_dev *dev, +static int mlx5_cmd_stub_modify_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_table *next_ft) { return 0; } -static int mlx5_cmd_stub_create_flow_group(struct mlx5_core_dev *dev, +static int mlx5_cmd_stub_create_flow_group(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 *in, - unsigned int *group_id) + struct mlx5_flow_group *fg) { return 0; } -static int mlx5_cmd_stub_destroy_flow_group(struct mlx5_core_dev *dev, +static int mlx5_cmd_stub_destroy_flow_group(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id) + struct mlx5_flow_group *fg) { return 0; } -static int mlx5_cmd_stub_create_fte(struct mlx5_core_dev *dev, +static int mlx5_cmd_stub_create_fte(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_group *group, struct fs_fte *fte) @@ -95,28 +91,29 @@ static int mlx5_cmd_stub_create_fte(struct mlx5_core_dev *dev, return 0; } -static int mlx5_cmd_stub_update_fte(struct mlx5_core_dev *dev, +static int mlx5_cmd_stub_update_fte(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id, + struct mlx5_flow_group *group, int modify_mask, struct fs_fte *fte) { return -EOPNOTSUPP; } -static int mlx5_cmd_stub_delete_fte(struct mlx5_core_dev *dev, +static int mlx5_cmd_stub_delete_fte(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct fs_fte *fte) { return 0; } -static int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev, +static int mlx5_cmd_update_root_ft(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 underlay_qpn, bool disconnect) { u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {0}; u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {0}; + struct mlx5_core_dev *dev = ns->dev; if ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) && underlay_qpn == 0) @@ -143,29 +140,26 @@ static int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev, return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } -static int mlx5_cmd_create_flow_table(struct mlx5_core_dev *dev, - u16 vport, - enum fs_flow_table_op_mod op_mod, - enum fs_flow_table_type type, - unsigned int level, +static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, unsigned int log_size, - struct mlx5_flow_table *next_ft, - unsigned int *table_id, u32 flags) + struct mlx5_flow_table *next_ft) { - int en_encap = !!(flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT); - int en_decap = !!(flags & MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); + int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT); + int en_decap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {0}; u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {0}; + struct mlx5_core_dev *dev = ns->dev; int err; MLX5_SET(create_flow_table_in, in, opcode, MLX5_CMD_OP_CREATE_FLOW_TABLE); - MLX5_SET(create_flow_table_in, in, table_type, type); - MLX5_SET(create_flow_table_in, in, flow_table_context.level, level); + MLX5_SET(create_flow_table_in, in, table_type, ft->type); + MLX5_SET(create_flow_table_in, in, flow_table_context.level, ft->level); MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, log_size); - if (vport) { - MLX5_SET(create_flow_table_in, in, vport_number, vport); + if (ft->vport) { + MLX5_SET(create_flow_table_in, in, vport_number, ft->vport); MLX5_SET(create_flow_table_in, in, other_vport, 1); } @@ -174,13 +168,18 @@ static int mlx5_cmd_create_flow_table(struct mlx5_core_dev *dev, MLX5_SET(create_flow_table_in, in, flow_table_context.reformat_en, en_encap); - switch (op_mod) { + switch (ft->op_mod) { case FS_FT_OP_MOD_NORMAL: if (next_ft) { MLX5_SET(create_flow_table_in, in, - flow_table_context.table_miss_action, 1); + flow_table_context.table_miss_action, + MLX5_FLOW_TABLE_MISS_ACTION_FWD); MLX5_SET(create_flow_table_in, in, flow_table_context.table_miss_id, next_ft->id); + } else { + MLX5_SET(create_flow_table_in, in, + flow_table_context.table_miss_action, + ns->def_miss_action); } break; @@ -195,16 +194,17 @@ static int mlx5_cmd_create_flow_table(struct mlx5_core_dev *dev, err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); if (!err) - *table_id = MLX5_GET(create_flow_table_out, out, - table_id); + ft->id = MLX5_GET(create_flow_table_out, out, + table_id); return err; } -static int mlx5_cmd_destroy_flow_table(struct mlx5_core_dev *dev, +static int mlx5_cmd_destroy_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft) { u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {0}; u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)] = {0}; + struct mlx5_core_dev *dev = ns->dev; MLX5_SET(destroy_flow_table_in, in, opcode, MLX5_CMD_OP_DESTROY_FLOW_TABLE); @@ -218,12 +218,13 @@ static int mlx5_cmd_destroy_flow_table(struct mlx5_core_dev *dev, return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } -static int mlx5_cmd_modify_flow_table(struct mlx5_core_dev *dev, +static int mlx5_cmd_modify_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_table *next_ft) { u32 in[MLX5_ST_SZ_DW(modify_flow_table_in)] = {0}; u32 out[MLX5_ST_SZ_DW(modify_flow_table_out)] = {0}; + struct mlx5_core_dev *dev = ns->dev; MLX5_SET(modify_flow_table_in, in, opcode, MLX5_CMD_OP_MODIFY_FLOW_TABLE); @@ -250,26 +251,29 @@ static int mlx5_cmd_modify_flow_table(struct mlx5_core_dev *dev, MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID); if (next_ft) { MLX5_SET(modify_flow_table_in, in, - flow_table_context.table_miss_action, 1); + flow_table_context.table_miss_action, + MLX5_FLOW_TABLE_MISS_ACTION_FWD); MLX5_SET(modify_flow_table_in, in, flow_table_context.table_miss_id, next_ft->id); } else { MLX5_SET(modify_flow_table_in, in, - flow_table_context.table_miss_action, 0); + flow_table_context.table_miss_action, + ns->def_miss_action); } } return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); } -static int mlx5_cmd_create_flow_group(struct mlx5_core_dev *dev, +static int mlx5_cmd_create_flow_group(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 *in, - unsigned int *group_id) + struct mlx5_flow_group *fg) { u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {0}; int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_core_dev *dev = ns->dev; int err; MLX5_SET(create_flow_group_in, in, opcode, @@ -283,23 +287,24 @@ static int mlx5_cmd_create_flow_group(struct mlx5_core_dev *dev, err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); if (!err) - *group_id = MLX5_GET(create_flow_group_out, out, - group_id); + fg->id = MLX5_GET(create_flow_group_out, out, + group_id); return err; } -static int mlx5_cmd_destroy_flow_group(struct mlx5_core_dev *dev, +static int mlx5_cmd_destroy_flow_group(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id) + struct mlx5_flow_group *fg) { u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)] = {0}; u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)] = {0}; + struct mlx5_core_dev *dev = ns->dev; MLX5_SET(destroy_flow_group_in, in, opcode, MLX5_CMD_OP_DESTROY_FLOW_GROUP); MLX5_SET(destroy_flow_group_in, in, table_type, ft->type); MLX5_SET(destroy_flow_group_in, in, table_id, ft->id); - MLX5_SET(destroy_flow_group_in, in, group_id, group_id); + MLX5_SET(destroy_flow_group_in, in, group_id, fg->id); if (ft->vport) { MLX5_SET(destroy_flow_group_in, in, vport_number, ft->vport); MLX5_SET(destroy_flow_group_in, in, other_vport, 1); @@ -505,23 +510,25 @@ err_out: return err; } -static int mlx5_cmd_create_fte(struct mlx5_core_dev *dev, +static int mlx5_cmd_create_fte(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_group *group, struct fs_fte *fte) { + struct mlx5_core_dev *dev = ns->dev; unsigned int group_id = group->id; return mlx5_cmd_set_fte(dev, 0, 0, ft, group_id, fte); } -static int mlx5_cmd_update_fte(struct mlx5_core_dev *dev, +static int mlx5_cmd_update_fte(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id, + struct mlx5_flow_group *fg, int modify_mask, struct fs_fte *fte) { int opmod; + struct mlx5_core_dev *dev = ns->dev; int atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev, flow_table_properties_nic_receive. flow_modify_en); @@ -529,15 +536,16 @@ static int mlx5_cmd_update_fte(struct mlx5_core_dev *dev, return -EOPNOTSUPP; opmod = 1; - return mlx5_cmd_set_fte(dev, opmod, modify_mask, ft, group_id, fte); + return mlx5_cmd_set_fte(dev, opmod, modify_mask, ft, fg->id, fte); } -static int mlx5_cmd_delete_fte(struct mlx5_core_dev *dev, +static int mlx5_cmd_delete_fte(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct fs_fte *fte) { u32 out[MLX5_ST_SZ_DW(delete_fte_out)] = {0}; u32 in[MLX5_ST_SZ_DW(delete_fte_in)] = {0}; + struct mlx5_core_dev *dev = ns->dev; MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY); MLX5_SET(delete_fte_in, in, table_type, ft->type); @@ -853,6 +861,7 @@ const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type typ case FS_FT_SNIFFER_RX: case FS_FT_SNIFFER_TX: case FS_FT_NIC_TX: + case FS_FT_RDMA_RX: return mlx5_fs_cmd_get_fw_cmds(); default: return mlx5_fs_cmd_get_stub_cmds(); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h index 6228ba7bfa1a..e340f9af2f5a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h @@ -36,45 +36,42 @@ #include "fs_core.h" struct mlx5_flow_cmds { - int (*create_flow_table)(struct mlx5_core_dev *dev, - u16 vport, - enum fs_flow_table_op_mod op_mod, - enum fs_flow_table_type type, - unsigned int level, unsigned int log_size, - struct mlx5_flow_table *next_ft, - unsigned int *table_id, u32 flags); - int (*destroy_flow_table)(struct mlx5_core_dev *dev, + int (*create_flow_table)(struct mlx5_flow_root_namespace *ns, + struct mlx5_flow_table *ft, + unsigned int log_size, + struct mlx5_flow_table *next_ft); + int (*destroy_flow_table)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft); - int (*modify_flow_table)(struct mlx5_core_dev *dev, + int (*modify_flow_table)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_table *next_ft); - int (*create_flow_group)(struct mlx5_core_dev *dev, + int (*create_flow_group)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 *in, - unsigned int *group_id); + struct mlx5_flow_group *fg); - int (*destroy_flow_group)(struct mlx5_core_dev *dev, + int (*destroy_flow_group)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id); + struct mlx5_flow_group *fg); - int (*create_fte)(struct mlx5_core_dev *dev, + int (*create_fte)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct mlx5_flow_group *fg, struct fs_fte *fte); - int (*update_fte)(struct mlx5_core_dev *dev, + int (*update_fte)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int group_id, + struct mlx5_flow_group *fg, int modify_mask, struct fs_fte *fte); - int (*delete_fte)(struct mlx5_core_dev *dev, + int (*delete_fte)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, struct fs_fte *fte); - int (*update_root_ft)(struct mlx5_core_dev *dev, + int (*update_root_ft)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, u32 underlay_qpn, bool disconnect); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 0be3eb86dd84..fb5b61727ee7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -403,7 +403,7 @@ static void del_hw_flow_table(struct fs_node *node) trace_mlx5_fs_del_ft(ft); if (node->active) { - err = root->cmds->destroy_flow_table(dev, ft); + err = root->cmds->destroy_flow_table(root, ft); if (err) mlx5_core_warn(dev, "flow steering can't destroy ft\n"); } @@ -435,7 +435,7 @@ static void modify_fte(struct fs_fte *fte) dev = get_dev(&fte->node); root = find_root(&ft->node); - err = root->cmds->update_fte(dev, ft, fg->id, fte->modify_mask, fte); + err = root->cmds->update_fte(root, ft, fg, fte->modify_mask, fte); if (err) mlx5_core_warn(dev, "%s can't del rule fg id=%d fte_index=%d\n", @@ -492,7 +492,7 @@ static void del_hw_fte(struct fs_node *node) dev = get_dev(&ft->node); root = find_root(&ft->node); if (node->active) { - err = root->cmds->delete_fte(dev, ft, fte); + err = root->cmds->delete_fte(root, ft, fte); if (err) mlx5_core_warn(dev, "flow steering can't delete fte in index %d of flow group id %d\n", @@ -532,7 +532,7 @@ static void del_hw_flow_group(struct fs_node *node) trace_mlx5_fs_del_fg(fg); root = find_root(&ft->node); - if (fg->node.active && root->cmds->destroy_flow_group(dev, ft, fg->id)) + if (fg->node.active && root->cmds->destroy_flow_group(root, ft, fg)) mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n", fg->id, ft->id); } @@ -783,7 +783,7 @@ static int connect_fts_in_prio(struct mlx5_core_dev *dev, fs_for_each_ft(iter, prio) { i++; - err = root->cmds->modify_flow_table(dev, iter, ft); + err = root->cmds->modify_flow_table(root, iter, ft); if (err) { mlx5_core_warn(dev, "Failed to modify flow table %d\n", iter->id); @@ -819,7 +819,7 @@ static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio struct mlx5_flow_root_namespace *root = find_root(&prio->node); struct mlx5_ft_underlay_qp *uqp; int min_level = INT_MAX; - int err; + int err = 0; u32 qpn; if (root->root_ft) @@ -831,11 +831,11 @@ static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio if (list_empty(&root->underlay_qpns)) { /* Don't set any QPN (zero) in case QPN list is empty */ qpn = 0; - err = root->cmds->update_root_ft(root->dev, ft, qpn, false); + err = root->cmds->update_root_ft(root, ft, qpn, false); } else { list_for_each_entry(uqp, &root->underlay_qpns, list) { qpn = uqp->qpn; - err = root->cmds->update_root_ft(root->dev, ft, + err = root->cmds->update_root_ft(root, ft, qpn, false); if (err) break; @@ -871,7 +871,7 @@ static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule, memcpy(&rule->dest_attr, dest, sizeof(*dest)); root = find_root(&ft->node); - err = root->cmds->update_fte(get_dev(&ft->node), ft, fg->id, + err = root->cmds->update_fte(root, ft, fg, modify_mask, fte); up_write_ref_node(&fte->node, false); @@ -1013,9 +1013,7 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa tree_init_node(&ft->node, del_hw_flow_table, del_sw_flow_table); log_table_sz = ft->max_fte ? ilog2(ft->max_fte) : 0; next_ft = find_next_chained_ft(fs_prio); - err = root->cmds->create_flow_table(root->dev, ft->vport, ft->op_mod, - ft->type, ft->level, log_table_sz, - next_ft, &ft->id, ft->flags); + err = root->cmds->create_flow_table(root, ft, log_table_sz, next_ft); if (err) goto free_ft; @@ -1032,7 +1030,7 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa trace_mlx5_fs_add_ft(ft); return ft; destroy_ft: - root->cmds->destroy_flow_table(root->dev, ft); + root->cmds->destroy_flow_table(root, ft); free_ft: kfree(ft); unlock_root: @@ -1114,7 +1112,6 @@ struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, start_flow_index); int end_index = MLX5_GET(create_flow_group_in, fg_in, end_flow_index); - struct mlx5_core_dev *dev = get_dev(&ft->node); struct mlx5_flow_group *fg; int err; @@ -1129,7 +1126,7 @@ struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, if (IS_ERR(fg)) return fg; - err = root->cmds->create_flow_group(dev, ft, fg_in, &fg->id); + err = root->cmds->create_flow_group(root, ft, fg_in, fg); if (err) { tree_put_node(&fg->node, false); return ERR_PTR(err); @@ -1269,11 +1266,9 @@ add_rule_fte(struct fs_fte *fte, fs_get_obj(ft, fg->node.parent); root = find_root(&fg->node); if (!(fte->status & FS_FTE_STATUS_EXISTING)) - err = root->cmds->create_fte(get_dev(&ft->node), - ft, fg, fte); + err = root->cmds->create_fte(root, ft, fg, fte); else - err = root->cmds->update_fte(get_dev(&ft->node), ft, fg->id, - modify_mask, fte); + err = root->cmds->update_fte(root, ft, fg, modify_mask, fte); if (err) goto free_handle; @@ -1339,7 +1334,6 @@ static int create_auto_flow_group(struct mlx5_flow_table *ft, struct mlx5_flow_group *fg) { struct mlx5_flow_root_namespace *root = find_root(&ft->node); - struct mlx5_core_dev *dev = get_dev(&ft->node); int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); void *match_criteria_addr; u8 src_esw_owner_mask_on; @@ -1369,7 +1363,7 @@ static int create_auto_flow_group(struct mlx5_flow_table *ft, memcpy(match_criteria_addr, fg->mask.match_criteria, sizeof(fg->mask.match_criteria)); - err = root->cmds->create_flow_group(dev, ft, in, &fg->id); + err = root->cmds->create_flow_group(root, ft, in, fg); if (!err) { fg->node.active = true; trace_mlx5_fs_add_fg(fg); @@ -1941,12 +1935,12 @@ static int update_root_ft_destroy(struct mlx5_flow_table *ft) if (list_empty(&root->underlay_qpns)) { /* Don't set any QPN (zero) in case QPN list is empty */ qpn = 0; - err = root->cmds->update_root_ft(root->dev, new_root_ft, + err = root->cmds->update_root_ft(root, new_root_ft, qpn, false); } else { list_for_each_entry(uqp, &root->underlay_qpns, list) { qpn = uqp->qpn; - err = root->cmds->update_root_ft(root->dev, + err = root->cmds->update_root_ft(root, new_root_ft, qpn, false); if (err) @@ -2060,6 +2054,10 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, if (steering->sniffer_tx_root_ns) return &steering->sniffer_tx_root_ns->ns; return NULL; + case MLX5_FLOW_NAMESPACE_RDMA_RX: + if (steering->rdma_rx_root_ns) + return &steering->rdma_rx_root_ns->ns; + return NULL; default: break; } @@ -2456,6 +2454,7 @@ void mlx5_cleanup_fs(struct mlx5_core_dev *dev) steering->fdb_sub_ns = NULL; cleanup_root_ns(steering->sniffer_rx_root_ns); cleanup_root_ns(steering->sniffer_tx_root_ns); + cleanup_root_ns(steering->rdma_rx_root_ns); cleanup_root_ns(steering->egress_root_ns); mlx5_cleanup_fc_stats(dev); kmem_cache_destroy(steering->ftes_cache); @@ -2497,6 +2496,25 @@ static int init_sniffer_rx_root_ns(struct mlx5_flow_steering *steering) return 0; } +static int init_rdma_rx_root_ns(struct mlx5_flow_steering *steering) +{ + struct fs_prio *prio; + + steering->rdma_rx_root_ns = create_root_ns(steering, FS_FT_RDMA_RX); + if (!steering->rdma_rx_root_ns) + return -ENOMEM; + + steering->rdma_rx_root_ns->def_miss_action = + MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN; + + /* Create single prio */ + prio = fs_create_prio(&steering->rdma_rx_root_ns->ns, 0, 1); + if (IS_ERR(prio)) { + cleanup_root_ns(steering->rdma_rx_root_ns); + return PTR_ERR(prio); + } + return 0; +} static int init_fdb_root_ns(struct mlx5_flow_steering *steering) { struct mlx5_flow_namespace *ns; @@ -2516,8 +2534,16 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) if (!steering->fdb_sub_ns) return -ENOMEM; + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BYPASS_PATH, + 1); + if (IS_ERR(maj_prio)) { + err = PTR_ERR(maj_prio); + goto out_err; + } + levels = 2 * FDB_MAX_PRIO * (FDB_MAX_CHAIN + 1); - maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns, 0, + maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns, + FDB_FAST_PATH, levels); if (IS_ERR(maj_prio)) { err = PTR_ERR(maj_prio); @@ -2542,7 +2568,7 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) steering->fdb_sub_ns[chain] = ns; } - maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, 1, 1); + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_SLOW_PATH, 1); if (IS_ERR(maj_prio)) { err = PTR_ERR(maj_prio); goto out_err; @@ -2725,6 +2751,13 @@ int mlx5_init_fs(struct mlx5_core_dev *dev) goto err; } + if (MLX5_CAP_FLOWTABLE_RDMA_RX(dev, ft_support) && + MLX5_CAP_FLOWTABLE_RDMA_RX(dev, table_miss_action_domain)) { + err = init_rdma_rx_root_ns(steering); + if (err) + goto err; + } + if (MLX5_IPSEC_DEV(dev) || MLX5_CAP_FLOWTABLE_NIC_TX(dev, ft_support)) { err = init_egress_root_ns(steering); if (err) @@ -2754,7 +2787,7 @@ int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn) goto update_ft_fail; } - err = root->cmds->update_root_ft(dev, root->root_ft, underlay_qpn, + err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn, false); if (err) { mlx5_core_warn(dev, "Failed adding underlay QPN (%u) to root FT err(%d)\n", @@ -2798,7 +2831,7 @@ int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn) goto out; } - err = root->cmds->update_root_ft(dev, root->root_ft, underlay_qpn, + err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn, true); if (err) mlx5_core_warn(dev, "Failed removing underlay QPN (%u) from root FT err(%d)\n", diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h index 87de0e4d9124..a08c3d09a50f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h @@ -67,6 +67,7 @@ enum fs_flow_table_type { FS_FT_FDB = 0X4, FS_FT_SNIFFER_RX = 0X5, FS_FT_SNIFFER_TX = 0X6, + FS_FT_RDMA_RX = 0X7, FS_FT_MAX_TYPE = FS_FT_SNIFFER_TX, }; @@ -90,6 +91,7 @@ struct mlx5_flow_steering { struct mlx5_flow_root_namespace **esw_ingress_root_ns; struct mlx5_flow_root_namespace *sniffer_tx_root_ns; struct mlx5_flow_root_namespace *sniffer_rx_root_ns; + struct mlx5_flow_root_namespace *rdma_rx_root_ns; struct mlx5_flow_root_namespace *egress_root_ns; }; @@ -150,7 +152,7 @@ struct mlx5_ft_underlay_qp { u32 qpn; }; -#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_800 +#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_a00 /* Calculate the fte_match_param length and without the reserved length. * Make sure the reserved field is the last. */ @@ -216,6 +218,7 @@ struct mlx5_flow_root_namespace { struct mutex chain_lock; struct list_head underlay_qpns; const struct mlx5_flow_cmds *cmds; + enum mlx5_flow_table_miss_action def_miss_action; }; int mlx5_init_fc_stats(struct mlx5_core_dev *dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index cb9fa3430c53..a2656f4008d9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -152,11 +152,11 @@ static void health_recover(struct work_struct *work) nic_state = mlx5_get_nic_state(dev); if (nic_state == MLX5_NIC_IFC_INVALID) { - dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n"); + mlx5_core_err(dev, "health recovery flow aborted since the nic state is invalid\n"); return; } - dev_err(&dev->pdev->dev, "starting health recovery flow\n"); + mlx5_core_err(dev, "starting health recovery flow\n"); mlx5_recover_device(dev); } @@ -180,8 +180,8 @@ static void health_care(struct work_struct *work) if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags)) schedule_delayed_work(&health->recover_work, recover_delay); else - dev_err(&dev->pdev->dev, - "new health works are not permitted at this stage\n"); + mlx5_core_err(dev, + "new health works are not permitted at this stage\n"); spin_unlock_irqrestore(&health->wq_lock, flags); } @@ -228,18 +228,22 @@ static void print_health_info(struct mlx5_core_dev *dev) return; for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) - dev_err(&dev->pdev->dev, "assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i)); + mlx5_core_err(dev, "assert_var[%d] 0x%08x\n", i, + ioread32be(h->assert_var + i)); - dev_err(&dev->pdev->dev, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr)); - dev_err(&dev->pdev->dev, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra)); + mlx5_core_err(dev, "assert_exit_ptr 0x%08x\n", + ioread32be(&h->assert_exit_ptr)); + mlx5_core_err(dev, "assert_callra 0x%08x\n", + ioread32be(&h->assert_callra)); sprintf(fw_str, "%d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); - dev_err(&dev->pdev->dev, "fw_ver %s\n", fw_str); - dev_err(&dev->pdev->dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id)); - dev_err(&dev->pdev->dev, "irisc_index %d\n", ioread8(&h->irisc_index)); - dev_err(&dev->pdev->dev, "synd 0x%x: %s\n", ioread8(&h->synd), hsynd_str(ioread8(&h->synd))); - dev_err(&dev->pdev->dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); + mlx5_core_err(dev, "fw_ver %s\n", fw_str); + mlx5_core_err(dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id)); + mlx5_core_err(dev, "irisc_index %d\n", ioread8(&h->irisc_index)); + mlx5_core_err(dev, "synd 0x%x: %s\n", ioread8(&h->synd), + hsynd_str(ioread8(&h->synd))); + mlx5_core_err(dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); fw = ioread32be(&h->fw_ver); - dev_err(&dev->pdev->dev, "raw fw_ver 0x%08x\n", fw); + mlx5_core_err(dev, "raw fw_ver 0x%08x\n", fw); } static unsigned long get_next_poll_jiffies(void) @@ -262,8 +266,7 @@ void mlx5_trigger_health_work(struct mlx5_core_dev *dev) if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) queue_work(health->wq, &health->work); else - dev_err(&dev->pdev->dev, - "new health works are not permitted at this stage\n"); + mlx5_core_err(dev, "new health works are not permitted at this stage\n"); spin_unlock_irqrestore(&health->wq_lock, flags); } @@ -284,7 +287,7 @@ static void poll_health(struct timer_list *t) health->prev = count; if (health->miss_counter == MAX_MISSES) { - dev_err(&dev->pdev->dev, "device's health compromised - reached miss count\n"); + mlx5_core_err(dev, "device's health compromised - reached miss count\n"); print_health_info(dev); } @@ -352,6 +355,13 @@ void mlx5_drain_health_recovery(struct mlx5_core_dev *dev) cancel_delayed_work_sync(&dev->priv.health.recover_work); } +void mlx5_health_flush(struct mlx5_core_dev *dev) +{ + struct mlx5_core_health *health = &dev->priv.health; + + flush_workqueue(health->wq); +} + void mlx5_health_cleanup(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; @@ -370,7 +380,7 @@ int mlx5_health_init(struct mlx5_core_dev *dev) return -ENOMEM; strcpy(name, "mlx5_health"); - strcat(name, dev_name(&dev->pdev->dev)); + strcat(name, dev_name(dev->device)); health->wq = create_singlethread_workqueue(name); kfree(name); if (!health->wq) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index 4eac42555c7d..ada1b7c0e0b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -68,6 +68,7 @@ static void mlx5i_build_nic_params(struct mlx5_core_dev *mdev, params->lro_en = false; params->hard_mtu = MLX5_IB_GRH_BYTES + MLX5_IPOIB_HARD_LEN; + params->tunneled_offload_en = false; } /* Called directly after IPoIB netdevice was created to initialize SW structs */ @@ -77,15 +78,14 @@ int mlx5i_init(struct mlx5_core_dev *mdev, void *ppriv) { struct mlx5e_priv *priv = mlx5i_epriv(netdev); - u16 max_mtu; int err; err = mlx5e_netdev_init(netdev, priv, mdev, profile, ppriv); if (err) return err; - mlx5_query_port_max_mtu(mdev, &max_mtu, 1); - netdev->mtu = max_mtu; + mlx5e_set_netdev_mtu_boundaries(priv); + netdev->mtu = netdev->max_mtu; mlx5e_build_nic_params(mdev, &priv->rss_params, &priv->channels.params, mlx5e_get_netdev_max_channels(netdev), diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c index 5633f8572800..8212bfd05733 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c @@ -122,7 +122,7 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, /* Handle add/replace event */ if (fi->fib_nhs == 1) { if (__mlx5_lag_is_active(ldev)) { - struct net_device *nh_dev = fi->fib_nh[0].nh_dev; + struct net_device *nh_dev = fi->fib_nh[0].fib_nh_dev; int i = mlx5_lag_dev_get_netdev_idx(ldev, nh_dev); mlx5_lag_set_port_affinity(ldev, ++i); @@ -134,10 +134,10 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, return; /* Verify next hops are ports of the same hca */ - if (!(fi->fib_nh[0].nh_dev == ldev->pf[0].netdev && - fi->fib_nh[1].nh_dev == ldev->pf[1].netdev) && - !(fi->fib_nh[0].nh_dev == ldev->pf[1].netdev && - fi->fib_nh[1].nh_dev == ldev->pf[0].netdev)) { + if (!(fi->fib_nh[0].fib_nh_dev == ldev->pf[0].netdev && + fi->fib_nh[1].fib_nh_dev == ldev->pf[1].netdev) && + !(fi->fib_nh[0].fib_nh_dev == ldev->pf[1].netdev && + fi->fib_nh[1].fib_nh_dev == ldev->pf[0].netdev)) { mlx5_core_warn(ldev->pf[0].dev, "Multipath offload require two ports of the same HCA\n"); return; } @@ -167,7 +167,7 @@ static void mlx5_lag_fib_nexthop_event(struct mlx5_lag *ldev, /* nh added/removed */ if (event == FIB_EVENT_NH_DEL) { - int i = mlx5_lag_dev_get_netdev_idx(ldev, fib_nh->nh_dev); + int i = mlx5_lag_dev_get_netdev_idx(ldev, fib_nh->fib_nh_dev); if (i >= 0) { i = (i + 1) % 2 + 1; /* peer port */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c index ca0ee9916e9e..0059b290e095 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c @@ -535,23 +535,16 @@ void mlx5_init_clock(struct mlx5_core_dev *mdev) do_div(ns, NSEC_PER_SEC / HZ); clock->overflow_period = ns; - mdev->clock_info_page = alloc_page(GFP_KERNEL); - if (mdev->clock_info_page) { - mdev->clock_info = kmap(mdev->clock_info_page); - if (!mdev->clock_info) { - __free_page(mdev->clock_info_page); - mlx5_core_warn(mdev, "failed to map clock page\n"); - } else { - mdev->clock_info->sign = 0; - mdev->clock_info->nsec = clock->tc.nsec; - mdev->clock_info->cycles = clock->tc.cycle_last; - mdev->clock_info->mask = clock->cycles.mask; - mdev->clock_info->mult = clock->nominal_c_mult; - mdev->clock_info->shift = clock->cycles.shift; - mdev->clock_info->frac = clock->tc.frac; - mdev->clock_info->overflow_period = - clock->overflow_period; - } + mdev->clock_info = + (struct mlx5_ib_clock_info *)get_zeroed_page(GFP_KERNEL); + if (mdev->clock_info) { + mdev->clock_info->nsec = clock->tc.nsec; + mdev->clock_info->cycles = clock->tc.cycle_last; + mdev->clock_info->mask = clock->cycles.mask; + mdev->clock_info->mult = clock->nominal_c_mult; + mdev->clock_info->shift = clock->cycles.shift; + mdev->clock_info->frac = clock->tc.frac; + mdev->clock_info->overflow_period = clock->overflow_period; } INIT_WORK(&clock->pps_info.out_work, mlx5_pps_out); @@ -599,8 +592,7 @@ void mlx5_cleanup_clock(struct mlx5_core_dev *mdev) cancel_delayed_work_sync(&clock->overflow_work); if (mdev->clock_info) { - kunmap(mdev->clock_info_page); - __free_page(mdev->clock_info_page); + free_page((unsigned long)mdev->clock_info); mdev->clock_info = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c index 40f4a19b1ce1..be69c1d7941a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c @@ -80,10 +80,8 @@ void mlx5_init_port_tun_entropy(struct mlx5_tun_entropy *tun_entropy, mlx5_query_port_tun_entropy(mdev, &entropy_flags); tun_entropy->num_enabling_entries = 0; tun_entropy->num_disabling_entries = 0; - tun_entropy->enabled = entropy_flags.calc_enabled; - tun_entropy->enabled = - (entropy_flags.calc_supported) ? - entropy_flags.calc_enabled : true; + tun_entropy->enabled = entropy_flags.calc_supported ? + entropy_flags.calc_enabled : true; } static int mlx5_set_entropy(struct mlx5_tun_entropy *tun_entropy, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c index 9a8fd762167b..b9d4f4e19ff9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c @@ -33,6 +33,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/mlx5/driver.h> +#include <net/vxlan.h> #include "mlx5_core.h" #include "vxlan.h" @@ -204,8 +205,8 @@ struct mlx5_vxlan *mlx5_vxlan_create(struct mlx5_core_dev *mdev) spin_lock_init(&vxlan->lock); hash_init(vxlan->htable); - /* Hardware adds 4789 by default */ - mlx5_vxlan_add_port(vxlan, 4789); + /* Hardware adds 4789 (IANA_VXLAN_UDP_PORT) by default */ + mlx5_vxlan_add_port(vxlan, IANA_VXLAN_UDP_PORT); return vxlan; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 76716419370d..61fa1d162d28 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -567,24 +567,23 @@ query_ex: static int set_hca_cap(struct mlx5_core_dev *dev) { - struct pci_dev *pdev = dev->pdev; int err; err = handle_hca_cap(dev); if (err) { - dev_err(&pdev->dev, "handle_hca_cap failed\n"); + mlx5_core_err(dev, "handle_hca_cap failed\n"); goto out; } err = handle_hca_cap_atomic(dev); if (err) { - dev_err(&pdev->dev, "handle_hca_cap_atomic failed\n"); + mlx5_core_err(dev, "handle_hca_cap_atomic failed\n"); goto out; } err = handle_hca_cap_odp(dev); if (err) { - dev_err(&pdev->dev, "handle_hca_cap_odp failed\n"); + mlx5_core_err(dev, "handle_hca_cap_odp failed\n"); goto out; } @@ -716,36 +715,28 @@ static int mlx5_core_set_issi(struct mlx5_core_dev *dev) return -EOPNOTSUPP; } -static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) +static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev, + const struct pci_device_id *id) { - struct pci_dev *pdev = dev->pdev; + struct mlx5_priv *priv = &dev->priv; int err = 0; - pci_set_drvdata(dev->pdev, dev); - strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); - priv->name[MLX5_MAX_NAME_LEN - 1] = 0; - - mutex_init(&priv->pgdir_mutex); - INIT_LIST_HEAD(&priv->pgdir_list); - spin_lock_init(&priv->mkey_lock); + priv->pci_dev_data = id->driver_data; - mutex_init(&priv->alloc_mutex); + pci_set_drvdata(dev->pdev, dev); + dev->bar_addr = pci_resource_start(pdev, 0); priv->numa_node = dev_to_node(&dev->pdev->dev); - if (mlx5_debugfs_root) - priv->dbg_root = - debugfs_create_dir(pci_name(pdev), mlx5_debugfs_root); - err = mlx5_pci_enable_device(dev); if (err) { - dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); - goto err_dbg; + mlx5_core_err(dev, "Cannot enable PCI device, aborting\n"); + return err; } err = request_bar(pdev); if (err) { - dev_err(&pdev->dev, "error requesting BARs, aborting\n"); + mlx5_core_err(dev, "error requesting BARs, aborting\n"); goto err_disable; } @@ -753,7 +744,7 @@ static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) err = set_dma_caps(pdev); if (err) { - dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n"); + mlx5_core_err(dev, "Failed setting DMA capabilities mask, aborting\n"); goto err_clr_master; } @@ -762,11 +753,11 @@ static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128)) mlx5_core_dbg(dev, "Enabling pci atomics failed\n"); - dev->iseg_base = pci_resource_start(dev->pdev, 0); + dev->iseg_base = dev->bar_addr; dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); if (!dev->iseg) { err = -ENOMEM; - dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n"); + mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n"); goto err_clr_master; } @@ -777,52 +768,47 @@ err_clr_master: release_bar(dev->pdev); err_disable: mlx5_pci_disable_device(dev); - -err_dbg: - debugfs_remove(priv->dbg_root); return err; } -static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv) +static void mlx5_pci_close(struct mlx5_core_dev *dev) { iounmap(dev->iseg); pci_clear_master(dev->pdev); release_bar(dev->pdev); mlx5_pci_disable_device(dev); - debugfs_remove_recursive(priv->dbg_root); } -static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) +static int mlx5_init_once(struct mlx5_core_dev *dev) { - struct pci_dev *pdev = dev->pdev; int err; - priv->devcom = mlx5_devcom_register_device(dev); - if (IS_ERR(priv->devcom)) - dev_err(&pdev->dev, "failed to register with devcom (0x%p)\n", - priv->devcom); + dev->priv.devcom = mlx5_devcom_register_device(dev); + if (IS_ERR(dev->priv.devcom)) + mlx5_core_err(dev, "failed to register with devcom (0x%p)\n", + dev->priv.devcom); err = mlx5_query_board_id(dev); if (err) { - dev_err(&pdev->dev, "query board id failed\n"); + mlx5_core_err(dev, "query board id failed\n"); goto err_devcom; } err = mlx5_eq_table_init(dev); if (err) { - dev_err(&pdev->dev, "failed to initialize eq\n"); + mlx5_core_err(dev, "failed to initialize eq\n"); goto err_devcom; } err = mlx5_events_init(dev); if (err) { - dev_err(&pdev->dev, "failed to initialize events\n"); + mlx5_core_err(dev, "failed to initialize events\n"); goto err_eq_cleanup; } err = mlx5_cq_debugfs_init(dev); if (err) { - dev_err(&pdev->dev, "failed to initialize cq debugfs\n"); + mlx5_core_err(dev, "failed to initialize cq debugfs\n"); goto err_events_cleanup; } @@ -838,31 +824,31 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) err = mlx5_init_rl_table(dev); if (err) { - dev_err(&pdev->dev, "Failed to init rate limiting\n"); + mlx5_core_err(dev, "Failed to init rate limiting\n"); goto err_tables_cleanup; } err = mlx5_mpfs_init(dev); if (err) { - dev_err(&pdev->dev, "Failed to init l2 table %d\n", err); + mlx5_core_err(dev, "Failed to init l2 table %d\n", err); goto err_rl_cleanup; } err = mlx5_eswitch_init(dev); if (err) { - dev_err(&pdev->dev, "Failed to init eswitch %d\n", err); + mlx5_core_err(dev, "Failed to init eswitch %d\n", err); goto err_mpfs_cleanup; } err = mlx5_sriov_init(dev); if (err) { - dev_err(&pdev->dev, "Failed to init sriov %d\n", err); + mlx5_core_err(dev, "Failed to init sriov %d\n", err); goto err_eswitch_cleanup; } err = mlx5_fpga_init(dev); if (err) { - dev_err(&pdev->dev, "Failed to init fpga device %d\n", err); + mlx5_core_err(dev, "Failed to init fpga device %d\n", err); goto err_sriov_cleanup; } @@ -912,93 +898,78 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) mlx5_devcom_unregister_device(dev->priv.devcom); } -static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, - bool boot) +static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot) { - struct pci_dev *pdev = dev->pdev; int err; - dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev); - mutex_lock(&dev->intf_state_mutex); - if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { - dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n", - __func__); - goto out; - } - - dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), - fw_rev_min(dev), fw_rev_sub(dev)); + mlx5_core_info(dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), + fw_rev_min(dev), fw_rev_sub(dev)); /* Only PFs hold the relevant PCIe information for this query */ if (mlx5_core_is_pf(dev)) pcie_print_link_status(dev->pdev); - /* on load removing any previous indication of internal error, device is - * up - */ - dev->state = MLX5_DEVICE_STATE_UP; - /* wait for firmware to accept initialization segments configurations */ err = wait_fw_init(dev, FW_PRE_INIT_TIMEOUT_MILI); if (err) { - dev_err(&dev->pdev->dev, "Firmware over %d MS in pre-initializing state, aborting\n", - FW_PRE_INIT_TIMEOUT_MILI); - goto out_err; + mlx5_core_err(dev, "Firmware over %d MS in pre-initializing state, aborting\n", + FW_PRE_INIT_TIMEOUT_MILI); + return err; } err = mlx5_cmd_init(dev); if (err) { - dev_err(&pdev->dev, "Failed initializing command interface, aborting\n"); - goto out_err; + mlx5_core_err(dev, "Failed initializing command interface, aborting\n"); + return err; } err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); if (err) { - dev_err(&dev->pdev->dev, "Firmware over %d MS in initializing state, aborting\n", - FW_INIT_TIMEOUT_MILI); + mlx5_core_err(dev, "Firmware over %d MS in initializing state, aborting\n", + FW_INIT_TIMEOUT_MILI); goto err_cmd_cleanup; } err = mlx5_core_enable_hca(dev, 0); if (err) { - dev_err(&pdev->dev, "enable hca failed\n"); + mlx5_core_err(dev, "enable hca failed\n"); goto err_cmd_cleanup; } err = mlx5_core_set_issi(dev); if (err) { - dev_err(&pdev->dev, "failed to set issi\n"); + mlx5_core_err(dev, "failed to set issi\n"); goto err_disable_hca; } err = mlx5_satisfy_startup_pages(dev, 1); if (err) { - dev_err(&pdev->dev, "failed to allocate boot pages\n"); + mlx5_core_err(dev, "failed to allocate boot pages\n"); goto err_disable_hca; } err = set_hca_ctrl(dev); if (err) { - dev_err(&pdev->dev, "set_hca_ctrl failed\n"); + mlx5_core_err(dev, "set_hca_ctrl failed\n"); goto reclaim_boot_pages; } err = set_hca_cap(dev); if (err) { - dev_err(&pdev->dev, "set_hca_cap failed\n"); + mlx5_core_err(dev, "set_hca_cap failed\n"); goto reclaim_boot_pages; } err = mlx5_satisfy_startup_pages(dev, 0); if (err) { - dev_err(&pdev->dev, "failed to allocate init pages\n"); + mlx5_core_err(dev, "failed to allocate init pages\n"); goto reclaim_boot_pages; } err = mlx5_cmd_init_hca(dev, sw_owner_id); if (err) { - dev_err(&pdev->dev, "init hca failed\n"); + mlx5_core_err(dev, "init hca failed\n"); goto reclaim_boot_pages; } @@ -1008,23 +979,50 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, err = mlx5_query_hca_caps(dev); if (err) { - dev_err(&pdev->dev, "query hca failed\n"); - goto err_stop_poll; + mlx5_core_err(dev, "query hca failed\n"); + goto stop_health; } - if (boot) { - err = mlx5_init_once(dev, priv); - if (err) { - dev_err(&pdev->dev, "sw objs init failed\n"); - goto err_stop_poll; - } + return 0; + +stop_health: + mlx5_stop_health_poll(dev, boot); +reclaim_boot_pages: + mlx5_reclaim_startup_pages(dev); +err_disable_hca: + mlx5_core_disable_hca(dev, 0); +err_cmd_cleanup: + mlx5_cmd_cleanup(dev); + + return err; +} + +static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot) +{ + int err; + + mlx5_stop_health_poll(dev, boot); + err = mlx5_cmd_teardown_hca(dev); + if (err) { + mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n"); + return err; } + mlx5_reclaim_startup_pages(dev); + mlx5_core_disable_hca(dev, 0); + mlx5_cmd_cleanup(dev); + + return 0; +} + +static int mlx5_load(struct mlx5_core_dev *dev) +{ + int err; dev->priv.uar = mlx5_get_uars_page(dev); if (IS_ERR(dev->priv.uar)) { - dev_err(&pdev->dev, "Failed allocating uar, aborting\n"); + mlx5_core_err(dev, "Failed allocating uar, aborting\n"); err = PTR_ERR(dev->priv.uar); - goto err_get_uars; + return err; } mlx5_events_start(dev); @@ -1032,132 +1030,155 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, err = mlx5_eq_table_create(dev); if (err) { - dev_err(&pdev->dev, "Failed to create EQs\n"); + mlx5_core_err(dev, "Failed to create EQs\n"); goto err_eq_table; } err = mlx5_fw_tracer_init(dev->tracer); if (err) { - dev_err(&pdev->dev, "Failed to init FW tracer\n"); + mlx5_core_err(dev, "Failed to init FW tracer\n"); goto err_fw_tracer; } err = mlx5_fpga_device_start(dev); if (err) { - dev_err(&pdev->dev, "fpga device start failed %d\n", err); + mlx5_core_err(dev, "fpga device start failed %d\n", err); goto err_fpga_start; } err = mlx5_accel_ipsec_init(dev); if (err) { - dev_err(&pdev->dev, "IPSec device start failed %d\n", err); + mlx5_core_err(dev, "IPSec device start failed %d\n", err); goto err_ipsec_start; } err = mlx5_accel_tls_init(dev); if (err) { - dev_err(&pdev->dev, "TLS device start failed %d\n", err); + mlx5_core_err(dev, "TLS device start failed %d\n", err); goto err_tls_start; } err = mlx5_init_fs(dev); if (err) { - dev_err(&pdev->dev, "Failed to init flow steering\n"); + mlx5_core_err(dev, "Failed to init flow steering\n"); goto err_fs; } err = mlx5_core_set_hca_defaults(dev); if (err) { - dev_err(&pdev->dev, "Failed to set hca defaults\n"); + mlx5_core_err(dev, "Failed to set hca defaults\n"); goto err_fs; } err = mlx5_sriov_attach(dev); if (err) { - dev_err(&pdev->dev, "sriov init failed %d\n", err); + mlx5_core_err(dev, "sriov init failed %d\n", err); goto err_sriov; } err = mlx5_ec_init(dev); if (err) { - dev_err(&pdev->dev, "Failed to init embedded CPU\n"); + mlx5_core_err(dev, "Failed to init embedded CPU\n"); goto err_ec; } - if (mlx5_device_registered(dev)) { - mlx5_attach_device(dev); - } else { - err = mlx5_register_device(dev); - if (err) { - dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); - goto err_reg_dev; - } - } - - set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); -out: - mutex_unlock(&dev->intf_state_mutex); - return 0; -err_reg_dev: - mlx5_ec_cleanup(dev); - err_ec: mlx5_sriov_detach(dev); - err_sriov: mlx5_cleanup_fs(dev); - err_fs: mlx5_accel_tls_cleanup(dev); - err_tls_start: mlx5_accel_ipsec_cleanup(dev); - err_ipsec_start: mlx5_fpga_device_stop(dev); - err_fpga_start: mlx5_fw_tracer_cleanup(dev->tracer); - err_fw_tracer: mlx5_eq_table_destroy(dev); - err_eq_table: mlx5_pagealloc_stop(dev); mlx5_events_stop(dev); - mlx5_put_uars_page(dev, priv->uar); + mlx5_put_uars_page(dev, dev->priv.uar); + return err; +} + +static void mlx5_unload(struct mlx5_core_dev *dev) +{ + mlx5_ec_cleanup(dev); + mlx5_sriov_detach(dev); + mlx5_cleanup_fs(dev); + mlx5_accel_ipsec_cleanup(dev); + mlx5_accel_tls_cleanup(dev); + mlx5_fpga_device_stop(dev); + mlx5_fw_tracer_cleanup(dev->tracer); + mlx5_eq_table_destroy(dev); + mlx5_pagealloc_stop(dev); + mlx5_events_stop(dev); + mlx5_put_uars_page(dev, dev->priv.uar); +} -err_get_uars: - if (boot) - mlx5_cleanup_once(dev); +static int mlx5_load_one(struct mlx5_core_dev *dev, bool boot) +{ + int err = 0; -err_stop_poll: - mlx5_stop_health_poll(dev, boot); - if (mlx5_cmd_teardown_hca(dev)) { - dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); - goto out_err; + dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev); + mutex_lock(&dev->intf_state_mutex); + if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { + mlx5_core_warn(dev, "interface is up, NOP\n"); + goto out; } + /* remove any previous indication of internal error */ + dev->state = MLX5_DEVICE_STATE_UP; -reclaim_boot_pages: - mlx5_reclaim_startup_pages(dev); + err = mlx5_function_setup(dev, boot); + if (err) + goto out; -err_disable_hca: - mlx5_core_disable_hca(dev, 0); + if (boot) { + err = mlx5_init_once(dev); + if (err) { + mlx5_core_err(dev, "sw objs init failed\n"); + goto function_teardown; + } + } -err_cmd_cleanup: - mlx5_cmd_cleanup(dev); + err = mlx5_load(dev); + if (err) + goto err_load; -out_err: + if (mlx5_device_registered(dev)) { + mlx5_attach_device(dev); + } else { + err = mlx5_register_device(dev); + if (err) { + mlx5_core_err(dev, "register device failed %d\n", err); + goto err_reg_dev; + } + } + + set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state); +out: + mutex_unlock(&dev->intf_state_mutex); + + return err; + +err_reg_dev: + mlx5_unload(dev); +err_load: + if (boot) + mlx5_cleanup_once(dev); +function_teardown: + mlx5_function_teardown(dev, boot); dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; mutex_unlock(&dev->intf_state_mutex); return err; } -static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, - bool cleanup) +static int mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup) { int err = 0; @@ -1166,8 +1187,8 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, mutex_lock(&dev->intf_state_mutex); if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { - dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", - __func__); + mlx5_core_warn(dev, "%s: interface is down, NOP\n", + __func__); if (cleanup) mlx5_cleanup_once(dev); goto out; @@ -1178,30 +1199,12 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, if (mlx5_device_registered(dev)) mlx5_detach_device(dev); - mlx5_ec_cleanup(dev); - mlx5_sriov_detach(dev); - mlx5_cleanup_fs(dev); - mlx5_accel_ipsec_cleanup(dev); - mlx5_accel_tls_cleanup(dev); - mlx5_fpga_device_stop(dev); - mlx5_fw_tracer_cleanup(dev->tracer); - mlx5_eq_table_destroy(dev); - mlx5_pagealloc_stop(dev); - mlx5_events_stop(dev); - mlx5_put_uars_page(dev, priv->uar); + mlx5_unload(dev); + if (cleanup) mlx5_cleanup_once(dev); - mlx5_stop_health_poll(dev, cleanup); - - err = mlx5_cmd_teardown_hca(dev); - if (err) { - dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); - goto out; - } - mlx5_reclaim_startup_pages(dev); - mlx5_core_disable_hca(dev, 0); - mlx5_cmd_cleanup(dev); + mlx5_function_teardown(dev, cleanup); out: mutex_unlock(&dev->intf_state_mutex); return err; @@ -1218,29 +1221,12 @@ static const struct devlink_ops mlx5_devlink_ops = { #endif }; -#define MLX5_IB_MOD "mlx5_ib" -static int init_one(struct pci_dev *pdev, - const struct pci_device_id *id) +static int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) { - struct mlx5_core_dev *dev; - struct devlink *devlink; - struct mlx5_priv *priv; + struct mlx5_priv *priv = &dev->priv; int err; - devlink = devlink_alloc(&mlx5_devlink_ops, sizeof(*dev)); - if (!devlink) { - dev_err(&pdev->dev, "kzalloc failed\n"); - return -ENOMEM; - } - - dev = devlink_priv(devlink); - priv = &dev->priv; - priv->pci_dev_data = id->driver_data; - - pci_set_drvdata(pdev, dev); - - dev->pdev = pdev; - dev->profile = &profile[prof_sel]; + dev->profile = &profile[profile_idx]; INIT_LIST_HEAD(&priv->ctx_list); spin_lock_init(&priv->ctx_lock); @@ -1252,25 +1238,75 @@ static int init_one(struct pci_dev *pdev, INIT_LIST_HEAD(&priv->bfregs.reg_head.list); INIT_LIST_HEAD(&priv->bfregs.wc_head.list); - err = mlx5_pci_init(dev, priv); - if (err) { - dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err); - goto clean_dev; + mutex_init(&priv->alloc_mutex); + mutex_init(&priv->pgdir_mutex); + INIT_LIST_HEAD(&priv->pgdir_list); + spin_lock_init(&priv->mkey_lock); + + priv->dbg_root = debugfs_create_dir(dev_name(dev->device), + mlx5_debugfs_root); + if (!priv->dbg_root) { + dev_err(dev->device, "mlx5_core: error, Cannot create debugfs dir, aborting\n"); + return -ENOMEM; } err = mlx5_health_init(dev); - if (err) { - dev_err(&pdev->dev, "mlx5_health_init failed with error code %d\n", err); - goto close_pci; - } + if (err) + goto err_health_init; err = mlx5_pagealloc_init(dev); if (err) goto err_pagealloc_init; - err = mlx5_load_one(dev, priv, true); + return 0; + +err_pagealloc_init: + mlx5_health_cleanup(dev); +err_health_init: + debugfs_remove(dev->priv.dbg_root); + + return err; +} + +static void mlx5_mdev_uninit(struct mlx5_core_dev *dev) +{ + mlx5_pagealloc_cleanup(dev); + mlx5_health_cleanup(dev); + debugfs_remove_recursive(dev->priv.dbg_root); +} + +#define MLX5_IB_MOD "mlx5_ib" +static int init_one(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct mlx5_core_dev *dev; + struct devlink *devlink; + int err; + + devlink = devlink_alloc(&mlx5_devlink_ops, sizeof(*dev)); + if (!devlink) { + dev_err(&pdev->dev, "kzalloc failed\n"); + return -ENOMEM; + } + + dev = devlink_priv(devlink); + dev->device = &pdev->dev; + dev->pdev = pdev; + + err = mlx5_mdev_init(dev, prof_sel); + if (err) + goto mdev_init_err; + + err = mlx5_pci_init(dev, pdev, id); + if (err) { + mlx5_core_err(dev, "mlx5_pci_init failed with error code %d\n", + err); + goto pci_init_err; + } + + err = mlx5_load_one(dev, true); if (err) { - dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err); + mlx5_core_err(dev, "mlx5_load_one failed with error code %d\n", + err); goto err_load_one; } @@ -1284,14 +1320,13 @@ static int init_one(struct pci_dev *pdev, return 0; clean_load: - mlx5_unload_one(dev, priv, true); + mlx5_unload_one(dev, true); + err_load_one: - mlx5_pagealloc_cleanup(dev); -err_pagealloc_init: - mlx5_health_cleanup(dev); -close_pci: - mlx5_pci_close(dev, priv); -clean_dev: + mlx5_pci_close(dev); +pci_init_err: + mlx5_mdev_uninit(dev); +mdev_init_err: devlink_free(devlink); return err; @@ -1301,20 +1336,18 @@ static void remove_one(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); struct devlink *devlink = priv_to_devlink(dev); - struct mlx5_priv *priv = &dev->priv; devlink_unregister(devlink); mlx5_unregister_device(dev); - if (mlx5_unload_one(dev, priv, true)) { - dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n"); - mlx5_health_cleanup(dev); + if (mlx5_unload_one(dev, true)) { + mlx5_core_err(dev, "mlx5_unload_one failed\n"); + mlx5_health_flush(dev); return; } - mlx5_pagealloc_cleanup(dev); - mlx5_health_cleanup(dev); - mlx5_pci_close(dev, priv); + mlx5_pci_close(dev); + mlx5_mdev_uninit(dev); devlink_free(devlink); } @@ -1322,12 +1355,11 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); - struct mlx5_priv *priv = &dev->priv; - dev_info(&pdev->dev, "%s was called\n", __func__); + mlx5_core_info(dev, "%s was called\n", __func__); mlx5_enter_error_state(dev, false); - mlx5_unload_one(dev, priv, false); + mlx5_unload_one(dev, false); /* In case of kernel call drain the health wq */ if (state) { mlx5_drain_health_wq(dev); @@ -1354,7 +1386,9 @@ static int wait_vital(struct pci_dev *pdev) count = ioread32be(health->health_counter); if (count && count != 0xffffffff) { if (last_count && last_count != count) { - dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i); + mlx5_core_info(dev, + "wait vital counter value 0x%x after %d iterations\n", + count, i); return 0; } last_count = count; @@ -1370,12 +1404,12 @@ static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) struct mlx5_core_dev *dev = pci_get_drvdata(pdev); int err; - dev_info(&pdev->dev, "%s was called\n", __func__); + mlx5_core_info(dev, "%s was called\n", __func__); err = mlx5_pci_enable_device(dev); if (err) { - dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n" - , __func__, err); + mlx5_core_err(dev, "%s: mlx5_pci_enable_device failed with error code: %d\n", + __func__, err); return PCI_ERS_RESULT_DISCONNECT; } @@ -1384,7 +1418,7 @@ static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) pci_save_state(pdev); if (wait_vital(pdev)) { - dev_err(&pdev->dev, "%s: wait_vital timed out\n", __func__); + mlx5_core_err(dev, "%s: wait_vital timed out\n", __func__); return PCI_ERS_RESULT_DISCONNECT; } @@ -1394,17 +1428,16 @@ static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) static void mlx5_pci_resume(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); - struct mlx5_priv *priv = &dev->priv; int err; - dev_info(&pdev->dev, "%s was called\n", __func__); + mlx5_core_info(dev, "%s was called\n", __func__); - err = mlx5_load_one(dev, priv, false); + err = mlx5_load_one(dev, false); if (err) - dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n" - , __func__, err); + mlx5_core_err(dev, "%s: mlx5_load_one failed with error code: %d\n", + __func__, err); else - dev_info(&pdev->dev, "%s: device recovered\n", __func__); + mlx5_core_info(dev, "%s: device recovered\n", __func__); } static const struct pci_error_handlers mlx5_err_handler = { @@ -1466,13 +1499,12 @@ succeed: static void shutdown(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); - struct mlx5_priv *priv = &dev->priv; int err; - dev_info(&pdev->dev, "Shutdown was called\n"); + mlx5_core_info(dev, "Shutdown was called\n"); err = mlx5_try_fast_unload(dev); if (err) - mlx5_unload_one(dev, priv, false); + mlx5_unload_one(dev, false); mlx5_pci_disable_device(dev); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 7b331674622c..22e69d4813e4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -41,6 +41,7 @@ #include <linux/ptp_clock_kernel.h> #include <linux/mlx5/cq.h> #include <linux/mlx5/fs.h> +#include <linux/mlx5/driver.h> #define DRIVER_NAME "mlx5_core" #define DRIVER_VERSION "5.0-0" @@ -48,44 +49,57 @@ extern uint mlx5_core_debug_mask; #define mlx5_core_dbg(__dev, format, ...) \ - dev_dbg(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format, \ + dev_dbg((__dev)->device, "%s:%d:(pid %d): " format, \ __func__, __LINE__, current->pid, \ ##__VA_ARGS__) -#define mlx5_core_dbg_once(__dev, format, ...) \ - dev_dbg_once(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format, \ - __func__, __LINE__, current->pid, \ +#define mlx5_core_dbg_once(__dev, format, ...) \ + dev_dbg_once((__dev)->device, \ + "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ ##__VA_ARGS__) -#define mlx5_core_dbg_mask(__dev, mask, format, ...) \ -do { \ - if ((mask) & mlx5_core_debug_mask) \ - mlx5_core_dbg(__dev, format, ##__VA_ARGS__); \ +#define mlx5_core_dbg_mask(__dev, mask, format, ...) \ +do { \ + if ((mask) & mlx5_core_debug_mask) \ + mlx5_core_dbg(__dev, format, ##__VA_ARGS__); \ } while (0) -#define mlx5_core_err(__dev, format, ...) \ - dev_err(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format, \ - __func__, __LINE__, current->pid, \ +#define mlx5_core_err(__dev, format, ...) \ + dev_err((__dev)->device, "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ ##__VA_ARGS__) -#define mlx5_core_err_rl(__dev, format, ...) \ - dev_err_ratelimited(&(__dev)->pdev->dev, \ - "%s:%d:(pid %d): " format, \ - __func__, __LINE__, current->pid, \ - ##__VA_ARGS__) +#define mlx5_core_err_rl(__dev, format, ...) \ + dev_err_ratelimited((__dev)->device, \ + "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ + ##__VA_ARGS__) -#define mlx5_core_warn(__dev, format, ...) \ - dev_warn(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format, \ - __func__, __LINE__, current->pid, \ - ##__VA_ARGS__) +#define mlx5_core_warn(__dev, format, ...) \ + dev_warn((__dev)->device, "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ + ##__VA_ARGS__) #define mlx5_core_warn_once(__dev, format, ...) \ - dev_warn_once(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format, \ + dev_warn_once((__dev)->device, "%s:%d:(pid %d): " format, \ __func__, __LINE__, current->pid, \ ##__VA_ARGS__) -#define mlx5_core_info(__dev, format, ...) \ - dev_info(&(__dev)->pdev->dev, format, ##__VA_ARGS__) +#define mlx5_core_warn_rl(__dev, format, ...) \ + dev_warn_ratelimited((__dev)->device, \ + "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ + ##__VA_ARGS__) + +#define mlx5_core_info(__dev, format, ...) \ + dev_info((__dev)->device, format, ##__VA_ARGS__) + +#define mlx5_core_info_rl(__dev, format, ...) \ + dev_info_ratelimited((__dev)->device, \ + "%s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, \ + ##__VA_ARGS__) enum { MLX5_CMD_DATA, /* print command payload only */ @@ -111,7 +125,6 @@ void mlx5_sriov_cleanup(struct mlx5_core_dev *dev); int mlx5_sriov_attach(struct mlx5_core_dev *dev); void mlx5_sriov_detach(struct mlx5_core_dev *dev); int mlx5_core_sriov_configure(struct pci_dev *dev, int num_vfs); -bool mlx5_sriov_is_enabled(struct mlx5_core_dev *dev); int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id); int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id); int mlx5_create_scheduling_element_cmd(struct mlx5_core_dev *dev, u8 hierarchy, @@ -176,6 +189,11 @@ int mlx5_firmware_flash(struct mlx5_core_dev *dev, const struct firmware *fw); void mlx5e_init(void); void mlx5e_cleanup(void); +static inline bool mlx5_sriov_is_enabled(struct mlx5_core_dev *dev) +{ + return pci_num_vf(dev->pdev) ? true : false; +} + static inline int mlx5_lag_is_lacp_owner(struct mlx5_core_dev *dev) { /* LACP owner conditions: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index 41025387ff2c..91bd258ecf1b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -200,7 +200,7 @@ static void free_4k(struct mlx5_core_dev *dev, u64 addr) rb_erase(&fwp->rb_node, &dev->priv.page_root); if (fwp->free_count != 1) list_del(&fwp->list); - dma_unmap_page(&dev->pdev->dev, addr & MLX5_U64_4K_PAGE_MASK, + dma_unmap_page(dev->device, addr & MLX5_U64_4K_PAGE_MASK, PAGE_SIZE, DMA_BIDIRECTIONAL); __free_page(fwp->page); kfree(fwp); @@ -211,11 +211,12 @@ static void free_4k(struct mlx5_core_dev *dev, u64 addr) static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) { + struct device *device = dev->device; + int nid = dev_to_node(device); struct page *page; u64 zero_addr = 1; u64 addr; int err; - int nid = dev_to_node(&dev->pdev->dev); page = alloc_pages_node(nid, GFP_HIGHUSER, 0); if (!page) { @@ -223,9 +224,8 @@ static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) return -ENOMEM; } map: - addr = dma_map_page(&dev->pdev->dev, page, 0, - PAGE_SIZE, DMA_BIDIRECTIONAL); - if (dma_mapping_error(&dev->pdev->dev, addr)) { + addr = dma_map_page(device, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); + if (dma_mapping_error(device, addr)) { mlx5_core_warn(dev, "failed dma mapping page\n"); err = -ENOMEM; goto err_mapping; @@ -240,8 +240,7 @@ map: err = insert_page(dev, addr, page, func_id); if (err) { mlx5_core_err(dev, "failed to track allocated page\n"); - dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, - DMA_BIDIRECTIONAL); + dma_unmap_page(device, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); } err_mapping: @@ -249,7 +248,7 @@ err_mapping: __free_page(page); if (zero_addr == 0) - dma_unmap_page(&dev->pdev->dev, zero_addr, PAGE_SIZE, + dma_unmap_page(device, zero_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); return err; @@ -600,8 +599,7 @@ int mlx5_wait_for_pages(struct mlx5_core_dev *dev, int *pages) return 0; } - mlx5_core_dbg(dev, "Waiting for %d pages from %s\n", prev_pages, - dev->priv.name); + mlx5_core_dbg(dev, "Waiting for %d pages\n", prev_pages); while (*pages) { if (time_after(jiffies, end)) { mlx5_core_warn(dev, "aborting while there are %d pending pages\n", *pages); @@ -614,6 +612,6 @@ int mlx5_wait_for_pages(struct mlx5_core_dev *dev, int *pages) msleep(50); } - mlx5_core_dbg(dev, "All pages received from %s\n", dev->priv.name); + mlx5_core_dbg(dev, "All pages received\n"); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index 21b7f05b16a5..cc262b30aed5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -293,15 +293,36 @@ static int mlx5_query_module_num(struct mlx5_core_dev *dev, int *module_num) return 0; } +static int mlx5_eeprom_page(int offset) +{ + if (offset < MLX5_EEPROM_PAGE_LENGTH) + /* Addresses between 0-255 - page 00 */ + return 0; + + /* Addresses between 256 - 639 belongs to pages 01, 02 and 03 + * For example, offset = 400 belongs to page 02: + * 1 + ((400 - 256)/128) = 2 + */ + return 1 + ((offset - MLX5_EEPROM_PAGE_LENGTH) / + MLX5_EEPROM_HIGH_PAGE_LENGTH); +} + +static int mlx5_eeprom_high_page_offset(int page_num) +{ + if (!page_num) /* Page 0 always start from low page */ + return 0; + + /* High page */ + return page_num * MLX5_EEPROM_HIGH_PAGE_LENGTH; +} + int mlx5_query_module_eeprom(struct mlx5_core_dev *dev, u16 offset, u16 size, u8 *data) { + int module_num, page_num, status, err; u32 out[MLX5_ST_SZ_DW(mcia_reg)]; u32 in[MLX5_ST_SZ_DW(mcia_reg)]; - int module_num; u16 i2c_addr; - int status; - int err; void *ptr = MLX5_ADDR_OF(mcia_reg, out, dword_0); err = mlx5_query_module_num(dev, &module_num); @@ -311,21 +332,24 @@ int mlx5_query_module_eeprom(struct mlx5_core_dev *dev, memset(in, 0, sizeof(in)); size = min_t(int, size, MLX5_EEPROM_MAX_BYTES); - if (offset < MLX5_EEPROM_PAGE_LENGTH && - offset + size > MLX5_EEPROM_PAGE_LENGTH) + /* Get the page number related to the given offset */ + page_num = mlx5_eeprom_page(offset); + + /* Set the right offset according to the page number, + * For page_num > 0, relative offset is always >= 128 (high page). + */ + offset -= mlx5_eeprom_high_page_offset(page_num); + + if (offset + size > MLX5_EEPROM_PAGE_LENGTH) /* Cross pages read, read until offset 256 in low page */ size -= offset + size - MLX5_EEPROM_PAGE_LENGTH; i2c_addr = MLX5_I2C_ADDR_LOW; - if (offset >= MLX5_EEPROM_PAGE_LENGTH) { - i2c_addr = MLX5_I2C_ADDR_HIGH; - offset -= MLX5_EEPROM_PAGE_LENGTH; - } MLX5_SET(mcia_reg, in, l, 0); MLX5_SET(mcia_reg, in, module, module_num); MLX5_SET(mcia_reg, in, i2c_device_address, i2c_addr); - MLX5_SET(mcia_reg, in, page_number, 0); + MLX5_SET(mcia_reg, in, page_number, page_num); MLX5_SET(mcia_reg, in, device_address, offset); MLX5_SET(mcia_reg, in, size, size); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/rdma.c b/drivers/net/ethernet/mellanox/mlx5/core/rdma.c new file mode 100644 index 000000000000..86f77456f873 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/rdma.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2019 Mellanox Technologies */ + +#include <linux/mlx5/vport.h> +#include <rdma/ib_verbs.h> +#include <net/addrconf.h> + +#include "lib/mlx5.h" +#include "eswitch.h" +#include "fs_core.h" +#include "rdma.h" + +static void mlx5_rdma_disable_roce_steering(struct mlx5_core_dev *dev) +{ + struct mlx5_core_roce *roce = &dev->priv.roce; + + if (!roce->ft) + return; + + mlx5_del_flow_rules(roce->allow_rule); + mlx5_destroy_flow_group(roce->fg); + mlx5_destroy_flow_table(roce->ft); +} + +static int mlx5_rdma_enable_roce_steering(struct mlx5_core_dev *dev) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_core_roce *roce = &dev->priv.roce; + struct mlx5_flow_handle *flow_rule = NULL; + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_namespace *ns = NULL; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_spec *spec; + struct mlx5_flow_table *ft; + struct mlx5_flow_group *fg; + void *match_criteria; + u32 *flow_group_in; + void *misc; + int err; + + if (!(MLX5_CAP_FLOWTABLE_RDMA_RX(dev, ft_support) && + MLX5_CAP_FLOWTABLE_RDMA_RX(dev, table_miss_action_domain))) + return -EOPNOTSUPP; + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + if (!flow_group_in) + return -ENOMEM; + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + kvfree(flow_group_in); + return -ENOMEM; + } + + ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_RDMA_RX); + if (!ns) { + mlx5_core_err(dev, "Failed to get RDMA RX namespace"); + err = -EOPNOTSUPP; + goto free; + } + + ft_attr.max_fte = 1; + ft = mlx5_create_flow_table(ns, &ft_attr); + if (IS_ERR(ft)) { + mlx5_core_err(dev, "Failed to create RDMA RX flow table"); + err = PTR_ERR(ft); + goto free; + } + + MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, + MLX5_MATCH_MISC_PARAMETERS); + match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, + match_criteria); + MLX5_SET_TO_ONES(fte_match_param, match_criteria, + misc_parameters.source_port); + + fg = mlx5_create_flow_group(ft, flow_group_in); + if (IS_ERR(fg)) { + err = PTR_ERR(fg); + mlx5_core_err(dev, "Failed to create RDMA RX flow group err(%d)\n", err); + goto destroy_flow_table; + } + + spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; + misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, + misc_parameters); + MLX5_SET(fte_match_set_misc, misc, source_port, + dev->priv.eswitch->manager_vport); + misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + misc_parameters); + MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; + flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, NULL, 0); + if (IS_ERR(flow_rule)) { + err = PTR_ERR(flow_rule); + mlx5_core_err(dev, "Failed to add RoCE allow rule, err=%d\n", + err); + goto destroy_flow_group; + } + + kvfree(spec); + kvfree(flow_group_in); + roce->ft = ft; + roce->fg = fg; + roce->allow_rule = flow_rule; + + return 0; + +destroy_flow_table: + mlx5_destroy_flow_table(ft); +destroy_flow_group: + mlx5_destroy_flow_group(fg); +free: + kvfree(spec); + kvfree(flow_group_in); + return err; +} + +static void mlx5_rdma_del_roce_addr(struct mlx5_core_dev *dev) +{ + mlx5_core_roce_gid_set(dev, 0, 0, 0, + NULL, NULL, false, 0, 0); +} + +static void mlx5_rdma_make_default_gid(struct mlx5_core_dev *dev, union ib_gid *gid) +{ + u8 hw_id[ETH_ALEN]; + + mlx5_query_nic_vport_mac_address(dev, 0, hw_id); + gid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); + addrconf_addr_eui48(&gid->raw[8], hw_id); +} + +static int mlx5_rdma_add_roce_addr(struct mlx5_core_dev *dev) +{ + union ib_gid gid; + u8 mac[ETH_ALEN]; + + mlx5_rdma_make_default_gid(dev, &gid); + return mlx5_core_roce_gid_set(dev, 0, + MLX5_ROCE_VERSION_1, + 0, gid.raw, mac, + false, 0, 1); +} + +void mlx5_rdma_disable_roce(struct mlx5_core_dev *dev) +{ + mlx5_rdma_disable_roce_steering(dev); + mlx5_rdma_del_roce_addr(dev); + mlx5_nic_vport_disable_roce(dev); +} + +void mlx5_rdma_enable_roce(struct mlx5_core_dev *dev) +{ + int err; + + err = mlx5_nic_vport_enable_roce(dev); + if (err) { + mlx5_core_err(dev, "Failed to enable RoCE: %d\n", err); + return; + } + + err = mlx5_rdma_add_roce_addr(dev); + if (err) { + mlx5_core_err(dev, "Failed to add RoCE address: %d\n", err); + goto disable_roce; + } + + err = mlx5_rdma_enable_roce_steering(dev); + if (err) { + mlx5_core_err(dev, "Failed to enable RoCE steering: %d\n", err); + goto del_roce_addr; + } + + return; + +del_roce_addr: + mlx5_rdma_del_roce_addr(dev); +disable_roce: + mlx5_nic_vport_disable_roce(dev); + return; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/rdma.h b/drivers/net/ethernet/mellanox/mlx5/core/rdma.h new file mode 100644 index 000000000000..750cff2a71a4 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/rdma.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2019 Mellanox Technologies. */ + +#ifndef __MLX5_RDMA_H__ +#define __MLX5_RDMA_H__ + +#include "mlx5_core.h" + +#ifdef CONFIG_MLX5_ESWITCH + +void mlx5_rdma_enable_roce(struct mlx5_core_dev *dev); +void mlx5_rdma_disable_roce(struct mlx5_core_dev *dev); + +#else /* CONFIG_MLX5_ESWITCH */ + +static inline void mlx5_rdma_enable_roce(struct mlx5_core_dev *dev) {} +static inline void mlx5_rdma_disable_roce(struct mlx5_core_dev *dev) {} + +#endif /* CONFIG_MLX5_ESWITCH */ +#endif /* __MLX5_RDMA_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index 7b23fa8d2d60..a249b3c3843d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -36,13 +36,6 @@ #include "mlx5_core.h" #include "eswitch.h" -bool mlx5_sriov_is_enabled(struct mlx5_core_dev *dev) -{ - struct mlx5_core_sriov *sriov = &dev->priv.sriov; - - return !!sriov->num_vfs; -} - static int sriov_restore_guids(struct mlx5_core_dev *dev, int vf) { struct mlx5_core_sriov *sriov = &dev->priv.sriov; @@ -151,33 +144,10 @@ out: mlx5_core_warn(dev, "timeout reclaiming VFs pages\n"); } -static int mlx5_pci_enable_sriov(struct pci_dev *pdev, int num_vfs) -{ - struct mlx5_core_dev *dev = pci_get_drvdata(pdev); - int err = 0; - - if (pci_num_vf(pdev)) { - mlx5_core_warn(dev, "Unable to enable pci sriov, already enabled\n"); - return -EBUSY; - } - - err = pci_enable_sriov(pdev, num_vfs); - if (err) - mlx5_core_warn(dev, "pci_enable_sriov failed : %d\n", err); - - return err; -} - -static void mlx5_pci_disable_sriov(struct pci_dev *pdev) -{ - pci_disable_sriov(pdev); -} - static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); - struct mlx5_core_sriov *sriov = &dev->priv.sriov; - int err = 0; + int err; err = mlx5_device_enable_sriov(dev, num_vfs); if (err) { @@ -185,42 +155,37 @@ static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) return err; } - err = mlx5_pci_enable_sriov(pdev, num_vfs); + err = pci_enable_sriov(pdev, num_vfs); if (err) { - mlx5_core_warn(dev, "mlx5_pci_enable_sriov failed : %d\n", err); + mlx5_core_warn(dev, "pci_enable_sriov failed : %d\n", err); mlx5_device_disable_sriov(dev); - return err; } - - sriov->num_vfs = num_vfs; - - return 0; + return err; } static void mlx5_sriov_disable(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); - struct mlx5_core_sriov *sriov = &dev->priv.sriov; - mlx5_pci_disable_sriov(pdev); + pci_disable_sriov(pdev); mlx5_device_disable_sriov(dev); - sriov->num_vfs = 0; } int mlx5_core_sriov_configure(struct pci_dev *pdev, int num_vfs) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); + struct mlx5_core_sriov *sriov = &dev->priv.sriov; int err = 0; mlx5_core_dbg(dev, "requested num_vfs %d\n", num_vfs); - if (!mlx5_core_is_pf(dev)) - return -EPERM; if (num_vfs) err = mlx5_sriov_enable(pdev, num_vfs); else mlx5_sriov_disable(pdev); + if (!err) + sriov->num_vfs = num_vfs; return err ? err : num_vfs; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c index c4d4b76096dc..b1068500f1df 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c @@ -182,16 +182,24 @@ out: } EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state); +int mlx5_core_create_tir_out(struct mlx5_core_dev *dev, + u32 *in, int inlen, + u32 *out, int outlen) +{ + MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR); + + return mlx5_cmd_exec(dev, in, inlen, out, outlen); +} +EXPORT_SYMBOL(mlx5_core_create_tir_out); + int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *tirn) { - u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {0}; + u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {}; int err; - MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR); - - memset(out, 0, sizeof(out)); - err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); + err = mlx5_core_create_tir_out(dev, in, inlen, + out, sizeof(out)); if (!err) *tirn = MLX5_GET(create_tir_out, out, tirn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/uar.c b/drivers/net/ethernet/mellanox/mlx5/core/uar.c index 94464723ff77..0d006224d7b0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/uar.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/uar.c @@ -79,7 +79,7 @@ static u64 uar2pfn(struct mlx5_core_dev *mdev, u32 index) else system_page_index = index; - return (pci_resource_start(mdev->pdev, 0) >> PAGE_SHIFT) + system_page_index; + return (mdev->bar_addr >> PAGE_SHIFT) + system_page_index; } static void up_rel_func(struct kref *kref) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index ef95feca9961..95cdc8cbcba4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -371,67 +371,6 @@ int mlx5_modify_nic_vport_mac_list(struct mlx5_core_dev *dev, } EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_mac_list); -int mlx5_query_nic_vport_vlans(struct mlx5_core_dev *dev, - u16 vport, - u16 vlans[], - int *size) -{ - u32 in[MLX5_ST_SZ_DW(query_nic_vport_context_in)]; - void *nic_vport_ctx; - int req_list_size; - int max_list_size; - int out_sz; - void *out; - int err; - int i; - - req_list_size = *size; - max_list_size = 1 << MLX5_CAP_GEN(dev, log_max_vlan_list); - if (req_list_size > max_list_size) { - mlx5_core_warn(dev, "Requested list size (%d) > (%d) max list size\n", - req_list_size, max_list_size); - req_list_size = max_list_size; - } - - out_sz = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in) + - req_list_size * MLX5_ST_SZ_BYTES(vlan_layout); - - memset(in, 0, sizeof(in)); - out = kzalloc(out_sz, GFP_KERNEL); - if (!out) - return -ENOMEM; - - MLX5_SET(query_nic_vport_context_in, in, opcode, - MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT); - MLX5_SET(query_nic_vport_context_in, in, allowed_list_type, - MLX5_NVPRT_LIST_TYPE_VLAN); - MLX5_SET(query_nic_vport_context_in, in, vport_number, vport); - - if (vport) - MLX5_SET(query_nic_vport_context_in, in, other_vport, 1); - - err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); - if (err) - goto out; - - nic_vport_ctx = MLX5_ADDR_OF(query_nic_vport_context_out, out, - nic_vport_context); - req_list_size = MLX5_GET(nic_vport_context, nic_vport_ctx, - allowed_list_size); - - *size = req_list_size; - for (i = 0; i < req_list_size; i++) { - void *vlan_addr = MLX5_ADDR_OF(nic_vport_context, - nic_vport_ctx, - current_uc_mac_address[i]); - vlans[i] = MLX5_GET(vlan_layout, vlan_addr, vlan); - } -out: - kfree(out); - return err; -} -EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_vlans); - int mlx5_modify_nic_vport_vlans(struct mlx5_core_dev *dev, u16 vlans[], int list_size) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wq.h b/drivers/net/ethernet/mellanox/mlx5/core/wq.h index ea934a48c90a..1f87cce421e0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/wq.h @@ -134,6 +134,11 @@ static inline void mlx5_wq_cyc_update_db_record(struct mlx5_wq_cyc *wq) *wq->db = cpu_to_be32(wq->wqe_ctr); } +static inline u16 mlx5_wq_cyc_get_ctr_wrap_cnt(struct mlx5_wq_cyc *wq, u16 ctr) +{ + return ctr >> wq->fbc.log_sz; +} + static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr) { return ctr & wq->fbc.sz_m1; @@ -243,6 +248,13 @@ static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix) return mlx5_frag_buf_get_wqe(&wq->fbc, ix); } +static inline u16 mlx5_wq_ll_get_wqe_next_ix(struct mlx5_wq_ll *wq, u16 ix) +{ + struct mlx5_wqe_srq_next_seg *wqe = mlx5_wq_ll_get_wqe(wq, ix); + + return be16_to_cpu(wqe->next_wqe_index); +} + static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next) { wq->head = head_next; diff --git a/drivers/net/ethernet/mellanox/mlxsw/Kconfig b/drivers/net/ethernet/mellanox/mlxsw/Kconfig index 9c195dfed031..7ccb950aa7d4 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/Kconfig +++ b/drivers/net/ethernet/mellanox/mlxsw/Kconfig @@ -4,6 +4,7 @@ config MLXSW_CORE tristate "Mellanox Technologies Switch ASICs support" + select NET_DEVLINK ---help--- This driver supports Mellanox Technologies Switch ASICs family. @@ -21,7 +22,6 @@ config MLXSW_CORE_HWMON config MLXSW_CORE_THERMAL bool "Thermal zone support for Mellanox Technologies Switch ASICs" depends on MLXSW_CORE && THERMAL - depends on !(MLXSW_CORE=y && THERMAL=m) default y ---help--- Say Y here if you want to automatically control fans speed according diff --git a/drivers/net/ethernet/mellanox/mlxsw/Makefile b/drivers/net/ethernet/mellanox/mlxsw/Makefile index a01d15546e37..c4dc72e1ce63 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/Makefile +++ b/drivers/net/ethernet/mellanox/mlxsw/Makefile @@ -28,8 +28,8 @@ mlxsw_spectrum-objs := spectrum.o spectrum_buffers.o \ spectrum1_mr_tcam.o spectrum2_mr_tcam.o \ spectrum_mr_tcam.o spectrum_mr.o \ spectrum_qdisc.o spectrum_span.o \ - spectrum_nve.o spectrum_nve_vxlan.o + spectrum_nve.o spectrum_nve_vxlan.o \ + spectrum_dpipe.o mlxsw_spectrum-$(CONFIG_MLXSW_SPECTRUM_DCB) += spectrum_dcb.o -mlxsw_spectrum-$(CONFIG_NET_DEVLINK) += spectrum_dpipe.o obj-$(CONFIG_MLXSW_MINIMAL) += mlxsw_minimal.o mlxsw_minimal-objs := minimal.o diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index d23d53c0e284..bcbe07ec22be 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -568,7 +568,7 @@ static int mlxsw_emad_init(struct mlxsw_core *mlxsw_core) if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX)) return 0; - emad_wq = alloc_workqueue("mlxsw_core_emad", WQ_MEM_RECLAIM, 0); + emad_wq = alloc_workqueue("mlxsw_core_emad", 0, 0); if (!emad_wq) return -ENOMEM; mlxsw_core->emad_wq = emad_wq; @@ -781,7 +781,8 @@ mlxsw_devlink_sb_pool_get(struct devlink *devlink, static int mlxsw_devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index, u16 pool_index, u32 size, - enum devlink_sb_threshold_type threshold_type) + enum devlink_sb_threshold_type threshold_type, + struct netlink_ext_ack *extack) { struct mlxsw_core *mlxsw_core = devlink_priv(devlink); struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver; @@ -789,7 +790,8 @@ mlxsw_devlink_sb_pool_set(struct devlink *devlink, if (!mlxsw_driver->sb_pool_set) return -EOPNOTSUPP; return mlxsw_driver->sb_pool_set(mlxsw_core, sb_index, - pool_index, size, threshold_type); + pool_index, size, threshold_type, + extack); } static void *__dl_port(struct devlink_port *devlink_port) @@ -829,7 +831,8 @@ static int mlxsw_devlink_sb_port_pool_get(struct devlink_port *devlink_port, static int mlxsw_devlink_sb_port_pool_set(struct devlink_port *devlink_port, unsigned int sb_index, u16 pool_index, - u32 threshold) + u32 threshold, + struct netlink_ext_ack *extack) { struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink); struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver; @@ -839,7 +842,7 @@ static int mlxsw_devlink_sb_port_pool_set(struct devlink_port *devlink_port, !mlxsw_core_port_check(mlxsw_core_port)) return -EOPNOTSUPP; return mlxsw_driver->sb_port_pool_set(mlxsw_core_port, sb_index, - pool_index, threshold); + pool_index, threshold, extack); } static int @@ -864,7 +867,8 @@ static int mlxsw_devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port, unsigned int sb_index, u16 tc_index, enum devlink_sb_pool_type pool_type, - u16 pool_index, u32 threshold) + u16 pool_index, u32 threshold, + struct netlink_ext_ack *extack) { struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink); struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver; @@ -875,7 +879,7 @@ mlxsw_devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port, return -EOPNOTSUPP; return mlxsw_driver->sb_tc_pool_bind_set(mlxsw_core_port, sb_index, tc_index, pool_type, - pool_index, threshold); + pool_index, threshold, extack); } static int mlxsw_devlink_sb_occ_snapshot(struct devlink *devlink, @@ -934,6 +938,46 @@ mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port, pool_type, p_cur, p_max); } +static int +mlxsw_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, + struct netlink_ext_ack *extack) +{ + struct mlxsw_core *mlxsw_core = devlink_priv(devlink); + char fw_info_psid[MLXSW_REG_MGIR_FW_INFO_PSID_SIZE]; + u32 hw_rev, fw_major, fw_minor, fw_sub_minor; + char mgir_pl[MLXSW_REG_MGIR_LEN]; + char buf[32]; + int err; + + err = devlink_info_driver_name_put(req, + mlxsw_core->bus_info->device_kind); + if (err) + return err; + + mlxsw_reg_mgir_pack(mgir_pl); + err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mgir), mgir_pl); + if (err) + return err; + mlxsw_reg_mgir_unpack(mgir_pl, &hw_rev, fw_info_psid, &fw_major, + &fw_minor, &fw_sub_minor); + + sprintf(buf, "%X", hw_rev); + err = devlink_info_version_fixed_put(req, "hw.revision", buf); + if (err) + return err; + + err = devlink_info_version_fixed_put(req, "fw.psid", fw_info_psid); + if (err) + return err; + + sprintf(buf, "%d.%d.%d", fw_major, fw_minor, fw_sub_minor); + err = devlink_info_version_running_put(req, "fw.version", buf); + if (err) + return err; + + return 0; +} + static int mlxsw_devlink_core_bus_device_reload(struct devlink *devlink, struct netlink_ext_ack *extack) { @@ -968,6 +1012,7 @@ static const struct devlink_ops mlxsw_devlink_ops = { .sb_occ_max_clear = mlxsw_devlink_sb_occ_max_clear, .sb_occ_port_pool_get = mlxsw_devlink_sb_occ_port_pool_get, .sb_occ_tc_port_bind_get = mlxsw_devlink_sb_occ_tc_port_bind_get, + .info_get = mlxsw_devlink_info_get, }; static int @@ -1718,7 +1763,11 @@ u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core, } EXPORT_SYMBOL(mlxsw_core_res_get); -int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port) +int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, + u32 port_number, bool split, + u32 split_port_subnumber, + const unsigned char *switch_id, + unsigned char switch_id_len) { struct devlink *devlink = priv_to_devlink(mlxsw_core); struct mlxsw_core_port *mlxsw_core_port = @@ -1727,6 +1776,9 @@ int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port) int err; mlxsw_core_port->local_port = local_port; + devlink_port_attrs_set(devlink_port, DEVLINK_PORT_FLAVOUR_PHYSICAL, + port_number, split, split_port_subnumber, + switch_id, switch_id_len); err = devlink_port_register(devlink, devlink_port, local_port); if (err) memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port)); @@ -1746,17 +1798,13 @@ void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port) EXPORT_SYMBOL(mlxsw_core_port_fini); void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port, - void *port_driver_priv, struct net_device *dev, - u32 port_number, bool split, - u32 split_port_subnumber) + void *port_driver_priv, struct net_device *dev) { struct mlxsw_core_port *mlxsw_core_port = &mlxsw_core->ports[local_port]; struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port; mlxsw_core_port->port_driver_priv = port_driver_priv; - devlink_port_attrs_set(devlink_port, DEVLINK_PORT_FLAVOUR_PHYSICAL, - port_number, split, split_port_subnumber); devlink_port_type_eth_set(devlink_port, dev); } EXPORT_SYMBOL(mlxsw_core_port_eth_set); @@ -1796,16 +1844,18 @@ enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core, } EXPORT_SYMBOL(mlxsw_core_port_type_get); -int mlxsw_core_port_get_phys_port_name(struct mlxsw_core *mlxsw_core, - u8 local_port, char *name, size_t len) + +struct devlink_port * +mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core, + u8 local_port) { struct mlxsw_core_port *mlxsw_core_port = &mlxsw_core->ports[local_port]; struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port; - return devlink_port_get_phys_port_name(devlink_port, name, len); + return devlink_port; } -EXPORT_SYMBOL(mlxsw_core_port_get_phys_port_name); +EXPORT_SYMBOL(mlxsw_core_port_devlink_port_get); static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core, const char *buf, size_t size) @@ -1958,10 +2008,10 @@ static int __init mlxsw_core_module_init(void) { int err; - mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, WQ_MEM_RECLAIM, 0); + mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, 0, 0); if (!mlxsw_wq) return -ENOMEM; - mlxsw_owq = alloc_ordered_workqueue("%s_ordered", WQ_MEM_RECLAIM, + mlxsw_owq = alloc_ordered_workqueue("%s_ordered", 0, mlxsw_core_driver_name); if (!mlxsw_owq) { err = -ENOMEM; diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index 8ec53f027575..917be621c904 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -164,20 +164,23 @@ void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core, u16 lag_id, u8 local_port); void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port); -int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port); +int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, + u32 port_number, bool split, + u32 split_port_subnumber, + const unsigned char *switch_id, + unsigned char switch_id_len); void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port); void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port, - void *port_driver_priv, struct net_device *dev, - u32 port_number, bool split, - u32 split_port_subnumber); + void *port_driver_priv, struct net_device *dev); void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port, void *port_driver_priv); void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port, void *port_driver_priv); enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core, u8 local_port); -int mlxsw_core_port_get_phys_port_name(struct mlxsw_core *mlxsw_core, - u8 local_port, char *name, size_t len); +struct devlink_port * +mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core, + u8 local_port); int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay); bool mlxsw_core_schedule_work(struct work_struct *work); @@ -251,13 +254,14 @@ struct mlxsw_driver { struct devlink_sb_pool_info *pool_info); int (*sb_pool_set)(struct mlxsw_core *mlxsw_core, unsigned int sb_index, u16 pool_index, u32 size, - enum devlink_sb_threshold_type threshold_type); + enum devlink_sb_threshold_type threshold_type, + struct netlink_ext_ack *extack); int (*sb_port_pool_get)(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 pool_index, u32 *p_threshold); int (*sb_port_pool_set)(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 pool_index, - u32 threshold); + u32 threshold, struct netlink_ext_ack *extack); int (*sb_tc_pool_bind_get)(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 tc_index, enum devlink_sb_pool_type pool_type, @@ -265,7 +269,8 @@ struct mlxsw_driver { int (*sb_tc_pool_bind_set)(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 tc_index, enum devlink_sb_pool_type pool_type, - u16 pool_index, u32 threshold); + u16 pool_index, u32 threshold, + struct netlink_ext_ack *extack); int (*sb_occ_snapshot)(struct mlxsw_core *mlxsw_core, unsigned int sb_index); int (*sb_occ_max_clear)(struct mlxsw_core *mlxsw_core, diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c index 00c390024350..cf2114273b72 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c @@ -51,33 +51,20 @@ static int mlxsw_m_port_dummy_open_stop(struct net_device *dev) return 0; } -static int -mlxsw_m_port_get_phys_port_name(struct net_device *dev, char *name, size_t len) -{ - struct mlxsw_m_port *mlxsw_m_port = netdev_priv(dev); - struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core; - u8 local_port = mlxsw_m_port->local_port; - - return mlxsw_core_port_get_phys_port_name(core, local_port, name, len); -} - -static int mlxsw_m_port_get_port_parent_id(struct net_device *dev, - struct netdev_phys_item_id *ppid) +static struct devlink_port * +mlxsw_m_port_get_devlink_port(struct net_device *dev) { struct mlxsw_m_port *mlxsw_m_port = netdev_priv(dev); struct mlxsw_m *mlxsw_m = mlxsw_m_port->mlxsw_m; - ppid->id_len = sizeof(mlxsw_m->base_mac); - memcpy(&ppid->id, &mlxsw_m->base_mac, ppid->id_len); - - return 0; + return mlxsw_core_port_devlink_port_get(mlxsw_m->core, + mlxsw_m_port->local_port); } static const struct net_device_ops mlxsw_m_port_netdev_ops = { .ndo_open = mlxsw_m_port_dummy_open_stop, .ndo_stop = mlxsw_m_port_dummy_open_stop, - .ndo_get_phys_port_name = mlxsw_m_port_get_phys_port_name, - .ndo_get_port_parent_id = mlxsw_m_port_get_port_parent_id, + .ndo_get_devlink_port = mlxsw_m_port_get_devlink_port, }; static int mlxsw_m_get_module_info(struct net_device *netdev, @@ -150,7 +137,10 @@ mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u8 local_port, u8 module) struct net_device *dev; int err; - err = mlxsw_core_port_init(mlxsw_m->core, local_port); + err = mlxsw_core_port_init(mlxsw_m->core, local_port, + module + 1, false, 0, + mlxsw_m->base_mac, + sizeof(mlxsw_m->base_mac)); if (err) { dev_err(mlxsw_m->bus_info->dev, "Port %d: Failed to init core port\n", local_port); @@ -190,7 +180,7 @@ mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u8 local_port, u8 module) } mlxsw_core_port_eth_set(mlxsw_m->core, mlxsw_m_port->local_port, - mlxsw_m_port, dev, module + 1, false, 0); + mlxsw_m_port, dev); return 0; diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h index ffee38e36ce8..8648ca171254 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h +++ b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h @@ -27,7 +27,7 @@ #define MLXSW_PCI_SW_RESET 0xF0010 #define MLXSW_PCI_SW_RESET_RST_BIT BIT(0) -#define MLXSW_PCI_SW_RESET_TIMEOUT_MSECS 13000 +#define MLXSW_PCI_SW_RESET_TIMEOUT_MSECS 20000 #define MLXSW_PCI_SW_RESET_WAIT_MSECS 100 #define MLXSW_PCI_FW_READY 0xA1844 #define MLXSW_PCI_FW_READY_MASK 0xFFFF diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index eb4c5e8964cd..e8002bfc1e8f 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -5210,6 +5210,42 @@ static inline void mlxsw_reg_pspa_pack(char *payload, u8 swid, u8 local_port) mlxsw_reg_pspa_sub_port_set(payload, 0); } +/* PPLR - Port Physical Loopback Register + * -------------------------------------- + * This register allows configuration of the port's loopback mode. + */ +#define MLXSW_REG_PPLR_ID 0x5018 +#define MLXSW_REG_PPLR_LEN 0x8 + +MLXSW_REG_DEFINE(pplr, MLXSW_REG_PPLR_ID, MLXSW_REG_PPLR_LEN); + +/* reg_pplr_local_port + * Local port number. + * Access: Index + */ +MLXSW_ITEM32(reg, pplr, local_port, 0x00, 16, 8); + +/* Phy local loopback. When set the port's egress traffic is looped back + * to the receiver and the port transmitter is disabled. + */ +#define MLXSW_REG_PPLR_LB_TYPE_BIT_PHY_LOCAL BIT(1) + +/* reg_pplr_lb_en + * Loopback enable. + * Access: RW + */ +MLXSW_ITEM32(reg, pplr, lb_en, 0x04, 0, 8); + +static inline void mlxsw_reg_pplr_pack(char *payload, u8 local_port, + bool phy_local) +{ + MLXSW_REG_ZERO(pplr, payload); + mlxsw_reg_pplr_local_port_set(payload, local_port); + mlxsw_reg_pplr_lb_en_set(payload, + phy_local ? + MLXSW_REG_PPLR_LB_TYPE_BIT_PHY_LOCAL : 0); +} + /* HTGT - Host Trap Group Table * ---------------------------- * Configures the properties for forwarding to CPU. @@ -8534,6 +8570,60 @@ static inline void mlxsw_reg_mpar_pack(char *payload, u8 local_port, mlxsw_reg_mpar_pa_id_set(payload, pa_id); } +/* MGIR - Management General Information Register + * ---------------------------------------------- + * MGIR register allows software to query the hardware and firmware general + * information. + */ +#define MLXSW_REG_MGIR_ID 0x9020 +#define MLXSW_REG_MGIR_LEN 0x9C + +MLXSW_REG_DEFINE(mgir, MLXSW_REG_MGIR_ID, MLXSW_REG_MGIR_LEN); + +/* reg_mgir_hw_info_device_hw_revision + * Access: RO + */ +MLXSW_ITEM32(reg, mgir, hw_info_device_hw_revision, 0x0, 16, 16); + +#define MLXSW_REG_MGIR_FW_INFO_PSID_SIZE 16 + +/* reg_mgir_fw_info_psid + * PSID (ASCII string). + * Access: RO + */ +MLXSW_ITEM_BUF(reg, mgir, fw_info_psid, 0x30, MLXSW_REG_MGIR_FW_INFO_PSID_SIZE); + +/* reg_mgir_fw_info_extended_major + * Access: RO + */ +MLXSW_ITEM32(reg, mgir, fw_info_extended_major, 0x44, 0, 32); + +/* reg_mgir_fw_info_extended_minor + * Access: RO + */ +MLXSW_ITEM32(reg, mgir, fw_info_extended_minor, 0x48, 0, 32); + +/* reg_mgir_fw_info_extended_sub_minor + * Access: RO + */ +MLXSW_ITEM32(reg, mgir, fw_info_extended_sub_minor, 0x4C, 0, 32); + +static inline void mlxsw_reg_mgir_pack(char *payload) +{ + MLXSW_REG_ZERO(mgir, payload); +} + +static inline void +mlxsw_reg_mgir_unpack(char *payload, u32 *hw_rev, char *fw_info_psid, + u32 *fw_major, u32 *fw_minor, u32 *fw_sub_minor) +{ + *hw_rev = mlxsw_reg_mgir_hw_info_device_hw_revision_get(payload); + mlxsw_reg_mgir_fw_info_psid_memcpy_from(payload, fw_info_psid); + *fw_major = mlxsw_reg_mgir_fw_info_extended_major_get(payload); + *fw_minor = mlxsw_reg_mgir_fw_info_extended_minor_get(payload); + *fw_sub_minor = mlxsw_reg_mgir_fw_info_extended_sub_minor_get(payload); +} + /* MRSR - Management Reset and Shutdown Register * --------------------------------------------- * MRSR register is used to reset or shutdown the switch or @@ -9927,6 +10017,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(pptb), MLXSW_REG(pbmc), MLXSW_REG(pspa), + MLXSW_REG(pplr), MLXSW_REG(htgt), MLXSW_REG(hpkt), MLXSW_REG(rgcr), @@ -9958,6 +10049,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(mcia), MLXSW_REG(mpat), MLXSW_REG(mpar), + MLXSW_REG(mgir), MLXSW_REG(mrsr), MLXSW_REG(mlcr), MLXSW_REG(mpsc), diff --git a/drivers/net/ethernet/mellanox/mlxsw/resources.h b/drivers/net/ethernet/mellanox/mlxsw/resources.h index 773ef7fdb285..33a9fc9ef6a4 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/resources.h +++ b/drivers/net/ethernet/mellanox/mlxsw/resources.h @@ -24,6 +24,8 @@ enum mlxsw_res_id { MLXSW_RES_ID_MAX_SYSTEM_PORT, MLXSW_RES_ID_MAX_LAG, MLXSW_RES_ID_MAX_LAG_MEMBERS, + MLXSW_RES_ID_LOCAL_PORTS_IN_1X, + MLXSW_RES_ID_LOCAL_PORTS_IN_2X, MLXSW_RES_ID_MAX_BUFFER_SIZE, MLXSW_RES_ID_CELL_SIZE, MLXSW_RES_ID_MAX_HEADROOM_SIZE, @@ -78,6 +80,8 @@ static u16 mlxsw_res_ids[] = { [MLXSW_RES_ID_MAX_SYSTEM_PORT] = 0x2502, [MLXSW_RES_ID_MAX_LAG] = 0x2520, [MLXSW_RES_ID_MAX_LAG_MEMBERS] = 0x2521, + [MLXSW_RES_ID_LOCAL_PORTS_IN_1X] = 0x2610, + [MLXSW_RES_ID_LOCAL_PORTS_IN_2X] = 0x2611, [MLXSW_RES_ID_MAX_BUFFER_SIZE] = 0x2802, /* Bytes */ [MLXSW_RES_ID_CELL_SIZE] = 0x2803, /* Bytes */ [MLXSW_RES_ID_MAX_HEADROOM_SIZE] = 0x2811, /* Bytes */ diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 9eb63300c1d3..dbb425717f5e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -21,7 +21,7 @@ #include <linux/dcbnl.h> #include <linux/inetdevice.h> #include <linux/netlink.h> -#include <linux/random.h> +#include <linux/jhash.h> #include <net/switchdev.h> #include <net/pkt_cls.h> #include <net/tc_act/tc_mirred.h> @@ -46,8 +46,8 @@ #define MLXSW_SP_FWREV_MINOR_TO_BRANCH(minor) ((minor) / 100) #define MLXSW_SP1_FWREV_MAJOR 13 -#define MLXSW_SP1_FWREV_MINOR 1910 -#define MLXSW_SP1_FWREV_SUBMINOR 622 +#define MLXSW_SP1_FWREV_MINOR 2000 +#define MLXSW_SP1_FWREV_SUBMINOR 1122 #define MLXSW_SP1_FWREV_CAN_RESET_MINOR 1702 static const struct mlxsw_fw_rev mlxsw_sp1_fw_rev = { @@ -1254,16 +1254,6 @@ static int mlxsw_sp_port_kill_vid(struct net_device *dev, return 0; } -static int mlxsw_sp_port_get_phys_port_name(struct net_device *dev, char *name, - size_t len) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - - return mlxsw_core_port_get_phys_port_name(mlxsw_sp_port->mlxsw_sp->core, - mlxsw_sp_port->local_port, - name, len); -} - static struct mlxsw_sp_port_mall_tc_entry * mlxsw_sp_port_mall_tc_entry_find(struct mlxsw_sp_port *port, unsigned long cookie) { @@ -1279,21 +1269,19 @@ mlxsw_sp_port_mall_tc_entry_find(struct mlxsw_sp_port *port, static int mlxsw_sp_port_add_cls_matchall_mirror(struct mlxsw_sp_port *mlxsw_sp_port, struct mlxsw_sp_port_mall_mirror_tc_entry *mirror, - const struct tc_action *a, + const struct flow_action_entry *act, bool ingress) { enum mlxsw_sp_span_type span_type; - struct net_device *to_dev; - to_dev = tcf_mirred_dev(a); - if (!to_dev) { + if (!act->dev) { netdev_err(mlxsw_sp_port->dev, "Could not find requested device\n"); return -EINVAL; } mirror->ingress = ingress; span_type = ingress ? MLXSW_SP_SPAN_INGRESS : MLXSW_SP_SPAN_EGRESS; - return mlxsw_sp_span_mirror_add(mlxsw_sp_port, to_dev, span_type, + return mlxsw_sp_span_mirror_add(mlxsw_sp_port, act->dev, span_type, true, &mirror->span_id); } @@ -1312,7 +1300,7 @@ mlxsw_sp_port_del_cls_matchall_mirror(struct mlxsw_sp_port *mlxsw_sp_port, static int mlxsw_sp_port_add_cls_matchall_sample(struct mlxsw_sp_port *mlxsw_sp_port, struct tc_cls_matchall_offload *cls, - const struct tc_action *a, + const struct flow_action_entry *act, bool ingress) { int err; @@ -1323,18 +1311,18 @@ mlxsw_sp_port_add_cls_matchall_sample(struct mlxsw_sp_port *mlxsw_sp_port, netdev_err(mlxsw_sp_port->dev, "sample already active\n"); return -EEXIST; } - if (tcf_sample_rate(a) > MLXSW_REG_MPSC_RATE_MAX) { + if (act->sample.rate > MLXSW_REG_MPSC_RATE_MAX) { netdev_err(mlxsw_sp_port->dev, "sample rate not supported\n"); return -EOPNOTSUPP; } rcu_assign_pointer(mlxsw_sp_port->sample->psample_group, - tcf_sample_psample_group(a)); - mlxsw_sp_port->sample->truncate = tcf_sample_truncate(a); - mlxsw_sp_port->sample->trunc_size = tcf_sample_trunc_size(a); - mlxsw_sp_port->sample->rate = tcf_sample_rate(a); + act->sample.psample_group); + mlxsw_sp_port->sample->truncate = act->sample.truncate; + mlxsw_sp_port->sample->trunc_size = act->sample.trunc_size; + mlxsw_sp_port->sample->rate = act->sample.rate; - err = mlxsw_sp_port_sample_set(mlxsw_sp_port, true, tcf_sample_rate(a)); + err = mlxsw_sp_port_sample_set(mlxsw_sp_port, true, act->sample.rate); if (err) goto err_port_sample_set; return 0; @@ -1360,10 +1348,10 @@ static int mlxsw_sp_port_add_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port, { struct mlxsw_sp_port_mall_tc_entry *mall_tc_entry; __be16 protocol = f->common.protocol; - const struct tc_action *a; + struct flow_action_entry *act; int err; - if (!tcf_exts_has_one_action(f->exts)) { + if (!flow_offload_has_one_action(&f->rule->action)) { netdev_err(mlxsw_sp_port->dev, "only singular actions are supported\n"); return -EOPNOTSUPP; } @@ -1373,19 +1361,21 @@ static int mlxsw_sp_port_add_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port, return -ENOMEM; mall_tc_entry->cookie = f->cookie; - a = tcf_exts_first_action(f->exts); + act = &f->rule->action.entries[0]; - if (is_tcf_mirred_egress_mirror(a) && protocol == htons(ETH_P_ALL)) { + if (act->id == FLOW_ACTION_MIRRED && protocol == htons(ETH_P_ALL)) { struct mlxsw_sp_port_mall_mirror_tc_entry *mirror; mall_tc_entry->type = MLXSW_SP_PORT_MALL_MIRROR; mirror = &mall_tc_entry->mirror; err = mlxsw_sp_port_add_cls_matchall_mirror(mlxsw_sp_port, - mirror, a, ingress); - } else if (is_tcf_sample(a) && protocol == htons(ETH_P_ALL)) { + mirror, act, + ingress); + } else if (act->id == FLOW_ACTION_SAMPLE && + protocol == htons(ETH_P_ALL)) { mall_tc_entry->type = MLXSW_SP_PORT_MALL_SAMPLE; err = mlxsw_sp_port_add_cls_matchall_sample(mlxsw_sp_port, f, - a, ingress); + act, ingress); } else { err = -EOPNOTSUPP; } @@ -1679,6 +1669,25 @@ static int mlxsw_sp_feature_hw_tc(struct net_device *dev, bool enable) return 0; } +static int mlxsw_sp_feature_loopback(struct net_device *dev, bool enable) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + char pplr_pl[MLXSW_REG_PPLR_LEN]; + int err; + + if (netif_running(dev)) + mlxsw_sp_port_admin_status_set(mlxsw_sp_port, false); + + mlxsw_reg_pplr_pack(pplr_pl, mlxsw_sp_port->local_port, enable); + err = mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(pplr), + pplr_pl); + + if (netif_running(dev)) + mlxsw_sp_port_admin_status_set(mlxsw_sp_port, true); + + return err; +} + typedef int (*mlxsw_sp_feature_handler)(struct net_device *dev, bool enable); static int mlxsw_sp_handle_feature(struct net_device *dev, @@ -1710,20 +1719,30 @@ static int mlxsw_sp_handle_feature(struct net_device *dev, static int mlxsw_sp_set_features(struct net_device *dev, netdev_features_t features) { - return mlxsw_sp_handle_feature(dev, features, NETIF_F_HW_TC, + netdev_features_t oper_features = dev->features; + int err = 0; + + err |= mlxsw_sp_handle_feature(dev, features, NETIF_F_HW_TC, mlxsw_sp_feature_hw_tc); + err |= mlxsw_sp_handle_feature(dev, features, NETIF_F_LOOPBACK, + mlxsw_sp_feature_loopback); + + if (err) { + dev->features = oper_features; + return -EINVAL; + } + + return 0; } -static int mlxsw_sp_port_get_port_parent_id(struct net_device *dev, - struct netdev_phys_item_id *ppid) +static struct devlink_port * +mlxsw_sp_port_get_devlink_port(struct net_device *dev) { struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - ppid->id_len = sizeof(mlxsw_sp->base_mac); - memcpy(&ppid->id, &mlxsw_sp->base_mac, ppid->id_len); - - return 0; + return mlxsw_core_port_devlink_port_get(mlxsw_sp->core, + mlxsw_sp_port->local_port); } static const struct net_device_ops mlxsw_sp_port_netdev_ops = { @@ -1739,9 +1758,8 @@ static const struct net_device_ops mlxsw_sp_port_netdev_ops = { .ndo_get_offload_stats = mlxsw_sp_port_get_offload_stats, .ndo_vlan_rx_add_vid = mlxsw_sp_port_add_vid, .ndo_vlan_rx_kill_vid = mlxsw_sp_port_kill_vid, - .ndo_get_phys_port_name = mlxsw_sp_port_get_phys_port_name, .ndo_set_features = mlxsw_sp_set_features, - .ndo_get_port_parent_id = mlxsw_sp_port_get_port_parent_id, + .ndo_get_devlink_port = mlxsw_sp_port_get_devlink_port, }; static void mlxsw_sp_port_get_drvinfo(struct net_device *dev, @@ -3126,11 +3144,11 @@ mlxsw_sp_port_set_link_ksettings(struct net_device *dev, if (err) return err; + mlxsw_sp_port->link.autoneg = autoneg; + if (!netif_running(dev)) return 0; - mlxsw_sp_port->link.autoneg = autoneg; - mlxsw_sp_port_admin_status_set(mlxsw_sp_port, false); mlxsw_sp_port_admin_status_set(mlxsw_sp_port, true); @@ -3316,7 +3334,7 @@ static int mlxsw_sp_port_ets_init(struct mlxsw_sp_port *mlxsw_sp_port) err = mlxsw_sp_port_ets_set(mlxsw_sp_port, MLXSW_REG_QEEC_HIERARCY_TC, i + 8, i, - false, 0); + true, 100); if (err) return err; } @@ -3391,7 +3409,10 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port, struct net_device *dev; int err; - err = mlxsw_core_port_init(mlxsw_sp->core, local_port); + err = mlxsw_core_port_init(mlxsw_sp->core, local_port, + module + 1, split, lane / width, + mlxsw_sp->base_mac, + sizeof(mlxsw_sp->base_mac)); if (err) { dev_err(mlxsw_sp->bus_info->dev, "Port %d: Failed to init core port\n", local_port); @@ -3462,7 +3483,7 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port, dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_LLTX | NETIF_F_SG | NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_TC; - dev->hw_features |= NETIF_F_HW_TC; + dev->hw_features |= NETIF_F_HW_TC | NETIF_F_LOOPBACK; dev->min_mtu = 0; dev->max_mtu = ETH_MAX_MTU; @@ -3573,8 +3594,7 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port, } mlxsw_core_port_eth_set(mlxsw_sp->core, mlxsw_sp_port->local_port, - mlxsw_sp_port, dev, module + 1, - mlxsw_sp_port->split, lane / width); + mlxsw_sp_port, dev); mlxsw_core_schedule_dw(&mlxsw_sp_port->periodic_hw_stats.update_dw, 0); return 0; @@ -3710,14 +3730,14 @@ static u8 mlxsw_sp_cluster_base_port_get(u8 local_port) } static int mlxsw_sp_port_split_create(struct mlxsw_sp *mlxsw_sp, u8 base_port, - u8 module, unsigned int count) + u8 module, unsigned int count, u8 offset) { u8 width = MLXSW_PORT_MODULE_MAX_WIDTH / count; int err, i; for (i = 0; i < count; i++) { - err = mlxsw_sp_port_create(mlxsw_sp, base_port + i, true, - module, width, i * width); + err = mlxsw_sp_port_create(mlxsw_sp, base_port + i * offset, + true, module, width, i * width); if (err) goto err_port_create; } @@ -3726,8 +3746,8 @@ static int mlxsw_sp_port_split_create(struct mlxsw_sp *mlxsw_sp, u8 base_port, err_port_create: for (i--; i >= 0; i--) - if (mlxsw_sp_port_created(mlxsw_sp, base_port + i)) - mlxsw_sp_port_remove(mlxsw_sp, base_port + i); + if (mlxsw_sp_port_created(mlxsw_sp, base_port + i * offset)) + mlxsw_sp_port_remove(mlxsw_sp, base_port + i * offset); return err; } @@ -3758,11 +3778,19 @@ static int mlxsw_sp_port_split(struct mlxsw_core *mlxsw_core, u8 local_port, struct netlink_ext_ack *extack) { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); + u8 local_ports_in_1x, local_ports_in_2x, offset; struct mlxsw_sp_port *mlxsw_sp_port; u8 module, cur_width, base_port; int i; int err; + if (!MLXSW_CORE_RES_VALID(mlxsw_core, LOCAL_PORTS_IN_1X) || + !MLXSW_CORE_RES_VALID(mlxsw_core, LOCAL_PORTS_IN_2X)) + return -EIO; + + local_ports_in_1x = MLXSW_CORE_RES_GET(mlxsw_core, LOCAL_PORTS_IN_1X); + local_ports_in_2x = MLXSW_CORE_RES_GET(mlxsw_core, LOCAL_PORTS_IN_2X); + mlxsw_sp_port = mlxsw_sp->ports[local_port]; if (!mlxsw_sp_port) { dev_err(mlxsw_sp->bus_info->dev, "Port number \"%d\" does not exist\n", @@ -3788,13 +3816,15 @@ static int mlxsw_sp_port_split(struct mlxsw_core *mlxsw_core, u8 local_port, /* Make sure we have enough slave (even) ports for the split. */ if (count == 2) { + offset = local_ports_in_2x; base_port = local_port; - if (mlxsw_sp->ports[base_port + 1]) { + if (mlxsw_sp->ports[base_port + local_ports_in_2x]) { netdev_err(mlxsw_sp_port->dev, "Invalid split configuration\n"); NL_SET_ERR_MSG_MOD(extack, "Invalid split configuration"); return -EINVAL; } } else { + offset = local_ports_in_1x; base_port = mlxsw_sp_cluster_base_port_get(local_port); if (mlxsw_sp->ports[base_port + 1] || mlxsw_sp->ports[base_port + 3]) { @@ -3805,10 +3835,11 @@ static int mlxsw_sp_port_split(struct mlxsw_core *mlxsw_core, u8 local_port, } for (i = 0; i < count; i++) - if (mlxsw_sp_port_created(mlxsw_sp, base_port + i)) - mlxsw_sp_port_remove(mlxsw_sp, base_port + i); + if (mlxsw_sp_port_created(mlxsw_sp, base_port + i * offset)) + mlxsw_sp_port_remove(mlxsw_sp, base_port + i * offset); - err = mlxsw_sp_port_split_create(mlxsw_sp, base_port, module, count); + err = mlxsw_sp_port_split_create(mlxsw_sp, base_port, module, count, + offset); if (err) { dev_err(mlxsw_sp->bus_info->dev, "Failed to create split ports\n"); goto err_port_split_create; @@ -3825,11 +3856,19 @@ static int mlxsw_sp_port_unsplit(struct mlxsw_core *mlxsw_core, u8 local_port, struct netlink_ext_ack *extack) { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); + u8 local_ports_in_1x, local_ports_in_2x, offset; struct mlxsw_sp_port *mlxsw_sp_port; u8 cur_width, base_port; unsigned int count; int i; + if (!MLXSW_CORE_RES_VALID(mlxsw_core, LOCAL_PORTS_IN_1X) || + !MLXSW_CORE_RES_VALID(mlxsw_core, LOCAL_PORTS_IN_2X)) + return -EIO; + + local_ports_in_1x = MLXSW_CORE_RES_GET(mlxsw_core, LOCAL_PORTS_IN_1X); + local_ports_in_2x = MLXSW_CORE_RES_GET(mlxsw_core, LOCAL_PORTS_IN_2X); + mlxsw_sp_port = mlxsw_sp->ports[local_port]; if (!mlxsw_sp_port) { dev_err(mlxsw_sp->bus_info->dev, "Port number \"%d\" does not exist\n", @@ -3847,6 +3886,11 @@ static int mlxsw_sp_port_unsplit(struct mlxsw_core *mlxsw_core, u8 local_port, cur_width = mlxsw_sp_port->mapping.width; count = cur_width == 1 ? 4 : 2; + if (count == 2) + offset = local_ports_in_2x; + else + offset = local_ports_in_1x; + base_port = mlxsw_sp_cluster_base_port_get(local_port); /* Determine which ports to remove. */ @@ -3854,8 +3898,8 @@ static int mlxsw_sp_port_unsplit(struct mlxsw_core *mlxsw_core, u8 local_port, base_port = base_port + 2; for (i = 0; i < count; i++) - if (mlxsw_sp_port_created(mlxsw_sp, base_port + i)) - mlxsw_sp_port_remove(mlxsw_sp, base_port + i); + if (mlxsw_sp_port_created(mlxsw_sp, base_port + i * offset)) + mlxsw_sp_port_remove(mlxsw_sp, base_port + i * offset); mlxsw_sp_port_unsplit_create(mlxsw_sp, base_port, count); @@ -4238,7 +4282,7 @@ static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp) u32 seed; int err; - get_random_bytes(&seed, sizeof(seed)); + seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); mlxsw_reg_slcr_pack(slcr_pl, MLXSW_REG_SLCR_LAG_HASH_SMAC | MLXSW_REG_SLCR_LAG_HASH_DMAC | MLXSW_REG_SLCR_LAG_HASH_ETHERTYPE | diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h index da6278b0caa4..8601b3041acd 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h @@ -371,13 +371,14 @@ int mlxsw_sp_sb_pool_get(struct mlxsw_core *mlxsw_core, struct devlink_sb_pool_info *pool_info); int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core, unsigned int sb_index, u16 pool_index, u32 size, - enum devlink_sb_threshold_type threshold_type); + enum devlink_sb_threshold_type threshold_type, + struct netlink_ext_ack *extack); int mlxsw_sp_sb_port_pool_get(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 pool_index, u32 *p_threshold); int mlxsw_sp_sb_port_pool_set(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 pool_index, - u32 threshold); + u32 threshold, struct netlink_ext_ack *extack); int mlxsw_sp_sb_tc_pool_bind_get(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 tc_index, enum devlink_sb_pool_type pool_type, @@ -385,7 +386,8 @@ int mlxsw_sp_sb_tc_pool_bind_get(struct mlxsw_core_port *mlxsw_core_port, int mlxsw_sp_sb_tc_pool_bind_set(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 tc_index, enum devlink_sb_pool_type pool_type, - u16 pool_index, u32 threshold); + u16 pool_index, u32 threshold, + struct netlink_ext_ack *extack); int mlxsw_sp_sb_occ_snapshot(struct mlxsw_core *mlxsw_core, unsigned int sb_index); int mlxsw_sp_sb_occ_max_clear(struct mlxsw_core *mlxsw_core, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c index 8811f6513e36..e993159e8e4c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c @@ -216,7 +216,6 @@ struct mlxsw_sp_acl_tcam_vregion { struct mlxsw_sp_acl_tcam_rehash_ctx ctx; } rehash; struct mlxsw_sp *mlxsw_sp; - bool failed_rollback; /* Indicates failed rollback during migration */ unsigned int ref_count; }; @@ -1256,11 +1255,8 @@ mlxsw_sp_acl_tcam_vchunk_migrate_start(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_tcam_chunk *new_chunk; new_chunk = mlxsw_sp_acl_tcam_chunk_create(mlxsw_sp, vchunk, region); - if (IS_ERR(new_chunk)) { - if (ctx->this_is_rollback) - vchunk->vregion->failed_rollback = true; + if (IS_ERR(new_chunk)) return PTR_ERR(new_chunk); - } vchunk->chunk2 = vchunk->chunk; vchunk->chunk = new_chunk; ctx->current_vchunk = vchunk; @@ -1318,8 +1314,13 @@ mlxsw_sp_acl_tcam_vchunk_migrate_one(struct mlxsw_sp *mlxsw_sp, err = mlxsw_sp_acl_tcam_ventry_migrate(mlxsw_sp, ventry, vchunk->chunk, credits); if (err) { - if (ctx->this_is_rollback) + if (ctx->this_is_rollback) { + /* Save the ventry which we ended with and try + * to continue later on. + */ + ctx->start_ventry = ventry; return err; + } /* Swap the chunk and chunk2 pointers so the follow-up * rollback call will see the original chunk pointer * in vchunk->chunk. @@ -1397,8 +1398,12 @@ mlxsw_sp_acl_tcam_vregion_migrate(struct mlxsw_sp *mlxsw_sp, ctx->this_is_rollback = true; err2 = mlxsw_sp_acl_tcam_vchunk_migrate_all(mlxsw_sp, vregion, ctx, credits); - if (err2) - vregion->failed_rollback = true; + if (err2) { + trace_mlxsw_sp_acl_tcam_vregion_rehash_rollback_failed(mlxsw_sp, + vregion); + dev_err(mlxsw_sp->bus_info->dev, "Failed to rollback during vregion migration fail\n"); + /* Let the rollback to be continued later on. */ + } } mutex_unlock(&vregion->lock); trace_mlxsw_sp_acl_tcam_vregion_migrate_end(mlxsw_sp, vregion); @@ -1423,8 +1428,6 @@ mlxsw_sp_acl_tcam_vregion_rehash_start(struct mlxsw_sp *mlxsw_sp, int err; trace_mlxsw_sp_acl_tcam_vregion_rehash(mlxsw_sp, vregion); - if (vregion->failed_rollback) - return -EBUSY; hints_priv = ops->region_rehash_hints_get(vregion->region->priv); if (IS_ERR(hints_priv)) @@ -1471,11 +1474,9 @@ mlxsw_sp_acl_tcam_vregion_rehash_end(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_tcam_region *unused_region = vregion->region2; const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; - if (!vregion->failed_rollback) { - vregion->region2 = NULL; - mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp, unused_region); - mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, unused_region); - } + vregion->region2 = NULL; + mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp, unused_region); + mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, unused_region); ops->region_rehash_hints_put(ctx->hints_priv); ctx->hints_priv = NULL; } @@ -1506,11 +1507,6 @@ mlxsw_sp_acl_tcam_vregion_rehash(struct mlxsw_sp *mlxsw_sp, ctx, credits); if (err) { dev_err(mlxsw_sp->bus_info->dev, "Failed to migrate vregion\n"); - if (vregion->failed_rollback) { - trace_mlxsw_sp_acl_tcam_vregion_rehash_dis(mlxsw_sp, - vregion); - dev_err(mlxsw_sp->bus_info->dev, "Failed to rollback during vregion migration fail\n"); - } } if (*credits >= 0) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c index 9a79b5e11597..8512dd49e420 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c @@ -6,6 +6,7 @@ #include <linux/dcbnl.h> #include <linux/if_ether.h> #include <linux/list.h> +#include <linux/netlink.h> #include "spectrum.h" #include "core.h" @@ -15,6 +16,8 @@ struct mlxsw_sp_sb_pr { enum mlxsw_reg_sbpr_mode mode; u32 size; + u8 freeze_mode:1, + freeze_size:1; }; struct mlxsw_cp_sb_occ { @@ -27,6 +30,8 @@ struct mlxsw_sp_sb_cm { u32 max_buff; u16 pool_index; struct mlxsw_cp_sb_occ occ; + u8 freeze_pool:1, + freeze_thresh:1; }; #define MLXSW_SP_SB_INFI -1U @@ -48,7 +53,12 @@ struct mlxsw_sp_sb_pool_des { u8 pool; }; -/* Order ingress pools before egress pools. */ +#define MLXSW_SP_SB_POOL_ING 0 +#define MLXSW_SP_SB_POOL_EGR 4 +#define MLXSW_SP_SB_POOL_EGR_MC 8 +#define MLXSW_SP_SB_POOL_ING_CPU 9 +#define MLXSW_SP_SB_POOL_EGR_CPU 10 + static const struct mlxsw_sp_sb_pool_des mlxsw_sp1_sb_pool_dess[] = { {MLXSW_REG_SBXX_DIR_INGRESS, 0}, {MLXSW_REG_SBXX_DIR_INGRESS, 1}, @@ -59,6 +69,8 @@ static const struct mlxsw_sp_sb_pool_des mlxsw_sp1_sb_pool_dess[] = { {MLXSW_REG_SBXX_DIR_EGRESS, 2}, {MLXSW_REG_SBXX_DIR_EGRESS, 3}, {MLXSW_REG_SBXX_DIR_EGRESS, 15}, + {MLXSW_REG_SBXX_DIR_INGRESS, 4}, + {MLXSW_REG_SBXX_DIR_EGRESS, 4}, }; static const struct mlxsw_sp_sb_pool_des mlxsw_sp2_sb_pool_dess[] = { @@ -70,6 +82,9 @@ static const struct mlxsw_sp_sb_pool_des mlxsw_sp2_sb_pool_dess[] = { {MLXSW_REG_SBXX_DIR_EGRESS, 1}, {MLXSW_REG_SBXX_DIR_EGRESS, 2}, {MLXSW_REG_SBXX_DIR_EGRESS, 3}, + {MLXSW_REG_SBXX_DIR_EGRESS, 15}, + {MLXSW_REG_SBXX_DIR_INGRESS, 4}, + {MLXSW_REG_SBXX_DIR_EGRESS, 4}, }; #define MLXSW_SP_SB_ING_TC_COUNT 8 @@ -93,6 +108,7 @@ struct mlxsw_sp_sb_vals { unsigned int pool_count; const struct mlxsw_sp_sb_pool_des *pool_dess; const struct mlxsw_sp_sb_pm *pms; + const struct mlxsw_sp_sb_pm *pms_cpu; const struct mlxsw_sp_sb_pr *prs; const struct mlxsw_sp_sb_mm *mms; const struct mlxsw_sp_sb_cm *cms_ingress; @@ -274,7 +290,7 @@ static int mlxsw_sp_port_pb_init(struct mlxsw_sp_port *mlxsw_sp_port) { const u32 pbs[] = { [0] = MLXSW_SP_PB_HEADROOM * mlxsw_sp_port->mapping.width, - [9] = 2 * MLXSW_PORT_MAX_MTU, + [9] = MLXSW_PORT_MAX_MTU, }; struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; char pbmc_pl[MLXSW_REG_PBMC_LEN]; @@ -389,45 +405,60 @@ static void mlxsw_sp_sb_ports_fini(struct mlxsw_sp *mlxsw_sp) .size = _size, \ } +#define MLXSW_SP_SB_PR_EXT(_mode, _size, _freeze_mode, _freeze_size) \ + { \ + .mode = _mode, \ + .size = _size, \ + .freeze_mode = _freeze_mode, \ + .freeze_size = _freeze_size, \ + } + #define MLXSW_SP1_SB_PR_INGRESS_SIZE 12440000 -#define MLXSW_SP1_SB_PR_INGRESS_MNG_SIZE (200 * 1000) #define MLXSW_SP1_SB_PR_EGRESS_SIZE 13232000 +#define MLXSW_SP1_SB_PR_CPU_SIZE (256 * 1000) +/* Order according to mlxsw_sp1_sb_pool_dess */ static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = { - /* Ingress pools. */ MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP1_SB_PR_INGRESS_SIZE), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), - MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, - MLXSW_SP1_SB_PR_INGRESS_MNG_SIZE), - /* Egress pools. */ - MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, - MLXSW_SP1_SB_PR_EGRESS_SIZE), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, + MLXSW_SP1_SB_PR_EGRESS_SIZE, true, false), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), - MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_STATIC, MLXSW_SP_SB_INFI, + true, true), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, + MLXSW_SP1_SB_PR_CPU_SIZE, true, false), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, + MLXSW_SP1_SB_PR_CPU_SIZE, true, false), }; #define MLXSW_SP2_SB_PR_INGRESS_SIZE 40960000 -#define MLXSW_SP2_SB_PR_INGRESS_MNG_SIZE (200 * 1000) #define MLXSW_SP2_SB_PR_EGRESS_SIZE 40960000 +#define MLXSW_SP2_SB_PR_CPU_SIZE (256 * 1000) +/* Order according to mlxsw_sp2_sb_pool_dess */ static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = { - /* Ingress pools. */ MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP2_SB_PR_INGRESS_SIZE), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), - MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, - MLXSW_SP2_SB_PR_INGRESS_MNG_SIZE), - /* Egress pools. */ - MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, - MLXSW_SP2_SB_PR_EGRESS_SIZE), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, + MLXSW_SP2_SB_PR_EGRESS_SIZE, true, false), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), + MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_STATIC, MLXSW_SP_SB_INFI, + true, true), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, + MLXSW_SP2_SB_PR_CPU_SIZE, true, false), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, + MLXSW_SP2_SB_PR_CPU_SIZE, true, false), }; static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp, @@ -462,83 +493,106 @@ static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp, .pool_index = _pool, \ } +#define MLXSW_SP_SB_CM_ING(_min_buff, _max_buff) \ + { \ + .min_buff = _min_buff, \ + .max_buff = _max_buff, \ + .pool_index = MLXSW_SP_SB_POOL_ING, \ + } + +#define MLXSW_SP_SB_CM_EGR(_min_buff, _max_buff) \ + { \ + .min_buff = _min_buff, \ + .max_buff = _max_buff, \ + .pool_index = MLXSW_SP_SB_POOL_EGR, \ + } + +#define MLXSW_SP_SB_CM_EGR_MC(_min_buff, _max_buff) \ + { \ + .min_buff = _min_buff, \ + .max_buff = _max_buff, \ + .pool_index = MLXSW_SP_SB_POOL_EGR_MC, \ + .freeze_pool = true, \ + .freeze_thresh = true, \ + } + static const struct mlxsw_sp_sb_cm mlxsw_sp1_sb_cms_ingress[] = { - MLXSW_SP_SB_CM(10000, 8, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, 0, 0), /* dummy, this PG does not exist */ - MLXSW_SP_SB_CM(20000, 1, 3), + MLXSW_SP_SB_CM_ING(10000, 8), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, 0), /* dummy, this PG does not exist */ + MLXSW_SP_SB_CM(10000, 8, MLXSW_SP_SB_POOL_ING_CPU), }; static const struct mlxsw_sp_sb_cm mlxsw_sp2_sb_cms_ingress[] = { - MLXSW_SP_SB_CM(0, 7, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN, 0), - MLXSW_SP_SB_CM(0, 0, 0), /* dummy, this PG does not exist */ - MLXSW_SP_SB_CM(20000, 1, 3), + MLXSW_SP_SB_CM_ING(0, 7), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), + MLXSW_SP_SB_CM_ING(0, 0), /* dummy, this PG does not exist */ + MLXSW_SP_SB_CM(10000, 8, MLXSW_SP_SB_POOL_ING_CPU), }; static const struct mlxsw_sp_sb_cm mlxsw_sp1_sb_cms_egress[] = { - MLXSW_SP_SB_CM(1500, 9, 4), - MLXSW_SP_SB_CM(1500, 9, 4), - MLXSW_SP_SB_CM(1500, 9, 4), - MLXSW_SP_SB_CM(1500, 9, 4), - MLXSW_SP_SB_CM(1500, 9, 4), - MLXSW_SP_SB_CM(1500, 9, 4), - MLXSW_SP_SB_CM(1500, 9, 4), - MLXSW_SP_SB_CM(1500, 9, 4), - MLXSW_SP_SB_CM(0, MLXSW_SP_SB_INFI, 8), - MLXSW_SP_SB_CM(0, MLXSW_SP_SB_INFI, 8), - MLXSW_SP_SB_CM(0, MLXSW_SP_SB_INFI, 8), - MLXSW_SP_SB_CM(0, MLXSW_SP_SB_INFI, 8), - MLXSW_SP_SB_CM(0, MLXSW_SP_SB_INFI, 8), - MLXSW_SP_SB_CM(0, MLXSW_SP_SB_INFI, 8), - MLXSW_SP_SB_CM(0, MLXSW_SP_SB_INFI, 8), - MLXSW_SP_SB_CM(0, MLXSW_SP_SB_INFI, 8), - MLXSW_SP_SB_CM(1, 0xff, 4), + MLXSW_SP_SB_CM_EGR(1500, 9), + MLXSW_SP_SB_CM_EGR(1500, 9), + MLXSW_SP_SB_CM_EGR(1500, 9), + MLXSW_SP_SB_CM_EGR(1500, 9), + MLXSW_SP_SB_CM_EGR(1500, 9), + MLXSW_SP_SB_CM_EGR(1500, 9), + MLXSW_SP_SB_CM_EGR(1500, 9), + MLXSW_SP_SB_CM_EGR(1500, 9), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR(1, 0xff), }; static const struct mlxsw_sp_sb_cm mlxsw_sp2_sb_cms_egress[] = { - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(0, 7, 4), - MLXSW_SP_SB_CM(1, 0xff, 4), + MLXSW_SP_SB_CM_EGR(0, 7), + MLXSW_SP_SB_CM_EGR(0, 7), + MLXSW_SP_SB_CM_EGR(0, 7), + MLXSW_SP_SB_CM_EGR(0, 7), + MLXSW_SP_SB_CM_EGR(0, 7), + MLXSW_SP_SB_CM_EGR(0, 7), + MLXSW_SP_SB_CM_EGR(0, 7), + MLXSW_SP_SB_CM_EGR(0, 7), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI), + MLXSW_SP_SB_CM_EGR(1, 0xff), }; -#define MLXSW_SP_CPU_PORT_SB_CM MLXSW_SP_SB_CM(0, 0, 4) +#define MLXSW_SP_CPU_PORT_SB_CM MLXSW_SP_SB_CM(0, 0, MLXSW_SP_SB_POOL_EGR_CPU) static const struct mlxsw_sp_sb_cm mlxsw_sp_cpu_port_sb_cms[] = { MLXSW_SP_CPU_PORT_SB_CM, - MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 4), - MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 4), - MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 4), - MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 4), - MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 4), + MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU), + MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU), + MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU), + MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU), + MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU), MLXSW_SP_CPU_PORT_SB_CM, - MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 4), + MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU), MLXSW_SP_CPU_PORT_SB_CM, MLXSW_SP_CPU_PORT_SB_CM, MLXSW_SP_CPU_PORT_SB_CM, @@ -646,79 +700,116 @@ static int mlxsw_sp_cpu_port_sb_cms_init(struct mlxsw_sp *mlxsw_sp) .max_buff = _max_buff, \ } +/* Order according to mlxsw_sp1_sb_pool_dess */ static const struct mlxsw_sp_sb_pm mlxsw_sp1_sb_pms[] = { - /* Ingress pools. */ MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX), MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), - MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX), - /* Egress pools. */ + MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), MLXSW_SP_SB_PM(0, 7), MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), MLXSW_SP_SB_PM(10000, 90000), + MLXSW_SP_SB_PM(0, 8), /* 50% occupancy */ + MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), }; +/* Order according to mlxsw_sp2_sb_pool_dess */ static const struct mlxsw_sp_sb_pm mlxsw_sp2_sb_pms[] = { - /* Ingress pools. */ MLXSW_SP_SB_PM(0, 7), MLXSW_SP_SB_PM(0, 0), MLXSW_SP_SB_PM(0, 0), - MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX), - /* Egress pools. */ + MLXSW_SP_SB_PM(0, 0), MLXSW_SP_SB_PM(0, 7), MLXSW_SP_SB_PM(0, 0), MLXSW_SP_SB_PM(0, 0), MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(10000, 90000), + MLXSW_SP_SB_PM(0, 8), /* 50% occupancy */ + MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN), }; -static int mlxsw_sp_port_sb_pms_init(struct mlxsw_sp_port *mlxsw_sp_port) +/* Order according to mlxsw_sp*_sb_pool_dess */ +static const struct mlxsw_sp_sb_pm mlxsw_sp_cpu_port_sb_pms[] = { + MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(0, 90000), + MLXSW_SP_SB_PM(0, 0), + MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX), +}; + +static int mlxsw_sp_sb_pms_init(struct mlxsw_sp *mlxsw_sp, u8 local_port, + const struct mlxsw_sp_sb_pm *pms, + bool skip_ingress) { - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - int i; - int err; + int i, err; for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) { - const struct mlxsw_sp_sb_pm *pm = &mlxsw_sp->sb_vals->pms[i]; + const struct mlxsw_sp_sb_pm *pm = &pms[i]; + const struct mlxsw_sp_sb_pool_des *des; u32 max_buff; u32 min_buff; + des = &mlxsw_sp->sb_vals->pool_dess[i]; + if (skip_ingress && des->dir == MLXSW_REG_SBXX_DIR_INGRESS) + continue; + min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, pm->min_buff); max_buff = pm->max_buff; if (mlxsw_sp_sb_pool_is_static(mlxsw_sp, i)) max_buff = mlxsw_sp_bytes_cells(mlxsw_sp, max_buff); - err = mlxsw_sp_sb_pm_write(mlxsw_sp, mlxsw_sp_port->local_port, - i, min_buff, max_buff); + err = mlxsw_sp_sb_pm_write(mlxsw_sp, local_port, i, min_buff, + max_buff); if (err) return err; } return 0; } -#define MLXSW_SP_SB_MM(_min_buff, _max_buff, _pool) \ +static int mlxsw_sp_port_sb_pms_init(struct mlxsw_sp_port *mlxsw_sp_port) +{ + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + + return mlxsw_sp_sb_pms_init(mlxsw_sp, mlxsw_sp_port->local_port, + mlxsw_sp->sb_vals->pms, false); +} + +static int mlxsw_sp_cpu_port_sb_pms_init(struct mlxsw_sp *mlxsw_sp) +{ + return mlxsw_sp_sb_pms_init(mlxsw_sp, 0, mlxsw_sp->sb_vals->pms_cpu, + true); +} + +#define MLXSW_SP_SB_MM(_min_buff, _max_buff) \ { \ .min_buff = _min_buff, \ .max_buff = _max_buff, \ - .pool_index = _pool, \ + .pool_index = MLXSW_SP_SB_POOL_EGR, \ } static const struct mlxsw_sp_sb_mm mlxsw_sp_sb_mms[] = { - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), - MLXSW_SP_SB_MM(0, 6, 4), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), + MLXSW_SP_SB_MM(0, 6), }; static int mlxsw_sp_sb_mms_init(struct mlxsw_sp *mlxsw_sp) @@ -752,21 +843,22 @@ static void mlxsw_sp_pool_count(struct mlxsw_sp *mlxsw_sp, { int i; - for (i = 0; i < mlxsw_sp->sb_vals->pool_count; ++i) + for (i = 0; i < mlxsw_sp->sb_vals->pool_count; ++i) { if (mlxsw_sp->sb_vals->pool_dess[i].dir == - MLXSW_REG_SBXX_DIR_EGRESS) - goto out; - WARN(1, "No egress pools\n"); + MLXSW_REG_SBXX_DIR_INGRESS) + (*p_ingress_len)++; + else + (*p_egress_len)++; + } -out: - *p_ingress_len = i; - *p_egress_len = mlxsw_sp->sb_vals->pool_count - i; + WARN(*p_egress_len == 0, "No egress pools\n"); } const struct mlxsw_sp_sb_vals mlxsw_sp1_sb_vals = { .pool_count = ARRAY_SIZE(mlxsw_sp1_sb_pool_dess), .pool_dess = mlxsw_sp1_sb_pool_dess, .pms = mlxsw_sp1_sb_pms, + .pms_cpu = mlxsw_sp_cpu_port_sb_pms, .prs = mlxsw_sp1_sb_prs, .mms = mlxsw_sp_sb_mms, .cms_ingress = mlxsw_sp1_sb_cms_ingress, @@ -782,6 +874,7 @@ const struct mlxsw_sp_sb_vals mlxsw_sp2_sb_vals = { .pool_count = ARRAY_SIZE(mlxsw_sp2_sb_pool_dess), .pool_dess = mlxsw_sp2_sb_pool_dess, .pms = mlxsw_sp2_sb_pms, + .pms_cpu = mlxsw_sp_cpu_port_sb_pms, .prs = mlxsw_sp2_sb_prs, .mms = mlxsw_sp_sb_mms, .cms_ingress = mlxsw_sp2_sb_cms_ingress, @@ -796,8 +889,8 @@ const struct mlxsw_sp_sb_vals mlxsw_sp2_sb_vals = { int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) { u32 max_headroom_size; - u16 ing_pool_count; - u16 eg_pool_count; + u16 ing_pool_count = 0; + u16 eg_pool_count = 0; int err; if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, CELL_SIZE)) @@ -831,6 +924,9 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) err = mlxsw_sp_cpu_port_sb_cms_init(mlxsw_sp); if (err) goto err_sb_cpu_port_sb_cms_init; + err = mlxsw_sp_cpu_port_sb_pms_init(mlxsw_sp); + if (err) + goto err_sb_cpu_port_pms_init; err = mlxsw_sp_sb_mms_init(mlxsw_sp); if (err) goto err_sb_mms_init; @@ -848,6 +944,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) err_devlink_sb_register: err_sb_mms_init: +err_sb_cpu_port_pms_init: err_sb_cpu_port_sb_cms_init: err_sb_prs_init: mlxsw_sp_sb_ports_fini(mlxsw_sp); @@ -897,16 +994,32 @@ int mlxsw_sp_sb_pool_get(struct mlxsw_core *mlxsw_core, int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core, unsigned int sb_index, u16 pool_index, u32 size, - enum devlink_sb_threshold_type threshold_type) + enum devlink_sb_threshold_type threshold_type, + struct netlink_ext_ack *extack) { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); u32 pool_size = mlxsw_sp_bytes_cells(mlxsw_sp, size); + const struct mlxsw_sp_sb_pr *pr; enum mlxsw_reg_sbpr_mode mode; - if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_BUFFER_SIZE)) + mode = (enum mlxsw_reg_sbpr_mode) threshold_type; + pr = &mlxsw_sp->sb_vals->prs[pool_index]; + + if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_BUFFER_SIZE)) { + NL_SET_ERR_MSG_MOD(extack, "Exceeded shared buffer size"); return -EINVAL; + } + + if (pr->freeze_mode && pr->mode != mode) { + NL_SET_ERR_MSG_MOD(extack, "Changing this pool's threshold type is forbidden"); + return -EINVAL; + }; + + if (pr->freeze_size && pr->size != size) { + NL_SET_ERR_MSG_MOD(extack, "Changing this pool's size is forbidden"); + return -EINVAL; + }; - mode = (enum mlxsw_reg_sbpr_mode) threshold_type; return mlxsw_sp_sb_pr_write(mlxsw_sp, pool_index, mode, pool_size, false); } @@ -924,7 +1037,8 @@ static u32 mlxsw_sp_sb_threshold_out(struct mlxsw_sp *mlxsw_sp, u16 pool_index, } static int mlxsw_sp_sb_threshold_in(struct mlxsw_sp *mlxsw_sp, u16 pool_index, - u32 threshold, u32 *p_max_buff) + u32 threshold, u32 *p_max_buff, + struct netlink_ext_ack *extack) { struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index); @@ -933,8 +1047,10 @@ static int mlxsw_sp_sb_threshold_in(struct mlxsw_sp *mlxsw_sp, u16 pool_index, val = threshold + MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET; if (val < MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN || - val > MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX) + val > MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX) { + NL_SET_ERR_MSG_MOD(extack, "Invalid dynamic threshold value"); return -EINVAL; + } *p_max_buff = val; } else { *p_max_buff = mlxsw_sp_bytes_cells(mlxsw_sp, threshold); @@ -960,7 +1076,7 @@ int mlxsw_sp_sb_port_pool_get(struct mlxsw_core_port *mlxsw_core_port, int mlxsw_sp_sb_port_pool_set(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 pool_index, - u32 threshold) + u32 threshold, struct netlink_ext_ack *extack) { struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_core_port_driver_priv(mlxsw_core_port); @@ -970,7 +1086,7 @@ int mlxsw_sp_sb_port_pool_set(struct mlxsw_core_port *mlxsw_core_port, int err; err = mlxsw_sp_sb_threshold_in(mlxsw_sp, pool_index, - threshold, &max_buff); + threshold, &max_buff, extack); if (err) return err; @@ -1001,22 +1117,41 @@ int mlxsw_sp_sb_tc_pool_bind_get(struct mlxsw_core_port *mlxsw_core_port, int mlxsw_sp_sb_tc_pool_bind_set(struct mlxsw_core_port *mlxsw_core_port, unsigned int sb_index, u16 tc_index, enum devlink_sb_pool_type pool_type, - u16 pool_index, u32 threshold) + u16 pool_index, u32 threshold, + struct netlink_ext_ack *extack) { struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_core_port_driver_priv(mlxsw_core_port); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; u8 local_port = mlxsw_sp_port->local_port; + const struct mlxsw_sp_sb_cm *cm; u8 pg_buff = tc_index; enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type; u32 max_buff; int err; - if (dir != mlxsw_sp->sb_vals->pool_dess[pool_index].dir) + if (dir != mlxsw_sp->sb_vals->pool_dess[pool_index].dir) { + NL_SET_ERR_MSG_MOD(extack, "Binding egress TC to ingress pool and vice versa is forbidden"); return -EINVAL; + } + + if (dir == MLXSW_REG_SBXX_DIR_INGRESS) + cm = &mlxsw_sp->sb_vals->cms_ingress[tc_index]; + else + cm = &mlxsw_sp->sb_vals->cms_egress[tc_index]; + + if (cm->freeze_pool && cm->pool_index != pool_index) { + NL_SET_ERR_MSG_MOD(extack, "Binding this TC to a different pool is forbidden"); + return -EINVAL; + } + + if (cm->freeze_thresh && cm->max_buff != threshold) { + NL_SET_ERR_MSG_MOD(extack, "Changing this TC's threshold is forbidden"); + return -EINVAL; + } err = mlxsw_sp_sb_threshold_in(mlxsw_sp, pool_index, - threshold, &max_buff); + threshold, &max_buff, extack); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.h index e689576231ab..246dbb3c0e1b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.h @@ -4,24 +4,9 @@ #ifndef _MLXSW_PIPELINE_H_ #define _MLXSW_PIPELINE_H_ -#if IS_ENABLED(CONFIG_NET_DEVLINK) - int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp); void mlxsw_sp_dpipe_fini(struct mlxsw_sp *mlxsw_sp); -#else - -static inline int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp) -{ - return 0; -} - -static inline void mlxsw_sp_dpipe_fini(struct mlxsw_sp *mlxsw_sp) -{ -} - -#endif - #define MLXSW_SP_DPIPE_TABLE_NAME_ERIF "mlxsw_erif" #define MLXSW_SP_DPIPE_TABLE_NAME_HOST4 "mlxsw_host4" #define MLXSW_SP_DPIPE_TABLE_NAME_HOST6 "mlxsw_host6" diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 52fed8c7bf1e..1cda8a248b12 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -13,9 +13,9 @@ #include <linux/socket.h> #include <linux/route.h> #include <linux/gcd.h> -#include <linux/random.h> #include <linux/if_macvlan.h> #include <linux/refcount.h> +#include <linux/jhash.h> #include <net/netevent.h> #include <net/neighbour.h> #include <net/arp.h> @@ -2371,7 +2371,7 @@ static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) MLXSW_REG_RAUHT_OP_WRITE_DELETE; } -static void +static int mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_neigh_entry *neigh_entry, enum mlxsw_reg_rauht_op op) @@ -2385,10 +2385,10 @@ mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, if (neigh_entry->counter_valid) mlxsw_reg_rauht_pack_counter(rauht_pl, neigh_entry->counter_index); - mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); + return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); } -static void +static int mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_neigh_entry *neigh_entry, enum mlxsw_reg_rauht_op op) @@ -2402,7 +2402,7 @@ mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, if (neigh_entry->counter_valid) mlxsw_reg_rauht_pack_counter(rauht_pl, neigh_entry->counter_index); - mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); + return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); } bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) @@ -2424,20 +2424,33 @@ mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_neigh_entry *neigh_entry, bool adding) { + enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); + int err; + if (!adding && !neigh_entry->connected) return; neigh_entry->connected = adding; if (neigh_entry->key.n->tbl->family == AF_INET) { - mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, - mlxsw_sp_rauht_op(adding)); + err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, + op); + if (err) + return; } else if (neigh_entry->key.n->tbl->family == AF_INET6) { if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) return; - mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, - mlxsw_sp_rauht_op(adding)); + err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, + op); + if (err) + return; } else { WARN_ON_ONCE(1); + return; } + + if (adding) + neigh_entry->key.n->flags |= NTF_OFFLOADED; + else + neigh_entry->key.n->flags &= ~NTF_OFFLOADED; } void @@ -2873,12 +2886,13 @@ mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, return false; list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { + struct fib6_nh *fib6_nh = &mlxsw_sp_rt6->rt->fib6_nh; struct in6_addr *gw; int ifindex, weight; - ifindex = mlxsw_sp_rt6->rt->fib6_nh.nh_dev->ifindex; - weight = mlxsw_sp_rt6->rt->fib6_nh.nh_weight; - gw = &mlxsw_sp_rt6->rt->fib6_nh.nh_gw; + ifindex = fib6_nh->fib_nh_dev->ifindex; + weight = fib6_nh->fib_nh_weight; + gw = &fib6_nh->fib_nh_gw6; if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, weight)) return false; @@ -2944,7 +2958,7 @@ mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) struct net_device *dev; list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { - dev = mlxsw_sp_rt6->rt->fib6_nh.nh_dev; + dev = mlxsw_sp_rt6->rt->fib6_nh.fib_nh_dev; val ^= dev->ifindex; } @@ -3610,7 +3624,7 @@ static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, const struct fib_nh *fib_nh, enum mlxsw_sp_ipip_type *p_ipipt) { - struct net_device *dev = fib_nh->nh_dev; + struct net_device *dev = fib_nh->fib_nh_dev; return dev && fib_nh->nh_parent->fib_type == RTN_UNICAST && @@ -3637,7 +3651,7 @@ static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp, struct fib_nh *fib_nh) { const struct mlxsw_sp_ipip_ops *ipip_ops; - struct net_device *dev = fib_nh->nh_dev; + struct net_device *dev = fib_nh->fib_nh_dev; struct mlxsw_sp_ipip_entry *ipip_entry; struct mlxsw_sp_rif *rif; int err; @@ -3681,18 +3695,18 @@ static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_nexthop *nh, struct fib_nh *fib_nh) { - struct net_device *dev = fib_nh->nh_dev; + struct net_device *dev = fib_nh->fib_nh_dev; struct in_device *in_dev; int err; nh->nh_grp = nh_grp; nh->key.fib_nh = fib_nh; #ifdef CONFIG_IP_ROUTE_MULTIPATH - nh->nh_weight = fib_nh->nh_weight; + nh->nh_weight = fib_nh->fib_nh_weight; #else nh->nh_weight = 1; #endif - memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw)); + memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); if (err) return err; @@ -3705,7 +3719,7 @@ static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, in_dev = __in_dev_get_rtnl(dev); if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && - fib_nh->nh_flags & RTNH_F_LINKDOWN) + fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) return 0; err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh); @@ -3804,7 +3818,7 @@ static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, const struct fib_info *fi) { - return fi->fib_nh->nh_scope == RT_SCOPE_LINK || + return fi->fib_nh->fib_nh_scope == RT_SCOPE_LINK || mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL); } @@ -3946,9 +3960,9 @@ mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; struct fib6_info *rt = mlxsw_sp_rt6->rt; - if (nh->rif && nh->rif->dev == rt->fib6_nh.nh_dev && + if (nh->rif && nh->rif->dev == rt->fib6_nh.fib_nh_dev && ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, - &rt->fib6_nh.nh_gw)) + &rt->fib6_nh.fib_nh_gw6)) return nh; continue; } @@ -3966,7 +3980,7 @@ mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE || fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP || fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP) { - nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD; + nh_grp->nexthops->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; return; } @@ -3974,9 +3988,9 @@ mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; if (nh->offloaded) - nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD; + nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; else - nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD; + nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; } } @@ -3992,7 +4006,7 @@ mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) for (i = 0; i < nh_grp->count; i++) { struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i]; - nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD; + nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; } } @@ -4008,19 +4022,20 @@ mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry) if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL || fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE) { list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, - list)->rt->fib6_nh.nh_flags |= RTNH_F_OFFLOAD; + list)->rt->fib6_nh.fib_nh_flags |= RTNH_F_OFFLOAD; return; } list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; + struct fib6_nh *fib6_nh = &mlxsw_sp_rt6->rt->fib6_nh; struct mlxsw_sp_nexthop *nh; nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); if (nh && nh->offloaded) - mlxsw_sp_rt6->rt->fib6_nh.nh_flags |= RTNH_F_OFFLOAD; + fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; else - mlxsw_sp_rt6->rt->fib6_nh.nh_flags &= ~RTNH_F_OFFLOAD; + fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; } } @@ -4035,7 +4050,7 @@ mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry) list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { struct fib6_info *rt = mlxsw_sp_rt6->rt; - rt->fib6_nh.nh_flags &= ~RTNH_F_OFFLOAD; + rt->fib6_nh.fib_nh_flags &= ~RTNH_F_OFFLOAD; } } @@ -4913,7 +4928,7 @@ static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) static bool mlxsw_sp_fib6_rt_can_mp(const struct fib6_info *rt) { /* RTF_CACHE routes are ignored */ - return (rt->fib6_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY; + return !(rt->fib6_flags & RTF_ADDRCONF) && rt->fib6_nh.fib_nh_gw_family; } static struct fib6_info * @@ -4972,8 +4987,8 @@ static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, const struct fib6_info *rt, enum mlxsw_sp_ipip_type *ret) { - return rt->fib6_nh.nh_dev && - mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh.nh_dev, ret); + return rt->fib6_nh.fib_nh_dev && + mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh.fib_nh_dev, ret); } static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp, @@ -4983,7 +4998,7 @@ static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp, { const struct mlxsw_sp_ipip_ops *ipip_ops; struct mlxsw_sp_ipip_entry *ipip_entry; - struct net_device *dev = rt->fib6_nh.nh_dev; + struct net_device *dev = rt->fib6_nh.fib_nh_dev; struct mlxsw_sp_rif *rif; int err; @@ -5026,11 +5041,11 @@ static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_nexthop *nh, const struct fib6_info *rt) { - struct net_device *dev = rt->fib6_nh.nh_dev; + struct net_device *dev = rt->fib6_nh.fib_nh_dev; nh->nh_grp = nh_grp; - nh->nh_weight = rt->fib6_nh.nh_weight; - memcpy(&nh->gw_addr, &rt->fib6_nh.nh_gw, sizeof(nh->gw_addr)); + nh->nh_weight = rt->fib6_nh.fib_nh_weight; + memcpy(&nh->gw_addr, &rt->fib6_nh.fib_nh_gw6, sizeof(nh->gw_addr)); mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); @@ -5053,7 +5068,7 @@ static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, const struct fib6_info *rt) { - return rt->fib6_flags & RTF_GATEWAY || + return rt->fib6_nh.fib_nh_gw_family || mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); } @@ -6035,6 +6050,10 @@ static int mlxsw_sp_router_fib_rule_event(unsigned long event, fr_info = container_of(info, struct fib_rule_notifier_info, info); rule = fr_info->rule; + /* Rule only affects locally generated traffic */ + if (rule->iifindex == info->net->loopback_dev->ifindex) + return 0; + switch (info->family) { case AF_INET: if (!fib4_rule_default(rule) && !rule->l3mdev) @@ -6086,10 +6105,20 @@ static int mlxsw_sp_router_fib_event(struct notifier_block *nb, return notifier_from_errno(err); break; case FIB_EVENT_ENTRY_ADD: + case FIB_EVENT_ENTRY_REPLACE: /* fall through */ + case FIB_EVENT_ENTRY_APPEND: /* fall through */ if (router->aborted) { NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route"); return notifier_from_errno(-EINVAL); } + if (info->family == AF_INET) { + struct fib_entry_notifier_info *fen_info = ptr; + + if (fen_info->fi->fib_nh_is_v6) { + NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); + return notifier_from_errno(-EINVAL); + } + } break; } @@ -6781,7 +6810,7 @@ static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp, /* A RIF is not created for macvlan netdevs. Their MAC is used to * populate the FDB */ - if (netif_is_macvlan(dev)) + if (netif_is_macvlan(dev) || netif_is_l3_master(dev)) return 0; for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { @@ -7808,7 +7837,7 @@ static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) char recr2_pl[MLXSW_REG_RECR2_LEN]; u32 seed; - get_random_bytes(&seed, sizeof(seed)); + seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); mlxsw_reg_recr2_pack(recr2_pl, seed); mlxsw_sp_mp4_hash_init(recr2_pl); mlxsw_sp_mp6_hash_init(recr2_pl); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c index 536c23c578c3..560a60e522f9 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c @@ -316,7 +316,11 @@ mlxsw_sp_span_gretap4_route(const struct net_device *to_dev, dev = rt->dst.dev; *saddrp = fl4.saddr; - *daddrp = rt->rt_gateway; + if (rt->rt_gw_family == AF_INET) + *daddrp = rt->rt_gw4; + /* can not offload if route has an IPv6 gateway */ + else if (rt->rt_gw_family == AF_INET6) + dev = NULL; out: ip_rt_put(rt); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c index f6ce386c3036..50111f228d77 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c @@ -1630,7 +1630,7 @@ static int mlxsw_sp_port_mdb_add(struct mlxsw_sp_port *mlxsw_sp_port, u16 fid_index; int err = 0; - if (switchdev_trans_ph_prepare(trans)) + if (switchdev_trans_ph_commit(trans)) return 0; bridge_port = mlxsw_sp_bridge_port_find(mlxsw_sp->bridge, orig_dev); diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchib.c b/drivers/net/ethernet/mellanox/mlxsw/switchib.c index bcf2e79a21c8..0d9356b3f65d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/switchib.c +++ b/drivers/net/ethernet/mellanox/mlxsw/switchib.c @@ -30,6 +30,7 @@ struct mlxsw_sib { struct mlxsw_sib_port **ports; struct mlxsw_core *core; const struct mlxsw_bus_info *bus_info; + u8 hw_id[ETH_ALEN]; }; struct mlxsw_sib_port { @@ -102,6 +103,18 @@ mlxsw_sib_tx_v1_hdr_construct(struct sk_buff *skb, mlxsw_tx_v1_hdr_type_set(txhdr, MLXSW_TXHDR_TYPE_CONTROL); } +static int mlxsw_sib_hw_id_get(struct mlxsw_sib *mlxsw_sib) +{ + char spad_pl[MLXSW_REG_SPAD_LEN] = {0}; + int err; + + err = mlxsw_reg_query(mlxsw_sib->core, MLXSW_REG(spad), spad_pl); + if (err) + return err; + mlxsw_reg_spad_base_mac_memcpy_from(spad_pl, mlxsw_sib->hw_id); + return 0; +} + static int mlxsw_sib_port_admin_status_set(struct mlxsw_sib_port *mlxsw_sib_port, bool is_up) @@ -267,7 +280,9 @@ static int mlxsw_sib_port_create(struct mlxsw_sib *mlxsw_sib, u8 local_port, { int err; - err = mlxsw_core_port_init(mlxsw_sib->core, local_port); + err = mlxsw_core_port_init(mlxsw_sib->core, local_port, + module + 1, false, 0, + mlxsw_sib->hw_id, sizeof(mlxsw_sib->hw_id)); if (err) { dev_err(mlxsw_sib->bus_info->dev, "Port %d: Failed to init core port\n", local_port); @@ -439,6 +454,12 @@ static int mlxsw_sib_init(struct mlxsw_core *mlxsw_core, mlxsw_sib->core = mlxsw_core; mlxsw_sib->bus_info = mlxsw_bus_info; + err = mlxsw_sib_hw_id_get(mlxsw_sib); + if (err) { + dev_err(mlxsw_sib->bus_info->dev, "Failed to get switch HW ID\n"); + return err; + } + err = mlxsw_sib_ports_create(mlxsw_sib); if (err) { dev_err(mlxsw_sib->bus_info->dev, "Failed to create ports\n"); diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c index 533fe6235b7c..fc4f19167262 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c +++ b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c @@ -379,26 +379,14 @@ mlxsw_sx_port_get_stats64(struct net_device *dev, stats->tx_dropped = tx_dropped; } -static int mlxsw_sx_port_get_phys_port_name(struct net_device *dev, char *name, - size_t len) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - - return mlxsw_core_port_get_phys_port_name(mlxsw_sx_port->mlxsw_sx->core, - mlxsw_sx_port->local_port, - name, len); -} - -static int mlxsw_sx_port_get_port_parent_id(struct net_device *dev, - struct netdev_phys_item_id *ppid) +static struct devlink_port * +mlxsw_sx_port_get_devlink_port(struct net_device *dev) { struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - ppid->id_len = sizeof(mlxsw_sx->hw_id); - memcpy(&ppid->id, &mlxsw_sx->hw_id, ppid->id_len); - - return 0; + return mlxsw_core_port_devlink_port_get(mlxsw_sx->core, + mlxsw_sx_port->local_port); } static const struct net_device_ops mlxsw_sx_port_netdev_ops = { @@ -407,8 +395,7 @@ static const struct net_device_ops mlxsw_sx_port_netdev_ops = { .ndo_start_xmit = mlxsw_sx_port_xmit, .ndo_change_mtu = mlxsw_sx_port_change_mtu, .ndo_get_stats64 = mlxsw_sx_port_get_stats64, - .ndo_get_phys_port_name = mlxsw_sx_port_get_phys_port_name, - .ndo_get_port_parent_id = mlxsw_sx_port_get_port_parent_id, + .ndo_get_devlink_port = mlxsw_sx_port_get_devlink_port, }; static void mlxsw_sx_port_get_drvinfo(struct net_device *dev, @@ -1102,7 +1089,7 @@ static int __mlxsw_sx_port_eth_create(struct mlxsw_sx *mlxsw_sx, u8 local_port, } mlxsw_core_port_eth_set(mlxsw_sx->core, mlxsw_sx_port->local_port, - mlxsw_sx_port, dev, module + 1, false, 0); + mlxsw_sx_port, dev); mlxsw_sx->ports[local_port] = mlxsw_sx_port; return 0; @@ -1127,7 +1114,9 @@ static int mlxsw_sx_port_eth_create(struct mlxsw_sx *mlxsw_sx, u8 local_port, { int err; - err = mlxsw_core_port_init(mlxsw_sx->core, local_port); + err = mlxsw_core_port_init(mlxsw_sx->core, local_port, + module + 1, false, 0, + mlxsw_sx->hw_id, sizeof(mlxsw_sx->hw_id)); if (err) { dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to init core port\n", local_port); |