summaryrefslogtreecommitdiffstats
path: root/drivers/net/ethernet/mediatek
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/ethernet/mediatek')
-rw-r--r--drivers/net/ethernet/mediatek/Kconfig2
-rw-r--r--drivers/net/ethernet/mediatek/Makefile2
-rw-r--r--drivers/net/ethernet/mediatek/mtk_eth_soc.c315
-rw-r--r--drivers/net/ethernet/mediatek/mtk_eth_soc.h73
-rw-r--r--drivers/net/ethernet/mediatek/mtk_ppe.c509
-rw-r--r--drivers/net/ethernet/mediatek/mtk_ppe.h288
-rw-r--r--drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c217
-rw-r--r--drivers/net/ethernet/mediatek/mtk_ppe_offload.c495
-rw-r--r--drivers/net/ethernet/mediatek/mtk_ppe_regs.h144
9 files changed, 1928 insertions, 117 deletions
diff --git a/drivers/net/ethernet/mediatek/Kconfig b/drivers/net/ethernet/mediatek/Kconfig
index 3362b148de23..c357c193378e 100644
--- a/drivers/net/ethernet/mediatek/Kconfig
+++ b/drivers/net/ethernet/mediatek/Kconfig
@@ -9,7 +9,9 @@ if NET_VENDOR_MEDIATEK
config NET_MEDIATEK_SOC
tristate "MediaTek SoC Gigabit Ethernet support"
+ depends on NET_DSA || !NET_DSA
select PHYLINK
+ select DIMLIB
help
This driver supports the gigabit ethernet MACs in the
MediaTek SoC family.
diff --git a/drivers/net/ethernet/mediatek/Makefile b/drivers/net/ethernet/mediatek/Makefile
index 3a777b4a6cd3..79d4cdbbcbf5 100644
--- a/drivers/net/ethernet/mediatek/Makefile
+++ b/drivers/net/ethernet/mediatek/Makefile
@@ -4,5 +4,5 @@
#
obj-$(CONFIG_NET_MEDIATEK_SOC) += mtk_eth.o
-mtk_eth-y := mtk_eth_soc.o mtk_sgmii.o mtk_eth_path.o
+mtk_eth-y := mtk_eth_soc.o mtk_sgmii.o mtk_eth_path.o mtk_ppe.o mtk_ppe_debugfs.o mtk_ppe_offload.o
obj-$(CONFIG_NET_MEDIATEK_STAR_EMAC) += mtk_star_emac.o
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index 01d3ee4b5829..ed4eacef17ce 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -19,6 +19,8 @@
#include <linux/interrupt.h>
#include <linux/pinctrl/devinfo.h>
#include <linux/phylink.h>
+#include <linux/jhash.h>
+#include <net/dsa.h>
#include "mtk_eth_soc.h"
@@ -85,7 +87,7 @@ static int mtk_mdio_busy_wait(struct mtk_eth *eth)
return 0;
if (time_after(jiffies, t_start + PHY_IAC_TIMEOUT))
break;
- usleep_range(10, 20);
+ cond_resched();
}
dev_err(eth->dev, "mdio: MDIO timeout\n");
@@ -776,13 +778,18 @@ static inline int mtk_max_buf_size(int frag_size)
return buf_size;
}
-static inline void mtk_rx_get_desc(struct mtk_rx_dma *rxd,
+static inline bool mtk_rx_get_desc(struct mtk_rx_dma *rxd,
struct mtk_rx_dma *dma_rxd)
{
- rxd->rxd1 = READ_ONCE(dma_rxd->rxd1);
rxd->rxd2 = READ_ONCE(dma_rxd->rxd2);
+ if (!(rxd->rxd2 & RX_DMA_DONE))
+ return false;
+
+ rxd->rxd1 = READ_ONCE(dma_rxd->rxd1);
rxd->rxd3 = READ_ONCE(dma_rxd->rxd3);
rxd->rxd4 = READ_ONCE(dma_rxd->rxd4);
+
+ return true;
}
/* the qdma core needs scratch memory to be setup */
@@ -857,7 +864,8 @@ static int txd_to_idx(struct mtk_tx_ring *ring, struct mtk_tx_dma *dma)
return ((void *)dma - (void *)ring->dma) / sizeof(*dma);
}
-static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf)
+static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
+ bool napi)
{
if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) {
@@ -889,8 +897,12 @@ static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf)
tx_buf->flags = 0;
if (tx_buf->skb &&
- (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC))
- dev_kfree_skb_any(tx_buf->skb);
+ (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC)) {
+ if (napi)
+ napi_consume_skb(tx_buf->skb, napi);
+ else
+ dev_kfree_skb_any(tx_buf->skb);
+ }
tx_buf->skb = NULL;
}
@@ -1068,7 +1080,7 @@ err_dma:
tx_buf = mtk_desc_to_tx_buf(ring, itxd);
/* unmap dma */
- mtk_tx_unmap(eth, tx_buf);
+ mtk_tx_unmap(eth, tx_buf, false);
itxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
if (!MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
@@ -1125,17 +1137,6 @@ static void mtk_wake_queue(struct mtk_eth *eth)
}
}
-static void mtk_stop_queue(struct mtk_eth *eth)
-{
- int i;
-
- for (i = 0; i < MTK_MAC_COUNT; i++) {
- if (!eth->netdev[i])
- continue;
- netif_stop_queue(eth->netdev[i]);
- }
-}
-
static netdev_tx_t mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct mtk_mac *mac = netdev_priv(dev);
@@ -1156,7 +1157,7 @@ static netdev_tx_t mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
tx_num = mtk_cal_txd_req(skb);
if (unlikely(atomic_read(&ring->free_count) <= tx_num)) {
- mtk_stop_queue(eth);
+ netif_stop_queue(dev);
netif_err(eth, tx_queued, dev,
"Tx Ring full when queue awake!\n");
spin_unlock(&eth->page_lock);
@@ -1182,7 +1183,7 @@ static netdev_tx_t mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
goto drop;
if (unlikely(atomic_read(&ring->free_count) <= ring->thresh))
- mtk_stop_queue(eth);
+ netif_stop_queue(dev);
spin_unlock(&eth->page_lock);
@@ -1238,17 +1239,19 @@ static void mtk_update_rx_cpu_idx(struct mtk_eth *eth)
static int mtk_poll_rx(struct napi_struct *napi, int budget,
struct mtk_eth *eth)
{
+ struct dim_sample dim_sample = {};
struct mtk_rx_ring *ring;
int idx;
struct sk_buff *skb;
u8 *data, *new_data;
struct mtk_rx_dma *rxd, trxd;
- int done = 0;
+ int done = 0, bytes = 0;
while (done < budget) {
struct net_device *netdev;
unsigned int pktlen;
dma_addr_t dma_addr;
+ u32 hash;
int mac;
ring = mtk_get_rx_ring(eth);
@@ -1259,18 +1262,16 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
rxd = &ring->dma[idx];
data = ring->data[idx];
- mtk_rx_get_desc(&trxd, rxd);
- if (!(trxd.rxd2 & RX_DMA_DONE))
+ if (!mtk_rx_get_desc(&trxd, rxd))
break;
/* find out which mac the packet come from. values start at 1 */
- if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
+ if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) ||
+ (trxd.rxd4 & RX_DMA_SPECIAL_TAG))
mac = 0;
- } else {
- mac = (trxd.rxd4 >> RX_DMA_FPORT_SHIFT) &
- RX_DMA_FPORT_MASK;
- mac--;
- }
+ else
+ mac = ((trxd.rxd4 >> RX_DMA_FPORT_SHIFT) &
+ RX_DMA_FPORT_MASK) - 1;
if (unlikely(mac < 0 || mac >= MTK_MAC_COUNT ||
!eth->netdev[mac]))
@@ -1298,17 +1299,18 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
goto release_desc;
}
+ dma_unmap_single(eth->dev, trxd.rxd1,
+ ring->buf_size, DMA_FROM_DEVICE);
+
/* receive data */
skb = build_skb(data, ring->frag_size);
if (unlikely(!skb)) {
- skb_free_frag(new_data);
+ skb_free_frag(data);
netdev->stats.rx_dropped++;
- goto release_desc;
+ goto skip_rx;
}
skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
- dma_unmap_single(eth->dev, trxd.rxd1,
- ring->buf_size, DMA_FROM_DEVICE);
pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
skb->dev = netdev;
skb_put(skb, pktlen);
@@ -1317,14 +1319,22 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
else
skb_checksum_none_assert(skb);
skb->protocol = eth_type_trans(skb, netdev);
+ bytes += pktlen;
+
+ hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY;
+ if (hash != MTK_RXD4_FOE_ENTRY) {
+ hash = jhash_1word(hash, 0);
+ skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
+ }
if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
- RX_DMA_VID(trxd.rxd3))
+ (trxd.rxd2 & RX_DMA_VTAG))
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
RX_DMA_VID(trxd.rxd3));
skb_record_rx_queue(skb, 0);
napi_gro_receive(napi, skb);
+skip_rx:
ring->data[idx] = new_data;
rxd->rxd1 = (unsigned int)dma_addr;
@@ -1348,6 +1358,12 @@ rx_done:
mtk_update_rx_cpu_idx(eth);
}
+ eth->rx_packets += done;
+ eth->rx_bytes += bytes;
+ dim_update_sample(eth->rx_events, eth->rx_packets, eth->rx_bytes,
+ &dim_sample);
+ net_dim(&eth->rx_dim, dim_sample);
+
return done;
}
@@ -1360,7 +1376,7 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget,
struct mtk_tx_buf *tx_buf;
u32 cpu, dma;
- cpu = mtk_r32(eth, MTK_QTX_CRX_PTR);
+ cpu = ring->last_free_ptr;
dma = mtk_r32(eth, MTK_QTX_DRX_PTR);
desc = mtk_qdma_phys_to_virt(ring, cpu);
@@ -1386,7 +1402,7 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget,
done[mac]++;
budget--;
}
- mtk_tx_unmap(eth, tx_buf);
+ mtk_tx_unmap(eth, tx_buf, true);
ring->last_free = desc;
atomic_inc(&ring->free_count);
@@ -1394,6 +1410,7 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget,
cpu = next_cpu;
}
+ ring->last_free_ptr = cpu;
mtk_w32(eth, cpu, MTK_QTX_CRX_PTR);
return budget;
@@ -1423,7 +1440,7 @@ static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget,
budget--;
}
- mtk_tx_unmap(eth, tx_buf);
+ mtk_tx_unmap(eth, tx_buf, true);
desc = &ring->dma[cpu];
ring->last_free = desc;
@@ -1440,6 +1457,7 @@ static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget,
static int mtk_poll_tx(struct mtk_eth *eth, int budget)
{
struct mtk_tx_ring *ring = &eth->tx_ring;
+ struct dim_sample dim_sample = {};
unsigned int done[MTK_MAX_DEVS];
unsigned int bytes[MTK_MAX_DEVS];
int total = 0, i;
@@ -1457,8 +1475,14 @@ static int mtk_poll_tx(struct mtk_eth *eth, int budget)
continue;
netdev_completed_queue(eth->netdev[i], done[i], bytes[i]);
total += done[i];
+ eth->tx_packets += done[i];
+ eth->tx_bytes += bytes[i];
}
+ dim_update_sample(eth->tx_events, eth->tx_packets, eth->tx_bytes,
+ &dim_sample);
+ net_dim(&eth->tx_dim, dim_sample);
+
if (mtk_queue_stopped(eth) &&
(atomic_read(&ring->free_count) > ring->thresh))
mtk_wake_queue(eth);
@@ -1480,7 +1504,6 @@ static void mtk_handle_status_irq(struct mtk_eth *eth)
static int mtk_napi_tx(struct napi_struct *napi, int budget)
{
struct mtk_eth *eth = container_of(napi, struct mtk_eth, tx_napi);
- u32 status, mask;
int tx_done = 0;
if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
@@ -1489,22 +1512,20 @@ static int mtk_napi_tx(struct napi_struct *napi, int budget)
tx_done = mtk_poll_tx(eth, budget);
if (unlikely(netif_msg_intr(eth))) {
- status = mtk_r32(eth, eth->tx_int_status_reg);
- mask = mtk_r32(eth, eth->tx_int_mask_reg);
dev_info(eth->dev,
- "done tx %d, intr 0x%08x/0x%x\n",
- tx_done, status, mask);
+ "done tx %d, intr 0x%08x/0x%x\n", tx_done,
+ mtk_r32(eth, eth->tx_int_status_reg),
+ mtk_r32(eth, eth->tx_int_mask_reg));
}
if (tx_done == budget)
return budget;
- status = mtk_r32(eth, eth->tx_int_status_reg);
- if (status & MTK_TX_DONE_INT)
+ if (mtk_r32(eth, eth->tx_int_status_reg) & MTK_TX_DONE_INT)
return budget;
- napi_complete(napi);
- mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
+ if (napi_complete_done(napi, tx_done))
+ mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
return tx_done;
}
@@ -1512,35 +1533,33 @@ static int mtk_napi_tx(struct napi_struct *napi, int budget)
static int mtk_napi_rx(struct napi_struct *napi, int budget)
{
struct mtk_eth *eth = container_of(napi, struct mtk_eth, rx_napi);
- u32 status, mask;
- int rx_done = 0;
- int remain_budget = budget;
+ int rx_done_total = 0;
mtk_handle_status_irq(eth);
-poll_again:
- mtk_w32(eth, MTK_RX_DONE_INT, MTK_PDMA_INT_STATUS);
- rx_done = mtk_poll_rx(napi, remain_budget, eth);
+ do {
+ int rx_done;
- if (unlikely(netif_msg_intr(eth))) {
- status = mtk_r32(eth, MTK_PDMA_INT_STATUS);
- mask = mtk_r32(eth, MTK_PDMA_INT_MASK);
- dev_info(eth->dev,
- "done rx %d, intr 0x%08x/0x%x\n",
- rx_done, status, mask);
- }
- if (rx_done == remain_budget)
- return budget;
+ mtk_w32(eth, MTK_RX_DONE_INT, MTK_PDMA_INT_STATUS);
+ rx_done = mtk_poll_rx(napi, budget - rx_done_total, eth);
+ rx_done_total += rx_done;
- status = mtk_r32(eth, MTK_PDMA_INT_STATUS);
- if (status & MTK_RX_DONE_INT) {
- remain_budget -= rx_done;
- goto poll_again;
- }
- napi_complete(napi);
- mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
+ if (unlikely(netif_msg_intr(eth))) {
+ dev_info(eth->dev,
+ "done rx %d, intr 0x%08x/0x%x\n", rx_done,
+ mtk_r32(eth, MTK_PDMA_INT_STATUS),
+ mtk_r32(eth, MTK_PDMA_INT_MASK));
+ }
- return rx_done + budget - remain_budget;
+ if (rx_done_total == budget)
+ return budget;
+
+ } while (mtk_r32(eth, MTK_PDMA_INT_STATUS) & MTK_RX_DONE_INT);
+
+ if (napi_complete_done(napi, rx_done_total))
+ mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
+
+ return rx_done_total;
}
static int mtk_tx_alloc(struct mtk_eth *eth)
@@ -1587,6 +1606,7 @@ static int mtk_tx_alloc(struct mtk_eth *eth)
atomic_set(&ring->free_count, MTK_DMA_SIZE - 2);
ring->next_free = &ring->dma[0];
ring->last_free = &ring->dma[MTK_DMA_SIZE - 1];
+ ring->last_free_ptr = (u32)(ring->phys + ((MTK_DMA_SIZE - 1) * sz));
ring->thresh = MAX_SKB_FRAGS;
/* make sure that all changes to the dma ring are flushed before we
@@ -1600,9 +1620,7 @@ static int mtk_tx_alloc(struct mtk_eth *eth)
mtk_w32(eth,
ring->phys + ((MTK_DMA_SIZE - 1) * sz),
MTK_QTX_CRX_PTR);
- mtk_w32(eth,
- ring->phys + ((MTK_DMA_SIZE - 1) * sz),
- MTK_QTX_DRX_PTR);
+ mtk_w32(eth, ring->last_free_ptr, MTK_QTX_DRX_PTR);
mtk_w32(eth, (QDMA_RES_THRES << 8) | QDMA_RES_THRES,
MTK_QTX_CFG(0));
} else {
@@ -1625,7 +1643,7 @@ static void mtk_tx_clean(struct mtk_eth *eth)
if (ring->buf) {
for (i = 0; i < MTK_DMA_SIZE; i++)
- mtk_tx_unmap(eth, &ring->buf[i]);
+ mtk_tx_unmap(eth, &ring->buf[i], false);
kfree(ring->buf);
ring->buf = NULL;
}
@@ -2015,25 +2033,22 @@ static int mtk_set_features(struct net_device *dev, netdev_features_t features)
/* wait for DMA to finish whatever it is doing before we start using it again */
static int mtk_dma_busy_wait(struct mtk_eth *eth)
{
- unsigned long t_start = jiffies;
+ unsigned int reg;
+ int ret;
+ u32 val;
- while (1) {
- if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
- if (!(mtk_r32(eth, MTK_QDMA_GLO_CFG) &
- (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)))
- return 0;
- } else {
- if (!(mtk_r32(eth, MTK_PDMA_GLO_CFG) &
- (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)))
- return 0;
- }
+ if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
+ reg = MTK_QDMA_GLO_CFG;
+ else
+ reg = MTK_PDMA_GLO_CFG;
- if (time_after(jiffies, t_start + MTK_DMA_BUSY_TIMEOUT))
- break;
- }
+ ret = readx_poll_timeout_atomic(__raw_readl, eth->base + reg, val,
+ !(val & (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)),
+ 5, MTK_DMA_BUSY_TIMEOUT_US);
+ if (ret)
+ dev_err(eth->dev, "DMA init timeout\n");
- dev_err(eth->dev, "DMA init timeout\n");
- return -1;
+ return ret;
}
static int mtk_dma_init(struct mtk_eth *eth)
@@ -2133,6 +2148,7 @@ static irqreturn_t mtk_handle_irq_rx(int irq, void *_eth)
{
struct mtk_eth *eth = _eth;
+ eth->rx_events++;
if (likely(napi_schedule_prep(&eth->rx_napi))) {
__napi_schedule(&eth->rx_napi);
mtk_rx_irq_disable(eth, MTK_RX_DONE_INT);
@@ -2145,6 +2161,7 @@ static irqreturn_t mtk_handle_irq_tx(int irq, void *_eth)
{
struct mtk_eth *eth = _eth;
+ eth->tx_events++;
if (likely(napi_schedule_prep(&eth->tx_napi))) {
__napi_schedule(&eth->tx_napi);
mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
@@ -2197,7 +2214,7 @@ static int mtk_start_dma(struct mtk_eth *eth)
if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
mtk_w32(eth,
MTK_TX_WB_DDONE | MTK_TX_DMA_EN |
- MTK_DMA_SIZE_16DWORDS | MTK_NDP_CO_PRO |
+ MTK_TX_BT_32DWORDS | MTK_NDP_CO_PRO |
MTK_RX_DMA_EN | MTK_RX_2B_OFFSET |
MTK_RX_BT_32DWORDS,
MTK_QDMA_GLO_CFG);
@@ -2233,6 +2250,9 @@ static void mtk_gdm_config(struct mtk_eth *eth, u32 config)
val |= config;
+ if (!i && eth->netdev[0] && netdev_uses_dsa(eth->netdev[0]))
+ val |= MTK_GDMA_SPECIAL_TAG;
+
mtk_w32(eth, val, MTK_GDMA_FWD_CFG(i));
}
/* Reset and enable PSE */
@@ -2255,12 +2275,17 @@ static int mtk_open(struct net_device *dev)
/* we run 2 netdevs on the same dma ring so we only bring it up once */
if (!refcount_read(&eth->dma_refcnt)) {
- int err = mtk_start_dma(eth);
+ u32 gdm_config = MTK_GDMA_TO_PDMA;
+ int err;
+ err = mtk_start_dma(eth);
if (err)
return err;
- mtk_gdm_config(eth, MTK_GDMA_TO_PDMA);
+ if (eth->soc->offload_version && mtk_ppe_start(&eth->ppe) == 0)
+ gdm_config = MTK_GDMA_TO_PPE;
+
+ mtk_gdm_config(eth, gdm_config);
napi_enable(&eth->tx_napi);
napi_enable(&eth->rx_napi);
@@ -2321,12 +2346,18 @@ static int mtk_stop(struct net_device *dev)
napi_disable(&eth->tx_napi);
napi_disable(&eth->rx_napi);
+ cancel_work_sync(&eth->rx_dim.work);
+ cancel_work_sync(&eth->tx_dim.work);
+
if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
mtk_stop_dma(eth, MTK_QDMA_GLO_CFG);
mtk_stop_dma(eth, MTK_PDMA_GLO_CFG);
mtk_dma_free(eth);
+ if (eth->soc->offload_version)
+ mtk_ppe_stop(&eth->ppe);
+
return 0;
}
@@ -2370,6 +2401,64 @@ err_disable_clks:
return ret;
}
+static void mtk_dim_rx(struct work_struct *work)
+{
+ struct dim *dim = container_of(work, struct dim, work);
+ struct mtk_eth *eth = container_of(dim, struct mtk_eth, rx_dim);
+ struct dim_cq_moder cur_profile;
+ u32 val, cur;
+
+ cur_profile = net_dim_get_rx_moderation(eth->rx_dim.mode,
+ dim->profile_ix);
+ spin_lock_bh(&eth->dim_lock);
+
+ val = mtk_r32(eth, MTK_PDMA_DELAY_INT);
+ val &= MTK_PDMA_DELAY_TX_MASK;
+ val |= MTK_PDMA_DELAY_RX_EN;
+
+ cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
+ val |= cur << MTK_PDMA_DELAY_RX_PTIME_SHIFT;
+
+ cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
+ val |= cur << MTK_PDMA_DELAY_RX_PINT_SHIFT;
+
+ mtk_w32(eth, val, MTK_PDMA_DELAY_INT);
+ mtk_w32(eth, val, MTK_QDMA_DELAY_INT);
+
+ spin_unlock_bh(&eth->dim_lock);
+
+ dim->state = DIM_START_MEASURE;
+}
+
+static void mtk_dim_tx(struct work_struct *work)
+{
+ struct dim *dim = container_of(work, struct dim, work);
+ struct mtk_eth *eth = container_of(dim, struct mtk_eth, tx_dim);
+ struct dim_cq_moder cur_profile;
+ u32 val, cur;
+
+ cur_profile = net_dim_get_tx_moderation(eth->tx_dim.mode,
+ dim->profile_ix);
+ spin_lock_bh(&eth->dim_lock);
+
+ val = mtk_r32(eth, MTK_PDMA_DELAY_INT);
+ val &= MTK_PDMA_DELAY_RX_MASK;
+ val |= MTK_PDMA_DELAY_TX_EN;
+
+ cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
+ val |= cur << MTK_PDMA_DELAY_TX_PTIME_SHIFT;
+
+ cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
+ val |= cur << MTK_PDMA_DELAY_TX_PINT_SHIFT;
+
+ mtk_w32(eth, val, MTK_PDMA_DELAY_INT);
+ mtk_w32(eth, val, MTK_QDMA_DELAY_INT);
+
+ spin_unlock_bh(&eth->dim_lock);
+
+ dim->state = DIM_START_MEASURE;
+}
+
static int mtk_hw_init(struct mtk_eth *eth)
{
int i, val, ret;
@@ -2391,9 +2480,6 @@ static int mtk_hw_init(struct mtk_eth *eth)
goto err_disable_pm;
}
- /* enable interrupt delay for RX */
- mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);
-
/* disable delay and normal interrupt */
mtk_tx_irq_disable(eth, ~0);
mtk_rx_irq_disable(eth, ~0);
@@ -2432,11 +2518,11 @@ static int mtk_hw_init(struct mtk_eth *eth)
/* Enable RX VLan Offloading */
mtk_w32(eth, 1, MTK_CDMP_EG_CTRL);
- /* enable interrupt delay for RX */
- mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);
+ /* set interrupt delays based on current Net DIM sample */
+ mtk_dim_rx(&eth->rx_dim.work);
+ mtk_dim_tx(&eth->tx_dim.work);
/* disable delay and normal interrupt */
- mtk_w32(eth, 0, MTK_QDMA_DELAY_INT);
mtk_tx_irq_disable(eth, ~0);
mtk_rx_irq_disable(eth, ~0);
@@ -2473,14 +2559,11 @@ static int __init mtk_init(struct net_device *dev)
{
struct mtk_mac *mac = netdev_priv(dev);
struct mtk_eth *eth = mac->hw;
- const char *mac_addr;
-
- mac_addr = of_get_mac_address(mac->of_node);
- if (!IS_ERR(mac_addr))
- ether_addr_copy(dev->dev_addr, mac_addr);
+ int ret;
- /* If the mac address is invalid, use random mac address */
- if (!is_valid_ether_addr(dev->dev_addr)) {
+ ret = of_get_mac_address(mac->of_node, dev->dev_addr);
+ if (ret) {
+ /* If the mac address is invalid, use random mac address */
eth_hw_addr_random(dev);
dev_err(eth->dev, "generated random MAC address %pM\n",
dev->dev_addr);
@@ -2832,6 +2915,7 @@ static const struct net_device_ops mtk_netdev_ops = {
#ifdef CONFIG_NET_POLL_CONTROLLER
.ndo_poll_controller = mtk_poll_controller,
#endif
+ .ndo_setup_tc = mtk_eth_setup_tc,
};
static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
@@ -2973,6 +3057,13 @@ static int mtk_probe(struct platform_device *pdev)
spin_lock_init(&eth->page_lock);
spin_lock_init(&eth->tx_irq_lock);
spin_lock_init(&eth->rx_irq_lock);
+ spin_lock_init(&eth->dim_lock);
+
+ eth->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+ INIT_WORK(&eth->rx_dim.work, mtk_dim_rx);
+
+ eth->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+ INIT_WORK(&eth->tx_dim.work, mtk_dim_tx);
if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
eth->ethsys = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
@@ -3088,6 +3179,17 @@ static int mtk_probe(struct platform_device *pdev)
goto err_free_dev;
}
+ if (eth->soc->offload_version) {
+ err = mtk_ppe_init(&eth->ppe, eth->dev,
+ eth->base + MTK_ETH_PPE_BASE, 2);
+ if (err)
+ goto err_free_dev;
+
+ err = mtk_eth_offload_init(eth);
+ if (err)
+ goto err_free_dev;
+ }
+
for (i = 0; i < MTK_MAX_DEVS; i++) {
if (!eth->netdev[i])
continue;
@@ -3162,6 +3264,7 @@ static const struct mtk_soc_data mt7621_data = {
.hw_features = MTK_HW_FEATURES,
.required_clks = MT7621_CLKS_BITMAP,
.required_pctl = false,
+ .offload_version = 2,
};
static const struct mtk_soc_data mt7622_data = {
@@ -3170,6 +3273,7 @@ static const struct mtk_soc_data mt7622_data = {
.hw_features = MTK_HW_FEATURES,
.required_clks = MT7622_CLKS_BITMAP,
.required_pctl = false,
+ .offload_version = 2,
};
static const struct mtk_soc_data mt7623_data = {
@@ -3177,6 +3281,7 @@ static const struct mtk_soc_data mt7623_data = {
.hw_features = MTK_HW_FEATURES,
.required_clks = MT7623_CLKS_BITMAP,
.required_pctl = true,
+ .offload_version = 2,
};
static const struct mtk_soc_data mt7629_data = {
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
index fd3cec8f06ba..11331b44ba07 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
@@ -15,12 +15,15 @@
#include <linux/u64_stats_sync.h>
#include <linux/refcount.h>
#include <linux/phylink.h>
+#include <linux/rhashtable.h>
+#include <linux/dim.h>
+#include "mtk_ppe.h"
#define MTK_QDMA_PAGE_SIZE 2048
#define MTK_MAX_RX_LENGTH 1536
#define MTK_MAX_RX_LENGTH_2K 2048
#define MTK_TX_DMA_BUF_LEN 0x3fff
-#define MTK_DMA_SIZE 256
+#define MTK_DMA_SIZE 512
#define MTK_NAPI_WEIGHT 64
#define MTK_MAC_COUNT 2
#define MTK_RX_ETH_HLEN (ETH_HLEN + ETH_FCS_LEN)
@@ -40,7 +43,8 @@
NETIF_F_HW_VLAN_CTAG_RX | \
NETIF_F_SG | NETIF_F_TSO | \
NETIF_F_TSO6 | \
- NETIF_F_IPV6_CSUM)
+ NETIF_F_IPV6_CSUM |\
+ NETIF_F_HW_TC)
#define MTK_HW_FEATURES_MT7628 (NETIF_F_SG | NETIF_F_RXCSUM)
#define NEXT_DESP_IDX(X, Y) (((X) + 1) & ((Y) - 1))
@@ -82,10 +86,12 @@
/* GDM Exgress Control Register */
#define MTK_GDMA_FWD_CFG(x) (0x500 + (x * 0x1000))
+#define MTK_GDMA_SPECIAL_TAG BIT(24)
#define MTK_GDMA_ICS_EN BIT(22)
#define MTK_GDMA_TCS_EN BIT(21)
#define MTK_GDMA_UCS_EN BIT(20)
#define MTK_GDMA_TO_PDMA 0x0
+#define MTK_GDMA_TO_PPE 0x4444
#define MTK_GDMA_DROP_ALL 0x7777
/* Unicast Filter MAC Address Register - Low */
@@ -132,13 +138,18 @@
/* PDMA Delay Interrupt Register */
#define MTK_PDMA_DELAY_INT 0xa0c
+#define MTK_PDMA_DELAY_RX_MASK GENMASK(15, 0)
#define MTK_PDMA_DELAY_RX_EN BIT(15)
-#define MTK_PDMA_DELAY_RX_PINT 4
#define MTK_PDMA_DELAY_RX_PINT_SHIFT 8
-#define MTK_PDMA_DELAY_RX_PTIME 4
-#define MTK_PDMA_DELAY_RX_DELAY \
- (MTK_PDMA_DELAY_RX_EN | MTK_PDMA_DELAY_RX_PTIME | \
- (MTK_PDMA_DELAY_RX_PINT << MTK_PDMA_DELAY_RX_PINT_SHIFT))
+#define MTK_PDMA_DELAY_RX_PTIME_SHIFT 0
+
+#define MTK_PDMA_DELAY_TX_MASK GENMASK(31, 16)
+#define MTK_PDMA_DELAY_TX_EN BIT(31)
+#define MTK_PDMA_DELAY_TX_PINT_SHIFT 24
+#define MTK_PDMA_DELAY_TX_PTIME_SHIFT 16
+
+#define MTK_PDMA_DELAY_PINT_MASK 0x7f
+#define MTK_PDMA_DELAY_PTIME_MASK 0xff
/* PDMA Interrupt Status Register */
#define MTK_PDMA_INT_STATUS 0xa20
@@ -198,12 +209,12 @@
#define MTK_RX_BT_32DWORDS (3 << 11)
#define MTK_NDP_CO_PRO BIT(10)
#define MTK_TX_WB_DDONE BIT(6)
-#define MTK_DMA_SIZE_16DWORDS (2 << 4)
+#define MTK_TX_BT_32DWORDS (3 << 4)
#define MTK_RX_DMA_BUSY BIT(3)
#define MTK_TX_DMA_BUSY BIT(1)
#define MTK_RX_DMA_EN BIT(2)
#define MTK_TX_DMA_EN BIT(0)
-#define MTK_DMA_BUSY_TIMEOUT HZ
+#define MTK_DMA_BUSY_TIMEOUT_US 1000000
/* QDMA Reset Index Register */
#define MTK_QDMA_RST_IDX 0x1A08
@@ -220,6 +231,7 @@
/* QDMA Interrupt Status Register */
#define MTK_QDMA_INT_STATUS 0x1A18
#define MTK_RX_DONE_DLY BIT(30)
+#define MTK_TX_DONE_DLY BIT(28)
#define MTK_RX_DONE_INT3 BIT(19)
#define MTK_RX_DONE_INT2 BIT(18)
#define MTK_RX_DONE_INT1 BIT(17)
@@ -229,8 +241,7 @@
#define MTK_TX_DONE_INT1 BIT(1)
#define MTK_TX_DONE_INT0 BIT(0)
#define MTK_RX_DONE_INT MTK_RX_DONE_DLY
-#define MTK_TX_DONE_INT (MTK_TX_DONE_INT0 | MTK_TX_DONE_INT1 | \
- MTK_TX_DONE_INT2 | MTK_TX_DONE_INT3)
+#define MTK_TX_DONE_INT MTK_TX_DONE_DLY
/* QDMA Interrupt grouping registers */
#define MTK_QDMA_INT_GRP1 0x1a20
@@ -296,15 +307,23 @@
#define RX_DMA_LSO BIT(30)
#define RX_DMA_PLEN0(_x) (((_x) & 0x3fff) << 16)
#define RX_DMA_GET_PLEN0(_x) (((_x) >> 16) & 0x3fff)
+#define RX_DMA_VTAG BIT(15)
/* QDMA descriptor rxd3 */
#define RX_DMA_VID(_x) ((_x) & 0xfff)
/* QDMA descriptor rxd4 */
+#define MTK_RXD4_FOE_ENTRY GENMASK(13, 0)
+#define MTK_RXD4_PPE_CPU_REASON GENMASK(18, 14)
+#define MTK_RXD4_SRC_PORT GENMASK(21, 19)
+#define MTK_RXD4_ALG GENMASK(31, 22)
+
+/* QDMA descriptor rxd4 */
#define RX_DMA_L4_VALID BIT(24)
#define RX_DMA_L4_VALID_PDMA BIT(30) /* when PDMA is used */
#define RX_DMA_FPORT_SHIFT 19
#define RX_DMA_FPORT_MASK 0x7
+#define RX_DMA_SPECIAL_TAG BIT(22)
/* PHY Indirect Access Control registers */
#define MTK_PHY_IAC 0x10004
@@ -623,6 +642,7 @@ struct mtk_tx_buf {
* @phys: The physical addr of tx_buf
* @next_free: Pointer to the next free descriptor
* @last_free: Pointer to the last free descriptor
+ * @last_free_ptr: Hardware pointer value of the last free descriptor
* @thresh: The threshold of minimum amount of free descriptors
* @free_count: QDMA uses a linked list. Track how many free descriptors
* are present
@@ -633,6 +653,7 @@ struct mtk_tx_ring {
dma_addr_t phys;
struct mtk_tx_dma *next_free;
struct mtk_tx_dma *last_free;
+ u32 last_free_ptr;
u16 thresh;
atomic_t free_count;
int dma_size;
@@ -802,6 +823,7 @@ struct mtk_soc_data {
u32 caps;
u32 required_clks;
bool required_pctl;
+ u8 offload_version;
netdev_features_t hw_features;
};
@@ -835,6 +857,7 @@ struct mtk_sgmii {
* @page_lock: Make sure that register operations are atomic
* @tx_irq__lock: Make sure that IRQ register operations are atomic
* @rx_irq__lock: Make sure that IRQ register operations are atomic
+ * @dim_lock: Make sure that Net DIM operations are atomic
* @dummy_dev: we run 2 netdevs on 1 physical DMA ring and need a
* dummy for NAPI to work
* @netdev: The netdev instances
@@ -853,6 +876,14 @@ struct mtk_sgmii {
* @rx_ring_qdma: Pointer to the memory holding info about the QDMA RX ring
* @tx_napi: The TX NAPI struct
* @rx_napi: The RX NAPI struct
+ * @rx_events: Net DIM RX event counter
+ * @rx_packets: Net DIM RX packet counter
+ * @rx_bytes: Net DIM RX byte counter
+ * @rx_dim: Net DIM RX context
+ * @tx_events: Net DIM TX event counter
+ * @tx_packets: Net DIM TX packet counter
+ * @tx_bytes: Net DIM TX byte counter
+ * @tx_dim: Net DIM TX context
* @scratch_ring: Newer SoCs need memory for a second HW managed TX ring
* @phy_scratch_ring: physical address of scratch_ring
* @scratch_head: The scratch memory that scratch_ring points to.
@@ -897,10 +928,25 @@ struct mtk_eth {
const struct mtk_soc_data *soc;
+ spinlock_t dim_lock;
+
+ u32 rx_events;
+ u32 rx_packets;
+ u32 rx_bytes;
+ struct dim rx_dim;
+
+ u32 tx_events;
+ u32 tx_packets;
+ u32 tx_bytes;
+ struct dim tx_dim;
+
u32 tx_int_mask_reg;
u32 tx_int_status_reg;
u32 rx_dma_l4_valid;
int ip_align;
+
+ struct mtk_ppe ppe;
+ struct rhashtable flow_table;
};
/* struct mtk_mac - the structure that holds the info about the MACs of the
@@ -945,4 +991,9 @@ int mtk_gmac_sgmii_path_setup(struct mtk_eth *eth, int mac_id);
int mtk_gmac_gephy_path_setup(struct mtk_eth *eth, int mac_id);
int mtk_gmac_rgmii_path_setup(struct mtk_eth *eth, int mac_id);
+int mtk_eth_offload_init(struct mtk_eth *eth);
+int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type,
+ void *type_data);
+
+
#endif /* MTK_ETH_H */
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c
new file mode 100644
index 000000000000..3ad10c793308
--- /dev/null
+++ b/drivers/net/ethernet/mediatek/mtk_ppe.c
@@ -0,0 +1,509 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
+
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/etherdevice.h>
+#include <linux/platform_device.h>
+#include "mtk_ppe.h"
+#include "mtk_ppe_regs.h"
+
+static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)
+{
+ writel(val, ppe->base + reg);
+}
+
+static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg)
+{
+ return readl(ppe->base + reg);
+}
+
+static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set)
+{
+ u32 val;
+
+ val = ppe_r32(ppe, reg);
+ val &= ~mask;
+ val |= set;
+ ppe_w32(ppe, reg, val);
+
+ return val;
+}
+
+static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val)
+{
+ return ppe_m32(ppe, reg, 0, val);
+}
+
+static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val)
+{
+ return ppe_m32(ppe, reg, val, 0);
+}
+
+static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
+{
+ int ret;
+ u32 val;
+
+ ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val,
+ !(val & MTK_PPE_GLO_CFG_BUSY),
+ 20, MTK_PPE_WAIT_TIMEOUT_US);
+
+ if (ret)
+ dev_err(ppe->dev, "PPE table busy");
+
+ return ret;
+}
+
+static void mtk_ppe_cache_clear(struct mtk_ppe *ppe)
+{
+ ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
+ ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
+}
+
+static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable)
+{
+ mtk_ppe_cache_clear(ppe);
+
+ ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN,
+ enable * MTK_PPE_CACHE_CTL_EN);
+}
+
+static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e)
+{
+ u32 hv1, hv2, hv3;
+ u32 hash;
+
+ switch (FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, e->ib1)) {
+ case MTK_PPE_PKT_TYPE_BRIDGE:
+ hv1 = e->bridge.src_mac_lo;
+ hv1 ^= ((e->bridge.src_mac_hi & 0xffff) << 16);
+ hv2 = e->bridge.src_mac_hi >> 16;
+ hv2 ^= e->bridge.dest_mac_lo;
+ hv3 = e->bridge.dest_mac_hi;
+ break;
+ case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
+ case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
+ hv1 = e->ipv4.orig.ports;
+ hv2 = e->ipv4.orig.dest_ip;
+ hv3 = e->ipv4.orig.src_ip;
+ break;
+ case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
+ case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
+ hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3];
+ hv1 ^= e->ipv6.ports;
+
+ hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2];
+ hv2 ^= e->ipv6.dest_ip[0];
+
+ hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1];
+ hv3 ^= e->ipv6.src_ip[0];
+ break;
+ case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
+ case MTK_PPE_PKT_TYPE_IPV6_6RD:
+ default:
+ WARN_ON_ONCE(1);
+ return MTK_PPE_HASH_MASK;
+ }
+
+ hash = (hv1 & hv2) | ((~hv1) & hv3);
+ hash = (hash >> 24) | ((hash & 0xffffff) << 8);
+ hash ^= hv1 ^ hv2 ^ hv3;
+ hash ^= hash >> 16;
+ hash <<= 1;
+ hash &= MTK_PPE_ENTRIES - 1;
+
+ return hash;
+}
+
+static inline struct mtk_foe_mac_info *
+mtk_foe_entry_l2(struct mtk_foe_entry *entry)
+{
+ int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
+
+ if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
+ return &entry->ipv6.l2;
+
+ return &entry->ipv4.l2;
+}
+
+static inline u32 *
+mtk_foe_entry_ib2(struct mtk_foe_entry *entry)
+{
+ int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
+
+ if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
+ return &entry->ipv6.ib2;
+
+ return &entry->ipv4.ib2;
+}
+
+int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto,
+ u8 pse_port, u8 *src_mac, u8 *dest_mac)
+{
+ struct mtk_foe_mac_info *l2;
+ u32 ports_pad, val;
+
+ memset(entry, 0, sizeof(*entry));
+
+ val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
+ FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) |
+ FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
+ MTK_FOE_IB1_BIND_TTL |
+ MTK_FOE_IB1_BIND_CACHE;
+ entry->ib1 = val;
+
+ val = FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) |
+ FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f) |
+ FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port);
+
+ if (is_multicast_ether_addr(dest_mac))
+ val |= MTK_FOE_IB2_MULTICAST;
+
+ ports_pad = 0xa5a5a500 | (l4proto & 0xff);
+ if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
+ entry->ipv4.orig.ports = ports_pad;
+ if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
+ entry->ipv6.ports = ports_pad;
+
+ if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) {
+ entry->ipv6.ib2 = val;
+ l2 = &entry->ipv6.l2;
+ } else {
+ entry->ipv4.ib2 = val;
+ l2 = &entry->ipv4.l2;
+ }
+
+ l2->dest_mac_hi = get_unaligned_be32(dest_mac);
+ l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4);
+ l2->src_mac_hi = get_unaligned_be32(src_mac);
+ l2->src_mac_lo = get_unaligned_be16(src_mac + 4);
+
+ if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
+ l2->etype = ETH_P_IPV6;
+ else
+ l2->etype = ETH_P_IP;
+
+ return 0;
+}
+
+int mtk_foe_entry_set_pse_port(struct mtk_foe_entry *entry, u8 port)
+{
+ u32 *ib2 = mtk_foe_entry_ib2(entry);
+ u32 val;
+
+ val = *ib2;
+ val &= ~MTK_FOE_IB2_DEST_PORT;
+ val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port);
+ *ib2 = val;
+
+ return 0;
+}
+
+int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool egress,
+ __be32 src_addr, __be16 src_port,
+ __be32 dest_addr, __be16 dest_port)
+{
+ int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
+ struct mtk_ipv4_tuple *t;
+
+ switch (type) {
+ case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
+ if (egress) {
+ t = &entry->ipv4.new;
+ break;
+ }
+ fallthrough;
+ case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
+ case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
+ t = &entry->ipv4.orig;
+ break;
+ case MTK_PPE_PKT_TYPE_IPV6_6RD:
+ entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr);
+ entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr);
+ return 0;
+ default:
+ WARN_ON_ONCE(1);
+ return -EINVAL;
+ }
+
+ t->src_ip = be32_to_cpu(src_addr);
+ t->dest_ip = be32_to_cpu(dest_addr);
+
+ if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
+ return 0;
+
+ t->src_port = be16_to_cpu(src_port);
+ t->dest_port = be16_to_cpu(dest_port);
+
+ return 0;
+}
+
+int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry,
+ __be32 *src_addr, __be16 src_port,
+ __be32 *dest_addr, __be16 dest_port)
+{
+ int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
+ u32 *src, *dest;
+ int i;
+
+ switch (type) {
+ case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
+ src = entry->dslite.tunnel_src_ip;
+ dest = entry->dslite.tunnel_dest_ip;
+ break;
+ case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
+ case MTK_PPE_PKT_TYPE_IPV6_6RD:
+ entry->ipv6.src_port = be16_to_cpu(src_port);
+ entry->ipv6.dest_port = be16_to_cpu(dest_port);
+ fallthrough;
+ case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
+ src = entry->ipv6.src_ip;
+ dest = entry->ipv6.dest_ip;
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < 4; i++)
+ src[i] = be32_to_cpu(src_addr[i]);
+ for (i = 0; i < 4; i++)
+ dest[i] = be32_to_cpu(dest_addr[i]);
+
+ return 0;
+}
+
+int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port)
+{
+ struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
+
+ l2->etype = BIT(port);
+
+ if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER))
+ entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1);
+ else
+ l2->etype |= BIT(8);
+
+ entry->ib1 &= ~MTK_FOE_IB1_BIND_VLAN_TAG;
+
+ return 0;
+}
+
+int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid)
+{
+ struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
+
+ switch (FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER, entry->ib1)) {
+ case 0:
+ entry->ib1 |= MTK_FOE_IB1_BIND_VLAN_TAG |
+ FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1);
+ l2->vlan1 = vid;
+ return 0;
+ case 1:
+ if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG)) {
+ l2->vlan1 = vid;
+ l2->etype |= BIT(8);
+ } else {
+ l2->vlan2 = vid;
+ entry->ib1 += FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1);
+ }
+ return 0;
+ default:
+ return -ENOSPC;
+ }
+}
+
+int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid)
+{
+ struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
+
+ if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER) ||
+ (entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG))
+ l2->etype = ETH_P_PPP_SES;
+
+ entry->ib1 |= MTK_FOE_IB1_BIND_PPPOE;
+ l2->pppoe_id = sid;
+
+ return 0;
+}
+
+static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry)
+{
+ return !(entry->ib1 & MTK_FOE_IB1_STATIC) &&
+ FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1) != MTK_FOE_STATE_BIND;
+}
+
+int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
+ u16 timestamp)
+{
+ struct mtk_foe_entry *hwe;
+ u32 hash;
+
+ timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP;
+ entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
+ entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp);
+
+ hash = mtk_ppe_hash_entry(entry);
+ hwe = &ppe->foe_table[hash];
+ if (!mtk_foe_entry_usable(hwe)) {
+ hwe++;
+ hash++;
+
+ if (!mtk_foe_entry_usable(hwe))
+ return -ENOSPC;
+ }
+
+ memcpy(&hwe->data, &entry->data, sizeof(hwe->data));
+ wmb();
+ hwe->ib1 = entry->ib1;
+
+ dma_wmb();
+
+ mtk_ppe_cache_clear(ppe);
+
+ return hash;
+}
+
+int mtk_ppe_init(struct mtk_ppe *ppe, struct device *dev, void __iomem *base,
+ int version)
+{
+ struct mtk_foe_entry *foe;
+
+ /* need to allocate a separate device, since it PPE DMA access is
+ * not coherent.
+ */
+ ppe->base = base;
+ ppe->dev = dev;
+ ppe->version = version;
+
+ foe = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*foe),
+ &ppe->foe_phys, GFP_KERNEL);
+ if (!foe)
+ return -ENOMEM;
+
+ ppe->foe_table = foe;
+
+ mtk_ppe_debugfs_init(ppe);
+
+ return 0;
+}
+
+static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)
+{
+ static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 };
+ int i, k;
+
+ memset(ppe->foe_table, 0, MTK_PPE_ENTRIES * sizeof(ppe->foe_table));
+
+ if (!IS_ENABLED(CONFIG_SOC_MT7621))
+ return;
+
+ /* skip all entries that cross the 1024 byte boundary */
+ for (i = 0; i < MTK_PPE_ENTRIES; i += 128)
+ for (k = 0; k < ARRAY_SIZE(skip); k++)
+ ppe->foe_table[i + skip[k]].ib1 |= MTK_FOE_IB1_STATIC;
+}
+
+int mtk_ppe_start(struct mtk_ppe *ppe)
+{
+ u32 val;
+
+ mtk_ppe_init_foe_table(ppe);
+ ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys);
+
+ val = MTK_PPE_TB_CFG_ENTRY_80B |
+ MTK_PPE_TB_CFG_AGE_NON_L4 |
+ MTK_PPE_TB_CFG_AGE_UNBIND |
+ MTK_PPE_TB_CFG_AGE_TCP |
+ MTK_PPE_TB_CFG_AGE_UDP |
+ MTK_PPE_TB_CFG_AGE_TCP_FIN |
+ FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS,
+ MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) |
+ FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE,
+ MTK_PPE_KEEPALIVE_DISABLE) |
+ FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) |
+ FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE,
+ MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) |
+ FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM,
+ MTK_PPE_ENTRIES_SHIFT);
+ ppe_w32(ppe, MTK_PPE_TB_CFG, val);
+
+ ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK,
+ MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6);
+
+ mtk_ppe_cache_enable(ppe, true);
+
+ val = MTK_PPE_FLOW_CFG_IP4_TCP_FRAG |
+ MTK_PPE_FLOW_CFG_IP4_UDP_FRAG |
+ MTK_PPE_FLOW_CFG_IP6_3T_ROUTE |
+ MTK_PPE_FLOW_CFG_IP6_5T_ROUTE |
+ MTK_PPE_FLOW_CFG_IP6_6RD |
+ MTK_PPE_FLOW_CFG_IP4_NAT |
+ MTK_PPE_FLOW_CFG_IP4_NAPT |
+ MTK_PPE_FLOW_CFG_IP4_DSLITE |
+ MTK_PPE_FLOW_CFG_L2_BRIDGE |
+ MTK_PPE_FLOW_CFG_IP4_NAT_FRAG;
+ ppe_w32(ppe, MTK_PPE_FLOW_CFG, val);
+
+ val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) |
+ FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3);
+ ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val);
+
+ val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) |
+ FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1);
+ ppe_w32(ppe, MTK_PPE_BIND_AGE0, val);
+
+ val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) |
+ FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7);
+ ppe_w32(ppe, MTK_PPE_BIND_AGE1, val);
+
+ val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF;
+ ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val);
+
+ val = MTK_PPE_BIND_LIMIT1_FULL |
+ FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1);
+ ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val);
+
+ val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) |
+ FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1);
+ ppe_w32(ppe, MTK_PPE_BIND_RATE, val);
+
+ /* enable PPE */
+ val = MTK_PPE_GLO_CFG_EN |
+ MTK_PPE_GLO_CFG_IP4_L4_CS_DROP |
+ MTK_PPE_GLO_CFG_IP4_CS_DROP |
+ MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE;
+ ppe_w32(ppe, MTK_PPE_GLO_CFG, val);
+
+ ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0);
+
+ return 0;
+}
+
+int mtk_ppe_stop(struct mtk_ppe *ppe)
+{
+ u32 val;
+ int i;
+
+ for (i = 0; i < MTK_PPE_ENTRIES; i++)
+ ppe->foe_table[i].ib1 = FIELD_PREP(MTK_FOE_IB1_STATE,
+ MTK_FOE_STATE_INVALID);
+
+ mtk_ppe_cache_enable(ppe, false);
+
+ /* disable offload engine */
+ ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN);
+ ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0);
+
+ /* disable aging */
+ val = MTK_PPE_TB_CFG_AGE_NON_L4 |
+ MTK_PPE_TB_CFG_AGE_UNBIND |
+ MTK_PPE_TB_CFG_AGE_TCP |
+ MTK_PPE_TB_CFG_AGE_UDP |
+ MTK_PPE_TB_CFG_AGE_TCP_FIN;
+ ppe_clear(ppe, MTK_PPE_TB_CFG, val);
+
+ return mtk_ppe_wait_busy(ppe);
+}
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h
new file mode 100644
index 000000000000..242fb8f2ae65
--- /dev/null
+++ b/drivers/net/ethernet/mediatek/mtk_ppe.h
@@ -0,0 +1,288 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
+
+#ifndef __MTK_PPE_H
+#define __MTK_PPE_H
+
+#include <linux/kernel.h>
+#include <linux/bitfield.h>
+
+#define MTK_ETH_PPE_BASE 0xc00
+
+#define MTK_PPE_ENTRIES_SHIFT 3
+#define MTK_PPE_ENTRIES (1024 << MTK_PPE_ENTRIES_SHIFT)
+#define MTK_PPE_HASH_MASK (MTK_PPE_ENTRIES - 1)
+#define MTK_PPE_WAIT_TIMEOUT_US 1000000
+
+#define MTK_FOE_IB1_UNBIND_TIMESTAMP GENMASK(7, 0)
+#define MTK_FOE_IB1_UNBIND_PACKETS GENMASK(23, 8)
+#define MTK_FOE_IB1_UNBIND_PREBIND BIT(24)
+
+#define MTK_FOE_IB1_BIND_TIMESTAMP GENMASK(14, 0)
+#define MTK_FOE_IB1_BIND_KEEPALIVE BIT(15)
+#define MTK_FOE_IB1_BIND_VLAN_LAYER GENMASK(18, 16)
+#define MTK_FOE_IB1_BIND_PPPOE BIT(19)
+#define MTK_FOE_IB1_BIND_VLAN_TAG BIT(20)
+#define MTK_FOE_IB1_BIND_PKT_SAMPLE BIT(21)
+#define MTK_FOE_IB1_BIND_CACHE BIT(22)
+#define MTK_FOE_IB1_BIND_TUNNEL_DECAP BIT(23)
+#define MTK_FOE_IB1_BIND_TTL BIT(24)
+
+#define MTK_FOE_IB1_PACKET_TYPE GENMASK(27, 25)
+#define MTK_FOE_IB1_STATE GENMASK(29, 28)
+#define MTK_FOE_IB1_UDP BIT(30)
+#define MTK_FOE_IB1_STATIC BIT(31)
+
+enum {
+ MTK_PPE_PKT_TYPE_IPV4_HNAPT = 0,
+ MTK_PPE_PKT_TYPE_IPV4_ROUTE = 1,
+ MTK_PPE_PKT_TYPE_BRIDGE = 2,
+ MTK_PPE_PKT_TYPE_IPV4_DSLITE = 3,
+ MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T = 4,
+ MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T = 5,
+ MTK_PPE_PKT_TYPE_IPV6_6RD = 7,
+};
+
+#define MTK_FOE_IB2_QID GENMASK(3, 0)
+#define MTK_FOE_IB2_PSE_QOS BIT(4)
+#define MTK_FOE_IB2_DEST_PORT GENMASK(7, 5)
+#define MTK_FOE_IB2_MULTICAST BIT(8)
+
+#define MTK_FOE_IB2_WHNAT_QID2 GENMASK(13, 12)
+#define MTK_FOE_IB2_WHNAT_DEVIDX BIT(16)
+#define MTK_FOE_IB2_WHNAT_NAT BIT(17)
+
+#define MTK_FOE_IB2_PORT_MG GENMASK(17, 12)
+
+#define MTK_FOE_IB2_PORT_AG GENMASK(23, 18)
+
+#define MTK_FOE_IB2_DSCP GENMASK(31, 24)
+
+#define MTK_FOE_VLAN2_WHNAT_BSS GEMMASK(5, 0)
+#define MTK_FOE_VLAN2_WHNAT_WCID GENMASK(13, 6)
+#define MTK_FOE_VLAN2_WHNAT_RING GENMASK(15, 14)
+
+enum {
+ MTK_FOE_STATE_INVALID,
+ MTK_FOE_STATE_UNBIND,
+ MTK_FOE_STATE_BIND,
+ MTK_FOE_STATE_FIN
+};
+
+struct mtk_foe_mac_info {
+ u16 vlan1;
+ u16 etype;
+
+ u32 dest_mac_hi;
+
+ u16 vlan2;
+ u16 dest_mac_lo;
+
+ u32 src_mac_hi;
+
+ u16 pppoe_id;
+ u16 src_mac_lo;
+};
+
+struct mtk_foe_bridge {
+ u32 dest_mac_hi;
+
+ u16 src_mac_lo;
+ u16 dest_mac_lo;
+
+ u32 src_mac_hi;
+
+ u32 ib2;
+
+ u32 _rsv[5];
+
+ u32 udf_tsid;
+ struct mtk_foe_mac_info l2;
+};
+
+struct mtk_ipv4_tuple {
+ u32 src_ip;
+ u32 dest_ip;
+ union {
+ struct {
+ u16 dest_port;
+ u16 src_port;
+ };
+ struct {
+ u8 protocol;
+ u8 _pad[3]; /* fill with 0xa5a5a5 */
+ };
+ u32 ports;
+ };
+};
+
+struct mtk_foe_ipv4 {
+ struct mtk_ipv4_tuple orig;
+
+ u32 ib2;
+
+ struct mtk_ipv4_tuple new;
+
+ u16 timestamp;
+ u16 _rsv0[3];
+
+ u32 udf_tsid;
+
+ struct mtk_foe_mac_info l2;
+};
+
+struct mtk_foe_ipv4_dslite {
+ struct mtk_ipv4_tuple ip4;
+
+ u32 tunnel_src_ip[4];
+ u32 tunnel_dest_ip[4];
+
+ u8 flow_label[3];
+ u8 priority;
+
+ u32 udf_tsid;
+
+ u32 ib2;
+
+ struct mtk_foe_mac_info l2;
+};
+
+struct mtk_foe_ipv6 {
+ u32 src_ip[4];
+ u32 dest_ip[4];
+
+ union {
+ struct {
+ u8 protocol;
+ u8 _pad[3]; /* fill with 0xa5a5a5 */
+ }; /* 3-tuple */
+ struct {
+ u16 dest_port;
+ u16 src_port;
+ }; /* 5-tuple */
+ u32 ports;
+ };
+
+ u32 _rsv[3];
+
+ u32 udf;
+
+ u32 ib2;
+ struct mtk_foe_mac_info l2;
+};
+
+struct mtk_foe_ipv6_6rd {
+ u32 src_ip[4];
+ u32 dest_ip[4];
+ u16 dest_port;
+ u16 src_port;
+
+ u32 tunnel_src_ip;
+ u32 tunnel_dest_ip;
+
+ u16 hdr_csum;
+ u8 dscp;
+ u8 ttl;
+
+ u8 flag;
+ u8 pad;
+ u8 per_flow_6rd_id;
+ u8 pad2;
+
+ u32 ib2;
+ struct mtk_foe_mac_info l2;
+};
+
+struct mtk_foe_entry {
+ u32 ib1;
+
+ union {
+ struct mtk_foe_bridge bridge;
+ struct mtk_foe_ipv4 ipv4;
+ struct mtk_foe_ipv4_dslite dslite;
+ struct mtk_foe_ipv6 ipv6;
+ struct mtk_foe_ipv6_6rd ipv6_6rd;
+ u32 data[19];
+ };
+};
+
+enum {
+ MTK_PPE_CPU_REASON_TTL_EXCEEDED = 0x02,
+ MTK_PPE_CPU_REASON_OPTION_HEADER = 0x03,
+ MTK_PPE_CPU_REASON_NO_FLOW = 0x07,
+ MTK_PPE_CPU_REASON_IPV4_FRAG = 0x08,
+ MTK_PPE_CPU_REASON_IPV4_DSLITE_FRAG = 0x09,
+ MTK_PPE_CPU_REASON_IPV4_DSLITE_NO_TCP_UDP = 0x0a,
+ MTK_PPE_CPU_REASON_IPV6_6RD_NO_TCP_UDP = 0x0b,
+ MTK_PPE_CPU_REASON_TCP_FIN_SYN_RST = 0x0c,
+ MTK_PPE_CPU_REASON_UN_HIT = 0x0d,
+ MTK_PPE_CPU_REASON_HIT_UNBIND = 0x0e,
+ MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED = 0x0f,
+ MTK_PPE_CPU_REASON_HIT_BIND_TCP_FIN = 0x10,
+ MTK_PPE_CPU_REASON_HIT_TTL_1 = 0x11,
+ MTK_PPE_CPU_REASON_HIT_BIND_VLAN_VIOLATION = 0x12,
+ MTK_PPE_CPU_REASON_KEEPALIVE_UC_OLD_HDR = 0x13,
+ MTK_PPE_CPU_REASON_KEEPALIVE_MC_NEW_HDR = 0x14,
+ MTK_PPE_CPU_REASON_KEEPALIVE_DUP_OLD_HDR = 0x15,
+ MTK_PPE_CPU_REASON_HIT_BIND_FORCE_CPU = 0x16,
+ MTK_PPE_CPU_REASON_TUNNEL_OPTION_HEADER = 0x17,
+ MTK_PPE_CPU_REASON_MULTICAST_TO_CPU = 0x18,
+ MTK_PPE_CPU_REASON_MULTICAST_TO_GMAC1_CPU = 0x19,
+ MTK_PPE_CPU_REASON_HIT_PRE_BIND = 0x1a,
+ MTK_PPE_CPU_REASON_PACKET_SAMPLING = 0x1b,
+ MTK_PPE_CPU_REASON_EXCEED_MTU = 0x1c,
+ MTK_PPE_CPU_REASON_PPE_BYPASS = 0x1e,
+ MTK_PPE_CPU_REASON_INVALID = 0x1f,
+};
+
+struct mtk_ppe {
+ struct device *dev;
+ void __iomem *base;
+ int version;
+
+ struct mtk_foe_entry *foe_table;
+ dma_addr_t foe_phys;
+
+ void *acct_table;
+};
+
+int mtk_ppe_init(struct mtk_ppe *ppe, struct device *dev, void __iomem *base,
+ int version);
+int mtk_ppe_start(struct mtk_ppe *ppe);
+int mtk_ppe_stop(struct mtk_ppe *ppe);
+
+static inline void
+mtk_foe_entry_clear(struct mtk_ppe *ppe, u16 hash)
+{
+ ppe->foe_table[hash].ib1 = 0;
+ dma_wmb();
+}
+
+static inline int
+mtk_foe_entry_timestamp(struct mtk_ppe *ppe, u16 hash)
+{
+ u32 ib1 = READ_ONCE(ppe->foe_table[hash].ib1);
+
+ if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND)
+ return -1;
+
+ return FIELD_GET(MTK_FOE_IB1_BIND_TIMESTAMP, ib1);
+}
+
+int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto,
+ u8 pse_port, u8 *src_mac, u8 *dest_mac);
+int mtk_foe_entry_set_pse_port(struct mtk_foe_entry *entry, u8 port);
+int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool orig,
+ __be32 src_addr, __be16 src_port,
+ __be32 dest_addr, __be16 dest_port);
+int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry,
+ __be32 *src_addr, __be16 src_port,
+ __be32 *dest_addr, __be16 dest_port);
+int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port);
+int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid);
+int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid);
+int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
+ u16 timestamp);
+int mtk_ppe_debugfs_init(struct mtk_ppe *ppe);
+
+#endif
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
new file mode 100644
index 000000000000..98b1d3577bcd
--- /dev/null
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
+
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+#include "mtk_eth_soc.h"
+
+struct mtk_flow_addr_info
+{
+ void *src, *dest;
+ u16 *src_port, *dest_port;
+ bool ipv6;
+};
+
+static const char *mtk_foe_entry_state_str(int state)
+{
+ static const char * const state_str[] = {
+ [MTK_FOE_STATE_INVALID] = "INV",
+ [MTK_FOE_STATE_UNBIND] = "UNB",
+ [MTK_FOE_STATE_BIND] = "BND",
+ [MTK_FOE_STATE_FIN] = "FIN",
+ };
+
+ if (state >= ARRAY_SIZE(state_str) || !state_str[state])
+ return "UNK";
+
+ return state_str[state];
+}
+
+static const char *mtk_foe_pkt_type_str(int type)
+{
+ static const char * const type_str[] = {
+ [MTK_PPE_PKT_TYPE_IPV4_HNAPT] = "IPv4 5T",
+ [MTK_PPE_PKT_TYPE_IPV4_ROUTE] = "IPv4 3T",
+ [MTK_PPE_PKT_TYPE_BRIDGE] = "L2",
+ [MTK_PPE_PKT_TYPE_IPV4_DSLITE] = "DS-LITE",
+ [MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T] = "IPv6 3T",
+ [MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T] = "IPv6 5T",
+ [MTK_PPE_PKT_TYPE_IPV6_6RD] = "6RD",
+ };
+
+ if (type >= ARRAY_SIZE(type_str) || !type_str[type])
+ return "UNKNOWN";
+
+ return type_str[type];
+}
+
+static void
+mtk_print_addr(struct seq_file *m, u32 *addr, bool ipv6)
+{
+ u32 n_addr[4];
+ int i;
+
+ if (!ipv6) {
+ seq_printf(m, "%pI4h", addr);
+ return;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(n_addr); i++)
+ n_addr[i] = htonl(addr[i]);
+ seq_printf(m, "%pI6", n_addr);
+}
+
+static void
+mtk_print_addr_info(struct seq_file *m, struct mtk_flow_addr_info *ai)
+{
+ mtk_print_addr(m, ai->src, ai->ipv6);
+ if (ai->src_port)
+ seq_printf(m, ":%d", *ai->src_port);
+ seq_printf(m, "->");
+ mtk_print_addr(m, ai->dest, ai->ipv6);
+ if (ai->dest_port)
+ seq_printf(m, ":%d", *ai->dest_port);
+}
+
+static int
+mtk_ppe_debugfs_foe_show(struct seq_file *m, void *private, bool bind)
+{
+ struct mtk_ppe *ppe = m->private;
+ int i;
+
+ for (i = 0; i < MTK_PPE_ENTRIES; i++) {
+ struct mtk_foe_entry *entry = &ppe->foe_table[i];
+ struct mtk_foe_mac_info *l2;
+ struct mtk_flow_addr_info ai = {};
+ unsigned char h_source[ETH_ALEN];
+ unsigned char h_dest[ETH_ALEN];
+ int type, state;
+ u32 ib2;
+
+
+ state = FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1);
+ if (!state)
+ continue;
+
+ if (bind && state != MTK_FOE_STATE_BIND)
+ continue;
+
+ type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
+ seq_printf(m, "%05x %s %7s", i,
+ mtk_foe_entry_state_str(state),
+ mtk_foe_pkt_type_str(type));
+
+ switch (type) {
+ case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
+ case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
+ ai.src_port = &entry->ipv4.orig.src_port;
+ ai.dest_port = &entry->ipv4.orig.dest_port;
+ fallthrough;
+ case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
+ ai.src = &entry->ipv4.orig.src_ip;
+ ai.dest = &entry->ipv4.orig.dest_ip;
+ break;
+ case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
+ ai.src_port = &entry->ipv6.src_port;
+ ai.dest_port = &entry->ipv6.dest_port;
+ fallthrough;
+ case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
+ case MTK_PPE_PKT_TYPE_IPV6_6RD:
+ ai.src = &entry->ipv6.src_ip;
+ ai.dest = &entry->ipv6.dest_ip;
+ ai.ipv6 = true;
+ break;
+ }
+
+ seq_printf(m, " orig=");
+ mtk_print_addr_info(m, &ai);
+
+ switch (type) {
+ case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
+ case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
+ ai.src_port = &entry->ipv4.new.src_port;
+ ai.dest_port = &entry->ipv4.new.dest_port;
+ fallthrough;
+ case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
+ ai.src = &entry->ipv4.new.src_ip;
+ ai.dest = &entry->ipv4.new.dest_ip;
+ seq_printf(m, " new=");
+ mtk_print_addr_info(m, &ai);
+ break;
+ }
+
+ if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) {
+ l2 = &entry->ipv6.l2;
+ ib2 = entry->ipv6.ib2;
+ } else {
+ l2 = &entry->ipv4.l2;
+ ib2 = entry->ipv4.ib2;
+ }
+
+ *((__be32 *)h_source) = htonl(l2->src_mac_hi);
+ *((__be16 *)&h_source[4]) = htons(l2->src_mac_lo);
+ *((__be32 *)h_dest) = htonl(l2->dest_mac_hi);
+ *((__be16 *)&h_dest[4]) = htons(l2->dest_mac_lo);
+
+ seq_printf(m, " eth=%pM->%pM etype=%04x"
+ " vlan=%d,%d ib1=%08x ib2=%08x\n",
+ h_source, h_dest, ntohs(l2->etype),
+ l2->vlan1, l2->vlan2, entry->ib1, ib2);
+ }
+
+ return 0;
+}
+
+static int
+mtk_ppe_debugfs_foe_show_all(struct seq_file *m, void *private)
+{
+ return mtk_ppe_debugfs_foe_show(m, private, false);
+}
+
+static int
+mtk_ppe_debugfs_foe_show_bind(struct seq_file *m, void *private)
+{
+ return mtk_ppe_debugfs_foe_show(m, private, true);
+}
+
+static int
+mtk_ppe_debugfs_foe_open_all(struct inode *inode, struct file *file)
+{
+ return single_open(file, mtk_ppe_debugfs_foe_show_all,
+ inode->i_private);
+}
+
+static int
+mtk_ppe_debugfs_foe_open_bind(struct inode *inode, struct file *file)
+{
+ return single_open(file, mtk_ppe_debugfs_foe_show_bind,
+ inode->i_private);
+}
+
+int mtk_ppe_debugfs_init(struct mtk_ppe *ppe)
+{
+ static const struct file_operations fops_all = {
+ .open = mtk_ppe_debugfs_foe_open_all,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+ };
+
+ static const struct file_operations fops_bind = {
+ .open = mtk_ppe_debugfs_foe_open_bind,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+ };
+
+ struct dentry *root;
+
+ root = debugfs_create_dir("mtk_ppe", NULL);
+ if (!root)
+ return -ENOMEM;
+
+ debugfs_create_file("entries", S_IRUGO, root, ppe, &fops_all);
+ debugfs_create_file("bind", S_IRUGO, root, ppe, &fops_bind);
+
+ return 0;
+}
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
new file mode 100644
index 000000000000..b5f68f66d42a
--- /dev/null
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
@@ -0,0 +1,495 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Felix Fietkau <nbd@nbd.name>
+ */
+
+#include <linux/if_ether.h>
+#include <linux/rhashtable.h>
+#include <linux/ip.h>
+#include <net/flow_offload.h>
+#include <net/pkt_cls.h>
+#include <net/dsa.h>
+#include "mtk_eth_soc.h"
+
+struct mtk_flow_data {
+ struct ethhdr eth;
+
+ union {
+ struct {
+ __be32 src_addr;
+ __be32 dst_addr;
+ } v4;
+ };
+
+ __be16 src_port;
+ __be16 dst_port;
+
+ struct {
+ u16 id;
+ __be16 proto;
+ u8 num;
+ } vlan;
+ struct {
+ u16 sid;
+ u8 num;
+ } pppoe;
+};
+
+struct mtk_flow_entry {
+ struct rhash_head node;
+ unsigned long cookie;
+ u16 hash;
+};
+
+static const struct rhashtable_params mtk_flow_ht_params = {
+ .head_offset = offsetof(struct mtk_flow_entry, node),
+ .key_offset = offsetof(struct mtk_flow_entry, cookie),
+ .key_len = sizeof(unsigned long),
+ .automatic_shrinking = true,
+};
+
+static u32
+mtk_eth_timestamp(struct mtk_eth *eth)
+{
+ return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP;
+}
+
+static int
+mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data,
+ bool egress)
+{
+ return mtk_foe_entry_set_ipv4_tuple(foe, egress,
+ data->v4.src_addr, data->src_port,
+ data->v4.dst_addr, data->dst_port);
+}
+
+static void
+mtk_flow_offload_mangle_eth(const struct flow_action_entry *act, void *eth)
+{
+ void *dest = eth + act->mangle.offset;
+ const void *src = &act->mangle.val;
+
+ if (act->mangle.offset > 8)
+ return;
+
+ if (act->mangle.mask == 0xffff) {
+ src += 2;
+ dest += 2;
+ }
+
+ memcpy(dest, src, act->mangle.mask ? 2 : 4);
+}
+
+
+static int
+mtk_flow_mangle_ports(const struct flow_action_entry *act,
+ struct mtk_flow_data *data)
+{
+ u32 val = ntohl(act->mangle.val);
+
+ switch (act->mangle.offset) {
+ case 0:
+ if (act->mangle.mask == ~htonl(0xffff))
+ data->dst_port = cpu_to_be16(val);
+ else
+ data->src_port = cpu_to_be16(val >> 16);
+ break;
+ case 2:
+ data->dst_port = cpu_to_be16(val);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int
+mtk_flow_mangle_ipv4(const struct flow_action_entry *act,
+ struct mtk_flow_data *data)
+{
+ __be32 *dest;
+
+ switch (act->mangle.offset) {
+ case offsetof(struct iphdr, saddr):
+ dest = &data->v4.src_addr;
+ break;
+ case offsetof(struct iphdr, daddr):
+ dest = &data->v4.dst_addr;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ memcpy(dest, &act->mangle.val, sizeof(u32));
+
+ return 0;
+}
+
+static int
+mtk_flow_get_dsa_port(struct net_device **dev)
+{
+#if IS_ENABLED(CONFIG_NET_DSA)
+ struct dsa_port *dp;
+
+ dp = dsa_port_from_netdev(*dev);
+ if (IS_ERR(dp))
+ return -ENODEV;
+
+ if (dp->cpu_dp->tag_ops->proto != DSA_TAG_PROTO_MTK)
+ return -ENODEV;
+
+ *dev = dp->cpu_dp->master;
+
+ return dp->index;
+#else
+ return -ENODEV;
+#endif
+}
+
+static int
+mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe,
+ struct net_device *dev)
+{
+ int pse_port, dsa_port;
+
+ dsa_port = mtk_flow_get_dsa_port(&dev);
+ if (dsa_port >= 0)
+ mtk_foe_entry_set_dsa(foe, dsa_port);
+
+ if (dev == eth->netdev[0])
+ pse_port = 1;
+ else if (dev == eth->netdev[1])
+ pse_port = 2;
+ else
+ return -EOPNOTSUPP;
+
+ mtk_foe_entry_set_pse_port(foe, pse_port);
+
+ return 0;
+}
+
+static int
+mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
+{
+ struct flow_rule *rule = flow_cls_offload_flow_rule(f);
+ struct flow_action_entry *act;
+ struct mtk_flow_data data = {};
+ struct mtk_foe_entry foe;
+ struct net_device *odev = NULL;
+ struct mtk_flow_entry *entry;
+ int offload_type = 0;
+ u16 addr_type = 0;
+ u32 timestamp;
+ u8 l4proto = 0;
+ int err = 0;
+ int hash;
+ int i;
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) {
+ struct flow_match_meta match;
+
+ flow_rule_match_meta(rule, &match);
+ } else {
+ return -EOPNOTSUPP;
+ }
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
+ struct flow_match_control match;
+
+ flow_rule_match_control(rule, &match);
+ addr_type = match.key->addr_type;
+ } else {
+ return -EOPNOTSUPP;
+ }
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
+ struct flow_match_basic match;
+
+ flow_rule_match_basic(rule, &match);
+ l4proto = match.key->ip_proto;
+ } else {
+ return -EOPNOTSUPP;
+ }
+
+ flow_action_for_each(i, act, &rule->action) {
+ switch (act->id) {
+ case FLOW_ACTION_MANGLE:
+ if (act->mangle.htype == FLOW_ACT_MANGLE_HDR_TYPE_ETH)
+ mtk_flow_offload_mangle_eth(act, &data.eth);
+ break;
+ case FLOW_ACTION_REDIRECT:
+ odev = act->dev;
+ break;
+ case FLOW_ACTION_CSUM:
+ break;
+ case FLOW_ACTION_VLAN_PUSH:
+ if (data.vlan.num == 1 ||
+ act->vlan.proto != htons(ETH_P_8021Q))
+ return -EOPNOTSUPP;
+
+ data.vlan.id = act->vlan.vid;
+ data.vlan.proto = act->vlan.proto;
+ data.vlan.num++;
+ break;
+ case FLOW_ACTION_VLAN_POP:
+ break;
+ case FLOW_ACTION_PPPOE_PUSH:
+ if (data.pppoe.num == 1)
+ return -EOPNOTSUPP;
+
+ data.pppoe.sid = act->pppoe.sid;
+ data.pppoe.num++;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+ }
+
+ switch (addr_type) {
+ case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
+ offload_type = MTK_PPE_PKT_TYPE_IPV4_HNAPT;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ if (!is_valid_ether_addr(data.eth.h_source) ||
+ !is_valid_ether_addr(data.eth.h_dest))
+ return -EINVAL;
+
+ err = mtk_foe_entry_prepare(&foe, offload_type, l4proto, 0,
+ data.eth.h_source,
+ data.eth.h_dest);
+ if (err)
+ return err;
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
+ struct flow_match_ports ports;
+
+ flow_rule_match_ports(rule, &ports);
+ data.src_port = ports.key->src;
+ data.dst_port = ports.key->dst;
+ } else {
+ return -EOPNOTSUPP;
+ }
+
+ if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
+ struct flow_match_ipv4_addrs addrs;
+
+ flow_rule_match_ipv4_addrs(rule, &addrs);
+
+ data.v4.src_addr = addrs.key->src;
+ data.v4.dst_addr = addrs.key->dst;
+
+ mtk_flow_set_ipv4_addr(&foe, &data, false);
+ }
+
+ flow_action_for_each(i, act, &rule->action) {
+ if (act->id != FLOW_ACTION_MANGLE)
+ continue;
+
+ switch (act->mangle.htype) {
+ case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
+ case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
+ err = mtk_flow_mangle_ports(act, &data);
+ break;
+ case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
+ err = mtk_flow_mangle_ipv4(act, &data);
+ break;
+ case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
+ /* handled earlier */
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ if (err)
+ return err;
+ }
+
+ if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
+ err = mtk_flow_set_ipv4_addr(&foe, &data, true);
+ if (err)
+ return err;
+ }
+
+ if (data.vlan.num == 1) {
+ if (data.vlan.proto != htons(ETH_P_8021Q))
+ return -EOPNOTSUPP;
+
+ mtk_foe_entry_set_vlan(&foe, data.vlan.id);
+ }
+ if (data.pppoe.num == 1)
+ mtk_foe_entry_set_pppoe(&foe, data.pppoe.sid);
+
+ err = mtk_flow_set_output_device(eth, &foe, odev);
+ if (err)
+ return err;
+
+ entry = kzalloc(sizeof(*entry), GFP_KERNEL);
+ if (!entry)
+ return -ENOMEM;
+
+ entry->cookie = f->cookie;
+ timestamp = mtk_eth_timestamp(eth);
+ hash = mtk_foe_entry_commit(&eth->ppe, &foe, timestamp);
+ if (hash < 0) {
+ err = hash;
+ goto free;
+ }
+
+ entry->hash = hash;
+ err = rhashtable_insert_fast(&eth->flow_table, &entry->node,
+ mtk_flow_ht_params);
+ if (err < 0)
+ goto clear_flow;
+
+ return 0;
+clear_flow:
+ mtk_foe_entry_clear(&eth->ppe, hash);
+free:
+ kfree(entry);
+ return err;
+}
+
+static int
+mtk_flow_offload_destroy(struct mtk_eth *eth, struct flow_cls_offload *f)
+{
+ struct mtk_flow_entry *entry;
+
+ entry = rhashtable_lookup(&eth->flow_table, &f->cookie,
+ mtk_flow_ht_params);
+ if (!entry)
+ return -ENOENT;
+
+ mtk_foe_entry_clear(&eth->ppe, entry->hash);
+ rhashtable_remove_fast(&eth->flow_table, &entry->node,
+ mtk_flow_ht_params);
+ kfree(entry);
+
+ return 0;
+}
+
+static int
+mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)
+{
+ struct mtk_flow_entry *entry;
+ int timestamp;
+ u32 idle;
+
+ entry = rhashtable_lookup(&eth->flow_table, &f->cookie,
+ mtk_flow_ht_params);
+ if (!entry)
+ return -ENOENT;
+
+ timestamp = mtk_foe_entry_timestamp(&eth->ppe, entry->hash);
+ if (timestamp < 0)
+ return -ETIMEDOUT;
+
+ idle = mtk_eth_timestamp(eth) - timestamp;
+ f->stats.lastused = jiffies - idle * HZ;
+
+ return 0;
+}
+
+static DEFINE_MUTEX(mtk_flow_offload_mutex);
+
+static int
+mtk_eth_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
+{
+ struct flow_cls_offload *cls = type_data;
+ struct net_device *dev = cb_priv;
+ struct mtk_mac *mac = netdev_priv(dev);
+ struct mtk_eth *eth = mac->hw;
+ int err;
+
+ if (!tc_can_offload(dev))
+ return -EOPNOTSUPP;
+
+ if (type != TC_SETUP_CLSFLOWER)
+ return -EOPNOTSUPP;
+
+ mutex_lock(&mtk_flow_offload_mutex);
+ switch (cls->command) {
+ case FLOW_CLS_REPLACE:
+ err = mtk_flow_offload_replace(eth, cls);
+ break;
+ case FLOW_CLS_DESTROY:
+ err = mtk_flow_offload_destroy(eth, cls);
+ break;
+ case FLOW_CLS_STATS:
+ err = mtk_flow_offload_stats(eth, cls);
+ break;
+ default:
+ err = -EOPNOTSUPP;
+ break;
+ }
+ mutex_unlock(&mtk_flow_offload_mutex);
+
+ return err;
+}
+
+static int
+mtk_eth_setup_tc_block(struct net_device *dev, struct flow_block_offload *f)
+{
+ struct mtk_mac *mac = netdev_priv(dev);
+ struct mtk_eth *eth = mac->hw;
+ static LIST_HEAD(block_cb_list);
+ struct flow_block_cb *block_cb;
+ flow_setup_cb_t *cb;
+
+ if (!eth->ppe.foe_table)
+ return -EOPNOTSUPP;
+
+ if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
+ return -EOPNOTSUPP;
+
+ cb = mtk_eth_setup_tc_block_cb;
+ f->driver_block_list = &block_cb_list;
+
+ switch (f->command) {
+ case FLOW_BLOCK_BIND:
+ block_cb = flow_block_cb_lookup(f->block, cb, dev);
+ if (block_cb) {
+ flow_block_cb_incref(block_cb);
+ return 0;
+ }
+ block_cb = flow_block_cb_alloc(cb, dev, dev, NULL);
+ if (IS_ERR(block_cb))
+ return PTR_ERR(block_cb);
+
+ flow_block_cb_add(block_cb, f);
+ list_add_tail(&block_cb->driver_list, &block_cb_list);
+ return 0;
+ case FLOW_BLOCK_UNBIND:
+ block_cb = flow_block_cb_lookup(f->block, cb, dev);
+ if (!block_cb)
+ return -ENOENT;
+
+ if (flow_block_cb_decref(block_cb)) {
+ flow_block_cb_remove(block_cb, f);
+ list_del(&block_cb->driver_list);
+ }
+ return 0;
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type,
+ void *type_data)
+{
+ if (type == TC_SETUP_FT)
+ return mtk_eth_setup_tc_block(dev, type_data);
+
+ return -EOPNOTSUPP;
+}
+
+int mtk_eth_offload_init(struct mtk_eth *eth)
+{
+ if (!eth->ppe.foe_table)
+ return 0;
+
+ return rhashtable_init(&eth->flow_table, &mtk_flow_ht_params);
+}
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
new file mode 100644
index 000000000000..0c45ea0900f1
--- /dev/null
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
+
+#ifndef __MTK_PPE_REGS_H
+#define __MTK_PPE_REGS_H
+
+#define MTK_PPE_GLO_CFG 0x200
+#define MTK_PPE_GLO_CFG_EN BIT(0)
+#define MTK_PPE_GLO_CFG_TSID_EN BIT(1)
+#define MTK_PPE_GLO_CFG_IP4_L4_CS_DROP BIT(2)
+#define MTK_PPE_GLO_CFG_IP4_CS_DROP BIT(3)
+#define MTK_PPE_GLO_CFG_TTL0_DROP BIT(4)
+#define MTK_PPE_GLO_CFG_PPE_BSWAP BIT(5)
+#define MTK_PPE_GLO_CFG_PSE_HASH_OFS BIT(6)
+#define MTK_PPE_GLO_CFG_MCAST_TB_EN BIT(7)
+#define MTK_PPE_GLO_CFG_FLOW_DROP_KA BIT(8)
+#define MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE BIT(9)
+#define MTK_PPE_GLO_CFG_UDP_LITE_EN BIT(10)
+#define MTK_PPE_GLO_CFG_UDP_LEN_DROP BIT(11)
+#define MTK_PPE_GLO_CFG_MCAST_ENTRIES GNEMASK(13, 12)
+#define MTK_PPE_GLO_CFG_BUSY BIT(31)
+
+#define MTK_PPE_FLOW_CFG 0x204
+#define MTK_PPE_FLOW_CFG_IP4_TCP_FRAG BIT(6)
+#define MTK_PPE_FLOW_CFG_IP4_UDP_FRAG BIT(7)
+#define MTK_PPE_FLOW_CFG_IP6_3T_ROUTE BIT(8)
+#define MTK_PPE_FLOW_CFG_IP6_5T_ROUTE BIT(9)
+#define MTK_PPE_FLOW_CFG_IP6_6RD BIT(10)
+#define MTK_PPE_FLOW_CFG_IP4_NAT BIT(12)
+#define MTK_PPE_FLOW_CFG_IP4_NAPT BIT(13)
+#define MTK_PPE_FLOW_CFG_IP4_DSLITE BIT(14)
+#define MTK_PPE_FLOW_CFG_L2_BRIDGE BIT(15)
+#define MTK_PPE_FLOW_CFG_IP_PROTO_BLACKLIST BIT(16)
+#define MTK_PPE_FLOW_CFG_IP4_NAT_FRAG BIT(17)
+#define MTK_PPE_FLOW_CFG_IP4_HASH_FLOW_LABEL BIT(18)
+#define MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY BIT(19)
+#define MTK_PPE_FLOW_CFG_IP6_HASH_GRE_KEY BIT(20)
+
+#define MTK_PPE_IP_PROTO_CHK 0x208
+#define MTK_PPE_IP_PROTO_CHK_IPV4 GENMASK(15, 0)
+#define MTK_PPE_IP_PROTO_CHK_IPV6 GENMASK(31, 16)
+
+#define MTK_PPE_TB_CFG 0x21c
+#define MTK_PPE_TB_CFG_ENTRY_NUM GENMASK(2, 0)
+#define MTK_PPE_TB_CFG_ENTRY_80B BIT(3)
+#define MTK_PPE_TB_CFG_SEARCH_MISS GENMASK(5, 4)
+#define MTK_PPE_TB_CFG_AGE_PREBIND BIT(6)
+#define MTK_PPE_TB_CFG_AGE_NON_L4 BIT(7)
+#define MTK_PPE_TB_CFG_AGE_UNBIND BIT(8)
+#define MTK_PPE_TB_CFG_AGE_TCP BIT(9)
+#define MTK_PPE_TB_CFG_AGE_UDP BIT(10)
+#define MTK_PPE_TB_CFG_AGE_TCP_FIN BIT(11)
+#define MTK_PPE_TB_CFG_KEEPALIVE GENMASK(13, 12)
+#define MTK_PPE_TB_CFG_HASH_MODE GENMASK(15, 14)
+#define MTK_PPE_TB_CFG_SCAN_MODE GENMASK(17, 16)
+#define MTK_PPE_TB_CFG_HASH_DEBUG GENMASK(19, 18)
+
+enum {
+ MTK_PPE_SCAN_MODE_DISABLED,
+ MTK_PPE_SCAN_MODE_CHECK_AGE,
+ MTK_PPE_SCAN_MODE_KEEPALIVE_AGE,
+};
+
+enum {
+ MTK_PPE_KEEPALIVE_DISABLE,
+ MTK_PPE_KEEPALIVE_UNICAST_CPU,
+ MTK_PPE_KEEPALIVE_DUP_CPU = 3,
+};
+
+enum {
+ MTK_PPE_SEARCH_MISS_ACTION_DROP,
+ MTK_PPE_SEARCH_MISS_ACTION_FORWARD = 2,
+ MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD = 3,
+};
+
+#define MTK_PPE_TB_BASE 0x220
+
+#define MTK_PPE_TB_USED 0x224
+#define MTK_PPE_TB_USED_NUM GENMASK(13, 0)
+
+#define MTK_PPE_BIND_RATE 0x228
+#define MTK_PPE_BIND_RATE_BIND GENMASK(15, 0)
+#define MTK_PPE_BIND_RATE_PREBIND GENMASK(31, 16)
+
+#define MTK_PPE_BIND_LIMIT0 0x22c
+#define MTK_PPE_BIND_LIMIT0_QUARTER GENMASK(13, 0)
+#define MTK_PPE_BIND_LIMIT0_HALF GENMASK(29, 16)
+
+#define MTK_PPE_BIND_LIMIT1 0x230
+#define MTK_PPE_BIND_LIMIT1_FULL GENMASK(13, 0)
+#define MTK_PPE_BIND_LIMIT1_NON_L4 GENMASK(23, 16)
+
+#define MTK_PPE_KEEPALIVE 0x234
+#define MTK_PPE_KEEPALIVE_TIME GENMASK(15, 0)
+#define MTK_PPE_KEEPALIVE_TIME_TCP GENMASK(23, 16)
+#define MTK_PPE_KEEPALIVE_TIME_UDP GENMASK(31, 24)
+
+#define MTK_PPE_UNBIND_AGE 0x238
+#define MTK_PPE_UNBIND_AGE_MIN_PACKETS GENMASK(31, 16)
+#define MTK_PPE_UNBIND_AGE_DELTA GENMASK(7, 0)
+
+#define MTK_PPE_BIND_AGE0 0x23c
+#define MTK_PPE_BIND_AGE0_DELTA_NON_L4 GENMASK(30, 16)
+#define MTK_PPE_BIND_AGE0_DELTA_UDP GENMASK(14, 0)
+
+#define MTK_PPE_BIND_AGE1 0x240
+#define MTK_PPE_BIND_AGE1_DELTA_TCP_FIN GENMASK(30, 16)
+#define MTK_PPE_BIND_AGE1_DELTA_TCP GENMASK(14, 0)
+
+#define MTK_PPE_HASH_SEED 0x244
+
+#define MTK_PPE_DEFAULT_CPU_PORT 0x248
+#define MTK_PPE_DEFAULT_CPU_PORT_MASK(_n) (GENMASK(2, 0) << ((_n) * 4))
+
+#define MTK_PPE_MTU_DROP 0x308
+
+#define MTK_PPE_VLAN_MTU0 0x30c
+#define MTK_PPE_VLAN_MTU0_NONE GENMASK(13, 0)
+#define MTK_PPE_VLAN_MTU0_1TAG GENMASK(29, 16)
+
+#define MTK_PPE_VLAN_MTU1 0x310
+#define MTK_PPE_VLAN_MTU1_2TAG GENMASK(13, 0)
+#define MTK_PPE_VLAN_MTU1_3TAG GENMASK(29, 16)
+
+#define MTK_PPE_VPM_TPID 0x318
+
+#define MTK_PPE_CACHE_CTL 0x320
+#define MTK_PPE_CACHE_CTL_EN BIT(0)
+#define MTK_PPE_CACHE_CTL_LOCK_CLR BIT(4)
+#define MTK_PPE_CACHE_CTL_REQ BIT(8)
+#define MTK_PPE_CACHE_CTL_CLEAR BIT(9)
+#define MTK_PPE_CACHE_CTL_CMD GENMASK(13, 12)
+
+#define MTK_PPE_MIB_CFG 0x334
+#define MTK_PPE_MIB_CFG_EN BIT(0)
+#define MTK_PPE_MIB_CFG_RD_CLR BIT(1)
+
+#define MTK_PPE_MIB_TB_BASE 0x338
+
+#define MTK_PPE_MIB_CACHE_CTL 0x350
+#define MTK_PPE_MIB_CACHE_CTL_EN BIT(0)
+#define MTK_PPE_MIB_CACHE_CTL_FLUSH BIT(2)
+
+#endif
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-07 22:52:43 +0100