diff options
| -rw-r--r-- | drivers/net/ethernet/sfc/Makefile | 1 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/efx.c | 40 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/efx.h | 107 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/falcon.c | 31 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/farch.c | 1108 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/filter.c | 1244 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/net_driver.h | 46 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/nic.h | 28 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/rx.c | 94 | ||||
| -rw-r--r-- | drivers/net/ethernet/sfc/siena.c | 16 |
10 files changed, 1448 insertions, 1267 deletions
diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile index ef7410f014d6..a61272661a73 100644 --- a/drivers/net/ethernet/sfc/Makefile +++ b/drivers/net/ethernet/sfc/Makefile @@ -1,5 +1,4 @@ sfc-y += efx.o nic.o farch.o falcon.o siena.o tx.o rx.o \ - filter.o \ selftest.o ethtool.o qt202x_phy.o mdio_10g.o \ tenxpress.o txc43128_phy.o falcon_boards.o \ mcdi.o mcdi_port.o mcdi_mon.o ptp.o diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 49d06ca79d7d..a2daaae266d7 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -611,7 +611,7 @@ static void efx_start_datapath(struct efx_nic *efx) /* RX filters also have scatter-enabled flags */ if (efx->rx_scatter != old_rx_scatter) - efx_filter_update_rx_scatter(efx); + efx->type->filter_update_rx_scatter(efx); /* We must keep at least one descriptor in a TX ring empty. * We could avoid this when the queue size does not exactly @@ -1499,6 +1499,44 @@ static void efx_remove_nic(struct efx_nic *efx) efx->type->remove(efx); } +static int efx_probe_filters(struct efx_nic *efx) +{ + int rc; + + spin_lock_init(&efx->filter_lock); + + rc = efx->type->filter_table_probe(efx); + if (rc) + return rc; + +#ifdef CONFIG_RFS_ACCEL + if (efx->type->offload_features & NETIF_F_NTUPLE) { + efx->rps_flow_id = kcalloc(efx->type->max_rx_ip_filters, + sizeof(*efx->rps_flow_id), + GFP_KERNEL); + if (!efx->rps_flow_id) { + efx->type->filter_table_remove(efx); + return -ENOMEM; + } + } +#endif + + return 0; +} + +static void efx_remove_filters(struct efx_nic *efx) +{ +#ifdef CONFIG_RFS_ACCEL + kfree(efx->rps_flow_id); +#endif + efx->type->filter_table_remove(efx); +} + +static void efx_restore_filters(struct efx_nic *efx) +{ + efx->type->filter_table_restore(efx); +} + /************************************************************************** * * NIC startup/shutdown diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h index 45de5b9fedbe..9e3573872e57 100644 --- a/drivers/net/ethernet/sfc/efx.h +++ b/drivers/net/ethernet/sfc/efx.h @@ -68,27 +68,92 @@ extern void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue); #define EFX_TXQ_MIN_ENT(efx) (2 * efx_tx_max_skb_descs(efx)) /* Filters */ -extern int efx_probe_filters(struct efx_nic *efx); -extern void efx_restore_filters(struct efx_nic *efx); -extern void efx_remove_filters(struct efx_nic *efx); -extern void efx_filter_update_rx_scatter(struct efx_nic *efx); -extern s32 efx_filter_insert_filter(struct efx_nic *efx, - struct efx_filter_spec *spec, - bool replace); -extern int efx_filter_remove_id_safe(struct efx_nic *efx, - enum efx_filter_priority priority, - u32 filter_id); -extern int efx_filter_get_filter_safe(struct efx_nic *efx, - enum efx_filter_priority priority, - u32 filter_id, struct efx_filter_spec *); -extern void efx_filter_clear_rx(struct efx_nic *efx, - enum efx_filter_priority priority); -extern u32 efx_filter_count_rx_used(struct efx_nic *efx, - enum efx_filter_priority priority); -extern u32 efx_filter_get_rx_id_limit(struct efx_nic *efx); -extern s32 efx_filter_get_rx_ids(struct efx_nic *efx, - enum efx_filter_priority priority, - u32 *buf, u32 size); + +/** + * efx_filter_insert_filter - add or replace a filter + * @efx: NIC in which to insert the filter + * @spec: Specification for the filter + * @replace_equal: Flag for whether the specified filter may replace an + * existing filter with equal priority + * + * On success, return the filter ID. + * On failure, return a negative error code. + * + * If an existing filter has equal match values to the new filter + * spec, then the new filter might replace it, depending on the + * relative priorities. If the existing filter has lower priority, or + * if @replace_equal is set and it has equal priority, then it is + * replaced. Otherwise the function fails, returning -%EPERM if + * the existing filter has higher priority or -%EEXIST if it has + * equal priority. + */ +static inline s32 efx_filter_insert_filter(struct efx_nic *efx, + struct efx_filter_spec *spec, + bool replace_equal) +{ + return efx->type->filter_insert(efx, spec, replace_equal); +} + +/** + * efx_filter_remove_id_safe - remove a filter by ID, carefully + * @efx: NIC from which to remove the filter + * @priority: Priority of filter, as passed to @efx_filter_insert_filter + * @filter_id: ID of filter, as returned by @efx_filter_insert_filter + * + * This function will range-check @filter_id, so it is safe to call + * with a value passed from userland. + */ +static inline int efx_filter_remove_id_safe(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 filter_id) +{ + return efx->type->filter_remove_safe(efx, priority, filter_id); +} + +/** + * efx_filter_get_filter_safe - retrieve a filter by ID, carefully + * @efx: NIC from which to remove the filter + * @priority: Priority of filter, as passed to @efx_filter_insert_filter + * @filter_id: ID of filter, as returned by @efx_filter_insert_filter + * @spec: Buffer in which to store filter specification + * + * This function will range-check @filter_id, so it is safe to call + * with a value passed from userland. + */ +static inline int +efx_filter_get_filter_safe(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 filter_id, struct efx_filter_spec *spec) +{ + return efx->type->filter_get_safe(efx, priority, filter_id, spec); +} + +/** + * efx_farch_filter_clear_rx - remove RX filters by priority + * @efx: NIC from which to remove the filters + * @priority: Maximum priority to remove + */ +static inline void efx_filter_clear_rx(struct efx_nic *efx, + enum efx_filter_priority priority) +{ + return efx->type->filter_clear_rx(efx, priority); +} + +static inline u32 efx_filter_count_rx_used(struct efx_nic *efx, + enum efx_filter_priority priority) +{ + return efx->type->filter_count_rx_used(efx, priority); +} +static inline u32 efx_filter_get_rx_id_limit(struct efx_nic *efx) +{ + return efx->type->filter_get_rx_id_limit(efx); +} +static inline s32 efx_filter_get_rx_ids(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 *buf, u32 size) +{ + return efx->type->filter_get_rx_ids(efx, priority, buf, size); +} #ifdef CONFIG_RFS_ACCEL extern int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, u16 rxq_index, u32 flow_id); diff --git a/drivers/net/ethernet/sfc/falcon.c b/drivers/net/ethernet/sfc/falcon.c index 0fd8a88905a6..6ea28f8e8792 100644 --- a/drivers/net/ethernet/sfc/falcon.c +++ b/drivers/net/ethernet/sfc/falcon.c @@ -2400,6 +2400,21 @@ const struct efx_nic_type falcon_a1_nic_type = { .ev_read_ack = efx_farch_ev_read_ack, .ev_test_generate = efx_farch_ev_test_generate, + /* We don't expose the filter table on Falcon A1 as it is not + * mapped into function 0, but these implementations still + * work with a degenerate case of all tables set to size 0. + */ + .filter_table_probe = efx_farch_filter_table_probe, + .filter_table_restore = efx_farch_filter_table_restore, + .filter_table_remove = efx_farch_filter_table_remove, + .filter_insert = efx_farch_filter_insert, + .filter_remove_safe = efx_farch_filter_remove_safe, + .filter_get_safe = efx_farch_filter_get_safe, + .filter_clear_rx = efx_farch_filter_clear_rx, + .filter_count_rx_used = efx_farch_filter_count_rx_used, + .filter_get_rx_id_limit = efx_farch_filter_get_rx_id_limit, + .filter_get_rx_ids = efx_farch_filter_get_rx_ids, + .revision = EFX_REV_FALCON_A1, .txd_ptr_tbl_base = FR_AA_TX_DESC_PTR_TBL_KER, .rxd_ptr_tbl_base = FR_AA_RX_DESC_PTR_TBL_KER, @@ -2468,6 +2483,21 @@ const struct efx_nic_type falcon_b0_nic_type = { .ev_process = efx_farch_ev_process, .ev_read_ack = efx_farch_ev_read_ack, .ev_test_generate = efx_farch_ev_test_generate, + .filter_table_probe = efx_farch_filter_table_probe, + .filter_table_restore = efx_farch_filter_table_restore, + .filter_table_remove = efx_farch_filter_table_remove, + .filter_update_rx_scatter = efx_farch_filter_update_rx_scatter, + .filter_insert = efx_farch_filter_insert, + .filter_remove_safe = efx_farch_filter_remove_safe, + .filter_get_safe = efx_farch_filter_get_safe, + .filter_clear_rx = efx_farch_filter_clear_rx, + .filter_count_rx_used = efx_farch_filter_count_rx_used, + .filter_get_rx_id_limit = efx_farch_filter_get_rx_id_limit, + .filter_get_rx_ids = efx_farch_filter_get_rx_ids, +#ifdef CONFIG_RFS_ACCEL + .filter_rfs_insert = efx_farch_filter_rfs_insert, + .filter_rfs_expire_one = efx_farch_filter_rfs_expire_one, +#endif .revision = EFX_REV_FALCON_B0, .txd_ptr_tbl_base = FR_BZ_TX_DESC_PTR_TBL, @@ -2483,5 +2513,6 @@ const struct efx_nic_type falcon_b0_nic_type = { .timer_period_max = 1 << FRF_AB_TC_TIMER_VAL_WIDTH, .offload_features = NETIF_F_IP_CSUM | NETIF_F_RXHASH | NETIF_F_NTUPLE, .mcdi_max_ver = -1, + .max_rx_ip_filters = FR_BZ_RX_FILTER_TBL0_ROWS, }; diff --git a/drivers/net/ethernet/sfc/farch.c b/drivers/net/ethernet/sfc/farch.c index 7f50882f11ab..19418045f3e0 100644 --- a/drivers/net/ethernet/sfc/farch.c +++ b/drivers/net/ethernet/sfc/farch.c @@ -1779,3 +1779,1111 @@ void efx_farch_init_common(struct efx_nic *efx) efx_writeo(efx, &temp, FR_BZ_TX_PACE); } } + +/************************************************************************** + * + * Filter tables + * + ************************************************************************** + */ + +/* "Fudge factors" - difference between programmed value and actual depth. + * Due to pipelined implementation we need to program H/W with a value that + * is larger than the hop limit we want. + */ +#define EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD 3 +#define EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL 1 + +/* Hard maximum search limit. Hardware will time-out beyond 200-something. + * We also need to avoid infinite loops in efx_farch_filter_search() when the + * table is full. + */ +#define EFX_FARCH_FILTER_CTL_SRCH_MAX 200 + +/* Don't try very hard to find space for performance hints, as this is + * counter-productive. */ +#define EFX_FARCH_FILTER_CTL_SRCH_HINT_MAX 5 + +enum efx_farch_filter_type { + EFX_FARCH_FILTER_TCP_FULL = 0, + EFX_FARCH_FILTER_TCP_WILD, + EFX_FARCH_FILTER_UDP_FULL, + EFX_FARCH_FILTER_UDP_WILD, + EFX_FARCH_FILTER_MAC_FULL = 4, + EFX_FARCH_FILTER_MAC_WILD, + EFX_FARCH_FILTER_UC_DEF = 8, + EFX_FARCH_FILTER_MC_DEF, + EFX_FARCH_FILTER_TYPE_COUNT, /* number of specific types */ +}; + +enum efx_farch_filter_table_id { + EFX_FARCH_FILTER_TABLE_RX_IP = 0, + EFX_FARCH_FILTER_TABLE_RX_MAC, + EFX_FARCH_FILTER_TABLE_RX_DEF, + EFX_FARCH_FILTER_TABLE_TX_MAC, + EFX_FARCH_FILTER_TABLE_COUNT, +}; + +enum efx_farch_filter_index { + EFX_FARCH_FILTER_INDEX_UC_DEF, + EFX_FARCH_FILTER_INDEX_MC_DEF, + EFX_FARCH_FILTER_SIZE_RX_DEF, +}; + +struct efx_farch_filter_spec { + u8 type:4; + u8 priority:4; + u8 flags; + u16 dmaq_id; + u32 data[3]; +}; + +struct efx_farch_filter_table { + enum efx_farch_filter_table_id id; + u32 offset; /* address of table relative to BAR */ + unsigned size; /* number of entries */ + unsigned step; /* step between entries */ + unsigned used; /* number currently used */ + unsigned long *used_bitmap; + struct efx_farch_filter_spec *spec; + unsigned search_limit[EFX_FARCH_FILTER_TYPE_COUNT]; +}; + +struct efx_farch_filter_state { + struct efx_farch_filter_table table[EFX_FARCH_FILTER_TABLE_COUNT]; +}; + +static void +efx_farch_filter_table_clear_entry(struct efx_nic *efx, + struct efx_farch_filter_table *table, + unsigned int filter_idx); + +/* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit + * key derived from the n-tuple. The initial LFSR state is 0xffff. */ +static u16 efx_farch_filter_hash(u32 key) +{ + u16 tmp; + + /* First 16 rounds */ + tmp = 0x1fff ^ key >> 16; + tmp = tmp ^ tmp >> 3 ^ tmp >> 6; + tmp = tmp ^ tmp >> 9; + /* Last 16 rounds */ + tmp = tmp ^ tmp << 13 ^ key; + tmp = tmp ^ tmp >> 3 ^ tmp >> 6; + return tmp ^ tmp >> 9; +} + +/* To allow for hash collisions, filter search continues at these + * increments from the first possible entry selected by the hash. */ +static u16 efx_farch_filter_increment(u32 key) +{ + return key * 2 - 1; +} + +static enum efx_farch_filter_table_id +efx_farch_filter_spec_table_id(const struct efx_farch_filter_spec *spec) +{ + BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != + (EFX_FARCH_FILTER_TCP_FULL >> 2)); + BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != + (EFX_FARCH_FILTER_TCP_WILD >> 2)); + BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != + (EFX_FARCH_FILTER_UDP_FULL >> 2)); + BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != + (EFX_FARCH_FILTER_UDP_WILD >> 2)); + BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_MAC != + (EFX_FARCH_FILTER_MAC_FULL >> 2)); + BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_MAC != + (EFX_FARCH_FILTER_MAC_WILD >> 2)); + BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_TX_MAC != + EFX_FARCH_FILTER_TABLE_RX_MAC + 2); + return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0); +} + +static void efx_farch_filter_push_rx_config(struct efx_nic *efx) +{ + struct efx_farch_filter_state *state = efx->filter_state; + struct efx_farch_filter_table *table; + efx_oword_t filter_ctl; + + efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); + + table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; + EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT, + table->search_limit[EFX_FARCH_FILTER_TCP_FULL] + + EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT, + table->search_limit[EFX_FARCH_FILTER_TCP_WILD] + + EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); + EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT, + table->search_limit[EFX_FARCH_FILTER_UDP_FULL] + + EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT, + table->search_limit[EFX_FARCH_FILTER_UDP_WILD] + + EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); + + table = &state->table[EFX_FARCH_FILTER_TABLE_RX_MAC]; + if (table->size) { + EFX_SET_OWORD_FIELD( + filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT, + table->search_limit[EFX_FARCH_FILTER_MAC_FULL] + + EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD( + filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT, + table->search_limit[EFX_FARCH_FILTER_MAC_WILD] + + EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); + } + + table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF]; + if (table->size) { + EFX_SET_OWORD_FIELD( + filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID, + table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].dmaq_id); + EFX_SET_OWORD_FIELD( + filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED, + !!(table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].flags & + EFX_FILTER_FLAG_RX_RSS)); + EFX_SET_OWORD_FIELD( + filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID, + table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].dmaq_id); + EFX_SET_OWORD_FIELD( + filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED, + !!(table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].flags & + EFX_FILTER_FLAG_RX_RSS)); + + /* There is a single bit to enable RX scatter for all + * unmatched packets. Only set it if scatter is + * enabled in both filter specs. + */ + EFX_SET_OWORD_FIELD( + filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, + !!(table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].flags & + table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].flags & + EFX_FILTER_FLAG_RX_SCATTER)); + } else if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { + /* We don't expose 'default' filters because unmatched + * packets always go to the queue number found in the + * RSS table. But we still need to set the RX scatter + * bit here. + */ + EFX_SET_OWORD_FIELD( + filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, + efx->rx_scatter); + } + + efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); +} + +static void efx_farch_filter_push_tx_limits(struct efx_nic *efx) +{ + struct efx_farch_filter_state *state = efx->filter_state; + struct efx_farch_filter_table *table; + efx_oword_t tx_cfg; + + efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG); + + table = &state->table[EFX_FARCH_FILTER_TABLE_TX_MAC]; + if (table->size) { + EFX_SET_OWORD_FIELD( + tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE, + table->search_limit[EFX_FARCH_FILTER_MAC_FULL] + + EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD( + tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE, + table->search_limit[EFX_FARCH_FILTER_MAC_WILD] + + EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); + } + + efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG); +} + +static int +efx_farch_filter_from_gen_spec(struct efx_farch_filter_spec *spec, + const struct efx_filter_spec *gen_spec) +{ + bool is_full = false; + + if ((gen_spec->flags & EFX_FILTER_FLAG_RX_RSS) && + gen_spec->rss_context != EFX_FILTER_RSS_CONTEXT_DEFAULT) + return -EINVAL; + + spec->priority = gen_spec->priority; + spec->flags = gen_spec->flags; + spec->dmaq_id = gen_spec->dmaq_id; + + switch (gen_spec->match_flags) { + case (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | + EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT): + is_full = true; + /* fall through */ + case (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT): { + __be32 rhost, host1, host2; + __be16 rport, port1, port2; + + EFX_BUG_ON_PARANOID(!(gen_spec->flags & EFX_FILTER_FLAG_RX)); + + if (gen_spec->ether_type != htons(ETH_P_IP)) + return -EPROTONOSUPPORT; + if (gen_spec->loc_port == 0 || + (is_full && gen_spec->rem_port == 0)) + return -EADDRNOTAVAIL; + switch (gen_spec->ip_proto) { + case IPPROTO_TCP: + spec->type = (is_full ? EFX_FARCH_FILTER_TCP_FULL : + EFX_FARCH_FILTER_TCP_WILD); + break; + case IPPROTO_UDP: + spec->type = (is_full ? EFX_FARCH_FILTER_UDP_FULL : + EFX_FARCH_FILTER_UDP_WILD); + break; + default: + return -EPROTONOSUPPORT; + } + + /* Filter is constructed in terms of source and destination, + * with the odd wrinkle that the ports are swapped in a UDP + * wildcard filter. We need to convert from local and remote + * (= zero for wildcard) addresses. + */ + rhost = is_full ? gen_spec->rem_host[0] : 0; + rport = is_full ? gen_spec->rem_port : 0; + host1 = rhost; + host2 = gen_spec->loc_host[0]; + if (!is_full && gen_spec->ip_proto == IPPROTO_UDP) { + port1 = gen_spec->loc_port; + port2 = rport; + } else { + port1 = rport; + port2 = gen_spec->loc_port; + } + spec->data[0] = ntohl(host1) << 16 | ntohs(port1); + spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16; + spec->data[2] = ntohl(host2); + + break; + } + + case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID: + is_full = true; + /* fall through */ + case EFX_FILTER_MATCH_LOC_MAC: + spec->type = (is_full ? EFX_FARCH_FILTER_MAC_FULL : + EFX_FARCH_FILTER_MAC_WILD); + spec->data[0] = is_full ? ntohs(gen_spec->outer_vid) : 0; + spec->data[1] = (gen_spec->loc_mac[2] << 24 | + gen_spec->loc_mac[3] << 16 | + gen_spec->loc_mac[4] << 8 | + gen_spec->loc_mac[5]); + spec->data[2] = (gen_spec->loc_mac[0] << 8 | + gen_spec->loc_mac[1]); + break; + + case EFX_FILTER_MATCH_LOC_MAC_IG: + spec->type = (is_multicast_ether_addr(gen_spec->loc_mac) ? + EFX_FARCH_FILTER_MC_DEF : + EFX_FARCH_FILTER_UC_DEF); + memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */ + break; + + default: + return -EPROTONOSUPPORT; + } + + return 0; +} + +static void +efx_farch_filter_to_gen_spec(struct efx_filter_spec *gen_spec, + const struct efx_farch_filter_spec *spec) +{ + bool is_full = false; + + /* *gen_spec should be completely initialised, to be consistent + * with efx_filter_init_{rx,tx}() and in case we want to copy + * it back to userland. + */ + memset(gen_spec, 0, sizeof(*gen_spec)); + + gen_spec->priority = spec->priority; + gen_spec->flags = spec->flags; + gen_spec->dmaq_id = spec->dmaq_id; + + switch (spec->type) { + case EFX_FARCH_FILTER_TCP_FULL: + case EFX_FARCH_FILTER_UDP_FULL: + is_full = true; + /* fall through */ + case EFX_FARCH_FILTER_TCP_WILD: + case EFX_FARCH_FILTER_UDP_WILD: { + __be32 host1, host2; + __be16 port1, port2; + + gen_spec->match_flags = + EFX_FILTER_MATCH_ETHER_TYPE | + EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT; + if (is_full) + gen_spec->match_flags |= (EFX_FILTER_MATCH_REM_HOST | + EFX_FILTER_MATCH_REM_PORT); + gen_spec->ether_type = htons(ETH_P_IP); + gen_spec->ip_proto = + (spec->type == EFX_FARCH_FILTER_TCP_FULL || + spec->type == EFX_FARCH_FILTER_TCP_WILD) ? + IPPROTO_TCP : IPPROTO_UDP; + + host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16); + port1 = htons(spec->data[0]); + host2 = htonl(spec->data[2]); + port2 = htons(spec->data[1] >> 16); + if (spec->flags & EFX_FILTER_FLAG_TX) { + gen_spec->loc_host[0] = host1; + gen_spec->rem_host[0] = host2; + } else { + gen_spec->loc_host[0] = host2; + gen_spec->rem_host[0] = host1; + } + if (!!(gen_spec->flags & EFX_FILTER_FLAG_TX) ^ + (!is_full && gen_spec->ip_proto == IPPROTO_UDP)) { + gen_spec->loc_port = port1; + gen_spec->rem_port = port2; + } else { + gen_spec->loc_port = port2; + gen_spec->rem_port = port1; + } + + break; + } + + case EFX_FARCH_FILTER_MAC_FULL: + is_full = true; + /* fall through */ + case EFX_FARCH_FILTER_MAC_WILD: + gen_spec->match_flags = EFX_FILTER_MATCH_LOC_MAC; + if (is_full) + gen_spec->match_flags |= EFX_FILTER_MATCH_OUTER_VID; + gen_spec->loc_mac[0] = spec->data[2] >> 8; + gen_spec->loc_mac[1] = spec->data[2]; + gen_spec->loc_mac[2] = spec->data[1] >> 24; + gen_spec->loc_mac[3] = spec->data[1] >> 16; + gen_spec->loc_mac[4] = spec->data[1] >> 8; + gen_spec->loc_mac[5] = spec->data[1]; + gen_spec->outer_vid = htons(spec->data[0]); + break; + + case EFX_FARCH_FILTER_UC_DEF: + case EFX_FARCH_FILTER_MC_DEF: + gen_spec->match_flags = EFX_FILTER_MATCH_LOC_MAC_IG; + gen_spec->loc_mac[0] = spec->type == EFX_FARCH_FILTER_MC_DEF; + break; + + default: + WARN_ON(1); + break; + } +} + +static void +efx_farch_filter_reset_rx_def(struct efx_nic *efx, unsigned filter_idx) +{ + struct efx_farch_filter_state *state = efx->filter_state; + struct efx_farch_filter_table *table = + &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF]; + struct efx_farch_filter_spec *spec = &table->spec[filter_idx]; + + /* If there's only one channel then disable RSS for non VF + * traffic, thereby allowing VFs to use RSS when the PF can't. + */ + spec->type = EFX_FARCH_FILTER_UC_DEF + filter_idx; + spec->priority = EFX_FILTER_PRI_MANUAL; + spec->flags = (EFX_FILTER_FLAG_RX | + (efx->n_rx_channels > 1 ? EFX_FILTER_FLAG_RX_RSS : 0) | + (efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0)); + spec->dmaq_id = 0; + table->used_bitmap[0] |= 1 << filter_idx; +} + +/* Build a filter entry and return its n-tuple key. */ +static u32 efx_farch_filter_build(efx_oword_t *filter, + struct efx_farch_filter_spec *spec) +{ + u32 data3; + + switch (efx_farch_filter_spec_table_id(spec)) { + case EFX_FARCH_FILTER_TABLE_RX_IP: { + bool is_udp = (spec->type == EFX_FARCH_FILTER_UDP_FULL || + spec->type == EFX_FARCH_FILTER_UDP_WILD); + EFX_POPULATE_OWORD_7( + *filter, + FRF_BZ_RSS_EN, + !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), + FRF_BZ_SCATTER_EN, + !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), + FRF_BZ_TCP_UDP, is_udp, + FRF_BZ_RXQ_ID, spec->dmaq_id, + EFX_DWORD_2, spec->data[2], + EFX_DWORD_1, spec->data[1], + EFX_DWORD_0, spec->data[0]); + data3 = is_udp; + break; + } + + case EFX_FARCH_FILTER_TABLE_RX_MAC: { + bool is_wild = spec->type == EFX_FARCH_FILTER_MAC_WILD; + EFX_POPULATE_OWORD_7( + *filter, + FRF_CZ_RMFT_RSS_EN, + !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), + FRF_CZ_RMFT_SCATTER_EN, + !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), + FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id, + FRF_CZ_RMFT_WILDCARD_MATCH, is_wild, + FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2], + FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1], + FRF_CZ_RMFT_VLAN_ID, spec->data[0]); + data3 = is_wild; + break; + } + + case EFX_FARCH_FILTER_TABLE_TX_MAC: { + bool is_wild = spec->type == EFX_FARCH_FILTER_MAC_WILD; + EFX_POPULATE_OWORD_5(*filter, + FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id, + FRF_CZ_TMFT_WILDCARD_MATCH, is_wild, + FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2], + FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1], + FRF_CZ_TMFT_VLAN_ID, spec->data[0]); + data3 = is_wild | spec->dmaq_id << 1; + break; + } + + default: + BUG(); + } + + return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3; +} + +static bool efx_farch_filter_equal(const struct efx_farch_filter_spec *left, + const struct efx_farch_filter_spec *right) +{ + if (left->type != right->type || + memcmp(left->data, right->data, sizeof(left->data))) + return false; + + if (left->flags & EFX_FILTER_FLAG_TX && + left->dmaq_id != right->dmaq_id) + return false; + + return true; +} + +/* + * Construct/deconstruct external filter IDs. At least the RX filter + * IDs must be ordered by matching priority, for RX NFC semantics. + * + * Deconstruction needs to be robust against invalid IDs so that + * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can + * accept user-provided IDs. + */ + +#define EFX_FARCH_FILTER_MATCH_PRI_COUNT 5 + +static const u8 efx_farch_filter_type_match_pri[EFX_FARCH_FILTER_TYPE_COUNT] = { + [EFX_FARCH_FILTER_TCP_FULL] = 0, + [EFX_FARCH_FILTER_UDP_FULL] = 0, + [EFX_FARCH_FILTER_TCP_WILD] = 1, + [EFX_FARCH_FILTER_UDP_WILD] = 1, + [EFX_FARCH_FILTER_MAC_FULL] = 2, + [EFX_FARCH_FILTER_MAC_WILD] = 3, + [EFX_FARCH_FILTER_UC_DEF] = 4, + [EFX_FARCH_FILTER_MC_DEF] = 4, +}; + +static const enum efx_farch_filter_table_id efx_farch_filter_range_table[] = { + EFX_FARCH_FILTER_TABLE_RX_IP, /* RX match pri 0 */ + EFX_FARCH_FILTER_TABLE_RX_IP, + EFX_FARCH_FILTER_TABLE_RX_MAC, + EFX_FARCH_FILTER_TABLE_RX_MAC, + EFX_FARCH_FILTER_TABLE_RX_DEF, /* RX match pri 4 */ + EFX_FARCH_FILTER_TABLE_TX_MAC, /* TX match pri 0 */ + EFX_FARCH_FILTER_TABLE_TX_MAC, /* TX match pri 1 */ +}; + +#define EFX_FARCH_FILTER_INDEX_WIDTH 13 +#define EFX_FARCH_FILTER_INDEX_MASK ((1 << EFX_FARCH_FILTER_INDEX_WIDTH) - 1) + +static inline u32 +efx_farch_filter_make_id(const struct efx_farch_filter_spec *spec, + unsigned int index) +{ + unsigned int range; + + range = efx_farch_filter_type_match_pri[spec->type]; + if (!(spec->flags & EFX_FILTER_FLAG_RX)) + range += EFX_FARCH_FILTER_MATCH_PRI_COUNT; + + return range << EFX_FARCH_FILTER_INDEX_WIDTH | index; +} + +static inline enum efx_farch_filter_table_id +efx_farch_filter_id_table_id(u32 id) +{ + unsigned int range = id >> EFX_FARCH_FILTER_INDEX_WIDTH; + + if (range < ARRAY_SIZE(efx_farch_filter_range_table)) + return efx_farch_filter_range_table[range]; + else + return EFX_FARCH_FILTER_TABLE_COUNT; /* invalid */ +} + +static inline unsigned int efx_farch_filter_id_index(u32 id) +{ + return id & EFX_FARCH_FILTER_INDEX_MASK; +} + +u32 efx_farch_filter_get_rx_id_limit(struct efx_nic *efx) +{ + struct efx_farch_filter_state *state = efx->filter_state; + unsigned int range = EFX_FARCH_FILTER_MATCH_PRI_COUNT - 1; + enum efx_farch_filter_table_id table_id; + + do { + table_id = efx_farch_filter_range_table[range]; + if (state->table[table_id].size != 0) + return range << EFX_FARCH_FILTER_INDEX_WIDTH | + state->table[table_id].size; + } while (range--); + + return 0; +} + +s32 efx_farch_filter_insert(struct efx_nic *efx, + struct efx_filter_spec *gen_spec, + bool replace_equal) +{ + struct efx_farch_filter_state *state = efx->filter_state; + struct efx_farch_filter_table *table; + struct efx_farch_filter_spec spec; + efx_oword_t filter; + int rep_index, ins_index; + unsigned int depth = 0; + int rc; + + rc = efx_farch_filter_from_gen_spec(&spec, gen_spec); + if (rc) + return rc; + + table = &state->table[efx_farch_filter_spec_table_id(&spec)]; + if (table->size == 0) + return -EINVAL; + + netif_vdbg(efx, hw, efx->net_dev, + "%s: type %d search_limit=%d", __func__, spec.type, + table->search_limit[spec.type]); + + if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) { + /* One filter spec per type */ + BUILD_BUG_ON(EFX_FARCH_FILTER_INDEX_UC_DEF != 0); + BUILD_BUG_ON(EFX_FARCH_FILTER_INDEX_MC_DEF != + EFX_FARCH_FILTER_MC_DEF - EFX_FARCH_FILTER_UC_DEF); + rep_index = spec.type - EFX_FARCH_FILTER_UC_DEF; + ins_index = rep_index; + + spin_lock_bh(&efx->filter_lock); + } else { + /* Search concurrently for + * (1) a filter to be replaced (rep_index): any filter + * with the same match values, up to the current + * search depth for this type, and + * (2) the insertion point (ins_index): (1) or any + * free slot before it or up to the maximum search + * depth for this priority + * We fail if we cannot find (2). + * + * We can stop once either + * (a) we find (1), in which case we have definitely + * found (2) as well; or + * (b) we have searched exhaustively for (1), and have + * either found (2) or searched exhaustively for it + */ + u32 key = efx_farch_filter_build(&filter, &spec); + unsigned int hash = efx_farch_filter_hash(key); + unsigned int incr = efx_farch_filter_increment(key); + unsigned int max_rep_depth = table->search_limit[spec.type]; + unsigned int max_ins_depth = + spec.priority <= EFX_FILTER_PRI_HINT ? + EFX_FARCH_FILTER_CTL_SRCH_HINT_MAX : + EFX_FARCH_FILTER_CTL_SRCH_MAX; + unsigned int i = hash & (table->size - 1); + + ins_index = -1; + depth = 1; + + spin_lock_bh(&efx->filter_lock); + + for (;;) { + if (!test_bit(i, table->used_bitmap)) { + if (ins_index < 0) + ins_index = i; + } else if (efx_farch_filter_equal(&spec, + &table->spec[i])) { + /* Case (a) */ + if (ins_index < 0) + ins_index = i; + rep_index = i; + break; + } + + if (depth >= max_rep_depth && + (ins_index >= 0 || depth >= max_ins_depth)) { + /* Case (b) */ + if (ins_index < 0) { + rc = -EBUSY; + goto out; + } + rep_index = -1; + break; + } + + i = (i + incr) & (table->size - 1); + ++depth; + } + } + + /* If we found a filter to be replaced, check whether we + * should do so + */ + if (rep_index >= 0) { + struct efx_farch_filter_spec *saved_spec = + &table->spec[rep_index]; + + if (spec.priority == saved_spec->priority && !replace_equal) { + rc = -EEXIST; + goto out; + } + if (spec.priority < saved_spec->priority) { + rc = -EPERM; + goto out; + } + } + + /* Insert the filter */ + if (ins_index != rep_index) { + __set_bit(ins_index, table->used_bitmap); + ++table->used; + } + table->spec[ins_index] = spec; + + if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) { + efx_farch_filter_push_rx_config(efx); + } else { + if (table->search_limit[spec.type] < depth) { + table->search_limit[spec.type] = depth; + if (spec.flags & EFX_FILTER_FLAG_TX) + efx_farch_filter_push_tx_limits(efx); + else + efx_farch_filter_push_rx_config(efx); + } + + efx_writeo(efx, &filter, + table->offset + table->step * ins_index); + + /* If we were able to replace a filter by inserting + * at a lower depth, clear the replaced filter + */ + if (ins_index != rep_index && rep_index >= 0) + efx_farch_filter_table_clear_entry(efx, table, + rep_index); + } + + netif_vdbg(efx, hw, efx->net_dev, + "%s: filter type %d index %d rxq %u set", + __func__, spec.type, ins_index, spec.dmaq_id); + rc = efx_farch_filter_make_id(&spec, ins_index); + +out: + spin_unlock_bh(&efx->filter_lock); + return rc; +} + +static void +efx_farch_filter_table_clear_entry(struct efx_nic *efx, + struct efx_farch_filter_table *table, + unsigned int filter_idx) +{ + static efx_oword_t filter; + + if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) { + /* RX default filters must always exist */ + efx_farch_filter_reset_rx_def(efx, filter_idx); + efx_farch_filter_push_rx_config(efx); + } else if (test_bit(filter_idx, table->used_bitmap)) { + __clear_bit(filter_idx, table->used_bitmap); + --table->used; + memset(&table->spec[filter_idx], 0, sizeof(table->spec[0])); + + efx_writeo(efx, &filter, + table->offset + table->step * filter_idx); + + /* If this filter required a greater search depth than + * any other, the search limit for its type can now be + * decreased. However, it is hard to determine that + * unless the table has become completely empty - in + * which case, all its search limits can be set to 0. + */ + if (unlikely(table->used == 0)) { + memset(table->search_limit, 0, + sizeof(table->search_limit)); + if (table->id == EFX_FARCH_FILTER_TABLE_TX_MAC) + efx_farch_filter_push_tx_limits(efx); + else + efx_farch_filter_push_rx_config(efx); + } + } +} + +int efx_farch_filter_remove_safe(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 filter_id) +{ + struct efx_farch_filter_state *state = efx->filter_state; + enum efx_farch_filter_table_id table_id; + struct efx_farch_filter_table *table; + unsigned int filter_idx; + struct efx_farch_filter_spec *spec; + int rc; + + table_id = efx_farch_filter_id_table_id(filter_id); + if ((unsigned int)table_id >= EFX_FARCH_FILTER_TABLE_COUNT) + return -ENOENT; + table = &state->table[table_id]; + + filter_idx = efx_farch_filter_id_index(filter_id); + if (filter_idx >= table->size) + return -ENOENT; + spec = &table->spec[filter_idx]; + + spin_lock_bh(&efx->filter_lock); + + if (test_bit(filter_idx, table->used_bitmap) && + spec->priority == priority) { + efx_farch_filter_table_clear_entry(efx, table, filter_idx); + rc = 0; + } else { + rc = -ENOENT; + } + + spin_unlock_bh(&efx->filter_lock); + + return rc; +} + +int efx_farch_filter_get_safe(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 filter_id, struct efx_filter_spec *spec_buf) +{ + struct efx_farch_filter_state *state = efx->filter_state; + enum efx_farch_filter_table_id table_id; + struct efx_farch_filter_table *table; + struct efx_farch_filter_spec *spec; + unsigned int filter_idx; + int rc; + + table_id = efx_farch_filter_id_table_id(filter_id); + if ((unsigned int)table_id >= EFX_FARCH_FILTER_TABLE_COUNT) + return -ENOENT; + table = &state->table[table_id]; + + filter_idx = efx_farch_filter_id_index(filter_id); + if (filter_idx >= table->size) + return -ENOENT; + spec = &table->spec[filter_idx]; + + spin_lock_bh(&efx->filter_lock); + + if (test_bit(filter_idx, table->used_bitmap) && + spec->priority == priority) { + efx_farch_filter_to_gen_spec(spec_buf, spec); + rc = 0; + } else { + rc = -ENOENT; + } + + spin_unlock_bh(&efx->filter_lock); + + return rc; +} + +static void +efx_farch_filter_table_clear(struct efx_nic *efx, + enum efx_farch_filter_table_id table_id, + enum efx_filter_priority priority) +{ + struct efx_farch_filter_state *state = efx->filter_state; + struct efx_farch_filter_table *table = &state->table[table_id]; + unsigned int filter_idx; + + spin_lock_bh(&efx->filter_lock); + for (filter_idx = 0; filter_idx < table->size; ++filter_idx) + if (table->spec[filter_idx].priority <= priority) + efx_farch_filter_table_clear_entry(efx, table, + filter_idx); + spin_unlock_bh(&efx->filter_lock); +} + +void efx_farch_filter_clear_rx(struct efx_nic *efx, + enum efx_filter_priority priority) +{ + efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_IP, + priority); + efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_MAC, + priority); +} + +u32 efx_farch_filter_count_rx_used(struct efx_nic *efx, + enum efx_filter_priority priority) +{ + struct efx_farch_filter_state *state = efx->filter_state; + enum efx_farch_filter_table_id table_id; + struct efx_farch_filter_table *table; + unsigned int filter_idx; + u32 count = 0; + + spin_lock_bh(&efx->filter_lock); + + for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP; + table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF; + table_id++) { + table = &state->table[table_id]; + for (filter_idx = 0; filter_idx < table->size; filter_idx++) { + if (test_bit(filter_idx, table->used_bitmap) && + table->spec[filter_idx].priority == priority) + ++count; + } + } + + spin_unlock_bh(&efx->filter_lock); + + return count; +} + +s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 *buf, u32 size) +{ + struct efx_farch_filter_state *state = efx->filter_state; + enum efx_farch_filter_table_id table_id; + struct efx_farch_filter_table *table; + unsigned int filter_idx; + s32 count = 0; + + spin_lock_bh(&efx->filter_lock); + + for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP; + table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF; + table_id++) { + table = &state->table[table_id]; + for (filter_idx = 0; filter_idx < table->size; filter_idx++) { + if (test_bit(filter_idx, table->used_bitmap) && + table->spec[filter_idx].priority == priority) { + if (count == size) { + count = -EMSGSIZE; + goto out; + } + buf[count++] = efx_farch_filter_make_id( + &table->spec[filter_idx], filter_idx); + } + } + } +out: + spin_unlock_bh(&efx->filter_lock); + + return count; +} + +/* Restore filter stater after reset */ +void efx_farch_filter_table_restore(struct efx_nic *efx) +{ + struct efx_farch_filter_state *state = efx->filter_state; + enum efx_farch_filter_table_id table_id; + struct efx_farch_filter_table *table; + efx_oword_t filter; + unsigned int filter_idx; + + spin_lock_bh(&efx->filter_lock); + + for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) { + table = &state->table[table_id]; + + /* Check whether this is a regular register table */ + if (table->step == 0) + continue; + + for (filter_idx = 0; filter_idx < table->size; filter_idx++) { + if (!test_bit(filter_idx, table->used_bitmap)) + continue; + efx_farch_filter_build(&filter, &table->spec[filter_idx]); + efx_writeo(efx, &filter, + table->offset + table->step * filter_idx); + } + } + + efx_farch_filter_push_rx_config(efx); + efx_farch_filter_push_tx_limits(efx); + + spin_unlock_bh(&efx->filter_lock); +} + +void efx_farch_filter_table_remove(struct efx_nic *efx) +{ + struct efx_farch_filter_state *state = efx->filter_state; + enum efx_farch_filter_table_id table_id; + + for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) { + kfree(state->table[table_id].used_bitmap); + vfree(state->table[table_id].spec); + } + kfree(state); +} + +int efx_farch_filter_table_probe(struct efx_nic *efx) +{ + struct efx_farch_filter_state *state; + struct efx_farch_filter_table *table; + unsigned table_id; + + state = kzalloc(sizeof(struct efx_farch_filter_state), GFP_KERNEL); + if (!state) + return -ENOMEM; + efx->filter_state = state; + + if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { + table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; + table->id = EFX_FARCH_FILTER_TABLE_RX_IP; + table->offset = FR_BZ_RX_FILTER_TBL0; + table->size = FR_BZ_RX_FILTER_TBL0_ROWS; + table->step = FR_BZ_RX_FILTER_TBL0_STEP; + } + + if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { + table = &state->table[EFX_FARCH_FILTER_TABLE_RX_MAC]; + table->id = EFX_FARCH_FILTER_TABLE_RX_MAC; + table->offset = FR_CZ_RX_MAC_FILTER_TBL0; + table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS; + table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP; + + table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF]; + table->id = EFX_FARCH_FILTER_TABLE_RX_DEF; + table->size = EFX_FARCH_FILTER_SIZE_RX_DEF; + + table = &state->table[EFX_FARCH_FILTER_TABLE_TX_MAC]; + table->id = EFX_FARCH_FILTER_TABLE_TX_MAC; + table->offset = FR_CZ_TX_MAC_FILTER_TBL0; + table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS; + table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP; + } + + for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) { + table = &state->table[table_id]; + if (table->size == 0) + continue; + table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size), + sizeof(unsigned long), + GFP_KERNEL); + if (!table->used_bitmap) + goto fail; + table->spec = vzalloc(table->size * sizeof(*table->spec)); + if (!table->spec) + goto fail; + } + + if (state->table[EFX_FARCH_FILTER_TABLE_RX_DEF].size) { + /* RX default filters must always exist */ + unsigned i; + for (i = 0; i < EFX_FARCH_FILTER_SIZE_RX_DEF; i++) + efx_farch_filter_reset_rx_def(efx, i); + } + + efx_farch_filter_push_rx_config(efx); + + return 0; + +fail: + efx_farch_filter_table_remove(efx); + return -ENOMEM; +} + +/* Update scatter enable flags for filters pointing to our own RX queues */ +void efx_farch_filter_update_rx_scatter(struct efx_nic *efx) +{ + struct efx_farch_filter_state *state = efx->filter_state; + enum efx_farch_filter_table_id table_id; + struct efx_farch_filter_table *table; + efx_oword_t filter; + unsigned int filter_idx; + + spin_lock_bh(&efx->filter_lock); + + for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP; + table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF; + table_id++) { + table = &state->table[table_id]; + + for (filter_idx = 0; filter_idx < table->size; filter_idx++) { + if (!test_bit(filter_idx, table->used_bitmap) || + table->spec[filter_idx].dmaq_id >= + efx->n_rx_channels) + continue; + + if (efx->rx_scatter) + table->spec[filter_idx].flags |= + EFX_FILTER_FLAG_RX_SCATTER; + else + table->spec[filter_idx].flags &= + ~EFX_FILTER_FLAG_RX_SCATTER; + + if (table_id == EFX_FARCH_FILTER_TABLE_RX_DEF) + /* Pushed by efx_farch_filter_push_rx_config() */ + continue; + + efx_farch_filter_build(&filter, &table->spec[filter_idx]); + efx_writeo(efx, &filter, + table->offset + table->step * filter_idx); + } + } + + efx_farch_filter_push_rx_config(efx); + + spin_unlock_bh(&efx->filter_lock); +} + +#ifdef CONFIG_RFS_ACCEL + +s32 efx_farch_filter_rfs_insert(struct efx_nic *efx, + struct efx_filter_spec *gen_spec) +{ + return efx_farch_filter_insert(efx, gen_spec, true); +} + +bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, + unsigned int index) +{ + struct efx_farch_filter_state *state = efx->filter_state; + struct efx_farch_filter_table *table = + &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; + + if (test_bit(index, table->used_bitmap) && + table->spec[index].priority == EFX_FILTER_PRI_HINT && + rps_may_expire_flow(efx->net_dev, table->spec[index].dmaq_id, + flow_id, index)) { + efx_farch_filter_table_clear_entry(efx, table, index); + return true; + } + + return false; +} + +#endif /* CONFIG_RFS_ACCEL */ diff --git a/drivers/net/ethernet/sfc/filter.c b/drivers/net/ethernet/sfc/filter.c deleted file mode 100644 index ad66376e9d8c..000000000000 --- a/drivers/net/ethernet/sfc/filter.c +++ /dev/null @@ -1,1244 +0,0 @@ -/**************************************************************************** - * Driver for Solarflare Solarstorm network controllers and boards - * Copyright 2005-2010 Solarflare Communications Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation, incorporated herein by reference. - */ - -#include <linux/in.h> -#include <net/ip.h> -#include "efx.h" -#include "filter.h" -#include "io.h" -#include "nic.h" -#include "farch_regs.h" - -/* "Fudge factors" - difference between programmed value and actual depth. - * Due to pipelined implementation we need to program H/W with a value that - * is larger than the hop limit we want. - */ -#define EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD 3 -#define EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL 1 - -/* Hard maximum search limit. Hardware will time-out beyond 200-something. - * We also need to avoid infinite loops in efx_farch_filter_search() when the - * table is full. - */ -#define EFX_FARCH_FILTER_CTL_SRCH_MAX 200 - -/* Don't try very hard to find space for performance hints, as this is - * counter-productive. */ -#define EFX_FARCH_FILTER_CTL_SRCH_HINT_MAX 5 - -enum efx_farch_filter_type { - EFX_FARCH_FILTER_TCP_FULL = 0, - EFX_FARCH_FILTER_TCP_WILD, - EFX_FARCH_FILTER_UDP_FULL, - EFX_FARCH_FILTER_UDP_WILD, - EFX_FARCH_FILTER_MAC_FULL = 4, - EFX_FARCH_FILTER_MAC_WILD, - EFX_FARCH_FILTER_UC_DEF = 8, - EFX_FARCH_FILTER_MC_DEF, - EFX_FARCH_FILTER_TYPE_COUNT, /* number of specific types */ -}; - -enum efx_farch_filter_table_id { - EFX_FARCH_FILTER_TABLE_RX_IP = 0, - EFX_FARCH_FILTER_TABLE_RX_MAC, - EFX_FARCH_FILTER_TABLE_RX_DEF, - EFX_FARCH_FILTER_TABLE_TX_MAC, - EFX_FARCH_FILTER_TABLE_COUNT, -}; - -enum efx_farch_filter_index { - EFX_FARCH_FILTER_INDEX_UC_DEF, - EFX_FARCH_FILTER_INDEX_MC_DEF, - EFX_FARCH_FILTER_SIZE_RX_DEF, -}; - -struct efx_farch_filter_spec { - u8 type:4; - u8 priority:4; - u8 flags; - u16 dmaq_id; - u32 data[3]; -}; - -struct efx_farch_filter_table { - enum efx_farch_filter_table_id id; - u32 offset; /* address of table relative to BAR */ - unsigned size; /* number of entries */ - unsigned step; /* step between entries */ - unsigned used; /* number currently used */ - unsigned long *used_bitmap; - struct efx_farch_filter_spec *spec; - unsigned search_limit[EFX_FARCH_FILTER_TYPE_COUNT]; -}; - -struct efx_farch_filter_state { - struct efx_farch_filter_table table[EFX_FARCH_FILTER_TABLE_COUNT]; -}; - -static void -efx_farch_filter_table_clear_entry(struct efx_nic *efx, - struct efx_farch_filter_table *table, - unsigned int filter_idx); - -/* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit - * key derived from the n-tuple. The initial LFSR state is 0xffff. */ -static u16 efx_farch_filter_hash(u32 key) -{ - u16 tmp; - - /* First 16 rounds */ - tmp = 0x1fff ^ key >> 16; - tmp = tmp ^ tmp >> 3 ^ tmp >> 6; - tmp = tmp ^ tmp >> 9; - /* Last 16 rounds */ - tmp = tmp ^ tmp << 13 ^ key; - tmp = tmp ^ tmp >> 3 ^ tmp >> 6; - return tmp ^ tmp >> 9; -} - -/* To allow for hash collisions, filter search continues at these - * increments from the first possible entry selected by the hash. */ -static u16 efx_farch_filter_increment(u32 key) -{ - return key * 2 - 1; -} - -static enum efx_farch_filter_table_id -efx_farch_filter_spec_table_id(const struct efx_farch_filter_spec *spec) -{ - BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != - (EFX_FARCH_FILTER_TCP_FULL >> 2)); - BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != - (EFX_FARCH_FILTER_TCP_WILD >> 2)); - BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != - (EFX_FARCH_FILTER_UDP_FULL >> 2)); - BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != - (EFX_FARCH_FILTER_UDP_WILD >> 2)); - BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_MAC != - (EFX_FARCH_FILTER_MAC_FULL >> 2)); - BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_MAC != - (EFX_FARCH_FILTER_MAC_WILD >> 2)); - BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_TX_MAC != - EFX_FARCH_FILTER_TABLE_RX_MAC + 2); - return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0); -} - -static void efx_farch_filter_push_rx_config(struct efx_nic *efx) -{ - struct efx_farch_filter_state *state = efx->filter_state; - struct efx_farch_filter_table *table; - efx_oword_t filter_ctl; - - efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); - - table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; - EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT, - table->search_limit[EFX_FARCH_FILTER_TCP_FULL] + - EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); - EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT, - table->search_limit[EFX_FARCH_FILTER_TCP_WILD] + - EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); - EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT, - table->search_limit[EFX_FARCH_FILTER_UDP_FULL] + - EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); - EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT, - table->search_limit[EFX_FARCH_FILTER_UDP_WILD] + - EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); - - table = &state->table[EFX_FARCH_FILTER_TABLE_RX_MAC]; - if (table->size) { - EFX_SET_OWORD_FIELD( - filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT, - table->search_limit[EFX_FARCH_FILTER_MAC_FULL] + - EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); - EFX_SET_OWORD_FIELD( - filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT, - table->search_limit[EFX_FARCH_FILTER_MAC_WILD] + - EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); - } - - table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF]; - if (table->size) { - EFX_SET_OWORD_FIELD( - filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID, - table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].dmaq_id); - EFX_SET_OWORD_FIELD( - filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED, - !!(table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].flags & - EFX_FILTER_FLAG_RX_RSS)); - EFX_SET_OWORD_FIELD( - filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID, - table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].dmaq_id); - EFX_SET_OWORD_FIELD( - filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED, - !!(table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].flags & - EFX_FILTER_FLAG_RX_RSS)); - - /* There is a single bit to enable RX scatter for all - * unmatched packets. Only set it if scatter is - * enabled in both filter specs. - */ - EFX_SET_OWORD_FIELD( - filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, - !!(table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].flags & - table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].flags & - EFX_FILTER_FLAG_RX_SCATTER)); - } else if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { - /* We don't expose 'default' filters because unmatched - * packets always go to the queue number found in the - * RSS table. But we still need to set the RX scatter - * bit here. - */ - EFX_SET_OWORD_FIELD( - filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, - efx->rx_scatter); - } - - efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); -} - -static void efx_farch_filter_push_tx_limits(struct efx_nic *efx) -{ - struct efx_farch_filter_state *state = efx->filter_state; - struct efx_farch_filter_table *table; - efx_oword_t tx_cfg; - - efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG); - - table = &state->table[EFX_FARCH_FILTER_TABLE_TX_MAC]; - if (table->size) { - EFX_SET_OWORD_FIELD( - tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE, - table->search_limit[EFX_FARCH_FILTER_MAC_FULL] + - EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); - EFX_SET_OWORD_FIELD( - tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE, - table->search_limit[EFX_FARCH_FILTER_MAC_WILD] + - EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); - } - - efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG); -} - -static int -efx_farch_filter_from_gen_spec(struct efx_farch_filter_spec *spec, - const struct efx_filter_spec *gen_spec) -{ - bool is_full = false; - - if ((gen_spec->flags & EFX_FILTER_FLAG_RX_RSS) && - gen_spec->rss_context != EFX_FILTER_RSS_CONTEXT_DEFAULT) - return -EINVAL; - - spec->priority = gen_spec->priority; - spec->flags = gen_spec->flags; - spec->dmaq_id = gen_spec->dmaq_id; - - switch (gen_spec->match_flags) { - case (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | - EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | - EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT): - is_full = true; - /* fall through */ - case (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | - EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT): { - __be32 rhost, host1, host2; - __be16 rport, port1, port2; - - EFX_BUG_ON_PARANOID(!(gen_spec->flags & EFX_FILTER_FLAG_RX)); - - if (gen_spec->ether_type != htons(ETH_P_IP)) - return -EPROTONOSUPPORT; - if (gen_spec->loc_port == 0 || - (is_full && gen_spec->rem_port == 0)) - return -EADDRNOTAVAIL; - switch (gen_spec->ip_proto) { - case IPPROTO_TCP: - spec->type = (is_full ? EFX_FARCH_FILTER_TCP_FULL : - EFX_FARCH_FILTER_TCP_WILD); - break; - case IPPROTO_UDP: - spec->type = (is_full ? EFX_FARCH_FILTER_UDP_FULL : - EFX_FARCH_FILTER_UDP_WILD); - break; - default: - return -EPROTONOSUPPORT; - } - - /* Filter is constructed in terms of source and destination, - * with the odd wrinkle that the ports are swapped in a UDP - * wildcard filter. We need to convert from local and remote - * (= zero for wildcard) addresses. - */ - rhost = is_full ? gen_spec->rem_host[0] : 0; - rport = is_full ? gen_spec->rem_port : 0; - host1 = rhost; - host2 = gen_spec->loc_host[0]; - if (!is_full && gen_spec->ip_proto == IPPROTO_UDP) { - port1 = gen_spec->loc_port; - port2 = rport; - } else { - port1 = rport; - port2 = gen_spec->loc_port; - } - spec->data[0] = ntohl(host1) << 16 | ntohs(port1); - spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16; - spec->data[2] = ntohl(host2); - - break; - } - - case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID: - is_full = true; - /* fall through */ - case EFX_FILTER_MATCH_LOC_MAC: - spec->type = (is_full ? EFX_FARCH_FILTER_MAC_FULL : - EFX_FARCH_FILTER_MAC_WILD); - spec->data[0] = is_full ? ntohs(gen_spec->outer_vid) : 0; - spec->data[1] = (gen_spec->loc_mac[2] << 24 | - gen_spec->loc_mac[3] << 16 | - gen_spec->loc_mac[4] << 8 | - gen_spec->loc_mac[5]); - spec->data[2] = (gen_spec->loc_mac[0] << 8 | - gen_spec->loc_mac[1]); - break; - - case EFX_FILTER_MATCH_LOC_MAC_IG: - spec->type = (is_multicast_ether_addr(gen_spec->loc_mac) ? - EFX_FARCH_FILTER_MC_DEF : - EFX_FARCH_FILTER_UC_DEF); - memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */ - break; - - default: - return -EPROTONOSUPPORT; - } - - return 0; -} - -static void -efx_farch_filter_to_gen_spec(struct efx_filter_spec *gen_spec, - const struct efx_farch_filter_spec *spec) -{ - bool is_full = false; - - /* *gen_spec should be completely initialised, to be consistent - * with efx_filter_init_{rx,tx}() and in case we want to copy - * it back to userland. - */ - memset(gen_spec, 0, sizeof(*gen_spec)); - - gen_spec->priority = spec->priority; - gen_spec->flags = spec->flags; - gen_spec->dmaq_id = spec->dmaq_id; - - switch (spec->type) { - case EFX_FARCH_FILTER_TCP_FULL: - case EFX_FARCH_FILTER_UDP_FULL: - is_full = true; - /* fall through */ - case EFX_FARCH_FILTER_TCP_WILD: - case EFX_FARCH_FILTER_UDP_WILD: { - __be32 host1, host2; - __be16 port1, port2; - - gen_spec->match_flags = - EFX_FILTER_MATCH_ETHER_TYPE | - EFX_FILTER_MATCH_IP_PROTO | - EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT; - if (is_full) - gen_spec->match_flags |= (EFX_FILTER_MATCH_REM_HOST | - EFX_FILTER_MATCH_REM_PORT); - gen_spec->ether_type = htons(ETH_P_IP); - gen_spec->ip_proto = - (spec->type == EFX_FARCH_FILTER_TCP_FULL || - spec->type == EFX_FARCH_FILTER_TCP_WILD) ? - IPPROTO_TCP : IPPROTO_UDP; - - host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16); - port1 = htons(spec->data[0]); - host2 = htonl(spec->data[2]); - port2 = htons(spec->data[1] >> 16); - if (spec->flags & EFX_FILTER_FLAG_TX) { - gen_spec->loc_host[0] = host1; - gen_spec->rem_host[0] = host2; - } else { - gen_spec->loc_host[0] = host2; - gen_spec->rem_host[0] = host1; - } - if (!!(gen_spec->flags & EFX_FILTER_FLAG_TX) ^ - (!is_full && gen_spec->ip_proto == IPPROTO_UDP)) { - gen_spec->loc_port = port1; - gen_spec->rem_port = port2; - } else { - gen_spec->loc_port = port2; - gen_spec->rem_port = port1; - } - - break; - } - - case EFX_FARCH_FILTER_MAC_FULL: - is_full = true; - /* fall through */ - case EFX_FARCH_FILTER_MAC_WILD: - gen_spec->match_flags = EFX_FILTER_MATCH_LOC_MAC; - if (is_full) - gen_spec->match_flags |= EFX_FILTER_MATCH_OUTER_VID; - gen_spec->loc_mac[0] = spec->data[2] >> 8; - gen_spec->loc_mac[1] = spec->data[2]; - gen_spec->loc_mac[2] = spec->data[1] >> 24; - gen_spec->loc_mac[3] = spec->data[1] >> 16; - gen_spec->loc_mac[4] = spec->data[1] >> 8; - gen_spec->loc_mac[5] = spec->data[1]; - gen_spec->outer_vid = htons(spec->data[0]); - break; - - case EFX_FARCH_FILTER_UC_DEF: - case EFX_FARCH_FILTER_MC_DEF: - gen_spec->match_flags = EFX_FILTER_MATCH_LOC_MAC_IG; - gen_spec->loc_mac[0] = spec->type == EFX_FARCH_FILTER_MC_DEF; - break; - - default: - WARN_ON(1); - break; - } -} - -static void -efx_farch_filter_reset_rx_def(struct efx_nic *efx, unsigned filter_idx) -{ - struct efx_farch_filter_state *state = efx->filter_state; - struct efx_farch_filter_table *table = - &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF]; - struct efx_farch_filter_spec *spec = &table->spec[filter_idx]; - - /* If there's only one channel then disable RSS for non VF - * traffic, thereby allowing VFs to use RSS when the PF can't. - */ - spec->type = EFX_FARCH_FILTER_UC_DEF + filter_idx; - spec->priority = EFX_FILTER_PRI_MANUAL; - spec->flags = (EFX_FILTER_FLAG_RX | - (efx->n_rx_channels > 1 ? EFX_FILTER_FLAG_RX_RSS : 0) | - (efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0)); - spec->dmaq_id = 0; - table->used_bitmap[0] |= 1 << filter_idx; -} - -/* Build a filter entry and return its n-tuple key. */ -static u32 efx_farch_filter_build(efx_oword_t *filter, - struct efx_farch_filter_spec *spec) -{ - u32 data3; - - switch (efx_farch_filter_spec_table_id(spec)) { - case EFX_FARCH_FILTER_TABLE_RX_IP: { - bool is_udp = (spec->type == EFX_FARCH_FILTER_UDP_FULL || - spec->type == EFX_FARCH_FILTER_UDP_WILD); - EFX_POPULATE_OWORD_7( - *filter, - FRF_BZ_RSS_EN, - !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), - FRF_BZ_SCATTER_EN, - !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), - FRF_BZ_TCP_UDP, is_udp, - FRF_BZ_RXQ_ID, spec->dmaq_id, - EFX_DWORD_2, spec->data[2], - EFX_DWORD_1, spec->data[1], - EFX_DWORD_0, spec->data[0]); - data3 = is_udp; - break; - } - - case EFX_FARCH_FILTER_TABLE_RX_MAC: { - bool is_wild = spec->type == EFX_FARCH_FILTER_MAC_WILD; - EFX_POPULATE_OWORD_7( - *filter, - FRF_CZ_RMFT_RSS_EN, - !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), - FRF_CZ_RMFT_SCATTER_EN, - !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), - FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id, - FRF_CZ_RMFT_WILDCARD_MATCH, is_wild, - FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2], - FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1], - FRF_CZ_RMFT_VLAN_ID, spec->data[0]); - data3 = is_wild; - break; - } - - case EFX_FARCH_FILTER_TABLE_TX_MAC: { - bool is_wild = spec->type == EFX_FARCH_FILTER_MAC_WILD; - EFX_POPULATE_OWORD_5(*filter, - FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id, - FRF_CZ_TMFT_WILDCARD_MATCH, is_wild, - FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2], - FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1], - FRF_CZ_TMFT_VLAN_ID, spec->data[0]); - data3 = is_wild | spec->dmaq_id << 1; - break; - } - - default: - BUG(); - } - - return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3; -} - -static bool efx_farch_filter_equal(const struct efx_farch_filter_spec *left, - const struct efx_farch_filter_spec *right) -{ - if (left->type != right->type || - memcmp(left->data, right->data, sizeof(left->data))) - return false; - - if (left->flags & EFX_FILTER_FLAG_TX && - left->dmaq_id != right->dmaq_id) - return false; - - return true; -} - -/* - * Construct/deconstruct external filter IDs. At least the RX filter - * IDs must be ordered by matching priority, for RX NFC semantics. - * - * Deconstruction needs to be robust against invalid IDs so that - * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can - * accept user-provided IDs. - */ - -#define EFX_FARCH_FILTER_MATCH_PRI_COUNT 5 - -static const u8 efx_farch_filter_type_match_pri[EFX_FARCH_FILTER_TYPE_COUNT] = { - [EFX_FARCH_FILTER_TCP_FULL] = 0, - [EFX_FARCH_FILTER_UDP_FULL] = 0, - [EFX_FARCH_FILTER_TCP_WILD] = 1, - [EFX_FARCH_FILTER_UDP_WILD] = 1, - [EFX_FARCH_FILTER_MAC_FULL] = 2, - [EFX_FARCH_FILTER_MAC_WILD] = 3, - [EFX_FARCH_FILTER_UC_DEF] = 4, - [EFX_FARCH_FILTER_MC_DEF] = 4, -}; - -static const enum efx_farch_filter_table_id efx_farch_filter_range_table[] = { - EFX_FARCH_FILTER_TABLE_RX_IP, /* RX match pri 0 */ - EFX_FARCH_FILTER_TABLE_RX_IP, - EFX_FARCH_FILTER_TABLE_RX_MAC, - EFX_FARCH_FILTER_TABLE_RX_MAC, - EFX_FARCH_FILTER_TABLE_RX_DEF, /* RX match pri 4 */ - EFX_FARCH_FILTER_TABLE_TX_MAC, /* TX match pri 0 */ - EFX_FARCH_FILTER_TABLE_TX_MAC, /* TX match pri 1 */ -}; - -#define EFX_FARCH_FILTER_INDEX_WIDTH 13 -#define EFX_FARCH_FILTER_INDEX_MASK ((1 << EFX_FARCH_FILTER_INDEX_WIDTH) - 1) - -static inline u32 -efx_farch_filter_make_id(const struct efx_farch_filter_spec *spec, - unsigned int index) -{ - unsigned int range; - - range = efx_farch_filter_type_match_pri[spec->type]; - if (!(spec->flags & EFX_FILTER_FLAG_RX)) - range += EFX_FARCH_FILTER_MATCH_PRI_COUNT; - - return range << EFX_FARCH_FILTER_INDEX_WIDTH | index; -} - -static inline enum efx_farch_filter_table_id -efx_farch_filter_id_table_id(u32 id) -{ - unsigned int range = id >> EFX_FARCH_FILTER_INDEX_WIDTH; - - if (range < ARRAY_SIZE(efx_farch_filter_range_table)) - return efx_farch_filter_range_table[range]; - else - return EFX_FARCH_FILTER_TABLE_COUNT; /* invalid */ -} - -static inline unsigned int efx_farch_filter_id_index(u32 id) -{ - return id & EFX_FARCH_FILTER_INDEX_MASK; -} - -u32 efx_filter_get_rx_id_limit(struct efx_nic *efx) -{ - struct efx_farch_filter_state *state = efx->filter_state; - unsigned int range = EFX_FARCH_FILTER_MATCH_PRI_COUNT - 1; - enum efx_farch_filter_table_id table_id; - - do { - table_id = efx_farch_filter_range_table[range]; - if (state->table[table_id].size != 0) - return range << EFX_FARCH_FILTER_INDEX_WIDTH | - state->table[table_id].size; - } while (range--); - - return 0; -} - -/** - * efx_filter_insert_filter - add or replace a filter - * @efx: NIC in which to insert the filter - * @spec: Specification for the filter - * @replace_equal: Flag for whether the specified filter may replace an - * existing filter with equal priority - * - * On success, return the filter ID. - * On failure, return a negative error code. - * - * If an existing filter has equal match values to the new filter - * spec, then the new filter might replace it, depending on the - * relative priorities. If the existing filter has lower priority, or - * if @replace_equal is set and it has equal priority, then it is - * replaced. Otherwise the function fails, returning -%EPERM if - * the existing filter has higher priority or -%EEXIST if it has - * equal priority. - */ -s32 efx_filter_insert_filter(struct efx_nic *efx, - struct efx_filter_spec *gen_spec, - bool replace_equal) -{ - struct efx_farch_filter_state *state = efx->filter_state; - struct efx_farch_filter_table *table; - struct efx_farch_filter_spec spec; - efx_oword_t filter; - int rep_index, ins_index; - unsigned int depth = 0; - int rc; - - rc = efx_farch_filter_from_gen_spec(&spec, gen_spec); - if (rc) - return rc; - - table = &state->table[efx_farch_filter_spec_table_id(&spec)]; - if (table->size == 0) - return -EINVAL; - - netif_vdbg(efx, hw, efx->net_dev, - "%s: type %d search_limit=%d", __func__, spec.type, - table->search_limit[spec.type]); - - if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) { - /* One filter spec per type */ - BUILD_BUG_ON(EFX_FARCH_FILTER_INDEX_UC_DEF != 0); - BUILD_BUG_ON(EFX_FARCH_FILTER_INDEX_MC_DEF != - EFX_FARCH_FILTER_MC_DEF - EFX_FARCH_FILTER_UC_DEF); - rep_index = spec.type - EFX_FARCH_FILTER_UC_DEF; - ins_index = rep_index; - - spin_lock_bh(&efx->filter_lock); - } else { - /* Search concurrently for - * (1) a filter to be replaced (rep_index): any filter - * with the same match values, up to the current - * search depth for this type, and - * (2) the insertion point (ins_index): (1) or any - * free slot before it or up to the maximum search - * depth for this priority - * We fail if we cannot find (2). - * - * We can stop once either - * (a) we find (1), in which case we have definitely - * found (2) as well; or - * (b) we have searched exhaustively for (1), and have - * either found (2) or searched exhaustively for it - */ - u32 key = efx_farch_filter_build(&filter, &spec); - unsigned int hash = efx_farch_filter_hash(key); - unsigned int incr = efx_farch_filter_increment(key); - unsigned int max_rep_depth = table->search_limit[spec.type]; - unsigned int max_ins_depth = - spec.priority <= EFX_FILTER_PRI_HINT ? - EFX_FARCH_FILTER_CTL_SRCH_HINT_MAX : - EFX_FARCH_FILTER_CTL_SRCH_MAX; - unsigned int i = hash & (table->size - 1); - - ins_index = -1; - depth = 1; - - spin_lock_bh(&efx->filter_lock); - - for (;;) { - if (!test_bit(i, table->used_bitmap)) { - if (ins_index < 0) - ins_index = i; - } else if (efx_farch_filter_equal(&spec, - &table->spec[i])) { - /* Case (a) */ - if (ins_index < 0) - ins_index = i; - rep_index = i; - break; - } - - if (depth >= max_rep_depth && - (ins_index >= 0 || depth >= max_ins_depth)) { - /* Case (b) */ - if (ins_index < 0) { - rc = -EBUSY; - goto out; - } - rep_index = -1; - break; - } - - i = (i + incr) & (table->size - 1); - ++depth; - } - } - - /* If we found a filter to be replaced, check whether we - * should do so - */ - if (rep_index >= 0) { - struct efx_farch_filter_spec *saved_spec = - &table->spec[rep_index]; - - if (spec.priority == saved_spec->priority && !replace_equal) { - rc = -EEXIST; - goto out; - } - if (spec.priority < saved_spec->priority) { - rc = -EPERM; - goto out; - } - } - - /* Insert the filter */ - if (ins_index != rep_index) { - __set_bit(ins_index, table->used_bitmap); - ++table->used; - } - table->spec[ins_index] = spec; - - if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) { - efx_farch_filter_push_rx_config(efx); - } else { - if (table->search_limit[spec.type] < depth) { - table->search_limit[spec.type] = depth; - if (spec.flags & EFX_FILTER_FLAG_TX) - efx_farch_filter_push_tx_limits(efx); - else - efx_farch_filter_push_rx_config(efx); - } - - efx_writeo(efx, &filter, - table->offset + table->step * ins_index); - - /* If we were able to replace a filter by inserting - * at a lower depth, clear the replaced filter - */ - if (ins_index != rep_index && rep_index >= 0) - efx_farch_filter_table_clear_entry(efx, table, - rep_index); - } - - netif_vdbg(efx, hw, efx->net_dev, - "%s: filter type %d index %d rxq %u set", - __func__, spec.type, ins_index, spec.dmaq_id); - rc = efx_farch_filter_make_id(&spec, ins_index); - -out: - spin_unlock_bh(&efx->filter_lock); - return rc; -} - -static void -efx_farch_filter_table_clear_entry(struct efx_nic *efx, - struct efx_farch_filter_table *table, - unsigned int filter_idx) -{ - static efx_oword_t filter; - - if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) { - /* RX default filters must always exist */ - efx_farch_filter_reset_rx_def(efx, filter_idx); - efx_farch_filter_push_rx_config(efx); - } else if (test_bit(filter_idx, table->used_bitmap)) { - __clear_bit(filter_idx, table->used_bitmap); - --table->used; - memset(&table->spec[filter_idx], 0, sizeof(table->spec[0])); - - efx_writeo(efx, &filter, - table->offset + table->step * filter_idx); - - /* If this filter required a greater search depth than - * any other, the search limit for its type can now be - * decreased. However, it is hard to determine that - * unless the table has become completely empty - in - * which case, all its search limits can be set to 0. - */ - if (unlikely(table->used == 0)) { - memset(table->search_limit, 0, - sizeof(table->search_limit)); - if (table->id == EFX_FARCH_FILTER_TABLE_TX_MAC) - efx_farch_filter_push_tx_limits(efx); - else - efx_farch_filter_push_rx_config(efx); - } - } -} - -/** - * efx_filter_remove_id_safe - remove a filter by ID, carefully - * @efx: NIC from which to remove the filter - * @priority: Priority of filter, as passed to @efx_filter_insert_filter - * @filter_id: ID of filter, as returned by @efx_filter_insert_filter - * - * This function will range-check @filter_id, so it is safe to call - * with a value passed from userland. - */ -int efx_filter_remove_id_safe(struct efx_nic *efx, - enum efx_filter_priority priority, - u32 filter_id) -{ - struct efx_farch_filter_state *state = efx->filter_state; - enum efx_farch_filter_table_id table_id; - struct efx_farch_filter_table *table; - unsigned int filter_idx; - struct efx_farch_filter_spec *spec; - int rc; - - table_id = efx_farch_filter_id_table_id(filter_id); - if ((unsigned int)table_id >= EFX_FARCH_FILTER_TABLE_COUNT) - return -ENOENT; - table = &state->table[table_id]; - - filter_idx = efx_farch_filter_id_index(filter_id); - if (filter_idx >= table->size) - return -ENOENT; - spec = &table->spec[filter_idx]; - - spin_lock_bh(&efx->filter_lock); - - if (test_bit(filter_idx, table->used_bitmap) && - spec->priority == priority) { - efx_farch_filter_table_clear_entry(efx, table, filter_idx); - rc = 0; - } else { - rc = -ENOENT; - } - - spin_unlock_bh(&efx->filter_lock); - - return rc; -} - -/** - * efx_filter_get_filter_safe - retrieve a filter by ID, carefully - * @efx: NIC from which to remove the filter - * @priority: Priority of filter, as passed to @efx_filter_insert_filter - * @filter_id: ID of filter, as returned by @efx_filter_insert_filter - * @spec: Buffer in which to store filter specification - * - * This function will range-check @filter_id, so it is safe to call - * with a value passed from userland. - */ -int efx_filter_get_filter_safe(struct efx_nic *efx, - enum efx_filter_priority priority, - u32 filter_id, struct efx_filter_spec *spec_buf) -{ - struct efx_farch_filter_state *state = efx->filter_state; - enum efx_farch_filter_table_id table_id; - struct efx_farch_filter_table *table; - struct efx_farch_filter_spec *spec; - unsigned int filter_idx; - int rc; - - table_id = efx_farch_filter_id_table_id(filter_id); - if ((unsigned int)table_id >= EFX_FARCH_FILTER_TABLE_COUNT) - return -ENOENT; - table = &state->table[table_id]; - - filter_idx = efx_farch_filter_id_index(filter_id); - if (filter_idx >= table->size) - return -ENOENT; - spec = &table->spec[filter_idx]; - - spin_lock_bh(&efx->filter_lock); - - if (test_bit(filter_idx, table->used_bitmap) && - spec->priority == priority) { - efx_farch_filter_to_gen_spec(spec_buf, spec); - rc = 0; - } else { - rc = -ENOENT; - } - - spin_unlock_bh(&efx->filter_lock); - - return rc; -} - -static void -efx_farch_filter_table_clear(struct efx_nic *efx, - enum efx_farch_filter_table_id table_id, - enum efx_filter_priority priority) -{ - struct efx_farch_filter_state *state = efx->filter_state; - struct efx_farch_filter_table *table = &state->table[table_id]; - unsigned int filter_idx; - - spin_lock_bh(&efx->filter_lock); - for (filter_idx = 0; filter_idx < table->size; ++filter_idx) - if (table->spec[filter_idx].priority <= priority) - efx_farch_filter_table_clear_entry(efx, table, - filter_idx); - spin_unlock_bh(&efx->filter_lock); -} - -/** - * efx_filter_clear_rx - remove RX filters by priority - * @efx: NIC from which to remove the filters - * @priority: Maximum priority to remove - */ -void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority) -{ - efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_IP, - priority); - efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_MAC, - priority); -} - -u32 efx_filter_count_rx_used(struct efx_nic *efx, - enum efx_filter_priority priority) -{ - struct efx_farch_filter_state *state = efx->filter_state; - enum efx_farch_filter_table_id table_id; - struct efx_farch_filter_table *table; - unsigned int filter_idx; - u32 count = 0; - - spin_lock_bh(&efx->filter_lock); - - for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP; - table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF; - table_id++) { - table = &state->table[table_id]; - for (filter_idx = 0; filter_idx < table->size; filter_idx++) { - if (test_bit(filter_idx, table->used_bitmap) && - table->spec[filter_idx].priority == priority) - ++count; - } - } - - spin_unlock_bh(&efx->filter_lock); - - return count; -} - -s32 efx_filter_get_rx_ids(struct efx_nic *efx, - enum efx_filter_priority priority, - u32 *buf, u32 size) -{ - struct efx_farch_filter_state *state = efx->filter_state; - enum efx_farch_filter_table_id table_id; - struct efx_farch_filter_table *table; - unsigned int filter_idx; - s32 count = 0; - - spin_lock_bh(&efx->filter_lock); - - for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP; - table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF; - table_id++) { - table = &state->table[table_id]; - for (filter_idx = 0; filter_idx < table->size; filter_idx++) { - if (test_bit(filter_idx, table->used_bitmap) && - table->spec[filter_idx].priority == priority) { - if (count == size) { - count = -EMSGSIZE; - goto out; - } - buf[count++] = efx_farch_filter_make_id( - &table->spec[filter_idx], filter_idx); - } - } - } -out: - spin_unlock_bh(&efx->filter_lock); - - return count; -} - -/* Restore filter stater after reset */ -void efx_restore_filters(struct efx_nic *efx) -{ - struct efx_farch_filter_state *state = efx->filter_state; - enum efx_farch_filter_table_id table_id; - struct efx_farch_filter_table *table; - efx_oword_t filter; - unsigned int filter_idx; - - spin_lock_bh(&efx->filter_lock); - - for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) { - table = &state->table[table_id]; - - /* Check whether this is a regular register table */ - if (table->step == 0) - continue; - - for (filter_idx = 0; filter_idx < table->size; filter_idx++) { - if (!test_bit(filter_idx, table->used_bitmap)) - continue; - efx_farch_filter_build(&filter, &table->spec[filter_idx]); - efx_writeo(efx, &filter, - table->offset + table->step * filter_idx); - } - } - - efx_farch_filter_push_rx_config(efx); - efx_farch_filter_push_tx_limits(efx); - - spin_unlock_bh(&efx->filter_lock); -} - -int efx_probe_filters(struct efx_nic *efx) -{ - struct efx_farch_filter_state *state; - struct efx_farch_filter_table *table; - unsigned table_id; - - state = kzalloc(sizeof(struct efx_farch_filter_state), GFP_KERNEL); - if (!state) - return -ENOMEM; - efx->filter_state = state; - - spin_lock_init(&efx->filter_lock); - - if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { -#ifdef CONFIG_RFS_ACCEL - efx->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS, - sizeof(*efx->rps_flow_id), - GFP_KERNEL); - if (!efx->rps_flow_id) - goto fail; -#endif - table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; - table->id = EFX_FARCH_FILTER_TABLE_RX_IP; - table->offset = FR_BZ_RX_FILTER_TBL0; - table->size = FR_BZ_RX_FILTER_TBL0_ROWS; - table->step = FR_BZ_RX_FILTER_TBL0_STEP; - } - - if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { - table = &state->table[EFX_FARCH_FILTER_TABLE_RX_MAC]; - table->id = EFX_FARCH_FILTER_TABLE_RX_MAC; - table->offset = FR_CZ_RX_MAC_FILTER_TBL0; - table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS; - table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP; - - table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF]; - table->id = EFX_FARCH_FILTER_TABLE_RX_DEF; - table->size = EFX_FARCH_FILTER_SIZE_RX_DEF; - - table = &state->table[EFX_FARCH_FILTER_TABLE_TX_MAC]; - table->id = EFX_FARCH_FILTER_TABLE_TX_MAC; - table->offset = FR_CZ_TX_MAC_FILTER_TBL0; - table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS; - table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP; - } - - for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) { - table = &state->table[table_id]; - if (table->size == 0) - continue; - table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size), - sizeof(unsigned long), - GFP_KERNEL); - if (!table->used_bitmap) - goto fail; - table->spec = vzalloc(table->size * sizeof(*table->spec)); - if (!table->spec) - goto fail; - } - - if (state->table[EFX_FARCH_FILTER_TABLE_RX_DEF].size) { - /* RX default filters must always exist */ - unsigned i; - for (i = 0; i < EFX_FARCH_FILTER_SIZE_RX_DEF; i++) - efx_farch_filter_reset_rx_def(efx, i); - } - - efx_farch_filter_push_rx_config(efx); - - return 0; - -fail: - efx_remove_filters(efx); - return -ENOMEM; -} - -void efx_remove_filters(struct efx_nic *efx) -{ - struct efx_farch_filter_state *state = efx->filter_state; - enum efx_farch_filter_table_id table_id; - - for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) { - kfree(state->table[table_id].used_bitmap); - vfree(state->table[table_id].spec); - } -#ifdef CONFIG_RFS_ACCEL - kfree(efx->rps_flow_id); -#endif - kfree(state); -} - -/* Update scatter enable flags for filters pointing to our own RX queues */ -void efx_filter_update_rx_scatter(struct efx_nic *efx) -{ - struct efx_farch_filter_state *state = efx->filter_state; - enum efx_farch_filter_table_id table_id; - struct efx_farch_filter_table *table; - efx_oword_t filter; - unsigned int filter_idx; - - spin_lock_bh(&efx->filter_lock); - - for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP; - table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF; - table_id++) { - table = &state->table[table_id]; - - for (filter_idx = 0; filter_idx < table->size; filter_idx++) { - if (!test_bit(filter_idx, table->used_bitmap) || - table->spec[filter_idx].dmaq_id >= - efx->n_rx_channels) - continue; - - if (efx->rx_scatter) - table->spec[filter_idx].flags |= - EFX_FILTER_FLAG_RX_SCATTER; - else - table->spec[filter_idx].flags &= - ~EFX_FILTER_FLAG_RX_SCATTER; - - if (table_id == EFX_FARCH_FILTER_TABLE_RX_DEF) - /* Pushed by efx_farch_filter_push_rx_config() */ - continue; - - efx_farch_filter_build(&filter, &table->spec[filter_idx]); - efx_writeo(efx, &filter, - table->offset + table->step * filter_idx); - } - } - - efx_farch_filter_push_rx_config(efx); - - spin_unlock_bh(&efx->filter_lock); -} - -#ifdef CONFIG_RFS_ACCEL - -int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, - u16 rxq_index, u32 flow_id) -{ - struct efx_nic *efx = netdev_priv(net_dev); - struct efx_channel *channel; - struct efx_filter_spec spec; - const struct iphdr *ip; - const __be16 *ports; - int nhoff; - int rc; - - nhoff = skb_network_offset(skb); - - if (skb->protocol == htons(ETH_P_8021Q)) { - EFX_BUG_ON_PARANOID(skb_headlen(skb) < - nhoff + sizeof(struct vlan_hdr)); - if (((const struct vlan_hdr *)skb->data + nhoff)-> - h_vlan_encapsulated_proto != htons(ETH_P_IP)) - return -EPROTONOSUPPORT; - - /* This is IP over 802.1q VLAN. We can't filter on the - * IP 5-tuple and the vlan together, so just strip the - * vlan header and filter on the IP part. - */ - nhoff += sizeof(struct vlan_hdr); - } else if (skb->protocol != htons(ETH_P_IP)) { - return -EPROTONOSUPPORT; - } - - /* RFS must validate the IP header length before calling us */ - EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip)); - ip = (const struct iphdr *)(skb->data + nhoff); - if (ip_is_fragment(ip)) - return -EPROTONOSUPPORT; - EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4); - ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); - - efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, - efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, - rxq_index); - rc = efx_filter_set_ipv4_full(&spec, ip->protocol, - ip->daddr, ports[1], ip->saddr, ports[0]); - if (rc) - return rc; - - rc = efx_filter_insert_filter(efx, &spec, true); - if (rc < 0) - return rc; - - /* Remember this so we can check whether to expire the filter later */ - efx->rps_flow_id[rc] = flow_id; - channel = efx_get_channel(efx, skb_get_rx_queue(skb)); - ++channel->rfs_filters_added; - - netif_info(efx, rx_status, efx->net_dev, - "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", - (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP", - &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]), - rxq_index, flow_id, rc); - - return rc; -} - -bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota) -{ - struct efx_farch_filter_state *state = efx->filter_state; - struct efx_farch_filter_table *table = - &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; - unsigned mask = table->size - 1; - unsigned index; - unsigned stop; - - if (!spin_trylock_bh(&efx->filter_lock)) - return false; - - index = efx->rps_expire_index; - stop = (index + quota) & mask; - - while (index != stop) { - if (test_bit(index, table->used_bitmap) && - table->spec[index].priority == EFX_FILTER_PRI_HINT && - rps_may_expire_flow(efx->net_dev, - table->spec[index].dmaq_id, - efx->rps_flow_id[index], index)) { - netif_info(efx, rx_status, efx->net_dev, - "expiring filter %d [flow %u]\n", - index, efx->rps_flow_id[index]); - efx_farch_filter_table_clear_entry(efx, table, index); - } - index = (index + 1) & mask; - } - - efx->rps_expire_index = stop; - - spin_unlock_bh(&efx->filter_lock); - return true; -} - -#endif /* CONFIG_RFS_ACCEL */ diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 5287a3cd639c..d35ce1410376 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -30,6 +30,7 @@ #include "enum.h" #include "bitfield.h" +#include "filter.h" /************************************************************************** * @@ -1025,6 +1026,24 @@ static inline unsigned int efx_port_num(struct efx_nic *efx) * @ev_process: Process events for a queue, up to the given NAPI quota * @ev_read_ack: Acknowledge read events on a queue, rearming its IRQ * @ev_test_generate: Generate a test event + * @filter_table_probe: Probe filter capabilities and set up filter software state + * @filter_table_restore: Restore filters removed from hardware + * @filter_table_remove: Remove filters from hardware and tear down software state + * @filter_update_rx_scatter: Update filters after change to rx scatter setting + * @filter_insert: add or replace a filter + * @filter_remove_safe: remove a filter by ID, carefully + * @filter_get_safe: retrieve a filter by ID, carefully + * @filter_clear_rx: remove RX filters by priority + * @filter_count_rx_used: Get the number of filters in use at a given priority + * @filter_get_rx_id_limit: Get maximum value of a filter id, plus 1 + * @filter_get_rx_ids: Get list of RX filters at a given priority + * @filter_rfs_insert: Add or replace a filter for RFS. This must be + * atomic. The hardware change may be asynchronous but should + * not be delayed for long. It may fail if this can't be done + * atomically. + * @filter_rfs_expire_one: Consider expiring a filter inserted for RFS. + * This must check whether the specified table entry is used by RFS + * and that rps_may_expire_flow() returns true for it. * @revision: Hardware architecture revision * @txd_ptr_tbl_base: TX descriptor ring base address * @rxd_ptr_tbl_base: RX descriptor ring base address @@ -1102,6 +1121,32 @@ struct efx_nic_type { int (*ev_process)(struct efx_channel *channel, int quota); void (*ev_read_ack)(struct efx_channel *channel); void (*ev_test_generate)(struct efx_channel *channel); + int (*filter_table_probe)(struct efx_nic *efx); + void (*filter_table_restore)(struct efx_nic *efx); + void (*filter_table_remove)(struct efx_nic *efx); + void (*filter_update_rx_scatter)(struct efx_nic *efx); + s32 (*filter_insert)(struct efx_nic *efx, + struct efx_filter_spec *spec, bool replace); + int (*filter_remove_safe)(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 filter_id); + int (*filter_get_safe)(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 filter_id, struct efx_filter_spec *); + void (*filter_clear_rx)(struct efx_nic *efx, + enum efx_filter_priority priority); + u32 (*filter_count_rx_used)(struct efx_nic *efx, + enum efx_filter_priority priority); + u32 (*filter_get_rx_id_limit)(struct efx_nic *efx); + s32 (*filter_get_rx_ids)(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 *buf, u32 size); +#ifdef CONFIG_RFS_ACCEL + s32 (*filter_rfs_insert)(struct efx_nic *efx, + struct efx_filter_spec *spec); + bool (*filter_rfs_expire_one)(struct efx_nic *efx, u32 flow_id, + unsigned int index); +#endif int revision; unsigned int txd_ptr_tbl_base; @@ -1117,6 +1162,7 @@ struct efx_nic_type { unsigned int timer_period_max; netdev_features_t offload_features; int mcdi_max_ver; + unsigned int max_rx_ip_filters; }; /************************************************************************** diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h index 25e25b635798..69298c918e97 100644 --- a/drivers/net/ethernet/sfc/nic.h +++ b/drivers/net/ethernet/sfc/nic.h @@ -404,6 +404,34 @@ extern int efx_farch_ev_process(struct efx_channel *channel, int quota); extern void efx_farch_ev_read_ack(struct efx_channel *channel); extern void efx_farch_ev_test_generate(struct efx_channel *channel); +/* Falcon/Siena filter operations */ +extern int efx_farch_filter_table_probe(struct efx_nic *efx); +extern void efx_farch_filter_table_restore(struct efx_nic *efx); +extern void efx_farch_filter_table_remove(struct efx_nic *efx); +extern void efx_farch_filter_update_rx_scatter(struct efx_nic *efx); +extern s32 efx_farch_filter_insert(struct efx_nic *efx, + struct efx_filter_spec *spec, bool replace); +extern int efx_farch_filter_remove_safe(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 filter_id); +extern int efx_farch_filter_get_safe(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 filter_id, struct efx_filter_spec *); +extern void efx_farch_filter_clear_rx(struct efx_nic *efx, + enum efx_filter_priority priority); +extern u32 efx_farch_filter_count_rx_used(struct efx_nic *efx, + enum efx_filter_priority priority); +extern u32 efx_farch_filter_get_rx_id_limit(struct efx_nic *efx); +extern s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx, + enum efx_filter_priority priority, + u32 *buf, u32 size); +#ifdef CONFIG_RFS_ACCEL +extern s32 efx_farch_filter_rfs_insert(struct efx_nic *efx, + struct efx_filter_spec *spec); +extern bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, + unsigned int index); +#endif + extern bool efx_nic_event_present(struct efx_channel *channel); /* Some statistics are computed as A - B where A and B each increase diff --git a/drivers/net/ethernet/sfc/rx.c b/drivers/net/ethernet/sfc/rx.c index f2b78cd2baf8..12990929e274 100644 --- a/drivers/net/ethernet/sfc/rx.c +++ b/drivers/net/ethernet/sfc/rx.c @@ -21,6 +21,7 @@ #include <net/checksum.h> #include "net_driver.h" #include "efx.h" +#include "filter.h" #include "nic.h" #include "selftest.h" #include "workarounds.h" @@ -802,3 +803,96 @@ module_param(rx_refill_threshold, uint, 0444); MODULE_PARM_DESC(rx_refill_threshold, "RX descriptor ring refill threshold (%)"); +#ifdef CONFIG_RFS_ACCEL + +int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, + u16 rxq_index, u32 flow_id) +{ + struct efx_nic *efx = netdev_priv(net_dev); + struct efx_channel *channel; + struct efx_filter_spec spec; + const struct iphdr *ip; + const __be16 *ports; + int nhoff; + int rc; + + nhoff = skb_network_offset(skb); + + if (skb->protocol == htons(ETH_P_8021Q)) { + EFX_BUG_ON_PARANOID(skb_headlen(skb) < + nhoff + sizeof(struct vlan_hdr)); + if (((const struct vlan_hdr *)skb->data + nhoff)-> + h_vlan_encapsulated_proto != htons(ETH_P_IP)) + return -EPROTONOSUPPORT; + + /* This is IP over 802.1q VLAN. We can't filter on the + * IP 5-tuple and the vlan together, so just strip the + * vlan header and filter on the IP part. + */ + nhoff += sizeof(struct vlan_hdr); + } else if (skb->protocol != htons(ETH_P_IP)) { + return -EPROTONOSUPPORT; + } + + /* RFS must validate the IP header length before calling us */ + EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip)); + ip = (const struct iphdr *)(skb->data + nhoff); + if (ip_is_fragment(ip)) + return -EPROTONOSUPPORT; + EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4); + ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); + + efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, + efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, + rxq_index); + rc = efx_filter_set_ipv4_full(&spec, ip->protocol, + ip->daddr, ports[1], ip->saddr, ports[0]); + if (rc) + return rc; + + rc = efx->type->filter_rfs_insert(efx, &spec); + if (rc < 0) + return rc; + + /* Remember this so we can check whether to expire the filter later */ + efx->rps_flow_id[rc] = flow_id; + channel = efx_get_channel(efx, skb_get_rx_queue(skb)); + ++channel->rfs_filters_added; + + netif_info(efx, rx_status, efx->net_dev, + "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", + (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP", + &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]), + rxq_index, flow_id, rc); + + return rc; +} + +bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota) +{ + bool (*expire_one)(struct efx_nic *efx, u32 flow_id, unsigned int index); + unsigned int index, size; + u32 flow_id; + + if (!spin_trylock_bh(&efx->filter_lock)) + return false; + + expire_one = efx->type->filter_rfs_expire_one; + index = efx->rps_expire_index; + size = efx->type->max_rx_ip_filters; + while (quota--) { + flow_id = efx->rps_flow_id[index]; + if (expire_one(efx, flow_id, index)) + netif_info(efx, rx_status, efx->net_dev, + "expired filter %d [flow %u]\n", + index, flow_id); + if (++index == size) + index = 0; + } + efx->rps_expire_index = index; + + spin_unlock_bh(&efx->filter_lock); + return true; +} + +#endif /* CONFIG_RFS_ACCEL */ diff --git a/drivers/net/ethernet/sfc/siena.c b/drivers/net/ethernet/sfc/siena.c index 23e573149bd6..5120cd8f706b 100644 --- a/drivers/net/ethernet/sfc/siena.c +++ b/drivers/net/ethernet/sfc/siena.c @@ -736,6 +736,21 @@ const struct efx_nic_type siena_a0_nic_type = { .ev_process = efx_farch_ev_process, .ev_read_ack = efx_farch_ev_read_ack, .ev_test_generate = efx_farch_ev_test_generate, + .filter_table_probe = efx_farch_filter_table_probe, + .filter_table_restore = efx_farch_filter_table_restore, + .filter_table_remove = efx_farch_filter_table_remove, + .filter_update_rx_scatter = efx_farch_filter_update_rx_scatter, + .filter_insert = efx_farch_filter_insert, + .filter_remove_safe = efx_farch_filter_remove_safe, + .filter_get_safe = efx_farch_filter_get_safe, + .filter_clear_rx = efx_farch_filter_clear_rx, + .filter_count_rx_used = efx_farch_filter_count_rx_used, + .filter_get_rx_id_limit = efx_farch_filter_get_rx_id_limit, + .filter_get_rx_ids = efx_farch_filter_get_rx_ids, +#ifdef CONFIG_RFS_ACCEL + .filter_rfs_insert = efx_farch_filter_rfs_insert, + .filter_rfs_expire_one = efx_farch_filter_rfs_expire_one, +#endif .revision = EFX_REV_SIENA_A0, .txd_ptr_tbl_base = FR_BZ_TX_DESC_PTR_TBL, @@ -752,4 +767,5 @@ const struct efx_nic_type siena_a0_nic_type = { .offload_features = (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXHASH | NETIF_F_NTUPLE), .mcdi_max_ver = 1, + .max_rx_ip_filters = FR_BZ_RX_FILTER_TBL0_ROWS, }; |