diff options
Diffstat (limited to 'drivers/net/ethernet/sfc/ef10.c')
| -rw-r--r-- | drivers/net/ethernet/sfc/ef10.c | 796 |
1 files changed, 360 insertions, 436 deletions
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index 75fbf58e421c..c4c45c94da77 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c @@ -28,9 +28,6 @@ enum { EFX_EF10_TEST = 1, EFX_EF10_REFILL, }; - -/* The reserved RSS context value */ -#define EFX_EF10_RSS_CONTEXT_INVALID 0xffffffff /* The maximum size of a shared RSS context */ /* TODO: this should really be from the mcdi protocol export */ #define EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE 64UL @@ -99,17 +96,15 @@ struct efx_ef10_filter_table { MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_MAXNUM * 2]; unsigned int rx_match_count; + struct rw_semaphore lock; /* Protects entries */ struct { unsigned long spec; /* pointer to spec plus flag bits */ -/* BUSY flag indicates that an update is in progress. AUTO_OLD is - * used to mark and sweep MAC filters for the device address lists. - */ -#define EFX_EF10_FILTER_FLAG_BUSY 1UL +/* AUTO_OLD is used to mark and sweep MAC filters for the device address lists. */ +/* unused flag 1UL */ #define EFX_EF10_FILTER_FLAG_AUTO_OLD 2UL #define EFX_EF10_FILTER_FLAGS 3UL u64 handle; /* firmware handle */ } *entry; - wait_queue_head_t waitq; /* Shadow of net_device address lists, guarded by mac_lock */ struct efx_ef10_dev_addr dev_uc_list[EFX_EF10_FILTER_DEV_UC_MAX]; struct efx_ef10_dev_addr dev_mc_list[EFX_EF10_FILTER_DEV_MC_MAX]; @@ -697,7 +692,7 @@ static int efx_ef10_probe(struct efx_nic *efx) } nic_data->warm_boot_count = rc; - nic_data->rx_rss_context = EFX_EF10_RSS_CONTEXT_INVALID; + efx->rss_context.context_id = EFX_EF10_RSS_CONTEXT_INVALID; nic_data->vport_id = EVB_PORT_ID_ASSIGNED; @@ -1489,8 +1484,8 @@ static int efx_ef10_init_nic(struct efx_nic *efx) } /* don't fail init if RSS setup doesn't work */ - rc = efx->type->rx_push_rss_config(efx, false, efx->rx_indir_table, NULL); - efx->rss_active = (rc == 0); + rc = efx->type->rx_push_rss_config(efx, false, + efx->rss_context.rx_indir_table, NULL); return 0; } @@ -1504,10 +1499,11 @@ static void efx_ef10_reset_mc_allocations(struct efx_nic *efx) /* All our allocations have been reset */ nic_data->must_realloc_vis = true; + nic_data->must_restore_rss_contexts = true; nic_data->must_restore_filters = true; nic_data->must_restore_piobufs = true; efx_ef10_forget_old_piobufs(efx); - nic_data->rx_rss_context = EFX_EF10_RSS_CONTEXT_INVALID; + efx->rss_context.context_id = EFX_EF10_RSS_CONTEXT_INVALID; /* Driver-created vswitches and vports must be re-created */ nic_data->must_probe_vswitching = true; @@ -2703,27 +2699,30 @@ static int efx_ef10_get_rss_flags(struct efx_nic *efx, u32 context, u32 *flags) * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we * just need to set the UDP ports flags (for both IP versions). */ -static void efx_ef10_set_rss_flags(struct efx_nic *efx, u32 context) +static void efx_ef10_set_rss_flags(struct efx_nic *efx, + struct efx_rss_context *ctx) { MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN); u32 flags; BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0); - if (efx_ef10_get_rss_flags(efx, context, &flags) != 0) + if (efx_ef10_get_rss_flags(efx, ctx->context_id, &flags) != 0) return; - MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, context); + MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, + ctx->context_id); flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN; flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN; MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags); if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, sizeof(inbuf), NULL, 0, NULL)) /* Succeeded, so UDP 4-tuple is now enabled */ - efx->rx_hash_udp_4tuple = true; + ctx->rx_hash_udp_4tuple = true; } -static int efx_ef10_alloc_rss_context(struct efx_nic *efx, u32 *context, - bool exclusive, unsigned *context_size) +static int efx_ef10_alloc_rss_context(struct efx_nic *efx, bool exclusive, + struct efx_rss_context *ctx, + unsigned *context_size) { MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN); MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); @@ -2739,7 +2738,7 @@ static int efx_ef10_alloc_rss_context(struct efx_nic *efx, u32 *context, EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE); if (!exclusive && rss_spread == 1) { - *context = EFX_EF10_RSS_CONTEXT_INVALID; + ctx->context_id = EFX_EF10_RSS_CONTEXT_INVALID; if (context_size) *context_size = 1; return 0; @@ -2762,29 +2761,26 @@ static int efx_ef10_alloc_rss_context(struct efx_nic *efx, u32 *context, if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) return -EIO; - *context = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); + ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); if (context_size) *context_size = rss_spread; if (nic_data->datapath_caps & 1 << MC_CMD_GET_CAPABILITIES_OUT_ADDITIONAL_RSS_MODES_LBN) - efx_ef10_set_rss_flags(efx, *context); + efx_ef10_set_rss_flags(efx, ctx); return 0; } -static void efx_ef10_free_rss_context(struct efx_nic *efx, u32 context) +static int efx_ef10_free_rss_context(struct efx_nic *efx, u32 context) { MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN); - int rc; MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, context); - - rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf), + return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf), NULL, 0, NULL); - WARN_ON(rc != 0); } static int efx_ef10_populate_rss_table(struct efx_nic *efx, u32 context, @@ -2796,15 +2792,15 @@ static int efx_ef10_populate_rss_table(struct efx_nic *efx, u32 context, MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, context); - BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) != + BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) != MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN); - /* This iterates over the length of efx->rx_indir_table, but copies - * bytes from rx_indir_table. That's because the latter is a pointer - * rather than an array, but should have the same length. - * The efx->rx_hash_key loop below is similar. + /* This iterates over the length of efx->rss_context.rx_indir_table, but + * copies bytes from rx_indir_table. That's because the latter is a + * pointer rather than an array, but should have the same length. + * The efx->rss_context.rx_hash_key loop below is similar. */ - for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); ++i) + for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i) MCDI_PTR(tablebuf, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] = (u8) rx_indir_table[i]; @@ -2816,9 +2812,9 @@ static int efx_ef10_populate_rss_table(struct efx_nic *efx, u32 context, MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, context); - BUILD_BUG_ON(ARRAY_SIZE(efx->rx_hash_key) != + BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); - for (i = 0; i < ARRAY_SIZE(efx->rx_hash_key); ++i) + for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i) MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i]; return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf, @@ -2827,27 +2823,27 @@ static int efx_ef10_populate_rss_table(struct efx_nic *efx, u32 context, static void efx_ef10_rx_free_indir_table(struct efx_nic *efx) { - struct efx_ef10_nic_data *nic_data = efx->nic_data; + int rc; - if (nic_data->rx_rss_context != EFX_EF10_RSS_CONTEXT_INVALID) - efx_ef10_free_rss_context(efx, nic_data->rx_rss_context); - nic_data->rx_rss_context = EFX_EF10_RSS_CONTEXT_INVALID; + if (efx->rss_context.context_id != EFX_EF10_RSS_CONTEXT_INVALID) { + rc = efx_ef10_free_rss_context(efx, efx->rss_context.context_id); + WARN_ON(rc != 0); + } + efx->rss_context.context_id = EFX_EF10_RSS_CONTEXT_INVALID; } static int efx_ef10_rx_push_shared_rss_config(struct efx_nic *efx, unsigned *context_size) { - u32 new_rx_rss_context; struct efx_ef10_nic_data *nic_data = efx->nic_data; - int rc = efx_ef10_alloc_rss_context(efx, &new_rx_rss_context, - false, context_size); + int rc = efx_ef10_alloc_rss_context(efx, false, &efx->rss_context, + context_size); if (rc != 0) return rc; - nic_data->rx_rss_context = new_rx_rss_context; nic_data->rx_rss_context_exclusive = false; - efx_set_default_rx_indir_table(efx); + efx_set_default_rx_indir_table(efx, &efx->rss_context); return 0; } @@ -2855,65 +2851,98 @@ static int efx_ef10_rx_push_exclusive_rss_config(struct efx_nic *efx, const u32 *rx_indir_table, const u8 *key) { + u32 old_rx_rss_context = efx->rss_context.context_id; struct efx_ef10_nic_data *nic_data = efx->nic_data; int rc; - u32 new_rx_rss_context; - if (nic_data->rx_rss_context == EFX_EF10_RSS_CONTEXT_INVALID || + if (efx->rss_context.context_id == EFX_EF10_RSS_CONTEXT_INVALID || !nic_data->rx_rss_context_exclusive) { - rc = efx_ef10_alloc_rss_context(efx, &new_rx_rss_context, - true, NULL); + rc = efx_ef10_alloc_rss_context(efx, true, &efx->rss_context, + NULL); if (rc == -EOPNOTSUPP) return rc; else if (rc != 0) goto fail1; - } else { - new_rx_rss_context = nic_data->rx_rss_context; } - rc = efx_ef10_populate_rss_table(efx, new_rx_rss_context, + rc = efx_ef10_populate_rss_table(efx, efx->rss_context.context_id, rx_indir_table, key); if (rc != 0) goto fail2; - if (nic_data->rx_rss_context != new_rx_rss_context) - efx_ef10_rx_free_indir_table(efx); - nic_data->rx_rss_context = new_rx_rss_context; + if (efx->rss_context.context_id != old_rx_rss_context && + old_rx_rss_context != EFX_EF10_RSS_CONTEXT_INVALID) + WARN_ON(efx_ef10_free_rss_context(efx, old_rx_rss_context) != 0); nic_data->rx_rss_context_exclusive = true; - if (rx_indir_table != efx->rx_indir_table) - memcpy(efx->rx_indir_table, rx_indir_table, - sizeof(efx->rx_indir_table)); - if (key != efx->rx_hash_key) - memcpy(efx->rx_hash_key, key, efx->type->rx_hash_key_size); + if (rx_indir_table != efx->rss_context.rx_indir_table) + memcpy(efx->rss_context.rx_indir_table, rx_indir_table, + sizeof(efx->rss_context.rx_indir_table)); + if (key != efx->rss_context.rx_hash_key) + memcpy(efx->rss_context.rx_hash_key, key, + efx->type->rx_hash_key_size); return 0; fail2: - if (new_rx_rss_context != nic_data->rx_rss_context) - efx_ef10_free_rss_context(efx, new_rx_rss_context); + if (old_rx_rss_context != efx->rss_context.context_id) { + WARN_ON(efx_ef10_free_rss_context(efx, efx->rss_context.context_id) != 0); + efx->rss_context.context_id = old_rx_rss_context; + } fail1: netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc); return rc; } -static int efx_ef10_rx_pull_rss_config(struct efx_nic *efx) +static int efx_ef10_rx_push_rss_context_config(struct efx_nic *efx, + struct efx_rss_context *ctx, + const u32 *rx_indir_table, + const u8 *key) +{ + int rc; + + WARN_ON(!mutex_is_locked(&efx->rss_lock)); + + if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) { + rc = efx_ef10_alloc_rss_context(efx, true, ctx, NULL); + if (rc) + return rc; + } + + if (!rx_indir_table) /* Delete this context */ + return efx_ef10_free_rss_context(efx, ctx->context_id); + + rc = efx_ef10_populate_rss_table(efx, ctx->context_id, + rx_indir_table, key); + if (rc) + return rc; + + memcpy(ctx->rx_indir_table, rx_indir_table, + sizeof(efx->rss_context.rx_indir_table)); + memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size); + + return 0; +} + +static int efx_ef10_rx_pull_rss_context_config(struct efx_nic *efx, + struct efx_rss_context *ctx) { - struct efx_ef10_nic_data *nic_data = efx->nic_data; MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN); MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN); MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN); size_t outlen; int rc, i; + WARN_ON(!mutex_is_locked(&efx->rss_lock)); + BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN != MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN); - if (nic_data->rx_rss_context == EFX_EF10_RSS_CONTEXT_INVALID) + if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) return -ENOENT; MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID, - nic_data->rx_rss_context); - BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) != + ctx->context_id); + BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN); rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, sizeof(inbuf), tablebuf, sizeof(tablebuf), &outlen); @@ -2923,13 +2952,13 @@ static int efx_ef10_rx_pull_rss_config(struct efx_nic *efx) if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN)) return -EIO; - for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++) - efx->rx_indir_table[i] = MCDI_PTR(tablebuf, + for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) + ctx->rx_indir_table[i] = MCDI_PTR(tablebuf, RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i]; MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID, - nic_data->rx_rss_context); - BUILD_BUG_ON(ARRAY_SIZE(efx->rx_hash_key) != + ctx->context_id); + BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, sizeof(inbuf), keybuf, sizeof(keybuf), &outlen); @@ -2939,13 +2968,50 @@ static int efx_ef10_rx_pull_rss_config(struct efx_nic *efx) if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN)) return -EIO; - for (i = 0; i < ARRAY_SIZE(efx->rx_hash_key); ++i) - efx->rx_hash_key[i] = MCDI_PTR( + for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i) + ctx->rx_hash_key[i] = MCDI_PTR( keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i]; return 0; } +static int efx_ef10_rx_pull_rss_config(struct efx_nic *efx) +{ + int rc; + + mutex_lock(&efx->rss_lock); + rc = efx_ef10_rx_pull_rss_context_config(efx, &efx->rss_context); + mutex_unlock(&efx->rss_lock); + return rc; +} + +static void efx_ef10_rx_restore_rss_contexts(struct efx_nic *efx) +{ + struct efx_ef10_nic_data *nic_data = efx->nic_data; + struct efx_rss_context *ctx; + int rc; + + WARN_ON(!mutex_is_locked(&efx->rss_lock)); + + if (!nic_data->must_restore_rss_contexts) + return; + + list_for_each_entry(ctx, &efx->rss_context.list, list) { + /* previous NIC RSS context is gone */ + ctx->context_id = EFX_EF10_RSS_CONTEXT_INVALID; + /* so try to allocate a new one */ + rc = efx_ef10_rx_push_rss_context_config(efx, ctx, + ctx->rx_indir_table, + ctx->rx_hash_key); + if (rc) + netif_warn(efx, probe, efx->net_dev, + "failed to restore RSS context %u, rc=%d" + "; RSS filters may fail to be applied\n", + ctx->user_id, rc); + } + nic_data->must_restore_rss_contexts = false; +} + static int efx_ef10_pf_rx_push_rss_config(struct efx_nic *efx, bool user, const u32 *rx_indir_table, const u8 *key) @@ -2956,7 +3022,7 @@ static int efx_ef10_pf_rx_push_rss_config(struct efx_nic *efx, bool user, return 0; if (!key) - key = efx->rx_hash_key; + key = efx->rss_context.rx_hash_key; rc = efx_ef10_rx_push_exclusive_rss_config(efx, rx_indir_table, key); @@ -2965,7 +3031,8 @@ static int efx_ef10_pf_rx_push_rss_config(struct efx_nic *efx, bool user, bool mismatch = false; size_t i; - for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table) && !mismatch; + for (i = 0; + i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch; i++) mismatch = rx_indir_table[i] != ethtool_rxfh_indir_default(i, efx->rss_spread); @@ -3000,11 +3067,9 @@ static int efx_ef10_vf_rx_push_rss_config(struct efx_nic *efx, bool user, const u8 *key __attribute__ ((unused))) { - struct efx_ef10_nic_data *nic_data = efx->nic_data; - if (user) return -EOPNOTSUPP; - if (nic_data->rx_rss_context != EFX_EF10_RSS_CONTEXT_INVALID) + if (efx->rss_context.context_id != EFX_EF10_RSS_CONTEXT_INVALID) return 0; return efx_ef10_rx_push_shared_rss_config(efx, NULL); } @@ -4109,6 +4174,7 @@ efx_ef10_filter_push_prep_set_match_fields(struct efx_nic *efx, static void efx_ef10_filter_push_prep(struct efx_nic *efx, const struct efx_filter_spec *spec, efx_dword_t *inbuf, u64 handle, + struct efx_rss_context *ctx, bool replacing) { struct efx_ef10_nic_data *nic_data = efx->nic_data; @@ -4116,11 +4182,16 @@ static void efx_ef10_filter_push_prep(struct efx_nic *efx, memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN); - /* Remove RSS flag if we don't have an RSS context. */ - if (flags & EFX_FILTER_FLAG_RX_RSS && - spec->rss_context == EFX_FILTER_RSS_CONTEXT_DEFAULT && - nic_data->rx_rss_context == EFX_EF10_RSS_CONTEXT_INVALID) - flags &= ~EFX_FILTER_FLAG_RX_RSS; + /* If RSS filter, caller better have given us an RSS context */ + if (flags & EFX_FILTER_FLAG_RX_RSS) { + /* We don't have the ability to return an error, so we'll just + * log a warning and disable RSS for the filter. + */ + if (WARN_ON_ONCE(!ctx)) + flags &= ~EFX_FILTER_FLAG_RX_RSS; + else if (WARN_ON_ONCE(ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID)) + flags &= ~EFX_FILTER_FLAG_RX_RSS; + } if (replacing) { MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, @@ -4146,21 +4217,18 @@ static void efx_ef10_filter_push_prep(struct efx_nic *efx, MC_CMD_FILTER_OP_IN_RX_MODE_RSS : MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE); if (flags & EFX_FILTER_FLAG_RX_RSS) - MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, - spec->rss_context != - EFX_FILTER_RSS_CONTEXT_DEFAULT ? - spec->rss_context : nic_data->rx_rss_context); + MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id); } static int efx_ef10_filter_push(struct efx_nic *efx, - const struct efx_filter_spec *spec, - u64 *handle, bool replacing) + const struct efx_filter_spec *spec, u64 *handle, + struct efx_rss_context *ctx, bool replacing) { MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN); int rc; - efx_ef10_filter_push_prep(efx, spec, inbuf, *handle, replacing); + efx_ef10_filter_push_prep(efx, spec, inbuf, *handle, ctx, replacing); rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), NULL); if (rc == 0) @@ -4249,25 +4317,35 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec, bool replace_equal) { - struct efx_ef10_filter_table *table = efx->filter_state; DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); + struct efx_ef10_nic_data *nic_data = efx->nic_data; + struct efx_ef10_filter_table *table; struct efx_filter_spec *saved_spec; + struct efx_rss_context *ctx = NULL; unsigned int match_pri, hash; unsigned int priv_flags; + bool rss_locked = false; bool replacing = false; + unsigned int depth, i; int ins_index = -1; DEFINE_WAIT(wait); bool is_mc_recip; s32 rc; + down_read(&efx->filter_sem); + table = efx->filter_state; + down_write(&table->lock); + /* For now, only support RX filters */ if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) != - EFX_FILTER_FLAG_RX) - return -EINVAL; + EFX_FILTER_FLAG_RX) { + rc = -EINVAL; + goto out_unlock; + } rc = efx_ef10_filter_pri(table, spec); if (rc < 0) - return rc; + goto out_unlock; match_pri = rc; hash = efx_ef10_filter_hash(spec); @@ -4275,80 +4353,71 @@ static s32 efx_ef10_filter_insert(struct efx_nic *efx, if (is_mc_recip) bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); + if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { + mutex_lock(&efx->rss_lock); + rss_locked = true; + if (spec->rss_context) + ctx = efx_find_rss_context_entry(efx, spec->rss_context); + else + ctx = &efx->rss_context; + if (!ctx) { + rc = -ENOENT; + goto out_unlock; + } + if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) { + rc = -EOPNOTSUPP; + goto out_unlock; + } + } + /* Find any existing filters with the same match tuple or - * else a free slot to insert at. If any of them are busy, - * we have to wait and retry. + * else a free slot to insert at. */ - for (;;) { - unsigned int depth = 1; - unsigned int i; - - spin_lock_bh(&efx->filter_lock); - - for (;;) { - i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1); - saved_spec = efx_ef10_filter_entry_spec(table, i); + for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { + i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1); + saved_spec = efx_ef10_filter_entry_spec(table, i); - if (!saved_spec) { - if (ins_index < 0) - ins_index = i; - } else if (efx_ef10_filter_equal(spec, saved_spec)) { - if (table->entry[i].spec & - EFX_EF10_FILTER_FLAG_BUSY) - break; - if (spec->priority < saved_spec->priority && - spec->priority != EFX_FILTER_PRI_AUTO) { - rc = -EPERM; - goto out_unlock; - } - if (!is_mc_recip) { - /* This is the only one */ - if (spec->priority == - saved_spec->priority && - !replace_equal) { - rc = -EEXIST; - goto out_unlock; - } - ins_index = i; - goto found; - } else if (spec->priority > - saved_spec->priority || - (spec->priority == - saved_spec->priority && - replace_equal)) { - if (ins_index < 0) - ins_index = i; - else - __set_bit(depth, mc_rem_map); - } + if (!saved_spec) { + if (ins_index < 0) + ins_index = i; + } else if (efx_ef10_filter_equal(spec, saved_spec)) { + if (spec->priority < saved_spec->priority && + spec->priority != EFX_FILTER_PRI_AUTO) { + rc = -EPERM; + goto out_unlock; } - - /* Once we reach the maximum search depth, use - * the first suitable slot or return -EBUSY if - * there was none - */ - if (depth == EFX_EF10_FILTER_SEARCH_LIMIT) { - if (ins_index < 0) { - rc = -EBUSY; + if (!is_mc_recip) { + /* This is the only one */ + if (spec->priority == + saved_spec->priority && + !replace_equal) { + rc = -EEXIST; goto out_unlock; } - goto found; + ins_index = i; + break; + } else if (spec->priority > + saved_spec->priority || + (spec->priority == + saved_spec->priority && + replace_equal)) { + if (ins_index < 0) + ins_index = i; + else + __set_bit(depth, mc_rem_map); } - - ++depth; } - - prepare_to_wait(&table->waitq, &wait, TASK_UNINTERRUPTIBLE); - spin_unlock_bh(&efx->filter_lock); - schedule(); } -found: - /* Create a software table entry if necessary, and mark it - * busy. We might yet fail to insert, but any attempt to - * insert a conflicting filter while we're waiting for the - * firmware must find the busy entry. + /* Once we reach the maximum search depth, use the first suitable + * slot, or return -EBUSY if there was none */ + if (ins_index < 0) { + rc = -EBUSY; + goto out_unlock; + } + + /* Create a software table entry if necessary. */ saved_spec = efx_ef10_filter_entry_spec(table, ins_index); if (saved_spec) { if (spec->priority == EFX_FILTER_PRI_AUTO && @@ -4372,28 +4441,19 @@ found: *saved_spec = *spec; priv_flags = 0; } - efx_ef10_filter_set_entry(table, ins_index, saved_spec, - priv_flags | EFX_EF10_FILTER_FLAG_BUSY); - - /* Mark lower-priority multicast recipients busy prior to removal */ - if (is_mc_recip) { - unsigned int depth, i; - - for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { - i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1); - if (test_bit(depth, mc_rem_map)) - table->entry[i].spec |= - EFX_EF10_FILTER_FLAG_BUSY; - } - } - - spin_unlock_bh(&efx->filter_lock); + efx_ef10_filter_set_entry(table, ins_index, saved_spec, priv_flags); + /* Actually insert the filter on the HW */ rc = efx_ef10_filter_push(efx, spec, &table->entry[ins_index].handle, - replacing); + ctx, replacing); + + if (rc == -EINVAL && nic_data->must_realloc_vis) + /* The MC rebooted under us, causing it to reject our filter + * insertion as pointing to an invalid VI (spec->dmaq_id). + */ + rc = -EAGAIN; /* Finalise the software table entry */ - spin_lock_bh(&efx->filter_lock); if (rc == 0) { if (replacing) { /* Update the fields that may differ */ @@ -4409,6 +4469,12 @@ found: } else if (!replacing) { kfree(saved_spec); saved_spec = NULL; + } else { + /* We failed to replace, so the old filter is still present. + * Roll back the software table to reflect this. In fact the + * efx_ef10_filter_set_entry() call below will do the right + * thing, so nothing extra is needed here. + */ } efx_ef10_filter_set_entry(table, ins_index, saved_spec, priv_flags); @@ -4430,7 +4496,6 @@ found: priv_flags = efx_ef10_filter_entry_flags(table, i); if (rc == 0) { - spin_unlock_bh(&efx->filter_lock); MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, @@ -4438,15 +4503,12 @@ found: rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), NULL, 0, NULL); - spin_lock_bh(&efx->filter_lock); } if (rc == 0) { kfree(saved_spec); saved_spec = NULL; priv_flags = 0; - } else { - priv_flags &= ~EFX_EF10_FILTER_FLAG_BUSY; } efx_ef10_filter_set_entry(table, i, saved_spec, priv_flags); @@ -4457,10 +4519,11 @@ found: if (rc == 0) rc = efx_ef10_make_filter_id(match_pri, ins_index); - wake_up_all(&table->waitq); out_unlock: - spin_unlock_bh(&efx->filter_lock); - finish_wait(&table->waitq, &wait); + if (rss_locked) + mutex_unlock(&efx->rss_lock); + up_write(&table->lock); + up_read(&efx->filter_sem); return rc; } @@ -4473,6 +4536,8 @@ static void efx_ef10_filter_update_rx_scatter(struct efx_nic *efx) * If !by_index, remove by ID * If by_index, remove by index * Filter ID may come from userland and must be range-checked. + * Caller must hold efx->filter_sem for read, and efx->filter_state->lock + * for write. */ static int efx_ef10_filter_remove_internal(struct efx_nic *efx, unsigned int priority_mask, @@ -4487,45 +4552,23 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx, DEFINE_WAIT(wait); int rc; - /* Find the software table entry and mark it busy. Don't - * remove it yet; any attempt to update while we're waiting - * for the firmware must find the busy entry. - */ - for (;;) { - spin_lock_bh(&efx->filter_lock); - if (!(table->entry[filter_idx].spec & - EFX_EF10_FILTER_FLAG_BUSY)) - break; - prepare_to_wait(&table->waitq, &wait, TASK_UNINTERRUPTIBLE); - spin_unlock_bh(&efx->filter_lock); - schedule(); - } - spec = efx_ef10_filter_entry_spec(table, filter_idx); if (!spec || (!by_index && efx_ef10_filter_pri(table, spec) != - efx_ef10_filter_get_unsafe_pri(filter_id))) { - rc = -ENOENT; - goto out_unlock; - } + efx_ef10_filter_get_unsafe_pri(filter_id))) + return -ENOENT; if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO && priority_mask == (1U << EFX_FILTER_PRI_AUTO)) { /* Just remove flags */ spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO; table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD; - rc = 0; - goto out_unlock; - } - - if (!(priority_mask & (1U << spec->priority))) { - rc = -ENOENT; - goto out_unlock; + return 0; } - table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY; - spin_unlock_bh(&efx->filter_lock); + if (!(priority_mask & (1U << spec->priority))) + return -ENOENT; if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) { /* Reset to an automatic filter */ @@ -4534,15 +4577,15 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx, new_spec.priority = EFX_FILTER_PRI_AUTO; new_spec.flags = (EFX_FILTER_FLAG_RX | - (efx_rss_enabled(efx) ? + (efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0)); new_spec.dmaq_id = 0; - new_spec.rss_context = EFX_FILTER_RSS_CONTEXT_DEFAULT; + new_spec.rss_context = 0; rc = efx_ef10_filter_push(efx, &new_spec, &table->entry[filter_idx].handle, + &efx->rss_context, true); - spin_lock_bh(&efx->filter_lock); if (rc == 0) *spec = new_spec; } else { @@ -4557,7 +4600,6 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx, rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), NULL, 0, NULL); - spin_lock_bh(&efx->filter_lock); if ((rc == 0) || (rc == -ENOENT)) { /* Filter removed OK or didn't actually exist */ kfree(spec); @@ -4569,11 +4611,6 @@ static int efx_ef10_filter_remove_internal(struct efx_nic *efx, } } - table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_BUSY; - wake_up_all(&table->waitq); -out_unlock: - spin_unlock_bh(&efx->filter_lock); - finish_wait(&table->waitq, &wait); return rc; } @@ -4581,17 +4618,33 @@ static int efx_ef10_filter_remove_safe(struct efx_nic *efx, enum efx_filter_priority priority, u32 filter_id) { - return efx_ef10_filter_remove_internal(efx, 1U << priority, - filter_id, false); + struct efx_ef10_filter_table *table; + int rc; + + down_read(&efx->filter_sem); + table = efx->filter_state; + down_write(&table->lock); + rc = efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id, + false); + up_write(&table->lock); + up_read(&efx->filter_sem); + return rc; } +/* Caller must hold efx->filter_sem for read */ static void efx_ef10_filter_remove_unsafe(struct efx_nic *efx, enum efx_filter_priority priority, u32 filter_id) { + struct efx_ef10_filter_table *table = efx->filter_state; + if (filter_id == EFX_EF10_FILTER_ID_INVALID) return; - efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id, true); + + down_write(&table->lock); + efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id, + true); + up_write(&table->lock); } static int efx_ef10_filter_get_safe(struct efx_nic *efx, @@ -4599,11 +4652,13 @@ static int efx_ef10_filter_get_safe(struct efx_nic *efx, u32 filter_id, struct efx_filter_spec *spec) { unsigned int filter_idx = efx_ef10_filter_get_unsafe_id(filter_id); - struct efx_ef10_filter_table *table = efx->filter_state; const struct efx_filter_spec *saved_spec; + struct efx_ef10_filter_table *table; int rc; - spin_lock_bh(&efx->filter_lock); + down_read(&efx->filter_sem); + table = efx->filter_state; + down_read(&table->lock); saved_spec = efx_ef10_filter_entry_spec(table, filter_idx); if (saved_spec && saved_spec->priority == priority && efx_ef10_filter_pri(table, saved_spec) == @@ -4613,13 +4668,15 @@ static int efx_ef10_filter_get_safe(struct efx_nic *efx, } else { rc = -ENOENT; } - spin_unlock_bh(&efx->filter_lock); + up_read(&table->lock); + up_read(&efx->filter_sem); return rc; } static int efx_ef10_filter_clear_rx(struct efx_nic *efx, - enum efx_filter_priority priority) + enum efx_filter_priority priority) { + struct efx_ef10_filter_table *table; unsigned int priority_mask; unsigned int i; int rc; @@ -4627,31 +4684,40 @@ static int efx_ef10_filter_clear_rx(struct efx_nic *efx, priority_mask = (((1U << (priority + 1)) - 1) & ~(1U << EFX_FILTER_PRI_AUTO)); + down_read(&efx->filter_sem); + table = efx->filter_state; + down_write(&table->lock); for (i = 0; i < HUNT_FILTER_TBL_ROWS; i++) { rc = efx_ef10_filter_remove_internal(efx, priority_mask, i, true); if (rc && rc != -ENOENT) - return rc; + break; + rc = 0; } - return 0; + up_write(&table->lock); + up_read(&efx->filter_sem); + return rc; } static u32 efx_ef10_filter_count_rx_used(struct efx_nic *efx, enum efx_filter_priority priority) { - struct efx_ef10_filter_table *table = efx->filter_state; + struct efx_ef10_filter_table *table; unsigned int filter_idx; s32 count = 0; - spin_lock_bh(&efx->filter_lock); + down_read(&efx->filter_sem); + table = efx->filter_state; + down_read(&table->lock); for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) { if (table->entry[filter_idx].spec && efx_ef10_filter_entry_spec(table, filter_idx)->priority == priority) ++count; } - spin_unlock_bh(&efx->filter_lock); + up_read(&table->lock); + up_read(&efx->filter_sem); return count; } @@ -4666,12 +4732,15 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx, enum efx_filter_priority priority, u32 *buf, u32 size) { - struct efx_ef10_filter_table *table = efx->filter_state; + struct efx_ef10_filter_table *table; struct efx_filter_spec *spec; unsigned int filter_idx; s32 count = 0; - spin_lock_bh(&efx->filter_lock); + down_read(&efx->filter_sem); + table = efx->filter_state; + down_read(&table->lock); + for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) { spec = efx_ef10_filter_entry_spec(table, filter_idx); if (spec && spec->priority == priority) { @@ -4685,201 +4754,42 @@ static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx, filter_idx); } } - spin_unlock_bh(&efx->filter_lock); + up_read(&table->lock); + up_read(&efx->filter_sem); return count; } #ifdef CONFIG_RFS_ACCEL -static efx_mcdi_async_completer efx_ef10_filter_rfs_insert_complete; - -static s32 efx_ef10_filter_rfs_insert(struct efx_nic *efx, - struct efx_filter_spec *spec) -{ - struct efx_ef10_filter_table *table = efx->filter_state; - MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); - struct efx_filter_spec *saved_spec; - unsigned int hash, i, depth = 1; - bool replacing = false; - int ins_index = -1; - u64 cookie; - s32 rc; - - /* Must be an RX filter without RSS and not for a multicast - * destination address (RFS only works for connected sockets). - * These restrictions allow us to pass only a tiny amount of - * data through to the completion function. - */ - EFX_WARN_ON_PARANOID(spec->flags != - (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_RX_SCATTER)); - EFX_WARN_ON_PARANOID(spec->priority != EFX_FILTER_PRI_HINT); - EFX_WARN_ON_PARANOID(efx_filter_is_mc_recipient(spec)); - - hash = efx_ef10_filter_hash(spec); - - spin_lock_bh(&efx->filter_lock); - - /* Find any existing filter with the same match tuple or else - * a free slot to insert at. If an existing filter is busy, - * we have to give up. - */ - for (;;) { - i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1); - saved_spec = efx_ef10_filter_entry_spec(table, i); - - if (!saved_spec) { - if (ins_index < 0) - ins_index = i; - } else if (efx_ef10_filter_equal(spec, saved_spec)) { - if (table->entry[i].spec & EFX_EF10_FILTER_FLAG_BUSY) { - rc = -EBUSY; - goto fail_unlock; - } - if (spec->priority < saved_spec->priority) { - rc = -EPERM; - goto fail_unlock; - } - ins_index = i; - break; - } - - /* Once we reach the maximum search depth, use the - * first suitable slot or return -EBUSY if there was - * none - */ - if (depth == EFX_EF10_FILTER_SEARCH_LIMIT) { - if (ins_index < 0) { - rc = -EBUSY; - goto fail_unlock; - } - break; - } - - ++depth; - } - - /* Create a software table entry if necessary, and mark it - * busy. We might yet fail to insert, but any attempt to - * insert a conflicting filter while we're waiting for the - * firmware must find the busy entry. - */ - saved_spec = efx_ef10_filter_entry_spec(table, ins_index); - if (saved_spec) { - replacing = true; - } else { - saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC); - if (!saved_spec) { - rc = -ENOMEM; - goto fail_unlock; - } - *saved_spec = *spec; - } - efx_ef10_filter_set_entry(table, ins_index, saved_spec, - EFX_EF10_FILTER_FLAG_BUSY); - - spin_unlock_bh(&efx->filter_lock); - - /* Pack up the variables needed on completion */ - cookie = replacing << 31 | ins_index << 16 | spec->dmaq_id; - - efx_ef10_filter_push_prep(efx, spec, inbuf, - table->entry[ins_index].handle, replacing); - efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), - MC_CMD_FILTER_OP_OUT_LEN, - efx_ef10_filter_rfs_insert_complete, cookie); - - return ins_index; - -fail_unlock: - spin_unlock_bh(&efx->filter_lock); - return rc; -} - -static void -efx_ef10_filter_rfs_insert_complete(struct efx_nic *efx, unsigned long cookie, - int rc, efx_dword_t *outbuf, - size_t outlen_actual) -{ - struct efx_ef10_filter_table *table = efx->filter_state; - unsigned int ins_index, dmaq_id; - struct efx_filter_spec *spec; - bool replacing; - - /* Unpack the cookie */ - replacing = cookie >> 31; - ins_index = (cookie >> 16) & (HUNT_FILTER_TBL_ROWS - 1); - dmaq_id = cookie & 0xffff; - - spin_lock_bh(&efx->filter_lock); - spec = efx_ef10_filter_entry_spec(table, ins_index); - if (rc == 0) { - table->entry[ins_index].handle = - MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE); - if (replacing) - spec->dmaq_id = dmaq_id; - } else if (!replacing) { - kfree(spec); - spec = NULL; - } - efx_ef10_filter_set_entry(table, ins_index, spec, 0); - spin_unlock_bh(&efx->filter_lock); - - wake_up_all(&table->waitq); -} - -static void -efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx, - unsigned long filter_idx, - int rc, efx_dword_t *outbuf, - size_t outlen_actual); - static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, unsigned int filter_idx) { - struct efx_ef10_filter_table *table = efx->filter_state; - struct efx_filter_spec *spec = - efx_ef10_filter_entry_spec(table, filter_idx); - MCDI_DECLARE_BUF(inbuf, - MC_CMD_FILTER_OP_IN_HANDLE_OFST + - MC_CMD_FILTER_OP_IN_HANDLE_LEN); - - if (!spec || - (table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAG_BUSY) || - spec->priority != EFX_FILTER_PRI_HINT || - !rps_may_expire_flow(efx->net_dev, spec->dmaq_id, - flow_id, filter_idx)) - return false; - - MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, - MC_CMD_FILTER_OP_IN_OP_REMOVE); - MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, - table->entry[filter_idx].handle); - if (efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 0, - efx_ef10_filter_rfs_expire_complete, filter_idx)) - return false; + struct efx_ef10_filter_table *table; + struct efx_filter_spec *spec; + bool ret; - table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY; - return true; -} + down_read(&efx->filter_sem); + table = efx->filter_state; + down_write(&table->lock); + spec = efx_ef10_filter_entry_spec(table, filter_idx); -static void -efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx, - unsigned long filter_idx, - int rc, efx_dword_t *outbuf, - size_t outlen_actual) -{ - struct efx_ef10_filter_table *table = efx->filter_state; - struct efx_filter_spec *spec = - efx_ef10_filter_entry_spec(table, filter_idx); + if (!spec || spec->priority != EFX_FILTER_PRI_HINT) { + ret = true; + goto out_unlock; + } - spin_lock_bh(&efx->filter_lock); - if (rc == 0) { - kfree(spec); - efx_ef10_filter_set_entry(table, filter_idx, NULL, 0); + if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, + flow_id, filter_idx)) { + ret = false; + goto out_unlock; } - table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_BUSY; - wake_up_all(&table->waitq); - spin_unlock_bh(&efx->filter_lock); + + ret = efx_ef10_filter_remove_internal(efx, 1U << spec->priority, + filter_idx, true) == 0; +out_unlock: + up_write(&table->lock); + up_read(&efx->filter_sem); + return ret; } #endif /* CONFIG_RFS_ACCEL */ @@ -5074,9 +4984,9 @@ static int efx_ef10_filter_table_probe(struct efx_nic *efx) table->vlan_filter = !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER); INIT_LIST_HEAD(&table->vlan_list); + init_rwsem(&table->lock); efx->filter_state = table; - init_waitqueue_head(&table->waitq); list_for_each_entry(vlan, &nic_data->vlan_list, list) { rc = efx_ef10_filter_add_vlan(efx, vlan->vid); @@ -5104,6 +5014,7 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx) unsigned int invalid_filters = 0, failed = 0; struct efx_ef10_filter_vlan *vlan; struct efx_filter_spec *spec; + struct efx_rss_context *ctx; unsigned int filter_idx; u32 mcdi_flags; int match_pri; @@ -5117,7 +5028,8 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx) if (!table) return; - spin_lock_bh(&efx->filter_lock); + down_write(&table->lock); + mutex_lock(&efx->rss_lock); for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) { spec = efx_ef10_filter_entry_spec(table, filter_idx); @@ -5133,20 +5045,32 @@ static void efx_ef10_filter_table_restore(struct efx_nic *efx) invalid_filters++; goto not_restored; } - if (spec->rss_context != EFX_FILTER_RSS_CONTEXT_DEFAULT && - spec->rss_context != nic_data->rx_rss_context) - netif_warn(efx, drv, efx->net_dev, - "Warning: unable to restore a filter with specific RSS context.\n"); - - table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY; - spin_unlock_bh(&efx->filter_lock); + if (spec->rss_context) + ctx = efx_find_rss_context_entry(efx, spec->rss_context); + else + ctx = &efx->rss_context; + if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { + if (!ctx) { + netif_warn(efx, drv, efx->net_dev, + "Warning: unable to restore a filter with nonexistent RSS context %u.\n", + spec->rss_context); + invalid_filters++; + goto not_restored; + } + if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) { + netif_warn(efx, drv, efx->net_dev, + "Warning: unable to restore a filter with RSS context %u as it was not created.\n", + spec->rss_context); + invalid_filters++; + goto not_restored; + } + } rc = efx_ef10_filter_push(efx, spec, &table->entry[filter_idx].handle, - false); + ctx, false); if (rc) failed++; - spin_lock_bh(&efx->filter_lock); if (rc) { not_restored: @@ -5158,13 +5082,11 @@ not_restored: kfree(spec); efx_ef10_filter_set_entry(table, filter_idx, NULL, 0); - } else { - table->entry[filter_idx].spec &= - ~EFX_EF10_FILTER_FLAG_BUSY; } } - spin_unlock_bh(&efx->filter_lock); + mutex_unlock(&efx->rss_lock); + up_write(&table->lock); /* This can happen validly if the MC's capabilities have changed, so * is not an error. @@ -5232,6 +5154,8 @@ static void efx_ef10_filter_mark_one_old(struct efx_nic *efx, uint16_t *id) struct efx_ef10_filter_table *table = efx->filter_state; unsigned int filter_idx; + efx_rwsem_assert_write_locked(&table->lock); + if (*id != EFX_EF10_FILTER_ID_INVALID) { filter_idx = efx_ef10_filter_get_unsafe_id(*id); if (!table->entry[filter_idx].spec) @@ -5267,10 +5191,10 @@ static void efx_ef10_filter_mark_old(struct efx_nic *efx) struct efx_ef10_filter_table *table = efx->filter_state; struct efx_ef10_filter_vlan *vlan; - spin_lock_bh(&efx->filter_lock); + down_write(&table->lock); list_for_each_entry(vlan, &table->vlan_list, list) _efx_ef10_filter_vlan_mark_old(efx, vlan); - spin_unlock_bh(&efx->filter_lock); + up_write(&table->lock); } static void efx_ef10_filter_uc_addr_list(struct efx_nic *efx) @@ -5547,10 +5471,7 @@ static int efx_ef10_filter_insert_def(struct efx_nic *efx, return rc; } -/* Remove filters that weren't renewed. Since nothing else changes the AUTO_OLD - * flag or removes these filters, we don't need to hold the filter_lock while - * scanning for these filters. - */ +/* Remove filters that weren't renewed. */ static void efx_ef10_filter_remove_old(struct efx_nic *efx) { struct efx_ef10_filter_table *table = efx->filter_state; @@ -5559,6 +5480,7 @@ static void efx_ef10_filter_remove_old(struct efx_nic *efx) int rc; int i; + down_write(&table->lock); for (i = 0; i < HUNT_FILTER_TBL_ROWS; i++) { if (READ_ONCE(table->entry[i].spec) & EFX_EF10_FILTER_FLAG_AUTO_OLD) { @@ -5570,6 +5492,7 @@ static void efx_ef10_filter_remove_old(struct efx_nic *efx) remove_failed++; } } + up_write(&table->lock); if (remove_failed) netif_info(efx, drv, efx->net_dev, @@ -6698,7 +6621,6 @@ const struct efx_nic_type efx_hunt_a0_vf_nic_type = { .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, #ifdef CONFIG_RFS_ACCEL - .filter_rfs_insert = efx_ef10_filter_rfs_insert, .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, #endif #ifdef CONFIG_SFC_MTD @@ -6784,6 +6706,9 @@ const struct efx_nic_type efx_hunt_a0_nic_type = { .tx_limit_len = efx_ef10_tx_limit_len, .rx_push_rss_config = efx_ef10_pf_rx_push_rss_config, .rx_pull_rss_config = efx_ef10_rx_pull_rss_config, + .rx_push_rss_context_config = efx_ef10_rx_push_rss_context_config, + .rx_pull_rss_context_config = efx_ef10_rx_pull_rss_context_config, + .rx_restore_rss_contexts = efx_ef10_rx_restore_rss_contexts, .rx_probe = efx_ef10_rx_probe, .rx_init = efx_ef10_rx_init, .rx_remove = efx_ef10_rx_remove, @@ -6808,7 +6733,6 @@ const struct efx_nic_type efx_hunt_a0_nic_type = { .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, #ifdef CONFIG_RFS_ACCEL - .filter_rfs_insert = efx_ef10_filter_rfs_insert, .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, #endif #ifdef CONFIG_SFC_MTD |