diff options
Diffstat (limited to 'drivers/hv')
| -rw-r--r-- | drivers/hv/channel.c | 143 | ||||
| -rw-r--r-- | drivers/hv/channel_mgmt.c | 29 | ||||
| -rw-r--r-- | drivers/hv/hv_balloon.c | 12 | ||||
| -rw-r--r-- | drivers/hv/hv_kvp.c | 2 | ||||
| -rw-r--r-- | drivers/hv/ring_buffer.c | 169 | ||||
| -rw-r--r-- | drivers/hv/vmbus_drv.c | 3 |
6 files changed, 133 insertions, 225 deletions
diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index e57cc40cb768..efd5db743319 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -177,6 +177,11 @@ int vmbus_open(struct vmbus_channel *newchannel, u32 send_ringbuffer_size, &vmbus_connection.chn_msg_list); spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + if (newchannel->rescind) { + err = -ENODEV; + goto error_free_gpadl; + } + ret = vmbus_post_msg(open_msg, sizeof(struct vmbus_channel_open_channel), true); @@ -421,6 +426,11 @@ int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer, spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + if (channel->rescind) { + ret = -ENODEV; + goto cleanup; + } + ret = vmbus_post_msg(gpadlmsg, msginfo->msgsize - sizeof(*msginfo), true); if (ret != 0) @@ -494,6 +504,10 @@ int vmbus_teardown_gpadl(struct vmbus_channel *channel, u32 gpadl_handle) list_add_tail(&info->msglistentry, &vmbus_connection.chn_msg_list); spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + if (channel->rescind) + goto post_msg_err; + ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_gpadl_teardown), true); @@ -647,9 +661,23 @@ void vmbus_close(struct vmbus_channel *channel) } EXPORT_SYMBOL_GPL(vmbus_close); -int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer, - u32 bufferlen, u64 requestid, - enum vmbus_packet_type type, u32 flags) +/** + * vmbus_sendpacket() - Send the specified buffer on the given channel + * @channel: Pointer to vmbus_channel structure. + * @buffer: Pointer to the buffer you want to receive the data into. + * @bufferlen: Maximum size of what the the buffer will hold + * @requestid: Identifier of the request + * @type: Type of packet that is being send e.g. negotiate, time + * packet etc. + * + * Sends data in @buffer directly to hyper-v via the vmbus + * This will send the data unparsed to hyper-v. + * + * Mainly used by Hyper-V drivers. + */ +int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer, + u32 bufferlen, u64 requestid, + enum vmbus_packet_type type, u32 flags) { struct vmpacket_descriptor desc; u32 packetlen = sizeof(struct vmpacket_descriptor) + bufferlen; @@ -676,42 +704,19 @@ int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer, return hv_ringbuffer_write(channel, bufferlist, num_vecs); } -EXPORT_SYMBOL(vmbus_sendpacket_ctl); - -/** - * vmbus_sendpacket() - Send the specified buffer on the given channel - * @channel: Pointer to vmbus_channel structure. - * @buffer: Pointer to the buffer you want to receive the data into. - * @bufferlen: Maximum size of what the the buffer will hold - * @requestid: Identifier of the request - * @type: Type of packet that is being send e.g. negotiate, time - * packet etc. - * - * Sends data in @buffer directly to hyper-v via the vmbus - * This will send the data unparsed to hyper-v. - * - * Mainly used by Hyper-V drivers. - */ -int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer, - u32 bufferlen, u64 requestid, - enum vmbus_packet_type type, u32 flags) -{ - return vmbus_sendpacket_ctl(channel, buffer, bufferlen, requestid, - type, flags); -} EXPORT_SYMBOL(vmbus_sendpacket); /* - * vmbus_sendpacket_pagebuffer_ctl - Send a range of single-page buffer + * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer * packets using a GPADL Direct packet type. This interface allows you * to control notifying the host. This will be useful for sending * batched data. Also the sender can control the send flags * explicitly. */ -int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel, - struct hv_page_buffer pagebuffers[], - u32 pagecount, void *buffer, u32 bufferlen, - u64 requestid, u32 flags) +int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel, + struct hv_page_buffer pagebuffers[], + u32 pagecount, void *buffer, u32 bufferlen, + u64 requestid) { int i; struct vmbus_channel_packet_page_buffer desc; @@ -736,7 +741,7 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel, /* Setup the descriptor */ desc.type = VM_PKT_DATA_USING_GPA_DIRECT; - desc.flags = flags; + desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */ desc.length8 = (u16)(packetlen_aligned >> 3); desc.transactionid = requestid; @@ -757,24 +762,6 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel, return hv_ringbuffer_write(channel, bufferlist, 3); } -EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer_ctl); - -/* - * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer - * packets using a GPADL Direct packet type. - */ -int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel, - struct hv_page_buffer pagebuffers[], - u32 pagecount, void *buffer, u32 bufferlen, - u64 requestid) -{ - u32 flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; - - return vmbus_sendpacket_pagebuffer_ctl(channel, pagebuffers, pagecount, - buffer, bufferlen, - requestid, flags); - -} EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer); /* @@ -814,62 +801,6 @@ int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel, } EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc); -/* - * vmbus_sendpacket_multipagebuffer - Send a multi-page buffer packet - * using a GPADL Direct packet type. - */ -int vmbus_sendpacket_multipagebuffer(struct vmbus_channel *channel, - struct hv_multipage_buffer *multi_pagebuffer, - void *buffer, u32 bufferlen, u64 requestid) -{ - struct vmbus_channel_packet_multipage_buffer desc; - u32 descsize; - u32 packetlen; - u32 packetlen_aligned; - struct kvec bufferlist[3]; - u64 aligned_data = 0; - u32 pfncount = NUM_PAGES_SPANNED(multi_pagebuffer->offset, - multi_pagebuffer->len); - - if (pfncount > MAX_MULTIPAGE_BUFFER_COUNT) - return -EINVAL; - - /* - * Adjust the size down since vmbus_channel_packet_multipage_buffer is - * the largest size we support - */ - descsize = sizeof(struct vmbus_channel_packet_multipage_buffer) - - ((MAX_MULTIPAGE_BUFFER_COUNT - pfncount) * - sizeof(u64)); - packetlen = descsize + bufferlen; - packetlen_aligned = ALIGN(packetlen, sizeof(u64)); - - - /* Setup the descriptor */ - desc.type = VM_PKT_DATA_USING_GPA_DIRECT; - desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; - desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */ - desc.length8 = (u16)(packetlen_aligned >> 3); - desc.transactionid = requestid; - desc.rangecount = 1; - - desc.range.len = multi_pagebuffer->len; - desc.range.offset = multi_pagebuffer->offset; - - memcpy(desc.range.pfn_array, multi_pagebuffer->pfn_array, - pfncount * sizeof(u64)); - - bufferlist[0].iov_base = &desc; - bufferlist[0].iov_len = descsize; - bufferlist[1].iov_base = buffer; - bufferlist[1].iov_len = bufferlen; - bufferlist[2].iov_base = &aligned_data; - bufferlist[2].iov_len = (packetlen_aligned - packetlen); - - return hv_ringbuffer_write(channel, bufferlist, 3); -} -EXPORT_SYMBOL_GPL(vmbus_sendpacket_multipagebuffer); - /** * vmbus_recvpacket() - Retrieve the user packet on the specified channel * @channel: Pointer to vmbus_channel structure. diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index dc590195a74e..060df71c2e8b 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -451,6 +451,12 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) /* Make sure this is a new offer */ mutex_lock(&vmbus_connection.channel_mutex); + /* + * Now that we have acquired the channel_mutex, + * we can release the potentially racing rescind thread. + */ + atomic_dec(&vmbus_connection.offer_in_progress); + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { if (!uuid_le_cmp(channel->offermsg.offer.if_type, newchannel->offermsg.offer.if_type) && @@ -481,7 +487,6 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) channel->num_sc++; spin_unlock_irqrestore(&channel->lock, flags); } else { - atomic_dec(&vmbus_connection.offer_in_progress); goto err_free_chan; } } @@ -510,7 +515,6 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) if (!fnew) { if (channel->sc_creation_callback != NULL) channel->sc_creation_callback(newchannel); - atomic_dec(&vmbus_connection.offer_in_progress); return; } @@ -541,7 +545,7 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) goto err_deq_chan; } - atomic_dec(&vmbus_connection.offer_in_progress); + newchannel->probe_done = true; return; err_deq_chan: @@ -873,8 +877,27 @@ static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) channel->rescind = true; spin_unlock_irqrestore(&channel->lock, flags); + /* + * Now that we have posted the rescind state, perform + * rescind related cleanup. + */ vmbus_rescind_cleanup(channel); + /* + * Now wait for offer handling to complete. + */ + while (READ_ONCE(channel->probe_done) == false) { + /* + * We wait here until any channel offer is currently + * being processed. + */ + msleep(1); + } + + /* + * At this point, the rescind handling can proceed safely. + */ + if (channel->device_obj) { if (channel->chn_rescind_callback) { channel->chn_rescind_callback(channel); diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c index f5728deff893..db0e6652d7ef 100644 --- a/drivers/hv/hv_balloon.c +++ b/drivers/hv/hv_balloon.c @@ -584,10 +584,6 @@ static int hv_memory_notifier(struct notifier_block *nb, unsigned long val, switch (val) { case MEM_ONLINE: - spin_lock_irqsave(&dm_device.ha_lock, flags); - dm_device.num_pages_onlined += mem->nr_pages; - spin_unlock_irqrestore(&dm_device.ha_lock, flags); - /* Fall through */ case MEM_CANCEL_ONLINE: if (dm_device.ha_waiting) { dm_device.ha_waiting = false; @@ -644,6 +640,9 @@ static void hv_page_online_one(struct hv_hotadd_state *has, struct page *pg) __online_page_set_limits(pg); __online_page_increment_counters(pg); __online_page_free(pg); + + WARN_ON_ONCE(!spin_is_locked(&dm_device.ha_lock)); + dm_device.num_pages_onlined++; } static void hv_bring_pgs_online(struct hv_hotadd_state *has, @@ -1036,8 +1035,8 @@ static void process_info(struct hv_dynmem_device *dm, struct dm_info_msg *msg) if (info_hdr->data_size == sizeof(__u64)) { __u64 *max_page_count = (__u64 *)&info_hdr[1]; - pr_info("INFO_TYPE_MAX_PAGE_CNT = %llu\n", - *max_page_count); + pr_info("Max. dynamic memory size: %llu MB\n", + (*max_page_count) >> (20 - PAGE_SHIFT)); } break; @@ -1656,6 +1655,7 @@ static int balloon_probe(struct hv_device *dev, } dm_device.state = DM_INITIALIZED; + last_post_time = jiffies; return 0; diff --git a/drivers/hv/hv_kvp.c b/drivers/hv/hv_kvp.c index 9a90b915b5be..5eed1e7da15c 100644 --- a/drivers/hv/hv_kvp.c +++ b/drivers/hv/hv_kvp.c @@ -304,7 +304,7 @@ static int process_ob_ipinfo(void *in_msg, void *out_msg, int op) strlen((char *)in->body.kvp_ip_val.adapter_id), UTF16_HOST_ENDIAN, (wchar_t *)out->kvp_ip_val.adapter_id, - MAX_IP_ADDR_SIZE); + MAX_ADAPTER_ID_SIZE); if (len < 0) return len; diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c index 1f450c39a9b0..12eb8caa4263 100644 --- a/drivers/hv/ring_buffer.c +++ b/drivers/hv/ring_buffer.c @@ -29,6 +29,7 @@ #include <linux/uio.h> #include <linux/vmalloc.h> #include <linux/slab.h> +#include <linux/prefetch.h> #include "hyperv_vmbus.h" @@ -94,30 +95,6 @@ hv_set_next_write_location(struct hv_ring_buffer_info *ring_info, ring_info->ring_buffer->write_index = next_write_location; } -/* Get the next read location for the specified ring buffer. */ -static inline u32 -hv_get_next_read_location(const struct hv_ring_buffer_info *ring_info) -{ - return ring_info->ring_buffer->read_index; -} - -/* - * Get the next read location + offset for the specified ring buffer. - * This allows the caller to skip. - */ -static inline u32 -hv_get_next_readlocation_withoffset(const struct hv_ring_buffer_info *ring_info, - u32 offset) -{ - u32 next = ring_info->ring_buffer->read_index; - - next += offset; - if (next >= ring_info->ring_datasize) - next -= ring_info->ring_datasize; - - return next; -} - /* Set the next read location for the specified ring buffer. */ static inline void hv_set_next_read_location(struct hv_ring_buffer_info *ring_info, @@ -142,29 +119,6 @@ hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info) } /* - * Helper routine to copy to source from ring buffer. - * Assume there is enough room. Handles wrap-around in src case only!! - */ -static u32 hv_copyfrom_ringbuffer( - const struct hv_ring_buffer_info *ring_info, - void *dest, - u32 destlen, - u32 start_read_offset) -{ - void *ring_buffer = hv_get_ring_buffer(ring_info); - u32 ring_buffer_size = hv_get_ring_buffersize(ring_info); - - memcpy(dest, ring_buffer + start_read_offset, destlen); - - start_read_offset += destlen; - if (start_read_offset >= ring_buffer_size) - start_read_offset -= ring_buffer_size; - - return start_read_offset; -} - - -/* * Helper routine to copy from source to ring buffer. * Assume there is enough room. Handles wrap-around in dest case only!! */ @@ -334,33 +288,22 @@ int hv_ringbuffer_write(struct vmbus_channel *channel, return 0; } -static inline void -init_cached_read_index(struct hv_ring_buffer_info *rbi) -{ - rbi->cached_read_index = rbi->ring_buffer->read_index; -} - int hv_ringbuffer_read(struct vmbus_channel *channel, void *buffer, u32 buflen, u32 *buffer_actual_len, u64 *requestid, bool raw) { - u32 bytes_avail_toread; - u32 next_read_location; - u64 prev_indices = 0; - struct vmpacket_descriptor desc; - u32 offset; - u32 packetlen; - struct hv_ring_buffer_info *inring_info = &channel->inbound; - - if (buflen <= 0) + struct vmpacket_descriptor *desc; + u32 packetlen, offset; + + if (unlikely(buflen == 0)) return -EINVAL; *buffer_actual_len = 0; *requestid = 0; - bytes_avail_toread = hv_get_bytes_to_read(inring_info); /* Make sure there is something to read */ - if (bytes_avail_toread < sizeof(desc)) { + desc = hv_pkt_iter_first(channel); + if (desc == NULL) { /* * No error is set when there is even no header, drivers are * supposed to analyze buffer_actual_len. @@ -368,48 +311,22 @@ int hv_ringbuffer_read(struct vmbus_channel *channel, return 0; } - init_cached_read_index(inring_info); - - next_read_location = hv_get_next_read_location(inring_info); - next_read_location = hv_copyfrom_ringbuffer(inring_info, &desc, - sizeof(desc), - next_read_location); - - offset = raw ? 0 : (desc.offset8 << 3); - packetlen = (desc.len8 << 3) - offset; + offset = raw ? 0 : (desc->offset8 << 3); + packetlen = (desc->len8 << 3) - offset; *buffer_actual_len = packetlen; - *requestid = desc.trans_id; - - if (bytes_avail_toread < packetlen + offset) - return -EAGAIN; + *requestid = desc->trans_id; - if (packetlen > buflen) + if (unlikely(packetlen > buflen)) return -ENOBUFS; - next_read_location = - hv_get_next_readlocation_withoffset(inring_info, offset); + /* since ring is double mapped, only one copy is necessary */ + memcpy(buffer, (const char *)desc + offset, packetlen); - next_read_location = hv_copyfrom_ringbuffer(inring_info, - buffer, - packetlen, - next_read_location); + /* Advance ring index to next packet descriptor */ + __hv_pkt_iter_next(channel, desc); - next_read_location = hv_copyfrom_ringbuffer(inring_info, - &prev_indices, - sizeof(u64), - next_read_location); - - /* - * Make sure all reads are done before we update the read index since - * the writer may start writing to the read area once the read index - * is updated. - */ - virt_mb(); - - /* Update the read index */ - hv_set_next_read_location(inring_info, next_read_location); - - hv_signal_on_read(channel); + /* Notify host of update */ + hv_pkt_iter_close(channel); return 0; } @@ -440,14 +357,16 @@ static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi) struct vmpacket_descriptor *hv_pkt_iter_first(struct vmbus_channel *channel) { struct hv_ring_buffer_info *rbi = &channel->inbound; - - /* set state for later hv_signal_on_read() */ - init_cached_read_index(rbi); + struct vmpacket_descriptor *desc; if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor)) return NULL; - return hv_get_ring_buffer(rbi) + rbi->priv_read_index; + desc = hv_get_ring_buffer(rbi) + rbi->priv_read_index; + if (desc) + prefetch((char *)desc + (desc->len8 << 3)); + + return desc; } EXPORT_SYMBOL_GPL(hv_pkt_iter_first); @@ -471,10 +390,7 @@ __hv_pkt_iter_next(struct vmbus_channel *channel, rbi->priv_read_index -= dsize; /* more data? */ - if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor)) - return NULL; - else - return hv_get_ring_buffer(rbi) + rbi->priv_read_index; + return hv_pkt_iter_first(channel); } EXPORT_SYMBOL_GPL(__hv_pkt_iter_next); @@ -484,6 +400,7 @@ EXPORT_SYMBOL_GPL(__hv_pkt_iter_next); void hv_pkt_iter_close(struct vmbus_channel *channel) { struct hv_ring_buffer_info *rbi = &channel->inbound; + u32 orig_write_sz = hv_get_bytes_to_write(rbi); /* * Make sure all reads are done before we update the read index since @@ -493,6 +410,40 @@ void hv_pkt_iter_close(struct vmbus_channel *channel) virt_rmb(); rbi->ring_buffer->read_index = rbi->priv_read_index; - hv_signal_on_read(channel); + /* + * Issue a full memory barrier before making the signaling decision. + * Here is the reason for having this barrier: + * If the reading of the pend_sz (in this function) + * were to be reordered and read before we commit the new read + * index (in the calling function) we could + * have a problem. If the host were to set the pending_sz after we + * have sampled pending_sz and go to sleep before we commit the + * read index, we could miss sending the interrupt. Issue a full + * memory barrier to address this. + */ + virt_mb(); + + /* If host has disabled notifications then skip */ + if (rbi->ring_buffer->interrupt_mask) + return; + + if (rbi->ring_buffer->feature_bits.feat_pending_send_sz) { + u32 pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz); + + /* + * If there was space before we began iteration, + * then host was not blocked. Also handles case where + * pending_sz is zero then host has nothing pending + * and does not need to be signaled. + */ + if (orig_write_sz > pending_sz) + return; + + /* If pending write will not fit, don't give false hope. */ + if (hv_get_bytes_to_write(rbi) < pending_sz) + return; + } + + vmbus_setevent(channel); } EXPORT_SYMBOL_GPL(hv_pkt_iter_close); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index c7e7d6db2d21..a9d49f6f6501 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -940,6 +940,9 @@ static void vmbus_chan_sched(struct hv_per_cpu_context *hv_cpu) if (channel->offermsg.child_relid != relid) continue; + if (channel->rescind) + continue; + switch (channel->callback_mode) { case HV_CALL_ISR: vmbus_channel_isr(channel); |