diff options
| -rw-r--r-- | drivers/infiniband/core/uverbs.h | 4 | ||||
| -rw-r--r-- | drivers/infiniband/core/uverbs_cmd.c | 890 | ||||
| -rw-r--r-- | drivers/infiniband/core/uverbs_main.c | 35 | ||||
| -rw-r--r-- | include/rdma/ib_verbs.h | 4 |
4 files changed, 557 insertions, 376 deletions
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index 3372d67ff139..bb9bee56a824 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -132,7 +132,7 @@ struct ib_ucq_object { u32 async_events_reported; }; -extern struct mutex ib_uverbs_idr_mutex; +extern spinlock_t ib_uverbs_idr_lock; extern struct idr ib_uverbs_pd_idr; extern struct idr ib_uverbs_mr_idr; extern struct idr ib_uverbs_mw_idr; @@ -141,6 +141,8 @@ extern struct idr ib_uverbs_cq_idr; extern struct idr ib_uverbs_qp_idr; extern struct idr ib_uverbs_srq_idr; +void idr_remove_uobj(struct idr *idp, struct ib_uobject *uobj); + struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file, int is_async, int *fd); void ib_uverbs_release_event_file(struct kref *ref); diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 403dd811ec7f..76bf61e9b552 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -50,7 +50,64 @@ (udata)->outlen = (olen); \ } while (0) -static int idr_add_uobj(struct idr *idr, void *obj, struct ib_uobject *uobj) +/* + * The ib_uobject locking scheme is as follows: + * + * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it + * needs to be held during all idr operations. When an object is + * looked up, a reference must be taken on the object's kref before + * dropping this lock. + * + * - Each object also has an rwsem. This rwsem must be held for + * reading while an operation that uses the object is performed. + * For example, while registering an MR, the associated PD's + * uobject.mutex must be held for reading. The rwsem must be held + * for writing while initializing or destroying an object. + * + * - In addition, each object has a "live" flag. If this flag is not + * set, then lookups of the object will fail even if it is found in + * the idr. This handles a reader that blocks and does not acquire + * the rwsem until after the object is destroyed. The destroy + * operation will set the live flag to 0 and then drop the rwsem; + * this will allow the reader to acquire the rwsem, see that the + * live flag is 0, and then drop the rwsem and its reference to + * object. The underlying storage will not be freed until the last + * reference to the object is dropped. + */ + +static void init_uobj(struct ib_uobject *uobj, u64 user_handle, + struct ib_ucontext *context) +{ + uobj->user_handle = user_handle; + uobj->context = context; + kref_init(&uobj->ref); + init_rwsem(&uobj->mutex); + uobj->live = 0; +} + +static void release_uobj(struct kref *kref) +{ + kfree(container_of(kref, struct ib_uobject, ref)); +} + +static void put_uobj(struct ib_uobject *uobj) +{ + kref_put(&uobj->ref, release_uobj); +} + +static void put_uobj_read(struct ib_uobject *uobj) +{ + up_read(&uobj->mutex); + put_uobj(uobj); +} + +static void put_uobj_write(struct ib_uobject *uobj) +{ + up_write(&uobj->mutex); + put_uobj(uobj); +} + +static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) { int ret; @@ -58,7 +115,9 @@ retry: if (!idr_pre_get(idr, GFP_KERNEL)) return -ENOMEM; + spin_lock(&ib_uverbs_idr_lock); ret = idr_get_new(idr, uobj, &uobj->id); + spin_unlock(&ib_uverbs_idr_lock); if (ret == -EAGAIN) goto retry; @@ -66,6 +125,121 @@ retry: return ret; } +void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) +{ + spin_lock(&ib_uverbs_idr_lock); + idr_remove(idr, uobj->id); + spin_unlock(&ib_uverbs_idr_lock); +} + +static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, + struct ib_ucontext *context) +{ + struct ib_uobject *uobj; + + spin_lock(&ib_uverbs_idr_lock); + uobj = idr_find(idr, id); + if (uobj) + kref_get(&uobj->ref); + spin_unlock(&ib_uverbs_idr_lock); + + return uobj; +} + +static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, + struct ib_ucontext *context) +{ + struct ib_uobject *uobj; + + uobj = __idr_get_uobj(idr, id, context); + if (!uobj) + return NULL; + + down_read(&uobj->mutex); + if (!uobj->live) { + put_uobj_read(uobj); + return NULL; + } + + return uobj; +} + +static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, + struct ib_ucontext *context) +{ + struct ib_uobject *uobj; + + uobj = __idr_get_uobj(idr, id, context); + if (!uobj) + return NULL; + + down_write(&uobj->mutex); + if (!uobj->live) { + put_uobj_write(uobj); + return NULL; + } + + return uobj; +} + +static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context) +{ + struct ib_uobject *uobj; + + uobj = idr_read_uobj(idr, id, context); + return uobj ? uobj->object : NULL; +} + +static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) +{ + return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context); +} + +static void put_pd_read(struct ib_pd *pd) +{ + put_uobj_read(pd->uobject); +} + +static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context) +{ + return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context); +} + +static void put_cq_read(struct ib_cq *cq) +{ + put_uobj_read(cq->uobject); +} + +static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) +{ + return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context); +} + +static void put_ah_read(struct ib_ah *ah) +{ + put_uobj_read(ah->uobject); +} + +static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) +{ + return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context); +} + +static void put_qp_read(struct ib_qp *qp) +{ + put_uobj_read(qp->uobject); +} + +static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) +{ + return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context); +} + +static void put_srq_read(struct ib_srq *srq) +{ + put_uobj_read(srq->uobject); +} + ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) @@ -296,7 +470,8 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, if (!uobj) return -ENOMEM; - uobj->context = file->ucontext; + init_uobj(uobj, 0, file->ucontext); + down_write(&uobj->mutex); pd = file->device->ib_dev->alloc_pd(file->device->ib_dev, file->ucontext, &udata); @@ -309,11 +484,10 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, pd->uobject = uobj; atomic_set(&pd->usecnt, 0); - mutex_lock(&ib_uverbs_idr_mutex); - - ret = idr_add_uobj(&ib_uverbs_pd_idr, pd, uobj); + uobj->object = pd; + ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); if (ret) - goto err_up; + goto err_idr; memset(&resp, 0, sizeof resp); resp.pd_handle = uobj->id; @@ -321,26 +495,27 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; - goto err_idr; + goto err_copy; } mutex_lock(&file->mutex); list_add_tail(&uobj->list, &file->ucontext->pd_list); mutex_unlock(&file->mutex); - mutex_unlock(&ib_uverbs_idr_mutex); + uobj->live = 1; + + up_write(&uobj->mutex); return in_len; -err_idr: - idr_remove(&ib_uverbs_pd_idr, uobj->id); +err_copy: + idr_remove_uobj(&ib_uverbs_pd_idr, uobj); -err_up: - mutex_unlock(&ib_uverbs_idr_mutex); +err_idr: ib_dealloc_pd(pd); err: - kfree(uobj); + put_uobj_write(uobj); return ret; } @@ -349,37 +524,34 @@ ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, int in_len, int out_len) { struct ib_uverbs_dealloc_pd cmd; - struct ib_pd *pd; struct ib_uobject *uobj; - int ret = -EINVAL; + int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&ib_uverbs_idr_mutex); + uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); + if (!uobj) + return -EINVAL; - pd = idr_find(&ib_uverbs_pd_idr, cmd.pd_handle); - if (!pd || pd->uobject->context != file->ucontext) - goto out; + ret = ib_dealloc_pd(uobj->object); + if (!ret) + uobj->live = 0; - uobj = pd->uobject; + put_uobj_write(uobj); - ret = ib_dealloc_pd(pd); if (ret) - goto out; + return ret; - idr_remove(&ib_uverbs_pd_idr, cmd.pd_handle); + idr_remove_uobj(&ib_uverbs_pd_idr, uobj); mutex_lock(&file->mutex); list_del(&uobj->list); mutex_unlock(&file->mutex); - kfree(uobj); + put_uobj(uobj); -out: - mutex_unlock(&ib_uverbs_idr_mutex); - - return ret ? ret : in_len; + return in_len; } ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, @@ -419,7 +591,8 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, if (!obj) return -ENOMEM; - obj->uobject.context = file->ucontext; + init_uobj(&obj->uobject, 0, file->ucontext); + down_write(&obj->uobject.mutex); /* * We ask for writable memory if any access flags other than @@ -436,23 +609,14 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, obj->umem.virt_base = cmd.hca_va; - mutex_lock(&ib_uverbs_idr_mutex); - - pd = idr_find(&ib_uverbs_pd_idr, cmd.pd_handle); - if (!pd || pd->uobject->context != file->ucontext) { - ret = -EINVAL; - goto err_up; - } - - if (!pd->device->reg_user_mr) { - ret = -ENOSYS; - goto err_up; - } + pd = idr_read_pd(cmd.pd_handle, file->ucontext); + if (!pd) + goto err_release; mr = pd->device->reg_user_mr(pd, &obj->umem, cmd.access_flags, &udata); if (IS_ERR(mr)) { ret = PTR_ERR(mr); - goto err_up; + goto err_put; } mr->device = pd->device; @@ -461,43 +625,48 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, atomic_inc(&pd->usecnt); atomic_set(&mr->usecnt, 0); - memset(&resp, 0, sizeof resp); - resp.lkey = mr->lkey; - resp.rkey = mr->rkey; - - ret = idr_add_uobj(&ib_uverbs_mr_idr, mr, &obj->uobject); + obj->uobject.object = mr; + ret = idr_add_uobj(&ib_uverbs_mr_idr, &obj->uobject); if (ret) goto err_unreg; + memset(&resp, 0, sizeof resp); + resp.lkey = mr->lkey; + resp.rkey = mr->rkey; resp.mr_handle = obj->uobject.id; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; - goto err_idr; + goto err_copy; } + put_pd_read(pd); + mutex_lock(&file->mutex); list_add_tail(&obj->uobject.list, &file->ucontext->mr_list); mutex_unlock(&file->mutex); - mutex_unlock(&ib_uverbs_idr_mutex); + obj->uobject.live = 1; + + up_write(&obj->uobject.mutex); return in_len; -err_idr: - idr_remove(&ib_uverbs_mr_idr, obj->uobject.id); +err_copy: + idr_remove_uobj(&ib_uverbs_mr_idr, &obj->uobject); err_unreg: ib_dereg_mr(mr); -err_up: - mutex_unlock(&ib_uverbs_idr_mutex); +err_put: + put_pd_read(pd); +err_release: ib_umem_release(file->device->ib_dev, &obj->umem); err_free: - kfree(obj); + put_uobj_write(&obj->uobject); return ret; } @@ -507,37 +676,40 @@ ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, { struct ib_uverbs_dereg_mr cmd; struct ib_mr *mr; + struct ib_uobject *uobj; struct ib_umem_object *memobj; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&ib_uverbs_idr_mutex); - - mr = idr_find(&ib_uverbs_mr_idr, cmd.mr_handle); - if (!mr || mr->uobject->context != file->ucontext) - goto out; + uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); + if (!uobj) + return -EINVAL; - memobj = container_of(mr->uobject, struct ib_umem_object, uobject); + memobj = container_of(uobj, struct ib_umem_object, uobject); + mr = uobj->object; ret = ib_dereg_mr(mr); + if (!ret) + uobj->live = 0; + + put_uobj_write(uobj); + if (ret) - goto out; + return ret; - idr_remove(&ib_uverbs_mr_idr, cmd.mr_handle); + idr_remove_uobj(&ib_uverbs_mr_idr, uobj); mutex_lock(&file->mutex); - list_del(&memobj->uobject.list); + list_del(&uobj->list); mutex_unlock(&file->mutex); ib_umem_release(file->device->ib_dev, &memobj->umem); - kfree(memobj); -out: - mutex_unlock(&ib_uverbs_idr_mutex); + put_uobj(uobj); - return ret ? ret : in_len; + return in_len; } ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, @@ -576,7 +748,7 @@ ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, struct ib_uverbs_create_cq cmd; struct ib_uverbs_create_cq_resp resp; struct ib_udata udata; - struct ib_ucq_object *uobj; + struct ib_ucq_object *obj; struct ib_uverbs_event_file *ev_file = NULL; struct ib_cq *cq; int ret; @@ -594,10 +766,13 @@ ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, if (cmd.comp_vector >= file->device->num_comp_vectors) return -EINVAL; - uobj = kmalloc(sizeof *uobj, GFP_KERNEL); - if (!uobj) + obj = kmalloc(sizeof *obj, GFP_KERNEL); + if (!obj) return -ENOMEM; + init_uobj(&obj->uobject, cmd.user_handle, file->ucontext); + down_write(&obj->uobject.mutex); + if (cmd.comp_channel >= 0) { ev_file = ib_uverbs_lookup_comp_file(cmd.comp_channel); if (!ev_file) { @@ -606,63 +781,64 @@ ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, } } - uobj->uobject.user_handle = cmd.user_handle; - uobj->uobject.context = file->ucontext; - uobj->uverbs_file = file; - uobj->comp_events_reported = 0; - uobj->async_events_reported = 0; - INIT_LIST_HEAD(&uobj->comp_list); - INIT_LIST_HEAD(&uobj->async_list); + obj->uverbs_file = file; + obj->comp_events_reported = 0; + obj->async_events_reported = 0; + INIT_LIST_HEAD(&obj->comp_list); + INIT_LIST_HEAD(&obj->async_list); cq = file->device->ib_dev->create_cq(file->device->ib_dev, cmd.cqe, file->ucontext, &udata); if (IS_ERR(cq)) { ret = PTR_ERR(cq); - goto err; + goto err_file; } cq->device = file->device->ib_dev; - cq->uobject = &uobj->uobject; + cq->uobject = &obj->uobject; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file; atomic_set(&cq->usecnt, 0); - mutex_lock(&ib_uverbs_idr_mutex); - - ret = idr_add_uobj(&ib_uverbs_cq_idr, cq, &uobj->uobject); + obj->uobject.object = cq; + ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); if (ret) - goto err_up; + goto err_free; memset(&resp, 0, sizeof resp); - resp.cq_handle = uobj->uobject.id; + resp.cq_handle = obj->uobject.id; resp.cqe = cq->cqe; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; - goto err_idr; + goto err_copy; } mutex_lock(&file->mutex); - list_add_tail(&uobj->uobject.list, &file->ucontext->cq_list); + list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); mutex_unlock(&file->mutex); - mutex_unlock(&ib_uverbs_idr_mutex); + obj->uobject.live = 1; + + up_write(&obj->uobject.mutex); return in_len; -err_idr: - idr_remove(&ib_uverbs_cq_idr, uobj->uobject.id); +err_copy: + idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); -err_up: - mutex_unlock(&ib_uverbs_idr_mutex); + +err_free: ib_destroy_cq(cq); -err: +err_file: if (ev_file) - ib_uverbs_release_ucq(file, ev_file, uobj); - kfree(uobj); + ib_uverbs_release_ucq(file, ev_file, obj); + +err: + put_uobj_write(&obj->uobject); return ret; } @@ -683,11 +859,9 @@ ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); - mutex_lock(&ib_uverbs_idr_mutex); - - cq = idr_find(&ib_uverbs_cq_idr, cmd.cq_handle); - if (!cq || cq->uobject->context != file->ucontext || !cq->device->resize_cq) - goto out; + cq = idr_read_cq(cmd.cq_handle, file->ucontext); + if (!cq) + return -EINVAL; ret = cq->device->resize_cq(cq, cmd.cqe, &udata); if (ret) @@ -701,7 +875,7 @@ ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, ret = -EFAULT; out: - mutex_unlock(&ib_uverbs_idr_mutex); + put_cq_read(cq); return ret ? ret : in_len; } @@ -712,6 +886,7 @@ ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, { struct ib_uverbs_poll_cq cmd; struct ib_uverbs_poll_cq_resp *resp; + struct ib_uobject *uobj; struct ib_cq *cq; struct ib_wc *wc; int ret = 0; @@ -732,15 +907,17 @@ ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, goto out_wc; } - mutex_lock(&ib_uverbs_idr_mutex); - cq = idr_find(&ib_uverbs_cq_idr, cmd.cq_handle); - if (!cq || cq->uobject->context != file->ucontext) { + uobj = idr_read_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); + if (!uobj) { ret = -EINVAL; goto out; } + cq = uobj->object; resp->count = ib_poll_cq(cq, cmd.ne, wc); + put_uobj_read(uobj); + for (i = 0; i < resp->count; i++) { resp->wc[i].wr_id = wc[i].wr_id; resp->wc[i].status = wc[i].status; @@ -762,7 +939,6 @@ ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, ret = -EFAULT; out: - mutex_unlock(&ib_uverbs_idr_mutex); kfree(resp); out_wc: @@ -775,22 +951,23 @@ ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, int out_len) { struct ib_uverbs_req_notify_cq cmd; + struct ib_uobject *uobj; struct ib_cq *cq; - int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&ib_uverbs_idr_mutex); - cq = idr_find(&ib_uverbs_cq_idr, cmd.cq_handle); - if (cq && cq->uobject->context == file->ucontext) { - ib_req_notify_cq(cq, cmd.solicited_only ? - IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); - ret = in_len; - } - mutex_unlock(&ib_uverbs_idr_mutex); + uobj = idr_read_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); + if (!uobj) + return -EINVAL; + cq = uobj->object; - return ret; + ib_req_notify_cq(cq, cmd.solicited_only ? + IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); + + put_uobj_read(uobj); + + return in_len; } ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, @@ -799,52 +976,50 @@ ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, { struct ib_uverbs_destroy_cq cmd; struct ib_uverbs_destroy_cq_resp resp; + struct ib_uobject *uobj; struct ib_cq *cq; - struct ib_ucq_object *uobj; + struct ib_ucq_object *obj; struct ib_uverbs_event_file *ev_file; - u64 user_handle; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - memset(&resp, 0, sizeof resp); - - mutex_lock(&ib_uverbs_idr_mutex); + uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); + if (!uobj) + return -EINVAL; + cq = uobj->object; + ev_file = cq->cq_context; + obj = container_of(cq->uobject, struct ib_ucq_object, uobject); - cq = idr_find(&ib_uverbs_cq_idr, cmd.cq_handle); - if (!cq || cq->uobject->context != file->ucontext) - goto out; + ret = ib_destroy_cq(cq); + if (!ret) + uobj->live = 0; - user_handle = cq->uobject->user_handle; - uobj = container_of(cq->uobject, struct ib_ucq_object, uobject); - ev_file = cq->cq_context; + put_uobj_write(uobj); - ret = ib_destroy_cq(cq); if (ret) - goto out; + return ret; - idr_remove(&ib_uverbs_cq_idr, cmd.cq_handle); + idr_remove_uobj(&ib_uverbs_cq_idr, uobj); mutex_lock(&file->mutex); - list_del(&uobj->uobject.list); + list_del(&uobj->list); mutex_unlock(&file->mutex); - ib_uverbs_release_ucq(file, ev_file, uobj); + ib_uverbs_release_ucq(file, ev_file, obj); - resp.comp_events_reported = uobj->comp_events_reported; - resp.async_events_reported = uobj->async_events_reported; + memset(&resp, 0, sizeof resp); + resp.comp_events_reported = obj->comp_events_reported; + resp.async_events_reported = obj->async_events_reported; - kfree(uobj); + put_uobj(uobj); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) - ret = -EFAULT; - -out: - mutex_unlock(&ib_uverbs_idr_mutex); + return -EFAULT; - return ret ? ret : in_len; + return in_len; } ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, @@ -854,7 +1029,7 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, struct ib_uverbs_create_qp cmd; struct ib_uverbs_create_qp_resp resp; struct ib_udata udata; - struct ib_uqp_object *uobj; + struct ib_uqp_object *obj; struct ib_pd *pd; struct ib_cq *scq, *rcq; struct ib_srq *srq; @@ -872,23 +1047,21 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); - uobj = kmalloc(sizeof *uobj, GFP_KERNEL); - if (!uobj) + obj = kmalloc(sizeof *obj, GFP_KERNEL); + if (!obj) return -ENOMEM; - mutex_lock(&ib_uverbs_idr_mutex); + init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext); + down_write(&obj->uevent.uobject.mutex); - pd = idr_find(&ib_uverbs_pd_idr, cmd.pd_handle); - scq = idr_find(&ib_uverbs_cq_idr, cmd.send_cq_handle); - rcq = idr_find(&ib_uverbs_cq_idr, cmd.recv_cq_handle); - srq = cmd.is_srq ? idr_find(&ib_uverbs_srq_idr, cmd.srq_handle) : NULL; + pd = idr_read_pd(cmd.pd_handle, file->ucontext); + scq = idr_read_cq(cmd.send_cq_handle, file->ucontext); + rcq = idr_read_cq(cmd.recv_cq_handle, file->ucontext); + srq = cmd.is_srq ? idr_read_srq(cmd.srq_handle, file->ucontext) : NULL; - if (!pd || pd->uobject->context != file->ucontext || - !scq || scq->uobject->context != file->ucontext || - !rcq || rcq->uobject->context != file->ucontext || - (cmd.is_srq && (!srq || srq->uobject->context != file->ucontext))) { + if (!pd || !scq || !rcq || (cmd.is_srq && !srq)) { ret = -EINVAL; - goto err_up; + goto err_put; } attr.event_handler = ib_uverbs_qp_event_handler; @@ -905,16 +1078,14 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, attr.cap.max_recv_sge = cmd.max_recv_sge; attr.cap.max_inline_data = cmd.max_inline_data; - uobj->uevent.uobject.user_handle = cmd.user_handle; - uobj->uevent.uobject.context = file->ucontext; - uobj->uevent.events_reported = 0; - INIT_LIST_HEAD(&uobj->uevent.event_list); - INIT_LIST_HEAD(&uobj->mcast_list); + obj->uevent.events_reported = 0; + INIT_LIST_HEAD(&obj->uevent.event_list); + INIT_LIST_HEAD(&obj->mcast_list); qp = pd->device->create_qp(pd, &attr, &udata); if (IS_ERR(qp)) { ret = PTR_ERR(qp); - goto err_up; + goto err_put; } qp->device = pd->device; @@ -922,7 +1093,7 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, qp->send_cq = attr.send_cq; qp->recv_cq = attr.recv_cq; qp->srq = attr.srq; - qp->uobject = &uobj->uevent.uobject; + qp->uobject = &obj->uevent.uobject; qp->event_handler = attr.event_handler; qp->qp_context = attr.qp_context; qp->qp_type = attr.qp_type; @@ -932,14 +1103,14 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, if (attr.srq) atomic_inc(&attr.srq->usecnt); - memset(&resp, 0, sizeof resp); - resp.qpn = qp->qp_num; - - ret = idr_add_uobj(&ib_uverbs_qp_idr, qp, &uobj->uevent.uobject); + obj->uevent.uobject.object = qp; + ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); if (ret) goto err_destroy; - resp.qp_handle = uobj->uevent.uobject.id; + memset(&resp, 0, sizeof resp); + resp.qpn = qp->qp_num; + resp.qp_handle = obj->uevent.uobject.id; resp.max_recv_sge = attr.cap.max_recv_sge; resp.max_send_sge = attr.cap.max_send_sge; resp.max_recv_wr = attr.cap.max_recv_wr; @@ -949,27 +1120,42 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; - goto err_idr; + goto err_copy; } + put_pd_read(pd); + put_cq_read(scq); + put_cq_read(rcq); + if (srq) + put_srq_read(srq); + mutex_lock(&file->mutex); - list_add_tail(&uobj->uevent.uobject.list, &file->ucontext->qp_list); + list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); mutex_unlock(&file->mutex); - mutex_unlock(&ib_uverbs_idr_mutex); + obj->uevent.uobject.live = 1; + + up_write(&obj->uevent.uobject.mutex); return in_len; -err_idr: - idr_remove(&ib_uverbs_qp_idr, uobj->uevent.uobject.id); +err_copy: + idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); err_destroy: ib_destroy_qp(qp); -err_up: - mutex_unlock(&ib_uverbs_idr_mutex); - - kfree(uobj); +err_put: + if (pd) + put_pd_read(pd); + if (scq) + put_cq_read(scq); + if (rcq) + put_cq_read(rcq); + if (srq) + put_srq_read(srq); + + put_uobj_write(&obj->uevent.uobject); return ret; } @@ -994,15 +1180,15 @@ ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, goto out; } - mutex_lock(&ib_uverbs_idr_mutex); - - qp = idr_find(&ib_uverbs_qp_idr, cmd.qp_handle); - if (qp && qp->uobject->context == file->ucontext) - ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); - else + qp = idr_read_qp(cmd.qp_handle, file->ucontext); + if (!qp) { ret = -EINVAL; + goto out; + } + + ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); - mutex_unlock(&ib_uverbs_idr_mutex); + put_qp_read(qp); if (ret) goto out; @@ -1089,10 +1275,8 @@ ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, if (!attr) return -ENOMEM; - mutex_lock(&ib_uverbs_idr_mutex); - - qp = idr_find(&ib_uverbs_qp_idr, cmd.qp_handle); - if (!qp || qp->uobject->context != file->ucontext) { + qp = idr_read_qp(cmd.qp_handle, file->ucontext); + if (!qp) { ret = -EINVAL; goto out; } @@ -1144,13 +1328,15 @@ ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; ret = ib_modify_qp(qp, attr, cmd.attr_mask); + + put_qp_read(qp); + if (ret) goto out; ret = in_len; out: - mutex_unlock(&ib_uverbs_idr_mutex); kfree(attr); return ret; @@ -1162,8 +1348,9 @@ ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, { struct ib_uverbs_destroy_qp cmd; struct ib_uverbs_destroy_qp_resp resp; + struct ib_uobject *uobj; struct ib_qp *qp; - struct ib_uqp_object *uobj; + struct ib_uqp_object *obj; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) @@ -1171,43 +1358,43 @@ ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, memset(&resp, 0, sizeof resp); - mutex_lock(&ib_uverbs_idr_mutex); - - qp = idr_find(&ib_uverbs_qp_idr, cmd.qp_handle); - if (!qp || qp->uobject->context != file->ucontext) - goto out; - - uobj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); + uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); + if (!uobj) + return -EINVAL; + qp = uobj->object; + obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); - if (!list_empty(&uobj->mcast_list)) { - ret = -EBUSY; - goto out; + if (!list_empty(&obj->mcast_list)) { + put_uobj_write(uobj); + return -EBUSY; } ret = ib_destroy_qp(qp); + if (!ret) + uobj->live = 0; + + put_uobj_write(uobj); + if (ret) - goto out; + return ret; - idr_remove(&ib_uverbs_qp_idr, cmd.qp_handle); + idr_remove_uobj(&ib_uverbs_qp_idr, uobj); mutex_lock(&file->mutex); - list_del(&uobj->uevent.uobject.list); + list_del(&uobj->list); mutex_unlock(&file->mutex); - ib_uverbs_release_uevent(file, &uobj->uevent); + ib_uverbs_release_uevent(file, &obj->uevent); - resp.events_reported = uobj->uevent.events_reported; + resp.events_reported = obj->uevent.events_reported; - kfree(uobj); + put_uobj(uobj); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) - ret = -EFAULT; - -out: - mutex_unlock(&ib_uverbs_idr_mutex); + return -EFAULT; - return ret ? ret : in_len; + return in_len; } ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, @@ -1220,6 +1407,7 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, struct ib_send_wr *wr = NULL, *last, *next, *bad_wr; struct ib_qp *qp; int i, sg_ind; + int is_ud; ssize_t ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) @@ -1236,12 +1424,11 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, if (!user_wr) return -ENOMEM; - mutex_lock(&ib_uverbs_idr_mutex); - - qp = idr_find(&ib_uverbs_qp_idr, cmd.qp_handle); - if (!qp || qp->uobject->context != file->ucontext) + qp = idr_read_qp(cmd.qp_handle, file->ucontext); + if (!qp) goto out; + is_ud = qp->qp_type == IB_QPT_UD; sg_ind = 0; last = NULL; for (i = 0; i < cmd.wr_count; ++i) { @@ -1249,12 +1436,12 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, buf + sizeof cmd + i * cmd.wqe_size, cmd.wqe_size)) { ret = -EFAULT; - goto out; + goto out_put; } if (user_wr->num_sge + sg_ind > cmd.sge_count) { ret = -EINVAL; - goto out; + goto out_put; } next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + @@ -1262,7 +1449,7 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, GFP_KERNEL); if (!next) { ret = -ENOMEM; - goto out; + goto out_put; } if (!last) @@ -1278,12 +1465,12 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, next->send_flags = user_wr->send_flags; next->imm_data = (__be32 __force) user_wr->imm_data; - if (qp->qp_type == IB_QPT_UD) { - next->wr.ud.ah = idr_find(&ib_uverbs_ah_idr, - user_wr->wr.ud.ah); + if (is_ud) { + next->wr.ud.ah = idr_read_ah(user_wr->wr.ud.ah, + file->ucontext); if (!next->wr.ud.ah) { ret = -EINVAL; - goto out; + goto out_put; } next->wr.ud.remote_qpn = user_wr->wr.ud.remote_qpn; next->wr.ud.remote_qkey = user_wr->wr.ud.remote_qkey; @@ -1320,7 +1507,7 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, sg_ind * sizeof (struct ib_sge), next->num_sge * sizeof (struct ib_sge))) { ret = -EFAULT; - goto out; + goto out_put; } sg_ind += next->num_sge; } else @@ -1340,10 +1527,13 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, &resp, sizeof resp)) ret = -EFAULT; -out: - mutex_unlock(&ib_uverbs_idr_mutex); +out_put: + put_qp_read(qp); +out: while (wr) { + if (is_ud && wr->wr.ud.ah) + put_ah_read(wr->wr.ud.ah); next = wr->next; kfree(wr); wr = next; @@ -1458,14 +1648,15 @@ ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, if (IS_ERR(wr)) return PTR_ERR(wr); - mutex_lock(&ib_uverbs_idr_mutex); - - qp = idr_find(&ib_uverbs_qp_idr, cmd.qp_handle); - if (!qp || qp->uobject->context != file->ucontext) + qp = idr_read_qp(cmd.qp_handle, file->ucontext); + if (!qp) goto out; resp.bad_wr = 0; ret = qp->device->post_recv(qp, wr, &bad_wr); + + put_qp_read(qp); + if (ret) for (next = wr; next; next = next->next) { ++resp.bad_wr; @@ -1479,8 +1670,6 @@ ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, ret = -EFAULT; out: - mutex_unlock(&ib_uverbs_idr_mutex); - while (wr) { next = wr->next; kfree(wr); @@ -1509,14 +1698,15 @@ ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, if (IS_ERR(wr)) return PTR_ERR(wr); - mutex_lock(&ib_uverbs_idr_mutex); - - srq = idr_find(&ib_uverbs_srq_idr, cmd.srq_handle); - if (!srq || srq->uobject->context != file->ucontext) + srq = idr_read_srq(cmd.srq_handle, file->ucontext); + if (!srq) goto out; resp.bad_wr = 0; ret = srq->device->post_srq_recv(srq, wr, &bad_wr); + + put_srq_read(srq); + if (ret) for (next = wr; next; next = next->next) { ++resp.bad_wr; @@ -1530,8 +1720,6 @@ ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, ret = -EFAULT; out: - mutex_unlock(&ib_uverbs_idr_mutex); - while (wr) { next = wr->next; kfree(wr); @@ -1563,17 +1751,15 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, if (!uobj) return -ENOMEM; - mutex_lock(&ib_uverbs_idr_mutex); + init_uobj(uobj, cmd.user_handle, file->ucontext); + down_write(&uobj->mutex); - pd = idr_find(&ib_uverbs_pd_idr, cmd.pd_handle); - if (!pd || pd->uobject->context != file->ucontext) { + pd = idr_read_pd(cmd.pd_handle, file->ucontext); + if (!pd) { ret = -EINVAL; - goto err_up; + goto err; } - uobj->user_handle = cmd.user_handle; - uobj->context = file->ucontext; - attr.dlid = cmd.attr.dlid; attr.sl = cmd.attr.sl; attr.src_path_bits = cmd.attr.src_path_bits; @@ -1589,12 +1775,13 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, ah = ib_create_ah(pd, &attr); if (IS_ERR(ah)) { ret = PTR_ERR(ah); - goto err_up; + goto err; } - ah->uobject = uobj; + ah->uobject = uobj; + uobj->object = ah; - ret = idr_add_uobj(&ib_uverbs_ah_idr, ah, uobj); + ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); if (ret) goto err_destroy; @@ -1603,27 +1790,29 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; - goto err_idr; + goto err_copy; } + put_pd_read(pd); + mutex_lock(&file->mutex); list_add_tail(&uobj->list, &file->ucontext->ah_list); mutex_unlock(&file->mutex); - mutex_unlock(&ib_uverbs_idr_mutex); + uobj->live = 1; + + up_write(&uobj->mutex); return in_len; -err_idr: - idr_remove(&ib_uverbs_ah_idr, uobj->id); +err_copy: + idr_remove_uobj(&ib_uverbs_ah_idr, uobj); err_destroy: ib_destroy_ah(ah); -err_up: - mutex_unlock(&ib_uverbs_idr_mutex); - - kfree(uobj); +err: + put_uobj_write(uobj); return ret; } @@ -1633,35 +1822,34 @@ ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, struct ib_uverbs_destroy_ah cmd; struct ib_ah *ah; struct ib_uobject *uobj; - int ret = -EINVAL; + int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&ib_uverbs_idr_mutex); + uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); + if (!uobj) + return -EINVAL; + ah = uobj->object; - ah = idr_find(&ib_uverbs_ah_idr, cmd.ah_handle); - if (!ah || ah->uobject->context != file->ucontext) - goto out; + ret = ib_destroy_ah(ah); + if (!ret) + uobj->live = 0; - uobj = ah->uobject; + put_uobj_write(uobj); - ret = ib_destroy_ah(ah); if (ret) - goto out; + return ret; - idr_remove(&ib_uverbs_ah_idr, cmd.ah_handle); + idr_remove_uobj(&ib_uverbs_ah_idr, uobj); mutex_lock(&file->mutex); list_del(&uobj->list); mutex_unlock(&file->mutex); - kfree(uobj); + put_uobj(uobj); -out: - mutex_unlock(&ib_uverbs_idr_mutex); - - return ret ? ret : in_len; + return in_len; } ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, @@ -1670,47 +1858,43 @@ ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, { struct ib_uverbs_attach_mcast cmd; struct ib_qp *qp; - struct ib_uqp_object *uobj; + struct ib_uqp_object *obj; struct ib_uverbs_mcast_entry *mcast; - int ret = -EINVAL; + int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&ib_uverbs_idr_mutex); - - qp = idr_find(&ib_uverbs_qp_idr, cmd.qp_handle); - if (!qp || qp->uobject->context != file->ucontext) - goto out; + qp = idr_read_qp(cmd.qp_handle, file->ucontext); + if (!qp) + return -EINVAL; - uobj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); + obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); - list_for_each_entry(mcast, &uobj->mcast_list, list) + list_for_each_entry(mcast, &obj->mcast_list, list) if (cmd.mlid == mcast->lid && !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { ret = 0; - goto out; + goto out_put; } mcast = kmalloc(sizeof *mcast, GFP_KERNEL); if (!mcast) { ret = -ENOMEM; - goto out; + goto out_put; } mcast->lid = cmd.mlid; memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); - if (!ret) { - uobj = container_of(qp->uobject, struct ib_uqp_object, - uevent.uobject); - list_add_tail(&mcast->list, &uobj->mcast_list); - } else + if (!ret) + list_add_tail(&mcast->list, &obj->mcast_list); + else kfree(mcast); -out: - mutex_unlock(&ib_uverbs_idr_mutex); +out_put: + put_qp_read(qp); return ret ? ret : in_len; } @@ -1720,7 +1904,7 @@ ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, int out_len) { struct ib_uverbs_detach_mcast cmd; - struct ib_uqp_object *uobj; + struct ib_uqp_object *obj; struct ib_qp *qp; struct ib_uverbs_mcast_entry *mcast; int ret = -EINVAL; @@ -1728,19 +1912,17 @@ ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&ib_uverbs_idr_mutex); - - qp = idr_find(&ib_uverbs_qp_idr, cmd.qp_handle); - if (!qp || qp->uobject->context != file->ucontext) - goto out; + qp = idr_read_qp(cmd.qp_handle, file->ucontext); + if (!qp) + return -EINVAL; ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); if (ret) - goto out; + goto out_put; - uobj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); + obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); - list_for_each_entry(mcast, &uobj->mcast_list, list) + list_for_each_entry(mcast, &obj->mcast_list, list) if (cmd.mlid == mcast->lid && !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { list_del(&mcast->list); @@ -1748,8 +1930,8 @@ ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, break; } -out: - mutex_unlock(&ib_uverbs_idr_mutex); +out_put: + put_qp_read(qp); return ret ? ret : in_len; } @@ -1761,7 +1943,7 @@ ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, struct ib_uverbs_create_srq cmd; struct ib_uverbs_create_srq_resp resp; struct ib_udata udata; - struct ib_uevent_object *uobj; + struct ib_uevent_object *obj; struct ib_pd *pd; struct ib_srq *srq; struct ib_srq_init_attr attr; @@ -1777,17 +1959,17 @@ ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); - uobj = kmalloc(sizeof *uobj, GFP_KERNEL); - if (!uobj) + obj = kmalloc(sizeof *obj, GFP_KERNEL); + if (!obj) return -ENOMEM; - mutex_lock(&ib_uverbs_idr_mutex); - - pd = idr_find(&ib_uverbs_pd_idr, cmd.pd_handle); + init_uobj(&obj->uobject, 0, file->ucontext); + down_write(&obj->uobject.mutex); - if (!pd || pd->uobject->context != file->ucontext) { + pd = idr_read_pd(cmd.pd_handle, file->ucontext); + if (!pd) { ret = -EINVAL; - goto err_up; + goto err; } attr.event_handler = ib_uverbs_srq_event_handler; @@ -1796,59 +1978,59 @@ ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, attr.attr.max_sge = cmd.max_sge; attr.attr.srq_limit = cmd.srq_limit; - uobj->uobject.user_handle = cmd.user_handle; - uobj->uobject.context = file->ucontext; - uobj->events_reported = 0; - INIT_LIST_HEAD(&uobj->event_list); + obj->events_reported = 0; + INIT_LIST_HEAD(&obj->event_list); srq = pd->device->create_srq(pd, &attr, &udata); if (IS_ERR(srq)) { ret = PTR_ERR(srq); - goto err_up; + goto err; } srq->device = pd->device; srq->pd = pd; - srq->uobject = &uobj->uobject; + srq->uobject = &obj->uobject; srq->event_handler = attr.event_handler; srq->srq_context = attr.srq_context; atomic_inc(&pd->usecnt); atomic_set(&srq->usecnt, 0); - memset(&resp, 0, sizeof resp); - - ret = idr_add_uobj(&ib_uverbs_srq_idr, srq, &uobj->uobject); + obj->uobject.object = srq; + ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uobject); if (ret) goto err_destroy; - resp.srq_handle = uobj->uobject.id; + memset(&resp, 0, sizeof resp); + resp.srq_handle = obj->uobject.id; resp.max_wr = attr.attr.max_wr; resp.max_sge = attr.attr.max_sge; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; - goto err_idr; + goto err_copy; } + put_pd_read(pd); + mutex_lock(&file->mutex); - list_add_tail(&uobj->uobject.list, &file->ucontext->srq_list); + list_add_tail(&obj->uobject.list, &file->ucontext->srq_list); mutex_unlock(&file->mutex); - mutex_unlock(&ib_uverbs_idr_mutex); + obj->uobject.live = 1; + + up_write(&obj->uobject.mutex); return in_len; -err_idr: - idr_remove(&ib_uverbs_srq_idr, uobj->uobject.id); +err_copy: + idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uobject); err_destroy: ib_destroy_srq(srq); -err_up: - mutex_unlock(&ib_uverbs_idr_mutex); - - kfree(uobj); +err: + put_uobj_write(&obj->uobject); return ret; } @@ -1864,21 +2046,16 @@ ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&ib_uverbs_idr_mutex); - - srq = idr_find(&ib_uverbs_srq_idr, cmd.srq_handle); - if (!srq || srq->uobject->context != file->ucontext) { - ret = -EINVAL; - goto out; - } + srq = idr_read_srq(cmd.srq_handle, file->ucontext); + if (!srq) + return -EINVAL; attr.max_wr = cmd.max_wr; attr.srq_limit = cmd.srq_limit; ret = ib_modify_srq(srq, &attr, cmd.attr_mask); -out: - mutex_unlock(&ib_uverbs_idr_mutex); + put_srq_read(srq); return ret ? ret : in_len; } @@ -1899,18 +2076,16 @@ ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&ib_uverbs_idr_mutex); + srq = idr_read_srq(cmd.srq_handle, file->ucontext); + if (!srq) + return -EINVAL; - srq = idr_find(&ib_uverbs_srq_idr, cmd.srq_handle); - if (srq && srq->uobject->context == file->ucontext) - ret = ib_query_srq(srq, &attr); - else - ret = -EINVAL; + ret = ib_query_srq(srq, &attr); - mutex_unlock(&ib_uverbs_idr_mutex); + put_srq_read(srq); if (ret) - goto out; + return ret; memset(&resp, 0, sizeof resp); @@ -1920,10 +2095,9 @@ ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) - ret = -EFAULT; + return -EFAULT; -out: - return ret ? ret : in_len; + return in_len; } ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, @@ -1932,45 +2106,45 @@ ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, { struct ib_uverbs_destroy_srq cmd; struct ib_uverbs_destroy_srq_resp resp; + struct ib_uobject *uobj; struct ib_srq *srq; - struct ib_uevent_object *uobj; + struct ib_uevent_object *obj; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&ib_uverbs_idr_mutex); - - memset(&resp, 0, sizeof resp); + uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); + if (!uobj) + return -EINVAL; + srq = uobj->object; + obj = container_of(uobj, struct ib_uevent_object, uobject); - srq = idr_find(&ib_uverbs_srq_idr, cmd.srq_handle); - if (!srq || srq->uobject->context != file->ucontext) - goto out; + ret = ib_destroy_srq(srq); + if (!ret) + uobj->live = 0; - uobj = container_of(srq->uobject, struct ib_uevent_object, uobject); + put_uobj_write(uobj); - ret = ib_destroy_srq(srq); if (ret) - goto out; + return ret; - idr_remove(&ib_uverbs_srq_idr, cmd.srq_handle); + idr_remove_uobj(&ib_uverbs_srq_idr, uobj); mutex_lock(&file->mutex); - list_del(&uobj->uobject.list); + list_del(&uobj->list); mutex_unlock(&file->mutex); - ib_uverbs_release_uevent(file, uobj); + ib_uverbs_release_uevent(file, obj); - resp.events_reported = uobj->events_reported; + memset(&resp, 0, sizeof resp); + resp.events_reported = obj->events_reported; - kfree(uobj); + put_uobj(uobj); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) ret = -EFAULT; -out: - mutex_unlock(&ib_uverbs_idr_mutex); - return ret ? ret : in_len; } diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index ff092a0a94da..5ec2d49e9bb6 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -66,7 +66,7 @@ enum { static struct class *uverbs_class; -DEFINE_MUTEX(ib_uverbs_idr_mutex); +DEFINE_SPINLOCK(ib_uverbs_idr_lock); DEFINE_IDR(ib_uverbs_pd_idr); DEFINE_IDR(ib_uverbs_mr_idr); DEFINE_IDR(ib_uverbs_mw_idr); @@ -183,21 +183,21 @@ static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file, if (!context) return 0; - mutex_lock(&ib_uverbs_idr_mutex); - list_for_each_entry_safe(uobj, tmp, &context->ah_list, list) { - struct ib_ah *ah = idr_find(&ib_uverbs_ah_idr, uobj->id); - idr_remove(&ib_uverbs_ah_idr, uobj->id); + struct ib_ah *ah = uobj->object; + + idr_remove_uobj(&ib_uverbs_ah_idr, uobj); ib_destroy_ah(ah); list_del(&uobj->list); kfree(uobj); } list_for_each_entry_safe(uobj, tmp, &context->qp_list, list) { - struct ib_qp *qp = idr_find(&ib_uverbs_qp_idr, uobj->id); + struct ib_qp *qp = uobj->object; struct ib_uqp_object *uqp = container_of(uobj, struct ib_uqp_object, uevent.uobject); - idr_remove(&ib_uverbs_qp_idr, uobj->id); + + idr_remove_uobj(&ib_uverbs_qp_idr, uobj); ib_uverbs_detach_umcast(qp, uqp); ib_destroy_qp(qp); list_del(&uobj->list); @@ -206,11 +206,12 @@ static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file, } list_for_each_entry_safe(uobj, tmp, &context->cq_list, list) { - struct ib_cq *cq = idr_find(&ib_uverbs_cq_idr, uobj->id); + struct ib_cq *cq = uobj->object; struct ib_uverbs_event_file *ev_file = cq->cq_context; struct ib_ucq_object *ucq = container_of(uobj, struct ib_ucq_object, uobject); - idr_remove(&ib_uverbs_cq_idr, uobj->id); + + idr_remove_uobj(&ib_uverbs_cq_idr, uobj); ib_destroy_cq(cq); list_del(&uobj->list); ib_uverbs_release_ucq(file, ev_file, ucq); @@ -218,10 +219,11 @@ static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file, } list_for_each_entry_safe(uobj, tmp, &context->srq_list, list) { - struct ib_srq *srq = idr_find(&ib_uverbs_srq_idr, uobj->id); + struct ib_srq *srq = uobj->object; struct ib_uevent_object *uevent = container_of(uobj, struct ib_uevent_object, uobject); - idr_remove(&ib_uverbs_srq_idr, uobj->id); + + idr_remove_uobj(&ib_uverbs_srq_idr, uobj); ib_destroy_srq(srq); list_del(&uobj->list); ib_uverbs_release_uevent(file, uevent); @@ -231,11 +233,11 @@ static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file, /* XXX Free MWs */ list_for_each_entry_safe(uobj, tmp, &context->mr_list, list) { - struct ib_mr *mr = idr_find(&ib_uverbs_mr_idr, uobj->id); + struct ib_mr *mr = uobj->object; struct ib_device *mrdev = mr->device; struct ib_umem_object *memobj; - idr_remove(&ib_uverbs_mr_idr, uobj->id); + idr_remove_uobj(&ib_uverbs_mr_idr, uobj); ib_dereg_mr(mr); memobj = container_of(uobj, struct ib_umem_object, uobject); @@ -246,15 +248,14 @@ static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file, } list_for_each_entry_safe(uobj, tmp, &context->pd_list, list) { - struct ib_pd *pd = idr_find(&ib_uverbs_pd_idr, uobj->id); - idr_remove(&ib_uverbs_pd_idr, uobj->id); + struct ib_pd *pd = uobj->object; + + idr_remove_uobj(&ib_uverbs_pd_idr, uobj); ib_dealloc_pd(pd); list_del(&uobj->list); kfree(uobj); } - mutex_unlock(&ib_uverbs_idr_mutex); - return context->device->dealloc_ucontext(context); } diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 7ced208edacf..ee1f3a355666 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -697,8 +697,12 @@ struct ib_ucontext { struct ib_uobject { u64 user_handle; /* handle given to us by userspace */ struct ib_ucontext *context; /* associated user context */ + void *object; /* containing object */ struct list_head list; /* link to context's list */ u32 id; /* index into kernel idr */ + struct kref ref; + struct rw_semaphore mutex; /* protects .live */ + int live; }; struct ib_umem { |