diff options
Diffstat (limited to 'block')
| -rw-r--r-- | block/Kconfig | 24 | ||||
| -rw-r--r-- | block/bfq-cgroup.c | 5 | ||||
| -rw-r--r-- | block/bfq-iosched.c | 800 | ||||
| -rw-r--r-- | block/bfq-iosched.h | 96 | ||||
| -rw-r--r-- | block/bfq-wf2q.c | 12 | ||||
| -rw-r--r-- | block/bio.c | 213 | ||||
| -rw-r--r-- | block/blk-merge.c | 135 | ||||
| -rw-r--r-- | block/blk.h | 2 | ||||
| -rw-r--r-- | block/elevator.c | 6 | ||||
| -rw-r--r-- | block/genhd.c | 48 | ||||
| -rw-r--r-- | block/opal_proto.h | 2 | ||||
| -rw-r--r-- | block/sed-opal.c | 716 |
12 files changed, 1219 insertions, 840 deletions
diff --git a/block/Kconfig b/block/Kconfig index 028bc085dac8..1b220101a9cb 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -26,30 +26,6 @@ menuconfig BLOCK if BLOCK -config LBDAF - bool "Support for large (2TB+) block devices and files" - depends on !64BIT - default y - help - Enable block devices or files of size 2TB and larger. - - This option is required to support the full capacity of large - (2TB+) block devices, including RAID, disk, Network Block Device, - Logical Volume Manager (LVM) and loopback. - - This option also enables support for single files larger than - 2TB. - - The ext4 filesystem requires that this feature be enabled in - order to support filesystems that have the huge_file feature - enabled. Otherwise, it will refuse to mount in the read-write - mode any filesystems that use the huge_file feature, which is - enabled by default by mke2fs.ext4. - - The GFS2 filesystem also requires this feature. - - If unsure, say Y. - config BLK_SCSI_REQUEST bool diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index c6113af31960..793c027ca60e 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -578,7 +578,8 @@ void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqg_and_blkg_get(bfqg); if (bfq_bfqq_busy(bfqq)) { - bfq_pos_tree_add_move(bfqd, bfqq); + if (unlikely(!bfqd->nonrot_with_queueing)) + bfq_pos_tree_add_move(bfqd, bfqq); bfq_activate_bfqq(bfqd, bfqq); } @@ -1102,7 +1103,7 @@ struct cftype bfq_blkcg_legacy_files[] = { }, #endif /* CONFIG_DEBUG_BLK_CGROUP */ - /* the same statictics which cover the bfqg and its descendants */ + /* the same statistics which cover the bfqg and its descendants */ { .name = "bfq.io_service_bytes_recursive", .private = (unsigned long)&blkcg_policy_bfq, diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 5ba1e0d841b4..b85a4ab8b9db 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -189,7 +189,7 @@ static const int bfq_default_max_budget = 16 * 1024; /* * When a sync request is dispatched, the queue that contains that * request, and all the ancestor entities of that queue, are charged - * with the number of sectors of the request. In constrast, if the + * with the number of sectors of the request. In contrast, if the * request is async, then the queue and its ancestor entities are * charged with the number of sectors of the request, multiplied by * the factor below. This throttles the bandwidth for async I/O, @@ -217,7 +217,7 @@ const int bfq_timeout = HZ / 8; * queue merging. * * As can be deduced from the low time limit below, queue merging, if - * successful, happens at the very beggining of the I/O of the involved + * successful, happens at the very beginning of the I/O of the involved * cooperating processes, as a consequence of the arrival of the very * first requests from each cooperator. After that, there is very * little chance to find cooperators. @@ -242,6 +242,14 @@ static struct kmem_cache *bfq_pool; blk_rq_sectors(rq) < BFQQ_SECT_THR_NONROT)) #define BFQQ_CLOSE_THR (sector_t)(8 * 1024) #define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 19) +/* + * Sync random I/O is likely to be confused with soft real-time I/O, + * because it is characterized by limited throughput and apparently + * isochronous arrival pattern. To avoid false positives, queues + * containing only random (seeky) I/O are prevented from being tagged + * as soft real-time. + */ +#define BFQQ_TOTALLY_SEEKY(bfqq) (bfqq->seek_history & -1) /* Min number of samples required to perform peak-rate update */ #define BFQ_RATE_MIN_SAMPLES 32 @@ -433,7 +441,7 @@ void bfq_schedule_dispatch(struct bfq_data *bfqd) /* * Lifted from AS - choose which of rq1 and rq2 that is best served now. - * We choose the request that is closesr to the head right now. Distance + * We choose the request that is closer to the head right now. Distance * behind the head is penalized and only allowed to a certain extent. */ static struct request *bfq_choose_req(struct bfq_data *bfqd, @@ -595,7 +603,16 @@ static bool bfq_too_late_for_merging(struct bfq_queue *bfqq) bfq_merge_time_limit); } -void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) +/* + * The following function is not marked as __cold because it is + * actually cold, but for the same performance goal described in the + * comments on the likely() at the beginning of + * bfq_setup_cooperator(). Unexpectedly, to reach an even lower + * execution time for the case where this function is not invoked, we + * had to add an unlikely() in each involved if(). + */ +void __cold +bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) { struct rb_node **p, *parent; struct bfq_queue *__bfqq; @@ -629,12 +646,19 @@ void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) } /* - * The following function returns true if every queue must receive the - * same share of the throughput (this condition is used when deciding - * whether idling may be disabled, see the comments in the function - * bfq_better_to_idle()). + * The following function returns false either if every active queue + * must receive the same share of the throughput (symmetric scenario), + * or, as a special case, if bfqq must receive a share of the + * throughput lower than or equal to the share that every other active + * queue must receive. If bfqq does sync I/O, then these are the only + * two cases where bfqq happens to be guaranteed its share of the + * throughput even if I/O dispatching is not plugged when bfqq remains + * temporarily empty (for more details, see the comments in the + * function bfq_better_to_idle()). For this reason, the return value + * of this function is used to check whether I/O-dispatch plugging can + * be avoided. * - * Such a scenario occurs when: + * The above first case (symmetric scenario) occurs when: * 1) all active queues have the same weight, * 2) all active queues belong to the same I/O-priority class, * 3) all active groups at the same level in the groups tree have the same @@ -654,30 +678,36 @@ void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) * support or the cgroups interface are not enabled, thus no state * needs to be maintained in this case. */ -static bool bfq_symmetric_scenario(struct bfq_data *bfqd) +static bool bfq_asymmetric_scenario(struct bfq_data *bfqd, + struct bfq_queue *bfqq) { + bool smallest_weight = bfqq && + bfqq->weight_counter && + bfqq->weight_counter == + container_of( + rb_first_cached(&bfqd->queue_weights_tree), + struct bfq_weight_counter, + weights_node); + /* * For queue weights to differ, queue_weights_tree must contain * at least two nodes. */ - bool varied_queue_weights = !RB_EMPTY_ROOT(&bfqd->queue_weights_tree) && - (bfqd->queue_weights_tree.rb_node->rb_left || - bfqd->queue_weights_tree.rb_node->rb_right); + bool varied_queue_weights = !smallest_weight && + !RB_EMPTY_ROOT(&bfqd->queue_weights_tree.rb_root) && + (bfqd->queue_weights_tree.rb_root.rb_node->rb_left || + bfqd->queue_weights_tree.rb_root.rb_node->rb_right); bool multiple_classes_busy = (bfqd->busy_queues[0] && bfqd->busy_queues[1]) || (bfqd->busy_queues[0] && bfqd->busy_queues[2]) || (bfqd->busy_queues[1] && bfqd->busy_queues[2]); - /* - * For queue weights to differ, queue_weights_tree must contain - * at least two nodes. - */ - return !(varied_queue_weights || multiple_classes_busy + return varied_queue_weights || multiple_classes_busy #ifdef CONFIG_BFQ_GROUP_IOSCHED || bfqd->num_groups_with_pending_reqs > 0 #endif - ); + ; } /* @@ -694,10 +724,11 @@ static bool bfq_symmetric_scenario(struct bfq_data *bfqd) * should be low too. */ void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, - struct rb_root *root) + struct rb_root_cached *root) { struct bfq_entity *entity = &bfqq->entity; - struct rb_node **new = &(root->rb_node), *parent = NULL; + struct rb_node **new = &(root->rb_root.rb_node), *parent = NULL; + bool leftmost = true; /* * Do not insert if the queue is already associated with a @@ -726,8 +757,10 @@ void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, } if (entity->weight < __counter->weight) new = &((*new)->rb_left); - else + else { new = &((*new)->rb_right); + leftmost = false; + } } bfqq->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), @@ -736,7 +769,7 @@ void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, /* * In the unlucky event of an allocation failure, we just * exit. This will cause the weight of queue to not be - * considered in bfq_symmetric_scenario, which, in its turn, + * considered in bfq_asymmetric_scenario, which, in its turn, * causes the scenario to be deemed wrongly symmetric in case * bfqq's weight would have been the only weight making the * scenario asymmetric. On the bright side, no unbalance will @@ -750,7 +783,8 @@ void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfqq->weight_counter->weight = entity->weight; rb_link_node(&bfqq->weight_counter->weights_node, parent, new); - rb_insert_color(&bfqq->weight_counter->weights_node, root); + rb_insert_color_cached(&bfqq->weight_counter->weights_node, root, + leftmost); inc_counter: bfqq->weight_counter->num_active++; @@ -765,7 +799,7 @@ inc_counter: */ void __bfq_weights_tree_remove(struct bfq_data *bfqd, struct bfq_queue *bfqq, - struct rb_root *root) + struct rb_root_cached *root) { if (!bfqq->weight_counter) return; @@ -774,7 +808,7 @@ void __bfq_weights_tree_remove(struct bfq_data *bfqd, if (bfqq->weight_counter->num_active > 0) goto reset_entity_pointer; - rb_erase(&bfqq->weight_counter->weights_node, root); + rb_erase_cached(&bfqq->weight_counter->weights_node, root); kfree(bfqq->weight_counter); reset_entity_pointer: @@ -889,7 +923,7 @@ static unsigned long bfq_serv_to_charge(struct request *rq, struct bfq_queue *bfqq) { if (bfq_bfqq_sync(bfqq) || bfqq->wr_coeff > 1 || - !bfq_symmetric_scenario(bfqq->bfqd)) + bfq_asymmetric_scenario(bfqq->bfqd, bfqq)) return blk_rq_sectors(rq); return blk_rq_sectors(rq) * bfq_async_charge_factor; @@ -955,7 +989,7 @@ static unsigned int bfq_wr_duration(struct bfq_data *bfqd) * of several files * mplayer took 23 seconds to start, if constantly weight-raised. * - * As for higher values than that accomodating the above bad + * As for higher values than that accommodating the above bad * scenario, tests show that higher values would often yield * the opposite of the desired result, i.e., would worsen * responsiveness by allowing non-interactive applications to @@ -994,6 +1028,7 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, else bfq_clear_bfqq_IO_bound(bfqq); + bfqq->entity.new_weight = bic->saved_weight; bfqq->ttime = bic->saved_ttime; bfqq->wr_coeff = bic->saved_wr_coeff; bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt; @@ -1041,8 +1076,18 @@ static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) hlist_for_each_entry_safe(item, n, &bfqd->burst_list, burst_list_node) hlist_del_init(&item->burst_list_node); - hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); - bfqd->burst_size = 1; + + /* + * Start the creation of a new burst list only if there is no + * active queue. See comments on the conditional invocation of + * bfq_handle_burst(). + */ + if (bfq_tot_busy_queues(bfqd) == 0) { + hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); + bfqd->burst_size = 1; + } else + bfqd->burst_size = 0; + bfqd->burst_parent_entity = bfqq->entity.parent; } @@ -1098,7 +1143,8 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) * many parallel threads/processes. Examples are systemd during boot, * or git grep. To help these processes get their job done as soon as * possible, it is usually better to not grant either weight-raising - * or device idling to their queues. + * or device idling to their queues, unless these queues must be + * protected from the I/O flowing through other active queues. * * In this comment we describe, firstly, the reasons why this fact * holds, and, secondly, the next function, which implements the main @@ -1110,7 +1156,10 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) * cumulatively served, the sooner the target job of these queues gets * completed. As a consequence, weight-raising any of these queues, * which also implies idling the device for it, is almost always - * counterproductive. In most cases it just lowers throughput. + * counterproductive, unless there are other active queues to isolate + * these new queues from. If there no other active queues, then + * weight-raising these new queues just lowers throughput in most + * cases. * * On the other hand, a burst of queue creations may be caused also by * the start of an application that does not consist of a lot of @@ -1144,14 +1193,16 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) * are very rare. They typically occur if some service happens to * start doing I/O exactly when the interactive task starts. * - * Turning back to the next function, it implements all the steps - * needed to detect the occurrence of a large burst and to properly - * mark all the queues belonging to it (so that they can then be - * treated in a different way). This goal is achieved by maintaining a - * "burst list" that holds, temporarily, the queues that belong to the - * burst in progress. The list is then used to mark these queues as - * belonging to a large burst if the burst does become large. The main - * steps are the following. + * Turning back to the next function, it is invoked only if there are + * no active queues (apart from active queues that would belong to the + * same, possible burst bfqq would belong to), and it implements all + * the steps needed to detect the occurrence of a large burst and to + * properly mark all the queues belonging to it (so that they can then + * be treated in a different way). This goal is achieved by + * maintaining a "burst list" that holds, temporarily, the queues that + * belong to the burst in progress. The list is then used to mark + * these queues as belonging to a large burst if the burst does become + * large. The main steps are the following. * * . when the very first queue is created, the queue is inserted into the * list (as it could be the first queue in a possible burst) @@ -1596,6 +1647,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, */ in_burst = bfq_bfqq_in_large_burst(bfqq); soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 && + !BFQQ_TOTALLY_SEEKY(bfqq) && !in_burst && time_is_before_jiffies(bfqq->soft_rt_next_start) && bfqq->dispatched == 0; @@ -1704,6 +1756,123 @@ static void bfq_add_request(struct request *rq) bfqq->queued[rq_is_sync(rq)]++; bfqd->queued++; + if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_sync(bfqq)) { + /* + * Periodically reset inject limit, to make sure that + * the latter eventually drops in case workload + * changes, see step (3) in the comments on + * bfq_update_inject_limit(). + */ + if (time_is_before_eq_jiffies(bfqq->decrease_time_jif + + msecs_to_jiffies(1000))) { + /* invalidate baseline total service time */ + bfqq->last_serv_time_ns = 0; + + /* + * Reset pointer in case we are waiting for + * some request completion. + */ + bfqd->waited_rq = NULL; + + /* + * If bfqq has a short think time, then start + * by setting the inject limit to 0 + * prudentially, because the service time of + * an injected I/O request may be higher than + * the think time of bfqq, and therefore, if + * one request was injected when bfqq remains + * empty, this injected request might delay + * the service of the next I/O request for + * bfqq significantly. In case bfqq can + * actually tolerate some injection, then the + * adaptive update will however raise the + * limit soon. This lucky circumstance holds + * exactly because bfqq has a short think + * time, and thus, after remaining empty, is + * likely to get new I/O enqueued---and then + * completed---before being expired. This is + * the very pattern that gives the + * limit-update algorithm the chance to + * measure the effect of injection on request + * service times, and then to update the limit + * accordingly. + * + * On the opposite end, if bfqq has a long + * think time, then start directly by 1, + * because: + * a) on the bright side, keeping at most one + * request in service in the drive is unlikely + * to cause any harm to the latency of bfqq's + * requests, as the service time of a single + * request is likely to be lower than the + * think time of bfqq; + * b) on the downside, after becoming empty, + * bfqq is likely to expire before getting its + * next request. With this request arrival + * pattern, it is very hard to sample total + * service times and update the inject limit + * accordingly (see comments on + * bfq_update_inject_limit()). So the limit is + * likely to be never, or at least seldom, + * updated. As a consequence, by setting the + * limit to 1, we avoid that no injection ever + * occurs with bfqq. On the downside, this + * proactive step further reduces chances to + * actually compute the baseline total service + * time. Thus it reduces chances to execute the + * limit-update algorithm and possibly raise the + * limit to more than 1. + */ + if (bfq_bfqq_has_short_ttime(bfqq)) + bfqq->inject_limit = 0; + else + bfqq->inject_limit = 1; + bfqq->decrease_time_jif = jiffies; + } + + /* + * The following conditions must hold to setup a new + * sampling of total service time, and then a new + * update of the inject limit: + * - bfqq is in service, because the total service + * time is evaluated only for the I/O requests of + * the queues in service; + * - this is the right occasion to compute or to + * lower the baseline total service time, because + * there are actually no requests in the drive, + * or + * the baseline total service time is available, and + * this is the right occasion to compute the other + * quantity needed to update the inject limit, i.e., + * the total service time caused by the amount of + * injection allowed by the current value of the + * limit. It is the right occasion because injection + * has actually been performed during the service + * hole, and there are still in-flight requests, + * which are very likely to be exactly the injected + * requests, or part of them; + * - the minimum interval for sampling the total + * service time and updating the inject limit has + * elapsed. + */ + if (bfqq == bfqd->in_service_queue && + (bfqd->rq_in_driver == 0 || + (bfqq->last_serv_time_ns > 0 && + bfqd->rqs_injected && bfqd->rq_in_driver > 0)) && + time_is_before_eq_jiffies(bfqq->decrease_time_jif + + msecs_to_jiffies(100))) { + bfqd->last_empty_occupied_ns = ktime_get_ns(); + /* + * Start the state machine for measuring the + * total service time of rq: setting + * wait_dispatch will cause bfqd->waited_rq to + * be set when rq will be dispatched. + */ + bfqd->wait_dispatch = true; + bfqd->rqs_injected = false; + } + } + elv_rb_add(&bfqq->sort_list, rq); /* @@ -1715,8 +1884,9 @@ static void bfq_add_request(struct request *rq) /* * Adjust priority tree position, if next_rq changes. + * See comments on bfq_pos_tree_add_move() for the unlikely(). */ - if (prev != bfqq->next_rq) + if (unlikely(!bfqd->nonrot_with_queueing && prev != bfqq->next_rq)) bfq_pos_tree_add_move(bfqd, bfqq); if (!bfq_bfqq_busy(bfqq)) /* switching to busy ... */ @@ -1856,7 +2026,9 @@ static void bfq_remove_request(struct request_queue *q, bfqq->pos_root = NULL; } } else { - bfq_pos_tree_add_move(bfqd, bfqq); + /* see comments on bfq_pos_tree_add_move() for the unlikely() */ + if (unlikely(!bfqd->nonrot_with_queueing)) + bfq_pos_tree_add_move(bfqd, bfqq); } if (rq->cmd_flags & REQ_META) @@ -1941,7 +2113,12 @@ static void bfq_request_merged(struct request_queue *q, struct request *req, */ if (prev != bfqq->next_rq) { bfq_updated_next_req(bfqd, bfqq); - bfq_pos_tree_add_move(bfqd, bfqq); + /* + * See comments on bfq_pos_tree_add_move() for + * the unlikely(). + */ + if (unlikely(!bfqd->nonrot_with_queueing)) + bfq_pos_tree_add_move(bfqd, bfqq); } } } @@ -2224,6 +2401,46 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, struct bfq_queue *in_service_bfqq, *new_bfqq; /* + * Do not perform queue merging if the device is non + * rotational and performs internal queueing. In fact, such a + * device reaches a high speed through internal parallelism + * and pipelining. This means that, to reach a high + * throughput, it must have many requests enqueued at the same + * time. But, in this configuration, the internal scheduling + * algorithm of the device does exactly the job of queue + * merging: it reorders requests so as to obtain as much as + * possible a sequential I/O pattern. As a consequence, with + * the workload generated by processes doing interleaved I/O, + * the throughput reached by the device is likely to be the + * same, with and without queue merging. + * + * Disabling merging also provides a remarkable benefit in + * terms of throughput. Merging tends to make many workloads + * artificially more uneven, because of shared queues + * remaining non empty for incomparably more time than + * non-merged queues. This may accentuate workload + * asymmetries. For example, if one of the queues in a set of + * merged queues has a higher weight than a normal queue, then + * the shared queue may inherit such a high weight and, by + * staying almost always active, may force BFQ to perform I/O + * plugging most of the time. This evidently makes it harder + * for BFQ to let the device reach a high throughput. + * + * Finally, the likely() macro below is not used because one + * of the two branches is more likely than the other, but to + * have the code path after the following if() executed as + * fast as possible for the case of a non rotational device + * with queueing. We want it because this is the fastest kind + * of device. On the opposite end, the likely() may lengthen + * the execution time of BFQ for the case of slower devices + * (rotational or at least without queueing). But in this case + * the execution time of BFQ matters very little, if not at + * all. + */ + if (likely(bfqd->nonrot_with_queueing)) + return NULL; + + /* * Prevent bfqq from being merged if it has been created too * long ago. The idea is that true cooperating processes, and * thus their associated bfq_queues, are supposed to be @@ -2286,6 +2503,7 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq) if (!bic) return; + bic->saved_weight = bfqq->entity.orig_weight; bic->saved_ttime = bfqq->ttime; bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq); bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); @@ -2374,6 +2592,16 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, * assignment causes no harm). */ new_bfqq->bic = NULL; + /* + * If the queue is shared, the pid is the pid of one of the associated + * processes. Which pid depends on the exact sequence of merge events + * the queue underwent. So printing such a pid is useless and confusing + * because it reports a random pid between those of the associated + * processes. + * We mark such a queue with a pid -1, and then print SHARED instead of + * a pid in logging messages. + */ + new_bfqq->pid = -1; bfqq->bic = NULL; /* release process reference to bfqq */ bfq_put_queue(bfqq); @@ -2408,8 +2636,8 @@ static bool bfq_allow_bio_merge(struct request_queue *q, struct request *rq, /* * bic still points to bfqq, then it has not yet been * redirected to some other bfq_queue, and a queue - * merge beween bfqq and new_bfqq can be safely - * fulfillled, i.e., bic can be redirected to new_bfqq + * merge between bfqq and new_bfqq can be safely + * fulfilled, i.e., bic can be redirected to new_bfqq * and bfqq can be put. */ bfq_merge_bfqqs(bfqd, bfqd->bio_bic, bfqq, @@ -2543,10 +2771,14 @@ static void bfq_arm_slice_timer(struct bfq_data *bfqd) * queue). */ if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && - bfq_symmetric_scenario(bfqd)) + !bfq_asymmetric_scenario(bfqd, bfqq)) sl = min_t(u64, sl, BFQ_MIN_TT); + else if (bfqq->wr_coeff > 1) + sl = max_t(u32, sl, 20ULL * NSEC_PER_MSEC); bfqd->last_idling_start = ktime_get(); + bfqd->last_idling_start_jiffies = jiffies; + hrtimer_start(&bfqd->idle_slice_timer, ns_to_ktime(sl), HRTIMER_MODE_REL); bfqg_stats_set_start_idle_time(bfqq_group(bfqq)); @@ -2848,8 +3080,10 @@ static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq) bfq_requeue_bfqq(bfqd, bfqq, true); /* * Resort priority tree of potential close cooperators. + * See comments on bfq_pos_tree_add_move() for the unlikely(). */ - bfq_pos_tree_add_move(bfqd, bfqq); + if (unlikely(!bfqd->nonrot_with_queueing)) + bfq_pos_tree_add_move(bfqd, bfqq); } /* @@ -3223,13 +3457,6 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); } -static bool bfq_bfqq_injectable(struct bfq_queue *bfqq) -{ - return BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && - blk_queue_nonrot(bfqq->bfqd->queue) && - bfqq->bfqd->hw_tag; -} - /** * bfq_bfqq_expire - expire a queue. * @bfqd: device owning the queue. @@ -3344,6 +3571,14 @@ void bfq_bfqq_expire(struct bfq_data *bfqd, slow, bfqq->dispatched, bfq_bfqq_has_short_ttime(bfqq)); /* + * bfqq expired, so no total service time needs to be computed + * any longer: reset state machine for measuring total service + * times. + */ + bfqd->rqs_injected = bfqd->wait_dispatch = false; + bfqd->waited_rq = NULL; + + /* * Increase, decrease or leave budget unchanged according to * reason. */ @@ -3352,8 +3587,6 @@ void bfq_bfqq_expire(struct bfq_data *bfqd, /* bfqq is gone, no more actions on it */ return; - bfqq->injected_service = 0; - /* mark bfqq as waiting a request only if a bic still points to it */ if (!bfq_bfqq_busy(bfqq) && reason != BFQQE_BUDGET_TIMEOUT && @@ -3497,8 +3730,9 @@ static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, } /* - * There is a case where idling must be performed not for - * throughput concerns, but to preserve service guarantees. + * There is a case where idling does not have to be performed for + * throughput concerns, but to preserve the throughput share of + * the process associated with bfqq. * * To introduce this case, we can note that allowing the drive * to enqueue more than one request at a time, and hence @@ -3514,77 +3748,83 @@ static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, * concern about per-process throughput distribution, and * makes its decisions only on a per-request basis. Therefore, * the service distribution enforced by the drive's internal - * scheduler is likely to coincide with the desired - * device-throughput distribution only in a completely - * symmetric scenario where: - * (i) each of these processes must get the same throughput as - * the others; - * (ii) the I/O of each process has the same properties, in - * terms of locality (sequential or random), direction - * (reads or writes), request sizes, greediness - * (from I/O-bound to sporadic), and so on. - * In fact, in such a scenario, the drive tends to treat - * the requests of each of these processes in about the same - * way as the requests of the others, and thus to provide - * each of these processes with about the same throughput - * (which is exactly the desired throughput distribution). In - * contrast, in any asymmetric scenario, device idling is - * certainly needed to guarantee that bfqq receives its - * assigned fraction of the device throughput (see [1] for - * details). - * The problem is that idling may significantly reduce - * throughput with certain combinations of types of I/O and - * devices. An important example is sync random I/O, on flash - * storage with command queueing. So, unless bfqq falls in the - * above cases where idling also boosts throughput, it would - * be important to check conditions (i) and (ii) accurately, - * so as to avoid idling when not strictly needed for service - * guarantees. + * scheduler is likely to coincide with the desired throughput + * distribution only in a completely symmetric, or favorably + * skewed scenario where: + * (i-a) each of these processes must get the same throughput as + * the others, + * (i-b) in case (i-a) does not hold, it holds that the process + * associated with bfqq must receive a lower or equal + * throughput than any of the other processes; + * (ii) the I/O of each process has the same properties, in + * terms of locality (sequential or random), direction + * (reads or writes), request sizes, greediness + * (from I/O-bound to sporadic), and so on; + + * In fact, in such a scenario, the drive tends to treat the requests + * of each process in about the same way as the requests of the + * others, and thus to provide each of these processes with about the + * same throughput. This is exactly the desired throughput + * distribution if (i-a) holds, or, if (i-b) holds instead, this is an + * even more convenient distribution for (the process associated with) + * bfqq. * - * Unfortunately, it is extremely difficult to thoroughly - * check condition (ii). And, in case there are active groups, - * it becomes very difficult to check condition (i) too. In - * fact, if there are active groups, then, for condition (i) - * to become false, it is enough that an active group contains - * more active processes or sub-groups than some other active - * group. More precisely, for condition (i) to hold because of - * such a group, it is not even necessary that the group is - * (still) active: it is sufficient that, even if the group - * has become inactive, some of its descendant processes still - * have some request already dispatched but still waiting for - * completion. In fact, requests have still to be guaranteed - * their share of the throughput even after being - * dispatched. In this respect, it is easy to show that, if a - * group frequently becomes inactive while still having - * in-flight requests, and if, when this happens, the group is - * not considered in the calculation of whether the scenario - * is asymmetric, then the group may fail to be guaranteed its - * fair share of the throughput (basically because idling may - * not be performed for the descendant processes of the group, - * but it had to be). We address this issue with the - * following bi-modal behavior, implemented in the function - * bfq_symmetric_scenario(). + * In contrast, in any asymmetric or unfavorable scenario, device + * idling (I/O-dispatch plugging) is certainly needed to guarantee + * that bfqq receives its assigned fraction of the device throughput + * (see [1] for details). + * + * The problem is that idling may significantly reduce throughput with + * certain combinations of types of I/O and devices. An important + * example is sync random I/O on flash storage with command + * queueing. So, unless bfqq falls in cases where idling also boosts + * throughput, it is important to check conditions (i-a), i(-b) and + * (ii) accurately, so as to avoid idling when not strictly needed for + * service guarantees. + * + * Unfortunately, it is extremely difficult to thoroughly check + * condition (ii). And, in case there are active groups, it becomes + * very difficult to check conditions (i-a) and (i-b) too. In fact, + * if there are active groups, then, for conditions (i-a) or (i-b) to + * become false 'indirectly', it is enough that an active group + * contains more active processes or sub-groups than some other active + * group. More precisely, for conditions (i-a) or (i-b) to become + * false because of such a group, it is not even necessary that the + * group is (still) active: it is sufficient that, even if the group + * has become inactive, some of its descendant processes still have + * some request already dispatched but still waiting for + * completion. In fact, requests have still to be guaranteed their + * share of the throughput even after being dispatched. In this + * respect, it is easy to show that, if a group frequently becomes + * inactive while still having in-flight requests, and if, when this + * happens, the group is not considered in the calculation of whether + * the scenario is asymmetric, then the group may fail to be + * guaranteed its fair share of the throughput (basically because + * idling may not be performed for the descendant processes of the + * group, but it had to be). We address this issue with the following + * bi-modal behavior, implemented in the function + * bfq_asymmetric_scenario(). * * If there are groups with requests waiting for completion * (as commented above, some of these groups may even be * already inactive), then the scenario is tagged as * asymmetric, conservatively, without checking any of the - * conditions (i) and (ii). So the device is idled for bfqq. + * conditions (i-a), (i-b) or (ii). So the device is idled for bfqq. * This behavior matches also the fact that groups are created * exactly if controlling I/O is a primary concern (to * preserve bandwidth and latency guarantees). * - * On the opposite end, if there are no groups with requests - * waiting for completion, then only condition (i) is actually - * controlled, i.e., provided that condition (i) holds, idling - * is not performed, regardless of whether condition (ii) - * holds. In other words, only if condition (i) does not hold, - * then idling is allowed, and the device tends to be - * prevented from queueing many requests, possibly of several - * processes. Since there are no groups with requests waiting - * for completion, then, to control condition (i) it is enough - * to check just whether all the queues with requests waiting - * for completion also have the same weight. + * On the opposite end, if there are no groups with requests waiting + * for completion, then only conditions (i-a) and (i-b) are actually + * controlled, i.e., provided that conditions (i-a) or (i-b) holds, + * idling is not performed, regardless of whether condition (ii) + * holds. In other words, only if conditions (i-a) and (i-b) do not + * hold, then idling is allowed, and the device tends to be prevented + * from queueing many requests, possibly of several processes. Since + * there are no groups with requests waiting for completion, then, to + * control conditions (i-a) and (i-b) it is enough to check just + * whether all the queues with requests waiting for completion also + * have the same weight. * * Not checking condition (ii) evidently exposes bfqq to the * risk of getting less throughput than its fair share. @@ -3636,7 +3876,7 @@ static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, * compound condition that is checked below for deciding * whether the scenario is asymmetric. To explain this * compound condition, we need to add that the function - * bfq_symmetric_scenario checks the weights of only + * bfq_asymmetric_scenario checks the weights of only * non-weight-raised queues, for efficiency reasons (see * comments on bfq_weights_tree_add()). Then the fact that * bfqq is weight-raised is checked explicitly here. More @@ -3664,7 +3904,7 @@ static bool idling_needed_for_service_guarantees(struct bfq_data *bfqd, return (bfqq->wr_coeff > 1 && bfqd->wr_busy_queues < bfq_tot_busy_queues(bfqd)) || - !bfq_symmetric_scenario(bfqd); + bfq_asymmetric_scenario(bfqd, bfqq); } /* @@ -3740,26 +3980,98 @@ static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_better_to_idle(bfqq); } -static struct bfq_queue *bfq_choose_bfqq_for_injection(struct bfq_data *bfqd) +/* + * This function chooses the queue from which to pick the next extra + * I/O request to inject, if it finds a compatible queue. See the + * comments on bfq_update_inject_limit() for details on the injection + * mechanism, and for the definitions of the quantities mentioned + * below. + */ +static struct bfq_queue * +bfq_choose_bfqq_for_injection(struct bfq_data *bfqd) { - struct bfq_queue *bfqq; + struct bfq_queue *bfqq, *in_serv_bfqq = bfqd->in_service_queue; + unsigned int limit = in_serv_bfqq->inject_limit; + /* + * If + * - bfqq is not weight-raised and therefore does not carry + * time-critical I/O, + * or + * - regardless of whether bfqq is weight-raised, bfqq has + * however a long think time, during which it can absorb the + * effect of an appropriate number of extra I/O requests + * from other queues (see bfq_update_inject_limit for + * details on the computation of this number); + * then injection can be performed without restrictions. + */ + bool in_serv_always_inject = in_serv_bfqq->wr_coeff == 1 || + !bfq_bfqq_has_short_ttime(in_serv_bfqq); /* - * A linear search; but, with a high probability, very few - * steps are needed to find a candidate queue, i.e., a queue - * with enough budget left for its next request. In fact: + * If + * - the baseline total service time could not be sampled yet, + * so the inject limit happens to be still 0, and + * - a lot of time has elapsed since the plugging of I/O + * dispatching started, so drive speed is being wasted + * significantly; + * then temporarily raise inject limit to one request. + */ + if (limit == 0 && in_serv_bfqq->last_serv_time_ns == 0 && + bfq_bfqq_wait_request(in_serv_bfqq) && + time_is_before_eq_jiffies(bfqd->last_idling_start_jiffies + + bfqd->bfq_slice_idle) + ) + limit = 1; + + if (bfqd->rq_in_driver >= limit) + return NULL; + + /* + * Linear search of the source queue for injection; but, with + * a high probability, very few steps are needed to find a + * candidate queue, i.e., a queue with enough budget left for + * its next request. In fact: * - BFQ dynamically updates the budget of every queue so as * to accommodate the expected backlog of the queue; * - if a queue gets all its requests dispatched as injected * service, then the queue is removed from the active list - * (and re-added only if it gets new requests, but with - * enough budget for its new backlog). + * (and re-added only if it gets new requests, but then it + * is assigned again enough budget for its new backlog). */ list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) if (!RB_EMPTY_ROOT(&bfqq->sort_list) && + (in_serv_always_inject || bfqq->wr_coeff > 1) && bfq_serv_to_charge(bfqq->next_rq, bfqq) <= - bfq_bfqq_budget_left(bfqq)) - return bfqq; + bfq_bfqq_budget_left(bfqq)) { + /* + * Allow for only one large in-flight request + * on non-rotational devices, for the + * following reason. On non-rotationl drives, + * large requests take much longer than + * smaller requests to be served. In addition, + * the drive prefers to serve large requests + * w.r.t. to small ones, if it can choose. So, + * having more than one large requests queued + * in the drive may easily make the next first + * request of the in-service queue wait for so + * long to break bfqq's service guarantees. On + * the bright side, large requests let the + * drive reach a very high throughput, even if + * there is only one in-flight large request + * at a time. + */ + if (blk_queue_nonrot(bfqd->queue) && + blk_rq_sectors(bfqq->next_rq) >= + BFQQ_SECT_THR_NONROT) + limit = min_t(unsigned int, 1, limit); + else + limit = in_serv_bfqq->inject_limit; + + if (bfqd->rq_in_driver < limit) { + bfqd->rqs_injected = true; + return bfqq; + } + } return NULL; } @@ -3846,14 +4158,32 @@ check_queue: * for a new request, or has requests waiting for a completion and * may idle after their completion, then keep it anyway. * - * Yet, to boost throughput, inject service from other queues if - * possible. + * Yet, inject service from other queues if it boosts + * throughput and is possible. */ if (bfq_bfqq_wait_request(bfqq) || (bfqq->dispatched != 0 && bfq_better_to_idle(bfqq))) { - if (bfq_bfqq_injectable(bfqq) && - bfqq->injected_service * bfqq->inject_coeff < - bfqq->entity.service * 10) + struct bfq_queue *async_bfqq = + bfqq->bic && bfqq->bic->bfqq[0] && + bfq_bfqq_busy(bfqq->bic->bfqq[0]) ? + bfqq->bic->bfqq[0] : NULL; + + /* + * If the process associated with bfqq has also async + * I/O pending, then inject it + * unconditionally. Injecting I/O from the same + * process can cause no harm to the process. On the + * contrary, it can only increase bandwidth and reduce + * latency for the process. + */ + if (async_bfqq && + icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic && + bfq_serv_to_charge(async_bfqq->next_rq, async_bfqq) <= + bfq_bfqq_budget_left(async_bfqq)) + bfqq = bfqq->bic->bfqq[0]; + else if (!idling_boosts_thr_without_issues(bfqd, bfqq) && + (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 || + !bfq_bfqq_has_short_ttime(bfqq))) bfqq = bfq_choose_bfqq_for_injection(bfqd); else bfqq = NULL; @@ -3945,15 +4275,15 @@ static struct request *bfq_dispatch_rq_from_bfqq(struct bfq_data *bfqd, bfq_bfqq_served(bfqq, service_to_charge); - bfq_dispatch_remove(bfqd->queue, rq); + if (bfqq == bfqd->in_service_queue && bfqd->wait_dispatch) { + bfqd->wait_dispatch = false; + bfqd->waited_rq = rq; + } - if (bfqq != bfqd->in_service_queue) { - if (likely(bfqd->in_service_queue)) - bfqd->in_service_queue->injected_service += - bfq_serv_to_charge(rq, bfqq); + bfq_dispatch_remove(bfqd->queue, rq); + if (bfqq != bfqd->in_service_queue) goto return_rq; - } /* * If weight raising has to terminate for bfqq, then next @@ -4384,13 +4714,6 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bfq_mark_bfqq_has_short_ttime(bfqq); bfq_mark_bfqq_sync(bfqq); bfq_mark_bfqq_just_created(bfqq); - /* - * Aggressively inject a lot of service: up to 90%. - * This coefficient remains constant during bfqq life, - * but this behavior might be changed, after enough - * testing and tuning. - */ - bfqq->inject_coeff = 1; } else bfq_clear_bfqq_sync(bfqq); @@ -4529,6 +4852,11 @@ bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, { bfqq->seek_history <<= 1; bfqq->seek_history |= BFQ_RQ_SEEKY(bfqd, bfqq->last_request_pos, rq); + + if (bfqq->wr_coeff > 1 && + bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && + BFQQ_TOTALLY_SEEKY(bfqq)) + bfq_bfqq_end_wr(bfqq); } static void bfq_update_has_short_ttime(struct bfq_data *bfqd, @@ -4823,6 +5151,9 @@ static void bfq_update_hw_tag(struct bfq_data *bfqd) bfqd->hw_tag = bfqd->max_rq_in_driver > BFQ_HW_QUEUE_THRESHOLD; bfqd->max_rq_in_driver = 0; bfqd->hw_tag_samples = 0; + + bfqd->nonrot_with_queueing = + blk_queue_nonrot(bfqd->queue) && bfqd->hw_tag; } static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) @@ -4950,6 +5281,147 @@ static void bfq_finish_requeue_request_body(struct bfq_queue *bfqq) } /* + * The processes associated with bfqq may happen to generate their + * cumulative I/O at a lower rate than the rate at which the device + * could serve the same I/O. This is rather probable, e.g., if only + * one process is associated with bfqq and the device is an SSD. It + * results in bfqq becoming often empty while in service. In this + * respect, if BFQ is allowed to switch to another queue when bfqq + * remains empty, then the device goes on being fed with I/O requests, + * and the throughput is not affected. In contrast, if BFQ is not + * allowed to switch to another queue---because bfqq is sync and + * I/O-dispatch needs to be plugged while bfqq is temporarily + * empty---then, during the service of bfqq, there will be frequent + * "service holes", i.e., time intervals during which bfqq gets empty + * and the device can only consume the I/O already queued in its + * hardware queues. During service holes, the device may even get to + * remaining idle. In the end, during the service of bfqq, the device + * is driven at a lower speed than the one it can reach with the kind + * of I/O flowing through bfqq. + * + * To counter this loss of throughput, BFQ implements a "request + * injection mechanism", which tries to fill the above service holes + * with I/O requests taken from other queues. The hard part in this + * mechanism is finding the right amount of I/O to inject, so as to + * both boost throughput and not break bfqq's bandwidth and latency + * guarantees. In this respect, the mechanism maintains a per-queue + * inject limit, computed as below. While bfqq is empty, the injection + * mechanism dispatches extra I/O requests only until the total number + * of I/O requests in flight---i.e., already dispatched but not yet + * completed---remains lower than this limit. + * + * A first definition comes in handy to introduce the algorithm by + * which the inject limit is computed. We define as first request for + * bfqq, an I/O request for bfqq that arrives while bfqq is in + * service, and causes bfqq to switch from empty to non-empty. The + * algorithm updates the limit as a function of the effect of + * injection on the service times of only the first requests of + * bfqq. The reason for this restriction is that these are the + * requests whose service time is affected most, because they are the + * first to arrive after injection possibly occurred. + * + * To evaluate the effect of injection, the algorithm measures the + * "total service time" of first requests. We define as total service + * time of an I/O request, the time that elapses since when the + * request is enqueued into bfqq, to when it is completed. This + * quantity allows the whole effect of injection to be measured. It is + * easy to see why. Suppose that some requests of other queues are + * actually injected while bfqq is empty, and that a new request R + * then arrives for bfqq. If the device does start to serve all or + * part of the injected requests during the service hole, then, + * because of this extra service, it may delay the next invocation of + * the dispatch hook of BFQ. Then, even after R gets eventually + * dispatched, the device may delay the actual service of R if it is + * still busy serving the extra requests, or if it decides to serve, + * before R, some extra request still present in its queues. As a + * conclusion, the cumulative extra delay caused by injection can be + * easily evaluated by just comparing the total service time of first + * requests with and without injection. + * + * The limit-update algorithm works as follows. On the arrival of a + * first request of bfqq, the algorithm measures the total time of the + * request only if one of the three cases below holds, and, for each + * case, it updates the limit as described below: + * + * (1) If there is no in-flight request. This gives a baseline for the + * total service time of the requests of bfqq. If the baseline has + * not been computed yet, then, after computing it, the limit is + * set to 1, to start boosting throughput, and to prepare the + * ground for the next case. If the baseline has already been + * computed, then it is updated, in case it results to be lower + * than the previous value. + * + * (2) If the limit is higher than 0 and there are in-flight + * requests. By comparing the total service time in this case with + * the above baseline, it is possible to know at which extent the + * current value of the limit is inflating the total service + * time. If the inflation is below a certain threshold, then bfqq + * is assumed to be suffering from no perceivable loss of its + * service guarantees, and the limit is even tentatively + * increased. If the inflation is above the threshold, then the + * limit is decreased. Due to the lack of any hysteresis, this + * logic makes the limit oscillate even in steady workload + * conditions. Yet we opted for it, because it is fast in reaching + * the best value for the limit, as a function of the current I/O + * workload. To reduce oscillations, this step is disabled for a + * short time interval after the limit happens to be decreased. + * + * (3) Periodically, after resetting the limit, to make sure that the + * limit eventually drops in case the workload changes. This is + * needed because, after the limit has gone safely up for a + * certain workload, it is impossible to guess whether the + * baseline total service time may have changed, without measuring + * it again without injection. A more effective version of this + * step might be to just sample the baseline, by interrupting + * injection only once, and then to reset/lower the limit only if + * the total service time with the current limit does happen to be + * too large. + * + * More details on each step are provided in the comments on the + * pieces of code that implement these steps: the branch handling the + * transition from empty to non empty in bfq_add_request(), the branch + * handling injection in bfq_select_queue(), and the function + * bfq_choose_bfqq_for_injection(). These comments also explain some + * exceptions, made by the injection mechanism in some special cases. + */ +static void bfq_update_inject_limit(struct bfq_data *bfqd, + struct bfq_queue *bfqq) +{ + u64 tot_time_ns = ktime_get_ns() - bfqd->last_empty_occupied_ns; + unsigned int old_limit = bfqq->inject_limit; + + if (bfqq->last_serv_time_ns > 0) { + u64 threshold = (bfqq->last_serv_time_ns * 3)>>1; + + if (tot_time_ns >= threshold && old_limit > 0) { + bfqq->inject_limit--; + bfqq->decrease_time_jif = jiffies; + } else if (tot_time_ns < threshold && + old_limit < bfqd->max_rq_in_driver<<1) + bfqq->inject_limit++; + } + + /* + * Either we still have to compute the base value for the + * total service time, and there seem to be the right + * conditions to do it, or we can lower the last base value + * computed. + */ + if ((bfqq->last_serv_time_ns == 0 && bfqd->rq_in_driver == 0) || + tot_time_ns < bfqq->last_serv_time_ns) { + bfqq->last_serv_time_ns = tot_time_ns; + /* + * Now we certainly have a base value: make sure we + * start trying injection. + */ + bfqq->inject_limit = max_t(unsigned int, 1, old_limit); + } + + /* update complete, not waiting for any request completion any longer */ + bfqd->waited_rq = NULL; +} + +/* * Handle either a requeue or a finish for rq. The things to do are * the same in both cases: all references to rq are to be dropped. In * particular, rq is considered completed from the point of view of @@ -4993,6 +5465,9 @@ static void bfq_finish_requeue_request(struct request *rq) spin_lock_irqsave(&bfqd->lock, flags); + if (rq == bfqd->waited_rq) + bfq_update_inject_limit(bfqd, bfqq); + bfq_completed_request(bfqq, bfqd); bfq_finish_requeue_request_body(bfqq); @@ -5156,7 +5631,7 @@ static void bfq_prepare_request(struct request *rq, struct bio *bio) * preparation is that, after the prepare_request hook is invoked for * rq, rq may still be transformed into a request with no icq, i.e., a * request not associated with any queue. No bfq hook is invoked to - * signal this tranformation. As a consequence, should these + * signal this transformation. As a consequence, should these * preparation operations be performed when the prepare_request hook * is invoked, and should rq be transformed one moment later, bfq * would end up in an inconsistent state, because it would have @@ -5247,7 +5722,29 @@ static struct bfq_queue *bfq_init_rq(struct request *rq) } } - if (unlikely(bfq_bfqq_just_created(bfqq))) + /* + * Consider bfqq as possibly belonging to a burst of newly + * created queues only if: + * 1) A burst is actually happening (bfqd->burst_size > 0) + * or + * 2) There is no other active queue. In fact, if, in + * contrast, there are active queues not belonging to the + * possible burst bfqq may belong to, then there is no gain + * in considering bfqq as belonging to a burst, and + * therefore in not weight-raising bfqq. See comments on + * bfq_handle_burst(). + * + * This filtering also helps eliminating false positives, + * occurring when bfqq does not belong to an actual large + * burst, but some background task (e.g., a service) happens + * to trigger the creation of new queues very close to when + * bfqq and its possible companion queues are created. See + * comments on bfq_handle_burst() for further details also on + * this issue. + */ + if (unlikely(bfq_bfqq_just_created(bfqq) && + (bfqd->burst_size > 0 || + bfq_tot_busy_queues(bfqd) == 0))) bfq_handle_burst(bfqd, bfqq); return bfqq; @@ -5507,7 +6004,7 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) HRTIMER_MODE_REL); bfqd->idle_slice_timer.function = bfq_idle_slice_timer; - bfqd->queue_weights_tree = RB_ROOT; + bfqd->queue_weights_tree = RB_ROOT_CACHED; bfqd->num_groups_with_pending_reqs = 0; INIT_LIST_HEAD(&bfqd->active_list); @@ -5515,6 +6012,7 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) INIT_HLIST_HEAD(&bfqd->burst_list); bfqd->hw_tag = -1; + bfqd->nonrot_with_queueing = blk_queue_nonrot(bfqd->queue); bfqd->bfq_max_budget = bfq_default_max_budget; diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h index 86394e503ca9..eba7cd449ab4 100644 --- a/block/bfq-iosched.h +++ b/block/bfq-iosched.h @@ -32,6 +32,8 @@ #define BFQ_DEFAULT_GRP_IOPRIO 0 #define BFQ_DEFAULT_GRP_CLASS IOPRIO_CLASS_BE +#define MAX_PID_STR_LENGTH 12 + /* * Soft real-time applications are extremely more latency sensitive * than interactive ones. Over-raise the weight of the former to @@ -89,7 +91,7 @@ struct bfq_service_tree { * expiration. This peculiar definition allows for the following * optimization, not yet exploited: while a given entity is still in * service, we already know which is the best candidate for next - * service among the other active entitities in the same parent + * service among the other active entities in the same parent * entity. We can then quickly compare the timestamps of the * in-service entity with those of such best candidate. * @@ -140,7 +142,7 @@ struct bfq_weight_counter { * * Unless cgroups are used, the weight value is calculated from the * ioprio to export the same interface as CFQ. When dealing with - * ``well-behaved'' queues (i.e., queues that do not spend too much + * "well-behaved" queues (i.e., queues that do not spend too much * time to consume their budget and have true sequential behavior, and * when there are no external factors breaking anticipation) the * relative weights at each level of the cgroups hierarchy should be @@ -240,6 +242,13 @@ struct bfq_queue { /* next ioprio and ioprio class if a change is in progress */ unsigned short new_ioprio, new_ioprio_class; + /* last total-service-time sample, see bfq_update_inject_limit() */ + u64 last_serv_time_ns; + /* limit for request injection */ + unsigned int inject_limit; + /* last time the inject limit has been decreased, in jiffies */ + unsigned long decrease_time_jif; + /* * Shared bfq_queue if queue is cooperating with one or more * other queues. @@ -357,29 +366,6 @@ struct bfq_queue { /* max service rate measured so far */ u32 max_service_rate; - /* - * Ratio between the service received by bfqq while it is in - * service, and the cumulative service (of requests of other - * queues) that may be injected while bfqq is empty but still - * in service. To increase precision, the coefficient is - * measured in tenths of unit. Here are some example of (1) - * ratios, (2) resulting percentages of service injected - * w.r.t. to the total service dispatched while bfqq is in - * service, and (3) corresponding values of the coefficient: - * 1 (50%) -> 10 - * 2 (33%) -> 20 - * 10 (9%) -> 100 - * 9.9 (9%) -> 99 - * 1.5 (40%) -> 15 - * 0.5 (66%) -> 5 - * 0.1 (90%) -> 1 - * - * So, if the coefficient is lower than 10, then - * injected service is more than bfqq service. - */ - unsigned int inject_coeff; - /* amount of service injected in current service slot */ - unsigned int injected_service; }; /** @@ -419,6 +405,15 @@ struct bfq_io_cq { bool was_in_burst_list; /* + * Save the weight when a merge occurs, to be able + * to restore it in case of split. If the weight is not + * correctly resumed when the queue is recycled, + * then the weight of the recycled queue could differ + * from the weight of the original queue. + */ + unsigned int saved_weight; + + /* * Similar to previous fields: save wr information. */ unsigned long saved_wr_coeff; @@ -450,7 +445,7 @@ struct bfq_data { * weight-raised @bfq_queue (see the comments to the functions * bfq_weights_tree_[add|remove] for further details). */ - struct rb_root queue_weights_tree; + struct rb_root_cached queue_weights_tree; /* * Number of groups with at least one descendant process that @@ -513,6 +508,9 @@ struct bfq_data { /* number of requests dispatched and waiting for completion */ int rq_in_driver; + /* true if the device is non rotational and performs queueing */ + bool nonrot_with_queueing; + /* * Maximum number of requests in driver in the last * @hw_tag_samples completed requests. @@ -544,6 +542,26 @@ struct bfq_data { /* time of last request completion (ns) */ u64 last_completion; + /* time of last transition from empty to non-empty (ns) */ + u64 last_empty_occupied_ns; + + /* + * Flag set to activate the sampling of the total service time + * of a just-arrived first I/O request (see + * bfq_update_inject_limit()). This will cause the setting of + * waited_rq when the request is finally dispatched. + */ + bool wait_dispatch; + /* + * If set, then bfq_update_inject_limit() is invoked when + * waited_rq is eventually completed. + */ + struct request *waited_rq; + /* + * True if some request has been injected during the last service hole. + */ + bool rqs_injected; + /* time of first rq dispatch in current observation interval (ns) */ u64 first_dispatch; /* time of last rq dispatch in current observation interval (ns) */ @@ -553,6 +571,7 @@ struct bfq_data { ktime_t last_budget_start; /* beginning of the last idle slice */ ktime_t last_idling_start; + unsigned long last_idling_start_jiffies; /* number of samples in current observation interval */ int peak_rate_samples; @@ -898,10 +917,10 @@ void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync); struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic); void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq); void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, - struct rb_root *root); + struct rb_root_cached *root); void __bfq_weights_tree_remove(struct bfq_data *bfqd, struct bfq_queue *bfqq, - struct rb_root *root); + struct rb_root_cached *root); void bfq_weights_tree_remove(struct bfq_data *bfqd, struct bfq_queue *bfqq); void bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, @@ -1008,13 +1027,23 @@ void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq); /* --------------- end of interface of B-WF2Q+ ---------------- */ /* Logging facilities. */ +static inline void bfq_pid_to_str(int pid, char *str, int len) +{ + if (pid != -1) + snprintf(str, len, "%d", pid); + else + snprintf(str, len, "SHARED-"); +} + #ifdef CONFIG_BFQ_GROUP_IOSCHED struct bfq_group *bfqq_group(struct bfq_queue *bfqq); #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \ + char pid_str[MAX_PID_STR_LENGTH]; \ + bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH); \ blk_add_cgroup_trace_msg((bfqd)->queue, \ bfqg_to_blkg(bfqq_group(bfqq))->blkcg, \ - "bfq%d%c " fmt, (bfqq)->pid, \ + "bfq%s%c " fmt, pid_str, \ bfq_bfqq_sync((bfqq)) ? 'S' : 'A', ##args); \ } while (0) @@ -1025,10 +1054,13 @@ struct bfq_group *bfqq_group(struct bfq_queue *bfqq); #else /* CONFIG_BFQ_GROUP_IOSCHED */ -#define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \ - blk_add_trace_msg((bfqd)->queue, "bfq%d%c " fmt, (bfqq)->pid, \ +#define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \ + char pid_str[MAX_PID_STR_LENGTH]; \ + bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH); \ + blk_add_trace_msg((bfqd)->queue, "bfq%s%c " fmt, pid_str, \ bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ - ##args) + ##args); \ +} while (0) #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do {} while (0) #endif /* CONFIG_BFQ_GROUP_IOSCHED */ diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c index ae4d000ac0af..48d899cfbe03 100644 --- a/block/bfq-wf2q.c +++ b/block/bfq-wf2q.c @@ -59,7 +59,7 @@ static bool bfq_update_parent_budget(struct bfq_entity *next_in_service); * bfq_update_next_in_service - update sd->next_in_service * @sd: sched_data for which to perform the update. * @new_entity: if not NULL, pointer to the entity whose activation, - * requeueing or repositionig triggered the invocation of + * requeueing or repositioning triggered the invocation of * this function. * @expiration: id true, this function is being invoked after the * expiration of the in-service entity @@ -90,7 +90,7 @@ static bool bfq_update_next_in_service(struct bfq_sched_data *sd, /* * If this update is triggered by the activation, requeueing - * or repositiong of an entity that does not coincide with + * or repositioning of an entity that does not coincide with * sd->next_in_service, then a full lookup in the active tree * can be avoided. In fact, it is enough to check whether the * just-modified entity has the same priority as @@ -737,7 +737,7 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); unsigned int prev_weight, new_weight; struct bfq_data *bfqd = NULL; - struct rb_root *root; + struct rb_root_cached *root; #ifdef CONFIG_BFQ_GROUP_IOSCHED struct bfq_sched_data *sd; struct bfq_group *bfqg; @@ -1396,7 +1396,7 @@ left: * In this first case, update the virtual time in @st too (see the * comments on this update inside the function). * - * In constrast, if there is an in-service entity, then return the + * In contrast, if there is an in-service entity, then return the * entity that would be set in service if not only the above * conditions, but also the next one held true: the currently * in-service entity, on expiration, @@ -1479,12 +1479,12 @@ static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, * is being invoked as a part of the expiration path * of the in-service queue. In this case, even if * sd->in_service_entity is not NULL, - * sd->in_service_entiy at this point is actually not + * sd->in_service_entity at this point is actually not * in service any more, and, if needed, has already * been properly queued or requeued into the right * tree. The reason why sd->in_service_entity is still * not NULL here, even if expiration is true, is that - * sd->in_service_entiy is reset as a last step in the + * sd->in_service_entity is reset as a last step in the * expiration path. So, if expiration is true, tell * __bfq_lookup_next_entity that there is no * sd->in_service_entity. diff --git a/block/bio.c b/block/bio.c index 716510ecd7ff..5959141d4e46 100644 --- a/block/bio.c +++ b/block/bio.c @@ -647,25 +647,70 @@ struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs) } EXPORT_SYMBOL(bio_clone_fast); +static inline bool page_is_mergeable(const struct bio_vec *bv, + struct page *page, unsigned int len, unsigned int off, + bool same_page) +{ + phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + + bv->bv_offset + bv->bv_len - 1; + phys_addr_t page_addr = page_to_phys(page); + + if (vec_end_addr + 1 != page_addr + off) + return false; + if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) + return false; + + if ((vec_end_addr & PAGE_MASK) != page_addr) { + if (same_page) + return false; + if (pfn_to_page(PFN_DOWN(vec_end_addr)) + 1 != page) + return false; + } + + return true; +} + +/* + * Check if the @page can be added to the current segment(@bv), and make + * sure to call it only if page_is_mergeable(@bv, @page) is true + */ +static bool can_add_page_to_seg(struct request_queue *q, + struct bio_vec *bv, struct page *page, unsigned len, + unsigned offset) +{ + unsigned long mask = queue_segment_boundary(q); + phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset; + phys_addr_t addr2 = page_to_phys(page) + offset + len - 1; + + if ((addr1 | mask) != (addr2 | mask)) + return false; + + if (bv->bv_len + len > queue_max_segment_size(q)) + return false; + + return true; +} + /** - * bio_add_pc_page - attempt to add page to bio + * __bio_add_pc_page - attempt to add page to passthrough bio * @q: the target queue * @bio: destination bio * @page: page to add * @len: vec entry length * @offset: vec entry offset + * @put_same_page: put the page if it is same with last added page * * Attempt to add a page to the bio_vec maplist. This can fail for a * number of reasons, such as the bio being full or target block device * limitations. The target block device must allow bio's up to PAGE_SIZE, * so it is always possible to add a single page to an empty bio. * - * This should only be used by REQ_PC bios. + * This should only be used by passthrough bios. */ -int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page - *page, unsigned int len, unsigned int offset) +int __bio_add_pc_page(struct request_queue *q, struct bio *bio, + struct page *page, unsigned int len, unsigned int offset, + bool put_same_page) { - int retried_segments = 0; struct bio_vec *bvec; /* @@ -683,11 +728,14 @@ int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page * a consecutive offset. Optimize this special case. */ if (bio->bi_vcnt > 0) { - struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1]; - - if (page == prev->bv_page && - offset == prev->bv_offset + prev->bv_len) { - prev->bv_len += len; + bvec = &bio->bi_io_vec[bio->bi_vcnt - 1]; + + if (page == bvec->bv_page && + offset == bvec->bv_offset + bvec->bv_len) { + if (put_same_page) + put_page(page); + bvec_merge: + bvec->bv_len += len; bio->bi_iter.bi_size += len; goto done; } @@ -696,13 +744,20 @@ int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page * If the queue doesn't support SG gaps and adding this * offset would create a gap, disallow it. */ - if (bvec_gap_to_prev(q, prev, offset)) + if (bvec_gap_to_prev(q, bvec, offset)) return 0; + + if (page_is_mergeable(bvec, page, len, offset, false) && + can_add_page_to_seg(q, bvec, page, len, offset)) + goto bvec_merge; } if (bio_full(bio)) return 0; + if (bio->bi_phys_segments >= queue_max_segments(q)) + return 0; + /* * setup the new entry, we might clear it again later if we * cannot add the page @@ -712,38 +767,19 @@ int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page bvec->bv_len = len; bvec->bv_offset = offset; bio->bi_vcnt++; - bio->bi_phys_segments++; bio->bi_iter.bi_size += len; - /* - * Perform a recount if the number of segments is greater - * than queue_max_segments(q). - */ - - while (bio->bi_phys_segments > queue_max_segments(q)) { - - if (retried_segments) - goto failed; - - retried_segments = 1; - blk_recount_segments(q, bio); - } - - /* If we may be able to merge these biovecs, force a recount */ - if (bio->bi_vcnt > 1 && biovec_phys_mergeable(q, bvec - 1, bvec)) - bio_clear_flag(bio, BIO_SEG_VALID); - done: + bio->bi_phys_segments = bio->bi_vcnt; + bio_set_flag(bio, BIO_SEG_VALID); return len; +} +EXPORT_SYMBOL(__bio_add_pc_page); - failed: - bvec->bv_page = NULL; - bvec->bv_len = 0; - bvec->bv_offset = 0; - bio->bi_vcnt--; - bio->bi_iter.bi_size -= len; - blk_recount_segments(q, bio); - return 0; +int bio_add_pc_page(struct request_queue *q, struct bio *bio, + struct page *page, unsigned int len, unsigned int offset) +{ + return __bio_add_pc_page(q, bio, page, len, offset, false); } EXPORT_SYMBOL(bio_add_pc_page); @@ -770,18 +806,12 @@ bool __bio_try_merge_page(struct bio *bio, struct page *page, if (bio->bi_vcnt > 0) { struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; - phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + - bv->bv_offset + bv->bv_len - 1; - phys_addr_t page_addr = page_to_phys(page); - - if (vec_end_addr + 1 != page_addr + off) - return false; - if (same_page && (vec_end_addr & PAGE_MASK) != page_addr) - return false; - bv->bv_len += len; - bio->bi_iter.bi_size += len; - return true; + if (page_is_mergeable(bv, page, len, off, same_page)) { + bv->bv_len += len; + bio->bi_iter.bi_size += len; + return true; + } } return false; } @@ -836,6 +866,26 @@ int bio_add_page(struct bio *bio, struct page *page, } EXPORT_SYMBOL(bio_add_page); +static void bio_get_pages(struct bio *bio) +{ + struct bvec_iter_all iter_all; + struct bio_vec *bvec; + int i; + + bio_for_each_segment_all(bvec, bio, i, iter_all) + get_page(bvec->bv_page); +} + +static void bio_release_pages(struct bio *bio) +{ + struct bvec_iter_all iter_all; + struct bio_vec *bvec; + int i; + + bio_for_each_segment_all(bvec, bio, i, iter_all) + put_page(bvec->bv_page); +} + static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter) { const struct bio_vec *bv = iter->bvec; @@ -848,20 +898,10 @@ static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter) len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count); size = bio_add_page(bio, bv->bv_page, len, bv->bv_offset + iter->iov_offset); - if (size == len) { - if (!bio_flagged(bio, BIO_NO_PAGE_REF)) { - struct page *page; - int i; - - mp_bvec_for_each_page(page, bv, i) - get_page(page); - } - - iov_iter_advance(iter, size); - return 0; - } - - return -EINVAL; + if (unlikely(size != len)) + return -EINVAL; + iov_iter_advance(iter, size); + return 0; } #define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) @@ -934,29 +974,24 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { const bool is_bvec = iov_iter_is_bvec(iter); - unsigned short orig_vcnt = bio->bi_vcnt; + int ret; - /* - * If this is a BVEC iter, then the pages are kernel pages. Don't - * release them on IO completion, if the caller asked us to. - */ - if (is_bvec && iov_iter_bvec_no_ref(iter)) - bio_set_flag(bio, BIO_NO_PAGE_REF); + if (WARN_ON_ONCE(bio->bi_vcnt)) + return -EINVAL; do { - int ret; - if (is_bvec) ret = __bio_iov_bvec_add_pages(bio, iter); else ret = __bio_iov_iter_get_pages(bio, iter); + } while (!ret && iov_iter_count(iter) && !bio_full(bio)); - if (unlikely(ret)) - return bio->bi_vcnt > orig_vcnt ? 0 : ret; - - } while (iov_iter_count(iter) && !bio_full(bio)); + if (iov_iter_bvec_no_ref(iter)) + bio_set_flag(bio, BIO_NO_PAGE_REF); + else + bio_get_pages(bio); - return 0; + return bio->bi_vcnt ? 0 : ret; } static void submit_bio_wait_endio(struct bio *bio) @@ -1388,21 +1423,14 @@ struct bio *bio_map_user_iov(struct request_queue *q, for (j = 0; j < npages; j++) { struct page *page = pages[j]; unsigned int n = PAGE_SIZE - offs; - unsigned short prev_bi_vcnt = bio->bi_vcnt; if (n > bytes) n = bytes; - if (!bio_add_pc_page(q, bio, page, n, offs)) + if (!__bio_add_pc_page(q, bio, page, n, offs, + true)) break; - /* - * check if vector was merged with previous - * drop page reference if needed - */ - if (bio->bi_vcnt == prev_bi_vcnt) - put_page(page); - added += n; bytes -= n; offs = 0; @@ -1659,16 +1687,6 @@ void bio_set_pages_dirty(struct bio *bio) } } -static void bio_release_pages(struct bio *bio) -{ - struct bio_vec *bvec; - int i; - struct bvec_iter_all iter_all; - - bio_for_each_segment_all(bvec, bio, i, iter_all) - put_page(bvec->bv_page); -} - /* * bio_check_pages_dirty() will check that all the BIO's pages are still dirty. * If they are, then fine. If, however, some pages are clean then they must @@ -2203,6 +2221,9 @@ static int __init init_bio(void) bio_slab_nr = 0; bio_slabs = kcalloc(bio_slab_max, sizeof(struct bio_slab), GFP_KERNEL); + + BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET); + if (!bio_slabs) panic("bio: can't allocate bios\n"); diff --git a/block/blk-merge.c b/block/blk-merge.c index 1c9d4f0f96ea..247b17f2a0f6 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -267,23 +267,6 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, goto split; } - if (bvprvp) { - if (seg_size + bv.bv_len > queue_max_segment_size(q)) - goto new_segment; - if (!biovec_phys_mergeable(q, bvprvp, &bv)) - goto new_segment; - - seg_size += bv.bv_len; - bvprv = bv; - bvprvp = &bvprv; - sectors += bv.bv_len >> 9; - - if (nsegs == 1 && seg_size > front_seg_size) - front_seg_size = seg_size; - - continue; - } -new_segment: if (nsegs == max_segs) goto split; @@ -370,12 +353,12 @@ EXPORT_SYMBOL(blk_queue_split); static unsigned int __blk_recalc_rq_segments(struct request_queue *q, struct bio *bio) { - struct bio_vec bv, bvprv = { NULL }; - int prev = 0; + struct bio_vec uninitialized_var(bv), bvprv = { NULL }; unsigned int seg_size, nr_phys_segs; unsigned front_seg_size; struct bio *fbio, *bbio; struct bvec_iter iter; + bool new_bio = false; if (!bio) return 0; @@ -396,7 +379,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, nr_phys_segs = 0; for_each_bio(bio) { bio_for_each_bvec(bv, bio, iter) { - if (prev) { + if (new_bio) { if (seg_size + bv.bv_len > queue_max_segment_size(q)) goto new_segment; @@ -404,7 +387,6 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, goto new_segment; seg_size += bv.bv_len; - bvprv = bv; if (nr_phys_segs == 1 && seg_size > front_seg_size) @@ -413,12 +395,15 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, continue; } new_segment: - bvprv = bv; - prev = 1; bvec_split_segs(q, &bv, &nr_phys_segs, &seg_size, &front_seg_size, NULL, UINT_MAX); + new_bio = false; } bbio = bio; + if (likely(bio->bi_iter.bi_size)) { + bvprv = bv; + new_bio = true; + } } fbio->bi_seg_front_size = front_seg_size; @@ -484,79 +469,85 @@ static unsigned blk_bvec_map_sg(struct request_queue *q, struct scatterlist **sg) { unsigned nbytes = bvec->bv_len; - unsigned nsegs = 0, total = 0, offset = 0; + unsigned nsegs = 0, total = 0; while (nbytes > 0) { - unsigned seg_size; - struct page *pg; - unsigned idx; + unsigned offset = bvec->bv_offset + total; + unsigned len = min(get_max_segment_size(q, offset), nbytes); *sg = blk_next_sg(sg, sglist); + sg_set_page(*sg, bvec->bv_page, len, offset); - seg_size = get_max_segment_size(q, bvec->bv_offset + total); - seg_size = min(nbytes, seg_size); - - offset = (total + bvec->bv_offset) % PAGE_SIZE; - idx = (total + bvec->bv_offset) / PAGE_SIZE; - pg = bvec_nth_page(bvec->bv_page, idx); - - sg_set_page(*sg, pg, seg_size, offset); - - total += seg_size; - nbytes -= seg_size; + total += len; + nbytes -= len; nsegs++; } return nsegs; } -static inline void -__blk_segment_map_sg(struct request_queue *q, struct bio_vec *bvec, - struct scatterlist *sglist, struct bio_vec *bvprv, - struct scatterlist **sg, int *nsegs) +static inline int __blk_bvec_map_sg(struct bio_vec bv, + struct scatterlist *sglist, struct scatterlist **sg) +{ + *sg = blk_next_sg(sg, sglist); + sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset); + return 1; +} + +/* only try to merge bvecs into one sg if they are from two bios */ +static inline bool +__blk_segment_map_sg_merge(struct request_queue *q, struct bio_vec *bvec, + struct bio_vec *bvprv, struct scatterlist **sg) { int nbytes = bvec->bv_len; - if (*sg) { - if ((*sg)->length + nbytes > queue_max_segment_size(q)) - goto new_segment; - if (!biovec_phys_mergeable(q, bvprv, bvec)) - goto new_segment; + if (!*sg) + return false; - (*sg)->length += nbytes; - } else { -new_segment: - if (bvec->bv_offset + bvec->bv_len <= PAGE_SIZE) { - *sg = blk_next_sg(sg, sglist); - sg_set_page(*sg, bvec->bv_page, nbytes, bvec->bv_offset); - (*nsegs) += 1; - } else - (*nsegs) += blk_bvec_map_sg(q, bvec, sglist, sg); - } - *bvprv = *bvec; -} + if ((*sg)->length + nbytes > queue_max_segment_size(q)) + return false; -static inline int __blk_bvec_map_sg(struct request_queue *q, struct bio_vec bv, - struct scatterlist *sglist, struct scatterlist **sg) -{ - *sg = sglist; - sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset); - return 1; + if (!biovec_phys_mergeable(q, bvprv, bvec)) + return false; + + (*sg)->length += nbytes; + + return true; } static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, struct scatterlist *sglist, struct scatterlist **sg) { - struct bio_vec bvec, bvprv = { NULL }; + struct bio_vec uninitialized_var(bvec), bvprv = { NULL }; struct bvec_iter iter; int nsegs = 0; + bool new_bio = false; - for_each_bio(bio) - bio_for_each_bvec(bvec, bio, iter) - __blk_segment_map_sg(q, &bvec, sglist, &bvprv, sg, - &nsegs); + for_each_bio(bio) { + bio_for_each_bvec(bvec, bio, iter) { + /* + * Only try to merge bvecs from two bios given we + * have done bio internal merge when adding pages + * to bio + */ + if (new_bio && + __blk_segment_map_sg_merge(q, &bvec, &bvprv, sg)) + goto next_bvec; + + if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE) + nsegs += __blk_bvec_map_sg(bvec, sglist, sg); + else + nsegs += blk_bvec_map_sg(q, &bvec, sglist, sg); + next_bvec: + new_bio = false; + } + if (likely(bio->bi_iter.bi_size)) { + bvprv = bvec; + new_bio = true; + } + } return nsegs; } @@ -572,9 +563,9 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq, int nsegs = 0; if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) - nsegs = __blk_bvec_map_sg(q, rq->special_vec, sglist, &sg); + nsegs = __blk_bvec_map_sg(rq->special_vec, sglist, &sg); else if (rq->bio && bio_op(rq->bio) == REQ_OP_WRITE_SAME) - nsegs = __blk_bvec_map_sg(q, bio_iovec(rq->bio), sglist, &sg); + nsegs = __blk_bvec_map_sg(bio_iovec(rq->bio), sglist, &sg); else if (rq->bio) nsegs = __blk_bios_map_sg(q, rq->bio, sglist, &sg); diff --git a/block/blk.h b/block/blk.h index 5d636ee41663..e27fd1512e4b 100644 --- a/block/blk.h +++ b/block/blk.h @@ -75,7 +75,7 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, if (addr1 + vec1->bv_len != addr2) return false; - if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2)) + if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) return false; if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) return false; diff --git a/block/elevator.c b/block/elevator.c index d6d835a08de6..2e5399d9f40f 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -509,8 +509,6 @@ void elv_unregister_queue(struct request_queue *q) int elv_register(struct elevator_type *e) { - char *def = ""; - /* create icq_cache if requested */ if (e->icq_size) { if (WARN_ON(e->icq_size < sizeof(struct io_cq)) || @@ -535,8 +533,8 @@ int elv_register(struct elevator_type *e) list_add_tail(&e->list, &elv_list); spin_unlock(&elv_list_lock); - printk(KERN_INFO "io scheduler %s registered%s\n", e->elevator_name, - def); + printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name); + return 0; } EXPORT_SYMBOL_GPL(elv_register); diff --git a/block/genhd.c b/block/genhd.c index 703267865f14..1d0d25f7b0fe 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1628,12 +1628,11 @@ static unsigned long disk_events_poll_jiffies(struct gendisk *disk) /* * If device-specific poll interval is set, always use it. If - * the default is being used, poll iff there are events which - * can't be monitored asynchronously. + * the default is being used, poll if the POLL flag is set. */ if (ev->poll_msecs >= 0) intv_msecs = ev->poll_msecs; - else if (disk->events & ~disk->async_events) + else if (disk->event_flags & DISK_EVENT_FLAG_POLL) intv_msecs = disk_events_dfl_poll_msecs; return msecs_to_jiffies(intv_msecs); @@ -1843,11 +1842,13 @@ static void disk_check_events(struct disk_events *ev, /* * Tell userland about new events. Only the events listed in - * @disk->events are reported. Unlisted events are processed the - * same internally but never get reported to userland. + * @disk->events are reported, and only if DISK_EVENT_FLAG_UEVENT + * is set. Otherwise, events are processed internally but never + * get reported to userland. */ for (i = 0; i < ARRAY_SIZE(disk_uevents); i++) - if (events & disk->events & (1 << i)) + if ((events & disk->events & (1 << i)) && + (disk->event_flags & DISK_EVENT_FLAG_UEVENT)) envp[nr_events++] = disk_uevents[i]; if (nr_events) @@ -1860,6 +1861,7 @@ static void disk_check_events(struct disk_events *ev, * * events : list of all supported events * events_async : list of events which can be detected w/o polling + * (always empty, only for backwards compatibility) * events_poll_msecs : polling interval, 0: disable, -1: system default */ static ssize_t __disk_events_show(unsigned int events, char *buf) @@ -1884,15 +1886,16 @@ static ssize_t disk_events_show(struct device *dev, { struct gendisk *disk = dev_to_disk(dev); + if (!(disk->event_flags & DISK_EVENT_FLAG_UEVENT)) + return 0; + return __disk_events_show(disk->events, buf); } static ssize_t disk_events_async_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct gendisk *disk = dev_to_disk(dev); - - return __disk_events_show(disk->async_events, buf); + return 0; } static ssize_t disk_events_poll_msecs_show(struct device *dev, @@ -1901,6 +1904,9 @@ static ssize_t disk_events_poll_msecs_show(struct device *dev, { struct gendisk *disk = dev_to_disk(dev); + if (!disk->ev) + return sprintf(buf, "-1\n"); + return sprintf(buf, "%ld\n", disk->ev->poll_msecs); } @@ -1917,6 +1923,9 @@ static ssize_t disk_events_poll_msecs_store(struct device *dev, if (intv < 0 && intv != -1) return -EINVAL; + if (!disk->ev) + return -ENODEV; + disk_block_events(disk); disk->ev->poll_msecs = intv; __disk_unblock_events(disk, true); @@ -1981,7 +1990,7 @@ static void disk_alloc_events(struct gendisk *disk) { struct disk_events *ev; - if (!disk->fops->check_events) + if (!disk->fops->check_events || !disk->events) return; ev = kzalloc(sizeof(*ev), GFP_KERNEL); @@ -2003,14 +2012,14 @@ static void disk_alloc_events(struct gendisk *disk) static void disk_add_events(struct gendisk *disk) { - if (!disk->ev) - return; - /* FIXME: error handling */ if (sysfs_create_files(&disk_to_dev(disk)->kobj, disk_events_attrs) < 0) pr_warn("%s: failed to create sysfs files for events\n", disk->disk_name); + if (!disk->ev) + return; + mutex_lock(&disk_events_mutex); list_add_tail(&disk->ev->node, &disk_events); mutex_unlock(&disk_events_mutex); @@ -2024,14 +2033,13 @@ static void disk_add_events(struct gendisk *disk) static void disk_del_events(struct gendisk *disk) { - if (!disk->ev) - return; + if (disk->ev) { + disk_block_events(disk); - disk_block_events(disk); - - mutex_lock(&disk_events_mutex); - list_del_init(&disk->ev->node); - mutex_unlock(&disk_events_mutex); + mutex_lock(&disk_events_mutex); + list_del_init(&disk->ev->node); + mutex_unlock(&disk_events_mutex); + } sysfs_remove_files(&disk_to_dev(disk)->kobj, disk_events_attrs); } diff --git a/block/opal_proto.h b/block/opal_proto.h index e20be8258854..b6e352cfe982 100644 --- a/block/opal_proto.h +++ b/block/opal_proto.h @@ -170,6 +170,8 @@ enum opal_token { OPAL_READLOCKED = 0x07, OPAL_WRITELOCKED = 0x08, OPAL_ACTIVEKEY = 0x0A, + /* lockingsp table */ + OPAL_LIFECYCLE = 0x06, /* locking info table */ OPAL_MAXRANGES = 0x04, /* mbr control */ diff --git a/block/sed-opal.c b/block/sed-opal.c index e0de4dd448b3..b1aa0cc25803 100644 --- a/block/sed-opal.c +++ b/block/sed-opal.c @@ -85,7 +85,6 @@ struct opal_dev { void *data; sec_send_recv *send_recv; - const struct opal_step *steps; struct mutex dev_lock; u16 comid; u32 hsn; @@ -157,7 +156,7 @@ static const u8 opaluid[][OPAL_UID_LENGTH] = { /* C_PIN_TABLE object ID's */ - [OPAL_C_PIN_MSID] = + [OPAL_C_PIN_MSID] = { 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x84, 0x02}, [OPAL_C_PIN_SID] = { 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x01}, @@ -181,7 +180,7 @@ static const u8 opaluid[][OPAL_UID_LENGTH] = { * Derived from: TCG_Storage_Architecture_Core_Spec_v2.01_r1.00 * Section: 6.3 Assigned UIDs */ -static const u8 opalmethod[][OPAL_UID_LENGTH] = { +static const u8 opalmethod[][OPAL_METHOD_LENGTH] = { [OPAL_PROPERTIES] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x01 }, [OPAL_STARTSESSION] = @@ -217,6 +216,7 @@ static const u8 opalmethod[][OPAL_UID_LENGTH] = { }; static int end_opal_session_error(struct opal_dev *dev); +static int opal_discovery0_step(struct opal_dev *dev); struct opal_suspend_data { struct opal_lock_unlock unlk; @@ -382,37 +382,50 @@ static void check_geometry(struct opal_dev *dev, const void *data) dev->lowest_lba = geo->lowest_aligned_lba; } -static int next(struct opal_dev *dev) +static int execute_step(struct opal_dev *dev, + const struct opal_step *step, size_t stepIndex) { - const struct opal_step *step; - int state = 0, error = 0; + int error = step->fn(dev, step->data); - do { - step = &dev->steps[state]; - if (!step->fn) - break; + if (error) { + pr_debug("Step %zu (%pS) failed with error %d: %s\n", + stepIndex, step->fn, error, + opal_error_to_human(error)); + } + + return error; +} - error = step->fn(dev, step->data); - if (error) { - pr_debug("Error on step function: %d with error %d: %s\n", - state, error, - opal_error_to_human(error)); - - /* For each OPAL command we do a discovery0 then we - * start some sort of session. - * If we haven't passed state 1 then there was an error - * on discovery0 or during the attempt to start a - * session. Therefore we shouldn't attempt to terminate - * a session, as one has not yet been created. - */ - if (state > 1) { - end_opal_session_error(dev); - return error; - } +static int execute_steps(struct opal_dev *dev, + const struct opal_step *steps, size_t n_steps) +{ + size_t state = 0; + int error; - } - state++; - } while (!error); + /* first do a discovery0 */ + error = opal_discovery0_step(dev); + if (error) + return error; + + for (state = 0; state < n_steps; state++) { + error = execute_step(dev, &steps[state], state); + if (error) + goto out_error; + } + + return 0; + +out_error: + /* + * For each OPAL command the first step in steps starts some sort of + * session. If an error occurred in the initial discovery0 or if an + * error occurred in the first step (and thus stopping the loop with + * state == 0) then there was an error before or during the attempt to + * start a session. Therefore we shouldn't attempt to terminate a + * session, as one has not yet been created. + */ + if (state > 0) + end_opal_session_error(dev); return error; } @@ -510,15 +523,32 @@ static int opal_discovery0(struct opal_dev *dev, void *data) return opal_discovery0_end(dev); } -static void add_token_u8(int *err, struct opal_dev *cmd, u8 tok) +static int opal_discovery0_step(struct opal_dev *dev) +{ + const struct opal_step discovery0_step = { + opal_discovery0, + }; + return execute_step(dev, &discovery0_step, 0); +} + +static bool can_add(int *err, struct opal_dev *cmd, size_t len) { if (*err) - return; - if (cmd->pos >= IO_BUFFER_LENGTH - 1) { - pr_debug("Error adding u8: end of buffer.\n"); + return false; + + if (len > IO_BUFFER_LENGTH || cmd->pos > IO_BUFFER_LENGTH - len) { + pr_debug("Error adding %zu bytes: end of buffer.\n", len); *err = -ERANGE; - return; + return false; } + + return true; +} + +static void add_token_u8(int *err, struct opal_dev *cmd, u8 tok) +{ + if (!can_add(err, cmd, 1)) + return; cmd->cmd[cmd->pos++] = tok; } @@ -551,7 +581,6 @@ static void add_medium_atom_header(struct opal_dev *cmd, bool bytestring, static void add_token_u64(int *err, struct opal_dev *cmd, u64 number) { - size_t len; int msb; @@ -563,9 +592,8 @@ static void add_token_u64(int *err, struct opal_dev *cmd, u64 number) msb = fls64(number); len = DIV_ROUND_UP(msb, 8); - if (cmd->pos >= IO_BUFFER_LENGTH - len - 1) { + if (!can_add(err, cmd, len + 1)) { pr_debug("Error adding u64: end of buffer.\n"); - *err = -ERANGE; return; } add_short_atom_header(cmd, false, false, len); @@ -573,24 +601,19 @@ static void add_token_u64(int *err, struct opal_dev *cmd, u64 number) add_token_u8(err, cmd, number >> (len * 8)); } -static void add_token_bytestring(int *err, struct opal_dev *cmd, - const u8 *bytestring, size_t len) +static u8 *add_bytestring_header(int *err, struct opal_dev *cmd, size_t len) { size_t header_len = 1; bool is_short_atom = true; - if (*err) - return; - if (len & ~SHORT_ATOM_LEN_MASK) { header_len = 2; is_short_atom = false; } - if (len >= IO_BUFFER_LENGTH - cmd->pos - header_len) { + if (!can_add(err, cmd, header_len + len)) { pr_debug("Error adding bytestring: end of buffer.\n"); - *err = -ERANGE; - return; + return NULL; } if (is_short_atom) @@ -598,9 +621,19 @@ static void add_token_bytestring(int *err, struct opal_dev *cmd, else add_medium_atom_header(cmd, true, false, len); - memcpy(&cmd->cmd[cmd->pos], bytestring, len); - cmd->pos += len; + return &cmd->cmd[cmd->pos]; +} + +static void add_token_bytestring(int *err, struct opal_dev *cmd, + const u8 *bytestring, size_t len) +{ + u8 *start; + start = add_bytestring_header(err, cmd, len); + if (!start) + return; + memcpy(start, bytestring, len); + cmd->pos += len; } static int build_locking_range(u8 *buffer, size_t length, u8 lr) @@ -623,7 +656,7 @@ static int build_locking_range(u8 *buffer, size_t length, u8 lr) static int build_locking_user(u8 *buffer, size_t length, u8 lr) { if (length > OPAL_UID_LENGTH) { - pr_debug("Can't build locking range user, Length OOB\n"); + pr_debug("Can't build locking range user. Length OOB\n"); return -ERANGE; } @@ -649,6 +682,9 @@ static int cmd_finalize(struct opal_dev *cmd, u32 hsn, u32 tsn) struct opal_header *hdr; int err = 0; + /* close the parameter list opened from cmd_start */ + add_token_u8(&err, cmd, OPAL_ENDLIST); + add_token_u8(&err, cmd, OPAL_ENDOFDATA); add_token_u8(&err, cmd, OPAL_STARTLIST); add_token_u8(&err, cmd, 0); @@ -687,6 +723,11 @@ static const struct opal_resp_tok *response_get_token( { const struct opal_resp_tok *tok; + if (!resp) { + pr_debug("Response is NULL\n"); + return ERR_PTR(-EINVAL); + } + if (n >= resp->num) { pr_debug("Token number doesn't exist: %d, resp: %d\n", n, resp->num); @@ -869,27 +910,19 @@ static size_t response_get_string(const struct parsed_resp *resp, int n, const char **store) { u8 skip; - const struct opal_resp_tok *token; + const struct opal_resp_tok *tok; *store = NULL; - if (!resp) { - pr_debug("Response is NULL\n"); + tok = response_get_token(resp, n); + if (IS_ERR(tok)) return 0; - } - - if (n >= resp->num) { - pr_debug("Response has %d tokens. Can't access %d\n", - resp->num, n); - return 0; - } - token = &resp->toks[n]; - if (token->type != OPAL_DTA_TOKENID_BYTESTRING) { + if (tok->type != OPAL_DTA_TOKENID_BYTESTRING) { pr_debug("Token is not a byte string!\n"); return 0; } - switch (token->width) { + switch (tok->width) { case OPAL_WIDTH_TINY: case OPAL_WIDTH_SHORT: skip = 1; @@ -905,37 +938,29 @@ static size_t response_get_string(const struct parsed_resp *resp, int n, return 0; } - *store = token->pos + skip; - return token->len - skip; + *store = tok->pos + skip; + return tok->len - skip; } static u64 response_get_u64(const struct parsed_resp *resp, int n) { - if (!resp) { - pr_debug("Response is NULL\n"); - return 0; - } + const struct opal_resp_tok *tok; - if (n >= resp->num) { - pr_debug("Response has %d tokens. Can't access %d\n", - resp->num, n); + tok = response_get_token(resp, n); + if (IS_ERR(tok)) return 0; - } - if (resp->toks[n].type != OPAL_DTA_TOKENID_UINT) { - pr_debug("Token is not unsigned it: %d\n", - resp->toks[n].type); + if (tok->type != OPAL_DTA_TOKENID_UINT) { + pr_debug("Token is not unsigned int: %d\n", tok->type); return 0; } - if (!(resp->toks[n].width == OPAL_WIDTH_TINY || - resp->toks[n].width == OPAL_WIDTH_SHORT)) { - pr_debug("Atom is not short or tiny: %d\n", - resp->toks[n].width); + if (tok->width != OPAL_WIDTH_TINY && tok->width != OPAL_WIDTH_SHORT) { + pr_debug("Atom is not short or tiny: %d\n", tok->width); return 0; } - return resp->toks[n].stored.u; + return tok->stored.u; } static bool response_token_matches(const struct opal_resp_tok *token, u8 match) @@ -991,6 +1016,27 @@ static void clear_opal_cmd(struct opal_dev *dev) memset(dev->cmd, 0, IO_BUFFER_LENGTH); } +static int cmd_start(struct opal_dev *dev, const u8 *uid, const u8 *method) +{ + int err = 0; + + clear_opal_cmd(dev); + set_comid(dev, dev->comid); + + add_token_u8(&err, dev, OPAL_CALL); + add_token_bytestring(&err, dev, uid, OPAL_UID_LENGTH); + add_token_bytestring(&err, dev, method, OPAL_METHOD_LENGTH); + + /* + * Every method call is followed by its parameters enclosed within + * OPAL_STARTLIST and OPAL_ENDLIST tokens. We automatically open the + * parameter list here and close it later in cmd_finalize. + */ + add_token_u8(&err, dev, OPAL_STARTLIST); + + return err; +} + static int start_opal_session_cont(struct opal_dev *dev) { u32 hsn, tsn; @@ -1050,24 +1096,47 @@ static int finalize_and_send(struct opal_dev *dev, cont_fn cont) return opal_send_recv(dev, cont); } +/* + * request @column from table @table on device @dev. On success, the column + * data will be available in dev->resp->tok[4] + */ +static int generic_get_column(struct opal_dev *dev, const u8 *table, + u64 column) +{ + int err; + + err = cmd_start(dev, table, opalmethod[OPAL_GET]); + + add_token_u8(&err, dev, OPAL_STARTLIST); + + add_token_u8(&err, dev, OPAL_STARTNAME); + add_token_u8(&err, dev, OPAL_STARTCOLUMN); + add_token_u64(&err, dev, column); + add_token_u8(&err, dev, OPAL_ENDNAME); + + add_token_u8(&err, dev, OPAL_STARTNAME); + add_token_u8(&err, dev, OPAL_ENDCOLUMN); + add_token_u64(&err, dev, column); + add_token_u8(&err, dev, OPAL_ENDNAME); + + add_token_u8(&err, dev, OPAL_ENDLIST); + + if (err) + return err; + + return finalize_and_send(dev, parse_and_check_status); +} + static int gen_key(struct opal_dev *dev, void *data) { u8 uid[OPAL_UID_LENGTH]; - int err = 0; - - clear_opal_cmd(dev); - set_comid(dev, dev->comid); + int err; memcpy(uid, dev->prev_data, min(sizeof(uid), dev->prev_d_len)); kfree(dev->prev_data); dev->prev_data = NULL; - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, uid, OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_GENKEY], - OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); - add_token_u8(&err, dev, OPAL_ENDLIST); + err = cmd_start(dev, uid, opalmethod[OPAL_GENKEY]); if (err) { pr_debug("Error building gen key command\n"); @@ -1105,62 +1174,39 @@ static int get_active_key_cont(struct opal_dev *dev) static int get_active_key(struct opal_dev *dev, void *data) { u8 uid[OPAL_UID_LENGTH]; - int err = 0; + int err; u8 *lr = data; - clear_opal_cmd(dev); - set_comid(dev, dev->comid); - err = build_locking_range(uid, sizeof(uid), *lr); if (err) return err; - err = 0; - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, uid, OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_GET], OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); - add_token_u8(&err, dev, OPAL_STARTLIST); - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 3); /* startCloumn */ - add_token_u8(&err, dev, 10); /* ActiveKey */ - add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 4); /* endColumn */ - add_token_u8(&err, dev, 10); /* ActiveKey */ - add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); - add_token_u8(&err, dev, OPAL_ENDLIST); - if (err) { - pr_debug("Error building get active key command\n"); + err = generic_get_column(dev, uid, OPAL_ACTIVEKEY); + if (err) return err; - } - return finalize_and_send(dev, get_active_key_cont); + return get_active_key_cont(dev); } static int generic_lr_enable_disable(struct opal_dev *dev, u8 *uid, bool rle, bool wle, bool rl, bool wl) { - int err = 0; + int err; - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, uid, OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_SET], OPAL_UID_LENGTH); + err = cmd_start(dev, uid, opalmethod[OPAL_SET]); - add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); add_token_u8(&err, dev, OPAL_VALUES); add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 5); /* ReadLockEnabled */ + add_token_u8(&err, dev, OPAL_READLOCKENABLED); add_token_u8(&err, dev, rle); add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 6); /* WriteLockEnabled */ + add_token_u8(&err, dev, OPAL_WRITELOCKENABLED); add_token_u8(&err, dev, wle); add_token_u8(&err, dev, OPAL_ENDNAME); @@ -1176,7 +1222,6 @@ static int generic_lr_enable_disable(struct opal_dev *dev, add_token_u8(&err, dev, OPAL_ENDLIST); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); return err; } @@ -1197,10 +1242,7 @@ static int setup_locking_range(struct opal_dev *dev, void *data) u8 uid[OPAL_UID_LENGTH]; struct opal_user_lr_setup *setup = data; u8 lr; - int err = 0; - - clear_opal_cmd(dev); - set_comid(dev, dev->comid); + int err; lr = setup->session.opal_key.lr; err = build_locking_range(uid, sizeof(uid), lr); @@ -1210,40 +1252,34 @@ static int setup_locking_range(struct opal_dev *dev, void *data) if (lr == 0) err = enable_global_lr(dev, uid, setup); else { - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, uid, OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_SET], - OPAL_UID_LENGTH); + err = cmd_start(dev, uid, opalmethod[OPAL_SET]); - add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); add_token_u8(&err, dev, OPAL_VALUES); add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 3); /* Ranges Start */ + add_token_u8(&err, dev, OPAL_RANGESTART); add_token_u64(&err, dev, setup->range_start); add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 4); /* Ranges length */ + add_token_u8(&err, dev, OPAL_RANGELENGTH); add_token_u64(&err, dev, setup->range_length); add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 5); /*ReadLockEnabled */ + add_token_u8(&err, dev, OPAL_READLOCKENABLED); add_token_u64(&err, dev, !!setup->RLE); add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 6); /*WriteLockEnabled*/ + add_token_u8(&err, dev, OPAL_WRITELOCKENABLED); add_token_u64(&err, dev, !!setup->WLE); add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_ENDLIST); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); - } if (err) { pr_debug("Error building Setup Locking range command.\n"); @@ -1261,29 +1297,21 @@ static int start_generic_opal_session(struct opal_dev *dev, u8 key_len) { u32 hsn; - int err = 0; + int err; if (key == NULL && auth != OPAL_ANYBODY_UID) return OPAL_INVAL_PARAM; - clear_opal_cmd(dev); - - set_comid(dev, dev->comid); hsn = GENERIC_HOST_SESSION_NUM; + err = cmd_start(dev, opaluid[OPAL_SMUID_UID], + opalmethod[OPAL_STARTSESSION]); - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, opaluid[OPAL_SMUID_UID], - OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_STARTSESSION], - OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u64(&err, dev, hsn); add_token_bytestring(&err, dev, opaluid[sp_type], OPAL_UID_LENGTH); add_token_u8(&err, dev, 1); switch (auth) { case OPAL_ANYBODY_UID: - add_token_u8(&err, dev, OPAL_ENDLIST); break; case OPAL_ADMIN1_UID: case OPAL_SID_UID: @@ -1296,7 +1324,6 @@ static int start_generic_opal_session(struct opal_dev *dev, add_token_bytestring(&err, dev, opaluid[auth], OPAL_UID_LENGTH); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); break; default: pr_debug("Cannot start Admin SP session with auth %d\n", auth); @@ -1324,6 +1351,7 @@ static int start_SIDASP_opal_session(struct opal_dev *dev, void *data) if (!key) { const struct opal_key *okey = data; + ret = start_generic_opal_session(dev, OPAL_SID_UID, OPAL_ADMINSP_UID, okey->key, @@ -1341,6 +1369,7 @@ static int start_SIDASP_opal_session(struct opal_dev *dev, void *data) static int start_admin1LSP_opal_session(struct opal_dev *dev, void *data) { struct opal_key *key = data; + return start_generic_opal_session(dev, OPAL_ADMIN1_UID, OPAL_LOCKINGSP_UID, key->key, key->key_len); @@ -1356,30 +1385,21 @@ static int start_auth_opal_session(struct opal_dev *dev, void *data) u8 *key = session->opal_key.key; u32 hsn = GENERIC_HOST_SESSION_NUM; - clear_opal_cmd(dev); - set_comid(dev, dev->comid); - - if (session->sum) { + if (session->sum) err = build_locking_user(lk_ul_user, sizeof(lk_ul_user), session->opal_key.lr); - if (err) - return err; - - } else if (session->who != OPAL_ADMIN1 && !session->sum) { + else if (session->who != OPAL_ADMIN1 && !session->sum) err = build_locking_user(lk_ul_user, sizeof(lk_ul_user), session->who - 1); - if (err) - return err; - } else + else memcpy(lk_ul_user, opaluid[OPAL_ADMIN1_UID], OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, opaluid[OPAL_SMUID_UID], - OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_STARTSESSION], - OPAL_UID_LENGTH); + if (err) + return err; + + err = cmd_start(dev, opaluid[OPAL_SMUID_UID], + opalmethod[OPAL_STARTSESSION]); - add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u64(&err, dev, hsn); add_token_bytestring(&err, dev, opaluid[OPAL_LOCKINGSP_UID], OPAL_UID_LENGTH); @@ -1392,7 +1412,6 @@ static int start_auth_opal_session(struct opal_dev *dev, void *data) add_token_u8(&err, dev, 3); add_token_bytestring(&err, dev, lk_ul_user, OPAL_UID_LENGTH); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); if (err) { pr_debug("Error building STARTSESSION command.\n"); @@ -1404,18 +1423,10 @@ static int start_auth_opal_session(struct opal_dev *dev, void *data) static int revert_tper(struct opal_dev *dev, void *data) { - int err = 0; - - clear_opal_cmd(dev); - set_comid(dev, dev->comid); + int err; - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, opaluid[OPAL_ADMINSP_UID], - OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_REVERT], - OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); - add_token_u8(&err, dev, OPAL_ENDLIST); + err = cmd_start(dev, opaluid[OPAL_ADMINSP_UID], + opalmethod[OPAL_REVERT]); if (err) { pr_debug("Error building REVERT TPER command.\n"); return err; @@ -1428,18 +1439,12 @@ static int internal_activate_user(struct opal_dev *dev, void *data) { struct opal_session_info *session = data; u8 uid[OPAL_UID_LENGTH]; - int err = 0; - - clear_opal_cmd(dev); - set_comid(dev, dev->comid); + int err; memcpy(uid, opaluid[OPAL_USER1_UID], OPAL_UID_LENGTH); uid[7] = session->who; - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, uid, OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_SET], OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); + err = cmd_start(dev, uid, opalmethod[OPAL_SET]); add_token_u8(&err, dev, OPAL_STARTNAME); add_token_u8(&err, dev, OPAL_VALUES); add_token_u8(&err, dev, OPAL_STARTLIST); @@ -1449,7 +1454,6 @@ static int internal_activate_user(struct opal_dev *dev, void *data) add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_ENDLIST); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); if (err) { pr_debug("Error building Activate UserN command.\n"); @@ -1463,20 +1467,12 @@ static int erase_locking_range(struct opal_dev *dev, void *data) { struct opal_session_info *session = data; u8 uid[OPAL_UID_LENGTH]; - int err = 0; - - clear_opal_cmd(dev); - set_comid(dev, dev->comid); + int err; if (build_locking_range(uid, sizeof(uid), session->opal_key.lr) < 0) return -ERANGE; - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, uid, OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_ERASE], - OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); - add_token_u8(&err, dev, OPAL_ENDLIST); + err = cmd_start(dev, uid, opalmethod[OPAL_ERASE]); if (err) { pr_debug("Error building Erase Locking Range Command.\n"); @@ -1488,26 +1484,20 @@ static int erase_locking_range(struct opal_dev *dev, void *data) static int set_mbr_done(struct opal_dev *dev, void *data) { u8 *mbr_done_tf = data; - int err = 0; + int err; - clear_opal_cmd(dev); - set_comid(dev, dev->comid); + err = cmd_start(dev, opaluid[OPAL_MBRCONTROL], + opalmethod[OPAL_SET]); - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, opaluid[OPAL_MBRCONTROL], - OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_SET], OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); add_token_u8(&err, dev, OPAL_VALUES); add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 2); /* Done */ + add_token_u8(&err, dev, OPAL_MBRDONE); add_token_u8(&err, dev, *mbr_done_tf); /* Done T or F */ add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_ENDLIST); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); if (err) { pr_debug("Error Building set MBR Done command\n"); @@ -1520,26 +1510,20 @@ static int set_mbr_done(struct opal_dev *dev, void *data) static int set_mbr_enable_disable(struct opal_dev *dev, void *data) { u8 *mbr_en_dis = data; - int err = 0; + int err; - clear_opal_cmd(dev); - set_comid(dev, dev->comid); + err = cmd_start(dev, opaluid[OPAL_MBRCONTROL], + opalmethod[OPAL_SET]); - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, opaluid[OPAL_MBRCONTROL], - OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_SET], OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); add_token_u8(&err, dev, OPAL_VALUES); add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 1); + add_token_u8(&err, dev, OPAL_MBRENABLE); add_token_u8(&err, dev, *mbr_en_dis); add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_ENDLIST); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); if (err) { pr_debug("Error Building set MBR done command\n"); @@ -1552,26 +1536,19 @@ static int set_mbr_enable_disable(struct opal_dev *dev, void *data) static int generic_pw_cmd(u8 *key, size_t key_len, u8 *cpin_uid, struct opal_dev *dev) { - int err = 0; + int err; - clear_opal_cmd(dev); - set_comid(dev, dev->comid); + err = cmd_start(dev, cpin_uid, opalmethod[OPAL_SET]); - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, cpin_uid, OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_SET], - OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); add_token_u8(&err, dev, OPAL_VALUES); add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 3); /* PIN */ + add_token_u8(&err, dev, OPAL_PIN); add_token_bytestring(&err, dev, key, key_len); add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_ENDLIST); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); return err; } @@ -1619,10 +1596,7 @@ static int add_user_to_lr(struct opal_dev *dev, void *data) u8 lr_buffer[OPAL_UID_LENGTH]; u8 user_uid[OPAL_UID_LENGTH]; struct opal_lock_unlock *lkul = data; - int err = 0; - - clear_opal_cmd(dev); - set_comid(dev, dev->comid); + int err; memcpy(lr_buffer, opaluid[OPAL_LOCKINGRANGE_ACE_RDLOCKED], OPAL_UID_LENGTH); @@ -1637,12 +1611,8 @@ static int add_user_to_lr(struct opal_dev *dev, void *data) user_uid[7] = lkul->session.who; - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, lr_buffer, OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_SET], - OPAL_UID_LENGTH); + err = cmd_start(dev, lr_buffer, opalmethod[OPAL_SET]); - add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); add_token_u8(&err, dev, OPAL_VALUES); @@ -1680,7 +1650,6 @@ static int add_user_to_lr(struct opal_dev *dev, void *data) add_token_u8(&err, dev, OPAL_ENDNAME); add_token_u8(&err, dev, OPAL_ENDLIST); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); if (err) { pr_debug("Error building add user to locking range command.\n"); @@ -1697,9 +1666,6 @@ static int lock_unlock_locking_range(struct opal_dev *dev, void *data) u8 read_locked = 1, write_locked = 1; int err = 0; - clear_opal_cmd(dev); - set_comid(dev, dev->comid); - if (build_locking_range(lr_buffer, sizeof(lr_buffer), lkul->session.opal_key.lr) < 0) return -ERANGE; @@ -1714,17 +1680,15 @@ static int lock_unlock_locking_range(struct opal_dev *dev, void *data) write_locked = 0; break; case OPAL_LK: - /* vars are initalized to locked */ + /* vars are initialized to locked */ break; default: pr_debug("Tried to set an invalid locking state... returning to uland\n"); return OPAL_INVAL_PARAM; } - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, lr_buffer, OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_SET], OPAL_UID_LENGTH); - add_token_u8(&err, dev, OPAL_STARTLIST); + err = cmd_start(dev, lr_buffer, opalmethod[OPAL_SET]); + add_token_u8(&err, dev, OPAL_STARTNAME); add_token_u8(&err, dev, OPAL_VALUES); add_token_u8(&err, dev, OPAL_STARTLIST); @@ -1741,7 +1705,6 @@ static int lock_unlock_locking_range(struct opal_dev *dev, void *data) add_token_u8(&err, dev, OPAL_ENDLIST); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); if (err) { pr_debug("Error building SET command.\n"); @@ -1775,7 +1738,7 @@ static int lock_unlock_locking_range_sum(struct opal_dev *dev, void *data) write_locked = 0; break; case OPAL_LK: - /* vars are initalized to locked */ + /* vars are initialized to locked */ break; default: pr_debug("Tried to set an invalid locking state.\n"); @@ -1796,17 +1759,10 @@ static int activate_lsp(struct opal_dev *dev, void *data) struct opal_lr_act *opal_act = data; u8 user_lr[OPAL_UID_LENGTH]; u8 uint_3 = 0x83; - int err = 0, i; - - clear_opal_cmd(dev); - set_comid(dev, dev->comid); - - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, opaluid[OPAL_LOCKINGSP_UID], - OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_ACTIVATE], - OPAL_UID_LENGTH); + int err, i; + err = cmd_start(dev, opaluid[OPAL_LOCKINGSP_UID], + opalmethod[OPAL_ACTIVATE]); if (opal_act->sum) { err = build_locking_range(user_lr, sizeof(user_lr), @@ -1814,7 +1770,6 @@ static int activate_lsp(struct opal_dev *dev, void *data) if (err) return err; - add_token_u8(&err, dev, OPAL_STARTLIST); add_token_u8(&err, dev, OPAL_STARTNAME); add_token_u8(&err, dev, uint_3); add_token_u8(&err, dev, 6); @@ -1829,11 +1784,6 @@ static int activate_lsp(struct opal_dev *dev, void *data) } add_token_u8(&err, dev, OPAL_ENDLIST); add_token_u8(&err, dev, OPAL_ENDNAME); - add_token_u8(&err, dev, OPAL_ENDLIST); - - } else { - add_token_u8(&err, dev, OPAL_STARTLIST); - add_token_u8(&err, dev, OPAL_ENDLIST); } if (err) { @@ -1844,17 +1794,19 @@ static int activate_lsp(struct opal_dev *dev, void *data) return finalize_and_send(dev, parse_and_check_status); } -static int get_lsp_lifecycle_cont(struct opal_dev *dev) +/* Determine if we're in the Manufactured Inactive or Active state */ +static int get_lsp_lifecycle(struct opal_dev *dev, void *data) { u8 lc_status; - int error = 0; + int err; - error = parse_and_check_status(dev); - if (error) - return error; + err = generic_get_column(dev, opaluid[OPAL_LOCKINGSP_UID], + OPAL_LIFECYCLE); + if (err) + return err; lc_status = response_get_u64(&dev->parsed, 4); - /* 0x08 is Manufacured Inactive */ + /* 0x08 is Manufactured Inactive */ /* 0x09 is Manufactured */ if (lc_status != OPAL_MANUFACTURED_INACTIVE) { pr_debug("Couldn't determine the status of the Lifecycle state\n"); @@ -1864,56 +1816,19 @@ static int get_lsp_lifecycle_cont(struct opal_dev *dev) return 0; } -/* Determine if we're in the Manufactured Inactive or Active state */ -static int get_lsp_lifecycle(struct opal_dev *dev, void *data) -{ - int err = 0; - - clear_opal_cmd(dev); - set_comid(dev, dev->comid); - - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, opaluid[OPAL_LOCKINGSP_UID], - OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_GET], OPAL_UID_LENGTH); - - add_token_u8(&err, dev, OPAL_STARTLIST); - add_token_u8(&err, dev, OPAL_STARTLIST); - - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 3); /* Start Column */ - add_token_u8(&err, dev, 6); /* Lifecycle Column */ - add_token_u8(&err, dev, OPAL_ENDNAME); - - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 4); /* End Column */ - add_token_u8(&err, dev, 6); /* Lifecycle Column */ - add_token_u8(&err, dev, OPAL_ENDNAME); - - add_token_u8(&err, dev, OPAL_ENDLIST); - add_token_u8(&err, dev, OPAL_ENDLIST); - - if (err) { - pr_debug("Error Building GET Lifecycle Status command\n"); - return err; - } - - return finalize_and_send(dev, get_lsp_lifecycle_cont); -} - -static int get_msid_cpin_pin_cont(struct opal_dev *dev) +static int get_msid_cpin_pin(struct opal_dev *dev, void *data) { const char *msid_pin; size_t strlen; - int error = 0; + int err; - error = parse_and_check_status(dev); - if (error) - return error; + err = generic_get_column(dev, opaluid[OPAL_C_PIN_MSID], OPAL_PIN); + if (err) + return err; strlen = response_get_string(&dev->parsed, 4, &msid_pin); if (!msid_pin) { - pr_debug("%s: Couldn't extract PIN from response\n", __func__); + pr_debug("Couldn't extract MSID_CPIN from response\n"); return OPAL_INVAL_PARAM; } @@ -1926,42 +1841,6 @@ static int get_msid_cpin_pin_cont(struct opal_dev *dev) return 0; } -static int get_msid_cpin_pin(struct opal_dev *dev, void *data) -{ - int err = 0; - - clear_opal_cmd(dev); - set_comid(dev, dev->comid); - - add_token_u8(&err, dev, OPAL_CALL); - add_token_bytestring(&err, dev, opaluid[OPAL_C_PIN_MSID], - OPAL_UID_LENGTH); - add_token_bytestring(&err, dev, opalmethod[OPAL_GET], OPAL_UID_LENGTH); - - add_token_u8(&err, dev, OPAL_STARTLIST); - add_token_u8(&err, dev, OPAL_STARTLIST); - - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 3); /* Start Column */ - add_token_u8(&err, dev, 3); /* PIN */ - add_token_u8(&err, dev, OPAL_ENDNAME); - - add_token_u8(&err, dev, OPAL_STARTNAME); - add_token_u8(&err, dev, 4); /* End Column */ - add_token_u8(&err, dev, 3); /* Lifecycle Column */ - add_token_u8(&err, dev, OPAL_ENDNAME); - - add_token_u8(&err, dev, OPAL_ENDLIST); - add_token_u8(&err, dev, OPAL_ENDLIST); - - if (err) { - pr_debug("Error building Get MSID CPIN PIN command.\n"); - return err; - } - - return finalize_and_send(dev, get_msid_cpin_pin_cont); -} - static int end_opal_session(struct opal_dev *dev, void *data) { int err = 0; @@ -1977,18 +1856,14 @@ static int end_opal_session(struct opal_dev *dev, void *data) static int end_opal_session_error(struct opal_dev *dev) { - const struct opal_step error_end_session[] = { - { end_opal_session, }, - { NULL, } + const struct opal_step error_end_session = { + end_opal_session, }; - dev->steps = error_end_session; - return next(dev); + return execute_step(dev, &error_end_session, 0); } -static inline void setup_opal_dev(struct opal_dev *dev, - const struct opal_step *steps) +static inline void setup_opal_dev(struct opal_dev *dev) { - dev->steps = steps; dev->tsn = 0; dev->hsn = 0; dev->prev_data = NULL; @@ -1996,15 +1871,11 @@ static inline void setup_opal_dev(struct opal_dev *dev, static int check_opal_support(struct opal_dev *dev) { - const struct opal_step steps[] = { - { opal_discovery0, }, - { NULL, } - }; int ret; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, steps); - ret = next(dev); + setup_opal_dev(dev); + ret = opal_discovery0_step(dev); dev->supported = !ret; mutex_unlock(&dev->dev_lock); return ret; @@ -2057,18 +1928,16 @@ static int opal_secure_erase_locking_range(struct opal_dev *dev, struct opal_session_info *opal_session) { const struct opal_step erase_steps[] = { - { opal_discovery0, }, { start_auth_opal_session, opal_session }, { get_active_key, &opal_session->opal_key.lr }, { gen_key, }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; int ret; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, erase_steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, erase_steps, ARRAY_SIZE(erase_steps)); mutex_unlock(&dev->dev_lock); return ret; } @@ -2077,17 +1946,15 @@ static int opal_erase_locking_range(struct opal_dev *dev, struct opal_session_info *opal_session) { const struct opal_step erase_steps[] = { - { opal_discovery0, }, { start_auth_opal_session, opal_session }, { erase_locking_range, opal_session }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; int ret; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, erase_steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, erase_steps, ARRAY_SIZE(erase_steps)); mutex_unlock(&dev->dev_lock); return ret; } @@ -2095,15 +1962,16 @@ static int opal_erase_locking_range(struct opal_dev *dev, static int opal_enable_disable_shadow_mbr(struct opal_dev *dev, struct opal_mbr_data *opal_mbr) { + u8 enable_disable = opal_mbr->enable_disable == OPAL_MBR_ENABLE ? + OPAL_TRUE : OPAL_FALSE; + const struct opal_step mbr_steps[] = { - { opal_discovery0, }, { start_admin1LSP_opal_session, &opal_mbr->key }, - { set_mbr_done, &opal_mbr->enable_disable }, + { set_mbr_done, &enable_disable }, { end_opal_session, }, { start_admin1LSP_opal_session, &opal_mbr->key }, - { set_mbr_enable_disable, &opal_mbr->enable_disable }, - { end_opal_session, }, - { NULL, } + { set_mbr_enable_disable, &enable_disable }, + { end_opal_session, } }; int ret; @@ -2112,8 +1980,8 @@ static int opal_enable_disable_shadow_mbr(struct opal_dev *dev, return -EINVAL; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, mbr_steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, mbr_steps, ARRAY_SIZE(mbr_steps)); mutex_unlock(&dev->dev_lock); return ret; } @@ -2130,7 +1998,7 @@ static int opal_save(struct opal_dev *dev, struct opal_lock_unlock *lk_unlk) suspend->lr = lk_unlk->session.opal_key.lr; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, NULL); + setup_opal_dev(dev); add_suspend_info(dev, suspend); mutex_unlock(&dev->dev_lock); return 0; @@ -2140,11 +2008,9 @@ static int opal_add_user_to_lr(struct opal_dev *dev, struct opal_lock_unlock *lk_unlk) { const struct opal_step steps[] = { - { opal_discovery0, }, { start_admin1LSP_opal_session, &lk_unlk->session.opal_key }, { add_user_to_lr, lk_unlk }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; int ret; @@ -2166,8 +2032,8 @@ static int opal_add_user_to_lr(struct opal_dev *dev, } mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, steps, ARRAY_SIZE(steps)); mutex_unlock(&dev->dev_lock); return ret; } @@ -2175,16 +2041,14 @@ static int opal_add_user_to_lr(struct opal_dev *dev, static int opal_reverttper(struct opal_dev *dev, struct opal_key *opal) { const struct opal_step revert_steps[] = { - { opal_discovery0, }, { start_SIDASP_opal_session, opal }, - { revert_tper, }, /* controller will terminate session */ - { NULL, } + { revert_tper, } /* controller will terminate session */ }; int ret; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, revert_steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, revert_steps, ARRAY_SIZE(revert_steps)); mutex_unlock(&dev->dev_lock); /* @@ -2201,37 +2065,34 @@ static int __opal_lock_unlock(struct opal_dev *dev, struct opal_lock_unlock *lk_unlk) { const struct opal_step unlock_steps[] = { - { opal_discovery0, }, { start_auth_opal_session, &lk_unlk->session }, { lock_unlock_locking_range, lk_unlk }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; const struct opal_step unlock_sum_steps[] = { - { opal_discovery0, }, { start_auth_opal_session, &lk_unlk->session }, { lock_unlock_locking_range_sum, lk_unlk }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; - dev->steps = lk_unlk->session.sum ? unlock_sum_steps : unlock_steps; - return next(dev); + if (lk_unlk->session.sum) + return execute_steps(dev, unlock_sum_steps, + ARRAY_SIZE(unlock_sum_steps)); + else + return execute_steps(dev, unlock_steps, + ARRAY_SIZE(unlock_steps)); } static int __opal_set_mbr_done(struct opal_dev *dev, struct opal_key *key) { - u8 mbr_done_tf = 1; - const struct opal_step mbrdone_step [] = { - { opal_discovery0, }, + u8 mbr_done_tf = OPAL_TRUE; + const struct opal_step mbrdone_step[] = { { start_admin1LSP_opal_session, key }, { set_mbr_done, &mbr_done_tf }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; - dev->steps = mbrdone_step; - return next(dev); + return execute_steps(dev, mbrdone_step, ARRAY_SIZE(mbrdone_step)); } static int opal_lock_unlock(struct opal_dev *dev, @@ -2252,14 +2113,12 @@ static int opal_lock_unlock(struct opal_dev *dev, static int opal_take_ownership(struct opal_dev *dev, struct opal_key *opal) { const struct opal_step owner_steps[] = { - { opal_discovery0, }, { start_anybodyASP_opal_session, }, { get_msid_cpin_pin, }, { end_opal_session, }, { start_SIDASP_opal_session, opal }, { set_sid_cpin_pin, opal }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; int ret; @@ -2267,21 +2126,20 @@ static int opal_take_ownership(struct opal_dev *dev, struct opal_key *opal) return -ENODEV; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, owner_steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, owner_steps, ARRAY_SIZE(owner_steps)); mutex_unlock(&dev->dev_lock); return ret; } -static int opal_activate_lsp(struct opal_dev *dev, struct opal_lr_act *opal_lr_act) +static int opal_activate_lsp(struct opal_dev *dev, + struct opal_lr_act *opal_lr_act) { const struct opal_step active_steps[] = { - { opal_discovery0, }, { start_SIDASP_opal_session, &opal_lr_act->key }, { get_lsp_lifecycle, }, { activate_lsp, opal_lr_act }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; int ret; @@ -2289,8 +2147,8 @@ static int opal_activate_lsp(struct opal_dev *dev, struct opal_lr_act *opal_lr_a return -EINVAL; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, active_steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, active_steps, ARRAY_SIZE(active_steps)); mutex_unlock(&dev->dev_lock); return ret; } @@ -2299,17 +2157,15 @@ static int opal_setup_locking_range(struct opal_dev *dev, struct opal_user_lr_setup *opal_lrs) { const struct opal_step lr_steps[] = { - { opal_discovery0, }, { start_auth_opal_session, &opal_lrs->session }, { setup_locking_range, opal_lrs }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; int ret; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, lr_steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, lr_steps, ARRAY_SIZE(lr_steps)); mutex_unlock(&dev->dev_lock); return ret; } @@ -2317,11 +2173,9 @@ static int opal_setup_locking_range(struct opal_dev *dev, static int opal_set_new_pw(struct opal_dev *dev, struct opal_new_pw *opal_pw) { const struct opal_step pw_steps[] = { - { opal_discovery0, }, { start_auth_opal_session, &opal_pw->session }, { set_new_pw, &opal_pw->new_user_pw }, - { end_opal_session, }, - { NULL } + { end_opal_session, } }; int ret; @@ -2332,8 +2186,8 @@ static int opal_set_new_pw(struct opal_dev *dev, struct opal_new_pw *opal_pw) return -EINVAL; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, pw_steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, pw_steps, ARRAY_SIZE(pw_steps)); mutex_unlock(&dev->dev_lock); return ret; } @@ -2342,11 +2196,9 @@ static int opal_activate_user(struct opal_dev *dev, struct opal_session_info *opal_session) { const struct opal_step act_steps[] = { - { opal_discovery0, }, { start_admin1LSP_opal_session, &opal_session->opal_key }, { internal_activate_user, opal_session }, - { end_opal_session, }, - { NULL, } + { end_opal_session, } }; int ret; @@ -2358,8 +2210,8 @@ static int opal_activate_user(struct opal_dev *dev, } mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, act_steps); - ret = next(dev); + setup_opal_dev(dev); + ret = execute_steps(dev, act_steps, ARRAY_SIZE(act_steps)); mutex_unlock(&dev->dev_lock); return ret; } @@ -2376,7 +2228,7 @@ bool opal_unlock_from_suspend(struct opal_dev *dev) return false; mutex_lock(&dev->dev_lock); - setup_opal_dev(dev, NULL); + setup_opal_dev(dev); list_for_each_entry(suspend, &dev->unlk_lst, node) { dev->tsn = 0; |