Line data Source code
1 : /*
2 : * Deadline i/o scheduler.
3 : *
4 : * Copyright (C) 2002 Jens Axboe <axboe@kernel.dk>
5 : */
6 : #include <linux/kernel.h>
7 : #include <linux/fs.h>
8 : #include <linux/blkdev.h>
9 : #include <linux/elevator.h>
10 : #include <linux/bio.h>
11 : #include <linux/module.h>
12 : #include <linux/slab.h>
13 : #include <linux/init.h>
14 : #include <linux/compiler.h>
15 : #include <linux/rbtree.h>
16 :
17 : /*
18 : * See Documentation/block/deadline-iosched.txt
19 : */
20 : static const int read_expire = HZ / 2; /* max time before a read is submitted. */
21 : static const int write_expire = 5 * HZ; /* ditto for writes, these limits are SOFT! */
22 : static const int writes_starved = 2; /* max times reads can starve a write */
23 : static const int fifo_batch = 16; /* # of sequential requests treated as one
24 1 : by the above parameters. For throughput. */
25 :
26 : struct deadline_data {
27 : /*
28 : * run time data
29 : */
30 :
31 : /*
32 : * requests (deadline_rq s) are present on both sort_list and fifo_list
33 : */
34 : struct rb_root sort_list[2];
35 : struct list_head fifo_list[2];
36 :
37 : /*
38 : * next in sort order. read, write or both are NULL
39 : */
40 : struct request *next_rq[2];
41 : unsigned int batching; /* number of sequential requests made */
42 : sector_t last_sector; /* head position */
43 : unsigned int starved; /* times reads have starved writes */
44 :
45 : /*
46 : * settings that change how the i/o scheduler behaves
47 : */
48 : int fifo_expire[2];
49 : int fifo_batch;
50 : int writes_starved;
51 : int front_merges;
52 : };
53 :
54 : static void deadline_move_request(struct deadline_data *, struct request *);
55 :
56 : static inline struct rb_root *
57 : deadline_rb_root(struct deadline_data *dd, struct request *rq)
58 : {
59 0 : return &dd->sort_list[rq_data_dir(rq)];
60 : }
61 :
62 : /*
63 : * get the request after `rq' in sector-sorted order
64 : */
65 : static inline struct request *
66 : deadline_latter_request(struct request *rq)
67 : {
68 0 : struct rb_node *node = rb_next(&rq->rb_node);
69 0 :
70 0 : if (node)
71 0 : return rb_entry_rq(node);
72 :
73 0 : return NULL;
74 : }
75 :
76 : static void
77 : deadline_add_rq_rb(struct deadline_data *dd, struct request *rq)
78 : {
79 0 : struct rb_root *root = deadline_rb_root(dd, rq);
80 0 : struct request *__alias;
81 0 :
82 0 : while (unlikely(__alias = elv_rb_add(root, rq)))
83 0 : deadline_move_request(dd, __alias);
84 0 : }
85 0 :
86 : static inline void
87 : deadline_del_rq_rb(struct deadline_data *dd, struct request *rq)
88 : {
89 0 : const int data_dir = rq_data_dir(rq);
90 0 :
91 0 : if (dd->next_rq[data_dir] == rq)
92 0 : dd->next_rq[data_dir] = deadline_latter_request(rq);
93 :
94 0 : elv_rb_del(deadline_rb_root(dd, rq), rq);
95 0 : }
96 :
97 : /*
98 : * add rq to rbtree and fifo
99 : */
100 : static void
101 : deadline_add_request(struct request_queue *q, struct request *rq)
102 : {
103 0 : struct deadline_data *dd = q->elevator->elevator_data;
104 0 : const int data_dir = rq_data_dir(rq);
105 :
106 0 : deadline_add_rq_rb(dd, rq);
107 :
108 : /*
109 : * set expire time and add to fifo list
110 : */
111 0 : rq_set_fifo_time(rq, jiffies + dd->fifo_expire[data_dir]);
112 0 : list_add_tail(&rq->queuelist, &dd->fifo_list[data_dir]);
113 0 : }
114 :
115 : /*
116 : * remove rq from rbtree and fifo.
117 : */
118 : static void deadline_remove_request(struct request_queue *q, struct request *rq)
119 : {
120 0 : struct deadline_data *dd = q->elevator->elevator_data;
121 :
122 0 : rq_fifo_clear(rq);
123 0 : deadline_del_rq_rb(dd, rq);
124 0 : }
125 :
126 : static int
127 : deadline_merge(struct request_queue *q, struct request **req, struct bio *bio)
128 : {
129 0 : struct deadline_data *dd = q->elevator->elevator_data;
130 0 : struct request *__rq;
131 0 : int ret;
132 0 :
133 0 : /*
134 0 : * check for front merge
135 0 : */
136 0 : if (dd->front_merges) {
137 0 : sector_t sector = bio->bi_sector + bio_sectors(bio);
138 :
139 0 : __rq = elv_rb_find(&dd->sort_list[bio_data_dir(bio)], sector);
140 0 : if (__rq) {
141 0 : BUG_ON(sector != blk_rq_pos(__rq));
142 :
143 0 : if (elv_rq_merge_ok(__rq, bio)) {
144 0 : ret = ELEVATOR_FRONT_MERGE;
145 0 : goto out;
146 : }
147 : }
148 : }
149 :
150 0 : return ELEVATOR_NO_MERGE;
151 0 : out:
152 0 : *req = __rq;
153 0 : return ret;
154 : }
155 :
156 : static void deadline_merged_request(struct request_queue *q,
157 : struct request *req, int type)
158 : {
159 0 : struct deadline_data *dd = q->elevator->elevator_data;
160 0 :
161 : /*
162 : * if the merge was a front merge, we need to reposition request
163 : */
164 0 : if (type == ELEVATOR_FRONT_MERGE) {
165 0 : elv_rb_del(deadline_rb_root(dd, req), req);
166 0 : deadline_add_rq_rb(dd, req);
167 : }
168 0 : }
169 :
170 : static void
171 : deadline_merged_requests(struct request_queue *q, struct request *req,
172 : struct request *next)
173 : {
174 0 : /*
175 0 : * if next expires before rq, assign its expire time to rq
176 : * and move into next position (next will be deleted) in fifo
177 : */
178 0 : if (!list_empty(&req->queuelist) && !list_empty(&next->queuelist)) {
179 0 : if (time_before(rq_fifo_time(next), rq_fifo_time(req))) {
180 0 : list_move(&req->queuelist, &next->queuelist);
181 0 : rq_set_fifo_time(req, rq_fifo_time(next));
182 : }
183 : }
184 :
185 : /*
186 : * kill knowledge of next, this one is a goner
187 : */
188 0 : deadline_remove_request(q, next);
189 0 : }
190 :
191 : /*
192 : * move request from sort list to dispatch queue.
193 : */
194 : static inline void
195 : deadline_move_to_dispatch(struct deadline_data *dd, struct request *rq)
196 : {
197 0 : struct request_queue *q = rq->q;
198 :
199 0 : deadline_remove_request(q, rq);
200 0 : elv_dispatch_add_tail(q, rq);
201 0 : }
202 :
203 : /*
204 : * move an entry to dispatch queue
205 : */
206 : static void
207 : deadline_move_request(struct deadline_data *dd, struct request *rq)
208 : {
209 0 : const int data_dir = rq_data_dir(rq);
210 0 :
211 0 : dd->next_rq[READ] = NULL;
212 0 : dd->next_rq[WRITE] = NULL;
213 0 : dd->next_rq[data_dir] = deadline_latter_request(rq);
214 :
215 0 : dd->last_sector = rq_end_sector(rq);
216 :
217 : /*
218 : * take it off the sort and fifo list, move
219 : * to dispatch queue
220 : */
221 0 : deadline_move_to_dispatch(dd, rq);
222 0 : }
223 :
224 : /*
225 : * deadline_check_fifo returns 0 if there are no expired requests on the fifo,
226 : * 1 otherwise. Requires !list_empty(&dd->fifo_list[data_dir])
227 : */
228 : static inline int deadline_check_fifo(struct deadline_data *dd, int ddir)
229 : {
230 0 : struct request *rq = rq_entry_fifo(dd->fifo_list[ddir].next);
231 0 :
232 : /*
233 : * rq is expired!
234 : */
235 0 : if (time_after(jiffies, rq_fifo_time(rq)))
236 0 : return 1;
237 :
238 0 : return 0;
239 : }
240 :
241 : /*
242 : * deadline_dispatch_requests selects the best request according to
243 : * read/write expire, fifo_batch, etc
244 : */
245 : static int deadline_dispatch_requests(struct request_queue *q, int force)
246 : {
247 0 : struct deadline_data *dd = q->elevator->elevator_data;
248 0 : const int reads = !list_empty(&dd->fifo_list[READ]);
249 0 : const int writes = !list_empty(&dd->fifo_list[WRITE]);
250 0 : struct request *rq;
251 0 : int data_dir;
252 0 :
253 0 : /*
254 0 : * batches are currently reads XOR writes
255 0 : */
256 0 : if (dd->next_rq[WRITE])
257 0 : rq = dd->next_rq[WRITE];
258 0 : else
259 0 : rq = dd->next_rq[READ];
260 :
261 0 : if (rq && dd->batching < dd->fifo_batch)
262 : /* we have a next request are still entitled to batch */
263 0 : goto dispatch_request;
264 :
265 : /*
266 : * at this point we are not running a batch. select the appropriate
267 : * data direction (read / write)
268 : */
269 :
270 0 : if (reads) {
271 0 : BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[READ]));
272 :
273 0 : if (writes && (dd->starved++ >= dd->writes_starved))
274 0 : goto dispatch_writes;
275 :
276 0 : data_dir = READ;
277 :
278 0 : goto dispatch_find_request;
279 : }
280 :
281 : /*
282 : * there are either no reads or writes have been starved
283 : */
284 :
285 0 : if (writes) {
286 0 : dispatch_writes:
287 0 : BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[WRITE]));
288 :
289 0 : dd->starved = 0;
290 :
291 0 : data_dir = WRITE;
292 :
293 0 : goto dispatch_find_request;
294 : }
295 :
296 0 : return 0;
297 0 :
298 : dispatch_find_request:
299 : /*
300 : * we are not running a batch, find best request for selected data_dir
301 : */
302 0 : if (deadline_check_fifo(dd, data_dir) || !dd->next_rq[data_dir]) {
303 : /*
304 : * A deadline has expired, the last request was in the other
305 : * direction, or we have run out of higher-sectored requests.
306 : * Start again from the request with the earliest expiry time.
307 : */
308 0 : rq = rq_entry_fifo(dd->fifo_list[data_dir].next);
309 : } else {
310 : /*
311 : * The last req was the same dir and we have a next request in
312 : * sort order. No expired requests so continue on from here.
313 : */
314 0 : rq = dd->next_rq[data_dir];
315 : }
316 :
317 0 : dd->batching = 0;
318 0 :
319 : dispatch_request:
320 : /*
321 : * rq is the selected appropriate request.
322 : */
323 0 : dd->batching++;
324 0 : deadline_move_request(dd, rq);
325 :
326 0 : return 1;
327 : }
328 :
329 : static int deadline_queue_empty(struct request_queue *q)
330 : {
331 0 : struct deadline_data *dd = q->elevator->elevator_data;
332 0 :
333 0 : return list_empty(&dd->fifo_list[WRITE])
334 0 : && list_empty(&dd->fifo_list[READ]);
335 : }
336 :
337 : static void deadline_exit_queue(struct elevator_queue *e)
338 : {
339 0 : struct deadline_data *dd = e->elevator_data;
340 0 :
341 0 : BUG_ON(!list_empty(&dd->fifo_list[READ]));
342 0 : BUG_ON(!list_empty(&dd->fifo_list[WRITE]));
343 0 :
344 0 : kfree(dd);
345 0 : }
346 :
347 : /*
348 : * initialize elevator private data (deadline_data).
349 : */
350 : static void *deadline_init_queue(struct request_queue *q)
351 : {
352 0 : struct deadline_data *dd;
353 0 :
354 0 : dd = kmalloc_node(sizeof(*dd), GFP_KERNEL | __GFP_ZERO, q->node);
355 0 : if (!dd)
356 0 : return NULL;
357 :
358 0 : INIT_LIST_HEAD(&dd->fifo_list[READ]);
359 0 : INIT_LIST_HEAD(&dd->fifo_list[WRITE]);
360 0 : dd->sort_list[READ] = RB_ROOT;
361 0 : dd->sort_list[WRITE] = RB_ROOT;
362 0 : dd->fifo_expire[READ] = read_expire;
363 0 : dd->fifo_expire[WRITE] = write_expire;
364 0 : dd->writes_starved = writes_starved;
365 0 : dd->front_merges = 1;
366 0 : dd->fifo_batch = fifo_batch;
367 0 : return dd;
368 : }
369 :
370 : /*
371 : * sysfs parts below
372 : */
373 :
374 : static ssize_t
375 : deadline_var_show(int var, char *page)
376 : {
377 15 : return sprintf(page, "%d\n", var);
378 : }
379 :
380 : static ssize_t
381 : deadline_var_store(int *var, const char *page, size_t count)
382 : {
383 10 : char *p = (char *) page;
384 5 :
385 10 : *var = simple_strtol(p, &p, 10);
386 5 : return count;
387 : }
388 :
389 : #define SHOW_FUNCTION(__FUNC, __VAR, __CONV) \
390 : static ssize_t __FUNC(struct elevator_queue *e, char *page) \
391 : { \
392 : struct deadline_data *dd = e->elevator_data; \
393 : int __data = __VAR; \
394 : if (__CONV) \
395 : __data = jiffies_to_msecs(__data); \
396 : return deadline_var_show(__data, (page)); \
397 : }
398 8 : SHOW_FUNCTION(deadline_read_expire_show, dd->fifo_expire[READ], 1);
399 8 : SHOW_FUNCTION(deadline_write_expire_show, dd->fifo_expire[WRITE], 1);
400 7 : SHOW_FUNCTION(deadline_writes_starved_show, dd->writes_starved, 0);
401 8 : SHOW_FUNCTION(deadline_front_merges_show, dd->front_merges, 0);
402 9 : SHOW_FUNCTION(deadline_fifo_batch_show, dd->fifo_batch, 0);
403 4 : #undef SHOW_FUNCTION
404 4 :
405 2 : #define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \
406 1 : static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) \
407 : { \
408 : struct deadline_data *dd = e->elevator_data; \
409 : int __data; \
410 : int ret = deadline_var_store(&__data, (page), count); \
411 : if (__data < (MIN)) \
412 : __data = (MIN); \
413 : else if (__data > (MAX)) \
414 : __data = (MAX); \
415 : if (__CONV) \
416 : *(__PTR) = msecs_to_jiffies(__data); \
417 : else \
418 : *(__PTR) = __data; \
419 : return ret; \
420 : }
421 11 : STORE_FUNCTION(deadline_read_expire_store, &dd->fifo_expire[READ], 0, INT_MAX, 1);
422 11 : STORE_FUNCTION(deadline_write_expire_store, &dd->fifo_expire[WRITE], 0, INT_MAX, 1);
423 7 : STORE_FUNCTION(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX, 0);
424 14 : STORE_FUNCTION(deadline_front_merges_store, &dd->front_merges, 0, 1, 0);
425 12 : STORE_FUNCTION(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX, 0);
426 3 : #undef STORE_FUNCTION
427 4 :
428 5 : #define DD_ATTR(name) \
429 4 : __ATTR(name, S_IRUGO|S_IWUSR, deadline_##name##_show, \
430 2 : deadline_##name##_store)
431 1 :
432 1 : static struct elv_fs_entry deadline_attrs[] = {
433 : DD_ATTR(read_expire),
434 : DD_ATTR(write_expire),
435 : DD_ATTR(writes_starved),
436 : DD_ATTR(front_merges),
437 : DD_ATTR(fifo_batch),
438 : __ATTR_NULL
439 : };
440 :
441 1 : static struct elevator_type iosched_deadline = {
442 : .ops = {
443 : .elevator_merge_fn = deadline_merge,
444 : .elevator_merged_fn = deadline_merged_request,
445 : .elevator_merge_req_fn = deadline_merged_requests,
446 : .elevator_dispatch_fn = deadline_dispatch_requests,
447 : .elevator_add_req_fn = deadline_add_request,
448 : .elevator_queue_empty_fn = deadline_queue_empty,
449 : .elevator_former_req_fn = elv_rb_former_request,
450 : .elevator_latter_req_fn = elv_rb_latter_request,
451 : .elevator_init_fn = deadline_init_queue,
452 : .elevator_exit_fn = deadline_exit_queue,
453 : },
454 :
455 : .elevator_attrs = deadline_attrs,
456 : .elevator_name = "deadline",
457 : .elevator_owner = THIS_MODULE,
458 : };
459 :
460 : static int __init deadline_init(void)
461 : {
462 1 : elv_register(&iosched_deadline);
463 :
464 1 : return 0;
465 : }
466 :
467 : static void __exit deadline_exit(void)
468 : {
469 2 : elv_unregister(&iosched_deadline);
470 2 : }
471 :
472 : module_init(deadline_init);
473 : module_exit(deadline_exit);
474 1 :
475 : MODULE_AUTHOR("Jens Axboe");
476 : MODULE_LICENSE("GPL");
477 : MODULE_DESCRIPTION("deadline IO scheduler");
|