LCOV - code coverage report
Current view: top level - block - deadline-iosched.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 33 175 18.9 %
Date: 2017-01-25 Functions: 14 30 46.7 %

          Line data    Source code
       1             : /*
       2             :  *  Deadline i/o scheduler.
       3             :  *
       4             :  *  Copyright (C) 2002 Jens Axboe <axboe@kernel.dk>
       5             :  */
       6             : #include <linux/kernel.h>
       7             : #include <linux/fs.h>
       8             : #include <linux/blkdev.h>
       9             : #include <linux/elevator.h>
      10             : #include <linux/bio.h>
      11             : #include <linux/module.h>
      12             : #include <linux/slab.h>
      13             : #include <linux/init.h>
      14             : #include <linux/compiler.h>
      15             : #include <linux/rbtree.h>
      16             : 
      17             : /*
      18             :  * See Documentation/block/deadline-iosched.txt
      19             :  */
      20             : static const int read_expire = HZ / 2;  /* max time before a read is submitted. */
      21             : static const int write_expire = 5 * HZ; /* ditto for writes, these limits are SOFT! */
      22             : static const int writes_starved = 2;    /* max times reads can starve a write */
      23             : static const int fifo_batch = 16;       /* # of sequential requests treated as one
      24           1 :                                      by the above parameters. For throughput. */
      25             : 
      26             : struct deadline_data {
      27             :         /*
      28             :          * run time data
      29             :          */
      30             : 
      31             :         /*
      32             :          * requests (deadline_rq s) are present on both sort_list and fifo_list
      33             :          */
      34             :         struct rb_root sort_list[2];    
      35             :         struct list_head fifo_list[2];
      36             : 
      37             :         /*
      38             :          * next in sort order. read, write or both are NULL
      39             :          */
      40             :         struct request *next_rq[2];
      41             :         unsigned int batching;          /* number of sequential requests made */
      42             :         sector_t last_sector;           /* head position */
      43             :         unsigned int starved;           /* times reads have starved writes */
      44             : 
      45             :         /*
      46             :          * settings that change how the i/o scheduler behaves
      47             :          */
      48             :         int fifo_expire[2];
      49             :         int fifo_batch;
      50             :         int writes_starved;
      51             :         int front_merges;
      52             : };
      53             : 
      54             : static void deadline_move_request(struct deadline_data *, struct request *);
      55             : 
      56             : static inline struct rb_root *
      57             : deadline_rb_root(struct deadline_data *dd, struct request *rq)
      58             : {
      59           0 :         return &dd->sort_list[rq_data_dir(rq)];
      60             : }
      61             : 
      62             : /*
      63             :  * get the request after `rq' in sector-sorted order
      64             :  */
      65             : static inline struct request *
      66             : deadline_latter_request(struct request *rq)
      67             : {
      68           0 :         struct rb_node *node = rb_next(&rq->rb_node);
      69           0 : 
      70           0 :         if (node)
      71           0 :                 return rb_entry_rq(node);
      72             : 
      73           0 :         return NULL;
      74             : }
      75             : 
      76             : static void
      77             : deadline_add_rq_rb(struct deadline_data *dd, struct request *rq)
      78             : {
      79           0 :         struct rb_root *root = deadline_rb_root(dd, rq);
      80           0 :         struct request *__alias;
      81           0 : 
      82           0 :         while (unlikely(__alias = elv_rb_add(root, rq)))
      83           0 :                 deadline_move_request(dd, __alias);
      84           0 : }
      85           0 : 
      86             : static inline void
      87             : deadline_del_rq_rb(struct deadline_data *dd, struct request *rq)
      88             : {
      89           0 :         const int data_dir = rq_data_dir(rq);
      90           0 : 
      91           0 :         if (dd->next_rq[data_dir] == rq)
      92           0 :                 dd->next_rq[data_dir] = deadline_latter_request(rq);
      93             : 
      94           0 :         elv_rb_del(deadline_rb_root(dd, rq), rq);
      95           0 : }
      96             : 
      97             : /*
      98             :  * add rq to rbtree and fifo
      99             :  */
     100             : static void
     101             : deadline_add_request(struct request_queue *q, struct request *rq)
     102             : {
     103           0 :         struct deadline_data *dd = q->elevator->elevator_data;
     104           0 :         const int data_dir = rq_data_dir(rq);
     105             : 
     106           0 :         deadline_add_rq_rb(dd, rq);
     107             : 
     108             :         /*
     109             :          * set expire time and add to fifo list
     110             :          */
     111           0 :         rq_set_fifo_time(rq, jiffies + dd->fifo_expire[data_dir]);
     112           0 :         list_add_tail(&rq->queuelist, &dd->fifo_list[data_dir]);
     113           0 : }
     114             : 
     115             : /*
     116             :  * remove rq from rbtree and fifo.
     117             :  */
     118             : static void deadline_remove_request(struct request_queue *q, struct request *rq)
     119             : {
     120           0 :         struct deadline_data *dd = q->elevator->elevator_data;
     121             : 
     122           0 :         rq_fifo_clear(rq);
     123           0 :         deadline_del_rq_rb(dd, rq);
     124           0 : }
     125             : 
     126             : static int
     127             : deadline_merge(struct request_queue *q, struct request **req, struct bio *bio)
     128             : {
     129           0 :         struct deadline_data *dd = q->elevator->elevator_data;
     130           0 :         struct request *__rq;
     131           0 :         int ret;
     132           0 : 
     133           0 :         /*
     134           0 :          * check for front merge
     135           0 :          */
     136           0 :         if (dd->front_merges) {
     137           0 :                 sector_t sector = bio->bi_sector + bio_sectors(bio);
     138             : 
     139           0 :                 __rq = elv_rb_find(&dd->sort_list[bio_data_dir(bio)], sector);
     140           0 :                 if (__rq) {
     141           0 :                         BUG_ON(sector != blk_rq_pos(__rq));
     142             : 
     143           0 :                         if (elv_rq_merge_ok(__rq, bio)) {
     144           0 :                                 ret = ELEVATOR_FRONT_MERGE;
     145           0 :                                 goto out;
     146             :                         }
     147             :                 }
     148             :         }
     149             : 
     150           0 :         return ELEVATOR_NO_MERGE;
     151           0 : out:
     152           0 :         *req = __rq;
     153           0 :         return ret;
     154             : }
     155             : 
     156             : static void deadline_merged_request(struct request_queue *q,
     157             :                                     struct request *req, int type)
     158             : {
     159           0 :         struct deadline_data *dd = q->elevator->elevator_data;
     160           0 : 
     161             :         /*
     162             :          * if the merge was a front merge, we need to reposition request
     163             :          */
     164           0 :         if (type == ELEVATOR_FRONT_MERGE) {
     165           0 :                 elv_rb_del(deadline_rb_root(dd, req), req);
     166           0 :                 deadline_add_rq_rb(dd, req);
     167             :         }
     168           0 : }
     169             : 
     170             : static void
     171             : deadline_merged_requests(struct request_queue *q, struct request *req,
     172             :                          struct request *next)
     173             : {
     174           0 :         /*
     175           0 :          * if next expires before rq, assign its expire time to rq
     176             :          * and move into next position (next will be deleted) in fifo
     177             :          */
     178           0 :         if (!list_empty(&req->queuelist) && !list_empty(&next->queuelist)) {
     179           0 :                 if (time_before(rq_fifo_time(next), rq_fifo_time(req))) {
     180           0 :                         list_move(&req->queuelist, &next->queuelist);
     181           0 :                         rq_set_fifo_time(req, rq_fifo_time(next));
     182             :                 }
     183             :         }
     184             : 
     185             :         /*
     186             :          * kill knowledge of next, this one is a goner
     187             :          */
     188           0 :         deadline_remove_request(q, next);
     189           0 : }
     190             : 
     191             : /*
     192             :  * move request from sort list to dispatch queue.
     193             :  */
     194             : static inline void
     195             : deadline_move_to_dispatch(struct deadline_data *dd, struct request *rq)
     196             : {
     197           0 :         struct request_queue *q = rq->q;
     198             : 
     199           0 :         deadline_remove_request(q, rq);
     200           0 :         elv_dispatch_add_tail(q, rq);
     201           0 : }
     202             : 
     203             : /*
     204             :  * move an entry to dispatch queue
     205             :  */
     206             : static void
     207             : deadline_move_request(struct deadline_data *dd, struct request *rq)
     208             : {
     209           0 :         const int data_dir = rq_data_dir(rq);
     210           0 : 
     211           0 :         dd->next_rq[READ] = NULL;
     212           0 :         dd->next_rq[WRITE] = NULL;
     213           0 :         dd->next_rq[data_dir] = deadline_latter_request(rq);
     214             : 
     215           0 :         dd->last_sector = rq_end_sector(rq);
     216             : 
     217             :         /*
     218             :          * take it off the sort and fifo list, move
     219             :          * to dispatch queue
     220             :          */
     221           0 :         deadline_move_to_dispatch(dd, rq);
     222           0 : }
     223             : 
     224             : /*
     225             :  * deadline_check_fifo returns 0 if there are no expired requests on the fifo,
     226             :  * 1 otherwise. Requires !list_empty(&dd->fifo_list[data_dir])
     227             :  */
     228             : static inline int deadline_check_fifo(struct deadline_data *dd, int ddir)
     229             : {
     230           0 :         struct request *rq = rq_entry_fifo(dd->fifo_list[ddir].next);
     231           0 : 
     232             :         /*
     233             :          * rq is expired!
     234             :          */
     235           0 :         if (time_after(jiffies, rq_fifo_time(rq)))
     236           0 :                 return 1;
     237             : 
     238           0 :         return 0;
     239             : }
     240             : 
     241             : /*
     242             :  * deadline_dispatch_requests selects the best request according to
     243             :  * read/write expire, fifo_batch, etc
     244             :  */
     245             : static int deadline_dispatch_requests(struct request_queue *q, int force)
     246             : {
     247           0 :         struct deadline_data *dd = q->elevator->elevator_data;
     248           0 :         const int reads = !list_empty(&dd->fifo_list[READ]);
     249           0 :         const int writes = !list_empty(&dd->fifo_list[WRITE]);
     250           0 :         struct request *rq;
     251           0 :         int data_dir;
     252           0 : 
     253           0 :         /*
     254           0 :          * batches are currently reads XOR writes
     255           0 :          */
     256           0 :         if (dd->next_rq[WRITE])
     257           0 :                 rq = dd->next_rq[WRITE];
     258           0 :         else
     259           0 :                 rq = dd->next_rq[READ];
     260             : 
     261           0 :         if (rq && dd->batching < dd->fifo_batch)
     262             :                 /* we have a next request are still entitled to batch */
     263           0 :                 goto dispatch_request;
     264             : 
     265             :         /*
     266             :          * at this point we are not running a batch. select the appropriate
     267             :          * data direction (read / write)
     268             :          */
     269             : 
     270           0 :         if (reads) {
     271           0 :                 BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[READ]));
     272             : 
     273           0 :                 if (writes && (dd->starved++ >= dd->writes_starved))
     274           0 :                         goto dispatch_writes;
     275             : 
     276           0 :                 data_dir = READ;
     277             : 
     278           0 :                 goto dispatch_find_request;
     279             :         }
     280             : 
     281             :         /*
     282             :          * there are either no reads or writes have been starved
     283             :          */
     284             : 
     285           0 :         if (writes) {
     286           0 : dispatch_writes:
     287           0 :                 BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[WRITE]));
     288             : 
     289           0 :                 dd->starved = 0;
     290             : 
     291           0 :                 data_dir = WRITE;
     292             : 
     293           0 :                 goto dispatch_find_request;
     294             :         }
     295             : 
     296           0 :         return 0;
     297           0 : 
     298             : dispatch_find_request:
     299             :         /*
     300             :          * we are not running a batch, find best request for selected data_dir
     301             :          */
     302           0 :         if (deadline_check_fifo(dd, data_dir) || !dd->next_rq[data_dir]) {
     303             :                 /*
     304             :                  * A deadline has expired, the last request was in the other
     305             :                  * direction, or we have run out of higher-sectored requests.
     306             :                  * Start again from the request with the earliest expiry time.
     307             :                  */
     308           0 :                 rq = rq_entry_fifo(dd->fifo_list[data_dir].next);
     309             :         } else {
     310             :                 /*
     311             :                  * The last req was the same dir and we have a next request in
     312             :                  * sort order. No expired requests so continue on from here.
     313             :                  */
     314           0 :                 rq = dd->next_rq[data_dir];
     315             :         }
     316             : 
     317           0 :         dd->batching = 0;
     318           0 : 
     319             : dispatch_request:
     320             :         /*
     321             :          * rq is the selected appropriate request.
     322             :          */
     323           0 :         dd->batching++;
     324           0 :         deadline_move_request(dd, rq);
     325             : 
     326           0 :         return 1;
     327             : }
     328             : 
     329             : static int deadline_queue_empty(struct request_queue *q)
     330             : {
     331           0 :         struct deadline_data *dd = q->elevator->elevator_data;
     332           0 : 
     333           0 :         return list_empty(&dd->fifo_list[WRITE])
     334           0 :                 && list_empty(&dd->fifo_list[READ]);
     335             : }
     336             : 
     337             : static void deadline_exit_queue(struct elevator_queue *e)
     338             : {
     339           0 :         struct deadline_data *dd = e->elevator_data;
     340           0 : 
     341           0 :         BUG_ON(!list_empty(&dd->fifo_list[READ]));
     342           0 :         BUG_ON(!list_empty(&dd->fifo_list[WRITE]));
     343           0 : 
     344           0 :         kfree(dd);
     345           0 : }
     346             : 
     347             : /*
     348             :  * initialize elevator private data (deadline_data).
     349             :  */
     350             : static void *deadline_init_queue(struct request_queue *q)
     351             : {
     352           0 :         struct deadline_data *dd;
     353           0 : 
     354           0 :         dd = kmalloc_node(sizeof(*dd), GFP_KERNEL | __GFP_ZERO, q->node);
     355           0 :         if (!dd)
     356           0 :                 return NULL;
     357             : 
     358           0 :         INIT_LIST_HEAD(&dd->fifo_list[READ]);
     359           0 :         INIT_LIST_HEAD(&dd->fifo_list[WRITE]);
     360           0 :         dd->sort_list[READ] = RB_ROOT;
     361           0 :         dd->sort_list[WRITE] = RB_ROOT;
     362           0 :         dd->fifo_expire[READ] = read_expire;
     363           0 :         dd->fifo_expire[WRITE] = write_expire;
     364           0 :         dd->writes_starved = writes_starved;
     365           0 :         dd->front_merges = 1;
     366           0 :         dd->fifo_batch = fifo_batch;
     367           0 :         return dd;
     368             : }
     369             : 
     370             : /*
     371             :  * sysfs parts below
     372             :  */
     373             : 
     374             : static ssize_t
     375             : deadline_var_show(int var, char *page)
     376             : {
     377          15 :         return sprintf(page, "%d\n", var);
     378             : }
     379             : 
     380             : static ssize_t
     381             : deadline_var_store(int *var, const char *page, size_t count)
     382             : {
     383          10 :         char *p = (char *) page;
     384           5 : 
     385          10 :         *var = simple_strtol(p, &p, 10);
     386           5 :         return count;
     387             : }
     388             : 
     389             : #define SHOW_FUNCTION(__FUNC, __VAR, __CONV)                            \
     390             : static ssize_t __FUNC(struct elevator_queue *e, char *page)             \
     391             : {                                                                       \
     392             :         struct deadline_data *dd = e->elevator_data;                 \
     393             :         int __data = __VAR;                                             \
     394             :         if (__CONV)                                                     \
     395             :                 __data = jiffies_to_msecs(__data);                      \
     396             :         return deadline_var_show(__data, (page));                       \
     397             : }
     398           8 : SHOW_FUNCTION(deadline_read_expire_show, dd->fifo_expire[READ], 1);
     399           8 : SHOW_FUNCTION(deadline_write_expire_show, dd->fifo_expire[WRITE], 1);
     400           7 : SHOW_FUNCTION(deadline_writes_starved_show, dd->writes_starved, 0);
     401           8 : SHOW_FUNCTION(deadline_front_merges_show, dd->front_merges, 0);
     402           9 : SHOW_FUNCTION(deadline_fifo_batch_show, dd->fifo_batch, 0);
     403           4 : #undef SHOW_FUNCTION
     404           4 : 
     405           2 : #define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV)                 \
     406           1 : static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) \
     407             : {                                                                       \
     408             :         struct deadline_data *dd = e->elevator_data;                 \
     409             :         int __data;                                                     \
     410             :         int ret = deadline_var_store(&__data, (page), count);               \
     411             :         if (__data < (MIN))                                          \
     412             :                 __data = (MIN);                                         \
     413             :         else if (__data > (MAX))                                     \
     414             :                 __data = (MAX);                                         \
     415             :         if (__CONV)                                                     \
     416             :                 *(__PTR) = msecs_to_jiffies(__data);                    \
     417             :         else                                                            \
     418             :                 *(__PTR) = __data;                                      \
     419             :         return ret;                                                     \
     420             : }
     421          11 : STORE_FUNCTION(deadline_read_expire_store, &dd->fifo_expire[READ], 0, INT_MAX, 1);
     422          11 : STORE_FUNCTION(deadline_write_expire_store, &dd->fifo_expire[WRITE], 0, INT_MAX, 1);
     423           7 : STORE_FUNCTION(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX, 0);
     424          14 : STORE_FUNCTION(deadline_front_merges_store, &dd->front_merges, 0, 1, 0);
     425          12 : STORE_FUNCTION(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX, 0);
     426           3 : #undef STORE_FUNCTION
     427           4 : 
     428           5 : #define DD_ATTR(name) \
     429           4 :         __ATTR(name, S_IRUGO|S_IWUSR, deadline_##name##_show, \
     430           2 :                                       deadline_##name##_store)
     431           1 : 
     432           1 : static struct elv_fs_entry deadline_attrs[] = {
     433             :         DD_ATTR(read_expire),
     434             :         DD_ATTR(write_expire),
     435             :         DD_ATTR(writes_starved),
     436             :         DD_ATTR(front_merges),
     437             :         DD_ATTR(fifo_batch),
     438             :         __ATTR_NULL
     439             : };
     440             : 
     441           1 : static struct elevator_type iosched_deadline = {
     442             :         .ops = {
     443             :                 .elevator_merge_fn =            deadline_merge,
     444             :                 .elevator_merged_fn =           deadline_merged_request,
     445             :                 .elevator_merge_req_fn =        deadline_merged_requests,
     446             :                 .elevator_dispatch_fn =         deadline_dispatch_requests,
     447             :                 .elevator_add_req_fn =          deadline_add_request,
     448             :                 .elevator_queue_empty_fn =      deadline_queue_empty,
     449             :                 .elevator_former_req_fn =       elv_rb_former_request,
     450             :                 .elevator_latter_req_fn =       elv_rb_latter_request,
     451             :                 .elevator_init_fn =             deadline_init_queue,
     452             :                 .elevator_exit_fn =             deadline_exit_queue,
     453             :         },
     454             : 
     455             :         .elevator_attrs = deadline_attrs,
     456             :         .elevator_name = "deadline",
     457             :         .elevator_owner = THIS_MODULE,
     458             : };
     459             : 
     460             : static int __init deadline_init(void)
     461             : {
     462           1 :         elv_register(&iosched_deadline);
     463             : 
     464           1 :         return 0;
     465             : }
     466             : 
     467             : static void __exit deadline_exit(void)
     468             : {
     469           2 :         elv_unregister(&iosched_deadline);
     470           2 : }
     471             : 
     472             : module_init(deadline_init);
     473             : module_exit(deadline_exit);
     474           1 : 
     475             : MODULE_AUTHOR("Jens Axboe");
     476             : MODULE_LICENSE("GPL");
     477             : MODULE_DESCRIPTION("deadline IO scheduler");

Generated by: LCOV version 1.10