00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00038 #include "libblock.h"
00039 #include "../../srv/vfs/vfs.h"
00040 #include <ipc/devmap.h>
00041 #include <ipc/bd.h>
00042 #include <ipc/services.h>
00043 #include <errno.h>
00044 #include <sys/mman.h>
00045 #include <async.h>
00046 #include <as.h>
00047 #include <assert.h>
00048 #include <fibril_synch.h>
00049 #include <adt/list.h>
00050 #include <adt/hash_table.h>
00051 #include <macros.h>
00052 #include <mem.h>
00053 #include <malloc.h>
00054 #include <stdio.h>
00055 #include <sys/typefmt.h>
00056 #include <stacktrace.h>
00057
00059 static FIBRIL_MUTEX_INITIALIZE(dcl_lock);
00061 static LIST_INITIALIZE(dcl_head);
00062
00063 #define CACHE_BUCKETS_LOG2 10
00064 #define CACHE_BUCKETS (1 << CACHE_BUCKETS_LOG2)
00065
00066 typedef struct {
00067 fibril_mutex_t lock;
00068 size_t lblock_size;
00069 unsigned blocks_cluster;
00070 unsigned block_count;
00071 unsigned blocks_cached;
00072 hash_table_t block_hash;
00073 link_t free_head;
00074 enum cache_mode mode;
00075 } cache_t;
00076
00077 typedef struct {
00078 link_t link;
00079 devmap_handle_t devmap_handle;
00080 int dev_phone;
00081 fibril_mutex_t comm_area_lock;
00082 void *comm_area;
00083 size_t comm_size;
00084 void *bb_buf;
00085 aoff64_t bb_addr;
00086 size_t pblock_size;
00087 cache_t *cache;
00088 } devcon_t;
00089
00090 static int read_blocks(devcon_t *devcon, aoff64_t ba, size_t cnt);
00091 static int write_blocks(devcon_t *devcon, aoff64_t ba, size_t cnt);
00092 static int get_block_size(int dev_phone, size_t *bsize);
00093 static int get_num_blocks(int dev_phone, aoff64_t *nblocks);
00094 static aoff64_t ba_ltop(devcon_t *devcon, aoff64_t lba);
00095
00096 static devcon_t *devcon_search(devmap_handle_t devmap_handle)
00097 {
00098 link_t *cur;
00099
00100 fibril_mutex_lock(&dcl_lock);
00101 for (cur = dcl_head.next; cur != &dcl_head; cur = cur->next) {
00102 devcon_t *devcon = list_get_instance(cur, devcon_t, link);
00103 if (devcon->devmap_handle == devmap_handle) {
00104 fibril_mutex_unlock(&dcl_lock);
00105 return devcon;
00106 }
00107 }
00108 fibril_mutex_unlock(&dcl_lock);
00109 return NULL;
00110 }
00111
00112 static int devcon_add(devmap_handle_t devmap_handle, int dev_phone, size_t bsize,
00113 void *comm_area, size_t comm_size)
00114 {
00115 link_t *cur;
00116 devcon_t *devcon;
00117
00118 if (comm_size < bsize)
00119 return EINVAL;
00120
00121 devcon = malloc(sizeof(devcon_t));
00122 if (!devcon)
00123 return ENOMEM;
00124
00125 link_initialize(&devcon->link);
00126 devcon->devmap_handle = devmap_handle;
00127 devcon->dev_phone = dev_phone;
00128 fibril_mutex_initialize(&devcon->comm_area_lock);
00129 devcon->comm_area = comm_area;
00130 devcon->comm_size = comm_size;
00131 devcon->bb_buf = NULL;
00132 devcon->bb_addr = 0;
00133 devcon->pblock_size = bsize;
00134 devcon->cache = NULL;
00135
00136 fibril_mutex_lock(&dcl_lock);
00137 for (cur = dcl_head.next; cur != &dcl_head; cur = cur->next) {
00138 devcon_t *d = list_get_instance(cur, devcon_t, link);
00139 if (d->devmap_handle == devmap_handle) {
00140 fibril_mutex_unlock(&dcl_lock);
00141 free(devcon);
00142 return EEXIST;
00143 }
00144 }
00145 list_append(&devcon->link, &dcl_head);
00146 fibril_mutex_unlock(&dcl_lock);
00147 return EOK;
00148 }
00149
00150 static void devcon_remove(devcon_t *devcon)
00151 {
00152 fibril_mutex_lock(&dcl_lock);
00153 list_remove(&devcon->link);
00154 fibril_mutex_unlock(&dcl_lock);
00155 }
00156
00157 int block_init(devmap_handle_t devmap_handle, size_t comm_size)
00158 {
00159 int rc;
00160 int dev_phone;
00161 void *comm_area;
00162 size_t bsize;
00163
00164 comm_area = mmap(NULL, comm_size, PROTO_READ | PROTO_WRITE,
00165 MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
00166 if (!comm_area) {
00167 return ENOMEM;
00168 }
00169
00170 dev_phone = devmap_device_connect(devmap_handle, IPC_FLAG_BLOCKING);
00171 if (dev_phone < 0) {
00172 munmap(comm_area, comm_size);
00173 return dev_phone;
00174 }
00175
00176 rc = async_share_out_start(dev_phone, comm_area,
00177 AS_AREA_READ | AS_AREA_WRITE);
00178 if (rc != EOK) {
00179 munmap(comm_area, comm_size);
00180 async_hangup(dev_phone);
00181 return rc;
00182 }
00183
00184 if (get_block_size(dev_phone, &bsize) != EOK) {
00185 munmap(comm_area, comm_size);
00186 async_hangup(dev_phone);
00187 return rc;
00188 }
00189
00190 rc = devcon_add(devmap_handle, dev_phone, bsize, comm_area, comm_size);
00191 if (rc != EOK) {
00192 munmap(comm_area, comm_size);
00193 async_hangup(dev_phone);
00194 return rc;
00195 }
00196
00197 return EOK;
00198 }
00199
00200 void block_fini(devmap_handle_t devmap_handle)
00201 {
00202 devcon_t *devcon = devcon_search(devmap_handle);
00203 assert(devcon);
00204
00205 if (devcon->cache)
00206 (void) block_cache_fini(devmap_handle);
00207
00208 devcon_remove(devcon);
00209
00210 if (devcon->bb_buf)
00211 free(devcon->bb_buf);
00212
00213 munmap(devcon->comm_area, devcon->comm_size);
00214 async_hangup(devcon->dev_phone);
00215
00216 free(devcon);
00217 }
00218
00219 int block_bb_read(devmap_handle_t devmap_handle, aoff64_t ba)
00220 {
00221 void *bb_buf;
00222 int rc;
00223
00224 devcon_t *devcon = devcon_search(devmap_handle);
00225 if (!devcon)
00226 return ENOENT;
00227 if (devcon->bb_buf)
00228 return EEXIST;
00229 bb_buf = malloc(devcon->pblock_size);
00230 if (!bb_buf)
00231 return ENOMEM;
00232
00233 fibril_mutex_lock(&devcon->comm_area_lock);
00234 rc = read_blocks(devcon, 0, 1);
00235 if (rc != EOK) {
00236 fibril_mutex_unlock(&devcon->comm_area_lock);
00237 free(bb_buf);
00238 return rc;
00239 }
00240 memcpy(bb_buf, devcon->comm_area, devcon->pblock_size);
00241 fibril_mutex_unlock(&devcon->comm_area_lock);
00242
00243 devcon->bb_buf = bb_buf;
00244 devcon->bb_addr = ba;
00245
00246 return EOK;
00247 }
00248
00249 void *block_bb_get(devmap_handle_t devmap_handle)
00250 {
00251 devcon_t *devcon = devcon_search(devmap_handle);
00252 assert(devcon);
00253 return devcon->bb_buf;
00254 }
00255
00256 static hash_index_t cache_hash(unsigned long *key)
00257 {
00258 return *key & (CACHE_BUCKETS - 1);
00259 }
00260
00261 static int cache_compare(unsigned long *key, hash_count_t keys, link_t *item)
00262 {
00263 block_t *b = hash_table_get_instance(item, block_t, hash_link);
00264 return b->lba == *key;
00265 }
00266
00267 static void cache_remove_callback(link_t *item)
00268 {
00269 }
00270
00271 static hash_table_operations_t cache_ops = {
00272 .hash = cache_hash,
00273 .compare = cache_compare,
00274 .remove_callback = cache_remove_callback
00275 };
00276
00277 int block_cache_init(devmap_handle_t devmap_handle, size_t size, unsigned blocks,
00278 enum cache_mode mode)
00279 {
00280 devcon_t *devcon = devcon_search(devmap_handle);
00281 cache_t *cache;
00282 if (!devcon)
00283 return ENOENT;
00284 if (devcon->cache)
00285 return EEXIST;
00286 cache = malloc(sizeof(cache_t));
00287 if (!cache)
00288 return ENOMEM;
00289
00290 fibril_mutex_initialize(&cache->lock);
00291 list_initialize(&cache->free_head);
00292 cache->lblock_size = size;
00293 cache->block_count = blocks;
00294 cache->blocks_cached = 0;
00295 cache->mode = mode;
00296
00297
00298 if (cache->lblock_size % devcon->pblock_size != 0) {
00299 free(cache);
00300 return ENOTSUP;
00301 }
00302
00303 cache->blocks_cluster = cache->lblock_size / devcon->pblock_size;
00304
00305 if (!hash_table_create(&cache->block_hash, CACHE_BUCKETS, 1,
00306 &cache_ops)) {
00307 free(cache);
00308 return ENOMEM;
00309 }
00310
00311 devcon->cache = cache;
00312 return EOK;
00313 }
00314
00315 int block_cache_fini(devmap_handle_t devmap_handle)
00316 {
00317 devcon_t *devcon = devcon_search(devmap_handle);
00318 cache_t *cache;
00319 int rc;
00320
00321 if (!devcon)
00322 return ENOENT;
00323 if (!devcon->cache)
00324 return EOK;
00325 cache = devcon->cache;
00326
00327
00328
00329
00330
00331
00332 while (!list_empty(&cache->free_head)) {
00333 block_t *b = list_get_instance(cache->free_head.next,
00334 block_t, free_link);
00335
00336 list_remove(&b->free_link);
00337 if (b->dirty) {
00338 memcpy(devcon->comm_area, b->data, b->size);
00339 rc = write_blocks(devcon, b->pba, cache->blocks_cluster);
00340 if (rc != EOK)
00341 return rc;
00342 }
00343
00344 unsigned long key = b->lba;
00345 hash_table_remove(&cache->block_hash, &key, 1);
00346
00347 free(b->data);
00348 free(b);
00349 }
00350
00351 hash_table_destroy(&cache->block_hash);
00352 devcon->cache = NULL;
00353 free(cache);
00354
00355 return EOK;
00356 }
00357
00358 #define CACHE_LO_WATERMARK 10
00359 #define CACHE_HI_WATERMARK 20
00360 static bool cache_can_grow(cache_t *cache)
00361 {
00362 if (cache->blocks_cached < CACHE_LO_WATERMARK)
00363 return true;
00364 if (!list_empty(&cache->free_head))
00365 return false;
00366 return true;
00367 }
00368
00369 static void block_initialize(block_t *b)
00370 {
00371 fibril_mutex_initialize(&b->lock);
00372 b->refcnt = 1;
00373 b->dirty = false;
00374 b->toxic = false;
00375 fibril_rwlock_initialize(&b->contents_lock);
00376 link_initialize(&b->free_link);
00377 link_initialize(&b->hash_link);
00378 }
00379
00392 int block_get(block_t **block, devmap_handle_t devmap_handle, aoff64_t ba, int flags)
00393 {
00394 devcon_t *devcon;
00395 cache_t *cache;
00396 block_t *b;
00397 link_t *l;
00398 unsigned long key = ba;
00399 int rc;
00400
00401 devcon = devcon_search(devmap_handle);
00402
00403 assert(devcon);
00404 assert(devcon->cache);
00405
00406 cache = devcon->cache;
00407
00408 retry:
00409 rc = EOK;
00410 b = NULL;
00411
00412 fibril_mutex_lock(&cache->lock);
00413 l = hash_table_find(&cache->block_hash, &key);
00414 if (l) {
00415 found:
00416
00417
00418
00419 b = hash_table_get_instance(l, block_t, hash_link);
00420 fibril_mutex_lock(&b->lock);
00421 if (b->refcnt++ == 0)
00422 list_remove(&b->free_link);
00423 if (b->toxic)
00424 rc = EIO;
00425 fibril_mutex_unlock(&b->lock);
00426 fibril_mutex_unlock(&cache->lock);
00427 } else {
00428
00429
00430
00431 if (cache_can_grow(cache)) {
00432
00433
00434
00435
00436
00437 b = malloc(sizeof(block_t));
00438 if (!b)
00439 goto recycle;
00440 b->data = malloc(cache->lblock_size);
00441 if (!b->data) {
00442 free(b);
00443 b = NULL;
00444 goto recycle;
00445 }
00446 cache->blocks_cached++;
00447 } else {
00448
00449
00450
00451 unsigned long temp_key;
00452 recycle:
00453 if (list_empty(&cache->free_head)) {
00454 fibril_mutex_unlock(&cache->lock);
00455 rc = ENOMEM;
00456 goto out;
00457 }
00458 l = cache->free_head.next;
00459 b = list_get_instance(l, block_t, free_link);
00460
00461 fibril_mutex_lock(&b->lock);
00462 if (b->dirty) {
00463
00464
00465
00466
00467
00468
00469
00470
00471
00472 list_remove(&b->free_link);
00473 list_append(&b->free_link, &cache->free_head);
00474 fibril_mutex_unlock(&cache->lock);
00475 fibril_mutex_lock(&devcon->comm_area_lock);
00476 memcpy(devcon->comm_area, b->data, b->size);
00477 rc = write_blocks(devcon, b->pba,
00478 cache->blocks_cluster);
00479 fibril_mutex_unlock(&devcon->comm_area_lock);
00480 if (rc != EOK) {
00481
00482
00483
00484
00485
00486
00487 fibril_mutex_unlock(&b->lock);
00488 goto retry;
00489 }
00490 b->dirty = false;
00491 if (!fibril_mutex_trylock(&cache->lock)) {
00492
00493
00494
00495
00496 fibril_mutex_unlock(&b->lock);
00497 goto retry;
00498 }
00499 l = hash_table_find(&cache->block_hash, &key);
00500 if (l) {
00501
00502
00503
00504
00505
00506
00507
00508
00509
00510 fibril_mutex_unlock(&b->lock);
00511 goto found;
00512 }
00513
00514 }
00515 fibril_mutex_unlock(&b->lock);
00516
00517
00518
00519
00520
00521 list_remove(&b->free_link);
00522 temp_key = b->lba;
00523 hash_table_remove(&cache->block_hash, &temp_key, 1);
00524 }
00525
00526 block_initialize(b);
00527 b->devmap_handle = devmap_handle;
00528 b->size = cache->lblock_size;
00529 b->lba = ba;
00530 b->pba = ba_ltop(devcon, b->lba);
00531 hash_table_insert(&cache->block_hash, &key, &b->hash_link);
00532
00533
00534
00535
00536
00537
00538 fibril_mutex_lock(&b->lock);
00539 fibril_mutex_unlock(&cache->lock);
00540
00541 if (!(flags & BLOCK_FLAGS_NOREAD)) {
00542
00543
00544
00545
00546 fibril_mutex_lock(&devcon->comm_area_lock);
00547 rc = read_blocks(devcon, b->pba, cache->blocks_cluster);
00548 memcpy(b->data, devcon->comm_area, cache->lblock_size);
00549 fibril_mutex_unlock(&devcon->comm_area_lock);
00550 if (rc != EOK)
00551 b->toxic = true;
00552 } else
00553 rc = EOK;
00554
00555 fibril_mutex_unlock(&b->lock);
00556 }
00557 out:
00558 if ((rc != EOK) && b) {
00559 assert(b->toxic);
00560 (void) block_put(b);
00561 b = NULL;
00562 }
00563 *block = b;
00564 return rc;
00565 }
00566
00575 int block_put(block_t *block)
00576 {
00577 devcon_t *devcon = devcon_search(block->devmap_handle);
00578 cache_t *cache;
00579 unsigned blocks_cached;
00580 enum cache_mode mode;
00581 int rc = EOK;
00582
00583 assert(devcon);
00584 assert(devcon->cache);
00585 assert(block->refcnt >= 1);
00586
00587 cache = devcon->cache;
00588
00589 retry:
00590 fibril_mutex_lock(&cache->lock);
00591 blocks_cached = cache->blocks_cached;
00592 mode = cache->mode;
00593 fibril_mutex_unlock(&cache->lock);
00594
00595
00596
00597
00598
00599
00600
00601
00602 fibril_mutex_lock(&block->lock);
00603 if (block->toxic)
00604 block->dirty = false;
00605 if (block->dirty && (block->refcnt == 1) &&
00606 (blocks_cached > CACHE_HI_WATERMARK || mode != CACHE_MODE_WB)) {
00607 fibril_mutex_lock(&devcon->comm_area_lock);
00608 memcpy(devcon->comm_area, block->data, block->size);
00609 rc = write_blocks(devcon, block->pba, cache->blocks_cluster);
00610 fibril_mutex_unlock(&devcon->comm_area_lock);
00611 block->dirty = false;
00612 }
00613 fibril_mutex_unlock(&block->lock);
00614
00615 fibril_mutex_lock(&cache->lock);
00616 fibril_mutex_lock(&block->lock);
00617 if (!--block->refcnt) {
00618
00619
00620
00621
00622
00623 if ((cache->blocks_cached > CACHE_HI_WATERMARK) ||
00624 (rc != EOK)) {
00625
00626
00627
00628
00629
00630 if (block->dirty) {
00631
00632
00633
00634
00635 block->refcnt++;
00636 fibril_mutex_unlock(&block->lock);
00637 fibril_mutex_unlock(&cache->lock);
00638 goto retry;
00639 }
00640
00641
00642
00643 unsigned long key = block->lba;
00644 hash_table_remove(&cache->block_hash, &key, 1);
00645 fibril_mutex_unlock(&block->lock);
00646 free(block->data);
00647 free(block);
00648 cache->blocks_cached--;
00649 fibril_mutex_unlock(&cache->lock);
00650 return rc;
00651 }
00652
00653
00654
00655 if (cache->mode != CACHE_MODE_WB && block->dirty) {
00656
00657
00658
00659
00660 block->refcnt++;
00661 fibril_mutex_unlock(&block->lock);
00662 fibril_mutex_unlock(&cache->lock);
00663 goto retry;
00664 }
00665 list_append(&block->free_link, &cache->free_head);
00666 }
00667 fibril_mutex_unlock(&block->lock);
00668 fibril_mutex_unlock(&cache->lock);
00669
00670 return rc;
00671 }
00672
00687 int block_seqread(devmap_handle_t devmap_handle, size_t *bufpos, size_t *buflen,
00688 aoff64_t *pos, void *dst, size_t size)
00689 {
00690 size_t offset = 0;
00691 size_t left = size;
00692 size_t block_size;
00693 devcon_t *devcon;
00694
00695 devcon = devcon_search(devmap_handle);
00696 assert(devcon);
00697 block_size = devcon->pblock_size;
00698
00699 fibril_mutex_lock(&devcon->comm_area_lock);
00700 while (left > 0) {
00701 size_t rd;
00702
00703 if (*bufpos + left < *buflen)
00704 rd = left;
00705 else
00706 rd = *buflen - *bufpos;
00707
00708 if (rd > 0) {
00709
00710
00711
00712
00713 memcpy(dst + offset, devcon->comm_area + *bufpos, rd);
00714 offset += rd;
00715 *bufpos += rd;
00716 *pos += rd;
00717 left -= rd;
00718 }
00719
00720 if (*bufpos == *buflen) {
00721
00722 int rc;
00723
00724 rc = read_blocks(devcon, *pos / block_size, 1);
00725 if (rc != EOK) {
00726 fibril_mutex_unlock(&devcon->comm_area_lock);
00727 return rc;
00728 }
00729
00730 *bufpos = 0;
00731 *buflen = block_size;
00732 }
00733 }
00734 fibril_mutex_unlock(&devcon->comm_area_lock);
00735
00736 return EOK;
00737 }
00738
00748 int block_read_direct(devmap_handle_t devmap_handle, aoff64_t ba, size_t cnt, void *buf)
00749 {
00750 devcon_t *devcon;
00751 int rc;
00752
00753 devcon = devcon_search(devmap_handle);
00754 assert(devcon);
00755
00756 fibril_mutex_lock(&devcon->comm_area_lock);
00757
00758 rc = read_blocks(devcon, ba, cnt);
00759 if (rc == EOK)
00760 memcpy(buf, devcon->comm_area, devcon->pblock_size * cnt);
00761
00762 fibril_mutex_unlock(&devcon->comm_area_lock);
00763
00764 return rc;
00765 }
00766
00776 int block_write_direct(devmap_handle_t devmap_handle, aoff64_t ba, size_t cnt,
00777 const void *data)
00778 {
00779 devcon_t *devcon;
00780 int rc;
00781
00782 devcon = devcon_search(devmap_handle);
00783 assert(devcon);
00784
00785 fibril_mutex_lock(&devcon->comm_area_lock);
00786
00787 memcpy(devcon->comm_area, data, devcon->pblock_size * cnt);
00788 rc = write_blocks(devcon, ba, cnt);
00789
00790 fibril_mutex_unlock(&devcon->comm_area_lock);
00791
00792 return rc;
00793 }
00794
00802 int block_get_bsize(devmap_handle_t devmap_handle, size_t *bsize)
00803 {
00804 devcon_t *devcon;
00805
00806 devcon = devcon_search(devmap_handle);
00807 assert(devcon);
00808
00809 return get_block_size(devcon->dev_phone, bsize);
00810 }
00811
00819 int block_get_nblocks(devmap_handle_t devmap_handle, aoff64_t *nblocks)
00820 {
00821 devcon_t *devcon;
00822
00823 devcon = devcon_search(devmap_handle);
00824 assert(devcon);
00825
00826 return get_num_blocks(devcon->dev_phone, nblocks);
00827 }
00828
00838 static int read_blocks(devcon_t *devcon, aoff64_t ba, size_t cnt)
00839 {
00840 int rc;
00841
00842 assert(devcon);
00843 rc = async_req_3_0(devcon->dev_phone, BD_READ_BLOCKS, LOWER32(ba),
00844 UPPER32(ba), cnt);
00845 if (rc != EOK) {
00846 printf("Error %d reading %zu blocks starting at block %" PRIuOFF64
00847 " from device handle %" PRIun "\n", rc, cnt, ba,
00848 devcon->devmap_handle);
00849 #ifndef NDEBUG
00850 stacktrace_print();
00851 #endif
00852 }
00853 return rc;
00854 }
00855
00865 static int write_blocks(devcon_t *devcon, aoff64_t ba, size_t cnt)
00866 {
00867 int rc;
00868
00869 assert(devcon);
00870 rc = async_req_3_0(devcon->dev_phone, BD_WRITE_BLOCKS, LOWER32(ba),
00871 UPPER32(ba), cnt);
00872 if (rc != EOK) {
00873 printf("Error %d writing %zu blocks starting at block %" PRIuOFF64
00874 " to device handle %" PRIun "\n", rc, cnt, ba, devcon->devmap_handle);
00875 #ifndef NDEBUG
00876 stacktrace_print();
00877 #endif
00878 }
00879 return rc;
00880 }
00881
00883 static int get_block_size(int dev_phone, size_t *bsize)
00884 {
00885 sysarg_t bs;
00886 int rc;
00887
00888 rc = async_req_0_1(dev_phone, BD_GET_BLOCK_SIZE, &bs);
00889 if (rc == EOK)
00890 *bsize = (size_t) bs;
00891
00892 return rc;
00893 }
00894
00896 static int get_num_blocks(int dev_phone, aoff64_t *nblocks)
00897 {
00898 sysarg_t nb_l, nb_h;
00899 int rc;
00900
00901 rc = async_req_0_2(dev_phone, BD_GET_NUM_BLOCKS, &nb_l, &nb_h);
00902 if (rc == EOK) {
00903 *nblocks = (aoff64_t) MERGE_LOUP32(nb_l, nb_h);
00904 }
00905
00906 return rc;
00907 }
00908
00910 static aoff64_t ba_ltop(devcon_t *devcon, aoff64_t lba)
00911 {
00912 assert(devcon->cache != NULL);
00913 return lba * devcon->cache->blocks_cluster;
00914 }
00915