From b34e6b52d110054cdf5532357261f6aea555fc32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=B1=E5=A4=A9=E9=BE=99=20=28Armink=29?= Date: Tue, 14 Mar 2023 22:02:11 +0800 Subject: [PATCH 1/2] [kvdb] Use partial GC function to reduce GC time. --- src/fdb_kvdb.c | 49 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/src/fdb_kvdb.c b/src/fdb_kvdb.c index dea370a..70234d0 100644 --- a/src/fdb_kvdb.c +++ b/src/fdb_kvdb.c @@ -119,7 +119,14 @@ struct alloc_kv_cb_args { uint32_t *empty_kv; }; +struct gc_cb_args { + fdb_kvdb_t db; + size_t cur_free_size; + size_t setting_free_size; +}; + static void gc_collect(fdb_kvdb_t db); +static void gc_collect_by_free_size(fdb_kvdb_t db, size_t free_size); #ifdef FDB_KV_USING_CACHE /* @@ -980,7 +987,7 @@ __retry: if ((empty_kv = alloc_kv(db, sector, kv_size)) == FAILED_ADDR) { if (db->gc_request && !already_gc) { FDB_DEBUG("Warning: Alloc an KV (size %" PRIu32 ") failed when new KV. Now will GC then retry.\n", (uint32_t)kv_size); - gc_collect(db); + gc_collect_by_free_size(db, kv_size); already_gc = true; goto __retry; } else if (already_gc) { @@ -1014,7 +1021,8 @@ static bool gc_check_cb(kv_sec_info_t sector, void *arg1, void *arg2) static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2) { struct fdb_kv kv; - fdb_kvdb_t db = arg1; + struct gc_cb_args *gc = (struct gc_cb_args *)arg1; + fdb_kvdb_t db = gc->db; if (sector->check_ok && (sector->status.dirty == FDB_SECTOR_DIRTY_TRUE || sector->status.dirty == FDB_SECTOR_DIRTY_GC)) { uint8_t status_table[FDB_DIRTY_STATUS_TABLE_SIZE]; @@ -1032,12 +1040,33 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2) } } while ((kv.addr.start = get_next_kv_addr(db, sector, &kv)) != FAILED_ADDR); format_sector(db, sector->addr, SECTOR_NOT_COMBINED); + gc->cur_free_size += db_sec_size(db) - SECTOR_HDR_DATA_SIZE; FDB_DEBUG("Collect a sector @0x%08" PRIX32 "\n", sector->addr); + if (gc->cur_free_size >= gc->setting_free_size) + return true; } return false; } +static void gc_collect_by_free_size(fdb_kvdb_t db, size_t free_size) +{ + struct kvdb_sec_info sector; + size_t empty_sec = 0; + struct gc_cb_args arg = {db, 0, free_size}; + + /* GC check the empty sector number */ + sector_iterator(db, §or, FDB_SECTOR_STORE_EMPTY, &empty_sec, NULL, gc_check_cb, false); + + /* do GC collect */ + FDB_DEBUG("The remain empty sector is %" PRIu32 ", GC threshold is %" PRIdLEAST16 ".\n", (uint32_t)empty_sec, FDB_GC_EMPTY_SEC_THRESHOLD); + if (empty_sec <= FDB_GC_EMPTY_SEC_THRESHOLD) { + sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, &arg, NULL, do_gc, false); + } + + db->gc_request = false; +} + /* * The GC will be triggered on the following scene: * 1. alloc an KV when the flash not has enough space @@ -1045,19 +1074,7 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2) */ static void gc_collect(fdb_kvdb_t db) { - struct kvdb_sec_info sector; - size_t empty_sec = 0; - - /* GC check the empty sector number */ - sector_iterator(db, §or, FDB_SECTOR_STORE_EMPTY, &empty_sec, NULL, gc_check_cb, false); - - /* do GC collect */ - FDB_DEBUG("The remain empty sector is %" PRIu32 ", GC threshold is %" PRIdLEAST16 ".\n", (uint32_t)empty_sec, FDB_GC_EMPTY_SEC_THRESHOLD); - if (empty_sec <= FDB_GC_EMPTY_SEC_THRESHOLD) { - sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, db, NULL, do_gc, false); - } - - db->gc_request = false; + gc_collect_by_free_size(db, db_max_size(db)); } static fdb_err_t align_write(fdb_kvdb_t db, uint32_t addr, const uint32_t *buf, size_t size) @@ -1221,7 +1238,7 @@ static fdb_err_t set_kv(fdb_kvdb_t db, const char *key, const void *value_buf, s } /* process the GC after set KV */ if (db->gc_request) { - gc_collect(db); + gc_collect_by_free_size(db, KV_HDR_DATA_SIZE + FDB_WG_ALIGN(strlen(key)) + FDB_WG_ALIGN(buf_len)); } } From e008c1a281cbff38a476beba477b1c901922863c Mon Sep 17 00:00:00 2001 From: shihang-zhang <77595854+shihang-zhang@users.noreply.github.com> Date: Fri, 21 Apr 2023 01:47:48 -0500 Subject: [PATCH 2/2] [kvdb] update the sector iterator by startswith the oldest sector (#206) Co-authored-by: shihang.zhang --- inc/fdb_def.h | 2 +- src/fdb_kvdb.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/inc/fdb_def.h b/inc/fdb_def.h index 67f9222..c9b9581 100644 --- a/inc/fdb_def.h +++ b/inc/fdb_def.h @@ -301,7 +301,7 @@ struct fdb_kvdb { #ifdef FDB_KV_AUTO_UPDATE uint32_t ver_num; /**< setting version number for update */ #endif - + uint32_t oldest_addr; void *user_data; }; typedef struct fdb_kvdb *fdb_kvdb_t; diff --git a/src/fdb_kvdb.c b/src/fdb_kvdb.c index 70234d0..5949631 100644 --- a/src/fdb_kvdb.c +++ b/src/fdb_kvdb.c @@ -125,6 +125,14 @@ struct gc_cb_args { size_t setting_free_size; }; +struct check_oldest_addr_cb_args { + fdb_kvdb_t db; + size_t sector_using_addr; + size_t sector_oldest_addr; + bool is_first_full_sector_found; + bool is_first_using_sector_found; +}; + static void gc_collect(fdb_kvdb_t db); static void gc_collect_by_free_size(fdb_kvdb_t db, size_t free_size); @@ -784,9 +792,11 @@ static void sector_iterator(fdb_kvdb_t db, kv_sec_info_t sector, fdb_sector_stor bool (*callback)(kv_sec_info_t sector, void *arg1, void *arg2), bool traversal_kv) { uint32_t sec_addr; + uint32_t sec_iterate_end_addr; /* search all sectors */ - sec_addr = 0; + sec_addr = db->oldest_addr; + sec_iterate_end_addr = db->oldest_addr; do { read_sector_info(db, sec_addr, sector, false); if (status == FDB_SECTOR_STORE_UNUSED || status == sector->status.store) { @@ -798,7 +808,12 @@ static void sector_iterator(fdb_kvdb_t db, kv_sec_info_t sector, fdb_sector_stor return; } } - } while ((sec_addr = get_next_sector_addr(db, sector)) != FAILED_ADDR); + /* if reach to the end, roll back to the first sector */ + if ((sec_addr = get_next_sector_addr(db, sector)) == FAILED_ADDR) { + sec_addr = 0; + } + + } while (sec_addr != sec_iterate_end_addr); } static bool sector_statistics_cb(kv_sec_info_t sector, void *arg1, void *arg2) @@ -1023,6 +1038,7 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2) struct fdb_kv kv; struct gc_cb_args *gc = (struct gc_cb_args *)arg1; fdb_kvdb_t db = gc->db; + uint32_t sec_addr; if (sector->check_ok && (sector->status.dirty == FDB_SECTOR_DIRTY_TRUE || sector->status.dirty == FDB_SECTOR_DIRTY_GC)) { uint8_t status_table[FDB_DIRTY_STATUS_TABLE_SIZE]; @@ -1042,6 +1058,15 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2) format_sector(db, sector->addr, SECTOR_NOT_COMBINED); gc->cur_free_size += db_sec_size(db) - SECTOR_HDR_DATA_SIZE; FDB_DEBUG("Collect a sector @0x%08" PRIX32 "\n", sector->addr); + /* update oldest_addr for next GC sector format */ + sec_addr = get_next_sector_addr(db, sector); + /*sec_addr reached db_max_size(db), roll back to the first sector*/ + if (sec_addr == FAILED_ADDR) { + db->oldest_addr = 0; + } else { + db->oldest_addr = sec_addr; + } + FDB_DEBUG("oldest_addr is @0x%08" PRIX32 "\n", db->oldest_addr); if (gc->cur_free_size >= gc->setting_free_size) return true; } @@ -1450,6 +1475,35 @@ static void kv_auto_update(fdb_kvdb_t db) } #endif /* FDB_KV_AUTO_UPDATE */ +static bool check_oldest_addr_cb(kv_sec_info_t sector, void *arg1, void *arg2) +{ + struct check_oldest_addr_cb_args *arg = (struct check_oldest_addr_cb_args *)arg1; + + if(sector->status.store == FDB_SECTOR_STORE_FULL) { + /* found the first full sector, if there is no full sector on the using sector's right, + * the first found full sector on the left of the using sector is the oldest */ + if (arg->is_first_full_sector_found == false) { + arg->is_first_full_sector_found = true; + arg->sector_oldest_addr = sector->addr; + } + + /* if there is full sector on the right of the using sector, + * the first full sector found on the right of the using sector is the oldest */ + if ((arg->is_first_using_sector_found == true) && (sector->addr > arg->sector_using_addr)) { + arg->sector_oldest_addr = sector->addr; + return true; + } + }else if(sector->status.store == FDB_SECTOR_STORE_USING) { + /* found the first using sector */ + if (arg->is_first_using_sector_found == false) { + arg->is_first_using_sector_found = true; + arg->sector_using_addr = sector->addr; + } + } + + return false; +} + static bool check_sec_hdr_cb(kv_sec_info_t sector, void *arg1, void *arg2) { if (!sector->check_ok) { @@ -1638,6 +1692,8 @@ fdb_err_t fdb_kvdb_init(fdb_kvdb_t db, const char *name, const char *path, struc void *user_data) { fdb_err_t result = FDB_NO_ERR; + struct kvdb_sec_info sector; + struct check_oldest_addr_cb_args arg = {db, 0, 0, false, false}; #ifdef FDB_KV_USING_CACHE size_t i; @@ -1660,6 +1716,11 @@ fdb_err_t fdb_kvdb_init(fdb_kvdb_t db, const char *name, const char *path, struc db->default_kvs.num = 0; db->default_kvs.kvs = NULL; } + + db->oldest_addr = 0; + sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, &arg, NULL, check_oldest_addr_cb, false); + db->oldest_addr = arg.sector_oldest_addr; + FDB_DEBUG("oldest_addr is @0x%08" PRIX32 "\n", db->oldest_addr); /* there is at least one empty sector for GC. */ FDB_ASSERT((FDB_GC_EMPTY_SEC_THRESHOLD > 0 && FDB_GC_EMPTY_SEC_THRESHOLD < SECTOR_NUM))