Embedded/FlashDB
1
0
Fork 0
mirror of https://github.com/armink/FlashDB.git synced 2025-01-16 20:12:52 +08:00
Code Issues Releases Wiki Activity

[kvdb] Improve the GC algorithm to solve the problem of failure to collection in some cases.

Browse Source
This commit is contained in:
armink 2024-10-05 22:53:52 +08:00
parent 92ed442cfc
commit 72a7905238
2 changed files with 258 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/fdb_kvdb.c
View File

@ -35,6 +35,8 @@
#define SECTOR_MAGIC_WORD 0x30424446
/* magic word(`K`, `V`, `0`, `0`) */
#define KV_MAGIC_WORD 0x3030564B
/* GC minimum number of empty sectors. GC will using at least 1 empty sector. */
#define GC_MIN_EMPTY_SEC_NUM 1
/* the sector remain threshold before full status */
#ifndef FDB_SEC_REMAIN_THRESHOLD
@ -135,9 +137,8 @@ struct alloc_kv_cb_args {
struct gc_cb_args {
fdb_kvdb_t db;
size_t cur_free_size;
size_t setting_free_size;
uint32_t traversed_len;
size_t last_gc_sec_addr;
};
static void gc_collect(fdb_kvdb_t db);
@ -1071,7 +1072,7 @@ __retry:
if ((empty_kv = alloc_kv(db, sector, kv_size)) == FAILED_ADDR) {
if (db->gc_request && !already_gc) {
FDB_INFO("Warning: Alloc an KV (size %" PRIu32 ") failed when new KV. Now will GC then retry.\n", (uint32_t)kv_size);
FDB_DEBUG("Alloc an KV (size %" PRIu32 ") failed when new KV. Now will GC then retry.\n", (uint32_t)kv_size);
gc_collect_by_free_size(db, kv_size);
already_gc = true;
goto __retry;
@ -1093,10 +1094,12 @@ static uint32_t new_kv_ex(fdb_kvdb_t db, kv_sec_info_t sector, size_t key_len, s
static bool gc_check_cb(kv_sec_info_t sector, void *arg1, void *arg2)
{
size_t *empty_sec = arg1;
size_t *empty_sec_num = arg1;
uint32_t *empty_sec_addr = arg2;
if (sector->check_ok) {
*empty_sec = *empty_sec + 1;
*empty_sec_num = *empty_sec_num + 1;
*empty_sec_addr = sector->addr;
}
return false;
@ -1108,6 +1111,7 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2)
struct fdb_kv kv;
struct gc_cb_args *gc = (struct gc_cb_args *)arg1;
fdb_kvdb_t db = gc->db;
uint32_t last_gc_sec_addr = 0;
if (sector->check_ok && (sector->status.dirty == FDB_SECTOR_DIRTY_TRUE || sector->status.dirty == FDB_SECTOR_DIRTY_GC)) {
uint8_t status_table[FDB_DIRTY_STATUS_TABLE_SIZE];
@ -1122,15 +1126,22 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2)
if (move_kv(db, &kv) != FDB_NO_ERR) {
FDB_INFO("Error: Moved the KV (%.*s) for GC failed.\n", kv.name_len, kv.name);
}
} else {
FDB_DEBUG("KV (%.*s) is garbage NOT need move, collect it.\n", kv.name_len, kv.name);
}
} while ((kv.addr.start = get_next_kv_addr(db, sector, &kv)) != FAILED_ADDR);
format_sector(db, sector->addr, SECTOR_NOT_COMBINED);
gc->cur_free_size += db_sec_size(db) - SECTOR_HDR_DATA_SIZE;
FDB_DEBUG("Collect a sector @0x%08" PRIX32 "\n", sector->addr);
last_gc_sec_addr = gc->last_gc_sec_addr;
gc->last_gc_sec_addr = sector->addr;
/* update oldest_addr for next GC sector format */
db_oldest_addr(db) = get_next_sector_addr(db, sector, 0);
if (gc->cur_free_size >= gc->setting_free_size)
return true;
FDB_DEBUG("Collect a sector @0x%08" PRIX32 "\n", sector->addr);
/* the collect new space is in last GC sector */
struct kvdb_sec_info last_gc_sector;
if (read_sector_info(db, last_gc_sec_addr, &last_gc_sector, true) == FDB_NO_ERR) {
if (last_gc_sector.remain > gc->setting_free_size)
return true;
}
}
return false;
@ -1139,15 +1150,17 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2)
static void gc_collect_by_free_size(fdb_kvdb_t db, size_t free_size)
{
struct kvdb_sec_info sector;
size_t empty_sec = 0;
struct gc_cb_args arg = { db, 0, free_size, 0 };
size_t empty_sec_num = 0;
/* an empty sector address */
uint32_t empty_sec_addr = 0;
/* GC check the empty sector number */
sector_iterator(db, &sector, FDB_SECTOR_STORE_EMPTY, &empty_sec, NULL, gc_check_cb, false);
sector_iterator(db, &sector, FDB_SECTOR_STORE_EMPTY, &empty_sec_num, &empty_sec_addr, gc_check_cb, false);
/* do GC collect */
FDB_DEBUG("The remain empty sector is %" PRIu32 ", GC threshold is %" PRIdLEAST16 ".\n", (uint32_t)empty_sec, FDB_GC_EMPTY_SEC_THRESHOLD);
if (empty_sec <= FDB_GC_EMPTY_SEC_THRESHOLD) {
FDB_DEBUG("The remain empty sector is %" PRIu32 ", GC threshold is %" PRIdLEAST16 ".\n", (uint32_t)empty_sec_num, FDB_GC_EMPTY_SEC_THRESHOLD);
if (empty_sec_num <= FDB_GC_EMPTY_SEC_THRESHOLD) {
struct gc_cb_args arg = { db, free_size, empty_sec_addr };
sector_iterator(db, &sector, FDB_SECTOR_STORE_UNUSED, &arg, NULL, do_gc, false);
}
@ -1580,7 +1593,7 @@ static bool check_sec_hdr_cb(kv_sec_info_t sector, void *arg1, void *arg2)
if (db->parent.not_formatable) {
return true;
} else {
FDB_INFO("Sector header info is incorrect. Auto format this sector (0x%08" PRIX32 ").\n", sector->addr);
FDB_DEBUG("Sector header info is incorrect. Auto format this sector (0x%08" PRIX32 ").\n", sector->addr);
format_sector(db, sector->addr, SECTOR_NOT_COMBINED);
}
}

260
tests/fdb_kvdb_tc.c
View File

@ -41,7 +41,8 @@
struct test_kv{
char name[32];
uint8_t value[TEST_KV_VALUE_LEN];
char *value;
size_t value_len;
uint32_t addr;
uint32_t saved_data_size;
bool is_changed;
@ -311,10 +312,11 @@ static int iter_all_kv(fdb_kvdb_t db, struct test_kv *kv_tbl, size_t len)
rt_strncpy(kv_tbl[index].name, cur_kv->name, 32);
kv_tbl[index].saved_data_size = data_size;
kv_tbl[index].addr = cur_kv->addr.start;
kv_tbl[index].value = (char *)rt_malloc(data_size);
if (kv_tbl[index].value == NULL)
RT_ASSERT(0 && "no memory for value");
/* read data */
fdb_blob_read((fdb_db_t)db, fdb_kv_to_blob(cur_kv, fdb_blob_make(&fdb_blob, &kv_tbl[index].value, data_size)));
uassert_true(kv_tbl[index].saved_data_size <= TEST_KV_VALUE_LEN);
fdb_blob_read((fdb_db_t)db, fdb_kv_to_blob(cur_kv, fdb_blob_make(&fdb_blob, kv_tbl[index].value, data_size)));
index++;
}
@ -323,13 +325,14 @@ static int iter_all_kv(fdb_kvdb_t db, struct test_kv *kv_tbl, size_t len)
static void test_save_fdb_by_kvs(const struct test_kv *kv_tbl, size_t len)
{
struct fdb_blob blob_obj, * blob = &blob_obj;
struct fdb_blob blob_obj, *blob = &blob_obj;
for (size_t i = 0; i < len; i++)
{
if (kv_tbl[i].is_changed)
{
fdb_kv_set_blob(&test_kvdb, kv_tbl[i].name, fdb_blob_make(blob, kv_tbl[i].value, TEST_KV_VALUE_LEN));
int result = fdb_kv_set_blob(&test_kvdb, kv_tbl[i].name, fdb_blob_make(blob, kv_tbl[i].value, kv_tbl[i].value_len));
uassert_true(result == FDB_NO_ERR);
}
}
}
@ -338,6 +341,8 @@ static void test_check_fdb_by_kvs(const struct test_kv *kv_tbl, size_t len)
{
static struct test_kv saved_kv_tbl[TEST_KV_MAX_NUM] = { 0 };
RT_ASSERT(len <= FDB_ARRAY_SIZE(saved_kv_tbl));
iter_all_kv(&test_kvdb, saved_kv_tbl, FDB_ARRAY_SIZE(saved_kv_tbl));
for (size_t i = 0, j = 0; i < len; i++)
@ -395,10 +400,10 @@ static void test_fdb_gc(void)
* +---------------------------------------------------------+
*/
static const struct test_kv kv_tbl[] = {
{"kv0", "0", 0, 0, 1},
{"kv1", "1", 0, 0, 1},
{"kv2", "2", 0, 0, 1},
{"kv3", "3", 1, 0, 1},
{"kv0", "0", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv3", "3", TEST_KV_VALUE_LEN, 1, 0, 1},
};
test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl));
@ -431,10 +436,10 @@ static void test_fdb_gc(void)
* +--------------+-------------+--------------+-------------+
*/
static const struct test_kv kv_tbl[] = {
{"kv1", "1", 0, 0, 0},
{"kv2", "2", 0, 0, 0},
{"kv0", "00", 1, 0, 1},
{"kv3", "33", 1, 0, 1},
{"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 0},
{"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 0},
{"kv0", "00", TEST_KV_VALUE_LEN, 1, 0, 1},
{"kv3", "33", TEST_KV_VALUE_LEN, 1, 0, 1},
};
test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl));
@ -511,10 +516,10 @@ static void test_fdb_gc(void)
*
*/
static const struct test_kv kv_tbl[] = {
{"kv0", "000", 2, 0, 1},
{"kv1", "111", 2, 0, 1},
{"kv2", "222", 2, 0, 1},
{"kv3", "333", 3, 0, 1},
{"kv0", "000", TEST_KV_VALUE_LEN, 2, 0, 1},
{"kv1", "111", TEST_KV_VALUE_LEN, 2, 0, 1},
{"kv2", "222", TEST_KV_VALUE_LEN, 2, 0, 1},
{"kv3", "333", TEST_KV_VALUE_LEN, 3, 0, 1},
};
test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl));
@ -611,10 +616,10 @@ static void test_fdb_gc(void)
* +--------------+-------------+--------------+-------------+
*/
static const struct test_kv kv_tbl[] = {
{"kv0", "0000", 3, 0, 1},
{"kv1", "1111", 3, 0, 1},
{"kv2", "2222", 0, 0, 1},
{"kv3", "3333", 0, 0, 1},
{"kv0", "0000", TEST_KV_VALUE_LEN, 3, 0, 1},
{"kv1", "1111", TEST_KV_VALUE_LEN, 3, 0, 1},
{"kv2", "2222", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv3", "3333", TEST_KV_VALUE_LEN, 0, 0, 1},
};
test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl));
@ -625,15 +630,209 @@ static void test_fdb_gc(void)
}
}
static void test_fdb_gc2(void)
{
fdb_kv_set_default(&test_kvdb);
{
/*
* prepare1: add 4 KVs
*
* +---------------------------------------------------------+
* | sector0 | sector1 | sector2 | sector3 |
* | using | using | empty | empty |
* +---------------------------------------------------------+
* | | | | |
* | kv0 | kv3 | | |
* | new | new | | |
* +----------------------------+ | |
* | | | | |
* | kv1 | | | |
* | new | | | |
* +--------------+ | | |
* | | | | |
* | kv2 | | | |
* | new | | | |
* +--------------+ | | |
* | | | | |
* +---------------------------------------------------------+
*/
static const struct test_kv kv_tbl[] = {
{"kv0", "0", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv3", "3", TEST_KV_VALUE_LEN, 1, 0, 1},
};
test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl));
uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0);
fdb_reboot();
uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0);
}
{
/*
* prepare2: change kv0 and kv3
*
* +--------------+-------------+--------------+-------------+
* | sector0 | sector1 | sector2 | sector3 |
* | using | using | empty | empty |
* +---------------------------------------------------------+
* | | | | |
* | kv0 | kv3 | | |
* | delete | delete | | |
* +----------------------------+ | |
* | | | | |
* | kv1 | kv0 | | |
* | new | new | | |
* +----------------------------+ | |
* | | | | |
* | kv2 | kv3 | | |
* | new | new | | |
* +----------------------------+ | |
* | | | | |
* +--------------+-------------+--------------+-------------+
*/
static const struct test_kv kv_tbl[] = {
{"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 0},
{"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 0},
{"kv0", "00", TEST_KV_VALUE_LEN, 1, 0, 1},
{"kv3", "33", TEST_KV_VALUE_LEN, 1, 0, 1},
};
test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl));
uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0);
fdb_reboot();
uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0);
}
{
/*
* prepare3: add big kv4
*
* +--------------+-------------+--------------+-------------+
* | sector0 | sector1 | sector2 | sector3 |
* | using | using | using | empty |
* +---------------------------------------------------------+
* | | | | |
* | kv0 | kv3 | | |
* | delete | delete | | |
* +----------------------------+ kv4 | |
* | | | new | |
* | kv1 | kv0 | | |
* | new | new | | |
* +----------------------------+ | |
* | | | | |
* | kv2 | kv3 |--------------| |
* | new | new | | |
* +----------------------------+ | |
* | | | | |
* +--------------+-------------+--------------+-------------+
*/
static const struct test_kv kv_tbl[] = {
{"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 0},
{"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 0},
{"kv0", "00", TEST_KV_VALUE_LEN, 1, 0, 0},
{"kv3", "33", TEST_KV_VALUE_LEN, 1, 0, 0},
{"kv4", "4", TEST_KV_VALUE_LEN * 3, 2, 0, 1},
};
test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl));
uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0);
fdb_reboot();
uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0);
}
{
/*
* add kv5, trigger GC
* step1: move kv1 and kv2
* +--------------+-------------+--------------+-------------+
* | sector0 | sector1 | sector2 | sector3 |
* | empty | using | using | using |
* +---------------------------------------------------------+
* | | | | |
* | | kv3 | | kv1 |
* | | delete | | new |
* | +-------------+ kv4 |-------------|
* | | | new | |
* | | kv0 | | kv2 |
* | | new | | new |
* | +-------------+ |-------------|
* | | | | |
* | | kv3 |--------------| |
* | | new | | |
* | +-------------+ | |
* | | | | |
* +--------------+-------------+--------------+-------------+
*
* step2: move kv0 and kv3
* +--------------+-------------+--------------+-------------+
* | sector0 | sector1 | sector2 | sector3 |
* | using | empty | using | using |
* +---------------------------------------------------------+
* | | | | |
* | kv3 | | | kv1 |
* | new | | | new |
* +--------------+ | kv4 |-------------|
* | | | new | |
* | | | | kv2 |
* | | | | new |
* | | | |-------------|
* | | | | |
* | | |--------------| kv0 |
* | | | | new |
* | | | |-------------|
* | | | | |
* +--------------+-------------+--------------+-------------+
*
* step3: add kv5
* +--------------+-------------+--------------+-------------+
* | sector0 | sector1 | sector2 | sector3 |
* | using | empty | using | using |
* +---------------------------------------------------------+
* | | | | |
* | kv3 | | | kv1 |
* | new | | | new |
* +--------------+ | |-------------|
* | | | kv4 | |
* | | | new | kv2 |
* | kv5 | | | new |
* | new | | |-------------|
* | | | | |
* | | |--------------| kv0 |
* |--------------| | | new |
* | | | |-------------|
* | | | | |
* +--------------+-------------+--------------+-------------+
*
*/
static const struct test_kv kv_tbl[] = {
{"kv3", "33", TEST_KV_VALUE_LEN, 0, 0, 0},
{"kv5", "5", TEST_KV_VALUE_LEN * 2, 0, 0, 1},
{"kv4", "4", TEST_KV_VALUE_LEN * 3, 2, 0, 0},
{"kv1", "1", TEST_KV_VALUE_LEN, 3, 0, 0},
{"kv2", "2", TEST_KV_VALUE_LEN, 3, 0, 0},
{"kv0", "00", TEST_KV_VALUE_LEN, 3, 0, 0},
};
test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl));
uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 2);
fdb_reboot();
uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 2);
}
}
static void test_fdb_scale_up(void)
{
fdb_kv_set_default(&test_kvdb);
static const struct test_kv old_kv_tbl[] = {
{"kv0", "0", 0, 0, 1},
{"kv1", "1", 0, 0, 1},
{"kv2", "2", 0, 0, 1},
{"kv3", "3", 1, 0, 1},
{"kv0", "0", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 1},
{"kv3", "3", TEST_KV_VALUE_LEN, 1, 0, 1},
};
/* save some data */
test_save_fdb_by_kvs(old_kv_tbl, FDB_ARRAY_SIZE(old_kv_tbl));
@ -647,10 +846,10 @@ static void test_fdb_scale_up(void)
/* save some new data */
static const struct test_kv new_kv_tbl[] = {
{"kv4", "4", 4, 0, 1},
{"kv5", "5", 4, 0, 1},
{"kv6", "6", 4, 0, 1},
{"kv7", "7", 5, 0, 1},
{"kv4", "4", TEST_KV_VALUE_LEN, 4, 0, 1},
{"kv5", "5", TEST_KV_VALUE_LEN, 4, 0, 1},
{"kv6", "6", TEST_KV_VALUE_LEN, 4, 0, 1},
{"kv7", "7", TEST_KV_VALUE_LEN, 5, 0, 1},
};
/* kv4: sector1, kv5: sector1, kv6: sector2, kv7: sector2 */
test_save_fdb_by_kvs(new_kv_tbl, FDB_ARRAY_SIZE(new_kv_tbl));
@ -698,6 +897,7 @@ static void testcase(void)
UTEST_UNIT_RUN(test_fdb_change_kv);
UTEST_UNIT_RUN(test_fdb_del_kv);
UTEST_UNIT_RUN(test_fdb_gc);
UTEST_UNIT_RUN(test_fdb_gc2);
UTEST_UNIT_RUN(test_fdb_scale_up);
UTEST_UNIT_RUN(test_fdb_kvdb_set_default);
UTEST_UNIT_RUN(test_fdb_kvdb_deinit);