From e214c917a5a66ffed7297784c43f815bac14b237 Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Thu, 9 May 2019 01:25:26 -0700 Subject: [PATCH 1/2] add prefill for slimcache, modify cmake config --- CMakeLists.txt | 37 +++++++++-------------- deps/ccommon/CMakeLists.txt | 27 +++++++++-------- src/server/slimcache/data/process.c | 47 +++++++++++++++++++++++++++++ src/server/slimcache/data/process.h | 14 ++++++++- src/server/twemcache/data/process.c | 8 ++--- src/server/twemcache/data/process.h | 2 +- 6 files changed, 93 insertions(+), 42 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 42d6e8d0b..fdc6ab8b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,6 @@ option(TARGET_RESPCLI "build resp-cli binary" ON) option(HAVE_RUST "build features written in rust" OFF) option(RUST_USE_MUSL "build rust deps against musl" OFF) option(BUILD_AND_INSTALL_CHECK "build our own version of check and link against it" OFF) -option(USE_PMEM "build persistent memory features" OFF) option(COVERAGE "code coverage" OFF) @@ -127,24 +126,16 @@ add_subdirectory(${CCOMMON_SOURCE_DIR} ${PROJECT_BINARY_DIR}/ccommon) include(FindPackageHandleStandardArgs) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake") -find_package(PkgConfig QUIET) - -if(PKG_CONFIG_FOUND) - pkg_check_modules(CHECK QUIET check>=0.10) -endif() - +find_package(Check) if(NOT CHECK_FOUND) - find_package(Check QUIET 0.10) -endif() - -if (USE_PMEM) - if(PKG_CONFIG_FOUND) - pkg_check_modules(LIBPMEM REQUIRED libpmem>=1.0) - else() - find_package(LIBPMEM REQUIRED 1.0) - endif() - link_directories(${LIBPMEM_LIBRARY_DIRS}) -endif(USE_PMEM) + message(WARNING "Check is required to build and run tests") +endif(NOT CHECK_FOUND) +if(CHECK_FOUND) + check_symbol_exists(ck_assert_int_eq check.h CHECK_WORKING) + if(NOT CHECK_WORKING) + message(WARNING "Check version too old to build tests") + endif(NOT CHECK_WORKING) +endif(CHECK_FOUND) find_package(Threads) @@ -163,11 +154,11 @@ include_directories(${include_directories} # server & (cli) client add_subdirectory(src) -# tests: always build last -if(CHECK_FOUND) - include_directories(${include_directories} ${CHECK_INCLUDES}) - add_subdirectory(test) -endif(CHECK_FOUND) +## tests: always build last +#if(CHECK_WORKING) +# include_directories(${include_directories} "${CHECK_INCLUDES}") +# add_subdirectory(test) +#endif(CHECK_WORKING) add_subdirectory(benchmarks) diff --git a/deps/ccommon/CMakeLists.txt b/deps/ccommon/CMakeLists.txt index f6ddf7597..748cf7871 100644 --- a/deps/ccommon/CMakeLists.txt +++ b/deps/ccommon/CMakeLists.txt @@ -140,15 +140,16 @@ endif(COVERAGE) include(FindPackageHandleStandardArgs) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake") -find_package(PkgConfig QUIET) - -if(PKG_CONFIG_FOUND) - pkg_check_modules(CHECK QUIET check>=0.10) -endif() - +find_package(Check) if(NOT CHECK_FOUND) - find_package(Check QUIET 0.10) -endif() + message(WARNING "Check is required to build and run tests") +endif(NOT CHECK_FOUND) +if(CHECK_FOUND) + check_symbol_exists(ck_assert_int_eq check.h CHECK_WORKING) + if(NOT CHECK_WORKING) + message(WARNING "Check version too old to build tests") + endif(NOT CHECK_WORKING) +endif(CHECK_FOUND) find_package(Threads) @@ -165,10 +166,10 @@ include_directories( add_subdirectory(src) -if(CHECK_FOUND) - include_directories(${include_directories} ${CHECK_INCLUDES}) - add_subdirectory(test) -endif(CHECK_FOUND) +#if(CHECK_WORKING) +# include_directories(${include_directories} "${CHECK_INCLUDES}") +# add_subdirectory(test) +#endif(CHECK_WORKING) if(HAVE_RUST) @@ -192,4 +193,4 @@ message(STATUS "HAVE_SIGNAME: " ${HAVE_SIGNAME}) message(STATUS "HAVE_BACKTRACE: " ${HAVE_BACKTRACE}) -message(STATUS "CHECK_FOUND: " ${CHECK_FOUND}) +message(STATUS "CHECK_WORKING: " ${CHECK_WORKING}) diff --git a/src/server/slimcache/data/process.c b/src/server/slimcache/data/process.c index e6ea696f3..35d6de10d 100644 --- a/src/server/slimcache/data/process.c +++ b/src/server/slimcache/data/process.c @@ -16,6 +16,45 @@ static bool process_init = false; static process_metrics_st *process_metrics = NULL; static bool allow_flush = ALLOW_FLUSH; +static bool prefill = PREFILL; +static uint8_t prefill_ksize; +static char prefill_kbuf[UINT8_MAX]; /* cuckoo storage has klen as uint8_t */ +static uint8_t prefill_vsize; +static char prefill_vbuf[UINT8_MAX]; /* cuckoo storage has vlen as uint8_t */ +static uint64_t prefill_nkey; + + +static void +_prefill_cuckoo(void) +{ + struct duration d; + struct bstring key, val; + struct item *it; + + duration_reset(&d); + key.len = prefill_ksize; + key.data = prefill_kbuf; + val.len = prefill_vsize; + val.data = prefill_vbuf; + + duration_start(&d); + for (uint32_t i = 0; i < prefill_nkey; ++i) { + /* print fixed-length key with leading 0's for padding */ + cc_snprintf(&prefill_kbuf, key.len + 1, "%.*d", key.len, i); + /* fill val, use the same value as key for now */ + cc_snprintf(&prefill_vbuf, val.len + 1, "%.*d", val.len, i); + /* insert into cuckoo/heap */ + it = cuckoo_insert(&key, &val, expire, + time_convert_proc_sec((time_i)INT32_MAX)); + ASSERT(it != NULL); + } + duration_stop(&d); + + log_info("prefilling cuckoo with %"PRIu64" keys, of key len %"PRIu8" & val " + "len %"PRIu8", in %.3f seconds", prefill_nkey, prefill_ksize, + prefill_vsize, duration_sec(&d)); +} + void process_setup(process_options_st *options, process_metrics_st *metrics) @@ -30,6 +69,14 @@ process_setup(process_options_st *options, process_metrics_st *metrics) if (options != NULL) { allow_flush = option_bool(&options->allow_flush); + prefill = option_bool(&options->prefill); + prefill_ksize = (uint8_t)option_uint(&options->prefill_ksize); + prefill_vsize = (uint8_t)option_uint(&options->prefill_vsize); + prefill_nkey = (uint64_t)option_uint(&options->prefill_nkey); + } + + if (prefill) { + _prefill_cuckoo(); } process_init = true; diff --git a/src/server/slimcache/data/process.h b/src/server/slimcache/data/process.h index 0f5ae1349..97dd7f922 100644 --- a/src/server/slimcache/data/process.h +++ b/src/server/slimcache/data/process.h @@ -9,7 +9,19 @@ /* name type default description */ #define PROCESS_OPTION(ACTION) \ - ACTION( allow_flush, OPTION_TYPE_BOOL, ALLOW_FLUSH, "allow flushing on the data port" ) + ACTION( allow_flush, OPTION_TYPE_BOOL, ALLOW_FLUSH, "allow flushing on the data port" )\ + ACTION( prefill, OPTION_TYPE_BOOL, PREFILL, "prefill data array" )\ + ACTION( prefill_ksize, OPTION_TYPE_UINT, PREFILL_KSIZE, "prefill key size" )\ + ACTION( prefill_vsize, OPTION_TYPE_UINT, PREFILL_VSIZE, "prefill val size" )\ + ACTION( prefill_nkey, OPTION_TYPE_UINT, PREFILL_NKEY, "prefill keys inserted" ) +/* For now, the prefill logic will populate the heap with keys and values of + * specified lengths, while the keys will be string representation of base-10 + * numeric values padded to the right length, i.e. keys will look like + * "000000", "000001", ..., "123456", and prefilling logic will always start + * from 0 and trying to insert the exact number of keys specified (underfill + * and eviction are therefore possible) depending on how cuckoo array is + * configured. + */ typedef struct { PROCESS_OPTION(OPTION_DECLARE) diff --git a/src/server/twemcache/data/process.c b/src/server/twemcache/data/process.c index b32b969f5..e09363835 100644 --- a/src/server/twemcache/data/process.c +++ b/src/server/twemcache/data/process.c @@ -27,8 +27,8 @@ static bool process_init = false; static process_metrics_st *process_metrics = NULL; static bool allow_flush = ALLOW_FLUSH; static bool prefill = PREFILL; -static uint32_t prefill_ksize; -static char prefill_kbuf[UINT8_MAX]; /* slab implementation has klen as unint8_t */ +static uint8_t prefill_ksize; +static char prefill_kbuf[UINT8_MAX]; /* slab implementation has klen as uint8_t */ static uint32_t prefill_vsize; /* val_buf size is arbitrary , update if want to warm up with larger objects */ static char prefill_vbuf[ITEM_SIZE_MAX]; @@ -62,7 +62,7 @@ _prefill_slab(void) } duration_stop(&d); - log_info("prefilling slab with %"PRIu64" keys, of key len %"PRIu32" & val " + log_info("prefilling slab with %"PRIu64" keys, of key len %"PRIu8" & val " "len %"PRIu32", in %.3f seconds", prefill_nkey, prefill_ksize, prefill_vsize, duration_sec(&d)); } @@ -82,7 +82,7 @@ process_setup(process_options_st *options, process_metrics_st *metrics) if (options != NULL) { allow_flush = option_bool(&options->allow_flush); prefill = option_bool(&options->prefill); - prefill_ksize = (uint32_t)option_uint(&options->prefill_ksize); + prefill_ksize = (uint8_t)option_uint(&options->prefill_ksize); prefill_vsize = (uint32_t)option_uint(&options->prefill_vsize); prefill_nkey = (uint64_t)option_uint(&options->prefill_nkey); } diff --git a/src/server/twemcache/data/process.h b/src/server/twemcache/data/process.h index 12bb6e95e..093125099 100644 --- a/src/server/twemcache/data/process.h +++ b/src/server/twemcache/data/process.h @@ -13,7 +13,7 @@ /* name type default description */ #define PROCESS_OPTION(ACTION) \ - ACTION( allow_flush, OPTION_TYPE_BOOL, ALLOW_FLUSH, "allow flush_all" )\ + ACTION( allow_flush, OPTION_TYPE_BOOL, ALLOW_FLUSH, "allow flush on data port")\ ACTION( prefill, OPTION_TYPE_BOOL, PREFILL, "prefill slabs with data" )\ ACTION( prefill_ksize, OPTION_TYPE_UINT, PREFILL_KSIZE, "prefill key size" )\ ACTION( prefill_vsize, OPTION_TYPE_UINT, PREFILL_VSIZE, "prefill val size" )\ From 87ce3c223ca6681a05ebaecacfb130fe6b121394 Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Sun, 12 May 2019 21:41:55 -0700 Subject: [PATCH 2/2] temp --- CMakeLists.txt | 2 +- benchmarks/CMakeLists.txt | 2 +- src/datapool/CMakeLists.txt | 2 +- src/server/slimcache/CMakeLists.txt | 1 + src/server/slimcache/data/process.c | 14 +++++++++----- src/server/slimcache/data/process.h | 5 +++++ src/server/slimds/CMakeLists.txt | 1 + src/storage/cuckoo/CMakeLists.txt | 1 - 8 files changed, 19 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fdc6ab8b0..827fbeee4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,7 @@ option(HAVE_STATS "stats enabled by default" ON) option(TARGET_PINGSERVER "build pingserver binary" ON) option(TARGET_DS "build dataserver binary" ON) -option(TARGET_SLIMDS "build slim dataserver binary" ON) +option(TARGET_SLIMDS "build slim dataserver binary" OFF) option(TARGET_SLIMCACHE "build slimcache binary" ON) option(TARGET_TWEMCACHE "build twemcache binary" ON) option(TARGET_CDB "build cdb binary (implies HAVE_RUST)" OFF) diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index fd55daef1..83d286ae5 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -4,6 +4,6 @@ target_include_directories(bench_micro PRIVATE ${CMAKE_SOURCE_DIR}/src ${CMAKE_SOURCE_DIR}/tests) -target_link_libraries(bench_micro cuckoo time) +target_link_libraries(bench_micro cuckoo datapool time) target_link_libraries(bench_micro ccommon-static) target_link_libraries(bench_micro pthread) diff --git a/src/datapool/CMakeLists.txt b/src/datapool/CMakeLists.txt index 4148d3e6d..8fca37840 100644 --- a/src/datapool/CMakeLists.txt +++ b/src/datapool/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(datapool datapool.h) +add_library(datapool) if(USE_PMEM) target_sources(datapool PRIVATE datapool_pmem.c) diff --git a/src/server/slimcache/CMakeLists.txt b/src/server/slimcache/CMakeLists.txt index 07af57ce6..47786a647 100644 --- a/src/server/slimcache/CMakeLists.txt +++ b/src/server/slimcache/CMakeLists.txt @@ -10,6 +10,7 @@ set(SOURCE set(MODULES core cuckoo + datapool protocol_admin protocol_memcache time diff --git a/src/server/slimcache/data/process.c b/src/server/slimcache/data/process.c index 35d6de10d..0e01aaccf 100644 --- a/src/server/slimcache/data/process.c +++ b/src/server/slimcache/data/process.c @@ -6,6 +6,7 @@ #include #include #include +#include