From 99002020422302180b37dde6fc61d1e07e5b59a3 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 25 Feb 2026 09:25:36 -0500 Subject: [PATCH 01/13] Integrate BULKI serializer to save and restore server checkpoints --- src/server/pdc_server.c | 846 +++++++++++++++++++++++++++------------- 1 file changed, 567 insertions(+), 279 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index abe35434d..9b0ae94c1 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -74,6 +74,9 @@ rocksdb_t *rocksdb_g; sqlite3 *sqlite3_db_g; #endif +#include "bulki.h" +#include "bulki_serde.h" + // Check how long PDC has run every OP_INTERVAL operations #define PDC_CHECKPOINT_CHK_OP_INTERVAL 2000 // Checkpoint every INTERVAL_SEC second and at least OP_INTERVAL operations @@ -1191,12 +1194,17 @@ PDC_Server_checkpoint() HashTablePair pair; char checkpoint_file[ADDR_MAX], checkpoint_file_local[ADDR_MAX], cmd[4096]; HashTableIterator hash_table_iter; - char * checkpoint; +// char * checkpoint; char * env_char; - uint64_t checkpoint_size; +// uint64_t checkpoint_size; bool use_tmpfs = false; FILE * file; + // BULKI-specific variables + BULKI *checkpoint_bulki = NULL; + size_t serialized_size = 0; + void * serialized_buffer = NULL; + #ifdef PDC_TIMING // Timing struct timeval pdc_timer_start; @@ -1223,31 +1231,42 @@ PDC_Server_checkpoint() if (pdc_server_rank_g == 0) LOG_INFO("Checkpoint file [%s]\n", checkpoint_file); - if (use_tmpfs) - file = fopen(checkpoint_file_local, "w+"); - else - file = fopen(checkpoint_file, "w+"); - - if (file == NULL) - PGOTO_ERROR(FAIL, "Checkpoint file open error"); + // Initialize BULKI structure - estimate initial capacity + checkpoint_bulki = BULKI_init(10); // todo: this should be 4 - // Checkpoint containers + // ========== Section 1: Checkpoint Containers ========== n_entry = hash_table_num_entries(container_hash_table_g); - fwrite(&n_entry, sizeof(int), 1, file); + + BULKI_Entity *containers_array = empty_BULKI_Array_Entity(); hash_table_iterate(container_hash_table_g, &hash_table_iter); while (n_entry != 0 && hash_table_iter_has_more(&hash_table_iter)) { pair = hash_table_iter_next(&hash_table_iter); cont_head = pair.value; + BULKI *container_entry = BULKI_init(2); + hash_key = PDC_get_hash_by_name(cont_head->cont_name); - fwrite(&hash_key, sizeof(uint32_t), 1, file); - fwrite(cont_head, sizeof(pdc_cont_hash_table_entry_t), 1, file); + BULKI_put(container_entry, + BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); + + // Store the container structure as binary blob + BULKI_put(container_entry, + BULKI_singleton_ENTITY("cont_data", PDC_STRING), + BULKI_ENTITY(cont_head, sizeof(pdc_cont_hash_table_entry_t), PDC_UINT8, PDC_CLS_ARRAY)); + + BULKI_ENTITY_append_BULKI(containers_array, container_entry); } - // DHT + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("containers", PDC_STRING), + containers_array); + + // ========== Section 2: Checkpoint Metadata Hash Table ========== n_entry = hash_table_num_entries(metadata_hash_table_g); - fwrite(&n_entry, sizeof(int), 1, file); + + BULKI_Entity *metadata_entries_array = empty_BULKI_Array_Entity(); hash_table_iterate(metadata_hash_table_g, &hash_table_iter); @@ -1255,83 +1274,218 @@ PDC_Server_checkpoint() pair = hash_table_iter_next(&hash_table_iter); head = pair.value; - fwrite(&head->n_obj, sizeof(int), 1, file); + BULKI *hash_entry = BULKI_init(3); + + // Store number of objects + BULKI_put(hash_entry, + BULKI_singleton_ENTITY("n_obj", PDC_STRING), + BULKI_ENTITY(&head->n_obj, 1, PDC_INT, PDC_CLS_ITEM)); + hash_key = PDC_get_hash_by_name(head->metadata->obj_name); - fwrite(&hash_key, sizeof(uint32_t), 1, file); + BULKI_put(hash_entry, + BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); + + // Array of metadata objects + BULKI_Entity *metadata_objs_array = empty_BULKI_Array_Entity(); // Iterate every metadata structure in current entry DL_FOREACH(head->metadata, elt) { - // Write entire metadata structure - fwrite(elt, sizeof(pdc_metadata_t), 1, file); + BULKI *metadata_obj = BULKI_init(4); // todo: maybe this should be 3 - // Write kv tags + // Store metadata structure + BULKI_put(metadata_obj, + BULKI_singleton_ENTITY("metadata", PDC_STRING), + BULKI_ENTITY(elt, sizeof(pdc_metadata_t), PDC_UINT8, PDC_CLS_ARRAY)); + + // ========== KV Tags ========== DL_COUNT(elt->kvtag_list_head, kvlist_elt, n_kvtag); - fwrite(&n_kvtag, sizeof(int), 1, file); + + BULKI_Entity *kvtags_array = empty_BULKI_Array_Entity(); DL_FOREACH(elt->kvtag_list_head, kvlist_elt) { + BULKI *kvtag_entry = BULKI_init(4); + key_len = strlen(kvlist_elt->kvtag->name) + 1; - fwrite(&key_len, sizeof(int), 1, file); - fwrite(kvlist_elt->kvtag->name, key_len, 1, file); - fwrite(&kvlist_elt->kvtag->size, sizeof(uint32_t), 1, file); - fwrite(&kvlist_elt->kvtag->type, sizeof(int8_t), 1, file); - fwrite(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, 1, file); + BULKI_put(kvtag_entry, + BULKI_singleton_ENTITY("key", PDC_STRING), + BULKI_singleton_ENTITY(kvlist_elt->kvtag->name, PDC_STRING)); + + BULKI_put(kvtag_entry, + BULKI_singleton_ENTITY("size", PDC_STRING), + BULKI_ENTITY(&kvlist_elt->kvtag->size, 1, PDC_UINT32, PDC_CLS_ITEM)); + + BULKI_put(kvtag_entry, + BULKI_singleton_ENTITY("type", PDC_STRING), + BULKI_ENTITY(&kvlist_elt->kvtag->type, 1, PDC_INT8, PDC_CLS_ITEM)); + + BULKI_put(kvtag_entry, + BULKI_singleton_ENTITY("value", PDC_STRING), + BULKI_ENTITY(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, + PDC_UINT8, PDC_CLS_ARRAY)); + + BULKI_ENTITY_append_BULKI(kvtags_array, kvtag_entry); } - // Write region info + BULKI_put(metadata_obj, + BULKI_singleton_ENTITY("kvtags", PDC_STRING), + kvtags_array); + + // ========== Storage Regions ========== n_region = 0; DL_COUNT(elt->storage_region_list_head, region_elt, n_region); - fwrite(&n_region, sizeof(int), 1, file); + + BULKI_Entity *regions_array = empty_BULKI_Array_Entity(); if (n_region > 0) { n_write_region = 0; DL_FOREACH(elt->storage_region_list_head, region_elt) { - fwrite(region_elt, sizeof(region_list_t), 1, file); - n_write_region++; + BULKI *region_entry = BULKI_init(3); + + // Store region structure + BULKI_put(region_entry, + BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); + + // Store histogram if present int has_hist = 0; if (region_elt->region_hist != NULL) has_hist = 1; - fwrite(&has_hist, sizeof(int), 1, file); + + BULKI_put(region_entry, + BULKI_singleton_ENTITY("has_hist", PDC_STRING), + BULKI_ENTITY(&has_hist, 1, PDC_INT, PDC_CLS_ITEM)); + if (has_hist == 1) { - fwrite(®ion_elt->region_hist->dtype, sizeof(int), 1, file); - fwrite(®ion_elt->region_hist->nbin, sizeof(int), 1, file); - fwrite(region_elt->region_hist->range, sizeof(double), - region_elt->region_hist->nbin * 2, file); - fwrite(region_elt->region_hist->bin, sizeof(uint64_t), region_elt->region_hist->nbin, - file); - fwrite(®ion_elt->region_hist->incr, sizeof(double), 1, file); + BULKI *histogram = BULKI_init(5); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("dtype", PDC_STRING), + BULKI_ENTITY(®ion_elt->region_hist->dtype, 1, PDC_INT, PDC_CLS_ITEM)); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("nbin", PDC_STRING), + BULKI_ENTITY(®ion_elt->region_hist->nbin, 1, PDC_INT, PDC_CLS_ITEM)); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("range", PDC_STRING), + BULKI_ENTITY(region_elt->region_hist->range, + region_elt->region_hist->nbin * 2, PDC_DOUBLE, PDC_CLS_ARRAY)); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("bin", PDC_STRING), + BULKI_ENTITY(region_elt->region_hist->bin, + region_elt->region_hist->nbin, PDC_UINT64, PDC_CLS_ARRAY)); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("incr", PDC_STRING), + BULKI_ENTITY(®ion_elt->region_hist->incr, 1, PDC_DOUBLE, PDC_CLS_ITEM)); + + BULKI_put(region_entry, + BULKI_singleton_ENTITY("histogram", PDC_STRING), + BULKI_ENTITY(histogram, 1, PDC_BULKI, PDC_CLS_ITEM)); } + + BULKI_ENTITY_append_BULKI(regions_array, region_entry); + n_write_region++; } if (n_write_region != n_region) LOG_ERROR("Error with number of regions\n"); } + + BULKI_put(metadata_obj, + BULKI_singleton_ENTITY("regions", PDC_STRING), + regions_array); + + BULKI_ENTITY_append_BULKI(metadata_objs_array, metadata_obj); + metadata_size++; region_count += n_region; } // End for metadata entry linked list - } // End for hash table metadata entry - // Note data server region are managed by data server instead of metadata server + BULKI_put(hash_entry, + BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), + metadata_objs_array); + + BULKI_ENTITY_append_BULKI(metadata_entries_array, hash_entry); + } // End for hash table metadata entry + + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), + metadata_entries_array); + + // ========== Section 3: Data Server Regions ========== data_server_region_t *region = NULL; DL_COUNT(dataserver_region_g, region, n_objs); - fwrite(&n_objs, sizeof(int), 1, file); + + BULKI_Entity *dataserver_regions_array = empty_BULKI_Array_Entity(); + DL_FOREACH(dataserver_region_g, region) { - fwrite(®ion->obj_id, sizeof(uint64_t), 1, file); + BULKI *dataserver_obj = BULKI_init(3); + + BULKI_put(dataserver_obj, + BULKI_singleton_ENTITY("obj_id", PDC_STRING), + BULKI_ENTITY(®ion->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); + DL_COUNT(region->region_storage_head, region_elt, n_region); - fwrite(&n_region, sizeof(int), 1, file); + + BULKI_Entity *ds_regions_array = empty_BULKI_Array_Entity(); DL_FOREACH(region->region_storage_head, region_elt) { - fwrite(region_elt, sizeof(region_list_t), 1, file); + BULKI *ds_region = BULKI_init(1); + BULKI_put(ds_region, + BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); + + BULKI_ENTITY_append_BULKI(ds_regions_array, ds_region); } + + BULKI_put(dataserver_obj, + BULKI_singleton_ENTITY("regions", PDC_STRING), + ds_regions_array); + + BULKI_ENTITY_append_BULKI(dataserver_regions_array, dataserver_obj); } - transfer_request_metadata_query_checkpoint(&checkpoint, &checkpoint_size); - fwrite(&checkpoint_size, sizeof(uint64_t), 1, file); - fwrite(checkpoint, checkpoint_size, 1, file); + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), + dataserver_regions_array); + // ========== Section 4: Transfer Request Metadata Query ========== + char * query_checkpoint; + uint64_t query_checkpoint_size; + transfer_request_metadata_query_checkpoint(&query_checkpoint, &query_checkpoint_size); + + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("transfer_query", PDC_STRING), + BULKI_ENTITY(query_checkpoint, query_checkpoint_size, PDC_UINT8, PDC_CLS_ARRAY)); + + // ========== Serialize and Write ========== + serialized_buffer = BULKI_serialize(checkpoint_bulki, &serialized_size); + + if (use_tmpfs) + file = fopen(checkpoint_file_local, "wb"); + else + file = fopen(checkpoint_file, "wb"); + + if (file == NULL) + PGOTO_ERROR(FAIL, "Checkpoint file open error"); + + // Write BULKI magic header for validation + uint32_t magic = 0x424C4B49; // "BLKI" in hex + fwrite(&magic, sizeof(uint32_t), 1, file); + + // Write serialized data + fwrite(serialized_buffer, serialized_size, 1, file); fclose(file); + // Clean up + BULKI_free(checkpoint_bulki, 1); + serialized_buffer = PDC_free(serialized_buffer); + if (use_tmpfs) { #ifdef PDC_TIMING gettimeofday(&pdc_timer_end_rank, 0); @@ -1358,8 +1512,8 @@ PDC_Server_checkpoint() #endif #ifdef PDC_TIMING - LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", metadata_size, region_count, - checkpoint_time_rank); + LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs (BULKI size: %zu bytes)\n", + metadata_size, region_count, checkpoint_time_rank, serialized_size); gettimeofday(&pdc_timer_end, 0); checkpoint_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); @@ -1403,17 +1557,24 @@ PDC_Server_restart(char *filename) FUNC_ENTER(NULL); perr_t ret_value = SUCCEED; - int n_entry, count, i, j, nobj = 0, all_nobj = 0, all_n_region, n_region, n_objs, total_region = 0, - n_kvtag, key_len; - int n_cont, all_cont; + int i, j, nobj = 0, all_nobj = 0, all_n_region, total_region = 0; + int all_cont; pdc_metadata_t * metadata, *elt; region_list_t * region_list; pdc_hash_table_entry_head * entry; pdc_cont_hash_table_entry_t *cont_entry; uint32_t * hash_key; unsigned idx; - uint64_t checkpoint_size; - char * checkpoint_buf; + + // BULKI-specific variables + BULKI * checkpoint_bulki = NULL; + BULKI_Entity * containers_array = NULL; + BULKI_Entity * metadata_entries_array = NULL; + BULKI_Entity * dataserver_regions_array = NULL; + void * file_buffer = NULL; + size_t file_size; + uint32_t magic; + #ifdef PDC_TIMING double start = MPI_Wtime(); #endif @@ -1421,275 +1582,402 @@ PDC_Server_restart(char *filename) // init hash table ret_value = PDC_Server_init_hash_table(); if (ret_value != SUCCEED) - PGOTO_ERROR(FAIL, "Error wtih PDC_Server_init_hash_table"); + PGOTO_ERROR(FAIL, "Error with PDC_Server_init_hash_table"); - FILE *file = fopen(filename, "r"); + // Read entire file into memory + FILE *file = fopen(filename, "rb"); if (file == NULL) PGOTO_ERROR(FAIL, "Error with fopen, filename: [%s]", filename); - if (fread(&n_cont, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_count\n"); - } - all_cont = n_cont; - while (n_cont > 0) { - hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); - if (fread(hash_key, sizeof(uint32_t), 1, file) != 1) { - LOG_ERROR("Read failed for hash_key\n"); - } - total_mem_usage_g += sizeof(uint32_t); - - // Reconstruct hash table - cont_entry = (pdc_cont_hash_table_entry_t *)PDC_malloc(sizeof(pdc_cont_hash_table_entry_t)); - total_mem_usage_g += sizeof(pdc_cont_hash_table_entry_t); - if (fread(cont_entry, sizeof(pdc_cont_hash_table_entry_t), 1, file) != 1) { - LOG_ERROR("Read failed for cont_entry\n"); - } + fseek(file, 0, SEEK_END); + file_size = ftell(file); + fseek(file, 0, SEEK_SET); -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_lock(&pdc_container_hash_table_mutex_g); -#endif - if (hash_table_insert(container_hash_table_g, hash_key, cont_entry) != 1) { - LOG_ERROR("Hash table insert failed\n"); - ret_value = FAIL; - } -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_unlock(&pdc_container_hash_table_mutex_g); -#endif + file_buffer = PDC_malloc(file_size); + if (fread(file_buffer, file_size, 1, file) != 1) { + LOG_ERROR("Failed to read checkpoint file\n"); + fclose(file); + PGOTO_ERROR(FAIL, "Read failed for checkpoint file"); + } + fclose(file); - n_cont--; - } // End while + // Validate magic header + memcpy(&magic, file_buffer, sizeof(uint32_t)); + if (magic != 0x424C4B49) { + LOG_ERROR("Invalid checkpoint file magic header\n"); + PGOTO_ERROR(FAIL, "Invalid checkpoint format"); + } - if (fread(&n_entry, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_entry\n"); + // Deserialize BULKI + checkpoint_bulki = BULKI_deserialize((char *)file_buffer + sizeof(uint32_t)); + if (checkpoint_bulki == NULL) { + LOG_ERROR("Failed to deserialize checkpoint file\n"); + PGOTO_ERROR(FAIL, "Deserialization failed"); } - while (n_entry > 0) { - if (fread(&count, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for count\n"); - } - hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); - if (fread(hash_key, sizeof(uint32_t), 1, file) != 1) { - LOG_ERROR("Read failed for hash_key\n"); - } - total_mem_usage_g += sizeof(uint32_t); - - // Reconstruct hash table - entry = (pdc_hash_table_entry_head *)PDC_malloc(sizeof(pdc_hash_table_entry_head)); - entry->n_obj = 0; - entry->bloom = NULL; - entry->metadata = NULL; - // Init hash table metadata (w/ bloom) with first obj - PDC_Server_hash_table_list_init(entry, hash_key); - - metadata = (pdc_metadata_t *)PDC_calloc(sizeof(pdc_metadata_t), count); - for (i = 0; i < count; i++) { - if (fread(metadata + i, sizeof(pdc_metadata_t), 1, file) != 1) { - LOG_ERROR("Read failed for metadata\n"); - } + // ========== Section 1: Restore Containers ========== + containers_array = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("containers", PDC_STRING)); - (metadata + i)->storage_region_list_head = NULL; - (metadata + i)->region_lock_head = NULL; - (metadata + i)->region_map_head = NULL; - (metadata + i)->region_buf_map_head = NULL; - (metadata + i)->bloom = NULL; - (metadata + i)->prev = NULL; - (metadata + i)->next = NULL; - (metadata + i)->kvtag_list_head = NULL; - (metadata + i)->all_storage_region_distributed = 0; - - // Read kv tags - if (fread(&n_kvtag, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_kvtag\n"); - } - for (j = 0; j < n_kvtag; j++) { - pdc_kvtag_list_t *kvtag_list = (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); - kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); - if (fread(&key_len, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for key_len\n"); - } - kvtag_list->kvtag->name = PDC_malloc(key_len); - if (fread((void *)(kvtag_list->kvtag->name), key_len, 1, file) != 1) { - LOG_ERROR("Read failed for kvtag_list->kvtag->name\n"); - } - if (fread(&kvtag_list->kvtag->size, sizeof(uint32_t), 1, file) != 1) { - LOG_ERROR("Read failed for kvtag_list->kvtag->size\n"); - } - if (fread(&kvtag_list->kvtag->type, sizeof(int8_t), 1, file) != 1) { - LOG_ERROR("Read failed for kvtag_list->kvtag->type\n"); - } - kvtag_list->kvtag->value = PDC_malloc(kvtag_list->kvtag->size); - if (fread(kvtag_list->kvtag->value, kvtag_list->kvtag->size, 1, file) != 1) { - LOG_ERROR("Read failed for kvtag_list->kvtag->value\n"); - } - DL_APPEND((metadata + i)->kvtag_list_head, kvtag_list); - } + if (containers_array != NULL && containers_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *cont_iter = Bent_iterator_init(containers_array, NULL, PDC_BULKI); + all_cont = containers_array->count; - if (fread(&n_region, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_region\n"); - } - if (n_region < 0) - PGOTO_ERROR(FAIL, "Checkpoint file region was less than 0"); + while (Bent_iterator_has_next_BULKI(cont_iter)) { + BULKI *container_entry = Bent_iterator_next_BULKI(cont_iter); - /* if (n_region == 0) */ - /* continue; */ + // Extract hash key + BULKI_Entity *hash_key_ent = BULKI_get(container_entry, + BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); + memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); + total_mem_usage_g += sizeof(uint32_t); - total_region += n_region; + // Extract container data + BULKI_Entity *cont_data_ent = BULKI_get(container_entry, + BULKI_singleton_ENTITY("cont_data", PDC_STRING)); + cont_entry = (pdc_cont_hash_table_entry_t *)PDC_malloc(sizeof(pdc_cont_hash_table_entry_t)); + memcpy(cont_entry, cont_data_ent->data, sizeof(pdc_cont_hash_table_entry_t)); + total_mem_usage_g += sizeof(pdc_cont_hash_table_entry_t); - for (j = 0; j < n_region; j++) { - region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - if (fread(region_list, sizeof(region_list_t), 1, file) != 1) { - LOG_ERROR("Read failed for region_list\n"); - } +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_lock(&pdc_container_hash_table_mutex_g); +#endif + if (hash_table_insert(container_hash_table_g, hash_key, cont_entry) != 1) { + LOG_ERROR("Hash table insert failed\n"); + ret_value = FAIL; + } +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_unlock(&pdc_container_hash_table_mutex_g); +#endif + } + } - int has_hist = 0; - if (fread(&has_hist, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for has_list\n"); - } - if (has_hist == 1) { - region_list->region_hist = (pdc_histogram_t *)PDC_malloc(sizeof(pdc_histogram_t)); - if (fread(®ion_list->region_hist->dtype, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->dtype\n"); - } - if (fread(®ion_list->region_hist->nbin, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->nbin\n"); - } - if (region_list->region_hist->nbin == 0) { - LOG_ERROR("Checkpoint file histogram size is 0\n"); + // ========== Section 2: Restore Metadata ========== + metadata_entries_array = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("metadata_entries", PDC_STRING)); + + if (metadata_entries_array != NULL && metadata_entries_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *entry_iter = Bent_iterator_init(metadata_entries_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(entry_iter)) { + BULKI *hash_entry = Bent_iterator_next_BULKI(entry_iter); + + // Extract n_obj + BULKI_Entity *n_obj_ent = BULKI_get(hash_entry, + BULKI_singleton_ENTITY("n_obj", PDC_STRING)); + int count; + memcpy(&count, n_obj_ent->data, sizeof(int)); + + // Extract hash key + BULKI_Entity *hash_key_ent = BULKI_get(hash_entry, + BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); + memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); + total_mem_usage_g += sizeof(uint32_t); + + // Reconstruct hash table entry + entry = (pdc_hash_table_entry_head *)PDC_malloc(sizeof(pdc_hash_table_entry_head)); + entry->n_obj = 0; + entry->bloom = NULL; + entry->metadata = NULL; + PDC_Server_hash_table_list_init(entry, hash_key); + + metadata = (pdc_metadata_t *)PDC_calloc(sizeof(pdc_metadata_t), count); + + // Extract metadata objects array + BULKI_Entity *metadata_objs_array = BULKI_get(hash_entry, + BULKI_singleton_ENTITY("metadata_objects", PDC_STRING)); + + if (metadata_objs_array != NULL && metadata_objs_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *obj_iter = Bent_iterator_init(metadata_objs_array, NULL, PDC_BULKI); + i = 0; + + while (Bent_iterator_has_next_BULKI(obj_iter) && i < count) { + BULKI *metadata_obj = Bent_iterator_next_BULKI(obj_iter); + + // Extract metadata structure + BULKI_Entity *metadata_ent = BULKI_get(metadata_obj, + BULKI_singleton_ENTITY("metadata", PDC_STRING)); + memcpy(metadata + i, metadata_ent->data, sizeof(pdc_metadata_t)); + + // Initialize pointers + (metadata + i)->storage_region_list_head = NULL; + (metadata + i)->region_lock_head = NULL; + (metadata + i)->region_map_head = NULL; + (metadata + i)->region_buf_map_head = NULL; + (metadata + i)->bloom = NULL; + (metadata + i)->prev = NULL; + (metadata + i)->next = NULL; + (metadata + i)->kvtag_list_head = NULL; + (metadata + i)->all_storage_region_distributed = 0; + + // ========== Restore KV Tags ========== + BULKI_Entity *kvtags_array = BULKI_get(metadata_obj, + BULKI_singleton_ENTITY("kvtags", PDC_STRING)); + + if (kvtags_array != NULL && kvtags_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *kvtag_iter = Bent_iterator_init(kvtags_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(kvtag_iter)) { + BULKI *kvtag_entry = Bent_iterator_next_BULKI(kvtag_iter); + + pdc_kvtag_list_t *kvtag_list = (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); + kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); + + // Extract key + BULKI_Entity *key_ent = BULKI_get(kvtag_entry, + BULKI_singleton_ENTITY("key", PDC_STRING)); + int key_len = strlen((char *)key_ent->data) + 1; + kvtag_list->kvtag->name = PDC_malloc(key_len); + memcpy(kvtag_list->kvtag->name, key_ent->data, key_len); + + // Extract size + BULKI_Entity *size_ent = BULKI_get(kvtag_entry, + BULKI_singleton_ENTITY("size", PDC_STRING)); + memcpy(&kvtag_list->kvtag->size, size_ent->data, sizeof(uint32_t)); + + // Extract type + BULKI_Entity *type_ent = BULKI_get(kvtag_entry, + BULKI_singleton_ENTITY("type", PDC_STRING)); + memcpy(&kvtag_list->kvtag->type, type_ent->data, sizeof(int8_t)); + + // Extract value + BULKI_Entity *value_ent = BULKI_get(kvtag_entry, + BULKI_singleton_ENTITY("value", PDC_STRING)); + kvtag_list->kvtag->value = PDC_malloc(kvtag_list->kvtag->size); + memcpy(kvtag_list->kvtag->value, value_ent->data, kvtag_list->kvtag->size); + + DL_APPEND((metadata + i)->kvtag_list_head, kvtag_list); + } } - region_list->region_hist->range = - (double *)PDC_malloc(sizeof(double) * region_list->region_hist->nbin * 2); - region_list->region_hist->bin = - (uint64_t *)PDC_malloc(sizeof(uint64_t) * region_list->region_hist->nbin); - - if (fread(region_list->region_hist->range, sizeof(double), - region_list->region_hist->nbin * 2, file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->range\n"); - } - if (fread(region_list->region_hist->bin, sizeof(uint64_t), region_list->region_hist->nbin, - file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->bin\n"); + // ========== Restore Storage Regions ========== + BULKI_Entity *regions_array = BULKI_get(metadata_obj, + BULKI_singleton_ENTITY("regions", PDC_STRING)); + + int n_region = 0; + if (regions_array != NULL && regions_array->pdc_type == PDC_BULKI) { + n_region = regions_array->count; + BULKI_Entity_Iterator *region_iter = Bent_iterator_init(regions_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(region_iter)) { + BULKI *region_entry = Bent_iterator_next_BULKI(region_iter); + + region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); + + // Extract region structure + BULKI_Entity *region_ent = BULKI_get(region_entry, + BULKI_singleton_ENTITY("region", PDC_STRING)); + memcpy(region_list, region_ent->data, sizeof(region_list_t)); + + // Extract histogram flag + BULKI_Entity *has_hist_ent = BULKI_get(region_entry, + BULKI_singleton_ENTITY("has_hist", PDC_STRING)); + int has_hist; + memcpy(&has_hist, has_hist_ent->data, sizeof(int)); + + if (has_hist == 1) { + BULKI_Entity *histogram_ent = BULKI_get(region_entry, + BULKI_singleton_ENTITY("histogram", PDC_STRING)); + + if (histogram_ent != NULL && histogram_ent->pdc_type == PDC_BULKI) { + BULKI *histogram = (BULKI *)histogram_ent->data; + + region_list->region_hist = (pdc_histogram_t *)PDC_malloc(sizeof(pdc_histogram_t)); + + BULKI_Entity *dtype_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("dtype", PDC_STRING)); + memcpy(®ion_list->region_hist->dtype, dtype_ent->data, sizeof(int)); + + BULKI_Entity *nbin_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("nbin", PDC_STRING)); + memcpy(®ion_list->region_hist->nbin, nbin_ent->data, sizeof(int)); + + if (region_list->region_hist->nbin == 0) { + LOG_ERROR("Checkpoint file histogram size is 0\n"); + } + + BULKI_Entity *range_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("range", PDC_STRING)); + region_list->region_hist->range = + (double *)PDC_malloc(sizeof(double) * region_list->region_hist->nbin * 2); + memcpy(region_list->region_hist->range, range_ent->data, + sizeof(double) * region_list->region_hist->nbin * 2); + + BULKI_Entity *bin_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("bin", PDC_STRING)); + region_list->region_hist->bin = + (uint64_t *)PDC_malloc(sizeof(uint64_t) * region_list->region_hist->nbin); + memcpy(region_list->region_hist->bin, bin_ent->data, + sizeof(uint64_t) * region_list->region_hist->nbin); + + BULKI_Entity *incr_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("incr", PDC_STRING)); + memcpy(®ion_list->region_hist->incr, incr_ent->data, sizeof(double)); + } + } + + // Initialize region_list fields + region_list->buf = NULL; + region_list->data_size = 1; + for (idx = 0; idx < region_list->ndim; idx++) + region_list->data_size *= region_list->count[idx]; + region_list->is_data_ready = 0; + region_list->shm_fd = 0; + region_list->meta = (metadata + i); + region_list->prev = NULL; + region_list->next = NULL; + region_list->overlap_storage_regions = NULL; + region_list->n_overlap_storage_region = 0; + hg_atomic_init32(&(region_list->buf_map_refcount), 0); + region_list->reg_dirty_from_buf = 0; + region_list->access_type = PDC_NA; + region_list->bulk_handle = NULL; + region_list->lock_handle = NULL; + region_list->addr = NULL; + region_list->obj_id = (metadata + i)->obj_id; + region_list->reg_id = 0; + region_list->from_obj_id = 0; + region_list->client_id = 0; + region_list->is_io_done = 0; + region_list->is_shm_closed = 0; + region_list->seq_id = 0; + region_list->sent_to_server = 0; + region_list->io_cache_region = NULL; + + memset(region_list->shm_addr, 0, ADDR_MAX); + memset(region_list->client_ids, 0, PDC_SERVER_MAX_PROC_PER_NODE * sizeof(uint32_t)); + + if (strstr(region_list->storage_location, "/global/cscratch") != NULL) { + region_list->data_loc_type = PDC_LUSTRE; + } + + DL_APPEND((metadata + i)->storage_region_list_head, region_list); + } } - if (fread(®ion_list->region_hist->incr, sizeof(double), 1, file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->incr\n"); - } - } - region_list->buf = NULL; - region_list->data_size = 1; - for (idx = 0; idx < region_list->ndim; idx++) - region_list->data_size *= region_list->count[idx]; - region_list->is_data_ready = 0; - region_list->shm_fd = 0; - region_list->meta = (metadata + i); - region_list->prev = NULL; - region_list->next = NULL; - region_list->overlap_storage_regions = NULL; - region_list->n_overlap_storage_region = 0; - hg_atomic_init32(&(region_list->buf_map_refcount), 0); - region_list->reg_dirty_from_buf = 0; - region_list->access_type = PDC_NA; - region_list->bulk_handle = NULL; - region_list->lock_handle = NULL; - region_list->addr = NULL; - region_list->obj_id = (metadata + i)->obj_id; - region_list->reg_id = 0; - region_list->from_obj_id = 0; - region_list->client_id = 0; - region_list->is_io_done = 0; - region_list->is_shm_closed = 0; - region_list->seq_id = 0; - region_list->sent_to_server = 0; - region_list->io_cache_region = NULL; - - memset(region_list->shm_addr, 0, ADDR_MAX); - memset(region_list->client_ids, 0, PDC_SERVER_MAX_PROC_PER_NODE * sizeof(uint32_t)); - - if (strstr(region_list->storage_location, "/global/cscratch") != NULL) { - region_list->data_loc_type = PDC_LUSTRE; - } - - DL_APPEND((metadata + i)->storage_region_list_head, region_list); - } // For j - total_region += n_region; + total_region += n_region; + DL_SORT((metadata + i)->storage_region_list_head, region_cmp); - DL_SORT((metadata + i)->storage_region_list_head, region_cmp); - } // For i + i++; + } + } - nobj += count; - total_mem_usage_g += sizeof(pdc_hash_table_entry_head); - total_mem_usage_g += (sizeof(pdc_metadata_t) * count); + nobj += count; + total_mem_usage_g += sizeof(pdc_hash_table_entry_head); + total_mem_usage_g += (sizeof(pdc_metadata_t) * count); - entry->metadata = NULL; + entry->metadata = NULL; - // Insert the previously read metadata to the linked list (hash table entry) - for (i = 0; i < count; i++) { - elt = metadata + i; - // Add to hash list and bloom filter - ret_value = PDC_Server_hash_table_list_insert(entry, elt); - if (ret_value != SUCCEED) - PGOTO_ERROR(FAIL, "Error with hash table recovering from checkpoint file"); + // Insert metadata to hash table + for (i = 0; i < count; i++) { + elt = metadata + i; + ret_value = PDC_Server_hash_table_list_insert(entry, elt); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "Error with hash table recovering from checkpoint file"); + } } - n_entry--; - } - - if (fread(&n_objs, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_objs\n"); } - for (i = 0; i < n_objs; ++i) { - data_server_region_t *new_obj_reg = - (data_server_region_t *)PDC_calloc(1, sizeof(struct data_server_region_t)); - new_obj_reg->fd = -1; - new_obj_reg->storage_location = (char *)PDC_malloc(sizeof(char) * ADDR_MAX); - if (fread(&new_obj_reg->obj_id, sizeof(uint64_t), 1, file) != 1) { - LOG_ERROR("Read failed for obj_id\n"); - } - if (fread(&n_region, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_region\n"); - } - DL_APPEND(dataserver_region_g, new_obj_reg); - for (j = 0; j < n_region; j++) { - region_list_t *new_region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - if (fread(new_region_list, sizeof(region_list_t), 1, file) != 1) { - LOG_ERROR("Read failed for new_region_list\n"); + // ========== Section 3: Restore Data Server Regions ========== + dataserver_regions_array = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING)); + + if (dataserver_regions_array != NULL && dataserver_regions_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *ds_iter = Bent_iterator_init(dataserver_regions_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(ds_iter)) { + BULKI *dataserver_obj = Bent_iterator_next_BULKI(ds_iter); + + data_server_region_t *new_obj_reg = + (data_server_region_t *)PDC_calloc(1, sizeof(struct data_server_region_t)); + new_obj_reg->fd = -1; + new_obj_reg->storage_location = (char *)PDC_malloc(sizeof(char) * ADDR_MAX); + + // Extract obj_id + BULKI_Entity *obj_id_ent = BULKI_get(dataserver_obj, + BULKI_singleton_ENTITY("obj_id", PDC_STRING)); + memcpy(&new_obj_reg->obj_id, obj_id_ent->data, sizeof(uint64_t)); + + // Extract regions + BULKI_Entity *ds_regions_array = BULKI_get(dataserver_obj, + BULKI_singleton_ENTITY("regions", PDC_STRING)); + + if (ds_regions_array != NULL && ds_regions_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *ds_region_iter = Bent_iterator_init(ds_regions_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(ds_region_iter)) { + BULKI *ds_region = Bent_iterator_next_BULKI(ds_region_iter); + + region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); + + BULKI_Entity *region_ent = BULKI_get(ds_region, + BULKI_singleton_ENTITY("region", PDC_STRING)); + memcpy(region_list, region_ent->data, sizeof(region_list_t)); + + // Initialize fields (similar to above) + region_list->buf = NULL; + region_list->data_size = 1; + for (idx = 0; idx < region_list->ndim; idx++) + region_list->data_size *= region_list->count[idx]; + region_list->is_data_ready = 0; + region_list->shm_fd = 0; + region_list->meta = NULL; + region_list->prev = NULL; + region_list->next = NULL; + region_list->overlap_storage_regions = NULL; + region_list->n_overlap_storage_region = 0; + hg_atomic_init32(&(region_list->buf_map_refcount), 0); + region_list->reg_dirty_from_buf = 0; + region_list->access_type = PDC_NA; + + DL_APPEND(new_obj_reg->region_storage_head, region_list); + } } - DL_APPEND(new_obj_reg->region_storage_head, new_region_list); + + DL_APPEND(dataserver_region_g, new_obj_reg); } } - if (fread(&checkpoint_size, sizeof(uint64_t), 1, file) != 1) { - LOG_ERROR("Read failed for checkpoint size\n"); - } - checkpoint_buf = (char *)PDC_malloc(checkpoint_size); - if (fread(checkpoint_buf, checkpoint_size, 1, file) != 1) { - LOG_ERROR("Read failed for checkpoint buf\n"); + // ========== Section 4: Restore Transfer Query ========== + BULKI_Entity *transfer_query_ent = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("transfer_query", PDC_STRING)); + if (transfer_query_ent != NULL) { + // Create a buffer copy from the BULKI entity data + char *checkpoint_buf = (char *)PDC_malloc(transfer_query_ent->count); + memcpy(checkpoint_buf, transfer_query_ent->data, transfer_query_ent->count); + + // Call the init function with the checkpoint buffer + transfer_request_metadata_query_init(pdc_server_size_g, checkpoint_buf); + + // Free the buffer + checkpoint_buf = (char *)PDC_free(checkpoint_buf); } - transfer_request_metadata_query_init(pdc_server_size_g, checkpoint_buf); - checkpoint_buf = (char *)PDC_free(checkpoint_buf); - fclose(file); - file = NULL; + // Clean up + BULKI_free(checkpoint_bulki, 1); + file_buffer = PDC_free(file_buffer); #ifdef ENABLE_MPI MPI_Reduce(&nobj, &all_nobj, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&total_region, &all_n_region, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else - all_nobj = nobj; - all_n_region = total_region; + all_nobj = nobj; + all_n_region = total_region; +#endif + +#ifdef PDC_TIMING + if (pdc_server_rank_g == 0) + LOG_ERROR("Total restart time = %.6f\n", MPI_Wtime() - start); #endif if (pdc_server_rank_g == 0) { - LOG_INFO("Server restarted from saved session, " - "successfully loaded %d containers, %d objects, %d regions...\n", + LOG_INFO("Recovered %d containers, %d objects, %d regions from checkpoint file \n", all_cont, all_nobj, all_n_region); } done: -#ifdef PDC_TIMING - pdc_server_timings->PDCserver_restart += MPI_Wtime() - start; -#endif - FUNC_LEAVE(ret_value); } From 0a53f026ceb0f4a14c4cc0a3800511dfe30d760f Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 27 Feb 2026 15:32:59 -0500 Subject: [PATCH 02/13] Add code to test metadata checkpointing Added code to add kv-tags at scale. Also added code to get and verify tags at scale. The later test code will be used to verify the metadata checkpointing after a server restart. --- src/tests/CMakeLists.txt | 2 + src/tests/tags/kvtag_add_scale.c | 224 ++++++++++++++++++ src/tests/tags/kvtag_get_verify_scale.c | 289 ++++++++++++++++++++++++ 3 files changed, 515 insertions(+) create mode 100644 src/tests/tags/kvtag_add_scale.c create mode 100644 src/tests/tags/kvtag_get_verify_scale.c diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 6f0482fdc..574d84d9f 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -136,6 +136,8 @@ if(BUILD_MPI_TESTING) tags/kvtag_affix_query_scale tags/kvtag_add_get_benchmark tags/kvtag_add_get_scale + tags/kvtag_add_scale + tags/kvtag_get_verify_scale ) if(PDC_ENABLE_IDIOMS) diff --git a/src/tests/tags/kvtag_add_scale.c b/src/tests/tags/kvtag_add_scale.c new file mode 100644 index 000000000..2814a9fbe --- /dev/null +++ b/src/tests/tags/kvtag_add_scale.c @@ -0,0 +1,224 @@ +/* +* Copyright Notice for +* Proactive Data Containers (PDC) Software Library and Utilities +* ----------------------------------------------------------------------------- + +*** Copyright Notice *** + +* Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the +* University of California, through Lawrence Berkeley National Laboratory, +* UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF +* Group (subject to receipt of any required approvals from the U.S. Dept. of +* Energy). All rights reserved. + +* If you have questions about your rights to use or distribute this software, +* please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + +* NOTICE. This Software was developed under funding from the U.S. Department of +* Energy and the U.S. Government consequently retains certain rights. As such, the +* U.S. Government has been granted for itself and others acting on its behalf a +* paid-up, nonexclusive, irrevocable, worldwide license in the Software to +* reproduce, distribute copies to the public, prepare derivative works, and +* perform publicly and display publicly, and to permit other to do so. +*/ + +#include +#include +#include +#include +#include +#include "pdc.h" +#include "pdc_client_connect.h" + +int +assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) +{ + if (rank > size || my_count == NULL || my_start == NULL) { + LOG_INFO("assign_work_to_rank(): Error with input\n"); + return -1; + } + if (nwork < size) { + if (rank < nwork) + *my_count = 1; + else + *my_count = 0; + (*my_start) = rank * (*my_count); + } + else { + (*my_count) = nwork / size; + (*my_start) = rank * (*my_count); + + // Last few ranks may have extra work + if (rank >= size - nwork % size) { + (*my_count)++; + (*my_start) += (rank - (size - nwork % size)); + } + } + + return 1; +} + +void +print_usage(char *name) +{ + /* Modified: Changed usage to only require n_obj and n_add_tag */ + LOG_JUST_PRINT("%s n_obj n_add_tag\n", name); +} + +int +main(int argc, char *argv[]) +{ + pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t * obj_ids; + int n_obj, n_add_tag, my_obj, my_obj_s, my_add_tag, my_add_tag_s; + int obj_1percent = 0, tag_1percent = 0; + int proc_num, my_rank, i, v; + char obj_name[128]; + double stime, total_time, percent_time; + pdc_kvtag_t kvtag; + int ret_value = SUCCEED; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &proc_num); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#endif + /* Modified: Changed argc check from 4 to 3 (removed n_query parameter) */ + if (argc < 3) { + if (my_rank == 0) + print_usage(argv[0]); + PGOTO_DONE(FAIL); + } + n_obj = atoi(argv[1]); + n_add_tag = atoi(argv[2]); + /* Removed: n_query assignment */ + + if (n_add_tag > n_obj) { + if (my_rank == 0) + LOG_ERROR("n_add_tag larger than n_obj! Exiting...\n"); + PGOTO_DONE(FAIL); + } + + assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); + /* Removed: assign_work_to_rank call for n_query */ + assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + + obj_1percent = my_obj / 100; + tag_1percent = my_add_tag / 100; + /* Removed: query_1percent calculation */ + + if (my_rank == 0) + /* Modified: Removed n_query from log message */ + LOG_INFO("Create %d obj, %d tags\n", my_obj, my_add_tag); + + // create a pdc + pdc = PDCinit("pdc"); + + // create a container property + cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + PGOTO_ERROR(FAIL, "Failed to create container property"); + + // create a container + cont = PDCcont_create("c1", cont_prop); + if (cont <= 0) + PGOTO_ERROR(FAIL, "Failed to create container"); + + // create an object property + obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop <= 0) + PGOTO_ERROR(FAIL, "Failed to create object property"); + + // Create a number of objects, add at least one tag to that object + obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + + for (i = 0; i < my_obj; i++) { + sprintf(obj_name, "obj%d", my_obj_s + i); + obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); + if (obj_ids[i] <= 0) + PGOTO_ERROR(FAIL, "Failed to create object"); + + if (i > 0 && i % obj_1percent == 0) { +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / obj_1percent; + int estimated_current_object_number = n_obj / 100 * current_percentage; + double tps = estimated_current_object_number / percent_time; + LOG_INFO("[OBJ PROGRESS %3d%% ] %11d objects, %7.2f seconds, TPS: %10.2f \n", + current_percentage, estimated_current_object_number, percent_time, tps); + } +#endif + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + + if (my_rank == 0) + LOG_INFO("Total time to create %11d objects: %7.2f , throughput %10.2f \n", n_obj, total_time, + n_obj / total_time); + + // Add tags + kvtag.name = "Group"; + kvtag.value = (void *)&v; + kvtag.type = PDC_INT; + kvtag.size = sizeof(int); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + for (i = 0; i < my_add_tag; i++) { + v = i + my_add_tag_s; + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) + PGOTO_ERROR(FAIL, "Failed to add a kvtag to o%d", i + my_obj_s); + + if (i % tag_1percent == 0) { +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / tag_1percent; + int estimated_current_tag_number = n_obj / 100 * current_percentage; + double tps = estimated_current_tag_number / percent_time; + LOG_INFO("[TAG PROGRESS %3d%% ] %11d tags, %7.2f seconds, TPS: %10.2f \n", current_percentage, + estimated_current_tag_number, percent_time, tps); + } +#endif + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + LOG_INFO("Total time to add tags to %11d objects: %7.2f , throughput %10.2f \n", n_add_tag, + total_time, n_add_tag / total_time); + + /* Removed: All query-related code (values allocation, get_tag loop, verification loop) */ + + /* Added: Free obj_ids array before cleanup */ + free(obj_ids); + + if (my_rank == 0) { + /* Modified: Changed message to reflect tag addition completion */ + LOG_INFO("Done adding tags\n"); + } + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return ret_value; +} \ No newline at end of file diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c new file mode 100644 index 000000000..60a79b10d --- /dev/null +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -0,0 +1,289 @@ +/* +* Copyright Notice for +* Proactive Data Containers (PDC) Software Library and Utilities +* ----------------------------------------------------------------------------- + +*** Copyright Notice *** + +* Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the +* University of California, through Lawrence Berkeley National Laboratory, +* UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF +* Group (subject to receipt of any required approvals from the U.S. Dept. of +* Energy). All rights reserved. + +* If you have questions about your rights to use or distribute this software, +* please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + +* NOTICE. This Software was developed under funding from the U.S. Department of +* Energy and the U.S. Government consequently retains certain rights. As such, the +* U.S. Government has been granted for itself and others acting on its behalf a +* paid-up, nonexclusive, irrevocable, worldwide license in the Software to +* reproduce, distribute copies to the public, prepare derivative works, and +* perform publicly and display publicly, and to permit other to do so. +*/ + +#include +#include +#include +#include +#include +#include "pdc.h" +#include "pdc_client_connect.h" + +int +assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) +{ + if (rank > size || my_count == NULL || my_start == NULL) { + LOG_INFO("assign_work_to_rank(): Error with input\n"); + return -1; + } + if (nwork < size) { + if (rank < nwork) + *my_count = 1; + else + *my_count = 0; + (*my_start) = rank * (*my_count); + } + else { + (*my_count) = nwork / size; + (*my_start) = rank * (*my_count); + + // Last few ranks may have extra work + if (rank >= size - nwork % size) { + (*my_count)++; + (*my_start) += (rank - (size - nwork % size)); + } + } + + return 1; +} + +void +print_usage(char *name) +{ + /* Modified: Changed usage to only require n_obj and n_query */ + LOG_JUST_PRINT("%s n_obj n_query\n", name); +} + +int +main(int argc, char *argv[]) +{ + pdcid_t pdc, cont_prop, cont; + pdcid_t * obj_ids; + /* Modified: Removed n_add_tag variable, kept n_query */ + int n_obj, n_query, my_obj, my_obj_s, my_query, my_query_s; + /* Removed: obj_1percent and tag_1percent */ + int query_1percent = 0; + int proc_num, my_rank, i; + char obj_name[128]; + double stime, total_time, percent_time; + pdc_kvtag_t kvtag; + void ** values; + pdc_var_type_t value_type; + size_t value_size; + int ret_value = SUCCEED; + /* Added: Counters for verification statistics */ + int verified_success = 0; + int verified_fail = 0; + int all_verified_success = 0; + int all_verified_fail = 0; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &proc_num); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#endif + /* Modified: Changed argc check from 4 to 3 (removed n_add_tag parameter) */ + if (argc < 3) { + if (my_rank == 0) + print_usage(argv[0]); + PGOTO_DONE(FAIL); + } + n_obj = atoi(argv[1]); + /* Removed: n_add_tag assignment */ + n_query = atoi(argv[2]); + + if (n_query > n_obj) { + if (my_rank == 0) + LOG_ERROR("n_query larger than n_obj! Exiting...\n"); + PGOTO_DONE(FAIL); + } + + /* Removed: assign_work_to_rank call for n_add_tag */ + assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); + assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + + /* Removed: obj_1percent and tag_1percent calculations */ + query_1percent = my_query / 100; + + if (my_rank == 0) + /* Modified: Log message shows only query count */ + LOG_INFO("Open %d obj, query %d tags\n", my_obj, my_query); + + // create a pdc + pdc = PDCinit("pdc"); + + // create a container property + cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + PGOTO_ERROR(FAIL, "Failed to create container property"); + + /* Added: Open existing container instead of creating new one */ + // Open the existing container + cont = PDCcont_open("c1", pdc); + if (cont <= 0) + PGOTO_ERROR(FAIL, "Failed to open container"); + + /* Removed: object property creation - not needed for opening existing objects */ + + /* Added: Open existing objects instead of creating new ones */ + // Open existing objects + obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + + /* Modified: Changed from PDCobj_create to PDCobj_open */ + for (i = 0; i < my_obj; i++) { + sprintf(obj_name, "obj%d", my_obj_s + i); + obj_ids[i] = PDCobj_open(obj_name, pdc); + if (obj_ids[i] <= 0) + PGOTO_ERROR(FAIL, "Failed to open object"); + + /* Added: Progress reporting for object opening (reusing original format) */ + if (i > 0 && query_1percent > 0 && i % query_1percent == 0) { +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / query_1percent; + int estimated_current_object_number = n_obj / 100 * current_percentage; + double tps = estimated_current_object_number / percent_time; + LOG_INFO("[OBJ PROGRESS %3d%% ] %11d objects, %7.2f seconds, TPS: %10.2f \n", + current_percentage, estimated_current_object_number, percent_time, tps); + } +#endif + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + + if (my_rank == 0) + /* Modified: Changed message from "create" to "open" */ + LOG_INFO("Total time to open %11d objects: %7.2f , throughput %10.2f \n", n_obj, total_time, + n_obj / total_time); + + /* Removed: All tag addition code (kvtag initialization and put_tag loop) */ + + /* Added: Initialize kvtag for query operations */ + // Setup kvtag for queries + kvtag.name = "Group"; + + values = (void **)calloc(my_query, sizeof(void *)); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + for (i = 0; i < my_query; i++) { + if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, + (void *)&value_size) < 0) + PGOTO_ERROR(FAIL, "Failed to get a kvtag from o%d\n", i + my_query_s); + + if (i % query_1percent == 0) { +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / query_1percent; + int estimated_current_query_number = n_obj / 100 * current_percentage; + double tps = estimated_current_query_number / percent_time; + LOG_INFO("[QRY PROGRESS %3d%% ] %11d queries, %7.2f seconds, TPS: %10.2f \n", + current_percentage, estimated_current_query_number, percent_time, tps); + } +#endif + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, + total_time, n_query / total_time); + + /* Modified: Changed expected value calculation and added verification counting */ + // The first program adds tags with value = i + my_add_tag_s + // Since we're querying the first my_query objects, and assuming they were tagged + // by ranks in order, we need to calculate the expected value + for (i = 0; i < my_query; i++) { + /* Modified: Calculate expected value based on how tags were originally assigned */ + // When tags were added, each rank tagged objects starting from my_add_tag_s + // For verification, we need to determine which rank added the tag and its offset + int expected_value = i + my_query_s; // Assuming tags were added in order starting from 0 + + /* Added: Count successful and failed verifications instead of immediate error */ + if (*(int *)(values[i]) == expected_value) { + verified_success++; + } else { + verified_fail++; + /* Optional: Log first few failures for debugging */ + if (verified_fail <= 10) { + LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", + i + my_query_s, expected_value, *(int *)(values[i])); + } + } + free(values[i]); + } + + free(values); + /* Added: Free obj_ids array */ + free(obj_ids); + + /* Added: Aggregate verification statistics across all ranks */ +#ifdef ENABLE_MPI + MPI_Reduce(&verified_success, &all_verified_success, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); +#else + all_verified_success = verified_success; + all_verified_fail = verified_fail; +#endif + + if (my_rank == 0) { + /* Modified: Enhanced completion message with verification statistics */ + LOG_INFO("==============================================================\n"); + LOG_INFO("Verification Summary:\n"); + LOG_INFO(" Total queries: %11d\n", n_query); + LOG_INFO(" Successfully verified: %11d (%6.2f%%)\n", + all_verified_success, + 100.0 * all_verified_success / n_query); + LOG_INFO(" Failed verification: %11d (%6.2f%%)\n", + all_verified_fail, + 100.0 * all_verified_fail / n_query); + LOG_INFO("==============================================================\n"); + + if (all_verified_fail > 0) { + LOG_ERROR("WARNING: %d objects failed verification!\n", all_verified_fail); + } else { + LOG_INFO("SUCCESS: All objects verified correctly!\n"); + } + } + + /* Added: Set return value based on verification results */ + if (verified_fail > 0) { + ret_value = FAIL; + } + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return ret_value; +} \ No newline at end of file From 5430bebf1ccfae0af0b2f5b159d22c0435cce35a Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 27 Feb 2026 15:54:17 -0500 Subject: [PATCH 03/13] Refactor kvtag add/get/verify test code --- src/tests/tags/kvtag_add_scale.c | 14 ++------ src/tests/tags/kvtag_get_verify_scale.c | 45 +++++++------------------ 2 files changed, 16 insertions(+), 43 deletions(-) diff --git a/src/tests/tags/kvtag_add_scale.c b/src/tests/tags/kvtag_add_scale.c index 2814a9fbe..4e00097f6 100644 --- a/src/tests/tags/kvtag_add_scale.c +++ b/src/tests/tags/kvtag_add_scale.c @@ -61,7 +61,7 @@ assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) void print_usage(char *name) { - /* Modified: Changed usage to only require n_obj and n_add_tag */ + // required parameters: n_obj and n_add_tag LOG_JUST_PRINT("%s n_obj n_add_tag\n", name); } @@ -83,7 +83,6 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - /* Modified: Changed argc check from 4 to 3 (removed n_query parameter) */ if (argc < 3) { if (my_rank == 0) print_usage(argv[0]); @@ -91,7 +90,6 @@ main(int argc, char *argv[]) } n_obj = atoi(argv[1]); n_add_tag = atoi(argv[2]); - /* Removed: n_query assignment */ if (n_add_tag > n_obj) { if (my_rank == 0) @@ -100,15 +98,12 @@ main(int argc, char *argv[]) } assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); - /* Removed: assign_work_to_rank call for n_query */ assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); obj_1percent = my_obj / 100; tag_1percent = my_add_tag / 100; - /* Removed: query_1percent calculation */ if (my_rank == 0) - /* Modified: Removed n_query from log message */ LOG_INFO("Create %d obj, %d tags\n", my_obj, my_add_tag); // create a pdc @@ -129,7 +124,7 @@ main(int argc, char *argv[]) if (obj_prop <= 0) PGOTO_ERROR(FAIL, "Failed to create object property"); - // Create a number of objects, add at least one tag to that object + // create a number of objects, add at least one tag to that object obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); #ifdef ENABLE_MPI @@ -205,13 +200,10 @@ main(int argc, char *argv[]) LOG_INFO("Total time to add tags to %11d objects: %7.2f , throughput %10.2f \n", n_add_tag, total_time, n_add_tag / total_time); - /* Removed: All query-related code (values allocation, get_tag loop, verification loop) */ - - /* Added: Free obj_ids array before cleanup */ free(obj_ids); if (my_rank == 0) { - /* Modified: Changed message to reflect tag addition completion */ + // confirming tag addition completion LOG_INFO("Done adding tags\n"); } diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index 60a79b10d..bdce6df81 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -61,7 +61,7 @@ assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) void print_usage(char *name) { - /* Modified: Changed usage to only require n_obj and n_query */ + // required parameters: n_obj and n_query LOG_JUST_PRINT("%s n_obj n_query\n", name); } @@ -70,9 +70,7 @@ main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont; pdcid_t * obj_ids; - /* Modified: Removed n_add_tag variable, kept n_query */ int n_obj, n_query, my_obj, my_obj_s, my_query, my_query_s; - /* Removed: obj_1percent and tag_1percent */ int query_1percent = 0; int proc_num, my_rank, i; char obj_name[128]; @@ -82,7 +80,7 @@ main(int argc, char *argv[]) pdc_var_type_t value_type; size_t value_size; int ret_value = SUCCEED; - /* Added: Counters for verification statistics */ + // counters for verification statistics int verified_success = 0; int verified_fail = 0; int all_verified_success = 0; @@ -93,14 +91,12 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - /* Modified: Changed argc check from 4 to 3 (removed n_add_tag parameter) */ if (argc < 3) { if (my_rank == 0) print_usage(argv[0]); PGOTO_DONE(FAIL); } n_obj = atoi(argv[1]); - /* Removed: n_add_tag assignment */ n_query = atoi(argv[2]); if (n_query > n_obj) { @@ -109,15 +105,12 @@ main(int argc, char *argv[]) PGOTO_DONE(FAIL); } - /* Removed: assign_work_to_rank call for n_add_tag */ assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); - /* Removed: obj_1percent and tag_1percent calculations */ query_1percent = my_query / 100; if (my_rank == 0) - /* Modified: Log message shows only query count */ LOG_INFO("Open %d obj, query %d tags\n", my_obj, my_query); // create a pdc @@ -128,15 +121,11 @@ main(int argc, char *argv[]) if (cont_prop <= 0) PGOTO_ERROR(FAIL, "Failed to create container property"); - /* Added: Open existing container instead of creating new one */ // Open the existing container cont = PDCcont_open("c1", pdc); if (cont <= 0) PGOTO_ERROR(FAIL, "Failed to open container"); - /* Removed: object property creation - not needed for opening existing objects */ - - /* Added: Open existing objects instead of creating new ones */ // Open existing objects obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); @@ -145,14 +134,14 @@ main(int argc, char *argv[]) stime = MPI_Wtime(); #endif - /* Modified: Changed from PDCobj_create to PDCobj_open */ + // open already created PDC objects and query tags for (i = 0; i < my_obj; i++) { sprintf(obj_name, "obj%d", my_obj_s + i); obj_ids[i] = PDCobj_open(obj_name, pdc); if (obj_ids[i] <= 0) PGOTO_ERROR(FAIL, "Failed to open object"); - /* Added: Progress reporting for object opening (reusing original format) */ + // progress reporting for object opening if (i > 0 && query_1percent > 0 && i % query_1percent == 0) { #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); @@ -174,13 +163,9 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) - /* Modified: Changed message from "create" to "open" */ LOG_INFO("Total time to open %11d objects: %7.2f , throughput %10.2f \n", n_obj, total_time, n_obj / total_time); - /* Removed: All tag addition code (kvtag initialization and put_tag loop) */ - - /* Added: Initialize kvtag for query operations */ // Setup kvtag for queries kvtag.name = "Group"; @@ -218,22 +203,19 @@ main(int argc, char *argv[]) LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, total_time, n_query / total_time); - /* Modified: Changed expected value calculation and added verification counting */ - // The first program adds tags with value = i + my_add_tag_s - // Since we're querying the first my_query objects, and assuming they were tagged - // by ranks in order, we need to calculate the expected value + // `kvtag_add_get_scale' program adds tags with value = i + my_add_tag_s + // calculate and checking the expected value for (i = 0; i < my_query; i++) { - /* Modified: Calculate expected value based on how tags were originally assigned */ - // When tags were added, each rank tagged objects starting from my_add_tag_s - // For verification, we need to determine which rank added the tag and its offset + // when tags were added, each rank tagged objects starting from my_add_tag_s + // for verification, we need to determine which rank added the tag and its offset int expected_value = i + my_query_s; // Assuming tags were added in order starting from 0 - /* Added: Count successful and failed verifications instead of immediate error */ + // count successful and failed verifications instead of immediate error if (*(int *)(values[i]) == expected_value) { verified_success++; } else { verified_fail++; - /* Optional: Log first few failures for debugging */ + // log first 10 failures for debugging if (verified_fail <= 10) { LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", i + my_query_s, expected_value, *(int *)(values[i])); @@ -243,10 +225,9 @@ main(int argc, char *argv[]) } free(values); - /* Added: Free obj_ids array */ free(obj_ids); - /* Added: Aggregate verification statistics across all ranks */ + // aggregate verification statistics across all ranks #ifdef ENABLE_MPI MPI_Reduce(&verified_success, &all_verified_success, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); @@ -256,7 +237,7 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) { - /* Modified: Enhanced completion message with verification statistics */ + // completion message with verification statistics LOG_INFO("==============================================================\n"); LOG_INFO("Verification Summary:\n"); LOG_INFO(" Total queries: %11d\n", n_query); @@ -275,7 +256,7 @@ main(int argc, char *argv[]) } } - /* Added: Set return value based on verification results */ + // set return value based on verification results if (verified_fail > 0) { ret_value = FAIL; } From 602c5e66734a81a94e395b19e5756d05c1db2f7f Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 17 Mar 2026 13:03:40 -0400 Subject: [PATCH 04/13] Add bulki-based new transfer request metadata query for checkpointing --- src/server/pdc_server.c | 43 ++-- ...dc_server_region_transfer_metadata_query.h | 5 + ...dc_server_region_transfer_metadata_query.c | 202 ++++++++++++++++++ 3 files changed, 235 insertions(+), 15 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 9b0ae94c1..4038e7217 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -949,7 +949,7 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) } else { // We are starting a brand new server - transfer_request_metadata_query_init(pdc_server_size_g, NULL); + transfer_request_metadata_query_init_bulki(pdc_server_size_g, NULL); if (is_hash_table_init_g != 1) { ret_value = PDC_Server_init_hash_table(); if (ret_value != SUCCEED) @@ -1454,14 +1454,18 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), dataserver_regions_array); - // ========== Section 4: Transfer Request Metadata Query ========== - char * query_checkpoint; - uint64_t query_checkpoint_size; - transfer_request_metadata_query_checkpoint(&query_checkpoint, &query_checkpoint_size); + // transfer request metadata query + BULKI *transfer_query_bulki = NULL; + ret_value = transfer_request_metadata_query_checkpoint_bulki(&transfer_query_bulki); + + if (ret_value != SUCCEED || transfer_query_bulki == NULL) { + LOG_ERROR("Failed to create transfer query checkpoint\n"); + PGOTO_ERROR(FAIL, "Transfer query checkpoint failed"); + } BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING), - BULKI_ENTITY(query_checkpoint, query_checkpoint_size, PDC_UINT8, PDC_CLS_ARRAY)); + BULKI_ENTITY(transfer_query_bulki, 1, PDC_BULKI, PDC_CLS_ITEM)); // ========== Serialize and Write ========== serialized_buffer = BULKI_serialize(checkpoint_bulki, &serialized_size); @@ -1940,19 +1944,28 @@ PDC_Server_restart(char *filename) } } - // ========== Section 4: Restore Transfer Query ========== + // restore transfer query BULKI_Entity *transfer_query_ent = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING)); - if (transfer_query_ent != NULL) { - // Create a buffer copy from the BULKI entity data - char *checkpoint_buf = (char *)PDC_malloc(transfer_query_ent->count); - memcpy(checkpoint_buf, transfer_query_ent->data, transfer_query_ent->count); - // Call the init function with the checkpoint buffer - transfer_request_metadata_query_init(pdc_server_size_g, checkpoint_buf); + if (transfer_query_ent != NULL && transfer_query_ent->pdc_type == PDC_BULKI) { + // Extract the nested BULKI containing transfer query data + BULKI *transfer_query_bulki = (BULKI *)transfer_query_ent->data; - // Free the buffer - checkpoint_buf = (char *)PDC_free(checkpoint_buf); + // initialize transfer query system with BULKI checkpoint + ret_value = transfer_request_metadata_query_init_bulki(pdc_server_size_g, transfer_query_bulki); + if (ret_value != SUCCEED) { + LOG_ERROR("Failed to restore transfer query from checkpoint\n"); + PGOTO_ERROR(FAIL, "Transfer query restoration failed"); + } + } + else { + // note: no transfer query data in checkpoint, initialize fresh + ret_value = transfer_request_metadata_query_init_bulki(pdc_server_size_g, NULL); + if (ret_value != SUCCEED) { + LOG_ERROR("Failed to initialize transfer query system\n"); + PGOTO_ERROR(FAIL, "Transfer query initialization failed"); + } } // Clean up diff --git a/src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h b/src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h index fc2ec3a7a..5755d5e4e 100644 --- a/src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h +++ b/src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h @@ -1,9 +1,14 @@ #ifndef PDC_SERVER_REGION_TRANSFER_METADATA_QUERY_H #define PDC_SERVER_REGION_TRANSFER_METADATA_QUERY_H +#include "bulki.h" +#include "bulki_serde.h" + perr_t transfer_request_metadata_query_init(int pdc_server_size_input, char *checkpoint); +perr_t transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *checkpoint_bulki); perr_t transfer_request_metadata_query_finalize(); perr_t transfer_request_metadata_query_checkpoint(char **checkpoint, uint64_t *checkpoint_size); +perr_t transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki); perr_t transfer_request_metadata_query_lookup_query_buf(uint64_t query_id, char **buf_ptr); uint64_t transfer_request_metadata_query_parse(int32_t n_objs, char *buf, uint8_t partition_type, uint64_t *total_buf_size_ptr); diff --git a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c index c66ecd905..8a2c8dc8e 100644 --- a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c +++ b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c @@ -53,6 +53,128 @@ static uint64_t transfer_request_metadata_query_append(uint64_t obj_id, int ndim static uint64_t metadata_query_buf_create(pdc_obj_region_metadata *regions, int size, uint64_t *total_buf_size_ptr); +perr_t +transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *checkpoint_bulki) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + + metadata_server_objs = NULL; + metadata_server_objs_end = NULL; + metadata_query_buf_head = NULL; + metadata_query_buf_end = NULL; + pdc_server_size = pdc_server_size_input; + data_server_bytes = (uint64_t *)PDC_calloc(pdc_server_size, sizeof(uint64_t)); + query_id_g = 100000; + + pthread_mutex_init(&metadata_query_mutex, NULL); + + if (checkpoint_bulki != NULL) { + BULKI_Entity *objects_array = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("objects", PDC_STRING)); + + if (objects_array == NULL || objects_array->pdc_type != PDC_BULKI) { + LOG_ERROR("Invalid transfer query checkpoint: missing or invalid 'objects' field\n"); + PGOTO_ERROR(FAIL, "Invalid checkpoint format"); + } + + BULKI_Entity_Iterator *obj_iter = Bent_iterator_init(objects_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(obj_iter)) { + BULKI *obj_bulki = Bent_iterator_next_BULKI(obj_iter); + + pdc_obj_metadata_pkg *obj_pkg = + (pdc_obj_metadata_pkg *)PDC_malloc(sizeof(pdc_obj_metadata_pkg)); + + BULKI_Entity *obj_id_ent = BULKI_get(obj_bulki, + BULKI_singleton_ENTITY("obj_id", PDC_STRING)); + if (obj_id_ent == NULL) { + LOG_ERROR("Missing obj_id in checkpoint object\n"); + PDC_free(obj_pkg); + continue; + } + memcpy(&obj_pkg->obj_id, obj_id_ent->data, sizeof(uint64_t)); + + BULKI_Entity *ndim_ent = BULKI_get(obj_bulki, + BULKI_singleton_ENTITY("ndim", PDC_STRING)); + if (ndim_ent == NULL) { + LOG_ERROR("Missing ndim in checkpoint object\n"); + PDC_free(obj_pkg); + continue; + } + memcpy(&obj_pkg->ndim, ndim_ent->data, sizeof(int)); + + obj_pkg->regions = NULL; + obj_pkg->regions_end = NULL; + obj_pkg->next = NULL; + + BULKI_Entity *regions_array = BULKI_get(obj_bulki, + BULKI_singleton_ENTITY("regions", PDC_STRING)); + + if (regions_array != NULL && regions_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *region_iter = Bent_iterator_init(regions_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(region_iter)) { + BULKI *region_bulki = Bent_iterator_next_BULKI(region_iter); + + pdc_region_metadata_pkg *region_pkg = + (pdc_region_metadata_pkg *)PDC_malloc(sizeof(pdc_region_metadata_pkg)); + + region_pkg->reg_offset = (uint64_t *)PDC_malloc(sizeof(uint64_t) * obj_pkg->ndim * 2); + region_pkg->reg_size = region_pkg->reg_offset + obj_pkg->ndim; + + BULKI_Entity *server_id_ent = BULKI_get(region_bulki, + BULKI_singleton_ENTITY("data_server_id", PDC_STRING)); + if (server_id_ent != NULL) { + memcpy(®ion_pkg->data_server_id, server_id_ent->data, sizeof(uint32_t)); + } else { + LOG_ERROR("Missing data_server_id in checkpoint region\n"); + PDC_free(region_pkg->reg_offset); + PDC_free(region_pkg); + continue; + } + + BULKI_Entity *offset_size_ent = BULKI_get(region_bulki, + BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING)); + if (offset_size_ent != NULL) { + memcpy(region_pkg->reg_offset, offset_size_ent->data, + sizeof(uint64_t) * obj_pkg->ndim * 2); + } else { + LOG_ERROR("Missing reg_offset_size in checkpoint region\n"); + PDC_free(region_pkg->reg_offset); + PDC_free(region_pkg); + continue; + } + + region_pkg->next = NULL; + + if (obj_pkg->regions == NULL) { + obj_pkg->regions = region_pkg; + obj_pkg->regions_end = region_pkg; + } else { + obj_pkg->regions_end->next = region_pkg; + obj_pkg->regions_end = region_pkg; + } + } + } + + if (metadata_server_objs == NULL) { + metadata_server_objs = obj_pkg; + metadata_server_objs_end = obj_pkg; + } else { + metadata_server_objs_end->next = obj_pkg; + metadata_server_objs_end = obj_pkg; + } + } + + LOG_DEBUG("Transfer query checkpoint restored successfully\n"); + } + +done: + FUNC_LEAVE(ret_value); +} + /** * Entry function for this class. Should be only called once at the beginning of Server init. * If checkpoint is not NULL, then load previously checkpointed metadata to static variables. @@ -167,6 +289,86 @@ transfer_request_metadata_query_finalize() FUNC_LEAVE(ret_value); } +perr_t +transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + pdc_obj_metadata_pkg * obj_temp; + pdc_region_metadata_pkg *region_temp; + int obj_count = 0; + BULKI * bulki = NULL; + +// if (checkpoint_bulki == NULL) { +// LOG_ERROR("checkpoint_bulki output parameter is NULL\n"); +// PGOTO_ERROR(FAIL, "Invalid parameter"); +// } + + pthread_mutex_lock(&metadata_query_mutex); + + bulki = BULKI_init(1); + BULKI_Entity *objects_array = empty_BULKI_Array_Entity(); + + obj_temp = metadata_server_objs; + while (obj_temp) { + BULKI *obj_bulki = BULKI_init(3); + + BULKI_put(obj_bulki, + BULKI_singleton_ENTITY("obj_id", PDC_STRING), + BULKI_ENTITY(&obj_temp->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); + + BULKI_put(obj_bulki, + BULKI_singleton_ENTITY("ndim", PDC_STRING), + BULKI_ENTITY(&obj_temp->ndim, 1, PDC_INT, PDC_CLS_ITEM)); + + BULKI_Entity *regions_array = empty_BULKI_Array_Entity(); + + region_temp = obj_temp->regions; + while (region_temp) { + BULKI *region_bulki = BULKI_init(2); + + BULKI_put(region_bulki, + BULKI_singleton_ENTITY("data_server_id", PDC_STRING), + BULKI_ENTITY(®ion_temp->data_server_id, 1, PDC_UINT32, PDC_CLS_ITEM)); + + BULKI_put(region_bulki, + BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), + BULKI_ENTITY(region_temp->reg_offset, obj_temp->ndim * 2, + PDC_UINT64, PDC_CLS_ARRAY)); + + BULKI_ENTITY_append_BULKI(regions_array, region_bulki); + region_temp = region_temp->next; + } + + BULKI_put(obj_bulki, + BULKI_singleton_ENTITY("regions", PDC_STRING), + regions_array); + + BULKI_ENTITY_append_BULKI(objects_array, obj_bulki); + + obj_count++; + obj_temp = obj_temp->next; + } + + BULKI_put(bulki, + BULKI_singleton_ENTITY("objects", PDC_STRING), + objects_array); + + pthread_mutex_unlock(&metadata_query_mutex); + + *checkpoint_bulki = bulki; + + LOG_DEBUG("Transfer query checkpoint created: %d objects\n", obj_count); + +done: + if (ret_value != SUCCEED && bulki != NULL) { + BULKI_free(bulki, 1); + } + + FUNC_LEAVE(ret_value); +} + /** * Checkpoint static variables in this file into a contiguous buffer. * checkpoint_size is the total number of bytes allocated. From 050a723f9e4e71abd09a8f5951f5072a41645bee Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 17 Mar 2026 13:11:06 -0400 Subject: [PATCH 05/13] Update call to bulki serialization and de-serialization directly to and from file instead of using a buffer Also added version control for the bulki-based checkpointing. --- src/server/pdc_server.c | 85 +++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 4038e7217..eeb365b85 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -77,6 +77,14 @@ sqlite3 *sqlite3_db_g; #include "bulki.h" #include "bulki_serde.h" +// PDC checkpoint version management +#define PDC_CHECKPOINT_MAGIC_2026 "PDC26.03" // Year 2026, month 03 +#define PDC_CHECKPOINT_MAGIC_2027 "PDC27.01" // Year 2027, month 01 + +#define PDC_CHECKPOINT_MAGIC_LEN 8 // All must be same length +#define PDC_CHECKPOINT_MAGIC_CURRENT PDC_CHECKPOINT_MAGIC_2026 +#define PDC_CHECKPOINT_VERSION_MATCH(m, v) (strncmp((m), (v), 8) == 0) + // Check how long PDC has run every OP_INTERVAL operations #define PDC_CHECKPOINT_CHK_OP_INTERVAL 2000 // Checkpoint every INTERVAL_SEC second and at least OP_INTERVAL operations @@ -1194,16 +1202,12 @@ PDC_Server_checkpoint() HashTablePair pair; char checkpoint_file[ADDR_MAX], checkpoint_file_local[ADDR_MAX], cmd[4096]; HashTableIterator hash_table_iter; -// char * checkpoint; char * env_char; -// uint64_t checkpoint_size; bool use_tmpfs = false; FILE * file; // BULKI-specific variables BULKI *checkpoint_bulki = NULL; - size_t serialized_size = 0; - void * serialized_buffer = NULL; #ifdef PDC_TIMING // Timing @@ -1234,7 +1238,12 @@ PDC_Server_checkpoint() // Initialize BULKI structure - estimate initial capacity checkpoint_bulki = BULKI_init(10); // todo: this should be 4 - // ========== Section 1: Checkpoint Containers ========== + // BULKI version number for validation + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("version_number", PDC_STRING), + BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); + + // checkpoint containers n_entry = hash_table_num_entries(container_hash_table_g); BULKI_Entity *containers_array = empty_BULKI_Array_Entity(); @@ -1468,8 +1477,6 @@ PDC_Server_checkpoint() BULKI_ENTITY(transfer_query_bulki, 1, PDC_BULKI, PDC_CLS_ITEM)); // ========== Serialize and Write ========== - serialized_buffer = BULKI_serialize(checkpoint_bulki, &serialized_size); - if (use_tmpfs) file = fopen(checkpoint_file_local, "wb"); else @@ -1478,17 +1485,13 @@ PDC_Server_checkpoint() if (file == NULL) PGOTO_ERROR(FAIL, "Checkpoint file open error"); - // Write BULKI magic header for validation - uint32_t magic = 0x424C4B49; // "BLKI" in hex - fwrite(&magic, sizeof(uint32_t), 1, file); - - // Write serialized data - fwrite(serialized_buffer, serialized_size, 1, file); - fclose(file); + // note: BULKI_Entity_serialize_to_file() still do in-memory serialization + // note: need to implement a streaming file writing without in-memory serialization + BULKI_serialize_to_file(checkpoint_bulki, file); + file = NULL; // file was closed by BULKI_serialize_to_file() // Clean up BULKI_free(checkpoint_bulki, 1); - serialized_buffer = PDC_free(serialized_buffer); if (use_tmpfs) { #ifdef PDC_TIMING @@ -1575,9 +1578,8 @@ PDC_Server_restart(char *filename) BULKI_Entity * containers_array = NULL; BULKI_Entity * metadata_entries_array = NULL; BULKI_Entity * dataserver_regions_array = NULL; - void * file_buffer = NULL; - size_t file_size; - uint32_t magic; + char magic[PDC_CHECKPOINT_MAGIC_LEN + 1]; + FILE * file = NULL; #ifdef PDC_TIMING double start = MPI_Wtime(); @@ -1588,38 +1590,39 @@ PDC_Server_restart(char *filename) if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "Error with PDC_Server_init_hash_table"); - // Read entire file into memory - FILE *file = fopen(filename, "rb"); - if (file == NULL) + // open file + file = fopen(filename, "rb"); + if (file == NULL) { PGOTO_ERROR(FAIL, "Error with fopen, filename: [%s]", filename); - - fseek(file, 0, SEEK_END); - file_size = ftell(file); - fseek(file, 0, SEEK_SET); - - file_buffer = PDC_malloc(file_size); - if (fread(file_buffer, file_size, 1, file) != 1) { - LOG_ERROR("Failed to read checkpoint file\n"); - fclose(file); - PGOTO_ERROR(FAIL, "Read failed for checkpoint file"); } - fclose(file); - // Validate magic header - memcpy(&magic, file_buffer, sizeof(uint32_t)); - if (magic != 0x424C4B49) { - LOG_ERROR("Invalid checkpoint file magic header\n"); - PGOTO_ERROR(FAIL, "Invalid checkpoint format"); - } + // Read and validate magic header +// fread(magic, 1, PDC_CHECKPOINT_MAGIC_LEN, file); +// magic[PDC_CHECKPOINT_MAGIC_LEN] = '\0'; +// if (!PDC_CHECKPOINT_VERSION_MATCH(magic, PDC_CHECKPOINT_MAGIC_CURRENT)) { +// LOG_ERROR("Invalid checkpoint file magic header: '%s'\n", magic); +// fclose(file); +// PGOTO_ERROR(FAIL, "Invalid checkpoint format"); +// } + + // Note: BULKI_deserialize_from_file will close the file + checkpoint_bulki = BULKI_deserialize_from_file(file); + file = NULL; // File was closed by BULKI_deserialize_from_file - // Deserialize BULKI - checkpoint_bulki = BULKI_deserialize((char *)file_buffer + sizeof(uint32_t)); if (checkpoint_bulki == NULL) { LOG_ERROR("Failed to deserialize checkpoint file\n"); PGOTO_ERROR(FAIL, "Deserialization failed"); } - // ========== Section 1: Restore Containers ========== + BULKI_Entity *version_entity = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING)); + int equal = BULKI_Entity_equal(version_entity, BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); + if (!equal) { + LOG_ERROR("Checkpoint version mismatch: expected '%s', found '%s'\n", + PDC_CHECKPOINT_MAGIC_CURRENT, version_entity ? (char *)version_entity->data : "NULL"); + PGOTO_ERROR(FAIL, "Checkpoint version mismatch"); + } + + // restore containers containers_array = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING)); From 7efb94527b414f6fdcd49b89c79a68660d219a4d Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 17 Mar 2026 13:12:24 -0400 Subject: [PATCH 06/13] Refactor bulki-based checkpointing integration in pdc-server --- src/server/pdc_server.c | 87 +++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index eeb365b85..ae9696f6c 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1235,8 +1235,8 @@ PDC_Server_checkpoint() if (pdc_server_rank_g == 0) LOG_INFO("Checkpoint file [%s]\n", checkpoint_file); - // Initialize BULKI structure - estimate initial capacity - checkpoint_bulki = BULKI_init(10); // todo: this should be 4 + // initialize BULKI structure - estimate initial capacity + checkpoint_bulki = BULKI_init(5); // BULKI version number for validation BULKI_put(checkpoint_bulki, @@ -1260,7 +1260,7 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); - // Store the container structure as binary blob + // store the container structure as binary blob BULKI_put(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING), BULKI_ENTITY(cont_head, sizeof(pdc_cont_hash_table_entry_t), PDC_UINT8, PDC_CLS_ARRAY)); @@ -1272,7 +1272,7 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("containers", PDC_STRING), containers_array); - // ========== Section 2: Checkpoint Metadata Hash Table ========== + // checkpoint metadata hash table n_entry = hash_table_num_entries(metadata_hash_table_g); BULKI_Entity *metadata_entries_array = empty_BULKI_Array_Entity(); @@ -1295,20 +1295,20 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); - // Array of metadata objects + // array of metadata objects BULKI_Entity *metadata_objs_array = empty_BULKI_Array_Entity(); - // Iterate every metadata structure in current entry + // iterate every metadata structure in current entry DL_FOREACH(head->metadata, elt) { - BULKI *metadata_obj = BULKI_init(4); // todo: maybe this should be 3 + BULKI *metadata_obj = BULKI_init(3); - // Store metadata structure + // store metadata structure BULKI_put(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING), BULKI_ENTITY(elt, sizeof(pdc_metadata_t), PDC_UINT8, PDC_CLS_ARRAY)); - // ========== KV Tags ========== + // kv tags DL_COUNT(elt->kvtag_list_head, kvlist_elt, n_kvtag); BULKI_Entity *kvtags_array = empty_BULKI_Array_Entity(); @@ -1316,7 +1316,6 @@ PDC_Server_checkpoint() { BULKI *kvtag_entry = BULKI_init(4); - key_len = strlen(kvlist_elt->kvtag->name) + 1; BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING), BULKI_singleton_ENTITY(kvlist_elt->kvtag->name, PDC_STRING)); @@ -1341,7 +1340,7 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("kvtags", PDC_STRING), kvtags_array); - // ========== Storage Regions ========== + // storage regions n_region = 0; DL_COUNT(elt->storage_region_list_head, region_elt, n_region); @@ -1352,12 +1351,12 @@ PDC_Server_checkpoint() { BULKI *region_entry = BULKI_init(3); - // Store region structure + // store region structure BULKI_put(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING), BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); - // Store histogram if present + // store histogram if present int has_hist = 0; if (region_elt->region_hist != NULL) has_hist = 1; @@ -1412,20 +1411,20 @@ PDC_Server_checkpoint() metadata_size++; region_count += n_region; - } // End for metadata entry linked list + } // end for metadata entry linked list BULKI_put(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), metadata_objs_array); BULKI_ENTITY_append_BULKI(metadata_entries_array, hash_entry); - } // End for hash table metadata entry + } // end for hash table metadata entry BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), metadata_entries_array); - // ========== Section 3: Data Server Regions ========== + // data server regions data_server_region_t *region = NULL; DL_COUNT(dataserver_region_g, region, n_objs); @@ -1433,7 +1432,7 @@ PDC_Server_checkpoint() DL_FOREACH(dataserver_region_g, region) { - BULKI *dataserver_obj = BULKI_init(3); + BULKI *dataserver_obj = BULKI_init(2); BULKI_put(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING), @@ -1519,8 +1518,8 @@ PDC_Server_checkpoint() #endif #ifdef PDC_TIMING - LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs (BULKI size: %zu bytes)\n", - metadata_size, region_count, checkpoint_time_rank, serialized_size); + LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", + metadata_size, region_count, checkpoint_time_rank); gettimeofday(&pdc_timer_end, 0); checkpoint_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); @@ -1660,7 +1659,7 @@ PDC_Server_restart(char *filename) } } - // ========== Section 2: Restore Metadata ========== + // restore metadata metadata_entries_array = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING)); @@ -1670,20 +1669,20 @@ PDC_Server_restart(char *filename) while (Bent_iterator_has_next_BULKI(entry_iter)) { BULKI *hash_entry = Bent_iterator_next_BULKI(entry_iter); - // Extract n_obj + // extract n_obj BULKI_Entity *n_obj_ent = BULKI_get(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING)); int count; memcpy(&count, n_obj_ent->data, sizeof(int)); - // Extract hash key + // extract hash key BULKI_Entity *hash_key_ent = BULKI_get(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING)); hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); total_mem_usage_g += sizeof(uint32_t); - // Reconstruct hash table entry + // reconstruct hash table entry entry = (pdc_hash_table_entry_head *)PDC_malloc(sizeof(pdc_hash_table_entry_head)); entry->n_obj = 0; entry->bloom = NULL; @@ -1692,7 +1691,7 @@ PDC_Server_restart(char *filename) metadata = (pdc_metadata_t *)PDC_calloc(sizeof(pdc_metadata_t), count); - // Extract metadata objects array + // extract metadata objects array BULKI_Entity *metadata_objs_array = BULKI_get(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING)); @@ -1703,12 +1702,12 @@ PDC_Server_restart(char *filename) while (Bent_iterator_has_next_BULKI(obj_iter) && i < count) { BULKI *metadata_obj = Bent_iterator_next_BULKI(obj_iter); - // Extract metadata structure + // extract metadata structure BULKI_Entity *metadata_ent = BULKI_get(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING)); memcpy(metadata + i, metadata_ent->data, sizeof(pdc_metadata_t)); - // Initialize pointers + // initialize pointers (metadata + i)->storage_region_list_head = NULL; (metadata + i)->region_lock_head = NULL; (metadata + i)->region_map_head = NULL; @@ -1719,7 +1718,7 @@ PDC_Server_restart(char *filename) (metadata + i)->kvtag_list_head = NULL; (metadata + i)->all_storage_region_distributed = 0; - // ========== Restore KV Tags ========== + // restore kv tags BULKI_Entity *kvtags_array = BULKI_get(metadata_obj, BULKI_singleton_ENTITY("kvtags", PDC_STRING)); @@ -1732,24 +1731,24 @@ PDC_Server_restart(char *filename) pdc_kvtag_list_t *kvtag_list = (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); - // Extract key + // extract key BULKI_Entity *key_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING)); int key_len = strlen((char *)key_ent->data) + 1; kvtag_list->kvtag->name = PDC_malloc(key_len); memcpy(kvtag_list->kvtag->name, key_ent->data, key_len); - // Extract size + // extract size BULKI_Entity *size_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING)); memcpy(&kvtag_list->kvtag->size, size_ent->data, sizeof(uint32_t)); - // Extract type + // extract type BULKI_Entity *type_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING)); memcpy(&kvtag_list->kvtag->type, type_ent->data, sizeof(int8_t)); - // Extract value + // extract value BULKI_Entity *value_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING)); kvtag_list->kvtag->value = PDC_malloc(kvtag_list->kvtag->size); @@ -1759,7 +1758,7 @@ PDC_Server_restart(char *filename) } } - // ========== Restore Storage Regions ========== + // restore storage regions BULKI_Entity *regions_array = BULKI_get(metadata_obj, BULKI_singleton_ENTITY("regions", PDC_STRING)); @@ -1773,12 +1772,12 @@ PDC_Server_restart(char *filename) region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - // Extract region structure + // extract region structure BULKI_Entity *region_ent = BULKI_get(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING)); memcpy(region_list, region_ent->data, sizeof(region_list_t)); - // Extract histogram flag + // extract histogram flag BULKI_Entity *has_hist_ent = BULKI_get(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING)); int has_hist; @@ -1825,7 +1824,7 @@ PDC_Server_restart(char *filename) } } - // Initialize region_list fields + // initialize region_list fields region_list->buf = NULL; region_list->data_size = 1; for (idx = 0; idx < region_list->ndim; idx++) @@ -1877,7 +1876,7 @@ PDC_Server_restart(char *filename) entry->metadata = NULL; - // Insert metadata to hash table + // insert metadata to hash table for (i = 0; i < count; i++) { elt = metadata + i; ret_value = PDC_Server_hash_table_list_insert(entry, elt); @@ -1887,7 +1886,7 @@ PDC_Server_restart(char *filename) } } - // ========== Section 3: Restore Data Server Regions ========== + // restore data server regions dataserver_regions_array = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING)); @@ -1902,12 +1901,12 @@ PDC_Server_restart(char *filename) new_obj_reg->fd = -1; new_obj_reg->storage_location = (char *)PDC_malloc(sizeof(char) * ADDR_MAX); - // Extract obj_id + // extract obj_id BULKI_Entity *obj_id_ent = BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING)); memcpy(&new_obj_reg->obj_id, obj_id_ent->data, sizeof(uint64_t)); - // Extract regions + // extract regions BULKI_Entity *ds_regions_array = BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING)); @@ -1923,7 +1922,7 @@ PDC_Server_restart(char *filename) BULKI_singleton_ENTITY("region", PDC_STRING)); memcpy(region_list, region_ent->data, sizeof(region_list_t)); - // Initialize fields (similar to above) + // initialize fields (similar to above) region_list->buf = NULL; region_list->data_size = 1; for (idx = 0; idx < region_list->ndim; idx++) @@ -1971,9 +1970,8 @@ PDC_Server_restart(char *filename) } } - // Clean up + // clean up BULKI_free(checkpoint_bulki, 1); - file_buffer = PDC_free(file_buffer); #ifdef ENABLE_MPI MPI_Reduce(&nobj, &all_nobj, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); @@ -1994,6 +1992,11 @@ PDC_Server_restart(char *filename) } done: + // ensure file is closed if error occurred before BULKI_deserialize_from_file + if (file != NULL) { + fclose(file); + } + FUNC_LEAVE(ret_value); } From 684bf327e625c40f7166c11078581584be97cba6 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 17 Mar 2026 13:44:56 -0400 Subject: [PATCH 07/13] Refactor kvtag-get-verify test code --- src/tests/tags/kvtag_get_verify_scale.c | 67 ++++++++----------------- 1 file changed, 21 insertions(+), 46 deletions(-) diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index bdce6df81..e624c95d5 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -62,7 +62,7 @@ void print_usage(char *name) { // required parameters: n_obj and n_query - LOG_JUST_PRINT("%s n_obj n_query\n", name); + LOG_JUST_PRINT("%s n_obj n_tag\n", name); } int @@ -70,8 +70,8 @@ main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont; pdcid_t * obj_ids; - int n_obj, n_query, my_obj, my_obj_s, my_query, my_query_s; - int query_1percent = 0; + int n_obj, n_query, my_obj, my_obj_s, n_tag, n_tag_s; + int obj_1percent = 0; int proc_num, my_rank, i; char obj_name[128]; double stime, total_time, percent_time; @@ -105,22 +105,17 @@ main(int argc, char *argv[]) PGOTO_DONE(FAIL); } - assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); + assign_work_to_rank(my_rank, proc_num, n_query, &n_tag, &n_tag_s); assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); - query_1percent = my_query / 100; + obj_1percent = my_obj / 100; if (my_rank == 0) - LOG_INFO("Open %d obj, query %d tags\n", my_obj, my_query); + LOG_INFO("Open %d obj, query %d tags\n", my_obj, n_tag); // create a pdc pdc = PDCinit("pdc"); - // create a container property - cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); - if (cont_prop <= 0) - PGOTO_ERROR(FAIL, "Failed to create container property"); - // Open the existing container cont = PDCcont_open("c1", pdc); if (cont <= 0) @@ -142,12 +137,12 @@ main(int argc, char *argv[]) PGOTO_ERROR(FAIL, "Failed to open object"); // progress reporting for object opening - if (i > 0 && query_1percent > 0 && i % query_1percent == 0) { + if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { - int current_percentage = i / query_1percent; + int current_percentage = i / obj_1percent; int estimated_current_object_number = n_obj / 100 * current_percentage; double tps = estimated_current_object_number / percent_time; LOG_INFO("[OBJ PROGRESS %3d%% ] %11d objects, %7.2f seconds, TPS: %10.2f \n", @@ -169,46 +164,18 @@ main(int argc, char *argv[]) // Setup kvtag for queries kvtag.name = "Group"; - values = (void **)calloc(my_query, sizeof(void *)); + values = (void **)calloc(n_tag, sizeof(void *)); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - for (i = 0; i < my_query; i++) { + for (i = 0; i < n_tag; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, (void *)&value_size) < 0) - PGOTO_ERROR(FAIL, "Failed to get a kvtag from o%d\n", i + my_query_s); - - if (i % query_1percent == 0) { -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - percent_time = MPI_Wtime() - stime; - if (my_rank == 0) { - int current_percentage = i / query_1percent; - int estimated_current_query_number = n_obj / 100 * current_percentage; - double tps = estimated_current_query_number / percent_time; - LOG_INFO("[QRY PROGRESS %3d%% ] %11d queries, %7.2f seconds, TPS: %10.2f \n", - current_percentage, estimated_current_query_number, percent_time, tps); - } -#endif - } - } - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - total_time = MPI_Wtime() - stime; -#endif - if (my_rank == 0) - LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, - total_time, n_query / total_time); + PGOTO_ERROR(FAIL, "Failed to get a kvtag from o%d\n", i + n_tag_s); - // `kvtag_add_get_scale' program adds tags with value = i + my_add_tag_s - // calculate and checking the expected value - for (i = 0; i < my_query; i++) { - // when tags were added, each rank tagged objects starting from my_add_tag_s - // for verification, we need to determine which rank added the tag and its offset - int expected_value = i + my_query_s; // Assuming tags were added in order starting from 0 + int expected_value = i + n_tag_s; // Assuming tags were added in order starting from 0 // count successful and failed verifications instead of immediate error if (*(int *)(values[i]) == expected_value) { @@ -218,12 +185,20 @@ main(int argc, char *argv[]) // log first 10 failures for debugging if (verified_fail <= 10) { LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", - i + my_query_s, expected_value, *(int *)(values[i])); + i + n_tag_s, expected_value, *(int *)(values[i])); } } free(values[i]); } +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, + total_time, n_query / total_time); + free(values); free(obj_ids); From 2c92c0b280edd5f3e8c5c01c67b56c9dc516f9e5 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 18 Mar 2026 00:44:02 -0400 Subject: [PATCH 08/13] Refactor code Added comment, fixed formatting suggested by GitHub clang-format style checker, brought back PDC_TIMING log from the stable version. --- src/server/pdc_server.c | 27 +++++++++---------------- src/tests/tags/kvtag_get_verify_scale.c | 11 +++++----- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index ae9696f6c..1b0d1ec21 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1465,7 +1465,6 @@ PDC_Server_checkpoint() // transfer request metadata query BULKI *transfer_query_bulki = NULL; ret_value = transfer_request_metadata_query_checkpoint_bulki(&transfer_query_bulki); - if (ret_value != SUCCEED || transfer_query_bulki == NULL) { LOG_ERROR("Failed to create transfer query checkpoint\n"); PGOTO_ERROR(FAIL, "Transfer query checkpoint failed"); @@ -1595,16 +1594,7 @@ PDC_Server_restart(char *filename) PGOTO_ERROR(FAIL, "Error with fopen, filename: [%s]", filename); } - // Read and validate magic header -// fread(magic, 1, PDC_CHECKPOINT_MAGIC_LEN, file); -// magic[PDC_CHECKPOINT_MAGIC_LEN] = '\0'; -// if (!PDC_CHECKPOINT_VERSION_MATCH(magic, PDC_CHECKPOINT_MAGIC_CURRENT)) { -// LOG_ERROR("Invalid checkpoint file magic header: '%s'\n", magic); -// fclose(file); -// PGOTO_ERROR(FAIL, "Invalid checkpoint format"); -// } - - // Note: BULKI_deserialize_from_file will close the file + // note: BULKI_deserialize_from_file will close the file checkpoint_bulki = BULKI_deserialize_from_file(file); file = NULL; // File was closed by BULKI_deserialize_from_file @@ -1613,11 +1603,13 @@ PDC_Server_restart(char *filename) PGOTO_ERROR(FAIL, "Deserialization failed"); } + // read and validate bulki version BULKI_Entity *version_entity = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING)); int equal = BULKI_Entity_equal(version_entity, BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); if (!equal) { LOG_ERROR("Checkpoint version mismatch: expected '%s', found '%s'\n", - PDC_CHECKPOINT_MAGIC_CURRENT, version_entity ? (char *)version_entity->data : "NULL"); + PDC_CHECKPOINT_MAGIC_CURRENT, + version_entity ? (char *)version_entity->data : "NULL"); PGOTO_ERROR(FAIL, "Checkpoint version mismatch"); } @@ -1981,17 +1973,16 @@ PDC_Server_restart(char *filename) all_n_region = total_region; #endif -#ifdef PDC_TIMING - if (pdc_server_rank_g == 0) - LOG_ERROR("Total restart time = %.6f\n", MPI_Wtime() - start); -#endif - if (pdc_server_rank_g == 0) { - LOG_INFO("Recovered %d containers, %d objects, %d regions from checkpoint file \n", + LOG_INFO("Server restarted from saved session, " + "successfully loaded %d containers, %d objects, %d regions...\n", all_cont, all_nobj, all_n_region); } done: +#ifdef PDC_TIMING + pdc_server_timings->PDCserver_restart += MPI_Wtime() - start; +#endif // ensure file is closed if error occurred before BULKI_deserialize_from_file if (file != NULL) { fclose(file); diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index e624c95d5..a7ffd21c1 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -180,7 +180,8 @@ main(int argc, char *argv[]) // count successful and failed verifications instead of immediate error if (*(int *)(values[i]) == expected_value) { verified_success++; - } else { + } + else { verified_fail++; // log first 10 failures for debugging if (verified_fail <= 10) { @@ -196,7 +197,7 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, + LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, n_obj, total_time, n_query / total_time); free(values); @@ -216,11 +217,9 @@ main(int argc, char *argv[]) LOG_INFO("==============================================================\n"); LOG_INFO("Verification Summary:\n"); LOG_INFO(" Total queries: %11d\n", n_query); - LOG_INFO(" Successfully verified: %11d (%6.2f%%)\n", - all_verified_success, + LOG_INFO(" Successfully verified: %11d (%6.2f%%)\n", all_verified_success, 100.0 * all_verified_success / n_query); - LOG_INFO(" Failed verification: %11d (%6.2f%%)\n", - all_verified_fail, + LOG_INFO(" Failed verification: %11d (%6.2f%%)\n", all_verified_fail, 100.0 * all_verified_fail / n_query); LOG_INFO("==============================================================\n"); From 65c546f56054839f1b3ed8989fa9f4377e5fa987 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 26 Mar 2026 20:59:01 +0000 Subject: [PATCH 09/13] Committing clang-format changes --- src/server/pdc_server.c | 284 ++++++++---------- ...dc_server_region_transfer_metadata_query.c | 85 +++--- src/tests/tags/kvtag_get_verify_scale.c | 41 +-- 3 files changed, 186 insertions(+), 224 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 1b0d1ec21..1a43b6ea7 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -78,11 +78,11 @@ sqlite3 *sqlite3_db_g; #include "bulki_serde.h" // PDC checkpoint version management -#define PDC_CHECKPOINT_MAGIC_2026 "PDC26.03" // Year 2026, month 03 -#define PDC_CHECKPOINT_MAGIC_2027 "PDC27.01" // Year 2027, month 01 +#define PDC_CHECKPOINT_MAGIC_2026 "PDC26.03" // Year 2026, month 03 +#define PDC_CHECKPOINT_MAGIC_2027 "PDC27.01" // Year 2027, month 01 -#define PDC_CHECKPOINT_MAGIC_LEN 8 // All must be same length -#define PDC_CHECKPOINT_MAGIC_CURRENT PDC_CHECKPOINT_MAGIC_2026 +#define PDC_CHECKPOINT_MAGIC_LEN 8 // All must be same length +#define PDC_CHECKPOINT_MAGIC_CURRENT PDC_CHECKPOINT_MAGIC_2026 #define PDC_CHECKPOINT_VERSION_MATCH(m, v) (strncmp((m), (v), 8) == 0) // Check how long PDC has run every OP_INTERVAL operations @@ -1239,8 +1239,7 @@ PDC_Server_checkpoint() checkpoint_bulki = BULKI_init(5); // BULKI version number for validation - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("version_number", PDC_STRING), + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING), BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); // checkpoint containers @@ -1256,21 +1255,17 @@ PDC_Server_checkpoint() BULKI *container_entry = BULKI_init(2); hash_key = PDC_get_hash_by_name(cont_head->cont_name); - BULKI_put(container_entry, - BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_put(container_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); // store the container structure as binary blob - BULKI_put(container_entry, - BULKI_singleton_ENTITY("cont_data", PDC_STRING), + BULKI_put(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING), BULKI_ENTITY(cont_head, sizeof(pdc_cont_hash_table_entry_t), PDC_UINT8, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI(containers_array, container_entry); } - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("containers", PDC_STRING), - containers_array); + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING), containers_array); // checkpoint metadata hash table n_entry = hash_table_num_entries(metadata_hash_table_g); @@ -1286,13 +1281,11 @@ PDC_Server_checkpoint() BULKI *hash_entry = BULKI_init(3); // Store number of objects - BULKI_put(hash_entry, - BULKI_singleton_ENTITY("n_obj", PDC_STRING), + BULKI_put(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING), BULKI_ENTITY(&head->n_obj, 1, PDC_INT, PDC_CLS_ITEM)); hash_key = PDC_get_hash_by_name(head->metadata->obj_name); - BULKI_put(hash_entry, - BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_put(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); // array of metadata objects @@ -1304,8 +1297,7 @@ PDC_Server_checkpoint() BULKI *metadata_obj = BULKI_init(3); // store metadata structure - BULKI_put(metadata_obj, - BULKI_singleton_ENTITY("metadata", PDC_STRING), + BULKI_put(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING), BULKI_ENTITY(elt, sizeof(pdc_metadata_t), PDC_UINT8, PDC_CLS_ARRAY)); // kv tags @@ -1316,29 +1308,23 @@ PDC_Server_checkpoint() { BULKI *kvtag_entry = BULKI_init(4); - BULKI_put(kvtag_entry, - BULKI_singleton_ENTITY("key", PDC_STRING), + BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING), BULKI_singleton_ENTITY(kvlist_elt->kvtag->name, PDC_STRING)); - BULKI_put(kvtag_entry, - BULKI_singleton_ENTITY("size", PDC_STRING), + BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING), BULKI_ENTITY(&kvlist_elt->kvtag->size, 1, PDC_UINT32, PDC_CLS_ITEM)); - BULKI_put(kvtag_entry, - BULKI_singleton_ENTITY("type", PDC_STRING), + BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING), BULKI_ENTITY(&kvlist_elt->kvtag->type, 1, PDC_INT8, PDC_CLS_ITEM)); - BULKI_put(kvtag_entry, - BULKI_singleton_ENTITY("value", PDC_STRING), - BULKI_ENTITY(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, - PDC_UINT8, PDC_CLS_ARRAY)); + BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING), + BULKI_ENTITY(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, PDC_UINT8, + PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI(kvtags_array, kvtag_entry); } - BULKI_put(metadata_obj, - BULKI_singleton_ENTITY("kvtags", PDC_STRING), - kvtags_array); + BULKI_put(metadata_obj, BULKI_singleton_ENTITY("kvtags", PDC_STRING), kvtags_array); // storage regions n_region = 0; @@ -1352,8 +1338,7 @@ PDC_Server_checkpoint() BULKI *region_entry = BULKI_init(3); // store region structure - BULKI_put(region_entry, - BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_put(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING), BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); // store histogram if present @@ -1361,37 +1346,30 @@ PDC_Server_checkpoint() if (region_elt->region_hist != NULL) has_hist = 1; - BULKI_put(region_entry, - BULKI_singleton_ENTITY("has_hist", PDC_STRING), + BULKI_put(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING), BULKI_ENTITY(&has_hist, 1, PDC_INT, PDC_CLS_ITEM)); if (has_hist == 1) { BULKI *histogram = BULKI_init(5); - BULKI_put(histogram, - BULKI_singleton_ENTITY("dtype", PDC_STRING), + BULKI_put(histogram, BULKI_singleton_ENTITY("dtype", PDC_STRING), BULKI_ENTITY(®ion_elt->region_hist->dtype, 1, PDC_INT, PDC_CLS_ITEM)); - BULKI_put(histogram, - BULKI_singleton_ENTITY("nbin", PDC_STRING), + BULKI_put(histogram, BULKI_singleton_ENTITY("nbin", PDC_STRING), BULKI_ENTITY(®ion_elt->region_hist->nbin, 1, PDC_INT, PDC_CLS_ITEM)); - BULKI_put(histogram, - BULKI_singleton_ENTITY("range", PDC_STRING), + BULKI_put(histogram, BULKI_singleton_ENTITY("range", PDC_STRING), BULKI_ENTITY(region_elt->region_hist->range, region_elt->region_hist->nbin * 2, PDC_DOUBLE, PDC_CLS_ARRAY)); - BULKI_put(histogram, - BULKI_singleton_ENTITY("bin", PDC_STRING), - BULKI_ENTITY(region_elt->region_hist->bin, - region_elt->region_hist->nbin, PDC_UINT64, PDC_CLS_ARRAY)); + BULKI_put(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING), + BULKI_ENTITY(region_elt->region_hist->bin, region_elt->region_hist->nbin, + PDC_UINT64, PDC_CLS_ARRAY)); - BULKI_put(histogram, - BULKI_singleton_ENTITY("incr", PDC_STRING), + BULKI_put(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING), BULKI_ENTITY(®ion_elt->region_hist->incr, 1, PDC_DOUBLE, PDC_CLS_ITEM)); - BULKI_put(region_entry, - BULKI_singleton_ENTITY("histogram", PDC_STRING), + BULKI_put(region_entry, BULKI_singleton_ENTITY("histogram", PDC_STRING), BULKI_ENTITY(histogram, 1, PDC_BULKI, PDC_CLS_ITEM)); } @@ -1403,9 +1381,7 @@ PDC_Server_checkpoint() LOG_ERROR("Error with number of regions\n"); } - BULKI_put(metadata_obj, - BULKI_singleton_ENTITY("regions", PDC_STRING), - regions_array); + BULKI_put(metadata_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), regions_array); BULKI_ENTITY_append_BULKI(metadata_objs_array, metadata_obj); @@ -1413,15 +1389,12 @@ PDC_Server_checkpoint() region_count += n_region; } // end for metadata entry linked list - BULKI_put(hash_entry, - BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), - metadata_objs_array); + BULKI_put(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), metadata_objs_array); BULKI_ENTITY_append_BULKI(metadata_entries_array, hash_entry); } // end for hash table metadata entry - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), metadata_entries_array); // data server regions @@ -1434,8 +1407,7 @@ PDC_Server_checkpoint() { BULKI *dataserver_obj = BULKI_init(2); - BULKI_put(dataserver_obj, - BULKI_singleton_ENTITY("obj_id", PDC_STRING), + BULKI_put(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING), BULKI_ENTITY(®ion->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); DL_COUNT(region->region_storage_head, region_elt, n_region); @@ -1444,34 +1416,29 @@ PDC_Server_checkpoint() DL_FOREACH(region->region_storage_head, region_elt) { BULKI *ds_region = BULKI_init(1); - BULKI_put(ds_region, - BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_put(ds_region, BULKI_singleton_ENTITY("region", PDC_STRING), BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI(ds_regions_array, ds_region); } - BULKI_put(dataserver_obj, - BULKI_singleton_ENTITY("regions", PDC_STRING), - ds_regions_array); + BULKI_put(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), ds_regions_array); BULKI_ENTITY_append_BULKI(dataserver_regions_array, dataserver_obj); } - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), dataserver_regions_array); // transfer request metadata query BULKI *transfer_query_bulki = NULL; - ret_value = transfer_request_metadata_query_checkpoint_bulki(&transfer_query_bulki); + ret_value = transfer_request_metadata_query_checkpoint_bulki(&transfer_query_bulki); if (ret_value != SUCCEED || transfer_query_bulki == NULL) { LOG_ERROR("Failed to create transfer query checkpoint\n"); PGOTO_ERROR(FAIL, "Transfer query checkpoint failed"); } - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("transfer_query", PDC_STRING), + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING), BULKI_ENTITY(transfer_query_bulki, 1, PDC_BULKI, PDC_CLS_ITEM)); // ========== Serialize and Write ========== @@ -1486,7 +1453,7 @@ PDC_Server_checkpoint() // note: BULKI_Entity_serialize_to_file() still do in-memory serialization // note: need to implement a streaming file writing without in-memory serialization BULKI_serialize_to_file(checkpoint_bulki, file); - file = NULL; // file was closed by BULKI_serialize_to_file() + file = NULL; // file was closed by BULKI_serialize_to_file() // Clean up BULKI_free(checkpoint_bulki, 1); @@ -1517,8 +1484,8 @@ PDC_Server_checkpoint() #endif #ifdef PDC_TIMING - LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", - metadata_size, region_count, checkpoint_time_rank); + LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", metadata_size, region_count, + checkpoint_time_rank); gettimeofday(&pdc_timer_end, 0); checkpoint_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); @@ -1561,9 +1528,9 @@ PDC_Server_restart(char *filename) { FUNC_ENTER(NULL); - perr_t ret_value = SUCCEED; - int i, j, nobj = 0, all_nobj = 0, all_n_region, total_region = 0; - int all_cont; + perr_t ret_value = SUCCEED; + int i, j, nobj = 0, all_nobj = 0, all_n_region, total_region = 0; + int all_cont; pdc_metadata_t * metadata, *elt; region_list_t * region_list; pdc_hash_table_entry_head * entry; @@ -1572,12 +1539,12 @@ PDC_Server_restart(char *filename) unsigned idx; // BULKI-specific variables - BULKI * checkpoint_bulki = NULL; - BULKI_Entity * containers_array = NULL; - BULKI_Entity * metadata_entries_array = NULL; - BULKI_Entity * dataserver_regions_array = NULL; - char magic[PDC_CHECKPOINT_MAGIC_LEN + 1]; - FILE * file = NULL; + BULKI * checkpoint_bulki = NULL; + BULKI_Entity *containers_array = NULL; + BULKI_Entity *metadata_entries_array = NULL; + BULKI_Entity *dataserver_regions_array = NULL; + char magic[PDC_CHECKPOINT_MAGIC_LEN + 1]; + FILE * file = NULL; #ifdef PDC_TIMING double start = MPI_Wtime(); @@ -1596,7 +1563,7 @@ PDC_Server_restart(char *filename) // note: BULKI_deserialize_from_file will close the file checkpoint_bulki = BULKI_deserialize_from_file(file); - file = NULL; // File was closed by BULKI_deserialize_from_file + file = NULL; // File was closed by BULKI_deserialize_from_file if (checkpoint_bulki == NULL) { LOG_ERROR("Failed to deserialize checkpoint file\n"); @@ -1604,36 +1571,36 @@ PDC_Server_restart(char *filename) } // read and validate bulki version - BULKI_Entity *version_entity = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING)); - int equal = BULKI_Entity_equal(version_entity, BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); + BULKI_Entity *version_entity = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING)); + int equal = BULKI_Entity_equal(version_entity, + BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); if (!equal) { - LOG_ERROR("Checkpoint version mismatch: expected '%s', found '%s'\n", - PDC_CHECKPOINT_MAGIC_CURRENT, + LOG_ERROR("Checkpoint version mismatch: expected '%s', found '%s'\n", PDC_CHECKPOINT_MAGIC_CURRENT, version_entity ? (char *)version_entity->data : "NULL"); PGOTO_ERROR(FAIL, "Checkpoint version mismatch"); } // restore containers - containers_array = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("containers", PDC_STRING)); + containers_array = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING)); if (containers_array != NULL && containers_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *cont_iter = Bent_iterator_init(containers_array, NULL, PDC_BULKI); - all_cont = containers_array->count; + all_cont = containers_array->count; while (Bent_iterator_has_next_BULKI(cont_iter)) { BULKI *container_entry = Bent_iterator_next_BULKI(cont_iter); // Extract hash key - BULKI_Entity *hash_key_ent = BULKI_get(container_entry, - BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + BULKI_Entity *hash_key_ent = + BULKI_get(container_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING)); hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); total_mem_usage_g += sizeof(uint32_t); // Extract container data - BULKI_Entity *cont_data_ent = BULKI_get(container_entry, - BULKI_singleton_ENTITY("cont_data", PDC_STRING)); + BULKI_Entity *cont_data_ent = + BULKI_get(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING)); cont_entry = (pdc_cont_hash_table_entry_t *)PDC_malloc(sizeof(pdc_cont_hash_table_entry_t)); memcpy(cont_entry, cont_data_ent->data, sizeof(pdc_cont_hash_table_entry_t)); total_mem_usage_g += sizeof(pdc_cont_hash_table_entry_t); @@ -1652,8 +1619,8 @@ PDC_Server_restart(char *filename) } // restore metadata - metadata_entries_array = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("metadata_entries", PDC_STRING)); + metadata_entries_array = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING)); if (metadata_entries_array != NULL && metadata_entries_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *entry_iter = Bent_iterator_init(metadata_entries_array, NULL, PDC_BULKI); @@ -1662,14 +1629,13 @@ PDC_Server_restart(char *filename) BULKI *hash_entry = Bent_iterator_next_BULKI(entry_iter); // extract n_obj - BULKI_Entity *n_obj_ent = BULKI_get(hash_entry, - BULKI_singleton_ENTITY("n_obj", PDC_STRING)); - int count; + BULKI_Entity *n_obj_ent = BULKI_get(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING)); + int count; memcpy(&count, n_obj_ent->data, sizeof(int)); // extract hash key - BULKI_Entity *hash_key_ent = BULKI_get(hash_entry, - BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + BULKI_Entity *hash_key_ent = + BULKI_get(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING)); hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); total_mem_usage_g += sizeof(uint32_t); @@ -1684,19 +1650,19 @@ PDC_Server_restart(char *filename) metadata = (pdc_metadata_t *)PDC_calloc(sizeof(pdc_metadata_t), count); // extract metadata objects array - BULKI_Entity *metadata_objs_array = BULKI_get(hash_entry, - BULKI_singleton_ENTITY("metadata_objects", PDC_STRING)); + BULKI_Entity *metadata_objs_array = + BULKI_get(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING)); if (metadata_objs_array != NULL && metadata_objs_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *obj_iter = Bent_iterator_init(metadata_objs_array, NULL, PDC_BULKI); - i = 0; + i = 0; while (Bent_iterator_has_next_BULKI(obj_iter) && i < count) { BULKI *metadata_obj = Bent_iterator_next_BULKI(obj_iter); // extract metadata structure - BULKI_Entity *metadata_ent = BULKI_get(metadata_obj, - BULKI_singleton_ENTITY("metadata", PDC_STRING)); + BULKI_Entity *metadata_ent = + BULKI_get(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING)); memcpy(metadata + i, metadata_ent->data, sizeof(pdc_metadata_t)); // initialize pointers @@ -1711,8 +1677,8 @@ PDC_Server_restart(char *filename) (metadata + i)->all_storage_region_distributed = 0; // restore kv tags - BULKI_Entity *kvtags_array = BULKI_get(metadata_obj, - BULKI_singleton_ENTITY("kvtags", PDC_STRING)); + BULKI_Entity *kvtags_array = + BULKI_get(metadata_obj, BULKI_singleton_ENTITY("kvtags", PDC_STRING)); if (kvtags_array != NULL && kvtags_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *kvtag_iter = Bent_iterator_init(kvtags_array, NULL, PDC_BULKI); @@ -1720,29 +1686,30 @@ PDC_Server_restart(char *filename) while (Bent_iterator_has_next_BULKI(kvtag_iter)) { BULKI *kvtag_entry = Bent_iterator_next_BULKI(kvtag_iter); - pdc_kvtag_list_t *kvtag_list = (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); - kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); + pdc_kvtag_list_t *kvtag_list = + (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); + kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); // extract key - BULKI_Entity *key_ent = BULKI_get(kvtag_entry, - BULKI_singleton_ENTITY("key", PDC_STRING)); - int key_len = strlen((char *)key_ent->data) + 1; + BULKI_Entity *key_ent = + BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING)); + int key_len = strlen((char *)key_ent->data) + 1; kvtag_list->kvtag->name = PDC_malloc(key_len); memcpy(kvtag_list->kvtag->name, key_ent->data, key_len); // extract size - BULKI_Entity *size_ent = BULKI_get(kvtag_entry, - BULKI_singleton_ENTITY("size", PDC_STRING)); + BULKI_Entity *size_ent = + BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING)); memcpy(&kvtag_list->kvtag->size, size_ent->data, sizeof(uint32_t)); // extract type - BULKI_Entity *type_ent = BULKI_get(kvtag_entry, - BULKI_singleton_ENTITY("type", PDC_STRING)); + BULKI_Entity *type_ent = + BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING)); memcpy(&kvtag_list->kvtag->type, type_ent->data, sizeof(int8_t)); // extract value - BULKI_Entity *value_ent = BULKI_get(kvtag_entry, - BULKI_singleton_ENTITY("value", PDC_STRING)); + BULKI_Entity *value_ent = + BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING)); kvtag_list->kvtag->value = PDC_malloc(kvtag_list->kvtag->size); memcpy(kvtag_list->kvtag->value, value_ent->data, kvtag_list->kvtag->size); @@ -1751,13 +1718,14 @@ PDC_Server_restart(char *filename) } // restore storage regions - BULKI_Entity *regions_array = BULKI_get(metadata_obj, - BULKI_singleton_ENTITY("regions", PDC_STRING)); + BULKI_Entity *regions_array = + BULKI_get(metadata_obj, BULKI_singleton_ENTITY("regions", PDC_STRING)); int n_region = 0; if (regions_array != NULL && regions_array->pdc_type == PDC_BULKI) { n_region = regions_array->count; - BULKI_Entity_Iterator *region_iter = Bent_iterator_init(regions_array, NULL, PDC_BULKI); + BULKI_Entity_Iterator *region_iter = + Bent_iterator_init(regions_array, NULL, PDC_BULKI); while (Bent_iterator_has_next_BULKI(region_iter)) { BULKI *region_entry = Bent_iterator_next_BULKI(region_iter); @@ -1765,53 +1733,54 @@ PDC_Server_restart(char *filename) region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); // extract region structure - BULKI_Entity *region_ent = BULKI_get(region_entry, - BULKI_singleton_ENTITY("region", PDC_STRING)); + BULKI_Entity *region_ent = + BULKI_get(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING)); memcpy(region_list, region_ent->data, sizeof(region_list_t)); // extract histogram flag - BULKI_Entity *has_hist_ent = BULKI_get(region_entry, - BULKI_singleton_ENTITY("has_hist", PDC_STRING)); + BULKI_Entity *has_hist_ent = + BULKI_get(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING)); int has_hist; memcpy(&has_hist, has_hist_ent->data, sizeof(int)); if (has_hist == 1) { - BULKI_Entity *histogram_ent = BULKI_get(region_entry, - BULKI_singleton_ENTITY("histogram", PDC_STRING)); + BULKI_Entity *histogram_ent = + BULKI_get(region_entry, BULKI_singleton_ENTITY("histogram", PDC_STRING)); if (histogram_ent != NULL && histogram_ent->pdc_type == PDC_BULKI) { BULKI *histogram = (BULKI *)histogram_ent->data; - region_list->region_hist = (pdc_histogram_t *)PDC_malloc(sizeof(pdc_histogram_t)); + region_list->region_hist = + (pdc_histogram_t *)PDC_malloc(sizeof(pdc_histogram_t)); - BULKI_Entity *dtype_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("dtype", PDC_STRING)); + BULKI_Entity *dtype_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("dtype", PDC_STRING)); memcpy(®ion_list->region_hist->dtype, dtype_ent->data, sizeof(int)); - BULKI_Entity *nbin_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("nbin", PDC_STRING)); + BULKI_Entity *nbin_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("nbin", PDC_STRING)); memcpy(®ion_list->region_hist->nbin, nbin_ent->data, sizeof(int)); if (region_list->region_hist->nbin == 0) { LOG_ERROR("Checkpoint file histogram size is 0\n"); } - BULKI_Entity *range_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("range", PDC_STRING)); - region_list->region_hist->range = - (double *)PDC_malloc(sizeof(double) * region_list->region_hist->nbin * 2); + BULKI_Entity *range_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("range", PDC_STRING)); + region_list->region_hist->range = (double *)PDC_malloc( + sizeof(double) * region_list->region_hist->nbin * 2); memcpy(region_list->region_hist->range, range_ent->data, sizeof(double) * region_list->region_hist->nbin * 2); - BULKI_Entity *bin_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("bin", PDC_STRING)); - region_list->region_hist->bin = - (uint64_t *)PDC_malloc(sizeof(uint64_t) * region_list->region_hist->nbin); + BULKI_Entity *bin_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING)); + region_list->region_hist->bin = (uint64_t *)PDC_malloc( + sizeof(uint64_t) * region_list->region_hist->nbin); memcpy(region_list->region_hist->bin, bin_ent->data, sizeof(uint64_t) * region_list->region_hist->nbin); - BULKI_Entity *incr_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("incr", PDC_STRING)); + BULKI_Entity *incr_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING)); memcpy(®ion_list->region_hist->incr, incr_ent->data, sizeof(double)); } } @@ -1845,7 +1814,8 @@ PDC_Server_restart(char *filename) region_list->io_cache_region = NULL; memset(region_list->shm_addr, 0, ADDR_MAX); - memset(region_list->client_ids, 0, PDC_SERVER_MAX_PROC_PER_NODE * sizeof(uint32_t)); + memset(region_list->client_ids, 0, + PDC_SERVER_MAX_PROC_PER_NODE * sizeof(uint32_t)); if (strstr(region_list->storage_location, "/global/cscratch") != NULL) { region_list->data_loc_type = PDC_LUSTRE; @@ -1870,7 +1840,7 @@ PDC_Server_restart(char *filename) // insert metadata to hash table for (i = 0; i < count; i++) { - elt = metadata + i; + elt = metadata + i; ret_value = PDC_Server_hash_table_list_insert(entry, elt); if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "Error with hash table recovering from checkpoint file"); @@ -1879,8 +1849,8 @@ PDC_Server_restart(char *filename) } // restore data server regions - dataserver_regions_array = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING)); + dataserver_regions_array = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING)); if (dataserver_regions_array != NULL && dataserver_regions_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *ds_iter = Bent_iterator_init(dataserver_regions_array, NULL, PDC_BULKI); @@ -1894,13 +1864,13 @@ PDC_Server_restart(char *filename) new_obj_reg->storage_location = (char *)PDC_malloc(sizeof(char) * ADDR_MAX); // extract obj_id - BULKI_Entity *obj_id_ent = BULKI_get(dataserver_obj, - BULKI_singleton_ENTITY("obj_id", PDC_STRING)); + BULKI_Entity *obj_id_ent = + BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING)); memcpy(&new_obj_reg->obj_id, obj_id_ent->data, sizeof(uint64_t)); // extract regions - BULKI_Entity *ds_regions_array = BULKI_get(dataserver_obj, - BULKI_singleton_ENTITY("regions", PDC_STRING)); + BULKI_Entity *ds_regions_array = + BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING)); if (ds_regions_array != NULL && ds_regions_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *ds_region_iter = Bent_iterator_init(ds_regions_array, NULL, PDC_BULKI); @@ -1910,8 +1880,8 @@ PDC_Server_restart(char *filename) region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - BULKI_Entity *region_ent = BULKI_get(ds_region, - BULKI_singleton_ENTITY("region", PDC_STRING)); + BULKI_Entity *region_ent = + BULKI_get(ds_region, BULKI_singleton_ENTITY("region", PDC_STRING)); memcpy(region_list, region_ent->data, sizeof(region_list_t)); // initialize fields (similar to above) @@ -1939,8 +1909,8 @@ PDC_Server_restart(char *filename) } // restore transfer query - BULKI_Entity *transfer_query_ent = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("transfer_query", PDC_STRING)); + BULKI_Entity *transfer_query_ent = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING)); if (transfer_query_ent != NULL && transfer_query_ent->pdc_type == PDC_BULKI) { // Extract the nested BULKI containing transfer query data @@ -1969,8 +1939,8 @@ PDC_Server_restart(char *filename) MPI_Reduce(&nobj, &all_nobj, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&total_region, &all_n_region, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else - all_nobj = nobj; - all_n_region = total_region; + all_nobj = nobj; + all_n_region = total_region; #endif if (pdc_server_rank_g == 0) { diff --git a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c index 8a2c8dc8e..440aa763c 100644 --- a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c +++ b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c @@ -71,8 +71,8 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che pthread_mutex_init(&metadata_query_mutex, NULL); if (checkpoint_bulki != NULL) { - BULKI_Entity *objects_array = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("objects", PDC_STRING)); + BULKI_Entity *objects_array = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("objects", PDC_STRING)); if (objects_array == NULL || objects_array->pdc_type != PDC_BULKI) { LOG_ERROR("Invalid transfer query checkpoint: missing or invalid 'objects' field\n"); @@ -84,11 +84,9 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che while (Bent_iterator_has_next_BULKI(obj_iter)) { BULKI *obj_bulki = Bent_iterator_next_BULKI(obj_iter); - pdc_obj_metadata_pkg *obj_pkg = - (pdc_obj_metadata_pkg *)PDC_malloc(sizeof(pdc_obj_metadata_pkg)); + pdc_obj_metadata_pkg *obj_pkg = (pdc_obj_metadata_pkg *)PDC_malloc(sizeof(pdc_obj_metadata_pkg)); - BULKI_Entity *obj_id_ent = BULKI_get(obj_bulki, - BULKI_singleton_ENTITY("obj_id", PDC_STRING)); + BULKI_Entity *obj_id_ent = BULKI_get(obj_bulki, BULKI_singleton_ENTITY("obj_id", PDC_STRING)); if (obj_id_ent == NULL) { LOG_ERROR("Missing obj_id in checkpoint object\n"); PDC_free(obj_pkg); @@ -96,8 +94,7 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che } memcpy(&obj_pkg->obj_id, obj_id_ent->data, sizeof(uint64_t)); - BULKI_Entity *ndim_ent = BULKI_get(obj_bulki, - BULKI_singleton_ENTITY("ndim", PDC_STRING)); + BULKI_Entity *ndim_ent = BULKI_get(obj_bulki, BULKI_singleton_ENTITY("ndim", PDC_STRING)); if (ndim_ent == NULL) { LOG_ERROR("Missing ndim in checkpoint object\n"); PDC_free(obj_pkg); @@ -105,12 +102,11 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che } memcpy(&obj_pkg->ndim, ndim_ent->data, sizeof(int)); - obj_pkg->regions = NULL; + obj_pkg->regions = NULL; obj_pkg->regions_end = NULL; - obj_pkg->next = NULL; + obj_pkg->next = NULL; - BULKI_Entity *regions_array = BULKI_get(obj_bulki, - BULKI_singleton_ENTITY("regions", PDC_STRING)); + BULKI_Entity *regions_array = BULKI_get(obj_bulki, BULKI_singleton_ENTITY("regions", PDC_STRING)); if (regions_array != NULL && regions_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *region_iter = Bent_iterator_init(regions_array, NULL, PDC_BULKI); @@ -122,25 +118,27 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che (pdc_region_metadata_pkg *)PDC_malloc(sizeof(pdc_region_metadata_pkg)); region_pkg->reg_offset = (uint64_t *)PDC_malloc(sizeof(uint64_t) * obj_pkg->ndim * 2); - region_pkg->reg_size = region_pkg->reg_offset + obj_pkg->ndim; + region_pkg->reg_size = region_pkg->reg_offset + obj_pkg->ndim; - BULKI_Entity *server_id_ent = BULKI_get(region_bulki, - BULKI_singleton_ENTITY("data_server_id", PDC_STRING)); + BULKI_Entity *server_id_ent = + BULKI_get(region_bulki, BULKI_singleton_ENTITY("data_server_id", PDC_STRING)); if (server_id_ent != NULL) { memcpy(®ion_pkg->data_server_id, server_id_ent->data, sizeof(uint32_t)); - } else { + } + else { LOG_ERROR("Missing data_server_id in checkpoint region\n"); PDC_free(region_pkg->reg_offset); PDC_free(region_pkg); continue; } - BULKI_Entity *offset_size_ent = BULKI_get(region_bulki, - BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING)); + BULKI_Entity *offset_size_ent = + BULKI_get(region_bulki, BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING)); if (offset_size_ent != NULL) { memcpy(region_pkg->reg_offset, offset_size_ent->data, sizeof(uint64_t) * obj_pkg->ndim * 2); - } else { + } + else { LOG_ERROR("Missing reg_offset_size in checkpoint region\n"); PDC_free(region_pkg->reg_offset); PDC_free(region_pkg); @@ -150,21 +148,23 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che region_pkg->next = NULL; if (obj_pkg->regions == NULL) { - obj_pkg->regions = region_pkg; + obj_pkg->regions = region_pkg; obj_pkg->regions_end = region_pkg; - } else { + } + else { obj_pkg->regions_end->next = region_pkg; - obj_pkg->regions_end = region_pkg; + obj_pkg->regions_end = region_pkg; } } } if (metadata_server_objs == NULL) { - metadata_server_objs = obj_pkg; + metadata_server_objs = obj_pkg; metadata_server_objs_end = obj_pkg; - } else { + } + else { metadata_server_objs_end->next = obj_pkg; - metadata_server_objs_end = obj_pkg; + metadata_server_objs_end = obj_pkg; } } @@ -298,28 +298,26 @@ transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) pdc_obj_metadata_pkg * obj_temp; pdc_region_metadata_pkg *region_temp; int obj_count = 0; - BULKI * bulki = NULL; + BULKI * bulki = NULL; -// if (checkpoint_bulki == NULL) { -// LOG_ERROR("checkpoint_bulki output parameter is NULL\n"); -// PGOTO_ERROR(FAIL, "Invalid parameter"); -// } + // if (checkpoint_bulki == NULL) { + // LOG_ERROR("checkpoint_bulki output parameter is NULL\n"); + // PGOTO_ERROR(FAIL, "Invalid parameter"); + // } pthread_mutex_lock(&metadata_query_mutex); - bulki = BULKI_init(1); + bulki = BULKI_init(1); BULKI_Entity *objects_array = empty_BULKI_Array_Entity(); obj_temp = metadata_server_objs; while (obj_temp) { BULKI *obj_bulki = BULKI_init(3); - BULKI_put(obj_bulki, - BULKI_singleton_ENTITY("obj_id", PDC_STRING), + BULKI_put(obj_bulki, BULKI_singleton_ENTITY("obj_id", PDC_STRING), BULKI_ENTITY(&obj_temp->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); - BULKI_put(obj_bulki, - BULKI_singleton_ENTITY("ndim", PDC_STRING), + BULKI_put(obj_bulki, BULKI_singleton_ENTITY("ndim", PDC_STRING), BULKI_ENTITY(&obj_temp->ndim, 1, PDC_INT, PDC_CLS_ITEM)); BULKI_Entity *regions_array = empty_BULKI_Array_Entity(); @@ -328,22 +326,17 @@ transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) while (region_temp) { BULKI *region_bulki = BULKI_init(2); - BULKI_put(region_bulki, - BULKI_singleton_ENTITY("data_server_id", PDC_STRING), + BULKI_put(region_bulki, BULKI_singleton_ENTITY("data_server_id", PDC_STRING), BULKI_ENTITY(®ion_temp->data_server_id, 1, PDC_UINT32, PDC_CLS_ITEM)); - BULKI_put(region_bulki, - BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), - BULKI_ENTITY(region_temp->reg_offset, obj_temp->ndim * 2, - PDC_UINT64, PDC_CLS_ARRAY)); + BULKI_put(region_bulki, BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), + BULKI_ENTITY(region_temp->reg_offset, obj_temp->ndim * 2, PDC_UINT64, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI(regions_array, region_bulki); region_temp = region_temp->next; } - BULKI_put(obj_bulki, - BULKI_singleton_ENTITY("regions", PDC_STRING), - regions_array); + BULKI_put(obj_bulki, BULKI_singleton_ENTITY("regions", PDC_STRING), regions_array); BULKI_ENTITY_append_BULKI(objects_array, obj_bulki); @@ -351,9 +344,7 @@ transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) obj_temp = obj_temp->next; } - BULKI_put(bulki, - BULKI_singleton_ENTITY("objects", PDC_STRING), - objects_array); + BULKI_put(bulki, BULKI_singleton_ENTITY("objects", PDC_STRING), objects_array); pthread_mutex_unlock(&metadata_query_mutex); diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index a7ffd21c1..ab98d093d 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -68,23 +68,23 @@ print_usage(char *name) int main(int argc, char *argv[]) { - pdcid_t pdc, cont_prop, cont; - pdcid_t * obj_ids; - int n_obj, n_query, my_obj, my_obj_s, n_tag, n_tag_s; - int obj_1percent = 0; - int proc_num, my_rank, i; - char obj_name[128]; - double stime, total_time, percent_time; - pdc_kvtag_t kvtag; - void ** values; + pdcid_t pdc, cont_prop, cont; + pdcid_t * obj_ids; + int n_obj, n_query, my_obj, my_obj_s, n_tag, n_tag_s; + int obj_1percent = 0; + int proc_num, my_rank, i; + char obj_name[128]; + double stime, total_time, percent_time; + pdc_kvtag_t kvtag; + void ** values; pdc_var_type_t value_type; size_t value_size; int ret_value = SUCCEED; // counters for verification statistics - int verified_success = 0; - int verified_fail = 0; - int all_verified_success = 0; - int all_verified_fail = 0; + int verified_success = 0; + int verified_fail = 0; + int all_verified_success = 0; + int all_verified_fail = 0; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); @@ -175,7 +175,7 @@ main(int argc, char *argv[]) (void *)&value_size) < 0) PGOTO_ERROR(FAIL, "Failed to get a kvtag from o%d\n", i + n_tag_s); - int expected_value = i + n_tag_s; // Assuming tags were added in order starting from 0 + int expected_value = i + n_tag_s; // Assuming tags were added in order starting from 0 // count successful and failed verifications instead of immediate error if (*(int *)(values[i]) == expected_value) { @@ -185,8 +185,8 @@ main(int argc, char *argv[]) verified_fail++; // log first 10 failures for debugging if (verified_fail <= 10) { - LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", - i + n_tag_s, expected_value, *(int *)(values[i])); + LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", i + n_tag_s, expected_value, + *(int *)(values[i])); } } free(values[i]); @@ -197,8 +197,8 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, n_obj, - total_time, n_query / total_time); + LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, + n_obj, total_time, n_query / total_time); free(values); free(obj_ids); @@ -209,7 +209,7 @@ main(int argc, char *argv[]) MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else all_verified_success = verified_success; - all_verified_fail = verified_fail; + all_verified_fail = verified_fail; #endif if (my_rank == 0) { @@ -225,7 +225,8 @@ main(int argc, char *argv[]) if (all_verified_fail > 0) { LOG_ERROR("WARNING: %d objects failed verification!\n", all_verified_fail); - } else { + } + else { LOG_INFO("SUCCESS: All objects verified correctly!\n"); } } From ca6b6369ee9f7931ff4b918775d04dc1c4d49872 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 26 Mar 2026 17:02:00 -0400 Subject: [PATCH 10/13] Move the bulki logs under build preprocessor --- src/commons/serde/bulki/bulki.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/commons/serde/bulki/bulki.c b/src/commons/serde/bulki/bulki.c index b9d4deaeb..372a784b5 100644 --- a/src/commons/serde/bulki/bulki.c +++ b/src/commons/serde/bulki/bulki.c @@ -624,7 +624,9 @@ BULKI_Entity_free(BULKI_Entity *bulk_entity, int free_struct) for (size_t i = 0; i < bulk_entity->count; i++) { BULKI_free(&bulki_array[i], 0); } +#ifdef BUKLI_DEBUG_LOG LOG_INFO("Freeing bulki_array 1\n"); +#endif bulki_array = NULL; } else if (bulk_entity->pdc_type == PDC_BULKI_ENT && bulk_entity->data != NULL) { @@ -632,7 +634,9 @@ BULKI_Entity_free(BULKI_Entity *bulk_entity, int free_struct) for (size_t i = 0; i < bulk_entity->count; i++) { BULKI_Entity_free(&bulki_entity_array[i], 0); } +#ifdef BUKLI_DEBUG_LOG LOG_INFO("Freeing bulki_array 2\n"); +#endif bulki_entity_array = NULL; } } @@ -640,15 +644,21 @@ BULKI_Entity_free(BULKI_Entity *bulk_entity, int free_struct) if (bulk_entity->pdc_type == PDC_BULKI && bulk_entity->data != NULL) { BULKI_free((BULKI *)bulk_entity->data, 0); bulk_entity->data = NULL; +#ifdef BUKLI_DEBUG_LOG LOG_INFO("Freeing bulki_item 1\n"); +#endif } } +#ifdef BUKLI_DEBUG_LOG LOG_INFO("Freeing bulk_entity\n"); +#endif if (bulk_entity->data != NULL) { +#ifdef BUKLI_DEBUG_LOG LOG_INFO( "bulki_entity->class: %d, bulki_entity->class: %d, bulki_entity->data: %p, bulki_entity: " "%p\n", bulk_entity->pdc_class, bulk_entity->pdc_type, bulk_entity->data, bulk_entity); +#endif bulk_entity->data = (void *)PDC_free(bulk_entity->data); bulk_entity->data = NULL; } From b995ac85bdc4535c27b55d910c61d7379aa3fcbe Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 26 Mar 2026 17:02:46 -0400 Subject: [PATCH 11/13] Add timer log for server restart --- src/server/pdc_server.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 1b0d1ec21..5a21a3e02 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1579,7 +1579,7 @@ PDC_Server_restart(char *filename) char magic[PDC_CHECKPOINT_MAGIC_LEN + 1]; FILE * file = NULL; -#ifdef PDC_TIMING +#if defined(PDC_TIMING) || defined(ENABLE_MPI) double start = MPI_Wtime(); #endif @@ -1983,6 +1983,13 @@ PDC_Server_restart(char *filename) #ifdef PDC_TIMING pdc_server_timings->PDCserver_restart += MPI_Wtime() - start; #endif + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + if (!pdc_server_rank_g) { + LOG_INFO("total restart time = %lf\n", MPI_Wtime() - start); + } +#endif // ensure file is closed if error occurred before BULKI_deserialize_from_file if (file != NULL) { fclose(file); From 0b76afce885b1ae3a8adff37f9f121db05b380ae Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 26 Mar 2026 17:04:01 -0400 Subject: [PATCH 12/13] Fix issue in printing server close time --- src/api/close_server.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/api/close_server.c b/src/api/close_server.c index 853826980..96f34c58a 100644 --- a/src/api/close_server.c +++ b/src/api/close_server.c @@ -42,11 +42,12 @@ main(int argc, char *argv[]) int rank; double start; MPI_Init(&argc, &argv); - +#endif + pdc = PDCinit("pdc"); +#ifdef ENABLE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &rank); start = MPI_Wtime(); #endif - pdc = PDCinit("pdc"); PDC_Client_close_all_server(); @@ -54,6 +55,7 @@ main(int argc, char *argv[]) LOG_ERROR("Failed to close PDC\n"); #ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); if (!rank) { LOG_INFO("total close time = %lf\n", MPI_Wtime() - start); } From 08bff24f6c94d4661af6981a567c12ba9d331e9d Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 15 May 2026 01:34:57 -0400 Subject: [PATCH 13/13] Fix log level to report the total checkpoint time --- src/server/pdc_server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 54892686d..2ec1f06ec 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1491,7 +1491,7 @@ PDC_Server_checkpoint() checkpoint_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); if (pdc_server_rank_g == 0) - LOG_ERROR("Rank[ ALL]: Total checkpoint time = %.6f\n", checkpoint_time); + LOG_INFO("Rank[ ALL]: Total checkpoint time = %.6f\n", checkpoint_time); #endif if (pdc_server_rank_g == 0) {