From 5f3a6dc02bb49d1c49ad8b198ee3147d0691b888 Mon Sep 17 00:00:00 2001 From: Vivien Kraus Date: Mon, 8 May 2023 14:13:06 +0200 Subject: Move the cache entry backend from hash database to append-only file. --- src/libdisfluid/disfluid-append-only-file.h | 1 + src/libdisfluid/disfluid-cache-entry.h | 439 ++++++++++++++++++---------- src/libdisfluid/disfluid-tests.h | 116 +++----- 3 files changed, 325 insertions(+), 231 deletions(-) diff --git a/src/libdisfluid/disfluid-append-only-file.h b/src/libdisfluid/disfluid-append-only-file.h index 65ad40c..c44e7be 100644 --- a/src/libdisfluid/disfluid-append-only-file.h +++ b/src/libdisfluid/disfluid-append-only-file.h @@ -22,6 +22,7 @@ MAYBE_UNUSED static int ao_file_commit_transaction (int fd); MAYBE_UNUSED static void ao_file_abort_transaction (int fd); # include +# include /* The file is structured in this way: diff --git a/src/libdisfluid/disfluid-cache-entry.h b/src/libdisfluid/disfluid-cache-entry.h index 76de29a..ca4f743 100644 --- a/src/libdisfluid/disfluid-cache-entry.h +++ b/src/libdisfluid/disfluid-cache-entry.h @@ -6,228 +6,353 @@ # include MAYBE_UNUSED static int -db_mark_cache_entry (const char *db_root, const string_desc_t * id); - -MAYBE_UNUSED static int -db_read_cache_entry (const char *db_root, const string_desc_t * id, +ao_cache_entry_read (int fd, size_t offset, struct timespec *request_date, - struct timespec *response_date, bool *invalidated, - string_desc_t * key_id, - string_desc_t * response_header_id, - string_desc_t * response_body_id); + struct timespec *response_date, + bool *invalidated, + bool read_key, + bool read_response_header, + bool read_response_body, + string_desc_t * key, + string_desc_t * response_header, + string_desc_t * response_body); MAYBE_UNUSED static int -db_write_cache_entry (const char *db_root, string_desc_t * id, - const struct timespec *request_date, - const struct timespec *response_date, bool invalidated, - const string_desc_t * key_id, - const string_desc_t * response_header_id, - const string_desc_t * response_body_id); +ao_cache_entry_push (int fd, size_t *offset, + const struct timespec *request_date, + const struct timespec *response_date, + bool invalidated, + const string_desc_t key, + const string_desc_t response_header, + const string_desc_t response_body); -# include "disfluid-db.h" +# include "disfluid-append-only-file.h" # include "safe-alloc.h" -MAYBE_UNUSED static int -db_mark_cache_entry (const char *db_root, const string_desc_t * id) -{ - struct timespec request_date = { 0 }, response_date = { 0 }; - bool invalidated = false; - string_desc_t key = { 0 }, header = { 0 }, body = { 0 }; - int error = db_read_cache_entry (db_root, id, &request_date, &response_date, - &invalidated, &key, &header, &body); - if (error < 0) - { - goto cleanup; - } - int key_error = db_mark_leaf (db_root, &key); - int header_error = db_mark_leaf (db_root, &header); - int body_error = db_mark_leaf (db_root, &body); - if (key_error < 0 || header_error < 0 || body_error < 0) - { - error = -1; - } -cleanup: - FREE (key._data); - FREE (header._data); - FREE (body._data); - return error; -} - /* The cache entry file is organized as follows: - - on 16 bytes, "disfluid c.entry", - on 8 bytes, the request date (timestamp in seconds), - on 4 bytes, the request date (number of nanoseconds), - on 8 bytes, the response date (timestamp in seconds), - on 4 bytes, the response date (number of nanoseconds), - - on 32 bytes, the key digest, - - on 32 bytes, the header digest, - - on 32 bytes, the body digest, + - on 8 bytes, the size of the key, + - on 8 bytes, the offset of the key, + - on 8 bytes, the size of the header, + - on 8 bytes, the offset of the header, + - on 8 bytes, the size of the body, + - on 8 bytes, the offset of the body, - 80 if the entry has been artificially invalidated, 00 otherwise. + - 7 bytes reserved. */ +struct cache_entry_header +{ + struct timespec request_date; + struct timespec response_date; + size_t key_size; + size_t key_offset; + size_t header_size; + size_t header_offset; + size_t body_size; + size_t body_offset; + bool invalidated; +}; + static int -parse_timespec (const char *data, struct timespec *ts) +ao_cache_entry_timespec_from_bytes (const uint8_t * bytes, + struct timespec *date) { - ts->tv_sec = 0; - ts->tv_nsec = 0; + date->tv_sec = 0; + date->tv_nsec = 0; for (size_t i = 0; i < 8; i++) { - ts->tv_sec *= 256; - ts->tv_sec += (uint8_t) data[i]; + date->tv_sec *= 256; + date->tv_sec += bytes[i]; } - for (size_t i = 0; i < 4; i++) + for (size_t j = 8; j < 12; j++) { - ts->tv_nsec *= 256; - ts->tv_nsec += (uint8_t) data[8 + i]; + date->tv_nsec *= 256; + date->tv_nsec += bytes[j]; } - if (ts->tv_nsec < 1000000000) + if (date->tv_nsec >= 1000000000) { - return 0; + return -1; } - return -1; + return 0; } static int -represent_timespec (char *data, const struct timespec *ts) +ao_cache_entry_timespec_to_bytes (const struct timespec *date, + uint8_t * bytes) { - struct timespec cp = {.tv_sec = ts->tv_sec,.tv_nsec = ts->tv_nsec }; + if (date->tv_nsec >= 1000000000) + { + return -1; + } + uint64_t date_sec = date->tv_sec; + uint32_t date_nsec = date->tv_nsec; for (size_t i = 8; i-- > 0;) { - data[i] = (char) (uint8_t) cp.tv_sec % 256; - cp.tv_sec /= 256; + bytes[i] = date_sec % 256; + date_sec /= 256; } - for (size_t i = 4; i-- > 0;) + for (size_t i = 12; i-- > 8;) { - data[8 + i] = (char) (uint8_t) cp.tv_nsec % 256; - cp.tv_nsec /= 256; + bytes[i] = date_nsec % 256; + date_nsec /= 256; } + return 0; } -MAYBE_UNUSED static int -db_read_cache_entry (const char *db_root, const string_desc_t * id, - struct timespec *request_date, - struct timespec *response_date, bool *invalidated, - string_desc_t * key_id, - string_desc_t * response_header_id, - string_desc_t * response_body_id) +static int +ao_cache_entry_offset_from_bytes (const uint8_t * bytes, size_t *length, + size_t *offset) { - string_desc_t file_data = { 0 }; - request_date->tv_sec = 0; - request_date->tv_nsec = 0; - response_date->tv_sec = 0; - response_date->tv_nsec = 0; - *invalidated = false; - assert (key_id->_data == NULL); - assert (response_header_id->_data == NULL); - assert (response_body_id->_data == NULL); - key_id->_nbytes = 0; - response_header_id->_nbytes = 0; - response_body_id->_nbytes = 0; - int error = db_read (db_root, id, &file_data); - if (error < 0) + *length = 0; + *offset = 0; + for (size_t i = 0; i < 8; i++) { - goto cleanup; + *length *= 256; + *length += bytes[i]; } - if (file_data._nbytes != 16 + 8 + 4 + 8 + 4 + 32 + 32 + 32 + 1) + for (size_t j = 8; j < 16; j++) { - error = -1; - goto cleanup; + *offset *= 256; + *offset += bytes[j]; } - if (memcmp (file_data._data, "disfluid c.entry", 16) != 0) + return 0; +} + +static int +ao_cache_entry_offset_to_bytes (size_t length, size_t offset, uint8_t * bytes) +{ + for (size_t i = 8; i-- > 0;) { - error = -1; - goto cleanup; + bytes[i] = length % 256; + length /= 256; } - const char *data = file_data._data + 16; - error = parse_timespec (data, request_date); - data += 12; - if (error != 0) + for (size_t i = 16; i-- > 8;) { - goto cleanup; + bytes[i] = offset % 256; + offset /= 256; } - error = parse_timespec (data, response_date); - data += 12; - if (error != 0) + return 0; +} + +static int +ao_cache_entry_from_bytes (const uint8_t * bytes, + struct cache_entry_header *entry) +{ + if (ao_cache_entry_timespec_from_bytes (bytes, &(entry->request_date)) < 0) { - goto cleanup; + return -1; } - if (ALLOC_N (key_id->_data, 32) < 0) + bytes += 12; + if (ao_cache_entry_timespec_from_bytes (bytes, &(entry->response_date)) < 0) { - error = -2; - goto cleanup; + return -1; } - key_id->_nbytes = 32; - memcpy (key_id->_data, data, 32); - data += 32; - if (ALLOC_N (response_header_id->_data, 32) < 0) + bytes += 12; + if (ao_cache_entry_offset_from_bytes + (bytes, &(entry->key_size), &(entry->key_offset)) < 0) { - error = -2; - goto cleanup; + return -1; } - response_header_id->_nbytes = 32; - memcpy (response_header_id->_data, data, 32); - data += 32; - if (ALLOC_N (response_body_id->_data, 32) < 0) + bytes += 16; + if (ao_cache_entry_offset_from_bytes + (bytes, &(entry->header_size), &(entry->header_offset)) < 0) { - error = -2; - goto cleanup; + return -1; + } + bytes += 16; + if (ao_cache_entry_offset_from_bytes + (bytes, &(entry->body_size), &(entry->body_offset)) < 0) + { + return -1; } - response_body_id->_nbytes = 32; - memcpy (response_body_id->_data, data, 32); - data += 32; - switch (*data) + bytes += 16; + if (*bytes >= 128) { - case 0: - break; - case 80: - *invalidated = true; - break; - default: - error = -1; - goto cleanup; + entry->invalidated = true; } -cleanup: - if (error != 0) + else { - FREE (key_id->_data); - FREE (response_header_id->_data); - FREE (response_body_id->_data); + entry->invalidated = false; } - FREE (file_data._data); - return error; + return 0; } -MAYBE_UNUSED static int -db_write_cache_entry (const char *db_root, string_desc_t * id, - const struct timespec *request_date, - const struct timespec *response_date, bool invalidated, - const string_desc_t * key_id, - const string_desc_t * response_header_id, - const string_desc_t * response_body_id) +static int +ao_cache_entry_to_bytes (const struct cache_entry_header *header, + uint8_t * dest) { - char file_data[16 + 8 + 4 + 8 + 4 + 32 + 32 + 32 + 1]; - if (key_id->_nbytes != 32 || response_header_id->_nbytes != 32 - || response_body_id->_nbytes != 32) + if (ao_cache_entry_timespec_to_bytes (&(header->request_date), dest) < 0) + { + return -1; + } + dest += 12; + if (ao_cache_entry_timespec_to_bytes (&(header->response_date), dest) < 0) { return -1; } - memcpy (file_data, "disfluid c.entry", 16); - represent_timespec (file_data + 16, request_date); - represent_timespec (file_data + 16 + 12, response_date); - memcpy (file_data + 16 + 12 + 12, key_id->_data, 32); - memcpy (file_data + 16 + 12 + 12 + 32, response_header_id->_data, 32); - memcpy (file_data + 16 + 12 + 12 + 32 + 32, response_body_id->_data, 32); - file_data[16 + 12 + 12 + 32 + 32 + 32] = 0; - if (invalidated) + dest += 12; + if (ao_cache_entry_offset_to_bytes + (header->key_size, header->key_offset, dest) < 0) { - file_data[16 + 12 + 12 + 32 + 32 + 32] = 80; + return -1; } - const string_desc_t sd_data = { - ._nbytes = sizeof (file_data), - ._data = file_data + dest += 16; + if (ao_cache_entry_offset_to_bytes + (header->header_size, header->header_offset, dest) < 0) + { + return -1; + } + dest += 16; + if (ao_cache_entry_offset_to_bytes + (header->body_size, header->body_offset, dest) < 0) + { + return -1; + } + dest += 16; + *dest = 0; + if (header->invalidated) + { + *dest = 128; + } + dest += 1; + for (size_t i = 0; i < 7; i++) + { + dest[i] = 0; + } + return 0; +} + +static int +ao_cache_entry_read (int fd, size_t offset, + struct timespec *request_date, + struct timespec *response_date, + bool *invalidated, + bool read_key, + bool read_response_header, + bool read_response_body, + string_desc_t * key, + string_desc_t * response_header, + string_desc_t * response_body) +{ + struct cache_entry_header header; + uint8_t header_bytes[12 * 2 + 16 * 3 + 1 + 7] = { 0 }; + string_desc_t data = {._nbytes = sizeof (header_bytes),._data = header_bytes }; - return db_write (db_root, id, &sd_data); + key->_data = NULL; + response_header->_data = NULL; + response_body->_data = NULL; + *invalidated = false; + request_date->tv_sec = request_date->tv_nsec = 0; + response_date->tv_sec = response_date->tv_nsec = 0; + if (ao_file_read (fd, offset, data) < 0) + { + return -1; + } + if (ao_cache_entry_from_bytes (header_bytes, &header) < 0) + { + return -1; + } + request_date->tv_sec = header.request_date.tv_sec; + request_date->tv_nsec = header.request_date.tv_nsec; + response_date->tv_sec = header.response_date.tv_sec; + response_date->tv_nsec = header.response_date.tv_nsec; + *invalidated = header.invalidated; + key->_nbytes = header.key_size; + response_header->_nbytes = header.header_size; + response_body->_nbytes = header.body_size; + if (read_key) + { + if (ALLOC_N (key->_data, key->_nbytes) < 0) + { + goto on_error; + } + if (ao_file_read (fd, header.key_offset, *key) < 0) + { + goto on_error; + } + } + if (read_response_header) + { + if (ALLOC_N (response_header->_data, response_header->_nbytes) < 0) + { + goto on_error; + } + if (ao_file_read (fd, header.header_offset, *response_header) < 0) + { + goto on_error; + } + } + if (read_response_body) + { + if (ALLOC_N (response_body->_data, response_body->_nbytes) < 0) + { + goto on_error; + } + if (ao_file_read (fd, header.body_offset, *response_body) < 0) + { + goto on_error; + } + } + return 0; + /* Unsuccessful exit: */ +on_error: + FREE (key->_data); + FREE (response_header->_data); + FREE (response_body->_data); + return -1; } +static int +ao_cache_entry_push (int fd, size_t *offset, + const struct timespec *request_date, + const struct timespec *response_date, + bool invalidated, + const string_desc_t key, + const string_desc_t response_header, + const string_desc_t response_body) +{ + struct cache_entry_header header = { + .request_date = { + .tv_sec = request_date->tv_sec, + .tv_nsec = request_date->tv_nsec}, + .response_date = { + .tv_sec = response_date->tv_sec, + .tv_nsec = response_date->tv_nsec}, + .key_size = key._nbytes, + .header_size = response_header._nbytes, + .body_size = response_body._nbytes, + .invalidated = invalidated + }; + if (ao_file_push_data (fd, key, &(header.key_offset)) < 0) + { + return -1; + } + if (ao_file_push_data (fd, response_header, &(header.header_offset)) < 0) + { + return -1; + } + if (ao_file_push_data (fd, response_body, &(header.body_offset)) < 0) + { + return -1; + } + uint8_t header_bytes[12 * 2 + 16 * 3 + 1 + 7] = { 0 }; + if (ao_cache_entry_to_bytes (&header, header_bytes) < 0) + { + return -1; + } + string_desc_t header_str = { + ._nbytes = sizeof (header_bytes), + ._data = header_bytes + }; + if (ao_file_push_data (fd, header_str, offset) < 0) + { + return -1; + } + return 0; +} #endif /* DISFLUID_DISFLUID_CACHE_ENTRY_INCLUDED */ diff --git a/src/libdisfluid/disfluid-tests.h b/src/libdisfluid/disfluid-tests.h index 8295572..0843404 100644 --- a/src/libdisfluid/disfluid-tests.h +++ b/src/libdisfluid/disfluid-tests.h @@ -87,46 +87,6 @@ START_TEST (test_check_version) END_TEST /* *INDENT-ON* */ -/* *INDENT-OFF* */ -START_TEST (test_read_invalid_magic_number) -/* *INDENT-ON* */ - -{ - char *database; - prepare_database (&database); - string_desc_t file_id = { 0 }; - string_desc_t lit_id = { 0 }; - char data[16 + 12 + 12 + 32 + 32 + 32 + 1] = { 0 }; - memcpy (data, "invalid magic no", 16); - const string_desc_t file_data = {._nbytes = sizeof (data),._data = data }; - static const char *lit_data = "hello :)"; - const string_desc_t lit = {._nbytes = strlen (lit_data),._data = - (char *) lit_data - }; - ck_assert_int_eq (db_write (database, &lit_id, &lit), 0); - memcpy (data + 16 + 12 + 12, lit_id._data, lit_id._nbytes); - memcpy (data + 16 + 12 + 12 + 32, lit_id._data, lit_id._nbytes); - memcpy (data + 16 + 12 + 12 + 32 + 32, lit_id._data, lit_id._nbytes); - FREE (lit_id._data); - ck_assert_int_eq (db_write (database, &file_id, &file_data), 0); - struct timespec request_date, response_date; - bool invalidated; - string_desc_t key_id = { 0 }, response_header_id = { 0 }, response_body_id = - { 0 }; - ck_assert_int_lt (db_read_cache_entry - (database, &file_id, &request_date, &response_date, - &invalidated, &key_id, &response_header_id, - &response_body_id), 0); - FREE (key_id._data); - FREE (response_header_id._data); - FREE (response_body_id._data); - FREE (file_id._data); - cleanup_database (&database); -} -/* *INDENT-OFF* */ -END_TEST -/* *INDENT-ON* */ - /* *INDENT-OFF* */ START_TEST (test_read_normal) /* *INDENT-ON* */ @@ -145,48 +105,57 @@ START_TEST (test_read_normal) const string_desc_t body = {._nbytes = strlen (body_value),._data = (char *) body_value }; - string_desc_t key_id = { 0 }; - string_desc_t header_id = { 0 }; - string_desc_t body_id = { 0 }; - string_desc_t entry_id = { 0 }; const struct timespec request_date = {.tv_sec = 12345,.tv_nsec = 678910 }; const struct timespec response_date = {.tv_sec = 111213,.tv_nsec = 141516 }; - char *database; - prepare_database (&database); - ck_assert_int_eq (db_write (database, &key_id, &key), 0); - ck_assert_int_eq (db_write (database, &header_id, &header), 0); - ck_assert_int_eq (db_write (database, &body_id, &body), 0); - ck_assert_int_eq (db_write_cache_entry - (database, &entry_id, &request_date, &response_date, true, - &key_id, &header_id, &body_id), 0); - string_desc_t check_key_id = { 0 }; - string_desc_t check_header_id = { 0 }; - string_desc_t check_body_id = { 0 }; + char filename[] = "/tmp/test-ao-cache-entry-XXXXXX"; + int file = mkstemp (filename); + ck_assert_int_ge (file, 0); + static const char *magic_data = "disfluid c.entry"; + const string_desc_t file_magic = { + ._data = (char *) magic_data, + ._nbytes = strlen (magic_data) + }; + int error = 0; + size_t top; + error = ao_file_prepare (file, file_magic); + error = ao_file_lock_for_writing (file, &top); + ck_assert_int_eq (error, 0); + error = + ao_cache_entry_push (file, &top, &request_date, &response_date, true, key, + header, body); + ck_assert_int_eq (error, 0); + error = ao_file_commit_transaction (file); + ck_assert_int_eq (error, 0); + string_desc_t check_key = { 0 }; + string_desc_t check_header = { 0 }; + string_desc_t check_body = { 0 }; struct timespec check_request_date, check_response_date; bool check_invalidated; - ck_assert_int_eq (db_read_cache_entry - (database, &entry_id, &check_request_date, - &check_response_date, &check_invalidated, &check_key_id, - &check_header_id, &check_body_id), 0); + error = + ao_cache_entry_read (file, top, &check_request_date, &check_response_date, + &check_invalidated, true, true, true, &check_key, + &check_header, &check_body); + ck_assert_int_eq (error, 0); ck_assert_int_eq (check_request_date.tv_sec, 12345); ck_assert_int_eq (check_request_date.tv_nsec, 678910); ck_assert_int_eq (check_response_date.tv_sec, 111213); ck_assert_int_eq (check_response_date.tv_nsec, 141516); ck_assert (check_invalidated); - ck_assert_int_eq (check_key_id._nbytes, 32); - ck_assert_int_eq (check_header_id._nbytes, 32); - ck_assert_int_eq (check_body_id._nbytes, 32); - ck_assert_int_eq (memcmp (check_key_id._data, key_id._data, 32), 0); - ck_assert_int_eq (memcmp (check_header_id._data, header_id._data, 32), 0); - ck_assert_int_eq (memcmp (check_body_id._data, body_id._data, 32), 0); - FREE (check_key_id._data); - FREE (check_header_id._data); - FREE (check_body_id._data); - FREE (key_id._data); - FREE (header_id._data); - FREE (body_id._data); - FREE (entry_id._data); - cleanup_database (&database); + ck_assert_int_eq (check_key._nbytes, strlen (key_value)); + ck_assert_int_eq (check_header._nbytes, strlen (header_value)); + ck_assert_int_eq (check_body._nbytes, strlen (body_value)); + ck_assert_int_eq (memcmp (check_key._data, key_value, strlen (key_value)), + 0); + ck_assert_int_eq (memcmp + (check_header._data, header_value, strlen (header_value)), + 0); + ck_assert_int_eq (memcmp + (check_body._data, body_value, strlen (body_value)), 0); + FREE (check_key._data); + FREE (check_header._data); + FREE (check_body._data); + remove (filename); + close (file); } /* *INDENT-OFF* */ END_TEST @@ -905,7 +874,6 @@ run_tests (size_t *n_tests, size_t *n_errors) suite_add_tcase (suite, general); TCase *cache_entry = tcase_create (_("disfluid cache entry files")); tcase_add_test (cache_entry, test_read_normal); - tcase_add_test (cache_entry, test_read_invalid_magic_number); suite_add_tcase (suite, cache_entry); TCase *cache_key = tcase_create (_("disfluid cache key")); tcase_add_test (cache_key, test_cache_key_no_host); -- cgit v1.2.3