summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVivien Kraus <vivien@planete-kraus.eu>2023-05-08 14:13:06 +0200
committerVivien Kraus <vivien@planete-kraus.eu>2023-05-10 00:01:20 +0200
commit5f3a6dc02bb49d1c49ad8b198ee3147d0691b888 (patch)
tree8798bae8f6ba991f207ef76c21f62c01f69439ea
parent832f61229ff6e833071758992808cdc610900af6 (diff)
Move the cache entry backend from hash database to append-only file.
-rw-r--r--src/libdisfluid/disfluid-append-only-file.h1
-rw-r--r--src/libdisfluid/disfluid-cache-entry.h439
-rw-r--r--src/libdisfluid/disfluid-tests.h116
3 files changed, 325 insertions, 231 deletions
diff --git a/src/libdisfluid/disfluid-append-only-file.h b/src/libdisfluid/disfluid-append-only-file.h
index 65ad40c..c44e7be 100644
--- a/src/libdisfluid/disfluid-append-only-file.h
+++ b/src/libdisfluid/disfluid-append-only-file.h
@@ -22,6 +22,7 @@ MAYBE_UNUSED static int ao_file_commit_transaction (int fd);
MAYBE_UNUSED static void ao_file_abort_transaction (int fd);
# include <unistd.h>
+# include <assert.h>
/* The file is structured in this way:
diff --git a/src/libdisfluid/disfluid-cache-entry.h b/src/libdisfluid/disfluid-cache-entry.h
index 76de29a..ca4f743 100644
--- a/src/libdisfluid/disfluid-cache-entry.h
+++ b/src/libdisfluid/disfluid-cache-entry.h
@@ -6,228 +6,353 @@
# include <stdbool.h>
MAYBE_UNUSED static int
-db_mark_cache_entry (const char *db_root, const string_desc_t * id);
-
-MAYBE_UNUSED static int
-db_read_cache_entry (const char *db_root, const string_desc_t * id,
+ao_cache_entry_read (int fd, size_t offset,
struct timespec *request_date,
- struct timespec *response_date, bool *invalidated,
- string_desc_t * key_id,
- string_desc_t * response_header_id,
- string_desc_t * response_body_id);
+ struct timespec *response_date,
+ bool *invalidated,
+ bool read_key,
+ bool read_response_header,
+ bool read_response_body,
+ string_desc_t * key,
+ string_desc_t * response_header,
+ string_desc_t * response_body);
MAYBE_UNUSED static int
-db_write_cache_entry (const char *db_root, string_desc_t * id,
- const struct timespec *request_date,
- const struct timespec *response_date, bool invalidated,
- const string_desc_t * key_id,
- const string_desc_t * response_header_id,
- const string_desc_t * response_body_id);
+ao_cache_entry_push (int fd, size_t *offset,
+ const struct timespec *request_date,
+ const struct timespec *response_date,
+ bool invalidated,
+ const string_desc_t key,
+ const string_desc_t response_header,
+ const string_desc_t response_body);
-# include "disfluid-db.h"
+# include "disfluid-append-only-file.h"
# include "safe-alloc.h"
-MAYBE_UNUSED static int
-db_mark_cache_entry (const char *db_root, const string_desc_t * id)
-{
- struct timespec request_date = { 0 }, response_date = { 0 };
- bool invalidated = false;
- string_desc_t key = { 0 }, header = { 0 }, body = { 0 };
- int error = db_read_cache_entry (db_root, id, &request_date, &response_date,
- &invalidated, &key, &header, &body);
- if (error < 0)
- {
- goto cleanup;
- }
- int key_error = db_mark_leaf (db_root, &key);
- int header_error = db_mark_leaf (db_root, &header);
- int body_error = db_mark_leaf (db_root, &body);
- if (key_error < 0 || header_error < 0 || body_error < 0)
- {
- error = -1;
- }
-cleanup:
- FREE (key._data);
- FREE (header._data);
- FREE (body._data);
- return error;
-}
-
/* The cache entry file is organized as follows:
- - on 16 bytes, "disfluid c.entry",
- on 8 bytes, the request date (timestamp in seconds),
- on 4 bytes, the request date (number of nanoseconds),
- on 8 bytes, the response date (timestamp in seconds),
- on 4 bytes, the response date (number of nanoseconds),
- - on 32 bytes, the key digest,
- - on 32 bytes, the header digest,
- - on 32 bytes, the body digest,
+ - on 8 bytes, the size of the key,
+ - on 8 bytes, the offset of the key,
+ - on 8 bytes, the size of the header,
+ - on 8 bytes, the offset of the header,
+ - on 8 bytes, the size of the body,
+ - on 8 bytes, the offset of the body,
- 80 if the entry has been artificially invalidated, 00 otherwise.
+ - 7 bytes reserved.
*/
+struct cache_entry_header
+{
+ struct timespec request_date;
+ struct timespec response_date;
+ size_t key_size;
+ size_t key_offset;
+ size_t header_size;
+ size_t header_offset;
+ size_t body_size;
+ size_t body_offset;
+ bool invalidated;
+};
+
static int
-parse_timespec (const char *data, struct timespec *ts)
+ao_cache_entry_timespec_from_bytes (const uint8_t * bytes,
+ struct timespec *date)
{
- ts->tv_sec = 0;
- ts->tv_nsec = 0;
+ date->tv_sec = 0;
+ date->tv_nsec = 0;
for (size_t i = 0; i < 8; i++)
{
- ts->tv_sec *= 256;
- ts->tv_sec += (uint8_t) data[i];
+ date->tv_sec *= 256;
+ date->tv_sec += bytes[i];
}
- for (size_t i = 0; i < 4; i++)
+ for (size_t j = 8; j < 12; j++)
{
- ts->tv_nsec *= 256;
- ts->tv_nsec += (uint8_t) data[8 + i];
+ date->tv_nsec *= 256;
+ date->tv_nsec += bytes[j];
}
- if (ts->tv_nsec < 1000000000)
+ if (date->tv_nsec >= 1000000000)
{
- return 0;
+ return -1;
}
- return -1;
+ return 0;
}
static int
-represent_timespec (char *data, const struct timespec *ts)
+ao_cache_entry_timespec_to_bytes (const struct timespec *date,
+ uint8_t * bytes)
{
- struct timespec cp = {.tv_sec = ts->tv_sec,.tv_nsec = ts->tv_nsec };
+ if (date->tv_nsec >= 1000000000)
+ {
+ return -1;
+ }
+ uint64_t date_sec = date->tv_sec;
+ uint32_t date_nsec = date->tv_nsec;
for (size_t i = 8; i-- > 0;)
{
- data[i] = (char) (uint8_t) cp.tv_sec % 256;
- cp.tv_sec /= 256;
+ bytes[i] = date_sec % 256;
+ date_sec /= 256;
}
- for (size_t i = 4; i-- > 0;)
+ for (size_t i = 12; i-- > 8;)
{
- data[8 + i] = (char) (uint8_t) cp.tv_nsec % 256;
- cp.tv_nsec /= 256;
+ bytes[i] = date_nsec % 256;
+ date_nsec /= 256;
}
+ return 0;
}
-MAYBE_UNUSED static int
-db_read_cache_entry (const char *db_root, const string_desc_t * id,
- struct timespec *request_date,
- struct timespec *response_date, bool *invalidated,
- string_desc_t * key_id,
- string_desc_t * response_header_id,
- string_desc_t * response_body_id)
+static int
+ao_cache_entry_offset_from_bytes (const uint8_t * bytes, size_t *length,
+ size_t *offset)
{
- string_desc_t file_data = { 0 };
- request_date->tv_sec = 0;
- request_date->tv_nsec = 0;
- response_date->tv_sec = 0;
- response_date->tv_nsec = 0;
- *invalidated = false;
- assert (key_id->_data == NULL);
- assert (response_header_id->_data == NULL);
- assert (response_body_id->_data == NULL);
- key_id->_nbytes = 0;
- response_header_id->_nbytes = 0;
- response_body_id->_nbytes = 0;
- int error = db_read (db_root, id, &file_data);
- if (error < 0)
+ *length = 0;
+ *offset = 0;
+ for (size_t i = 0; i < 8; i++)
{
- goto cleanup;
+ *length *= 256;
+ *length += bytes[i];
}
- if (file_data._nbytes != 16 + 8 + 4 + 8 + 4 + 32 + 32 + 32 + 1)
+ for (size_t j = 8; j < 16; j++)
{
- error = -1;
- goto cleanup;
+ *offset *= 256;
+ *offset += bytes[j];
}
- if (memcmp (file_data._data, "disfluid c.entry", 16) != 0)
+ return 0;
+}
+
+static int
+ao_cache_entry_offset_to_bytes (size_t length, size_t offset, uint8_t * bytes)
+{
+ for (size_t i = 8; i-- > 0;)
{
- error = -1;
- goto cleanup;
+ bytes[i] = length % 256;
+ length /= 256;
}
- const char *data = file_data._data + 16;
- error = parse_timespec (data, request_date);
- data += 12;
- if (error != 0)
+ for (size_t i = 16; i-- > 8;)
{
- goto cleanup;
+ bytes[i] = offset % 256;
+ offset /= 256;
}
- error = parse_timespec (data, response_date);
- data += 12;
- if (error != 0)
+ return 0;
+}
+
+static int
+ao_cache_entry_from_bytes (const uint8_t * bytes,
+ struct cache_entry_header *entry)
+{
+ if (ao_cache_entry_timespec_from_bytes (bytes, &(entry->request_date)) < 0)
{
- goto cleanup;
+ return -1;
}
- if (ALLOC_N (key_id->_data, 32) < 0)
+ bytes += 12;
+ if (ao_cache_entry_timespec_from_bytes (bytes, &(entry->response_date)) < 0)
{
- error = -2;
- goto cleanup;
+ return -1;
}
- key_id->_nbytes = 32;
- memcpy (key_id->_data, data, 32);
- data += 32;
- if (ALLOC_N (response_header_id->_data, 32) < 0)
+ bytes += 12;
+ if (ao_cache_entry_offset_from_bytes
+ (bytes, &(entry->key_size), &(entry->key_offset)) < 0)
{
- error = -2;
- goto cleanup;
+ return -1;
}
- response_header_id->_nbytes = 32;
- memcpy (response_header_id->_data, data, 32);
- data += 32;
- if (ALLOC_N (response_body_id->_data, 32) < 0)
+ bytes += 16;
+ if (ao_cache_entry_offset_from_bytes
+ (bytes, &(entry->header_size), &(entry->header_offset)) < 0)
{
- error = -2;
- goto cleanup;
+ return -1;
+ }
+ bytes += 16;
+ if (ao_cache_entry_offset_from_bytes
+ (bytes, &(entry->body_size), &(entry->body_offset)) < 0)
+ {
+ return -1;
}
- response_body_id->_nbytes = 32;
- memcpy (response_body_id->_data, data, 32);
- data += 32;
- switch (*data)
+ bytes += 16;
+ if (*bytes >= 128)
{
- case 0:
- break;
- case 80:
- *invalidated = true;
- break;
- default:
- error = -1;
- goto cleanup;
+ entry->invalidated = true;
}
-cleanup:
- if (error != 0)
+ else
{
- FREE (key_id->_data);
- FREE (response_header_id->_data);
- FREE (response_body_id->_data);
+ entry->invalidated = false;
}
- FREE (file_data._data);
- return error;
+ return 0;
}
-MAYBE_UNUSED static int
-db_write_cache_entry (const char *db_root, string_desc_t * id,
- const struct timespec *request_date,
- const struct timespec *response_date, bool invalidated,
- const string_desc_t * key_id,
- const string_desc_t * response_header_id,
- const string_desc_t * response_body_id)
+static int
+ao_cache_entry_to_bytes (const struct cache_entry_header *header,
+ uint8_t * dest)
{
- char file_data[16 + 8 + 4 + 8 + 4 + 32 + 32 + 32 + 1];
- if (key_id->_nbytes != 32 || response_header_id->_nbytes != 32
- || response_body_id->_nbytes != 32)
+ if (ao_cache_entry_timespec_to_bytes (&(header->request_date), dest) < 0)
+ {
+ return -1;
+ }
+ dest += 12;
+ if (ao_cache_entry_timespec_to_bytes (&(header->response_date), dest) < 0)
{
return -1;
}
- memcpy (file_data, "disfluid c.entry", 16);
- represent_timespec (file_data + 16, request_date);
- represent_timespec (file_data + 16 + 12, response_date);
- memcpy (file_data + 16 + 12 + 12, key_id->_data, 32);
- memcpy (file_data + 16 + 12 + 12 + 32, response_header_id->_data, 32);
- memcpy (file_data + 16 + 12 + 12 + 32 + 32, response_body_id->_data, 32);
- file_data[16 + 12 + 12 + 32 + 32 + 32] = 0;
- if (invalidated)
+ dest += 12;
+ if (ao_cache_entry_offset_to_bytes
+ (header->key_size, header->key_offset, dest) < 0)
{
- file_data[16 + 12 + 12 + 32 + 32 + 32] = 80;
+ return -1;
}
- const string_desc_t sd_data = {
- ._nbytes = sizeof (file_data),
- ._data = file_data
+ dest += 16;
+ if (ao_cache_entry_offset_to_bytes
+ (header->header_size, header->header_offset, dest) < 0)
+ {
+ return -1;
+ }
+ dest += 16;
+ if (ao_cache_entry_offset_to_bytes
+ (header->body_size, header->body_offset, dest) < 0)
+ {
+ return -1;
+ }
+ dest += 16;
+ *dest = 0;
+ if (header->invalidated)
+ {
+ *dest = 128;
+ }
+ dest += 1;
+ for (size_t i = 0; i < 7; i++)
+ {
+ dest[i] = 0;
+ }
+ return 0;
+}
+
+static int
+ao_cache_entry_read (int fd, size_t offset,
+ struct timespec *request_date,
+ struct timespec *response_date,
+ bool *invalidated,
+ bool read_key,
+ bool read_response_header,
+ bool read_response_body,
+ string_desc_t * key,
+ string_desc_t * response_header,
+ string_desc_t * response_body)
+{
+ struct cache_entry_header header;
+ uint8_t header_bytes[12 * 2 + 16 * 3 + 1 + 7] = { 0 };
+ string_desc_t data = {._nbytes = sizeof (header_bytes),._data = header_bytes
};
- return db_write (db_root, id, &sd_data);
+ key->_data = NULL;
+ response_header->_data = NULL;
+ response_body->_data = NULL;
+ *invalidated = false;
+ request_date->tv_sec = request_date->tv_nsec = 0;
+ response_date->tv_sec = response_date->tv_nsec = 0;
+ if (ao_file_read (fd, offset, data) < 0)
+ {
+ return -1;
+ }
+ if (ao_cache_entry_from_bytes (header_bytes, &header) < 0)
+ {
+ return -1;
+ }
+ request_date->tv_sec = header.request_date.tv_sec;
+ request_date->tv_nsec = header.request_date.tv_nsec;
+ response_date->tv_sec = header.response_date.tv_sec;
+ response_date->tv_nsec = header.response_date.tv_nsec;
+ *invalidated = header.invalidated;
+ key->_nbytes = header.key_size;
+ response_header->_nbytes = header.header_size;
+ response_body->_nbytes = header.body_size;
+ if (read_key)
+ {
+ if (ALLOC_N (key->_data, key->_nbytes) < 0)
+ {
+ goto on_error;
+ }
+ if (ao_file_read (fd, header.key_offset, *key) < 0)
+ {
+ goto on_error;
+ }
+ }
+ if (read_response_header)
+ {
+ if (ALLOC_N (response_header->_data, response_header->_nbytes) < 0)
+ {
+ goto on_error;
+ }
+ if (ao_file_read (fd, header.header_offset, *response_header) < 0)
+ {
+ goto on_error;
+ }
+ }
+ if (read_response_body)
+ {
+ if (ALLOC_N (response_body->_data, response_body->_nbytes) < 0)
+ {
+ goto on_error;
+ }
+ if (ao_file_read (fd, header.body_offset, *response_body) < 0)
+ {
+ goto on_error;
+ }
+ }
+ return 0;
+ /* Unsuccessful exit: */
+on_error:
+ FREE (key->_data);
+ FREE (response_header->_data);
+ FREE (response_body->_data);
+ return -1;
}
+static int
+ao_cache_entry_push (int fd, size_t *offset,
+ const struct timespec *request_date,
+ const struct timespec *response_date,
+ bool invalidated,
+ const string_desc_t key,
+ const string_desc_t response_header,
+ const string_desc_t response_body)
+{
+ struct cache_entry_header header = {
+ .request_date = {
+ .tv_sec = request_date->tv_sec,
+ .tv_nsec = request_date->tv_nsec},
+ .response_date = {
+ .tv_sec = response_date->tv_sec,
+ .tv_nsec = response_date->tv_nsec},
+ .key_size = key._nbytes,
+ .header_size = response_header._nbytes,
+ .body_size = response_body._nbytes,
+ .invalidated = invalidated
+ };
+ if (ao_file_push_data (fd, key, &(header.key_offset)) < 0)
+ {
+ return -1;
+ }
+ if (ao_file_push_data (fd, response_header, &(header.header_offset)) < 0)
+ {
+ return -1;
+ }
+ if (ao_file_push_data (fd, response_body, &(header.body_offset)) < 0)
+ {
+ return -1;
+ }
+ uint8_t header_bytes[12 * 2 + 16 * 3 + 1 + 7] = { 0 };
+ if (ao_cache_entry_to_bytes (&header, header_bytes) < 0)
+ {
+ return -1;
+ }
+ string_desc_t header_str = {
+ ._nbytes = sizeof (header_bytes),
+ ._data = header_bytes
+ };
+ if (ao_file_push_data (fd, header_str, offset) < 0)
+ {
+ return -1;
+ }
+ return 0;
+}
#endif /* DISFLUID_DISFLUID_CACHE_ENTRY_INCLUDED */
diff --git a/src/libdisfluid/disfluid-tests.h b/src/libdisfluid/disfluid-tests.h
index 8295572..0843404 100644
--- a/src/libdisfluid/disfluid-tests.h
+++ b/src/libdisfluid/disfluid-tests.h
@@ -88,46 +88,6 @@ END_TEST
/* *INDENT-ON* */
/* *INDENT-OFF* */
-START_TEST (test_read_invalid_magic_number)
-/* *INDENT-ON* */
-
-{
- char *database;
- prepare_database (&database);
- string_desc_t file_id = { 0 };
- string_desc_t lit_id = { 0 };
- char data[16 + 12 + 12 + 32 + 32 + 32 + 1] = { 0 };
- memcpy (data, "invalid magic no", 16);
- const string_desc_t file_data = {._nbytes = sizeof (data),._data = data };
- static const char *lit_data = "hello :)";
- const string_desc_t lit = {._nbytes = strlen (lit_data),._data =
- (char *) lit_data
- };
- ck_assert_int_eq (db_write (database, &lit_id, &lit), 0);
- memcpy (data + 16 + 12 + 12, lit_id._data, lit_id._nbytes);
- memcpy (data + 16 + 12 + 12 + 32, lit_id._data, lit_id._nbytes);
- memcpy (data + 16 + 12 + 12 + 32 + 32, lit_id._data, lit_id._nbytes);
- FREE (lit_id._data);
- ck_assert_int_eq (db_write (database, &file_id, &file_data), 0);
- struct timespec request_date, response_date;
- bool invalidated;
- string_desc_t key_id = { 0 }, response_header_id = { 0 }, response_body_id =
- { 0 };
- ck_assert_int_lt (db_read_cache_entry
- (database, &file_id, &request_date, &response_date,
- &invalidated, &key_id, &response_header_id,
- &response_body_id), 0);
- FREE (key_id._data);
- FREE (response_header_id._data);
- FREE (response_body_id._data);
- FREE (file_id._data);
- cleanup_database (&database);
-}
-/* *INDENT-OFF* */
-END_TEST
-/* *INDENT-ON* */
-
-/* *INDENT-OFF* */
START_TEST (test_read_normal)
/* *INDENT-ON* */
@@ -145,48 +105,57 @@ START_TEST (test_read_normal)
const string_desc_t body = {._nbytes = strlen (body_value),._data =
(char *) body_value
};
- string_desc_t key_id = { 0 };
- string_desc_t header_id = { 0 };
- string_desc_t body_id = { 0 };
- string_desc_t entry_id = { 0 };
const struct timespec request_date = {.tv_sec = 12345,.tv_nsec = 678910 };
const struct timespec response_date = {.tv_sec = 111213,.tv_nsec = 141516 };
- char *database;
- prepare_database (&database);
- ck_assert_int_eq (db_write (database, &key_id, &key), 0);
- ck_assert_int_eq (db_write (database, &header_id, &header), 0);
- ck_assert_int_eq (db_write (database, &body_id, &body), 0);
- ck_assert_int_eq (db_write_cache_entry
- (database, &entry_id, &request_date, &response_date, true,
- &key_id, &header_id, &body_id), 0);
- string_desc_t check_key_id = { 0 };
- string_desc_t check_header_id = { 0 };
- string_desc_t check_body_id = { 0 };
+ char filename[] = "/tmp/test-ao-cache-entry-XXXXXX";
+ int file = mkstemp (filename);
+ ck_assert_int_ge (file, 0);
+ static const char *magic_data = "disfluid c.entry";
+ const string_desc_t file_magic = {
+ ._data = (char *) magic_data,
+ ._nbytes = strlen (magic_data)
+ };
+ int error = 0;
+ size_t top;
+ error = ao_file_prepare (file, file_magic);
+ error = ao_file_lock_for_writing (file, &top);
+ ck_assert_int_eq (error, 0);
+ error =
+ ao_cache_entry_push (file, &top, &request_date, &response_date, true, key,
+ header, body);
+ ck_assert_int_eq (error, 0);
+ error = ao_file_commit_transaction (file);
+ ck_assert_int_eq (error, 0);
+ string_desc_t check_key = { 0 };
+ string_desc_t check_header = { 0 };
+ string_desc_t check_body = { 0 };
struct timespec check_request_date, check_response_date;
bool check_invalidated;
- ck_assert_int_eq (db_read_cache_entry
- (database, &entry_id, &check_request_date,
- &check_response_date, &check_invalidated, &check_key_id,
- &check_header_id, &check_body_id), 0);
+ error =
+ ao_cache_entry_read (file, top, &check_request_date, &check_response_date,
+ &check_invalidated, true, true, true, &check_key,
+ &check_header, &check_body);
+ ck_assert_int_eq (error, 0);
ck_assert_int_eq (check_request_date.tv_sec, 12345);
ck_assert_int_eq (check_request_date.tv_nsec, 678910);
ck_assert_int_eq (check_response_date.tv_sec, 111213);
ck_assert_int_eq (check_response_date.tv_nsec, 141516);
ck_assert (check_invalidated);
- ck_assert_int_eq (check_key_id._nbytes, 32);
- ck_assert_int_eq (check_header_id._nbytes, 32);
- ck_assert_int_eq (check_body_id._nbytes, 32);
- ck_assert_int_eq (memcmp (check_key_id._data, key_id._data, 32), 0);
- ck_assert_int_eq (memcmp (check_header_id._data, header_id._data, 32), 0);
- ck_assert_int_eq (memcmp (check_body_id._data, body_id._data, 32), 0);
- FREE (check_key_id._data);
- FREE (check_header_id._data);
- FREE (check_body_id._data);
- FREE (key_id._data);
- FREE (header_id._data);
- FREE (body_id._data);
- FREE (entry_id._data);
- cleanup_database (&database);
+ ck_assert_int_eq (check_key._nbytes, strlen (key_value));
+ ck_assert_int_eq (check_header._nbytes, strlen (header_value));
+ ck_assert_int_eq (check_body._nbytes, strlen (body_value));
+ ck_assert_int_eq (memcmp (check_key._data, key_value, strlen (key_value)),
+ 0);
+ ck_assert_int_eq (memcmp
+ (check_header._data, header_value, strlen (header_value)),
+ 0);
+ ck_assert_int_eq (memcmp
+ (check_body._data, body_value, strlen (body_value)), 0);
+ FREE (check_key._data);
+ FREE (check_header._data);
+ FREE (check_body._data);
+ remove (filename);
+ close (file);
}
/* *INDENT-OFF* */
END_TEST
@@ -905,7 +874,6 @@ run_tests (size_t *n_tests, size_t *n_errors)
suite_add_tcase (suite, general);
TCase *cache_entry = tcase_create (_("disfluid cache entry files"));
tcase_add_test (cache_entry, test_read_normal);
- tcase_add_test (cache_entry, test_read_invalid_magic_number);
suite_add_tcase (suite, cache_entry);
TCase *cache_key = tcase_create (_("disfluid cache key"));
tcase_add_test (cache_key, test_cache_key_no_host);