summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/journal-remote/journal-remote-write.c2
-rw-r--r--src/journal-remote/journal-remote.c4
-rw-r--r--src/journal/journal-file.c30
-rw-r--r--src/journal/journal-file.h5
-rw-r--r--src/journal/journald-server.c6
-rw-r--r--src/journal/sd-journal.c2
-rw-r--r--src/journal/test-journal-flush.c2
-rw-r--r--src/journal/test-journal-interleaving.c8
-rw-r--r--src/journal/test-journal-stream.c6
-rw-r--r--src/journal/test-journal-verify.c6
-rw-r--r--src/journal/test-journal.c102
11 files changed, 141 insertions, 32 deletions
diff --git a/src/journal-remote/journal-remote-write.c b/src/journal-remote/journal-remote-write.c
index c5c6abbd5c..2836ba8b48 100644
--- a/src/journal-remote/journal-remote-write.c
+++ b/src/journal-remote/journal-remote-write.c
@@ -22,7 +22,7 @@
#include "journal-remote.h"
static int do_rotate(JournalFile **f, bool compress, bool seal) {
- int r = journal_file_rotate(f, compress, seal, NULL);
+ int r = journal_file_rotate(f, compress, (uint64_t) -1, seal, NULL);
if (r < 0) {
if (*f)
log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);
diff --git a/src/journal-remote/journal-remote.c b/src/journal-remote/journal-remote.c
index 428725223d..ff46023920 100644
--- a/src/journal-remote/journal-remote.c
+++ b/src/journal-remote/journal-remote.c
@@ -185,7 +185,7 @@ static int open_output(Writer *w, const char* host) {
r = journal_file_open_reliably(output,
O_RDWR|O_CREAT, 0640,
- arg_compress, arg_seal,
+ arg_compress, (uint64_t) -1, arg_seal,
&w->metrics,
w->mmap, NULL,
NULL, &w->journal);
@@ -729,7 +729,7 @@ static int setup_microhttpd_server(RemoteServer *s,
}
r = sd_event_add_time(s->events, &d->timer_event,
- CLOCK_MONOTONIC, UINT64_MAX, 0,
+ CLOCK_MONOTONIC, (uint64_t) -1, 0,
null_timer_event_handler, d);
if (r < 0) {
log_error_errno(r, "Failed to add timer_event: %m");
diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c
index 5643c0578d..5fc61b16e0 100644
--- a/src/journal/journal-file.c
+++ b/src/journal/journal-file.c
@@ -51,7 +51,8 @@
#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem))
#define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem))
-#define COMPRESSION_SIZE_THRESHOLD (512ULL)
+#define DEFAULT_COMPRESS_THRESHOLD (512ULL)
+#define MIN_COMPRESS_THRESHOLD (8ULL)
/* This is the minimum journal file size */
#define JOURNAL_FILE_SIZE_MIN (512ULL*1024ULL) /* 512 KiB */
@@ -1552,7 +1553,7 @@ static int journal_file_append_data(
o->data.hash = htole64(hash);
#if HAVE_XZ || HAVE_LZ4
- if (JOURNAL_FILE_COMPRESS(f) && size >= COMPRESSION_SIZE_THRESHOLD) {
+ if (JOURNAL_FILE_COMPRESS(f) && size >= f->compress_threshold_bytes) {
size_t rsize = 0;
compression = compress_blob(data, size, o->data.payload, size - 1, &rsize);
@@ -3211,6 +3212,7 @@ int journal_file_open(
int flags,
mode_t mode,
bool compress,
+ uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
@@ -3222,6 +3224,7 @@ int journal_file_open(
JournalFile *f;
void *h;
int r;
+ char bytes[FORMAT_BYTES_MAX];
assert(ret);
assert(fd >= 0 || fname);
@@ -3247,10 +3250,20 @@ int journal_file_open(
#elif HAVE_XZ
f->compress_xz = compress;
#endif
+
+ if (compress_threshold_bytes == (uint64_t) -1)
+ f->compress_threshold_bytes = DEFAULT_COMPRESS_THRESHOLD;
+ else
+ f->compress_threshold_bytes = MAX(MIN_COMPRESS_THRESHOLD, compress_threshold_bytes);
+
#if HAVE_GCRYPT
f->seal = seal;
#endif
+ log_debug("Journal effective settings seal=%s compress=%s compress_threshold_bytes=%s",
+ yes_no(f->seal), yes_no(JOURNAL_FILE_COMPRESS(f)),
+ format_bytes(bytes, sizeof(bytes), f->compress_threshold_bytes));
+
if (mmap_cache)
f->mmap = mmap_cache_ref(mmap_cache);
else {
@@ -3435,7 +3448,7 @@ fail:
return r;
}
-int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes) {
+int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes) {
_cleanup_free_ char *p = NULL;
size_t l;
JournalFile *old_file, *new_file = NULL;
@@ -3489,7 +3502,9 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred
* we archive them */
old_file->defrag_on_close = true;
- r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress, seal, NULL, old_file->mmap, deferred_closes, old_file, &new_file);
+ r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress,
+ compress_threshold_bytes, seal, NULL, old_file->mmap, deferred_closes,
+ old_file, &new_file);
if (deferred_closes &&
set_put(deferred_closes, old_file) >= 0)
@@ -3506,6 +3521,7 @@ int journal_file_open_reliably(
int flags,
mode_t mode,
bool compress,
+ uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
@@ -3517,7 +3533,8 @@ int journal_file_open_reliably(
size_t l;
_cleanup_free_ char *p = NULL;
- r = journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
+ r = journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache,
+ deferred_closes, template, ret);
if (!IN_SET(r,
-EBADMSG, /* Corrupted */
-ENODATA, /* Truncated */
@@ -3559,7 +3576,8 @@ int journal_file_open_reliably(
log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname);
- return journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
+ return journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache,
+ deferred_closes, template, ret);
}
int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) {
diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h
index 67abf8da49..04d35f5813 100644
--- a/src/journal/journal-file.h
+++ b/src/journal/journal-file.h
@@ -124,6 +124,7 @@ typedef struct JournalFile {
unsigned last_seen_generation;
+ uint64_t compress_threshold_bytes;
#if HAVE_XZ || HAVE_LZ4
void *compress_buffer;
size_t compress_buffer_size;
@@ -153,6 +154,7 @@ int journal_file_open(
int flags,
mode_t mode,
bool compress,
+ uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
@@ -169,6 +171,7 @@ int journal_file_open_reliably(
int flags,
mode_t mode,
bool compress,
+ uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
@@ -246,7 +249,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
void journal_file_dump(JournalFile *f);
void journal_file_print_header(JournalFile *f);
-int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes);
+int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes);
void journal_file_post_change(JournalFile *f);
int journal_file_enable_post_change_timer(JournalFile *f, sd_event *e, usec_t t);
diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c
index 5cd58e8a77..570450d321 100644
--- a/src/journal/journald-server.c
+++ b/src/journal/journald-server.c
@@ -280,9 +280,9 @@ static int open_journal(
assert(ret);
if (reliably)
- r = journal_file_open_reliably(fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
+ r = journal_file_open_reliably(fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
else
- r = journal_file_open(-1, fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
+ r = journal_file_open(-1, fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
if (r < 0)
return r;
@@ -463,7 +463,7 @@ static int do_rotate(
if (!*f)
return -EINVAL;
- r = journal_file_rotate(f, s->compress, seal, s->deferred_closes);
+ r = journal_file_rotate(f, s->compress, (uint64_t) -1, seal, s->deferred_closes);
if (r < 0) {
if (*f)
return log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 11dbd83f2d..79ffd9cb3f 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -1337,7 +1337,7 @@ static int add_any_file(
goto finish;
}
- r = journal_file_open(fd, path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, NULL, &f);
+ r = journal_file_open(fd, path, O_RDONLY, 0, false, 0, false, NULL, j->mmap, NULL, NULL, &f);
if (r < 0) {
log_debug_errno(r, "Failed to open journal file %s: %m", path);
goto finish;
diff --git a/src/journal/test-journal-flush.c b/src/journal/test-journal-flush.c
index f9b7b75ef1..e302fd61f1 100644
--- a/src/journal/test-journal-flush.c
+++ b/src/journal/test-journal-flush.c
@@ -39,7 +39,7 @@ int main(int argc, char *argv[]) {
assert_se(mkdtemp(dn));
fn = strappend(dn, "/test.journal");
- r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, NULL, &new_journal);
+ r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal);
assert_se(r >= 0);
r = sd_journal_open(&j, 0);
diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c
index d87bdbdd32..f8b8d1e6d8 100644
--- a/src/journal/test-journal-interleaving.c
+++ b/src/journal/test-journal-interleaving.c
@@ -52,7 +52,7 @@ _noreturn_ static void log_assert_errno(const char *text, int error, const char
static JournalFile *test_open(const char *name) {
JournalFile *f;
- assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, NULL, &f));
+ assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f));
return f;
}
@@ -217,7 +217,7 @@ static void test_sequence_numbers(void) {
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644,
- true, false, NULL, NULL, NULL, NULL, &one) == 0);
+ true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
append_number(one, 1, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
@@ -234,7 +234,7 @@ static void test_sequence_numbers(void) {
memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t));
assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644,
- true, false, NULL, NULL, NULL, one, &two) == 0);
+ true, (uint64_t) -1, false, NULL, NULL, NULL, one, &two) == 0);
assert_se(two->header->state == STATE_ONLINE);
assert_se(!sd_id128_equal(two->header->file_id, one->header->file_id));
@@ -265,7 +265,7 @@ static void test_sequence_numbers(void) {
seqnum = 0;
assert_se(journal_file_open(-1, "two.journal", O_RDWR, 0,
- true, false, NULL, NULL, NULL, NULL, &two) == 0);
+ true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id));
diff --git a/src/journal/test-journal-stream.c b/src/journal/test-journal-stream.c
index 73ed6e5dcb..279301c67d 100644
--- a/src/journal/test-journal-stream.c
+++ b/src/journal/test-journal-stream.c
@@ -93,9 +93,9 @@ int main(int argc, char *argv[]) {
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
- assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &one) == 0);
- assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &two) == 0);
- assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &three) == 0);
+ assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
+ assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
+ assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &three) == 0);
for (i = 0; i < N_ENTRIES; i++) {
char *p, *q;
diff --git a/src/journal/test-journal-verify.c b/src/journal/test-journal-verify.c
index fbb75e43e3..89389579f3 100644
--- a/src/journal/test-journal-verify.c
+++ b/src/journal/test-journal-verify.c
@@ -56,7 +56,7 @@ static int raw_verify(const char *fn, const char *verification_key) {
JournalFile *f;
int r;
- r = journal_file_open(-1, fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f);
+ r = journal_file_open(-1, fn, O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f);
if (r < 0)
return r;
@@ -89,7 +89,7 @@ int main(int argc, char *argv[]) {
log_info("Generating...");
- assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+ assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
for (n = 0; n < N_ENTRIES; n++) {
struct iovec iovec;
@@ -112,7 +112,7 @@ int main(int argc, char *argv[]) {
log_info("Verifying...");
- assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+ assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
/* journal_file_print_header(f); */
journal_file_dump(f);
diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c
index 517c9102a6..3b9917b4a8 100644
--- a/src/journal/test-journal.c
+++ b/src/journal/test-journal.c
@@ -43,7 +43,7 @@ static void test_non_empty(void) {
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
- assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f) == 0);
+ assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f) == 0);
dual_timestamp_get(&ts);
@@ -105,8 +105,8 @@ static void test_non_empty(void) {
assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0);
- journal_file_rotate(&f, true, true, NULL);
- journal_file_rotate(&f, true, true, NULL);
+ journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
+ journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
(void) journal_file_close(f);
@@ -132,13 +132,13 @@ static void test_empty(void) {
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
- assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, NULL, &f1) == 0);
+ assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f1) == 0);
- assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &f2) == 0);
+ assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f2) == 0);
- assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, NULL, &f3) == 0);
+ assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f3) == 0);
- assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f4) == 0);
+ assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f4) == 0);
journal_file_print_header(f1);
puts("");
@@ -165,6 +165,91 @@ static void test_empty(void) {
(void) journal_file_close(f4);
}
+#if HAVE_XZ || HAVE_LZ4
+static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
+ dual_timestamp ts;
+ JournalFile *f;
+ struct iovec iovec;
+ Object *o;
+ uint64_t p;
+ char t[] = "/tmp/journal-XXXXXX";
+ char data[2048] = {0};
+ bool is_compressed;
+ int r;
+
+ assert_se(data_size <= sizeof(data));
+
+ log_set_max_level(LOG_DEBUG);
+
+ assert_se(mkdtemp(t));
+ assert_se(chdir(t) >= 0);
+
+ assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0);
+
+ dual_timestamp_get(&ts);
+
+ iovec.iov_base = (void*) data;
+ iovec.iov_len = data_size;
+ assert_se(journal_file_append_entry(f, &ts, &iovec, 1, NULL, NULL, NULL) == 0);
+
+#if HAVE_GCRYPT
+ journal_file_append_tag(f);
+#endif
+ journal_file_dump(f);
+
+ /* We have to partially reimplement some of the dump logic, because the normal next_entry does the
+ * decompression for us. */
+ p = le64toh(f->header->header_size);
+ while (true) {
+ r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o);
+ assert_se(r == 0);
+ if (o->object.type == OBJECT_DATA)
+ break;
+
+ assert_se(p < le64toh(f->header->tail_object_offset));
+ p = p + ALIGN64(le64toh(o->object.size));
+ }
+
+ is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0;
+
+ (void) journal_file_close(f);
+
+ log_info("Done...");
+
+ if (arg_keep)
+ log_info("Not removing %s", t);
+ else {
+ journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
+
+ assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+ }
+
+ puts("------------------------------------------------------------");
+
+ return is_compressed;
+}
+
+static void test_min_compress_size(void) {
+ /* Note that XZ will actually fail to compress anything under 80 bytes, so you have to choose the limits
+ * carefully */
+
+ /* DEFAULT_MIN_COMPRESS_SIZE is 512 */
+ assert_se(!check_compressed((uint64_t) -1, 255));
+ assert_se(check_compressed((uint64_t) -1, 513));
+
+ /* compress everything */
+ assert_se(check_compressed(0, 96));
+ assert_se(check_compressed(8, 96));
+
+ /* Ensure we don't try to compress less than 8 bytes */
+ assert_se(!check_compressed(0, 7));
+
+ /* check boundary conditions */
+ assert_se(check_compressed(256, 256));
+ assert_se(!check_compressed(256, 255));
+}
+#endif
+
int main(int argc, char *argv[]) {
arg_keep = argc > 1;
@@ -174,6 +259,9 @@ int main(int argc, char *argv[]) {
test_non_empty();
test_empty();
+#if HAVE_XZ || HAVE_LZ4
+ test_min_compress_size();
+#endif
return 0;
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-11 15:01:49 +0100