wal_recover() treated every CRC-mismatched frame as a crash-consistent
torn tail: it truncated the file from that frame to EOF unconditionally.
A real crash can only ever tear the very last frame (nothing runs after
a crash to write more good frames past a torn one), but non-crash
corruption (disk bit rot, a stray write) can hit a frame in the middle
of the log while fully valid, already-durable frames still follow it.
In that case the old code silently and permanently destroyed every
committed ledger entry after the corruption point.
wal_recover() now looks ahead past a CRC-mismatched frame: if the rest
of the file parses as a complete, valid frame chain, this cannot be a
crash-torn tail, so it returns WAL_ERR_CORRUPT_MID_LOG with the corrupt
frame's offset (wal_recovery_stats_t.corrupt_frame_offset) and leaves
the file completely untouched instead of truncating. Genuine
crash-torn tails (corruption with nothing valid after it) still
truncate exactly as before.
Threaded through services/ledger-core/src/wal_ffi.rs as
WalError::CorruptMidLog{offset}, which Ledger::open already propagates
as a hard boot failure via the existing WalError->LedgerError mapping
— ledger-core now refuses to boot on this condition rather than
silently losing data.
Tested at both layers: services/wal/tests/test_wal.c (C, via
wal_recover() directly) and services/ledger-core/src/wal_ffi.rs +
services/ledger-core/tests/recovery.rs (Rust, through the safe wrapper
and through Ledger::open end-to-end), each with a sibling test proving
the existing single-frame-at-the-true-tail case still truncates as
before.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
132 lines
6.6 KiB
C
132 lines
6.6 KiB
C
#ifndef RATION_WAL_H
|
|
#define RATION_WAL_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
/*
|
|
* Crash-safe write-ahead log.
|
|
*
|
|
* Format: an append-only sequence of length-prefixed, CRC32-checksummed
|
|
* frames (see frame.h). Every wal_append() fsync()s the file before
|
|
* returning, so a successful append is durable on disk even if the
|
|
* process is killed immediately afterward.
|
|
*
|
|
* Crash recovery: a process can die mid-append, leaving a "torn" frame at
|
|
* the tail of the file (a partially written header and/or payload, or a
|
|
* fully-written-but-not-fsynced frame with a bad CRC). wal_recover()
|
|
* detects this by CRC/length validation rather than treating it as a
|
|
* fatal error: every frame up to the first invalid one is replayed via
|
|
* the callback, and the torn tail is truncated off the file so the log
|
|
* is left clean and ready for further appends.
|
|
*/
|
|
|
|
typedef struct wal wal_t;
|
|
|
|
typedef enum {
|
|
WAL_OK = 0,
|
|
WAL_ERR_IO = -1, /* open/read/write/fsync/truncate failed */
|
|
WAL_ERR_INVALID = -2, /* bad arguments, or recovery callback aborted */
|
|
WAL_ERR_CORRUPT = -3, /* frame failed structural/CRC validation */
|
|
WAL_ERR_EOF = -4, /* wal_read_next: no more frames to read */
|
|
WAL_ERR_TOO_LARGE = -5, /* payload exceeds WAL_MAX_PAYLOAD_SIZE */
|
|
WAL_ERR_NOMEM = -6, /* allocation failure */
|
|
WAL_ERR_BUFFER_SMALL = -7, /* wal_read_next: caller buffer too small */
|
|
WAL_ERR_CORRUPT_MID_LOG = -8, /* wal_recover: a frame failed CRC validation
|
|
* but one or more fully-valid frames were
|
|
* found after it - this is NOT a
|
|
* crash-consistent torn tail (a crash can
|
|
* never leave good frames after a torn
|
|
* one), so the file is left completely
|
|
* untouched (no truncation) rather than
|
|
* risking silent loss of durable data.
|
|
* See wal_recovery_stats_t.corrupt_frame_offset
|
|
* for where to start manual investigation. */
|
|
} wal_status_t;
|
|
|
|
typedef struct {
|
|
size_t frames_recovered; /* valid frames replayed, in append order */
|
|
size_t bytes_recovered; /* sum of payload bytes across those frames */
|
|
int torn_tail_dropped; /* 1 if a trailing invalid frame was discarded */
|
|
size_t torn_bytes_dropped; /* bytes truncated off the file for the torn tail */
|
|
size_t corrupt_frame_offset; /* valid only when wal_recover() returns
|
|
* WAL_ERR_CORRUPT_MID_LOG: the byte offset
|
|
* of the first frame that failed CRC
|
|
* validation despite valid frames existing
|
|
* after it. Zero and meaningless otherwise. */
|
|
} wal_recovery_stats_t;
|
|
|
|
/* Invoked once per valid frame during wal_recover(), in append order.
|
|
* `payload` is valid only for the duration of the call. Return nonzero to
|
|
* abort recovery early (wal_recover then returns WAL_ERR_INVALID; frames
|
|
* already delivered to the callback are unaffected, but nothing after
|
|
* the aborted frame is truncated or replayed). */
|
|
typedef int (*wal_recover_cb)(const uint8_t *payload, uint32_t len, void *ctx);
|
|
|
|
/* Open (creating if necessary) the WAL file at `path` for append/read.
|
|
* The write cursor is positioned at the current end of file - callers
|
|
* should run wal_recover() on `path` first so that end-of-file is a
|
|
* clean frame boundary, not a torn tail. Returns NULL on failure and, if
|
|
* `out_status` is non-NULL, stores the reason. */
|
|
wal_t *wal_open(const char *path, wal_status_t *out_status);
|
|
|
|
/* Append one frame containing `payload[0..len)`. Computes the CRC,
|
|
* writes the header+payload, and fsync()s before returning WAL_OK. On
|
|
* any failure the file is left unchanged from the caller's point of view
|
|
* (either nothing was written, or what was written will be cleaned up by
|
|
* a future wal_recover() as a torn tail). */
|
|
wal_status_t wal_append(wal_t *w, const uint8_t *payload, uint32_t len);
|
|
|
|
/* Read the next frame sequentially (independent cursor from the append
|
|
* position) into `buf` (capacity `buf_cap`). On WAL_OK, `*out_len` is set
|
|
* to the payload length actually written to `buf`. Returns WAL_ERR_EOF
|
|
* once all appended frames have been read. Returns WAL_ERR_BUFFER_SMALL
|
|
* (without advancing the cursor) if `buf_cap` is smaller than the next
|
|
* frame's payload - retry with a larger buffer. */
|
|
wal_status_t wal_read_next(wal_t *w, uint8_t *buf, uint32_t buf_cap, uint32_t *out_len);
|
|
|
|
/* Force any buffered data to stable storage. wal_append() already does
|
|
* this internally after every write, so this is mainly useful for tests
|
|
* and callers that want an explicit durability checkpoint. */
|
|
wal_status_t wal_sync(wal_t *w);
|
|
|
|
/* Close the handle and release associated resources. No-op if `w` is NULL. */
|
|
void wal_close(wal_t *w);
|
|
|
|
/* Scan the WAL file at `path` from the beginning, validating and
|
|
* replaying every well-formed frame via `cb`. If the file ends with a
|
|
* torn frame (incomplete header, incomplete payload, an implausible
|
|
* length field, or a CRC-mismatched frame with nothing valid after it),
|
|
* that frame and anything after it is discarded: the file is truncated
|
|
* back to the end of the last valid frame so it is immediately safe to
|
|
* wal_open() and keep appending to.
|
|
*
|
|
* A CRC-mismatched frame that has one or more fully-valid frames AFTER
|
|
* it is handled differently and deliberately NOT truncated: a real
|
|
* crash can only ever tear the very last frame (nothing runs to write
|
|
* more good frames past a torn one), so a bad-CRC frame with valid data
|
|
* after it can only be non-crash corruption (disk bit rot, a stray
|
|
* write). Discarding it the same way as a torn tail would silently and
|
|
* permanently destroy already-durable, previously-fsynced frames.
|
|
* Instead this returns WAL_ERR_CORRUPT_MID_LOG, leaves the file
|
|
* byte-for-byte untouched, and reports the corrupt frame's byte offset
|
|
* via `stats->corrupt_frame_offset` so an operator can investigate
|
|
* rather than losing the data silently.
|
|
*
|
|
* `stats`, if non-NULL, is filled in with what was recovered (or, on
|
|
* WAL_ERR_CORRUPT_MID_LOG, with what was recovered up to the corrupt
|
|
* frame plus its offset). Intended to be called once at startup, before
|
|
* wal_open(). */
|
|
wal_status_t wal_recover(const char *path, wal_recover_cb cb, void *ctx,
|
|
wal_recovery_stats_t *stats);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif /* RATION_WAL_H */
|