Repository navigation
Bug: lenient WAL replay drops intact commits after a damaged record #1497
Description
This work was produced with the help of language models.
With throw_on_wal_replay_failure=False, the engine's default and Python's from its next release (ladybug-python#77), recovery truncates the WAL at the first record it cannot read, which suits a torn tail but loses every intact commit after damage in the middle.
On 0.22.1, after one byte of commit 2 of 3 is flipped (first script below), every lenient open shows only commit 1, [[1, 100]], while a strict open refuses with Checksum verification failed, the WAL file is corrupted. and keeps the WAL.
A read that fails once (an EIO injected on 0.22.0; 0.22.1 has the same replay code) drops the intact commits after it the same way, and so does a record of a type the build does not know, such as the ABORT record #1380 adds.
A read-only transaction open during a checkpoint keeps its CHECKPOINT record in the WAL. Damage before that record makes recovery drop it and apply earlier commits again: with commits adding k = 1, 2 and 3, the record between the last two, and one byte of commit 2 flipped, a lenient open on 0.22.1 returns k = 1, 1, 2 with no error (second script below).
Expected: lenient replay drops only a torn tail and refuses anything else, keeping the WAL.
Repro (Python, ladybug==0.22.1)
import os
import tempfile
import ladybug as lb
path = os.path.join(tempfile.mkdtemp(), "db")
with lb.Database(path) as db, lb.Connection(db) as conn:
conn.execute("CALL force_checkpoint_on_close=false")
conn.execute("CALL auto_checkpoint=false")
conn.execute("CREATE NODE TABLE T(id INT64 PRIMARY KEY)")
sizes = []
for commit in (1, 2, 3):
conn.execute(f"UNWIND range({commit * 1000}, {commit * 1000 + 99}) AS i CREATE (:T {{id: i}})")
sizes.append(os.path.getsize(path + ".wal"))
offset = (sizes[0] + sizes[1]) // 2
with open(path + ".wal", "r+b") as wal:
wal.seek(offset)
byte = wal.read(1)[0]
wal.seek(offset)
wal.write(bytes([byte ^ 0xFF]))
query = "MATCH (t:T) RETURN t.id / 1000 AS c, count(*) ORDER BY c"
for label, strict in (("strict open", True), ("lenient open", False), ("reopen", False)):
try:
with lb.Database(path, throw_on_wal_replay_failure=strict) as db, lb.Connection(db) as conn:
print(f"{label}: {conn.execute(query).get_all()}")
except RuntimeError as e:
print(f"{label}: {e}")Output:
strict open: Storage exception: Checksum verification failed, the WAL file is corrupted.
lenient open: [[1, 100]]
reopen: [[1, 100]]
Repro with a CHECKPOINT record (Python, ladybug==0.22.1)
import os
import tempfile
import ladybug as lb
path = os.path.join(tempfile.mkdtemp(), "db")
create = "MATCH (a:T {{id: 1}}) CREATE (a)-[:R {{k: {}}}]->(a)"
with lb.Database(path) as db:
conn = lb.Connection(db)
conn.execute("CALL force_checkpoint_on_close=false")
conn.execute("CALL auto_checkpoint=false")
conn.execute("CREATE NODE TABLE T(id INT64 PRIMARY KEY)")
conn.execute("CREATE REL TABLE R(FROM T TO T, k INT64)")
conn.execute("CREATE (:T {id: 1})")
conn.execute("CHECKPOINT")
conn.execute(create.format(1))
commit2_start = os.path.getsize(path + ".wal")
conn.execute(create.format(2))
reader = lb.Connection(db)
reader.execute("BEGIN TRANSACTION READ ONLY")
conn.execute("CHECKPOINT")
conn.execute(create.format(3))
reader.execute("COMMIT")
reader.close()
conn.close()
with open(path + ".wal", "r+b") as wal:
wal.seek(commit2_start + 9)
byte = wal.read(1)[0]
wal.seek(commit2_start + 9)
wal.write(bytes([byte ^ 0xFF]))
query = "MATCH (:T)-[r:R]->(:T) RETURN r.k ORDER BY r.k"
for label, strict in (("strict open", True), ("lenient open", False), ("reopen", False)):
try:
with lb.Database(path, throw_on_wal_replay_failure=strict) as db, lb.Connection(db) as conn:
print(f"{label}: {conn.execute(query).get_all()}")
except RuntimeError as e:
print(f"{label}: {e}")Output:
strict open: Storage exception: Checksum verification failed, the WAL file is corrupted.
lenient open: [[1], [1], [2]]
reopen: [[1], [1], [2]]
Activity