diff --git a/di/tplogutils/init.q b/di/tplogutils/init.q new file mode 100644 index 00000000..3220f689 --- /dev/null +++ b/di/tplogutils/init.q @@ -0,0 +1,72 @@ +/ header to build deserialisable msg +header:8#-8!(`upd;`trade;()); +/ first part of tp update msg +updmsg:`char$10#8_-8!(`upd;`trade;()); +/ size of default chunk to read (10MB) +chunk:10*1024*1024; +/ don't let single read exceed this +maxchunk:8*chunk; + +check:{[logfile;lastmsgtoreplay] + / logfile (symbol) is the handle to the logsfile + / lastmsgtoreplay (long) is index position of the last message to be replayed from the log + / check if the logfile is corrupt + loginfo:-11!(-2;logfile); + :$[1 = count loginfo; + / - the log file is good so return the good log file handle + :logfile; + loginfo[0] <= lastmsgtoreplay + 1; + :logfile; + repair[logfile] + ] + }; + +repair:{[logfile] + / - append ".good" to the "good" log file + goodlog: `$ string[logfile],".good"; + / - create file and open handle to it + goodlogh: hopen goodlog set (); + / - loop through the file in chunks + repairover[logfile;goodlogh] over `start`size!(0j;chunk); + / - return goodlog + goodlog + }; + +repairover:{[logfile;goodlogh;d] + / logfile (symbol) is the handle to the logsfile + / goodlogh (int) is the handle to the "good" log file + / d (dictionary) has two keys start and size, the point to start reading from and size of chunk to read + / read bytes from + x:read1 logfile,d`start`size; + / find the start points of upd messages + u: ss[`char$x;updmsg]; + / nothing in this block + if[not count u; + / EOF - we're done + if[hcount[logfile] <= sum d`start`size;:d]; + / move on bytes + :@[d;`start;+;d`size]]; + / split bytes into msgs + m: u _ x; + / message sizes as bytes + mz: 0x0 vs' `int$ 8 + ms: count each m; + / set msg size at correct part of hdr + hd: @[header;7 6 5 4;:;] each mz; + / try and deserialize each msg + g: @[(1b;)@-9!;;(0b;)@] each hd,'m; + / write good msgs to the "good" log + goodlogh g[;1] where k:g[;0]; + / saw msg(s) but couldn't read + if[not any k; + / read as much as we dare, give up + if[maxchunk <= d`size; + :@[d;`start`size;:;(sum d`start`size;chunk)]]; + / read a bigger chunk + :@[d;`size;*;2]]; + / move to the end of the last good msg + ns: d[`start] + sums[ms] last where k; + :@[d;`start`size;:;(ns;chunk)]; + }; + +export:([check;repair]) + diff --git a/di/tplogutils/test.csv b/di/tplogutils/test.csv new file mode 100644 index 00000000..c0b8cb2c --- /dev/null +++ b/di/tplogutils/test.csv @@ -0,0 +1,21 @@ +action,ms,bytes,lang,code,repeat,minver,comment +before,0,0,q,tplogutils:use`di.tplogutils,1,,Initialize module +before,0,0,q,os:use`di.os,1,,Initialize module +/ note location of the test.q file might be different, built under assumption using di module file system +before,0,0,q,"system ""l "", os.abspath[""di/tplogutils/test.q""]",1,1,load additional testing functions / dependencies +run,0,0,q,testrepairandreplay[],1,1, +run,0,0,q,testrepairrecoversmessages[],1,1, +run,0,0,q,testrepaircreatesgoodfile[],1,1, +run,0,0,q,testcheckvalidlog[],1,1, +run,0,0,q,testcheckcorruptsufficientmessages[],1,1, +run,0,0,q,testrepaircreatesgoodfile[],1,1, +run,0,0,q,testrepairrecoversmessages[],1,1, +run,0,0,q,testchecktriggersrepair[],1,1, +run,0,0,q,testrepairgarbageatend[],1,1, +run,0,0,q,testmultiplecorruptsections[],1,1, +run,0,0,q,testcompletelycorruptlog[],1,1, +run,0,0,q,testemptylog[],1,1, +run,0,0,q,testrepairandreplay[],1,1, +run,0,0,q,testlargefilehandling[],1,1, +run,0,0,q,testrepaircreatesgoodfile[],1,1, +run,0,0,q,testsequentialoperations[],1,1, diff --git a/di/tplogutils/test.q b/di/tplogutils/test.q new file mode 100644 index 00000000..5c579cc3 --- /dev/null +++ b/di/tplogutils/test.q @@ -0,0 +1,360 @@ +/ ============================================================================= +/ TEST HELPERS +/ ============================================================================= + +upd:{[t;x] t upsert x}; +trade:([] time:`timestamp$(); sym:`symbol$(); price:`float$(); size:`long$()); + +/ @function createvalidlog +/ @description Create a valid tickerplant log file for testing +/ @param filepath {symbol} Path where to create the log file +/ @param msgcount {long} Number of messages to write +createvalidlog:{[filepath;msgcount] + / create test table + trade:([] time:.z.p + til msgcount; sym:msgcount?`AAPL`GOOGL`MSFT`AMZN`TSLA; price:100+msgcount?100.0; size:100+msgcount?1000); + / create log file and write messages + h:hopen filepath set (); + {[h;i;t] h enlist (`upd;`trade;value t[i])} [h;;trade] each til msgcount; + hclose h; + }; + +/ @function createcorruptlog +/ @description Create a log file with valid messages followed by corruption +/ @param filepath {symbol} Path where to create the log file +/ @param msgcount {long} Number of messages in log file +/ @param corruptpos {long} Message position where to insert corruption +createcorruptlog:{[filepath;msgcount;corruptpos] + / create test table + trade:([] time:.z.p + til msgcount; sym:msgcount?`AAPL`GOOGL`MSFT`AMZN`TSLA; price:100+msgcount?100.0; size:100+msgcount?1000); + / create log file and write messages + h:hopen filepath set (); + {[h;i;t;corruptpos] + if[=[i;corruptpos]; + data:enlist (`upd;`trade;value t[i]); + databytes:-18!data; + data_bytes[10+til 20]:`byte$(20?50); + :h data_bytes; + ] + h enlist (`upd;`trade;value t[i]) + } [h;;trade;corruptpos] each til msgcount; + hclose h; + }; + +/ @function countLogMessages +/ @description Count number of messages in a log file +/ @param filepath {symbol} Path to log file +/ @returns {long} Number of messages in the log +countlogmessages:{[filepath] + count -11!(1;filepath) + }; + +/ @function cleanup +/ @description Delete test files +/ @param filepaths {symbol[]} List of file paths to delete +cleanup:{[filepaths] + {[fp] @[hdel;fp;{}]} each filepaths; + }; + +/ ============================================================================= +/ BASIC FUNCTIONALITY TESTS +/ ============================================================================= + +/ @test Valid log file tplogutils.check returns original filepath +testcheckvalidlog: { + testfile:`:test_valid.log; + msgcount:10; + + / setup + createvalidlog[testfile;msgcount]; + + / test + result:tplogutils.check[testfile;msgcount-1]; + + / assert + passes:result~testfile; + + / cleanup + cleanup enlist testfile; + + / Return + passes + }; + +/ @test tplogutils.check returns original when enough good messages exist +testcheckcorruptsufficientmessages:{ + testfile:`:test_corrupt_sufficient.log; + validmsgcount:20; + lastmsgtoreplay:10j; + + / setup: corrupt after position where we have enough good messages + createcorruptlog[testfile;validmsgcount;500]; + + / test + result:tplogutils.check[testfile;lastmsgtoreplay]; + + / assert - should return original since we have enough good messages + goodmsgcount:first -11!(-2;testfile); + passes:(result~testfile) and (goodmsgcount > lastmsgtoreplay); + + / cleanup + cleanup enlist testfile; + + passes + }; + +/ @test tplogutils.repair creates .good file with correct name +testrepaircreatesgoodfile: { + testfile:`:test_tplogutils.repair.log; + expectedgoodfile:`$string[testfile],".good"; + + / setup + createcorruptlog[testfile;15;150]; + + / test + result:tplogutils.repair[testfile]; + + / assert + namecorrect:result~expectedgoodfile; + fileexists:not ()~key expectedgoodfile; + passes:namecorrect and fileexists; + + / cleanup + cleanup (testfile;expectedgoodfile); + + passes + }; + +/ @test tplogutils.repair recovers valid messages from corrupt log +testrepairrecoversmessages: { + testfile:`:test_recover.log; + goodfile:`$string[testfile],".good"; + validmsgcount:20; + + / setup + createcorruptlog[testfile;validmsgcount;250]; + + / test + tplogutils.repair[testfile]; + + / Count messages in good file + recoveredcount:countlogmessages[goodfile]; + + / assert - should recover at least some messages + passes:(recoveredcount>0) and (recoveredcount<=validmsgcount); + + / cleanup + cleanup (testfile;goodfile); + + passes + }; + +/ @test tplogutils.check triggers tplogutils.repair when insufficient good messages +testchecktriggersrepair: { + testfile:`:test_tplogutils.check_tplogutils.repair.log; + goodfile:`$string[testfile],".good"; + validmsgcount:10; + lastmsgtoreplay:15j; / Need more messages than available good ones + + / setup - corrupt early so not enough good messages + createcorruptlog[testfile;validmsgcount;100]; + + / test + result:tplogutils.check[testfile;lastmsgtoreplay]; + + / assert + triggerstplogutils.repair:result~goodfile; + filecreated:not ()~key goodfile; + passes:triggerstplogutils.repair and filecreated; + + / cleanup + cleanup (testfile;goodfile); + + passes + }; + +/ ============================================================================= +/ EDGE CASE TESTS +/ ============================================================================= + +/ @test tplogutils.repair handles garbage at end of file +testrepairgarbageatend: { + testfile:`:test_garbage_end.log; + goodfile:`$string[testfile],".good"; + + / setup - create log and append garbage at end + createvalidlog[testfile;10]; + bytes:read1 testfile; + testfile set bytes,100#0x00; + + / test + result:tplogutils.repair[testfile]; + + / assert + namecorrect:result~goodfile; + hasMessages:countlogmessages[goodfile]>0; + passes:namecorrect and hasMessages; + + / cleanup + cleanup (testfile;goodfile); + + passes + }; + +/ @test Handles multiple corruption points +testmultiplecorruptsections: { + testfile:`:test_multi_corrupt.log; + goodfile:`$string[testfile],".good"; + + / setup - create log with corruption in middle + createvalidlog[testfile;30]; + bytes:read1 testfile; + + / insert corruption at position (should have valid messages before and after) + if[200 < count bytes; + corrupted:bytes[til 200],10#0xFF,bytes[210+til count[bytes]-210]; + testfile set corrupted; + ]; + + / test + result:tplogutils.repair[testfile]; + + / assert - should create file and recover something + fileCorrect:result~goodfile; + fileExists:not ()~key goodfile; + passes:fileCorrect and fileExists; + + / cleanup + cleanup (testfile;goodfile); + + passes + }; + +/ @test Completely corrupt log creates empty .good file +testcompletelycorruptlog: { + testfile:`:test_all_corrupt.log; + goodfile:`$string[testfile],".good"; + + / setup - create completely corrupt file + testfile set 1000#0x00; + + / test + result:tplogutils.repair[testfile]; + + / assert - should create .good file even if empty/minimal + namecorrect:result~goodfile; + fileExists:not ()~key goodfile; + passes:namecorrect and fileExists; + + / cleanup + cleanup (testfile;goodfile); + + passes + }; + +/ @test Empty log file handling +testemptylog: { + testfile:`:test_empty.log; + + / setup - create empty file + testfile set 0#0x00; + + / test - should not crash + result:tplogutils.check[testfile;0j]; + + / If we got here without error, test passes + passes:1b; + + / cleanup + cleanup enlist testfile; + + passes + }; + +/ ============================================================================= +/ CONFIGURATION TESTS +/ ============================================================================= + +/ @test Module metadata is present +testmoduleinfo: { + hasname:`name in key info; + hasversion:`version in key info; + hasdesc:`description in key info; + + hasname and hasversion and hasdesc + }; + +/ ============================================================================= +/ INTEGRATION TESTS +/ ============================================================================= + +/ @test tplogutils.repair then replay workflow +testrepairandreplay: { + testfile:`:test_replay.log; + goodfile:`$string[testfile],".good"; + + / setup + createcorruptlog[testfile;20;200]; + + / test - tplogutils.repair and try to replay + tplogutils.repair[testfile]; + + / This should not throw an error if the .good file is valid + replayOk:@[{-11!(1;x);1b};goodfile;{0b}]; + + / cleanup + cleanup (testfile;goodfile); + + replayOk + }; + +/ @test Large file handling (performance test) +testlargefilehandling: { + testfile:`:test_large.log; + goodfile:`$string[testfile],".good"; + msgcount:500; / Reasonable size for testing + + / setup + createcorruptlog[testfile;msgcount;5000]; + + / test - measure time + start:.z.p; + result:tplogutils.repair[testfile]; + elapsed:`second$.z.p-start; + + / assert - should complete and create file + completed:result~goodfile; + reasonable:elapsed<30; / Should complete in under 30 seconds + passes:completed and reasonable; + + / cleanup + cleanup (testfile;goodfile); + + passes + }; + +/ @test Sequential tplogutils.check and tplogutils.repair calls +testsequentialoperations: { + testfile:`:test_sequential.log; + goodfile:`$string[testfile],".good"; + + / setup + createcorruptlog[testfile;15;150]; + + / test - tplogutils.check then tplogutils.repair + tplogutils.checkresult:tplogutils.check[testfile;20j]; + + / if tplogutils.check triggered tplogutils.repair, goodfile should exist + / if not, manually tplogutils.repair + if[not tplogutils.checkresult~goodfile; + tplogutils.repair[testfile]; + ]; + + / assert - .good file should exist in either case + passes:not ()~key goodfile; + + / cleanup + cleanup (testfile;goodfile); + + passes + }; + diff --git a/di/tplogutils/tplogutils.md b/di/tplogutils/tplogutils.md new file mode 100644 index 00000000..dad45812 --- /dev/null +++ b/di/tplogutils/tplogutils.md @@ -0,0 +1,210 @@ +# `tplogutils` – Tickerplant Log Check & Repair Utilities for kdb+/q + +A small utility module for **checking** and **best‑effort repairing** tickerplant-style log files by scanning raw bytes for update-message boundaries, attempting to deserialize candidate messages, and writing any recoverable messages into a new `*.good` logfile. + +> **Note:** As currently implemented, recovery is keyed off the signature of `(`upd;`trade;...)` (see **Configuration**). If your logs contain other tables or message shapes, you may need to adapt the signature constants. + +--- + +## :sparkles: Features + +- Check whether a logfile should be used as-is or repaired (based on the logic in `check`). +- Repair a corrupt logfile by extracting messages that can be successfully deserialized. +- Chunked scanning to avoid loading large files into memory. +- Adaptive read sizing when no valid messages are found in a chunk. +- Produces a new `.good` output file (append-only write during recovery). +- Includes a test suite (`test.q`, `test.csv`) that generates valid/corrupt logs and validates recovery outcomes. + +--- + +## :file_folder: Directory contents + +- `init.q` – module implementation (constants + `check`, `repair`) +- `tplogutils.md` – documentation (you can replace/rename to `README.md` if desired) +- `test.q` – tests + helpers for creating valid/corrupted logs +- `test.csv` – test manifest for your project’s test harness + +--- + +## :inbox_tray: Loading + +### KDB-X (supports `use`) +If you are using KDB-X (where `use` exists), load the module using the symbol that matches your `QPATH` layout. + +If your `QPATH` includes the `di` directory (e.g. `~/kdbx-modules/di`), a common pattern is: + +```q +tplogutils:use`tplogutils +``` + +--- + +## :gear: Configuration + +These constants are defined at the top of `init.q`: + +| Name | Type | Description | +|------------|-------------|-------------| +| `HEADER` | byte list | Template bytes used to build a deserialisable message header. | +| `UPDMSG` | char list | Prefix used to detect candidate update messages within raw bytes. | +| `CHUNK` | long | Default chunk size (bytes) to read (10MB). | +| `MAXCHUNK` | long | Maximum chunk size for a single read attempt (`8 * CHUNK`). | + +### Current default signature + +The module sets `UPDMSG` based on the serialized form of: + +```q +(`upd;`trade;()) +``` + +This means: +- it is geared toward logs containing `upd` messages for the `trade` table +- logs containing other table names or different update call shapes may not be recovered unless you adjust the signature logic + +--- + +## :wrench: Functions + +### Summary + +| Function | Description | +|----------|-------------| +| `check[logfile;lastmsgtoreplay]` | Returns `logfile` if it should be used as-is per `check` logic, otherwise triggers `repair` and returns `.good`. | +| `repair[logfile]` | Creates `.good` and writes any recoverable messages into it. Returns the new filename. | +--- + +### `check` + +```q +tplogutils.check[logfile; lastmsgtoreplay] +``` + +**Parameters** + +| Parameter | Type | Description | +|----------:|------|-------------| +| `logfile` | symbol | Path to logfile as a symbol (e.g. ```:tp.log```), as used by `-11!`, `hcount`, `read1`, etc. | +| `lastmsgtoreplay` | long | Index position of the last message the caller intends to replay. | + +**Behavior (as implemented)** +- inspects logfile info via `-11!(-2; logfile)` +- returns either: + - the original `logfile`, or + - a repaired logfile produced by `repair[logfile]` + +**Returns** +- `logfile` **or** `.good` + +--- + +### `repair` + +```q +tplogutils.repair[logfile] +``` + +**Purpose** +Create a “good” logfile containing only recoverable messages. + +**Behavior (as implemented)** +- writes output to `.good` +- processes the input logfile in chunks +- for each chunk: + - searches for occurrences of the configured `UPDMSG` signature + - splits the chunk into candidate messages + - constructs a header for each candidate + - attempts to deserialize each candidate + - writes successfully decoded messages into the output logfile + +**Returns** +- symbol path of the repaired logfile (e.g. ```:tp.log.good```) + +--- + +## :rocket: Typical usage + +### Repair-if-needed flow + +```q +/ Load module +tplogutils:use`tplogutils + +/ Decide whether to repair +log:`:tp.log +safe:tplogutils.check[log; 0j] + +/ safe is either `:tp.log or `:tp.log.good +safe +``` + +### Always repair + +```q +tplogutils:use`tplogutils + +log:`:tp.log +good:tplogutils.repair log +good +``` + +--- + +## :test_tube: Tests + +The module includes `test.q` and `test.csv`. + +### What the tests do (high level) + +`test.q` provides helpers to: +- create a valid log by writing records shaped like `enlist (`upd;`trade; rowData)` +- create a corrupt log by introducing byte-level corruption into one record +- verify that `check` and `repair` behave as expected across scenarios: + - valid logs + - corruption with enough valid messages + - corruption requiring repair + - garbage at end-of-file + - multiple corrupt sections + - completely corrupt logs + - empty logs + - sequential operations + +### Running tests manually + +```q +/ Load module +tplogutils:use`tplogutils + +/ Load tests +\l /path/to/kdbx-modules/di/tplogutils/test.q + +/ Run a few key tests +test_check_valid_log[] +test_repair_creates_good_file[] +test_repair_recovers_messages[] +test_repair_garbage_at_end[] +``` + +> **Note:** If the tests refer to `tplogsutil` but you loaded the module as `tplogutils`, either: +> - load the module into a `tplogsutil` variable as well, or +> - update the test references to `tplogutils`. + +--- + +## :bulb: Notes & limitations + +- **Best-effort recovery only:** The repair process only keeps messages that can be successfully deserialized by the module’s decode attempt. +- **Signature-specific:** The scan is currently tuned to the prefix of `(`upd;`trade;...)`. +- **Chunk-boundary sensitivity:** Recovery depends on being able to locate the message signature within the bytes read for a given chunk. +- **Validate output:** Always validate that `.good` replays correctly in your environment before using it as a production recovery artifact. + +--- + +## :package: Exported symbols + +The module exports: + +```q +export:([check;repair]) +``` + diff --git a/test_empty.log b/test_empty.log new file mode 100644 index 00000000..9e77be4b Binary files /dev/null and b/test_empty.log differ