Add archivist incremental sync state
Request: Add archivist support for remembering which IPO archive stages have already been synced and which stages should be updated next. Changes: - Add sync_runs, ticker_sync_state, sync_tasks, and price_performance tables to the archive schema. - Add scripts/update_sync_state.py to derive per-ticker stage status and rebuild the next-sync task queue. - Export the new sync-state tables as Git-friendly CSV snapshots. - Document the incremental archive flow in the archivist skill and README. Verification: - Ran scripts/bootstrap_historical_data.py. - Ran scripts/update_sync_state.py with a deterministic as-of timestamp. - Checked SQLite integrity and DB-to-snapshot row counts with Python sqlite3. - Parsed Python scripts with ast.parse. - Ran git diff --check and checked for temporary SQLite/cache files. Next useful context: - Current derived queue has 2 open tasks for 06658 and 15 waiting_until_due tasks for future stages.
This commit is contained in:
@@ -53,6 +53,23 @@ CREATE TABLE IF NOT EXISTS ipo_demand (
|
||||
notes TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS price_performance (
|
||||
performance_id TEXT PRIMARY KEY,
|
||||
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
|
||||
stage TEXT NOT NULL,
|
||||
source_id TEXT,
|
||||
as_of_date TEXT NOT NULL,
|
||||
open_price_hkd REAL,
|
||||
high_price_hkd REAL,
|
||||
low_price_hkd REAL,
|
||||
close_price_hkd REAL,
|
||||
return_pct REAL,
|
||||
turnover_hkd_m REAL,
|
||||
data_as_of TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
UNIQUE (ticker, stage)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_refs (
|
||||
source_id TEXT PRIMARY KEY,
|
||||
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
|
||||
@@ -81,3 +98,45 @@ CREATE TABLE IF NOT EXISTS data_gaps (
|
||||
created_at TEXT NOT NULL,
|
||||
notes TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_runs (
|
||||
sync_run_id TEXT PRIMARY KEY,
|
||||
mode TEXT NOT NULL,
|
||||
as_of TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT,
|
||||
status TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
CHECK (status IN ('running', 'complete', 'failed'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticker_sync_state (
|
||||
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
|
||||
stage TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
required INTEGER NOT NULL DEFAULT 1,
|
||||
due_date TEXT,
|
||||
completed_at TEXT,
|
||||
last_source_id TEXT,
|
||||
data_gap_id TEXT,
|
||||
last_sync_run_id TEXT REFERENCES sync_runs(sync_run_id),
|
||||
updated_at TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
PRIMARY KEY (ticker, stage),
|
||||
CHECK (status IN ('complete', 'pending_not_due', 'pending_due', 'blocked', 'not_applicable')),
|
||||
CHECK (required IN (0, 1))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
|
||||
stage TEXT NOT NULL,
|
||||
task_type TEXT NOT NULL,
|
||||
task_status TEXT NOT NULL,
|
||||
due_date TEXT,
|
||||
data_gap_id TEXT,
|
||||
last_sync_run_id TEXT REFERENCES sync_runs(sync_run_id),
|
||||
updated_at TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
CHECK (task_status IN ('open', 'waiting_until_due', 'blocked'))
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user