Add archivist incremental sync state

Request:
Add archivist support for remembering which IPO archive stages have already been synced and which stages should be updated next.

Changes:
- Add sync_runs, ticker_sync_state, sync_tasks, and price_performance tables to the archive schema.
- Add scripts/update_sync_state.py to derive per-ticker stage status and rebuild the next-sync task queue.
- Export the new sync-state tables as Git-friendly CSV snapshots.
- Document the incremental archive flow in the archivist skill and README.

Verification:
- Ran scripts/bootstrap_historical_data.py.
- Ran scripts/update_sync_state.py with a deterministic as-of timestamp.
- Checked SQLite integrity and DB-to-snapshot row counts with Python sqlite3.
- Parsed Python scripts with ast.parse.
- Ran git diff --check and checked for temporary SQLite/cache files.

Next useful context:
- Current derived queue has 2 open tasks for 06658 and 15 waiting_until_due tasks for future stages.
This commit is contained in:
2026-06-15 06:29:54 +00:00
parent fb715cd446
commit 08db218b6d
10 changed files with 601 additions and 3 deletions
+59
View File
@@ -53,6 +53,23 @@ CREATE TABLE IF NOT EXISTS ipo_demand (
notes TEXT
);
CREATE TABLE IF NOT EXISTS price_performance (
performance_id TEXT PRIMARY KEY,
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
stage TEXT NOT NULL,
source_id TEXT,
as_of_date TEXT NOT NULL,
open_price_hkd REAL,
high_price_hkd REAL,
low_price_hkd REAL,
close_price_hkd REAL,
return_pct REAL,
turnover_hkd_m REAL,
data_as_of TEXT NOT NULL,
notes TEXT,
UNIQUE (ticker, stage)
);
CREATE TABLE IF NOT EXISTS source_refs (
source_id TEXT PRIMARY KEY,
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
@@ -81,3 +98,45 @@ CREATE TABLE IF NOT EXISTS data_gaps (
created_at TEXT NOT NULL,
notes TEXT
);
CREATE TABLE IF NOT EXISTS sync_runs (
sync_run_id TEXT PRIMARY KEY,
mode TEXT NOT NULL,
as_of TEXT NOT NULL,
started_at TEXT NOT NULL,
finished_at TEXT,
status TEXT NOT NULL,
notes TEXT,
CHECK (status IN ('running', 'complete', 'failed'))
);
CREATE TABLE IF NOT EXISTS ticker_sync_state (
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
stage TEXT NOT NULL,
status TEXT NOT NULL,
required INTEGER NOT NULL DEFAULT 1,
due_date TEXT,
completed_at TEXT,
last_source_id TEXT,
data_gap_id TEXT,
last_sync_run_id TEXT REFERENCES sync_runs(sync_run_id),
updated_at TEXT NOT NULL,
notes TEXT,
PRIMARY KEY (ticker, stage),
CHECK (status IN ('complete', 'pending_not_due', 'pending_due', 'blocked', 'not_applicable')),
CHECK (required IN (0, 1))
);
CREATE TABLE IF NOT EXISTS sync_tasks (
task_id TEXT PRIMARY KEY,
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
stage TEXT NOT NULL,
task_type TEXT NOT NULL,
task_status TEXT NOT NULL,
due_date TEXT,
data_gap_id TEXT,
last_sync_run_id TEXT REFERENCES sync_runs(sync_run_id),
updated_at TEXT NOT NULL,
notes TEXT,
CHECK (task_status IN ('open', 'waiting_until_due', 'blocked'))
);