Bootstrap HK IPO historical archive
Request: - Use the project archivist workflow to update historical IPO data. Changes: - Add an embedded SQLite archive at data/hk_ipo.sqlite. - Add schema/hk_ipo.schema.sql and scripts/bootstrap_historical_data.py for reproducible archive generation. - Archive HKEXnews source PDFs for 06658, 06675, and 06106 under repo-relative data/raw paths. - Export Git-friendly snapshots for ipo_master, offering_terms, ipo_demand, source_refs, and data_gaps. - Add .gitignore rules for Python cache and SQLite transient files. Verification: - Re-ran the bootstrap script successfully. - Ran PRAGMA integrity_check on the SQLite database. - Verified source_refs paths are repo-relative, files exist, and SHA-256 hashes match. - Verified snapshot row counts match SQLite table counts. - Ran git diff --check and searched generated durable files for machine-specific absolute paths.
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
__pycache__/
|
||||
data/*.sqlite-journal
|
||||
data/*.sqlite-shm
|
||||
data/*.sqlite-wal
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,4 @@
|
||||
gap_id,ticker,stage,field_name,reason,expected_resolution_date,created_at,notes
|
||||
06106_allotment_results_pending_2026_06_15,06106,T1_allotment,ipo_demand,Allotment results were expected on 2026-06-23 and were not available in this seed archive.,2026-06-23,2026-06-15T06:15:00Z,Update after the HKEXnews allotment results announcement is published.
|
||||
06106_full_prospectus_classification_2026_06_15,06106,T0_prospectus,full_prospectus_local_path,The archived 2026061500011 PDF is an offering announcement/notice; the separately archived 2026061500013 PDF needs document-role verification before detailed extraction.,,2026-06-15T06:15:00Z,Keep both official files in raw archive until classification is confirmed.
|
||||
06675_allotment_results_pending_2026_06_15,06675,T1_allotment,ipo_demand,Allotment results were expected on 2026-06-16 and were not available in this seed archive.,2026-06-16,2026-06-15T06:15:00Z,Update after the HKEXnews allotment results announcement is published.
|
||||
|
@@ -0,0 +1,2 @@
|
||||
demand_id,ticker,source_id,stage_date,valid_applications,successful_applications,public_oversubscription_times,international_placees,international_oversubscription_times,final_hk_offer_shares,final_international_offer_shares,data_as_of,notes
|
||||
06658_allotment_2026_06_12,06658,06658_allotment_results_2026_06_12,2026-06-12,180507,11465,6586.73,64,2.64,1146500,10317600,2026-06-15T06:15:00Z,Claw-back shown as N/A in the HKEXnews allotment results.
|
||||
|
@@ -0,0 +1,4 @@
|
||||
ticker,company_name_en,company_name_zh,stock_short_name,exchange,board,status,listing_date,application_start_date,application_end_date,allotment_results_expected_date,industry_label,data_as_of,notes
|
||||
06106,"Shanghai Seer Intelligent Technology Co., Ltd.",上海仙工智能科技股份有限公司,,HKEX,Main Board,open_for_subscription,2026-06-24,2026-06-15,2026-06-18,2026-06-23,Industrial intelligent robots / robot controllers,2026-06-15T06:15:00Z,Seeded from HKEXnews global offering announcement; full prospectus source classification needs follow-up.
|
||||
06658,"Liuliumei Co., Ltd.",溜溜梅股份有限公司,LIULIUMEI,HKEX,Main Board,listed,2026-06-15,2026-06-05,2026-06-10,2026-06-12,Snack food / preserved fruit,2026-06-15T06:15:00Z,Seeded from HKEXnews prospectus and allotment results.
|
||||
06675,"SENASIC Electronics Technology Co., Ltd.",琻捷電子科技(江蘇)股份有限公司,,HKEX,Main Board,pending_listing,2026-06-17,2026-06-09,2026-06-12,2026-06-16,Automotive wireless sensing SoC / semiconductors,2026-06-15T06:15:00Z,Seeded from HKEXnews prospectus and global offering announcement; allotment results not yet archived.
|
||||
|
@@ -0,0 +1,4 @@
|
||||
ticker,source_id,prospectus_date,offer_price_hkd,board_lot,min_subscription_amount_hkd,global_offer_shares,hk_offer_shares_initial,international_offer_shares_initial,public_offer_pct_initial,over_allotment_offer_shares,offer_size_adjustment_offer_shares,market_cap_hkd_m,gross_proceeds_hkd_m,net_proceeds_hkd_m,issued_shares_upon_listing,data_as_of
|
||||
06106,06106_prospectus_notice_2026_06_15,2026-06-15,101.6,50,5131.24,10497300,524900,9972400,0.05,1574550,1574550,,,,,2026-06-15T06:15:00Z
|
||||
06658,06658_prospectus_2026_06_05,2026-06-05,43.58,100,4401.96,11464100,1146500,10317600,0.1,,,3434.59,499.6,440.1,78811208,2026-06-15T06:15:00Z
|
||||
06675,06675_global_offering_announcement_2026_06_09,2026-06-09,18.36,200,3709.04,53407000,5340800,48066200,0.1,8011000,,6959.2,,906.7,379041820,2026-06-15T06:15:00Z
|
||||
|
@@ -0,0 +1,7 @@
|
||||
source_id,ticker,source_type,title,path_base,local_path,url,file_sha256,source_date,archived_at,notes
|
||||
06106_prospectus_candidate_2026_06_15,06106,prospectus_candidate_pending_verification,"Shanghai Seer Intelligent Technology Co., Ltd. Prospectus Candidate",repo_root,data/raw/06106/prospectus_candidate_2026-06-15.pdf,https://www1.hkexnews.hk/listedco/listconews/sehk/2026/0615/2026061500013.pdf,e8b129296563e43b7834be9d59ac41926fbaeb4f088da2c908b1f04b4151967b,2026-06-15,2026-06-15T06:15:00Z,Downloaded from HKEXnews; document role should be verified before using for detailed fact extraction.
|
||||
06106_prospectus_notice_2026_06_15,06106,prospectus_notice,"Shanghai Seer Intelligent Technology Co., Ltd. Prospectus Notice",repo_root,data/raw/06106/prospectus_notice_2026-06-15.pdf,https://www1.hkexnews.hk/listedco/listconews/sehk/2026/0615/2026061500011.pdf,510983deaba5614975a57c5e77d3ea83af071a24609c28cd3f89914e1649bff5,2026-06-15,2026-06-15T06:15:00Z,HKEXnews announcement containing global offering terms and timetable.
|
||||
06658_allotment_results_2026_06_12,06658,allotment_results,"Liuliumei Co., Ltd. Announcement of Allotment Results",repo_root,data/raw/06658/allotment_results_2026-06-12.pdf,https://www1.hkexnews.hk/listedco/listconews/sehk/2026/0612/2026061202100.pdf,bb305cf55cc87809ecd845ea44243c4f41fcfaa31dbf496580e2ed8fc06d54a0,2026-06-12,2026-06-15T06:15:00Z,HKEXnews allotment results.
|
||||
06658_prospectus_2026_06_05,06658,prospectus,"Liuliumei Co., Ltd. Prospectus",repo_root,data/raw/06658/prospectus_2026-06-05.pdf,https://www1.hkexnews.hk/listedco/listconews/sehk/2026/0605/2026060500023.pdf,e928dd8082e8aaf28156a46f64c98bee308d8ae4d10a9571a4531a3f9a8f0eb1,2026-06-05,2026-06-15T06:15:00Z,HKEXnews prospectus.
|
||||
06675_global_offering_announcement_2026_06_09,06675,global_offering_announcement,"SENASIC Electronics Technology Co., Ltd. Global Offering Announcement",repo_root,data/raw/06675/global_offering_announcement_2026-06-09.pdf,https://www.hkexnews.hk/listedco/listconews/sehk/2026/0609/2026060900009.pdf,a6b0c03d6b7a42cab0865aa0abf6dfa2dd80e6d16e392d73ddd3cd3839f7aeff,2026-06-09,2026-06-15T06:15:00Z,HKEXnews global offering announcement.
|
||||
06675_prospectus_2026_06_09,06675,prospectus,"SENASIC Electronics Technology Co., Ltd. Prospectus",repo_root,data/raw/06675/prospectus_2026-06-09.pdf,https://www.hkexnews.hk/listedco/listconews/sehk/2026/0609/2026060900029.pdf,0c0c634786b7e7da921dd631fa7ba696043fae4ab29cf29dcc5f9e976c53b160,2026-06-09,2026-06-15T06:15:00Z,HKEXnews prospectus.
|
||||
|
@@ -0,0 +1,83 @@
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ipo_master (
|
||||
ticker TEXT PRIMARY KEY,
|
||||
company_name_en TEXT NOT NULL,
|
||||
company_name_zh TEXT,
|
||||
stock_short_name TEXT,
|
||||
exchange TEXT NOT NULL DEFAULT 'HKEX',
|
||||
board TEXT NOT NULL DEFAULT 'Main Board',
|
||||
status TEXT NOT NULL,
|
||||
listing_date TEXT,
|
||||
application_start_date TEXT,
|
||||
application_end_date TEXT,
|
||||
allotment_results_expected_date TEXT,
|
||||
industry_label TEXT,
|
||||
data_as_of TEXT NOT NULL,
|
||||
notes TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS offering_terms (
|
||||
ticker TEXT PRIMARY KEY REFERENCES ipo_master(ticker),
|
||||
source_id TEXT NOT NULL,
|
||||
prospectus_date TEXT,
|
||||
offer_price_hkd REAL,
|
||||
board_lot INTEGER,
|
||||
min_subscription_amount_hkd REAL,
|
||||
global_offer_shares INTEGER,
|
||||
hk_offer_shares_initial INTEGER,
|
||||
international_offer_shares_initial INTEGER,
|
||||
public_offer_pct_initial REAL,
|
||||
over_allotment_offer_shares INTEGER,
|
||||
offer_size_adjustment_offer_shares INTEGER,
|
||||
market_cap_hkd_m REAL,
|
||||
gross_proceeds_hkd_m REAL,
|
||||
net_proceeds_hkd_m REAL,
|
||||
issued_shares_upon_listing INTEGER,
|
||||
data_as_of TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ipo_demand (
|
||||
demand_id TEXT PRIMARY KEY,
|
||||
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
|
||||
source_id TEXT NOT NULL,
|
||||
stage_date TEXT NOT NULL,
|
||||
valid_applications INTEGER,
|
||||
successful_applications INTEGER,
|
||||
public_oversubscription_times REAL,
|
||||
international_placees INTEGER,
|
||||
international_oversubscription_times REAL,
|
||||
final_hk_offer_shares INTEGER,
|
||||
final_international_offer_shares INTEGER,
|
||||
data_as_of TEXT NOT NULL,
|
||||
notes TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_refs (
|
||||
source_id TEXT PRIMARY KEY,
|
||||
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
|
||||
source_type TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
path_base TEXT NOT NULL DEFAULT 'repo_root',
|
||||
local_path TEXT NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
file_sha256 TEXT,
|
||||
source_date TEXT,
|
||||
archived_at TEXT NOT NULL,
|
||||
notes TEXT,
|
||||
CHECK (path_base = 'repo_root'),
|
||||
CHECK (local_path NOT LIKE '/%'),
|
||||
CHECK (local_path NOT LIKE './%'),
|
||||
CHECK (local_path NOT LIKE '%\%')
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_gaps (
|
||||
gap_id TEXT PRIMARY KEY,
|
||||
ticker TEXT NOT NULL REFERENCES ipo_master(ticker),
|
||||
stage TEXT NOT NULL,
|
||||
field_name TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
expected_resolution_date TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
notes TEXT
|
||||
);
|
||||
@@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bootstrap the project-local HK IPO historical archive."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ARCHIVE_AS_OF = "2026-06-15T06:15:00Z"
|
||||
DB_PATH = Path("data/hk_ipo.sqlite")
|
||||
SCHEMA_PATH = Path("schema/hk_ipo.schema.sql")
|
||||
SNAPSHOT_DIR = Path("data/snapshots")
|
||||
|
||||
|
||||
IPO_MASTER = [
|
||||
{
|
||||
"ticker": "06658",
|
||||
"company_name_en": "Liuliumei Co., Ltd.",
|
||||
"company_name_zh": "溜溜梅股份有限公司",
|
||||
"stock_short_name": "LIULIUMEI",
|
||||
"status": "listed",
|
||||
"listing_date": "2026-06-15",
|
||||
"application_start_date": "2026-06-05",
|
||||
"application_end_date": "2026-06-10",
|
||||
"allotment_results_expected_date": "2026-06-12",
|
||||
"industry_label": "Snack food / preserved fruit",
|
||||
"data_as_of": ARCHIVE_AS_OF,
|
||||
"notes": "Seeded from HKEXnews prospectus and allotment results.",
|
||||
},
|
||||
{
|
||||
"ticker": "06675",
|
||||
"company_name_en": "SENASIC Electronics Technology Co., Ltd.",
|
||||
"company_name_zh": "琻捷電子科技(江蘇)股份有限公司",
|
||||
"stock_short_name": None,
|
||||
"status": "pending_listing",
|
||||
"listing_date": "2026-06-17",
|
||||
"application_start_date": "2026-06-09",
|
||||
"application_end_date": "2026-06-12",
|
||||
"allotment_results_expected_date": "2026-06-16",
|
||||
"industry_label": "Automotive wireless sensing SoC / semiconductors",
|
||||
"data_as_of": ARCHIVE_AS_OF,
|
||||
"notes": "Seeded from HKEXnews prospectus and global offering announcement; allotment results not yet archived.",
|
||||
},
|
||||
{
|
||||
"ticker": "06106",
|
||||
"company_name_en": "Shanghai Seer Intelligent Technology Co., Ltd.",
|
||||
"company_name_zh": "上海仙工智能科技股份有限公司",
|
||||
"stock_short_name": None,
|
||||
"status": "open_for_subscription",
|
||||
"listing_date": "2026-06-24",
|
||||
"application_start_date": "2026-06-15",
|
||||
"application_end_date": "2026-06-18",
|
||||
"allotment_results_expected_date": "2026-06-23",
|
||||
"industry_label": "Industrial intelligent robots / robot controllers",
|
||||
"data_as_of": ARCHIVE_AS_OF,
|
||||
"notes": "Seeded from HKEXnews global offering announcement; full prospectus source classification needs follow-up.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
OFFERING_TERMS = [
|
||||
{
|
||||
"ticker": "06658",
|
||||
"source_id": "06658_prospectus_2026_06_05",
|
||||
"prospectus_date": "2026-06-05",
|
||||
"offer_price_hkd": 43.58,
|
||||
"board_lot": 100,
|
||||
"min_subscription_amount_hkd": 4401.96,
|
||||
"global_offer_shares": 11464100,
|
||||
"hk_offer_shares_initial": 1146500,
|
||||
"international_offer_shares_initial": 10317600,
|
||||
"public_offer_pct_initial": 0.10,
|
||||
"over_allotment_offer_shares": None,
|
||||
"offer_size_adjustment_offer_shares": None,
|
||||
"market_cap_hkd_m": 3434.59,
|
||||
"gross_proceeds_hkd_m": 499.6,
|
||||
"net_proceeds_hkd_m": 440.1,
|
||||
"issued_shares_upon_listing": 78811208,
|
||||
"data_as_of": ARCHIVE_AS_OF,
|
||||
},
|
||||
{
|
||||
"ticker": "06675",
|
||||
"source_id": "06675_global_offering_announcement_2026_06_09",
|
||||
"prospectus_date": "2026-06-09",
|
||||
"offer_price_hkd": 18.36,
|
||||
"board_lot": 200,
|
||||
"min_subscription_amount_hkd": 3709.04,
|
||||
"global_offer_shares": 53407000,
|
||||
"hk_offer_shares_initial": 5340800,
|
||||
"international_offer_shares_initial": 48066200,
|
||||
"public_offer_pct_initial": 0.10,
|
||||
"over_allotment_offer_shares": 8011000,
|
||||
"offer_size_adjustment_offer_shares": None,
|
||||
"market_cap_hkd_m": 6959.2,
|
||||
"gross_proceeds_hkd_m": None,
|
||||
"net_proceeds_hkd_m": 906.7,
|
||||
"issued_shares_upon_listing": 379041820,
|
||||
"data_as_of": ARCHIVE_AS_OF,
|
||||
},
|
||||
{
|
||||
"ticker": "06106",
|
||||
"source_id": "06106_prospectus_notice_2026_06_15",
|
||||
"prospectus_date": "2026-06-15",
|
||||
"offer_price_hkd": 101.60,
|
||||
"board_lot": 50,
|
||||
"min_subscription_amount_hkd": 5131.24,
|
||||
"global_offer_shares": 10497300,
|
||||
"hk_offer_shares_initial": 524900,
|
||||
"international_offer_shares_initial": 9972400,
|
||||
"public_offer_pct_initial": 0.05,
|
||||
"over_allotment_offer_shares": 1574550,
|
||||
"offer_size_adjustment_offer_shares": 1574550,
|
||||
"market_cap_hkd_m": None,
|
||||
"gross_proceeds_hkd_m": None,
|
||||
"net_proceeds_hkd_m": None,
|
||||
"issued_shares_upon_listing": None,
|
||||
"data_as_of": ARCHIVE_AS_OF,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
IPO_DEMAND = [
|
||||
{
|
||||
"demand_id": "06658_allotment_2026_06_12",
|
||||
"ticker": "06658",
|
||||
"source_id": "06658_allotment_results_2026_06_12",
|
||||
"stage_date": "2026-06-12",
|
||||
"valid_applications": 180507,
|
||||
"successful_applications": 11465,
|
||||
"public_oversubscription_times": 6586.73,
|
||||
"international_placees": 64,
|
||||
"international_oversubscription_times": 2.64,
|
||||
"final_hk_offer_shares": 1146500,
|
||||
"final_international_offer_shares": 10317600,
|
||||
"data_as_of": ARCHIVE_AS_OF,
|
||||
"notes": "Claw-back shown as N/A in the HKEXnews allotment results.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
SOURCES = [
|
||||
{
|
||||
"source_id": "06658_prospectus_2026_06_05",
|
||||
"ticker": "06658",
|
||||
"source_type": "prospectus",
|
||||
"title": "Liuliumei Co., Ltd. Prospectus",
|
||||
"local_path": "data/raw/06658/prospectus_2026-06-05.pdf",
|
||||
"url": "https://www1.hkexnews.hk/listedco/listconews/sehk/2026/0605/2026060500023.pdf",
|
||||
"source_date": "2026-06-05",
|
||||
"notes": "HKEXnews prospectus.",
|
||||
},
|
||||
{
|
||||
"source_id": "06658_allotment_results_2026_06_12",
|
||||
"ticker": "06658",
|
||||
"source_type": "allotment_results",
|
||||
"title": "Liuliumei Co., Ltd. Announcement of Allotment Results",
|
||||
"local_path": "data/raw/06658/allotment_results_2026-06-12.pdf",
|
||||
"url": "https://www1.hkexnews.hk/listedco/listconews/sehk/2026/0612/2026061202100.pdf",
|
||||
"source_date": "2026-06-12",
|
||||
"notes": "HKEXnews allotment results.",
|
||||
},
|
||||
{
|
||||
"source_id": "06675_prospectus_2026_06_09",
|
||||
"ticker": "06675",
|
||||
"source_type": "prospectus",
|
||||
"title": "SENASIC Electronics Technology Co., Ltd. Prospectus",
|
||||
"local_path": "data/raw/06675/prospectus_2026-06-09.pdf",
|
||||
"url": "https://www.hkexnews.hk/listedco/listconews/sehk/2026/0609/2026060900029.pdf",
|
||||
"source_date": "2026-06-09",
|
||||
"notes": "HKEXnews prospectus.",
|
||||
},
|
||||
{
|
||||
"source_id": "06675_global_offering_announcement_2026_06_09",
|
||||
"ticker": "06675",
|
||||
"source_type": "global_offering_announcement",
|
||||
"title": "SENASIC Electronics Technology Co., Ltd. Global Offering Announcement",
|
||||
"local_path": "data/raw/06675/global_offering_announcement_2026-06-09.pdf",
|
||||
"url": "https://www.hkexnews.hk/listedco/listconews/sehk/2026/0609/2026060900009.pdf",
|
||||
"source_date": "2026-06-09",
|
||||
"notes": "HKEXnews global offering announcement.",
|
||||
},
|
||||
{
|
||||
"source_id": "06106_prospectus_notice_2026_06_15",
|
||||
"ticker": "06106",
|
||||
"source_type": "prospectus_notice",
|
||||
"title": "Shanghai Seer Intelligent Technology Co., Ltd. Prospectus Notice",
|
||||
"local_path": "data/raw/06106/prospectus_notice_2026-06-15.pdf",
|
||||
"url": "https://www1.hkexnews.hk/listedco/listconews/sehk/2026/0615/2026061500011.pdf",
|
||||
"source_date": "2026-06-15",
|
||||
"notes": "HKEXnews announcement containing global offering terms and timetable.",
|
||||
},
|
||||
{
|
||||
"source_id": "06106_prospectus_candidate_2026_06_15",
|
||||
"ticker": "06106",
|
||||
"source_type": "prospectus_candidate_pending_verification",
|
||||
"title": "Shanghai Seer Intelligent Technology Co., Ltd. Prospectus Candidate",
|
||||
"local_path": "data/raw/06106/prospectus_candidate_2026-06-15.pdf",
|
||||
"url": "https://www1.hkexnews.hk/listedco/listconews/sehk/2026/0615/2026061500013.pdf",
|
||||
"source_date": "2026-06-15",
|
||||
"notes": "Downloaded from HKEXnews; document role should be verified before using for detailed fact extraction.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
DATA_GAPS = [
|
||||
{
|
||||
"gap_id": "06675_allotment_results_pending_2026_06_15",
|
||||
"ticker": "06675",
|
||||
"stage": "T1_allotment",
|
||||
"field_name": "ipo_demand",
|
||||
"reason": "Allotment results were expected on 2026-06-16 and were not available in this seed archive.",
|
||||
"expected_resolution_date": "2026-06-16",
|
||||
"created_at": ARCHIVE_AS_OF,
|
||||
"notes": "Update after the HKEXnews allotment results announcement is published.",
|
||||
},
|
||||
{
|
||||
"gap_id": "06106_allotment_results_pending_2026_06_15",
|
||||
"ticker": "06106",
|
||||
"stage": "T1_allotment",
|
||||
"field_name": "ipo_demand",
|
||||
"reason": "Allotment results were expected on 2026-06-23 and were not available in this seed archive.",
|
||||
"expected_resolution_date": "2026-06-23",
|
||||
"created_at": ARCHIVE_AS_OF,
|
||||
"notes": "Update after the HKEXnews allotment results announcement is published.",
|
||||
},
|
||||
{
|
||||
"gap_id": "06106_full_prospectus_classification_2026_06_15",
|
||||
"ticker": "06106",
|
||||
"stage": "T0_prospectus",
|
||||
"field_name": "full_prospectus_local_path",
|
||||
"reason": "The archived 2026061500011 PDF is an offering announcement/notice; the separately archived 2026061500013 PDF needs document-role verification before detailed extraction.",
|
||||
"expected_resolution_date": None,
|
||||
"created_at": ARCHIVE_AS_OF,
|
||||
"notes": "Keep both official files in raw archive until classification is confirmed.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def repo_root() -> Path:
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def hash_file(relative_path: str) -> str:
|
||||
path = repo_root() / relative_path
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def ensure_relative_path(relative_path: str) -> None:
|
||||
path = Path(relative_path)
|
||||
if path.is_absolute() or relative_path.startswith("./") or "\\" in relative_path:
|
||||
raise ValueError(f"Path must be repo-relative POSIX style: {relative_path}")
|
||||
if not (repo_root() / relative_path).exists():
|
||||
raise FileNotFoundError(relative_path)
|
||||
|
||||
|
||||
def upsert_rows(conn: sqlite3.Connection, table: str, rows: list[dict[str, object]]) -> None:
|
||||
if not rows:
|
||||
return
|
||||
columns = list(rows[0])
|
||||
placeholders = ", ".join("?" for _ in columns)
|
||||
assignments = ", ".join(f"{column}=excluded.{column}" for column in columns)
|
||||
sql = (
|
||||
f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({placeholders}) "
|
||||
f"ON CONFLICT DO UPDATE SET {assignments}"
|
||||
)
|
||||
conn.executemany(sql, ([row[column] for column in columns] for row in rows))
|
||||
|
||||
|
||||
def export_snapshot(conn: sqlite3.Connection, table: str) -> None:
|
||||
SNAPSHOT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cursor = conn.execute(f"SELECT * FROM {table} ORDER BY 1")
|
||||
columns = [description[0] for description in cursor.description]
|
||||
with (SNAPSHOT_DIR / f"{table}.csv").open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.writer(handle)
|
||||
writer.writerow(columns)
|
||||
writer.writerows(cursor.fetchall())
|
||||
|
||||
|
||||
def main() -> None:
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with sqlite3.connect(DB_PATH) as conn:
|
||||
conn.executescript(SCHEMA_PATH.read_text(encoding="utf-8"))
|
||||
upsert_rows(conn, "ipo_master", IPO_MASTER)
|
||||
upsert_rows(conn, "offering_terms", OFFERING_TERMS)
|
||||
upsert_rows(conn, "ipo_demand", IPO_DEMAND)
|
||||
|
||||
source_rows = []
|
||||
for source in SOURCES:
|
||||
ensure_relative_path(source["local_path"])
|
||||
source_rows.append(
|
||||
{
|
||||
**source,
|
||||
"path_base": "repo_root",
|
||||
"file_sha256": hash_file(source["local_path"]),
|
||||
"archived_at": ARCHIVE_AS_OF,
|
||||
}
|
||||
)
|
||||
upsert_rows(conn, "source_refs", source_rows)
|
||||
upsert_rows(conn, "data_gaps", DATA_GAPS)
|
||||
|
||||
for table in [
|
||||
"ipo_master",
|
||||
"offering_terms",
|
||||
"ipo_demand",
|
||||
"source_refs",
|
||||
"data_gaps",
|
||||
]:
|
||||
export_snapshot(conn, table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user