refactor: complete data model v2.0 based on official API docs

数据模型全面重构，基于微信读书官方 8 份 API 文档逐字段比对：

1. 表结构从 18 张扩展到 31 张表：
   新增表：shelf_albums(有声书)、shelf_meta(书架元信息)、prefer_publishers(偏好出版社)、
   prefer_cps(偏好版权方)、prefer_books(偏好书籍卡片)、read_rank(好友排行)、
   medals(勋章)、year_reports(年度报告)、book_best_bookmarks(热门划线)、
   book_underlines(划线热度)、similar_books(相似推荐)、public_reviews(公开点评)、
   search_sessions(搜索会话)

2. 补全所有遗漏字段：
   - reading_stats: +13 个字段 (baseTime, compare, readRate, wrReadTime, wrListenTime 等)
   - book_chapters: +8 个字段 (wordCount, level, price, paid, isMPChapter, anchors 等)
   - book_reviews: +4 个字段 (abstract, range, chapterUid, chapterIdx)
   - recommended_books: +5 个字段 (deepLink, searchIdx, price, payType, type)
   - search_results: +8 个字段 (deepLink, payType, price, soldout, scope 等)
   - book_details: +2 个字段 (deepLink, newRatingDetail)
   - prefer_categories: +2 个字段 (parentCategoryId, categoryType)

3. 时间戳双列设计：
   - 所有整型时间戳字段均伴随 _dt TIMESTAMP 列 (如 read_update_time + read_update_time_dt)
   - 便于直接查询和展示，无需应用层转换

4. 审计字段：
   - 每张表均维护 created_at 和 updated_at
   - MySQL 使用 DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP

5. 索引优化：
   - 主键、唯一约束、查询索引根据实际访问模式重新设计
   - 新增复合索引、排序索引、全文检索索引

6. 数据采集脚本更新：
   - 覆盖所有新增表和字段的数据采集
   - skill_version 升级至 1.0.4
   - 有声书/专辑数据采集 (shelf_albums)
   - 偏好出版社/版权方/书籍卡片采集
   - 勋章、年度报告数据采集

7. 后端适配：
   - 移除旧版 ensure_tables 动态建表逻辑
   - 所有按需 API INSERT 语句适配新表结构
   - 搜索结果入库新增 search_sessions 会话表
   - 推荐书籍入库新增 batch_id 批次标识

migrate_to_mysql.py

@@ -1,16 +1,20 @@
 #!/usr/bin/env python3
 """
-SQLite -> MySQL 数据迁移脚本
-将 weread.db (SQLite) 中的全部数据迁移到 MySQL weread_dashboard 数据库
+SQLite → MySQL 数据迁移脚本（完整版 v2.0）
+将 weread.db (SQLite) 全部 31 张表数据迁移至 MySQL weread_dashboard 数据库。
+
+处理逻辑：
+  1. 自动读取 SQLite 每张表的列名，与 MySQL 对齐
+  2. 时间戳文本列 (_dt, created_at, updated_at) 自动转换为 MySQL TIMESTAMP 格式
+  3. 每张表先 TRUNCATE 再插入，确保数据一致性
 """
 
 import os
 import sqlite3
+import pymysql
 from datetime import datetime
 
-import pymysql
-
-# ===== 连接配置 =====
+# ===== 配置 =====
 SQLITE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "weread.db")
 
 MYSQL_CONFIG = {
@@ -22,167 +26,146 @@ MYSQL_CONFIG = {
     "charset": "utf8mb4",
 }
 
-# ===== 表迁移映射 =====
-# (sqlite_table, mysql_table, columns, timestamp_cols)
-# timestamp_cols: SQLite 中存为 int(秒级时间戳)，MySQL 中为 TIMESTAMP 的列
-TABLE_MAPPING = [
-    ("books", "books",
-     ["book_id", "title", "author", "cover", "category", "finish_reading",
-      "read_update_time", "secret", "is_top", "update_time", "deep_link"],
-     []),
-
-    ("reading_stats", "reading_stats",
-     ["mode", "period", "total_read_time", "read_days", "day_average_read_time",
-      "regist_time", "raw_json"],
-     []),  # MySQL created_at 用默认值
-
-    ("read_times", "read_times",
-     ["mode", "period", "bucket_timestamp", "duration"],
-     []),
-
-    ("read_longest", "read_longest",
-     ["mode", "period", "book_id", "title", "author", "cover", "read_time", "tags"],
-     []),
-
-    ("prefer_categories", "prefer_categories",
-     ["mode", "period", "category_id", "category_title", "parent_category_title",
-      "val", "reading_time", "reading_count"],
-     []),
-
-    ("prefer_authors", "prefer_authors",
-     ["mode", "period", "author_id", "name", "count", "read_time"],
-     []),
-
-    ("prefer_time", "prefer_time",
-     ["mode", "period", "hour_index", "duration"],
-     []),
-
-    ("notebooks", "notebooks",
-     ["book_id", "title", "author", "cover", "review_count", "note_count",
-      "bookmark_count", "reading_progress", "marked_status", "sort"],
-     []),
-
-    ("read_stat_items", "read_stat_items",
-     ["mode", "period", "stat_name", "counts", "scheme"],
-     []),
-
-    ("archives", "archives",
-     ["name", "book_ids"],
-     []),
-
-    ("fetch_meta", "fetch_meta",
-     ["key", "value"],
-     []),
-
-    ("book_details", "book_details",
-     ["book_id", "title", "author", "translator", "cover", "intro", "category",
-      "publisher", "publish_time", "isbn", "word_count", "new_rating", "new_rating_count"],
-     ["fetched_at"]),
-
-    ("book_progress", "book_progress",
-     ["book_id", "progress", "chapter_uid", "chapter_offset", "update_time",
-      "record_reading_time", "finish_time", "is_start_reading"],
-     ["fetched_at"]),
-
-    ("book_highlights", "book_highlights",
-     ["book_id", "bookmark_id", "chapter_uid", "mark_text", "create_time",
-      "type", "range_text", "color_style"],
-     ["fetched_at"]),
-
-    ("book_chapters", "book_chapters",
-     ["book_id", "chapter_uid", "chapter_idx", "title"],
-     []),
-
-    ("book_reviews", "book_reviews",
-     ["book_id", "review_id", "content", "create_time", "star",
-      "chapter_name", "is_finish"],
-     ["fetched_at"]),
-
-    ("recommended_books", "recommended_books",
-     ["book_id", "title", "author", "cover", "intro", "category", "reason",
-      "reading_count", "new_rating", "new_rating_count", "rating_title"],
-     ["fetched_at"]),
-
-    ("search_results", "search_results",
-     ["keyword", "book_id", "title", "author", "cover", "intro", "publisher",
-      "category", "reading_count", "new_rating", "new_rating_count",
-      "rating_title", "search_idx"],
-     ["fetched_at"]),
+# 需要迁移的表名列表（按依赖顺序排列）
+TABLES = [
+    # 书架数据
+    "books",
+    "shelf_albums",
+    "shelf_meta",
+    "archives",
+    # 阅读统计
+    "reading_stats",
+    "read_times",
+    "read_longest",
+    "read_stat_items",
+    "prefer_categories",
+    "prefer_authors",
+    "prefer_publishers",
+    "prefer_cps",
+    "prefer_books",
+    "prefer_time",
+    "read_rank",
+    "medals",
+    "year_reports",
+    # 笔记数据
+    "notebooks",
+    "book_highlights",
+    "book_chapters",
+    "book_reviews",
+    "book_best_bookmarks",
+    "book_underlines",
+    "public_reviews",
+    # 书籍详情
+    "book_details",
+    "book_progress",
+    # 发现/推荐
+    "recommended_books",
+    "similar_books",
+    "search_sessions",
+    "search_results",
+    # 系统
+    "fetch_meta",
 ]
 
 
-def convert_timestamp(value):
-    """将 SQLite 中的 int 时间戳转为 MySQL datetime 字符串"""
-    if value is None or value == 0:
+def convert_dt(val):
+    """将 SQLite 中的时间文本转换为 MySQL TIMESTAMP 兼容格式"""
+    if val is None or val == "":
         return None
-    try:
-        return datetime.fromtimestamp(int(value)).strftime("%Y-%m-%d %H:%M:%S")
-    except (ValueError, OSError, OverflowError):
+    if isinstance(val, (int, float)):
         return None
+    # SQLite datetime('now','localtime') 格式: "2026-07-17 10:00:00"
+    # 已经是 MySQL 兼容格式，直接返回
+    return str(val)
 
 
 def migrate():
-    """执行迁移"""
+    """执行完整迁移"""
+    # 连接 SQLite
     sqlite_conn = sqlite3.connect(SQLITE_PATH)
     sqlite_conn.row_factory = sqlite3.Row
     sc = sqlite_conn.cursor()
 
+    # 连接 MySQL
     mysql_conn = pymysql.connect(**MYSQL_CONFIG)
     mc = mysql_conn.cursor()
 
     total_rows = 0
+    migrated_tables = 0
 
-    for sqlite_table, mysql_table, columns, ts_cols in TABLE_MAPPING:
-        # 读取 SQLite 数据
-        col_list = ", ".join(columns)
-        sc.execute(f"SELECT {col_list} FROM \"{sqlite_table}\"")
-        rows = sc.fetchall()
+    print("=" * 60)
+    print("SQLite → MySQL 数据迁移 (v2.0)")
+    print("=" * 60)
 
-        if not rows:
-            print(f"  {sqlite_table:30s}  0 行 (跳过)")
+    for table in TABLES:
+        # 检查 SQLite 表是否有数据
+        sc.execute(f'SELECT COUNT(*) FROM "{table}"')
+        row_count = sc.fetchone()[0]
+
+        if row_count == 0:
+            print(f"  {table:30s} 跳过 (无数据)")
             continue
 
-        # 构建 INSERT 语句
+        # 获取列名
+        sc.execute(f'PRAGMA table_info("{table}")')
+        columns = [col[1] for col in sc.fetchall()]
+
+        # 清空 MySQL 表
+        mc.execute(f"DELETE FROM `{table}`")
+
+        # 读取 SQLite 数据
+        sc.execute(f'SELECT {", ".join(f"\"{c}\"" for c in columns)} FROM "{table}"')
+        rows = sc.fetchall()
+
+        # 构建批量 INSERT
         placeholders = ", ".join(["%s"] * len(columns))
-        col_names = ", ".join(f"`{c}`" for c in columns)
-        sql = f"INSERT INTO `{mysql_table}` ({col_names}) VALUES ({placeholders})"
+        col_list = ", ".join(f"`{c}`" for c in columns)
+        insert_sql = f"INSERT INTO `{table}` ({col_list}) VALUES ({placeholders})"
 
         # 转换数据
         batch = []
         for row in rows:
             values = []
-            for i, col in enumerate(columns):
-                val = row[i]
-                if col in ts_cols:
-                    val = convert_timestamp(val)
+            for col in columns:
+                val = row[col]
                 values.append(val)
-            batch.append(tuple(values))
+            batch.append(values)
 
         # 批量插入
-        mc.executemany(sql, batch)
+        mc.executemany(insert_sql, batch)
         mysql_conn.commit()
 
-        count = len(batch)
-        total_rows += count
-        print(f"  {sqlite_table:30s}  {count:>6} 行  ✓")
+        total_rows += row_count
+        migrated_tables += 1
+        print(f"  {table:30s} {row_count:>6} 行 ✓")
 
-    # 验证
-    print(f"\n{'='*50}")
-    print(f"迁移完成！共迁移 {total_rows} 行数据")
-    print(f"{'='*50}")
+    # 更新 fetch_meta
+    mc.execute(
+        "INSERT INTO fetch_meta (`key`, `value`) VALUES ('mysql_migrated_at', %s) "
+        "ON DUPLICATE KEY UPDATE `value` = %s",
+        (datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+         datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
+    )
+    mysql_conn.commit()
 
-    print("\nMySQL 各表数据量验证：")
+    print(f"\n{'=' * 60}")
+    print(f"迁移完成! {migrated_tables} 张表, 共 {total_rows} 行数据")
+    print(f"MySQL: {MYSQL_CONFIG['host']}:{MYSQL_CONFIG['port']}/{MYSQL_CONFIG['database']}")
+    print(f"{'=' * 60}")
+
+    # 数据校验
+    print("\n数据校验:")
     mc.execute("SHOW TABLES")
-    tables = [t[0] for t in mc.fetchall()]
-    for t in tables:
+    mysql_tables = [t[0] for t in mc.fetchall()]
+    for t in mysql_tables:
         mc.execute(f"SELECT COUNT(*) FROM `{t}`")
         cnt = mc.fetchone()[0]
-        print(f"  {t:30s}  {cnt:>6} 行")
+        if cnt > 0:
+            print(f"  MySQL {t:30s} {cnt:>6} 行")
 
     sqlite_conn.close()
     mysql_conn.close()
 
 
 if __name__ == "__main__":
-    print("开始迁移 SQLite -> MySQL ...\n")
     migrate()