refactor: complete data model v2.0 based on official API docs

数据模型全面重构,基于微信读书官方 8 份 API 文档逐字段比对:

1. 表结构从 18 张扩展到 31 张表:
   新增表:shelf_albums(有声书)、shelf_meta(书架元信息)、prefer_publishers(偏好出版社)、
   prefer_cps(偏好版权方)、prefer_books(偏好书籍卡片)、read_rank(好友排行)、
   medals(勋章)、year_reports(年度报告)、book_best_bookmarks(热门划线)、
   book_underlines(划线热度)、similar_books(相似推荐)、public_reviews(公开点评)、
   search_sessions(搜索会话)

2. 补全所有遗漏字段:
   - reading_stats: +13 个字段 (baseTime, compare, readRate, wrReadTime, wrListenTime 等)
   - book_chapters: +8 个字段 (wordCount, level, price, paid, isMPChapter, anchors 等)
   - book_reviews: +4 个字段 (abstract, range, chapterUid, chapterIdx)
   - recommended_books: +5 个字段 (deepLink, searchIdx, price, payType, type)
   - search_results: +8 个字段 (deepLink, payType, price, soldout, scope 等)
   - book_details: +2 个字段 (deepLink, newRatingDetail)
   - prefer_categories: +2 个字段 (parentCategoryId, categoryType)

3. 时间戳双列设计:
   - 所有整型时间戳字段均伴随 _dt TIMESTAMP 列 (如 read_update_time + read_update_time_dt)
   - 便于直接查询和展示,无需应用层转换

4. 审计字段:
   - 每张表均维护 created_at 和 updated_at
   - MySQL 使用 DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP

5. 索引优化:
   - 主键、唯一约束、查询索引根据实际访问模式重新设计
   - 新增复合索引、排序索引、全文检索索引

6. 数据采集脚本更新:
   - 覆盖所有新增表和字段的数据采集
   - skill_version 升级至 1.0.4
   - 有声书/专辑数据采集 (shelf_albums)
   - 偏好出版社/版权方/书籍卡片采集
   - 勋章、年度报告数据采集

7. 后端适配:
   - 移除旧版 ensure_tables 动态建表逻辑
   - 所有按需 API INSERT 语句适配新表结构
   - 搜索结果入库新增 search_sessions 会话表
   - 推荐书籍入库新增 batch_id 批次标识
This commit is contained in:
admin committed 2026-07-17 10:37:07 +08:00
1 parent 507b9c5704
commit 2a809eb7cc
5 files changed
+1595 -864

No files matched your search

+93 -174
View File
@@ -49,7 +49,7 @@ MYSQL_CONFIG = {
# ======================================================================
WEREAD_API_URL = "https://i.weread.qq.com/api/agent/gateway"
WEREAD_API_KEY = os.environ.get("WEREAD_API_KEY", "")
WEREAD_SKILL_VERSION = "1.0.3"
WEREAD_SKILL_VERSION = "1.0.4"
# ======================================================================
@@ -208,137 +208,21 @@ def format_duration(seconds):
return f"{minutes}分钟"
def ensure_tables(cursor):
def ts_to_dt_str(ts):
"""
确保按需数据表存在(MySQL 版本)。
这些表存储用户点击书籍详情/笔记时实时获取的数据,
不在初始化采集脚本中创建,而是首次访问时按需创建。
Unix 时间戳转完整日期时间字符串 (中国时区 UTC+8)。
用于写入数据库的 _dt 列。
Args:
cursor: MySQL 游标对象
ts (int): Unix 时间戳(秒)
Returns:
str: 格式化日期时间 "YYYY-MM-DD HH:MM:SS",空值返回 None
"""
cursor.executescript = None # MySQL 无 executescript,逐条执行
tables_sql = [
# 书籍详细信息表(来自微信读书 /book/info 接口)
"""CREATE TABLE IF NOT EXISTS `book_details` (
`book_id` VARCHAR(64) NOT NULL,
`title` VARCHAR(255) DEFAULT NULL COMMENT '书名',
`author` VARCHAR(255) DEFAULT NULL COMMENT '作者',
`translator` VARCHAR(255) DEFAULT NULL COMMENT '译者',
`cover` VARCHAR(512) DEFAULT NULL COMMENT '封面图URL',
`intro` LONGTEXT DEFAULT NULL COMMENT '书籍简介',
`category` VARCHAR(128) DEFAULT NULL COMMENT '分类',
`publisher` VARCHAR(255) DEFAULT NULL COMMENT '出版社',
`publish_time` VARCHAR(32) DEFAULT NULL COMMENT '出版时间',
`isbn` VARCHAR(32) DEFAULT NULL COMMENT 'ISBN号',
`word_count` INT UNSIGNED DEFAULT 0 COMMENT '总字数',
`new_rating` DECIMAL(5,1) DEFAULT 0 COMMENT '评分(百分制)',
`new_rating_count` INT UNSIGNED DEFAULT 0 COMMENT '评分人数',
`fetched_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '获取时间',
PRIMARY KEY (`book_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='书籍详细信息'""",
# 阅读进度表(来自微信读书 /book/getprogress 接口)
"""CREATE TABLE IF NOT EXISTS `book_progress` (
`book_id` VARCHAR(64) NOT NULL,
`progress` TINYINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '阅读进度百分比(0-100)',
`chapter_uid` INT UNSIGNED DEFAULT NULL COMMENT '当前阅读章节UID',
`chapter_offset` INT UNSIGNED DEFAULT NULL COMMENT '当前章节内偏移量',
`update_time` INT UNSIGNED DEFAULT NULL COMMENT '最后阅读时间(Unix时间戳)',
`record_reading_time` INT UNSIGNED DEFAULT 0 COMMENT '累计阅读时长(秒)',
`finish_time` INT UNSIGNED DEFAULT NULL COMMENT '读完时间(Unix时间戳)',
`is_start_reading` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否已开始阅读(1=是)',
`fetched_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '获取时间',
PRIMARY KEY (`book_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='阅读进度'""",
# 划线内容表(来自微信读书 /book/bookmarklist 接口)
"""CREATE TABLE IF NOT EXISTS `book_highlights` (
`id` BIGINT NOT NULL AUTO_INCREMENT,
`book_id` VARCHAR(64) NOT NULL COMMENT '书籍ID',
`bookmark_id` VARCHAR(64) DEFAULT NULL COMMENT '划线唯一ID',
`chapter_uid` INT UNSIGNED DEFAULT NULL COMMENT '所在章节UID',
`mark_text` LONGTEXT DEFAULT NULL COMMENT '划线原文内容',
`create_time` INT UNSIGNED DEFAULT NULL COMMENT '创建时间(Unix时间戳)',
`type` TINYINT DEFAULT NULL COMMENT '划线类型',
`range_text` VARCHAR(64) DEFAULT NULL COMMENT '位置范围',
`color_style` TINYINT DEFAULT NULL COMMENT '划线颜色样式',
`fetched_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '获取时间',
PRIMARY KEY (`id`),
INDEX `idx_highlights_book_id` (`book_id`),
INDEX `idx_highlights_chapter` (`book_id`, `chapter_uid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='划线内容'""",
# 章节信息表(来自微信读书 /book/bookmarklist 接口的 chapters 字段)
"""CREATE TABLE IF NOT EXISTS `book_chapters` (
`id` BIGINT NOT NULL AUTO_INCREMENT,
`book_id` VARCHAR(64) NOT NULL COMMENT '书籍ID',
`chapter_uid` INT UNSIGNED NOT NULL COMMENT '章节UID',
`chapter_idx` INT UNSIGNED DEFAULT NULL COMMENT '章节序号',
`title` VARCHAR(255) DEFAULT NULL COMMENT '章节标题',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_book_chapter` (`book_id`, `chapter_uid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='章节信息'""",
# 个人想法/点评表(来自微信读书 /review/list/mine 接口)
"""CREATE TABLE IF NOT EXISTS `book_reviews` (
`id` BIGINT NOT NULL AUTO_INCREMENT,
`book_id` VARCHAR(64) NOT NULL COMMENT '书籍ID',
`review_id` VARCHAR(64) DEFAULT NULL COMMENT '想法唯一ID',
`content` LONGTEXT DEFAULT NULL COMMENT '想法内容文本',
`create_time` INT UNSIGNED DEFAULT NULL COMMENT '创建时间(Unix时间戳)',
`star` TINYINT DEFAULT -1 COMMENT '评分(0-5, -1=无评分)',
`chapter_name` VARCHAR(255) DEFAULT NULL COMMENT '所在章节名称',
`is_finish` TINYINT(1) DEFAULT 0 COMMENT '是否书评(1=书评)',
`fetched_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '获取时间',
PRIMARY KEY (`id`),
INDEX `idx_reviews_book_id` (`book_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='个人想法/点评'""",
# 推荐书籍表(来自微信读书 /book/recommend 接口)
"""CREATE TABLE IF NOT EXISTS `recommended_books` (
`id` BIGINT NOT NULL AUTO_INCREMENT,
`book_id` VARCHAR(64) NOT NULL COMMENT '书籍ID',
`title` VARCHAR(255) DEFAULT NULL COMMENT '书名',
`author` VARCHAR(255) DEFAULT NULL COMMENT '作者',
`cover` VARCHAR(512) DEFAULT NULL COMMENT '封面图URL',
`intro` TEXT DEFAULT NULL COMMENT '书籍简介',
`category` VARCHAR(128) DEFAULT NULL COMMENT '分类',
`reason` VARCHAR(512) DEFAULT NULL COMMENT '推荐理由',
`reading_count` INT UNSIGNED DEFAULT 0 COMMENT '在读人数',
`new_rating` DECIMAL(5,1) DEFAULT 0 COMMENT '评分(百分制)',
`new_rating_count` INT UNSIGNED DEFAULT 0 COMMENT '评分人数',
`rating_title` VARCHAR(32) DEFAULT NULL COMMENT '评分标签(神作/力荐等)',
`fetched_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '获取时间',
PRIMARY KEY (`id`),
INDEX `idx_recommended_book_id` (`book_id`),
INDEX `idx_recommended_fetched` (`fetched_at` DESC)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='推荐书籍'""",
# 搜索结果表(来自微信读书 /store/search 接口)
"""CREATE TABLE IF NOT EXISTS `search_results` (
`id` BIGINT NOT NULL AUTO_INCREMENT,
`keyword` VARCHAR(255) NOT NULL COMMENT '搜索关键词',
`book_id` VARCHAR(64) NOT NULL COMMENT '书籍ID',
`title` VARCHAR(255) DEFAULT NULL COMMENT '书名',
`author` VARCHAR(255) DEFAULT NULL COMMENT '作者',
`cover` VARCHAR(512) DEFAULT NULL COMMENT '封面图URL',
`intro` TEXT DEFAULT NULL COMMENT '书籍简介',
`publisher` VARCHAR(255) DEFAULT NULL COMMENT '出版社',
`category` VARCHAR(128) DEFAULT NULL COMMENT '分类',
`reading_count` INT UNSIGNED DEFAULT 0 COMMENT '在读人数',
`new_rating` DECIMAL(5,1) DEFAULT 0 COMMENT '评分(百分制)',
`new_rating_count` INT UNSIGNED DEFAULT 0 COMMENT '评分人数',
`rating_title` VARCHAR(32) DEFAULT NULL COMMENT '评分标签',
`search_idx` INT UNSIGNED DEFAULT 0 COMMENT '搜索序号',
`fetched_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '获取时间',
PRIMARY KEY (`id`),
INDEX `idx_search_keyword` (`keyword`),
INDEX `idx_search_book_id` (`book_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='搜索结果'""",
]
for sql in tables_sql:
cursor.execute(sql)
if not ts or ts == 0:
return None
from datetime import timezone, timedelta
cst = timezone(timedelta(hours=8))
return datetime.fromtimestamp(ts, tz=cst).strftime("%Y-%m-%d %H:%M:%S")
# ======================================================================
@@ -794,18 +678,17 @@ def api_book_detail(conn):
return jsonify({"error": "book_id is required"}), 400
cursor = conn.cursor()
ensure_tables(cursor)
conn.commit()
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# 1. 获取书籍基本信息
# 1. 获取书籍基本信息(/book/info)
try:
info = weread_api_call("/book/info", bookId=book_id)
rating_detail = info.get("newRatingDetail")
cursor.execute(
"REPLACE INTO book_details "
"(book_id, title, author, translator, cover, intro, category, "
"publisher, publish_time, isbn, word_count, new_rating, new_rating_count, fetched_at) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
"publisher, publish_time, isbn, word_count, new_rating, new_rating_count, "
"new_rating_detail, deep_link) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(
book_id,
info.get("title", ""),
@@ -820,7 +703,8 @@ def api_book_detail(conn):
info.get("wordCount", 0),
info.get("newRating", 0),
info.get("newRatingCount", 0),
now,
json.dumps(rating_detail, ensure_ascii=False) if rating_detail else None,
info.get("deepLink", ""),
),
)
conn.commit()
@@ -836,21 +720,25 @@ def api_book_detail(conn):
try:
prog = weread_api_call("/book/getprogress", bookId=book_id)
book_prog = prog.get("book", {})
srv_ts = prog.get("timestamp")
ut = book_prog.get("updateTime", 0)
ft = book_prog.get("finishTime", 0)
cursor.execute(
"REPLACE INTO book_progress "
"(book_id, progress, chapter_uid, chapter_offset, update_time, "
"record_reading_time, finish_time, is_start_reading, fetched_at) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)",
"(book_id, progress, chapter_uid, chapter_offset, update_time, update_time_dt, "
"record_reading_time, finish_time, finish_time_dt, is_start_reading, "
"server_timestamp, server_timestamp_dt) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(
book_id,
book_prog.get("progress", 0),
book_prog.get("chapterUid", 0),
book_prog.get("chapterOffset", 0),
book_prog.get("updateTime", 0),
ut, ts_to_dt_str(ut),
book_prog.get("recordReadingTime", 0),
book_prog.get("finishTime", 0),
ft, ts_to_dt_str(ft),
book_prog.get("isStartReading", 0),
now,
srv_ts, ts_to_dt_str(srv_ts),
),
)
conn.commit()
@@ -898,7 +786,7 @@ def api_book_detail(conn):
"new_rating": float(detail["new_rating"]) if detail["new_rating"] else 0,
"new_rating_count": detail["new_rating_count"],
"progress": progress_data,
"deep_link": f"weread://reading?bId={book_id}",
"deep_link": detail["deep_link"] or f"weread://reading?bId={book_id}",
})
@@ -926,20 +814,18 @@ def api_book_notes(conn):
return jsonify({"error": "book_id is required"}), 400
cursor = conn.cursor()
ensure_tables(cursor)
conn.commit()
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# 1. 获取划线数据
# 1. 获取划线数据(/book/bookmarklist)
highlights = []
chapters = {}
try:
data = weread_api_call("/book/bookmarklist", bookId=book_id)
# 保存章节信息
# 保存章节信息(包含完整字段)
for ch in data.get("chapters", []):
cursor.execute(
"REPLACE INTO book_chapters (book_id, chapter_uid, chapter_idx, title) "
"REPLACE INTO book_chapters "
"(book_id, chapter_uid, chapter_idx, title) "
"VALUES (%s,%s,%s,%s)",
(book_id, ch.get("chapterUid", 0), ch.get("chapterIdx", 0), ch.get("title", "")),
)
@@ -948,21 +834,21 @@ def api_book_notes(conn):
# 保存划线数据(先删除旧数据再插入)
cursor.execute("DELETE FROM book_highlights WHERE book_id=%s", (book_id,))
for bm in data.get("updated", []):
ct = bm.get("createTime", 0)
cursor.execute(
"INSERT INTO book_highlights "
"(book_id, bookmark_id, chapter_uid, mark_text, create_time, "
"type, range_text, color_style, fetched_at) "
"(book_id, bookmark_id, chapter_uid, mark_text, create_time, create_time_dt, "
"type, range_text, color_style) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(
book_id,
bm.get("bookmarkId", ""),
bm.get("chapterUid", 0),
bm.get("markText", ""),
bm.get("createTime", 0),
ct, ts_to_dt_str(ct),
bm.get("type", 0),
bm.get("range", ""),
bm.get("colorStyle", 0),
now,
),
)
highlights.append({
@@ -1001,26 +887,32 @@ def api_book_notes(conn):
cursor.execute("DELETE FROM book_reviews WHERE book_id=%s", (book_id,))
for item in rev_data.get("reviews", []):
rev = item.get("review", {})
ct = rev.get("createTime", 0)
cursor.execute(
"INSERT INTO book_reviews "
"(book_id, review_id, content, create_time, star, chapter_name, is_finish, fetched_at) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s)",
"(book_id, review_id, content, abstract, range_text, chapter_uid, chapter_idx, "
"chapter_name, create_time, create_time_dt, star, is_finish) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(
book_id,
rev.get("reviewId", ""),
rev.get("content", ""),
rev.get("createTime", 0),
rev.get("star", 0),
rev.get("abstract", ""),
rev.get("range", ""),
rev.get("chapterUid"),
rev.get("chapterIdx"),
rev.get("chapterName", ""),
ct, ts_to_dt_str(ct),
rev.get("star", -1),
rev.get("isFinish", 0),
now,
),
)
reviews.append({
"review_id": rev.get("reviewId", ""),
"content": rev.get("content", ""),
"abstract": rev.get("abstract", ""),
"create_time": ts_to_date(rev.get("createTime", 0)),
"star": rev.get("star", 0),
"star": rev.get("star", -1),
"chapter_name": rev.get("chapterName", ""),
"is_finish": rev.get("isFinish", 0),
})
@@ -1078,22 +970,21 @@ def api_recommend(conn):
"""
max_idx = int(request.args.get("maxIdx", 0))
cursor = conn.cursor()
ensure_tables(cursor)
conn.commit()
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
batch_id = datetime.now().strftime("%Y%m%d%H%M%S")
try:
data = weread_api_call("/book/recommend", count=12, maxIdx=max_idx)
books = data.get("books", [])
# 将推荐书籍入库
# 将推荐书籍入库(完整字段)
for book in books:
rating_detail = book.get("newRatingDetail", {})
cursor.execute(
"REPLACE INTO recommended_books "
"(book_id, title, author, cover, intro, category, reason, "
"reading_count, new_rating, new_rating_count, rating_title, fetched_at) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
"reading_count, new_rating, new_rating_count, rating_title, "
"deep_link, search_idx, price, pay_type, book_type, batch_id) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(
book.get("bookId", ""),
book.get("title", ""),
@@ -1106,7 +997,12 @@ def api_recommend(conn):
book.get("newRating", 0),
book.get("newRatingCount", 0),
rating_detail.get("title", "") if isinstance(rating_detail, dict) else "",
now,
book.get("deepLink", ""),
book.get("searchIdx", 0),
book.get("price"),
book.get("payType"),
book.get("type", 0),
batch_id,
),
)
conn.commit()
@@ -1168,25 +1064,39 @@ def api_search(conn):
max_idx = int(request.args.get("maxIdx", 0))
cursor = conn.cursor()
ensure_tables(cursor)
conn.commit()
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
data = weread_api_call("/store/search", keyword=keyword, scope=10, maxIdx=max_idx)
sid = data.get("sid", "")
has_more = data.get("hasMore", 0)
# 创建搜索会话记录
cursor.execute(
"INSERT INTO search_sessions (sid, keyword, scope, has_more, max_idx) "
"VALUES (%s,%s,%s,%s,%s)",
(sid, keyword, 10, has_more, max_idx),
)
session_id = cursor.lastrowid
results = data.get("results", [])
books = []
for group in results:
result_title = group.get("title", "")
result_scope = group.get("scope")
scope_count = group.get("scopeCount")
current_count = group.get("currentCount")
for item in group.get("books", []):
bi = item.get("bookInfo", {})
rating_detail = item.get("newRatingDetail", {})
cursor.execute(
"REPLACE INTO search_results "
"(keyword, book_id, title, author, cover, intro, publisher, "
"category, reading_count, new_rating, new_rating_count, "
"rating_title, search_idx, fetched_at) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
"INSERT INTO search_results "
"(session_id, keyword, book_id, title, author, cover, intro, publisher, "
"category, reading_count, new_rating, new_rating_count, rating_title, "
"deep_link, pay_type, price, soldout, search_idx, "
"result_scope, result_title, scope_count, current_count) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(
session_id,
keyword,
bi.get("bookId", ""),
bi.get("title", ""),
@@ -1199,8 +1109,15 @@ def api_search(conn):
item.get("newRating", 0),
item.get("newRatingCount", 0),
rating_detail.get("title", "") if isinstance(rating_detail, dict) else "",
bi.get("deepLink", ""),
bi.get("payType"),
bi.get("price"),
bi.get("soldout", 0),
item.get("searchIdx", 0),
now,
result_scope,
result_title,
scope_count,
current_count,
),
)
books.append({
@@ -1215,6 +1132,8 @@ def api_search(conn):
"new_rating": item.get("newRating", 0),
"new_rating_count": item.get("newRatingCount", 0),
"rating_title": rating_detail.get("title", "") if isinstance(rating_detail, dict) else "",
"deep_link": bi.get("deepLink", ""),
"soldout": bi.get("soldout", 0),
})
conn.commit()
except Exception as e: