mirror of
https://github.com/KingHWJ/WeReadBoard.git
synced 2026-11-03 18:24:58 +08:00
admin
ebfa3a1852
新增功能： - 书籍详情弹窗：点击书架书籍展示详细信息（书名、作者、出版社、ISBN、字数、评分、简介、阅读进度），数据实时从微信读书 API 获取并入库 - 笔记详情弹窗：点击有笔记的书展示划线和想法/点评，按章节分组展示，数据实时获取并入库 - 推荐页面：个性化推荐（为你推荐）+ 关键词搜索书籍，数据从微信读书 API 获取并入库 - 修复 Flask 中文 URL 参数编码问题 后端新增 API： - GET /api/book/detail?book_id=xxx — 书籍详情 - GET /api/book/notes?book_id=xxx — 笔记详情（划线+想法） - GET /api/recommend?maxIdx=0 — 个性化推荐 - GET /api/search?keyword=xxx — 搜索书籍 数据库新增表： - book_details, book_progress, book_highlights, book_chapters - book_reviews, recommended_books, search_results MySQL 版本建表语句 schema_mysql.sql： - 18 张表的完整 DDL，含索引、字符集 utf8mb4、字段注释 - 适用于后续从 SQLite 迁移至 MySQL
593 lines
17 KiB
Python
593 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
微信读书数据采集脚本
|
||
从微信读书 API 拉取数据并存入本地 SQLite 数据库
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import time
|
||
import urllib.request
|
||
|
||
API_URL = "https://i.weread.qq.com/api/agent/gateway"
|
||
API_KEY = os.environ.get("WEREAD_API_KEY", "")
|
||
SKILL_VERSION = "1.0.3"
|
||
DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "weread.db")
|
||
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
|
||
|
||
def api_call(api_name, **kwargs):
|
||
"""调用微信读书 API"""
|
||
body = {"api_name": api_name, "skill_version": SKILL_VERSION}
|
||
body.update(kwargs)
|
||
req = urllib.request.Request(
|
||
API_URL,
|
||
data=json.dumps(body).encode("utf-8"),
|
||
headers={
|
||
"Authorization": f"Bearer {API_KEY}",
|
||
"Content-Type": "application/json",
|
||
},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
return json.loads(resp.read().decode("utf-8"))
|
||
|
||
|
||
def init_db(db_path):
|
||
"""初始化数据库表结构"""
|
||
conn = sqlite3.connect(db_path)
|
||
c = conn.cursor()
|
||
|
||
# 书架书籍表
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS books (
|
||
book_id TEXT PRIMARY KEY,
|
||
title TEXT,
|
||
author TEXT,
|
||
cover TEXT,
|
||
category TEXT,
|
||
finish_reading INTEGER DEFAULT 0,
|
||
read_update_time INTEGER,
|
||
secret INTEGER DEFAULT 0,
|
||
is_top INTEGER DEFAULT 0,
|
||
update_time INTEGER,
|
||
deep_link TEXT
|
||
)
|
||
""")
|
||
|
||
# 阅读统计概要表
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS reading_stats (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
mode TEXT,
|
||
period TEXT,
|
||
total_read_time INTEGER,
|
||
read_days INTEGER,
|
||
day_average_read_time REAL,
|
||
regist_time INTEGER,
|
||
raw_json TEXT
|
||
)
|
||
""")
|
||
|
||
# 阅读时长时间序列（按年/月/日分桶）
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS read_times (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
mode TEXT,
|
||
period TEXT,
|
||
bucket_timestamp INTEGER,
|
||
duration INTEGER
|
||
)
|
||
""")
|
||
|
||
# 阅读时长最长的书籍排行
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS read_longest (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
mode TEXT,
|
||
period TEXT,
|
||
book_id TEXT,
|
||
title TEXT,
|
||
author TEXT,
|
||
cover TEXT,
|
||
read_time INTEGER,
|
||
tags TEXT
|
||
)
|
||
""")
|
||
|
||
# 偏好分类
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS prefer_categories (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
mode TEXT,
|
||
period TEXT,
|
||
category_id INTEGER,
|
||
category_title TEXT,
|
||
parent_category_title TEXT,
|
||
val REAL,
|
||
reading_time INTEGER,
|
||
reading_count INTEGER
|
||
)
|
||
""")
|
||
|
||
# 偏好作者
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS prefer_authors (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
mode TEXT,
|
||
period TEXT,
|
||
author_id INTEGER,
|
||
name TEXT,
|
||
count INTEGER,
|
||
read_time TEXT
|
||
)
|
||
""")
|
||
|
||
# 24小时阅读时段分布
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS prefer_time (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
mode TEXT,
|
||
period TEXT,
|
||
hour_index INTEGER,
|
||
duration INTEGER
|
||
)
|
||
""")
|
||
|
||
# 笔记本概览
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS notebooks (
|
||
book_id TEXT PRIMARY KEY,
|
||
title TEXT,
|
||
author TEXT,
|
||
cover TEXT,
|
||
review_count INTEGER DEFAULT 0,
|
||
note_count INTEGER DEFAULT 0,
|
||
bookmark_count INTEGER DEFAULT 0,
|
||
reading_progress INTEGER DEFAULT 0,
|
||
marked_status INTEGER DEFAULT 0,
|
||
sort INTEGER
|
||
)
|
||
""")
|
||
|
||
# 阅读统计摘要项
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS read_stat_items (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
mode TEXT,
|
||
period TEXT,
|
||
stat_name TEXT,
|
||
counts TEXT,
|
||
scheme TEXT
|
||
)
|
||
""")
|
||
|
||
# 书单/归档
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS archives (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
name TEXT,
|
||
book_ids TEXT
|
||
)
|
||
""")
|
||
|
||
# 数据采集元信息
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS fetch_meta (
|
||
key TEXT PRIMARY KEY,
|
||
value TEXT
|
||
)
|
||
""")
|
||
|
||
# ===== 按需获取的详细数据表 =====
|
||
|
||
# 书籍详细信息（来自 /book/info）
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS book_details (
|
||
book_id TEXT PRIMARY KEY,
|
||
title TEXT,
|
||
author TEXT,
|
||
translator TEXT,
|
||
cover TEXT,
|
||
intro TEXT,
|
||
category TEXT,
|
||
publisher TEXT,
|
||
publish_time TEXT,
|
||
isbn TEXT,
|
||
word_count INTEGER,
|
||
new_rating REAL,
|
||
new_rating_count INTEGER,
|
||
fetched_at INTEGER
|
||
)
|
||
""")
|
||
|
||
# 阅读进度（来自 /book/getprogress）
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS book_progress (
|
||
book_id TEXT PRIMARY KEY,
|
||
progress INTEGER,
|
||
chapter_uid INTEGER,
|
||
chapter_offset INTEGER,
|
||
update_time INTEGER,
|
||
record_reading_time INTEGER,
|
||
finish_time INTEGER,
|
||
is_start_reading INTEGER,
|
||
fetched_at INTEGER
|
||
)
|
||
""")
|
||
|
||
# 划线内容（来自 /book/bookmarklist）
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS book_highlights (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
book_id TEXT,
|
||
bookmark_id TEXT,
|
||
chapter_uid INTEGER,
|
||
mark_text TEXT,
|
||
create_time INTEGER,
|
||
type INTEGER,
|
||
range_text TEXT,
|
||
color_style INTEGER,
|
||
fetched_at INTEGER
|
||
)
|
||
""")
|
||
|
||
# 章节信息（来自 /book/bookmarklist 的 chapters）
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS book_chapters (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
book_id TEXT,
|
||
chapter_uid INTEGER,
|
||
chapter_idx INTEGER,
|
||
title TEXT,
|
||
UNIQUE(book_id, chapter_uid)
|
||
)
|
||
""")
|
||
|
||
# 个人想法/点评（来自 /review/list/mine）
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS book_reviews (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
book_id TEXT,
|
||
review_id TEXT,
|
||
content TEXT,
|
||
create_time INTEGER,
|
||
star INTEGER,
|
||
chapter_name TEXT,
|
||
is_finish INTEGER,
|
||
fetched_at INTEGER
|
||
)
|
||
""")
|
||
|
||
# 推荐书籍（来自 /book/recommend）
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS recommended_books (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
book_id TEXT,
|
||
title TEXT,
|
||
author TEXT,
|
||
cover TEXT,
|
||
intro TEXT,
|
||
category TEXT,
|
||
reason TEXT,
|
||
reading_count INTEGER,
|
||
new_rating REAL,
|
||
new_rating_count INTEGER,
|
||
rating_title TEXT,
|
||
fetched_at INTEGER
|
||
)
|
||
""")
|
||
|
||
# 搜索结果（来自 /store/search）
|
||
c.execute("""
|
||
CREATE TABLE IF NOT EXISTS search_results (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
keyword TEXT,
|
||
book_id TEXT,
|
||
title TEXT,
|
||
author TEXT,
|
||
cover TEXT,
|
||
intro TEXT,
|
||
publisher TEXT,
|
||
category TEXT,
|
||
reading_count INTEGER,
|
||
new_rating REAL,
|
||
new_rating_count INTEGER,
|
||
rating_title TEXT,
|
||
search_idx INTEGER,
|
||
fetched_at INTEGER
|
||
)
|
||
""")
|
||
|
||
conn.commit()
|
||
return conn
|
||
|
||
|
||
def fetch_shelf(conn):
|
||
"""拉取书架数据"""
|
||
print(" 拉取书架数据...")
|
||
data = api_call("/shelf/sync")
|
||
c = conn.cursor()
|
||
|
||
books = data.get("books", [])
|
||
for book in books:
|
||
c.execute(
|
||
"""INSERT OR REPLACE INTO books
|
||
(book_id, title, author, cover, category, finish_reading,
|
||
read_update_time, secret, is_top, update_time, deep_link)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||
(
|
||
book.get("bookId"),
|
||
book.get("title", ""),
|
||
book.get("author", ""),
|
||
book.get("cover", ""),
|
||
book.get("category", ""),
|
||
book.get("finishReading", 0),
|
||
book.get("readUpdateTime", 0),
|
||
book.get("secret", 0),
|
||
book.get("isTop", 0),
|
||
book.get("updateTime", 0),
|
||
book.get("deepLink", ""),
|
||
),
|
||
)
|
||
|
||
# 保存书单
|
||
archives = data.get("archive", [])
|
||
c.execute("DELETE FROM archives")
|
||
for arch in archives:
|
||
c.execute(
|
||
"INSERT INTO archives (name, book_ids) VALUES (?, ?)",
|
||
(arch.get("name", ""), json.dumps(arch.get("bookIds", []))),
|
||
)
|
||
|
||
conn.commit()
|
||
mp_flag = "yes" if data.get("mp") else "no"
|
||
print(f" 书架: {len(books)} 本电子书, {len(data.get('albums', []))} 个专辑, MP={mp_flag}, {len(archives)} 个书单")
|
||
|
||
|
||
def fetch_readdata(conn, mode, period_label):
|
||
"""拉取阅读统计数据"""
|
||
print(f" 拉取阅读统计 [{mode}]...")
|
||
params = {"mode": mode}
|
||
if mode != "overall":
|
||
params["baseTime"] = 0
|
||
data = api_call("/readdata/detail", **params)
|
||
c = conn.cursor()
|
||
|
||
total_read_time = data.get("totalReadTime", 0)
|
||
read_days = data.get("readDays", 0)
|
||
day_avg = data.get("dayAverageReadTime", 0)
|
||
regist_time = data.get("registTime", 0)
|
||
|
||
c.execute(
|
||
"""INSERT INTO reading_stats
|
||
(mode, period, total_read_time, read_days, day_average_read_time,
|
||
regist_time, raw_json)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||
(
|
||
mode,
|
||
period_label,
|
||
total_read_time,
|
||
read_days,
|
||
day_avg,
|
||
regist_time,
|
||
json.dumps(data, ensure_ascii=False),
|
||
),
|
||
)
|
||
|
||
# 保存 readTimes 分桶数据
|
||
read_times = data.get("readTimes", {})
|
||
for ts, duration in read_times.items():
|
||
c.execute(
|
||
"INSERT INTO read_times (mode, period, bucket_timestamp, duration) VALUES (?, ?, ?, ?)",
|
||
(mode, period_label, int(ts), duration),
|
||
)
|
||
|
||
# 保存 dailyReadTimes（如果有）
|
||
daily_times = data.get("dailyReadTimes", {})
|
||
for ts, duration in daily_times.items():
|
||
c.execute(
|
||
"INSERT INTO read_times (mode, period, bucket_timestamp, duration) VALUES (?, ?, ?, ?)",
|
||
(mode + "_daily", period_label, int(ts), duration),
|
||
)
|
||
|
||
# 保存阅读最久的书籍
|
||
for item in data.get("readLongest", []):
|
||
book = item.get("book", {})
|
||
album = item.get("albumInfo", {})
|
||
c.execute(
|
||
"""INSERT INTO read_longest
|
||
(mode, period, book_id, title, author, cover, read_time, tags)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||
(
|
||
mode,
|
||
period_label,
|
||
book.get("bookId") or album.get("albumId", ""),
|
||
book.get("title") or album.get("name", ""),
|
||
book.get("author") or album.get("authorName", ""),
|
||
book.get("cover") or album.get("cover", ""),
|
||
item.get("readTime", 0),
|
||
json.dumps(item.get("tags", []), ensure_ascii=False),
|
||
),
|
||
)
|
||
|
||
# 保存偏好分类
|
||
for cat in data.get("preferCategory", []):
|
||
c.execute(
|
||
"""INSERT INTO prefer_categories
|
||
(mode, period, category_id, category_title, parent_category_title,
|
||
val, reading_time, reading_count)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||
(
|
||
mode,
|
||
period_label,
|
||
cat.get("categoryId", 0),
|
||
cat.get("categoryTitle", ""),
|
||
cat.get("parentCategoryTitle", ""),
|
||
cat.get("val", 0),
|
||
cat.get("readingTime", 0),
|
||
cat.get("readingCount", 0),
|
||
),
|
||
)
|
||
|
||
# 保存偏好作者
|
||
for author in data.get("preferAuthor", []):
|
||
c.execute(
|
||
"""INSERT INTO prefer_authors
|
||
(mode, period, author_id, name, count, read_time)
|
||
VALUES (?, ?, ?, ?, ?, ?)""",
|
||
(
|
||
mode,
|
||
period_label,
|
||
author.get("authorId", 0),
|
||
author.get("name", ""),
|
||
author.get("count", 0),
|
||
author.get("readTime", ""),
|
||
),
|
||
)
|
||
|
||
# 保存24小时时段分布
|
||
prefer_time = data.get("preferTime", [])
|
||
# preferTime 从6点开始到次日5点，映射到 hour_index
|
||
for i, duration in enumerate(prefer_time):
|
||
hour = (6 + i) % 24
|
||
c.execute(
|
||
"INSERT INTO prefer_time (mode, period, hour_index, duration) VALUES (?, ?, ?, ?)",
|
||
(mode, period_label, hour, duration),
|
||
)
|
||
|
||
# 保存统计摘要项
|
||
for stat in data.get("readStat", []):
|
||
c.execute(
|
||
"""INSERT INTO read_stat_items
|
||
(mode, period, stat_name, counts, scheme)
|
||
VALUES (?, ?, ?, ?, ?)""",
|
||
(
|
||
mode,
|
||
period_label,
|
||
stat.get("stat", ""),
|
||
stat.get("counts", ""),
|
||
stat.get("scheme", ""),
|
||
),
|
||
)
|
||
|
||
conn.commit()
|
||
print(f" [{mode}] 总时长: {total_read_time}秒 ({total_read_time//3600}小时), 阅读天数: {read_days}")
|
||
|
||
|
||
def fetch_notebooks(conn):
|
||
"""拉取笔记本概览（分页拉取全部）"""
|
||
print(" 拉取笔记本数据...")
|
||
c = conn.cursor()
|
||
c.execute("DELETE FROM notebooks")
|
||
|
||
count = 100
|
||
last_sort = None
|
||
total = 0
|
||
|
||
while True:
|
||
params = {"count": count}
|
||
if last_sort is not None:
|
||
params["lastSort"] = last_sort
|
||
data = api_call("/user/notebooks", **params)
|
||
|
||
books = data.get("books", [])
|
||
for item in books:
|
||
book = item.get("book", {})
|
||
c.execute(
|
||
"""INSERT OR REPLACE INTO notebooks
|
||
(book_id, title, author, cover, review_count, note_count,
|
||
bookmark_count, reading_progress, marked_status, sort)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||
(
|
||
item.get("bookId", ""),
|
||
book.get("title", ""),
|
||
book.get("author", ""),
|
||
book.get("cover", ""),
|
||
item.get("reviewCount", 0),
|
||
item.get("noteCount", 0),
|
||
item.get("bookmarkCount", 0),
|
||
item.get("readingProgress", 0),
|
||
item.get("markedStatus", 0),
|
||
item.get("sort", 0),
|
||
),
|
||
)
|
||
total += 1
|
||
|
||
if data.get("hasMore", 0) == 1 and books:
|
||
last_sort = books[-1].get("sort")
|
||
else:
|
||
break
|
||
time.sleep(0.3)
|
||
|
||
conn.commit()
|
||
print(f" 笔记本: {total} 本有笔记的书")
|
||
|
||
|
||
def main():
|
||
if not API_KEY:
|
||
print("错误: 请先设置 WEREAD_API_KEY 环境变量")
|
||
print(" export WEREAD_API_KEY=wrk-xxxxxxxx")
|
||
return
|
||
|
||
print("=" * 50)
|
||
print("微信读书数据采集")
|
||
print("=" * 50)
|
||
|
||
# 初始化数据库
|
||
print(f"\n[1/4] 初始化数据库: {DB_PATH}")
|
||
conn = init_db(DB_PATH)
|
||
|
||
# 清空旧数据（保留表结构）
|
||
for table in [
|
||
"reading_stats", "read_times", "read_longest", "prefer_categories",
|
||
"prefer_authors", "prefer_time", "read_stat_items",
|
||
]:
|
||
conn.execute(f"DELETE FROM {table}")
|
||
conn.commit()
|
||
|
||
# 拉取书架
|
||
print("\n[2/4] 拉取书架数据")
|
||
fetch_shelf(conn)
|
||
|
||
# 拉取阅读统计（overall + 当前年）
|
||
print("\n[3/4] 拉取阅读统计数据")
|
||
fetch_readdata(conn, "overall", "overall")
|
||
current_year = time.localtime().tm_year
|
||
fetch_readdata(conn, "annually", str(current_year))
|
||
|
||
# 拉取笔记本
|
||
print("\n[4/4] 拉取笔记本数据")
|
||
fetch_notebooks(conn)
|
||
|
||
# 记录采集时间
|
||
c = conn.cursor()
|
||
c.execute(
|
||
"INSERT OR REPLACE INTO fetch_meta (key, value) VALUES (?, ?)",
|
||
("last_fetch_time", time.strftime("%Y-%m-%d %H:%M:%S")),
|
||
)
|
||
conn.commit()
|
||
|
||
# 统计
|
||
c.execute("SELECT COUNT(*) FROM books")
|
||
book_count = c.fetchone()[0]
|
||
c.execute("SELECT COUNT(*) FROM notebooks")
|
||
notebook_count = c.fetchone()[0]
|
||
c.execute("SELECT SUM(total_read_time) FROM reading_stats WHERE mode='overall'")
|
||
total_time = c.fetchone()[0] or 0
|
||
|
||
print("\n" + "=" * 50)
|
||
print("采集完成!")
|
||
print(f" 书架书籍: {book_count} 本")
|
||
print(f" 笔记本: {notebook_count} 本")
|
||
print(f" 总阅读时长: {total_time // 3600} 小时 {(total_time % 3600) // 60} 分钟")
|
||
print(f" 数据库: {DB_PATH}")
|
||
print("=" * 50)
|
||
|
||
conn.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|