From f38d264366dbf6dda3d47365c3e3c28f46bb544e Mon Sep 17 00:00:00 2001 From: imsyy Date: Wed, 4 Dec 2024 17:47:48 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=A6=84=20refactor:=20=E8=B0=83=E6=95=B4?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 19 ++++--- .github/workflows/docker.yml | 17 +++++- Dockerfile | 28 ++++++++-- README.md | 22 +++++++- package.json | 5 +- pnpm-lock.yaml | 33 ++++++------ src/config.ts | 2 + src/index.ts | 3 +- src/routes/36kr.ts | 11 ++-- src/routes/51cto.ts | 11 ++-- src/routes/52pojie.ts | 11 ++-- src/routes/acfun.ts | 11 ++-- src/routes/baidu.ts | 11 ++-- src/routes/bilibili.ts | 11 ++-- src/routes/coolapk.ts | 11 ++-- src/routes/csdn.ts | 11 ++-- src/routes/douban-group.ts | 11 ++-- src/routes/douban-movie.ts | 11 ++-- src/routes/douyin.ts | 11 ++-- src/routes/earthquake.ts | 11 ++-- src/routes/genshin.ts | 11 ++-- src/routes/hellogithub.ts | 11 ++-- src/routes/history.ts | 11 ++-- src/routes/honkai.ts | 11 ++-- src/routes/hostloc.ts | 11 ++-- src/routes/hupu.ts | 11 ++-- src/routes/huxiu.ts | 11 ++-- src/routes/ifanr.ts | 11 ++-- src/routes/ithome-xijiayi.ts | 11 ++-- src/routes/ithome.ts | 11 ++-- src/routes/jianshu.ts | 11 ++-- src/routes/juejin.ts | 11 ++-- src/routes/lol.ts | 11 ++-- src/routes/miyoushe.ts | 11 ++-- src/routes/netease-news.ts | 11 ++-- src/routes/ngabbs.ts | 11 ++-- src/routes/nodeseek.ts | 11 ++-- src/routes/qq-news.ts | 11 ++-- src/routes/sina-news.ts | 11 ++-- src/routes/sina.ts | 11 ++-- src/routes/sspai.ts | 11 ++-- src/routes/starrail.ts | 11 ++-- src/routes/thepaper.ts | 11 ++-- src/routes/tieba.ts | 11 ++-- src/routes/toutiao.ts | 11 ++-- src/routes/v2ex.ts | 11 ++-- src/routes/weatheralarm.ts | 14 ++--- src/routes/weibo.ts | 11 ++-- src/routes/weread.ts | 11 ++-- src/routes/zhihu-daily.ts | 11 ++-- src/routes/zhihu.ts | 11 ++-- src/types.d.ts | 3 +- src/utils/getData.ts | 100 +++++++++++++++++++++++++++-------- src/utils/parseRSS.ts | 2 +- 54 files changed, 348 insertions(+), 362 deletions(-) diff --git a/.env.example b/.env.example index 421e5fa..acf4c6c 100644 --- a/.env.example +++ b/.env.example @@ -1,24 +1,27 @@ # 服务端口 -PORT = 6688 +PORT=6688 # 允许的域名 -ALLOWED_DOMAIN = "*" +ALLOWED_DOMAIN="*" # 允许的主域名,填写格式为 imsyy.top ## 若填写该项,将忽略 ALLOWED_DOMAIN -ALLOWED_HOST = "" +ALLOWED_HOST="" # ROBOT -DISALLOW_ROBOT = true +DISALLOW_ROBOT=true # 缓存时长( 秒 ) -CACHE_TTL = 3600 +CACHE_TTL=3600 # 请求超时( 毫秒 ) -REQUEST_TIMEOUT = 6000 +REQUEST_TIMEOUT=6000 # 是否输出日志 -USE_LOG_FILE = true +USE_LOG_FILE=true # RSS Mode -RSS_MODE = false \ No newline at end of file +RSS_MODE=false + +# Puppeteer +USE_PUPPETEER=false \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 621119f..b8d65a8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,7 +43,7 @@ jobs: imsyy/dailyhot-api ghcr.io/${{ github.repository }} - - name: Build and push + - name: Build and push regular image (no Puppeteer) uses: docker/build-push-action@v5 with: context: . @@ -54,3 +54,18 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + - name: Build and push Puppeteer image (with Puppeteer) + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + platforms: | + linux/amd64 + linux/arm64 + push: true + tags: imsyy/dailyhot-api:web-latest + build-args: | + USE_PUPPETEER=true + labels: ${{ steps.meta.outputs.labels }} + diff --git a/Dockerfile b/Dockerfile index cf36fe8..6eb0096 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,13 +3,27 @@ FROM node:20-alpine AS base ENV NODE_ENV=docker # 安装 Puppeteer 所需的依赖库 -RUN apk add libc6-compat -# RUN apk add chromium nss freetype harfbuzz ca-certificates +RUN apk add --no-cache \ + libc6-compat \ + nss \ + freetype \ + harfbuzz \ + ca-certificates + +# 判断是否需要安装 Chromium +ARG USE_PUPPETEER=false +RUN if [ "$USE_PUPPETEER" = "true" ]; then \ + apk add --no-cache chromium; \ + fi # 配置 Chromium -# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true -# ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser +# 清理缓存 +RUN rm -rf /var/cache/apk/* + +# 构建阶段 FROM base AS builder RUN npm install -g pnpm @@ -21,13 +35,16 @@ COPY public ./public # add .env.example to .env RUN [ ! -e ".env" ] && cp .env.example .env || true +RUN if [ "$USE_PUPPETEER" = "true" ]; then \ + sed -i 's/^USE_PUPPETEER=false/USE_PUPPETEER=true/' .env; \ + fi RUN pnpm install RUN pnpm build RUN pnpm prune --production +# 运行阶段 FROM base AS runner -WORKDIR /app # 创建用户和组 RUN addgroup --system --gid 114514 nodejs @@ -35,6 +52,7 @@ RUN adduser --system --uid 114514 hono # 创建日志目录 RUN mkdir -p /app/logs && chown -R hono:nodejs /app/logs +RUN ln -s /app/logs /logs # 复制文件 COPY --from=builder --chown=hono:nodejs /app/node_modules /app/node_modules diff --git a/README.md b/README.md index 398001f..c1c187d 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ | 虎扑 | 步行街热帖 | hupu | ![](https://img.shields.io/website.svg?label=hupu&url=https://api-hot.imsyy.top/hupu&cacheSeconds=7200) | | 爱范儿 | 快讯 | ifanr | ![](https://img.shields.io/website.svg?label=ifanr&url=https://api-hot.imsyy.top/ifanr&cacheSeconds=7200) | | 英雄联盟 | 更新公告 | lol | ![](https://img.shields.io/website.svg?label=lol&url=https://api-hot.imsyy.top/lol&cacheSeconds=7200) | +| 米游社 | 最新消息 | miyoushe | ![](https://img.shields.io/website.svg?label=miyoushe&url=https://api-hot.imsyy.top/miyoushe&cacheSeconds=7200) | | 原神 | 最新消息 | genshin | ![](https://img.shields.io/website.svg?label=genshin&url=https://api-hot.imsyy.top/genshin&cacheSeconds=7200) | | 崩坏3 | 最新动态 | honkai | ![](https://img.shields.io/website.svg?label=honkai&url=https://api-hot.imsyy.top/honkai&cacheSeconds=7200) | | 崩坏:星穹铁道 | 最新动态 | starrail | ![](https://img.shields.io/website.svg?label=starrail&url=https://api-hot.imsyy.top/starrail&cacheSeconds=7200) | @@ -83,6 +84,8 @@ 本项目支持 `Node.js` 调用,可在安装完成后调用 `serveHotApi` 来开启服务器 +> 该方式无法使用部分需要 Puppeteer 环境的接口 + ```bash pnpm add dailyhot-api ``` @@ -100,6 +103,8 @@ serveHotApi(3000); ## ⚙️ 部署 +由于部分接口无法通过接口调用等方式获取数据,故采用 `Puppeteer` 来实现,但由于使用后会造成 **内存占用过大或镜像过大**,可选择性开启,详情请参考下方说明。 + 具体使用说明可参考 [我的博客](https://blog.imsyy.top/posts/2024/0408),下方仅讲解基础操作: ### Docker 部署 @@ -111,8 +116,11 @@ serveHotApi(3000); ```bash # 构建 docker build -t dailyhot-api . +# 构建 Puppeteer 版 +docker build --build-arg USE_PUPPETEER=true -t dailyhot-api . + # 运行 -docker run -p 6688:6688 -d dailyhot-api +docker run --restart always -p 6688:6688 -d dailyhot-api # 或使用 Docker Compose docker-compose up -d ``` @@ -122,8 +130,11 @@ docker-compose up -d ```bash # 拉取 docker pull imsyy/dailyhot-api:latest +# 拉取 Puppeteer 版 +docker pull imsyy/dailyhot-api:web-latest + # 运行 -docker run -p 6688:6688 -d imsyy/dailyhot-api:latest +docker run --restart always -p 6688:6688 -d imsyy/dailyhot-api:latest ``` ### 手动部署 @@ -148,7 +159,10 @@ npm install #### 开发 ```bash +# 标准运行 npm run dev +# 采用 Puppeteer 运行 +npm run dev:web ``` 成功启动后程序会在控制台输出可访问的地址 @@ -157,7 +171,11 @@ npm run dev ```bash npm run build + +# 标准运行 npm run start +# 采用 Puppeteer 运行 +npm run start:web ``` 成功启动后程序会在控制台输出可访问的地址 diff --git a/package.json b/package.json index 1d56f6c..43d22a4 100644 --- a/package.json +++ b/package.json @@ -30,9 +30,11 @@ "format": "prettier --write .", "lint": "eslint .", "dev": "cross-env NODE_ENV=development tsx watch --no-cache src/index.ts", + "dev:web": "cross-env NODE_ENV=development USE_PUPPETEER=true tsx watch --no-cache src/index.ts", "dev:cache": "cross-env NODE_ENV=development tsx watch src/index.ts", "build": "tsc --project tsconfig.json", - "start": "cross-env NODE_ENV=development tsx dist/src/index.js" + "start": "cross-env NODE_ENV=development tsx dist/index.js", + "start:web": "cross-env NODE_ENV=development USE_PUPPETEER=true tsx dist/index.js" }, "type": "module", "dependencies": { @@ -46,6 +48,7 @@ "hono": "^4.6.12", "md5": "^2.3.0", "node-cache": "^5.1.2", + "puppeteer": "^23.10.0", "puppeteer-cluster": "^0.24.0", "rss-parser": "^3.13.0", "winston": "^3.17.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 487fc93..e08a49d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,9 +38,12 @@ importers: node-cache: specifier: ^5.1.2 version: 5.1.2 + puppeteer: + specifier: ^23.10.0 + version: 23.10.0(typescript@5.7.2) puppeteer-cluster: specifier: ^0.24.0 - version: 0.24.0(puppeteer@23.9.0(typescript@5.7.2)) + version: 0.24.0(puppeteer@23.10.0(typescript@5.7.2)) rss-parser: specifier: ^3.13.0 version: 3.13.0 @@ -324,8 +327,8 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} - '@puppeteer/browsers@2.4.1': - resolution: {integrity: sha512-0kdAbmic3J09I6dT8e9vE2JOCSt13wHCW5x/ly8TSt2bDtuIWe2TgLZZDHdcziw9AVCzflMAXCrVyRIhIs44Ng==} + '@puppeteer/browsers@2.5.0': + resolution: {integrity: sha512-6TQAc/5uRILE6deixJ1CR8rXyTbzXIXNgO1D0Woi9Bqicz2FV5iKP3BHYEg6o4UATCMcbQQ0jbmeaOkn/HQk2w==} engines: {node: '>=18'} hasBin: true @@ -1117,12 +1120,12 @@ packages: peerDependencies: puppeteer: '>=22.0.0' - puppeteer-core@23.9.0: - resolution: {integrity: sha512-hLVrav2HYMVdK0YILtfJwtnkBAwNOztUdR4aJ5YKDvgsbtagNr6urUJk9HyjRA9e+PaLI3jzJ0wM7A4jSZ7Qxw==} + puppeteer-core@23.10.0: + resolution: {integrity: sha512-7pv6kFget4Iki0RLBDowi35vaccz73XpC6/FAnfCrYa2IXVUlBHfZw+HGWzlvvTGwM9HHd32rgCrIDzil3kj+w==} engines: {node: '>=18'} - puppeteer@23.9.0: - resolution: {integrity: sha512-WfB8jGwFV+qrD9dcJJVvWPFJBU6kxeu2wxJz9WooDGfM3vIiKLgzImEDBxUQnCBK/2cXB3d4dV6gs/LLpgfLDg==} + puppeteer@23.10.0: + resolution: {integrity: sha512-vLpEvUM8POKBX4j6y/yyD4oGRhS1oBAbMn3lYpz1yeakhRsA8IhF5QmjXwAIQYGdR/INxyFiY6kQDoUtLGDP3g==} engines: {node: '>=18'} hasBin: true @@ -1552,7 +1555,7 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.17.1 - '@puppeteer/browsers@2.4.1': + '@puppeteer/browsers@2.5.0': dependencies: debug: 4.3.7 extract-zip: 2.0.1 @@ -2430,16 +2433,16 @@ snapshots: punycode@2.3.1: {} - puppeteer-cluster@0.24.0(puppeteer@23.9.0(typescript@5.7.2)): + puppeteer-cluster@0.24.0(puppeteer@23.10.0(typescript@5.7.2)): dependencies: debug: 4.3.7 - puppeteer: 23.9.0(typescript@5.7.2) + puppeteer: 23.10.0(typescript@5.7.2) transitivePeerDependencies: - supports-color - puppeteer-core@23.9.0: + puppeteer-core@23.10.0: dependencies: - '@puppeteer/browsers': 2.4.1 + '@puppeteer/browsers': 2.5.0 chromium-bidi: 0.8.0(devtools-protocol@0.0.1367902) debug: 4.3.7 devtools-protocol: 0.0.1367902 @@ -2450,13 +2453,13 @@ snapshots: - supports-color - utf-8-validate - puppeteer@23.9.0(typescript@5.7.2): + puppeteer@23.10.0(typescript@5.7.2): dependencies: - '@puppeteer/browsers': 2.4.1 + '@puppeteer/browsers': 2.5.0 chromium-bidi: 0.8.0(devtools-protocol@0.0.1367902) cosmiconfig: 9.0.0(typescript@5.7.2) devtools-protocol: 0.0.1367902 - puppeteer-core: 23.9.0 + puppeteer-core: 23.10.0 typed-query-selector: 2.12.0 transitivePeerDependencies: - bufferutil diff --git a/src/config.ts b/src/config.ts index 2d3e721..39b4af3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -12,6 +12,7 @@ export type Config = { ALLOWED_HOST: string; USE_LOG_FILE: boolean; RSS_MODE: boolean; + USE_PUPPETEER: boolean; }; // 验证并提取环境变量 @@ -49,4 +50,5 @@ export const config: Config = { ALLOWED_HOST: getEnvVariable("ALLOWED_HOST") || "imsyy.top", USE_LOG_FILE: getBooleanEnvVariable("USE_LOG_FILE", true), RSS_MODE: getBooleanEnvVariable("RSS_MODE", false), + USE_PUPPETEER: getBooleanEnvVariable("USE_PUPPETEER", false), }; diff --git a/src/index.ts b/src/index.ts index 458b05d..7d1fc20 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,5 @@ import { serve } from "@hono/node-server"; import { config } from "./config.js"; -import packageJson from "../package.json"; import logger from "./utils/logger.js"; import app from "./app.js"; @@ -11,8 +10,8 @@ const serveHotApi: (port?: number) => void = (port: number = config.PORT) => { fetch: app.fetch, port, }); - logger.info(`📦 Version: ${packageJson.version}`); logger.info(`🔥 DailyHot API 成功在端口 ${port} 上运行`); + logger.info(`💻 Puppeteer: ${config.USE_PUPPETEER}`); logger.info(`🔗 Local: 👉 http://localhost:${port}`); return apiServer; } catch (error) { diff --git a/src/routes/36kr.ts b/src/routes/36kr.ts index 273fc58..8561f5e 100644 --- a/src/routes/36kr.ts +++ b/src/routes/36kr.ts @@ -12,7 +12,7 @@ const typeMap: Record = { export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "hot"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "36kr", title: "36氪", @@ -24,10 +24,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://m.36kr.com/hot-list-m", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -59,8 +57,7 @@ const getList = async (options: Options, noCache: boolean): Promise)[type || "hot"]]; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["36kr"]) => { const item = v.templateMaterial; return { diff --git a/src/routes/51cto.ts b/src/routes/51cto.ts index 2dc2b08..311fa33 100644 --- a/src/routes/51cto.ts +++ b/src/routes/51cto.ts @@ -5,16 +5,14 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "51cto", title: "51CTO", type: "推荐榜", link: "https://www.51cto.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -41,8 +39,7 @@ const getList = async (noCache: boolean): Promise => { }); const list = result.data.data.data.list; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["51cto"]) => ({ id: v.source_id, title: v.title, diff --git a/src/routes/52pojie.ts b/src/routes/52pojie.ts index c9adc48..da504fd 100644 --- a/src/routes/52pojie.ts +++ b/src/routes/52pojie.ts @@ -5,7 +5,7 @@ import { getTime } from "../utils/getTime.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "hot"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "52pojie", title: "吾爱破解", @@ -22,10 +22,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://www.52pojie.cn/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData?.data?.length || 0, + ...listData, }; return routeData; }; @@ -47,8 +45,7 @@ const getList = async (options: Options, noCache: boolean): Promise ({ id: v.guid || i, title: v.title || "", diff --git a/src/routes/acfun.ts b/src/routes/acfun.ts index a38e960..0399700 100644 --- a/src/routes/acfun.ts +++ b/src/routes/acfun.ts @@ -27,7 +27,7 @@ const rangeMap: Record = { export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "-1"; const range = c.req.query("range") || "DAY"; - const { fromCache, data, updateTime } = await getList({ type, range }, noCache); + const listData = await getList({ type, range }, noCache); const routeData: RouterData = { name: "acfun", title: "AcFun", @@ -44,10 +44,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://www.acfun.cn/rank/list/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -64,8 +62,7 @@ const getList = async (options: Options, noCache: boolean): Promise ({ id: v.dougaId, title: v.contentTitle, diff --git a/src/routes/baidu.ts b/src/routes/baidu.ts index f8ca763..b6528ba 100644 --- a/src/routes/baidu.ts +++ b/src/routes/baidu.ts @@ -13,7 +13,7 @@ const typeMap: Record = { export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "realtime"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "baidu", title: "百度", @@ -25,10 +25,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://top.baidu.com/board", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -49,8 +47,7 @@ const getList = async (options: Options, noCache: boolean): Promise ({ id: v.index, title: v.word, diff --git a/src/routes/bilibili.ts b/src/routes/bilibili.ts index 18e9d7c..5305da5 100644 --- a/src/routes/bilibili.ts +++ b/src/routes/bilibili.ts @@ -22,7 +22,7 @@ const typeMap: Record = { export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "0"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "bilibili", title: "哔哩哔哩", @@ -35,10 +35,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://www.bilibili.com/v/popular/rank/all", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -89,8 +87,7 @@ const getList = async (options: Options, noCache: boolean): Promise ({ id: v.bvid, title: v.title, diff --git a/src/routes/coolapk.ts b/src/routes/coolapk.ts index 183d3c0..3497d3d 100644 --- a/src/routes/coolapk.ts +++ b/src/routes/coolapk.ts @@ -4,16 +4,14 @@ import { get } from "../utils/getData.js"; import { genHeaders } from "../utils/getToken/coolapk.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "coolapk", title: "酷安", type: "热榜", link: "https://www.coolapk.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -27,8 +25,7 @@ const getList = async (noCache: boolean) => { }); const list = result.data.data; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["coolapk"]) => ({ id: v.id, title: v.message, diff --git a/src/routes/csdn.ts b/src/routes/csdn.ts index c8e07aa..8386f38 100644 --- a/src/routes/csdn.ts +++ b/src/routes/csdn.ts @@ -4,17 +4,15 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "csdn", title: "CSDN", type: "排行榜", description: "专业开发者社区", link: "https://www.csdn.net/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -24,8 +22,7 @@ const getList = async (noCache: boolean): Promise => { const result = await get({ url, noCache }); const list = result.data.data; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["csdn"]) => ({ id: v.productId, title: v.articleTitle, diff --git a/src/routes/douban-group.ts b/src/routes/douban-group.ts index f9d5e88..c741f8b 100644 --- a/src/routes/douban-group.ts +++ b/src/routes/douban-group.ts @@ -4,16 +4,14 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "douban-group", title: "豆瓣讨论", type: "讨论精选", link: "https://www.douban.com/group/explore", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -50,8 +48,7 @@ const getList = async (noCache: boolean) => { }; }); return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: listData, }; }; diff --git a/src/routes/douban-movie.ts b/src/routes/douban-movie.ts index 25eff35..257f3d3 100644 --- a/src/routes/douban-movie.ts +++ b/src/routes/douban-movie.ts @@ -3,16 +3,14 @@ import { load } from "cheerio"; import { get } from "../utils/getData.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "douban-movie", title: "豆瓣电影", type: "新片榜", link: "https://movie.douban.com/chart", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -58,8 +56,7 @@ const getList = async (noCache: boolean) => { }; }); return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: listData, }; }; diff --git a/src/routes/douyin.ts b/src/routes/douyin.ts index b35dd02..3310c1c 100644 --- a/src/routes/douyin.ts +++ b/src/routes/douyin.ts @@ -4,17 +4,15 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "douyin", title: "抖音", type: "热榜", description: "实时上升热点", link: "https://www.douyin.com", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -47,8 +45,7 @@ const getList = async (noCache: boolean) => { }); const list = result.data.data.word_list; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["douyin"]) => ({ id: v.sentence_id, title: v.word, diff --git a/src/routes/earthquake.ts b/src/routes/earthquake.ts index 5f016ea..fa742a0 100644 --- a/src/routes/earthquake.ts +++ b/src/routes/earthquake.ts @@ -14,16 +14,14 @@ const mappings: Record = { }; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "earthquake", title: "中国地震台", type: "地震速报", link: "https://news.ceic.ac.cn/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -35,8 +33,7 @@ const getList = async (noCache: boolean) => { const match = result.data.match(regex); const list = match && match[1] ? JSON.parse(match[1]) : []; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["earthquake"]) => { const contentBuilder = []; const { NEW_DID, LOCATION_C, M } = v; diff --git a/src/routes/genshin.ts b/src/routes/genshin.ts index 36d7750..79d2738 100644 --- a/src/routes/genshin.ts +++ b/src/routes/genshin.ts @@ -5,7 +5,7 @@ import { getTime } from "../utils/getTime.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "1"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "genshin", title: "原神", @@ -21,10 +21,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://www.miyoushe.com/ys/home/28", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -35,8 +33,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.list; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["miyoushe"]) => { const data = v.post; return { diff --git a/src/routes/hellogithub.ts b/src/routes/hellogithub.ts index 2c34420..2719ff9 100644 --- a/src/routes/hellogithub.ts +++ b/src/routes/hellogithub.ts @@ -5,7 +5,7 @@ import { getTime } from "../utils/getTime.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const sort = c.req.query("sort") || "featured"; - const { fromCache, data, updateTime } = await getList({ sort }, noCache); + const listData = await getList({ sort }, noCache); const routeData: RouterData = { name: "hellogithub", title: "HelloGitHub", @@ -21,10 +21,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://hellogithub.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -35,8 +33,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["hellogithub"]) => ({ id: v.item_id, title: v.title, diff --git a/src/routes/history.ts b/src/routes/history.ts index 5e6263b..f180def 100644 --- a/src/routes/history.ts +++ b/src/routes/history.ts @@ -8,7 +8,7 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { // 获取日期 const day = c.req.query("day") || getCurrentDateTime(true).day; const month = c.req.query("month") || getCurrentDateTime(true).month; - const { fromCache, data, updateTime } = await getList({ month, day }, noCache); + const listData = await getList({ month, day }, noCache); const routeData: RouterData = { name: "history", title: "历史上的今天", @@ -18,10 +18,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { day: "日期", }, link: "https://baike.baidu.com/calendar", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -39,8 +37,7 @@ const getList = async (options: Options, noCache: boolean) => { }); const list = monthStr ? result.data[monthStr][monthStr + dayStr] : []; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["history"], index: number) => ({ id: index, title: load(v.title).text().trim(), diff --git a/src/routes/honkai.ts b/src/routes/honkai.ts index c0a2572..d6882a1 100644 --- a/src/routes/honkai.ts +++ b/src/routes/honkai.ts @@ -5,7 +5,7 @@ import { getTime } from "../utils/getTime.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "1"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "honkai", title: "崩坏3", @@ -21,10 +21,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://www.miyoushe.com/bh3/home/6", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -35,8 +33,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.list; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["miyoushe"]) => { const data = v.post; return { diff --git a/src/routes/hostloc.ts b/src/routes/hostloc.ts index 0b0ae38..1319cc4 100644 --- a/src/routes/hostloc.ts +++ b/src/routes/hostloc.ts @@ -12,7 +12,7 @@ const typeMap: Record = { export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "hot"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "hostloc", title: "全球主机交流", @@ -24,10 +24,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://hostloc.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -49,8 +47,7 @@ const getList = async (options: Options, noCache: boolean) => { }; const list = await parseData(); return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v, i) => ({ id: v.guid || i, title: v.title || "", diff --git a/src/routes/hupu.ts b/src/routes/hupu.ts index 7727309..0e44524 100644 --- a/src/routes/hupu.ts +++ b/src/routes/hupu.ts @@ -4,7 +4,7 @@ import { get } from "../utils/getData.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "1"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "hupu", title: "虎扑", @@ -22,10 +22,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://bbs.hupu.com/all-gambia", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -36,8 +34,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.topicThreads; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["hupu"]) => ({ id: v.tid, title: v.title, diff --git a/src/routes/huxiu.ts b/src/routes/huxiu.ts index 9a6a9bf..7a04b32 100644 --- a/src/routes/huxiu.ts +++ b/src/routes/huxiu.ts @@ -4,16 +4,14 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "huxiu", title: "虎嗅", type: "24小时", link: "https://www.huxiu.com/moment/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -38,8 +36,7 @@ const getList = async (noCache: boolean) => { const matchResult = result.data.match(pattern); const jsonObject = JSON.parse(matchResult[1]).moment.momentList.moment_list.datalist; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: jsonObject.map((v: RouterType["huxiu"]) => ({ id: v.object_id, title: titleProcessing(v.content).title, diff --git a/src/routes/ifanr.ts b/src/routes/ifanr.ts index 450e3a8..6ae8c65 100644 --- a/src/routes/ifanr.ts +++ b/src/routes/ifanr.ts @@ -4,17 +4,15 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "ifanr", title: "爱范儿", type: "快讯", description: "15秒了解全球新鲜事", link: "https://www.ifanr.com/digest/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -24,8 +22,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.objects; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["ifanr"]) => ({ id: v.id, title: v.post_title, diff --git a/src/routes/ithome-xijiayi.ts b/src/routes/ithome-xijiayi.ts index d960112..31d1180 100644 --- a/src/routes/ithome-xijiayi.ts +++ b/src/routes/ithome-xijiayi.ts @@ -4,17 +4,15 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "ithome-xijiayi", title: "IT之家「喜加一」", type: "最新动态", description: "最新最全的「喜加一」游戏动态尽在这里!", link: "https://www.ithome.com/zt/xijiayi", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -51,8 +49,7 @@ const getList = async (noCache: boolean) => { }; }); return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: listData, }; }; diff --git a/src/routes/ithome.ts b/src/routes/ithome.ts index 2a353a8..7c4f23b 100644 --- a/src/routes/ithome.ts +++ b/src/routes/ithome.ts @@ -4,17 +4,15 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "ithome", title: "IT之家", type: "热榜", description: "爱科技,爱这里 - 前沿科技新闻网站", link: "https://m.ithome.com/rankm/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -51,8 +49,7 @@ const getList = async (noCache: boolean) => { }; }); return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: listData, }; }; diff --git a/src/routes/jianshu.ts b/src/routes/jianshu.ts index f8ea5fa..482af82 100644 --- a/src/routes/jianshu.ts +++ b/src/routes/jianshu.ts @@ -3,17 +3,15 @@ import { load } from "cheerio"; import { get } from "../utils/getData.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "jianshu", title: "简书", type: "热门推荐", description: "一个优质的创作社区", link: "https://www.jianshu.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -52,8 +50,7 @@ const getList = async (noCache: boolean) => { }; }); return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: listData, }; }; diff --git a/src/routes/juejin.ts b/src/routes/juejin.ts index 7c73a01..0d9caf1 100644 --- a/src/routes/juejin.ts +++ b/src/routes/juejin.ts @@ -3,16 +3,14 @@ import type { RouterType } from "../router.types.js"; import { get } from "../utils/getData.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "juejin", title: "稀土掘金", type: "文章榜", link: "https://juejin.cn/hot/articles", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -22,8 +20,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["juejin"]) => ({ id: v.content.content_id, title: v.content.title, diff --git a/src/routes/lol.ts b/src/routes/lol.ts index 491bb9e..46ce7a8 100644 --- a/src/routes/lol.ts +++ b/src/routes/lol.ts @@ -4,16 +4,14 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "lol", title: "英雄联盟", type: "更新公告", link: "https://lol.qq.com/gicp/news/423/2/1334/1.html", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -24,8 +22,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.result; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["lol"]) => ({ id: v.iDocID, title: v.sTitle, diff --git a/src/routes/miyoushe.ts b/src/routes/miyoushe.ts index e601eb2..2d4b1fe 100644 --- a/src/routes/miyoushe.ts +++ b/src/routes/miyoushe.ts @@ -25,7 +25,7 @@ const typeMap: Record = { export const handleRoute = async (c: ListContext, noCache: boolean) => { const game = c.req.query("game") || "1"; const type = c.req.query("type") || "1"; - const { fromCache, data, updateTime } = await getList({ game, type }, noCache); + const listData = await getList({ game, type }, noCache); const routeData: RouterData = { name: "miyoushe", title: `米游社 · ${gameMap[game]}`, @@ -41,10 +41,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://www.miyoushe.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -55,8 +53,7 @@ const getList = async (options: Options, noCache: boolean): Promise { const data = v.post; return { diff --git a/src/routes/netease-news.ts b/src/routes/netease-news.ts index 4f85010..db30210 100644 --- a/src/routes/netease-news.ts +++ b/src/routes/netease-news.ts @@ -4,16 +4,14 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "netease-news", title: "网易新闻", type: "热点榜", link: "https://m.163.com/hot", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -23,8 +21,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.list; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["netease-news"]) => ({ id: v.docid, title: v.title, diff --git a/src/routes/ngabbs.ts b/src/routes/ngabbs.ts index 135ed1a..dabfbb9 100644 --- a/src/routes/ngabbs.ts +++ b/src/routes/ngabbs.ts @@ -4,17 +4,15 @@ import { post } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "ngabbs", title: "NGA", type: "论坛热帖", description: "精英玩家俱乐部", link: "https://ngabbs.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -42,8 +40,7 @@ const getList = async (noCache: boolean) => { }); const list = result.data.result[0]; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["ngabbs"]) => ({ id: v.tid, title: v.subject, diff --git a/src/routes/nodeseek.ts b/src/routes/nodeseek.ts index e6aaf50..2301a04 100644 --- a/src/routes/nodeseek.ts +++ b/src/routes/nodeseek.ts @@ -5,7 +5,7 @@ import { parseStringPromise } from "xml2js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "nodeseek", title: "NodeSeek", @@ -19,10 +19,8 @@ export const handleRoute = async (_: undefined, noCache: boolean) => { }, }, link: "https://www.nodeseek.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -33,8 +31,7 @@ const getList = async (noCache: boolean) => { const rssData = await parseStringPromise(result.data); const list = rssData.rss.channel[0].item; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["nodeseek"]) => ({ id: v.guid[0]._, title: v.title[0], diff --git a/src/routes/qq-news.ts b/src/routes/qq-news.ts index 7fcd960..8db35f2 100644 --- a/src/routes/qq-news.ts +++ b/src/routes/qq-news.ts @@ -4,16 +4,14 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "qq-news", title: "腾讯新闻", type: "热点榜", link: "https://news.qq.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -23,8 +21,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.idlist[0].newslist.slice(1); return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["qq-news"]) => ({ id: v.id, title: v.title, diff --git a/src/routes/sina-news.ts b/src/routes/sina-news.ts index 6734fa8..e5fd3a3 100644 --- a/src/routes/sina-news.ts +++ b/src/routes/sina-news.ts @@ -64,7 +64,7 @@ const listType = { export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "1"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "sina-news", title: "新浪新闻", @@ -76,10 +76,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://sinanews.sina.cn/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -122,8 +120,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = parseData(result.data).data; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["sina-news"]) => ({ id: v.id, title: v.title, diff --git a/src/routes/sina.ts b/src/routes/sina.ts index 24edb95..a00e23e 100644 --- a/src/routes/sina.ts +++ b/src/routes/sina.ts @@ -5,7 +5,7 @@ import { get } from "../utils/getData.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "1"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "sina", title: "新浪网", @@ -28,10 +28,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://sinanews.sina.cn/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -42,8 +40,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.hotList; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["sina"]) => { const base = v.base; const info = v.info; diff --git a/src/routes/sspai.ts b/src/routes/sspai.ts index 58f1344..208302c 100644 --- a/src/routes/sspai.ts +++ b/src/routes/sspai.ts @@ -5,7 +5,7 @@ import { getTime } from "../utils/getTime.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "热门文章"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "sspai", title: "少数派", @@ -17,10 +17,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://sspai.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -31,8 +29,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["sspai"]) => ({ id: v.id, title: v.title, diff --git a/src/routes/starrail.ts b/src/routes/starrail.ts index 160953c..8bb8ce2 100644 --- a/src/routes/starrail.ts +++ b/src/routes/starrail.ts @@ -5,7 +5,7 @@ import { getTime } from "../utils/getTime.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "1"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "starrail", title: "崩坏:星穹铁道", @@ -21,10 +21,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://www.miyoushe.com/sr/home/53", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -35,8 +33,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.list; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["miyoushe"]) => { const data = v.post; return { diff --git a/src/routes/thepaper.ts b/src/routes/thepaper.ts index b050a57..da77ec5 100644 --- a/src/routes/thepaper.ts +++ b/src/routes/thepaper.ts @@ -4,16 +4,14 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "thepaper", title: "澎湃新闻", type: "热榜", link: "https://www.thepaper.cn/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -23,8 +21,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.hotNews; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["thepaper"]) => ({ id: v.contId, title: v.name, diff --git a/src/routes/tieba.ts b/src/routes/tieba.ts index 4c25a4f..0c06af7 100644 --- a/src/routes/tieba.ts +++ b/src/routes/tieba.ts @@ -4,17 +4,15 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "tieba", title: "百度贴吧", type: "热议榜", description: "全球领先的中文社区", link: "https://tieba.baidu.com/hottopic/browse/topicList", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -24,8 +22,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.bang_topic.topic_list; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["tieba"]) => ({ id: v.topic_id, title: v.topic_name, diff --git a/src/routes/toutiao.ts b/src/routes/toutiao.ts index 98adae7..d4ad6df 100644 --- a/src/routes/toutiao.ts +++ b/src/routes/toutiao.ts @@ -4,16 +4,14 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "toutiao", title: "今日头条", type: "热榜", link: "https://www.toutiao.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -23,8 +21,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["toutiao"]) => ({ id: v.ClusterIdStr, title: v.Title, diff --git a/src/routes/v2ex.ts b/src/routes/v2ex.ts index c67b6c6..2d97c87 100644 --- a/src/routes/v2ex.ts +++ b/src/routes/v2ex.ts @@ -4,7 +4,7 @@ import { get } from "../utils/getData.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const type = c.req.query("type") || "hot"; - const { fromCache, data, updateTime } = await getList({ type }, noCache); + const listData = await getList({ type }, noCache); const routeData: RouterData = { name: "v2ex", title: "V2EX", @@ -19,10 +19,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "https://www.v2ex.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -33,8 +31,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["v2ex"]) => ({ id: v.id, title: v.title, diff --git a/src/routes/weatheralarm.ts b/src/routes/weatheralarm.ts index caf7e96..ac3b951 100644 --- a/src/routes/weatheralarm.ts +++ b/src/routes/weatheralarm.ts @@ -5,11 +5,11 @@ import { getTime } from "../utils/getTime.js"; export const handleRoute = async (c: ListContext, noCache: boolean) => { const province = c.req.query("province") || ""; - const { fromCache, data, type, updateTime } = await getList({ province }, noCache); + const listData = await getList({ province }, noCache); const routeData: RouterData = { name: "weatheralarm", title: "中央气象台", - type: type || "全国气象预警", + type: `${province || "全国"}气象预警`, params: { province: { name: "预警区域", @@ -17,10 +17,8 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => { }, }, link: "http://nmc.cn/publish/alarm.html", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -31,9 +29,7 @@ const getList = async (options: Options, noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data.page.list; return { - fromCache: result.fromCache, - updateTime: result.updateTime, - type: province + "气象预警", + ...result, data: list.map((v: RouterType["weatheralarm"]) => ({ id: v.alertid, title: v.title, diff --git a/src/routes/weibo.ts b/src/routes/weibo.ts index ac3399a..4c0f37e 100644 --- a/src/routes/weibo.ts +++ b/src/routes/weibo.ts @@ -4,17 +4,15 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "weibo", title: "微博", type: "热搜榜", description: "实时热点,每分钟更新一次", link: "https://s.weibo.com/top/summary/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -24,8 +22,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache, ttl: 60 }); const list = result.data.data.realtime; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["weibo"]) => { const key = v.word_scheme ? v.word_scheme : `#${v.word}`; return { diff --git a/src/routes/weread.ts b/src/routes/weread.ts index 375e29d..0bce300 100644 --- a/src/routes/weread.ts +++ b/src/routes/weread.ts @@ -5,16 +5,14 @@ import getWereadID from "../utils/getToken/weread.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "weread", title: "微信读书", type: "飙升榜", link: "https://weread.qq.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -31,8 +29,7 @@ const getList = async (noCache: boolean) => { }); const list = result.data.books; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["weread"]) => { const data = v.bookInfo; return { diff --git a/src/routes/zhihu-daily.ts b/src/routes/zhihu-daily.ts index e64311b..d51cf31 100644 --- a/src/routes/zhihu-daily.ts +++ b/src/routes/zhihu-daily.ts @@ -3,17 +3,15 @@ import type { RouterType } from "../router.types.js"; import { get } from "../utils/getData.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "zhihu-daily", title: "知乎日报", type: "推荐榜", description: "每天三次,每次七分钟", link: "https://daily.zhihu.com/", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -30,8 +28,7 @@ const getList = async (noCache: boolean) => { }); const list = result.data.stories.filter((el: RouterType["zhihu-daily"]) => el.type === 0); return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["zhihu-daily"]) => ({ id: v.id, title: v.title, diff --git a/src/routes/zhihu.ts b/src/routes/zhihu.ts index 04e2f25..05ff792 100644 --- a/src/routes/zhihu.ts +++ b/src/routes/zhihu.ts @@ -4,16 +4,14 @@ import { get } from "../utils/getData.js"; import { getTime } from "../utils/getTime.js"; export const handleRoute = async (_: undefined, noCache: boolean) => { - const { fromCache, data, updateTime } = await getList(noCache); + const listData = await getList(noCache); const routeData: RouterData = { name: "zhihu", title: "知乎", type: "热榜", link: "https://www.zhihu.com/hot", - total: data?.length || 0, - updateTime, - fromCache, - data, + total: listData.data?.length || 0, + ...listData, }; return routeData; }; @@ -23,8 +21,7 @@ const getList = async (noCache: boolean) => { const result = await get({ url, noCache }); const list = result.data.data; return { - fromCache: result.fromCache, - updateTime: result.updateTime, + ...result, data: list.map((v: RouterType["zhihu"]) => { const data = v.target; return { diff --git a/src/types.d.ts b/src/types.d.ts index 0973798..3a1b0ea 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -18,9 +18,10 @@ export interface ListItem { // 路由接口数据 export interface RouterResType { - updateTime: string; + updateTime: string | number; fromCache: boolean; data: ListItem[]; + message?: string; } // 路由数据 diff --git a/src/utils/getData.ts b/src/utils/getData.ts index 2cd5850..05137db 100644 --- a/src/utils/getData.ts +++ b/src/utils/getData.ts @@ -1,4 +1,5 @@ -import type { Get, Post, Web } from "../types.ts"; +import type { Get, Post, Web } from "../types.js"; +import type { Page } from "puppeteer"; import { config } from "../config.js"; import { getCache, setCache, delCache } from "./cache.js"; import { Cluster } from "puppeteer-cluster"; @@ -12,24 +13,50 @@ const request = axios.create({ withCredentials: true, }); -// puppeteer-cluster -export const createCluster = async () => { - return await Cluster.launch({ - concurrency: Cluster.CONCURRENCY_BROWSER, - maxConcurrency: 5, - // puppeteer - puppeteerOptions: { - headless: true, - args: ["--no-sandbox", "--disable-setuid-sandbox"], - }, - }); +// Puppeteer Cluster +let cluster: Cluster | null = null; + +/** + * 创建 Puppeteer Cluster + * @returns {Promise} Puppeteer Cluster + */ +export const createCluster = async (): Promise => { + if (cluster) return cluster; + try { + cluster = await Cluster.launch({ + concurrency: Cluster.CONCURRENCY_BROWSER, + maxConcurrency: 5, + puppeteerOptions: { + headless: true, + args: [ + "--no-sandbox", + "--disable-gpu", + "--disable-setuid-sandbox", + "--disable-dev-shm-usage", + ], + }, + }); + logger.info("Puppeteer Cluster 已成功初始化"); + } catch (error) { + logger.warn("启动 Puppeteer Cluster 失败,可能是没有 Chromium 安装", error); + cluster = null; + } + return cluster; }; -// Cluster -const cluster = await createCluster(); - -// Cluster configuration -cluster.task(async ({ page, data: { url, userAgent } }) => { +/** + * 处理任务 + * @param page 页面 + * @param data 数据 + * @returns + */ +const taskHandler = async ({ + page, + data: { url, userAgent }, +}: { + page: Page; + data: { url: string; userAgent?: string }; +}): Promise => { // 用户代理 if (userAgent) await page.setUserAgent(userAgent); // 请求拦截 @@ -45,9 +72,9 @@ cluster.task(async ({ page, data: { url, userAgent } }) => { }); // 加载页面 await page.goto(url, { waitUntil: "networkidle0", timeout: config.REQUEST_TIMEOUT }); - const pageContent = await page.content(); - return pageContent; -}); + // 返回页面内容 + return page.content(); +}; // 请求拦截 request.interceptors.request.use( @@ -137,12 +164,25 @@ export const post = async (options: Post) => { } }; -// puppeteer +// WEB - Puppeteer export const web = async (options: Web) => { const { url, noCache, ttl = config.CACHE_TTL, userAgent } = options; logger.info("使用 Puppeteer 发起页面请求", options); + if (config.USE_PUPPETEER === false) { + return { + fromCache: false, + data: [], + message: "Puppeteer is not enabled, please set USE_PUPPETEER=true", + updateTime: 0, + }; + } + // 初始化 Cluster + const clusterInstance = await createCluster(); + if (!clusterInstance) { + logger.error("Cluster 初始化失败,无法继续"); + throw new Error("Cluster 初始化失败"); + } try { - if (!cluster) throw new Error("Cluster is not initialized"); // 检查缓存 if (noCache) { delCache(url); @@ -155,7 +195,7 @@ export const web = async (options: Web) => { } // 缓存不存在时使用 Puppeteer 请求页面 logger.info("启动浏览器请求页面", { url }); - const pageContent = await cluster.execute({ url, userAgent }); + const pageContent = await clusterInstance.execute({ url, userAgent }, taskHandler); // 存储新获取的数据到缓存 const updateTime = new Date().toISOString(); setCache(url, { data: pageContent, updateTime }, ttl); @@ -167,3 +207,17 @@ export const web = async (options: Web) => { throw error; } }; + +/** + * 关闭 Cluster + */ +export const closeCluster = async () => { + if (cluster) { + try { + await cluster.close(); + logger.info("Puppeteer Cluster 已成功关闭"); + } catch (error) { + logger.error("关闭 Puppeteer Cluster 时出错", error); + } + } +}; diff --git a/src/utils/parseRSS.ts b/src/utils/parseRSS.ts index 7f9cedc..184ccae 100644 --- a/src/utils/parseRSS.ts +++ b/src/utils/parseRSS.ts @@ -25,8 +25,8 @@ export const parseRSS = async (rssContent: string) => { try { new URL(url); return true; + // eslint-disable-next-line @typescript-eslint/no-unused-vars } catch (error) { - console.error(error); return false; } };