feat: 新增 吾爱破解、果壳、快手

This commit is contained in:
imsyy
2024-12-05 15:28:28 +08:00
parent f38d264366
commit 161b6b612b
22 changed files with 288 additions and 1050 deletions

View File

@@ -21,7 +21,4 @@ REQUEST_TIMEOUT=6000
USE_LOG_FILE=true
# RSS Mode
RSS_MODE=false
# Puppeteer
USE_PUPPETEER=false
RSS_MODE=false

View File

@@ -43,7 +43,7 @@ jobs:
imsyy/dailyhot-api
ghcr.io/${{ github.repository }}
- name: Build and push regular image (no Puppeteer)
- name: Build and push regular image
uses: docker/build-push-action@v5
with:
context: .
@@ -54,18 +54,3 @@ jobs:
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
- name: Build and push Puppeteer image (with Puppeteer)
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
platforms: |
linux/amd64
linux/arm64
push: true
tags: imsyy/dailyhot-api:web-latest
build-args: |
USE_PUPPETEER=true
labels: ${{ steps.meta.outputs.labels }}

View File

@@ -2,24 +2,6 @@ FROM node:20-alpine AS base
ENV NODE_ENV=docker
# 安装 Puppeteer 所需的依赖库
RUN apk add --no-cache \
libc6-compat \
nss \
freetype \
harfbuzz \
ca-certificates
# 判断是否需要安装 Chromium
ARG USE_PUPPETEER=false
RUN if [ "$USE_PUPPETEER" = "true" ]; then \
apk add --no-cache chromium; \
fi
# 配置 Chromium
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# 清理缓存
RUN rm -rf /var/cache/apk/*
@@ -33,12 +15,10 @@ COPY package*json tsconfig.json pnpm-lock.yaml .env.example ./
COPY src ./src
COPY public ./public
# add .env.example to .env
# 复制环境变量
RUN [ ! -e ".env" ] && cp .env.example .env || true
RUN if [ "$USE_PUPPETEER" = "true" ]; then \
sed -i 's/^USE_PUPPETEER=false/USE_PUPPETEER=true/' .env; \
fi
# 安装依赖
RUN pnpm install
RUN pnpm build
RUN pnpm prune --production

View File

@@ -30,7 +30,7 @@
<details>
<summary>查看全部接口</summary>
> 例站点运行于海外服务器,部分国内站点可能存在访问异常,请以实际情况为准
> 例站点运行于海外服务器,部分国内站点可能存在访问异常,请以实际情况为准
| **站点** | **类别** | **调用名称** | **状态** |
| ---------------- | ------------ | -------------- | --------------------------------------------------------------------------------------------------------------------------- |
@@ -41,6 +41,7 @@
| 知乎日报 | 推荐榜 | zhihu-daily | ![](https://img.shields.io/website.svg?label=zhihu-daily&url=https://api-hot.imsyy.top/zhihu-daily&cacheSeconds=7200) |
| 百度 | 热搜榜 | baidu | ![](https://img.shields.io/website.svg?label=baidu&url=https://api-hot.imsyy.top/baidu&cacheSeconds=7200) |
| 抖音 | 热点榜 | douyin | ![](https://img.shields.io/website.svg?label=douyin&url=https://api-hot.imsyy.top/douyin&cacheSeconds=7200) |
| 快手 | 热点榜 | kuaishou | ![](https://img.shields.io/website.svg?label=kuaishou&url=https://api-hot.imsyy.top/kuaishou&cacheSeconds=7200) |
| 豆瓣电影 | 新片榜 | douban-movie | ![](https://img.shields.io/website.svg?label=douban-movie&url=https://api-hot.imsyy.top/douban-movie&cacheSeconds=7200) |
| 豆瓣讨论小组 | 讨论精选 | douban-group | ![](https://img.shields.io/website.svg?label=douban-group&url=https://api-hot.imsyy.top/douban-group&cacheSeconds=7200) |
| 百度贴吧 | 热议榜 | tieba | ![](https://img.shields.io/website.svg?label=tieba&url=https://api-hot.imsyy.top/tieba&cacheSeconds=7200) |
@@ -48,6 +49,7 @@
| IT之家 | 热榜 | ithome | ![](https://img.shields.io/website.svg?label=ithome&url=https://api-hot.imsyy.top/ithome&cacheSeconds=7200) |
| IT之家「喜加一」 | 最新动态 | ithome-xijiayi | ![](https://img.shields.io/website.svg?label=ithome-xijiayi&url=https://api-hot.imsyy.top/ithome-xijiayi&cacheSeconds=7200) |
| 简书 | 热门推荐 | jianshu | ![](https://img.shields.io/website.svg?label=jianshu&url=https://api-hot.imsyy.top/jianshu&cacheSeconds=7200) |
| 果壳 | 热门文章 | guokr | ![](https://img.shields.io/website.svg?label=guokr&url=https://api-hot.imsyy.top/guokr&cacheSeconds=7200) |
| 澎湃新闻 | 热榜 | thepaper | ![](https://img.shields.io/website.svg?label=thepaper&url=https://api-hot.imsyy.top/thepaper&cacheSeconds=7200) |
| 今日头条 | 热榜 | toutiao | ![](https://img.shields.io/website.svg?label=toutiao&url=https://api-hot.imsyy.top/toutiao&cacheSeconds=7200) |
| 36 氪 | 热榜 | 36kr | ![](https://img.shields.io/website.svg?label=36kr&url=https://api-hot.imsyy.top/36kr&cacheSeconds=7200) |
@@ -66,7 +68,7 @@
| 虎扑 | 步行街热帖 | hupu | ![](https://img.shields.io/website.svg?label=hupu&url=https://api-hot.imsyy.top/hupu&cacheSeconds=7200) |
| 爱范儿 | 快讯 | ifanr | ![](https://img.shields.io/website.svg?label=ifanr&url=https://api-hot.imsyy.top/ifanr&cacheSeconds=7200) |
| 英雄联盟 | 更新公告 | lol | ![](https://img.shields.io/website.svg?label=lol&url=https://api-hot.imsyy.top/lol&cacheSeconds=7200) |
| 米游社 | 最新消息 | miyoushe | ![](https://img.shields.io/website.svg?label=miyoushe&url=https://api-hot.imsyy.top/miyoushe&cacheSeconds=7200) |
| 米游社 | 最新消息 | miyoushe | ![](https://img.shields.io/website.svg?label=miyoushe&url=https://api-hot.imsyy.top/miyoushe&cacheSeconds=7200) |
| 原神 | 最新消息 | genshin | ![](https://img.shields.io/website.svg?label=genshin&url=https://api-hot.imsyy.top/genshin&cacheSeconds=7200) |
| 崩坏3 | 最新动态 | honkai | ![](https://img.shields.io/website.svg?label=honkai&url=https://api-hot.imsyy.top/honkai&cacheSeconds=7200) |
| 崩坏:星穹铁道 | 最新动态 | starrail | ![](https://img.shields.io/website.svg?label=starrail&url=https://api-hot.imsyy.top/starrail&cacheSeconds=7200) |
@@ -103,8 +105,6 @@ serveHotApi(3000);
## ⚙️ 部署
由于部分接口无法通过接口调用等方式获取数据,故采用 `Puppeteer` 来实现,但由于使用后会造成 **内存占用过大或镜像过大**,可选择性开启,详情请参考下方说明。
具体使用说明可参考 [我的博客](https://blog.imsyy.top/posts/2024/0408),下方仅讲解基础操作:
### Docker 部署
@@ -116,8 +116,6 @@ serveHotApi(3000);
```bash
# 构建
docker build -t dailyhot-api .
# 构建 Puppeteer 版
docker build --build-arg USE_PUPPETEER=true -t dailyhot-api .
# 运行
docker run --restart always -p 6688:6688 -d dailyhot-api
@@ -130,8 +128,6 @@ docker-compose up -d
```bash
# 拉取
docker pull imsyy/dailyhot-api:latest
# 拉取 Puppeteer 版
docker pull imsyy/dailyhot-api:web-latest
# 运行
docker run --restart always -p 6688:6688 -d imsyy/dailyhot-api:latest
@@ -139,7 +135,7 @@ docker run --restart always -p 6688:6688 -d imsyy/dailyhot-api:latest
### 手动部署
最直接的方式,您可以按照以下步骤将 DailyHotApi 部署在您的电脑、服务器或者其他任何地方
最直接的方式,您可以按照以下步骤将 `DailyHotApi` 部署在您的电脑、服务器或者其他任何地方
#### 安装
@@ -159,10 +155,7 @@ npm install
#### 开发
```bash
# 标准运行
npm run dev
# 采用 Puppeteer 运行
npm run dev:web
```
成功启动后程序会在控制台输出可访问的地址
@@ -171,11 +164,7 @@ npm run dev:web
```bash
npm run build
# 标准运行
npm run start
# 采用 Puppeteer 运行
npm run start:web
```
成功启动后程序会在控制台输出可访问的地址

View File

@@ -30,11 +30,9 @@
"format": "prettier --write .",
"lint": "eslint .",
"dev": "cross-env NODE_ENV=development tsx watch --no-cache src/index.ts",
"dev:web": "cross-env NODE_ENV=development USE_PUPPETEER=true tsx watch --no-cache src/index.ts",
"dev:cache": "cross-env NODE_ENV=development tsx watch src/index.ts",
"build": "tsc --project tsconfig.json",
"start": "cross-env NODE_ENV=development tsx dist/index.js",
"start:web": "cross-env NODE_ENV=development USE_PUPPETEER=true tsx dist/index.js"
"start": "cross-env NODE_ENV=development tsx dist/index.js"
},
"type": "module",
"dependencies": {
@@ -46,18 +44,18 @@
"dotenv": "^16.4.6",
"feed": "^4.2.2",
"hono": "^4.6.12",
"iconv-lite": "^0.6.3",
"md5": "^2.3.0",
"node-cache": "^5.1.2",
"puppeteer": "^23.10.0",
"puppeteer-cluster": "^0.24.0",
"rss-parser": "^3.13.0",
"winston": "^3.17.0",
"xml2js": "^0.6.2"
"user-agents": "^1.1.379",
"winston": "^3.17.0"
},
"devDependencies": {
"@eslint/js": "^9.16.0",
"@types/md5": "^2.3.5",
"@types/node": "^22.10.1",
"@types/user-agents": "^1.0.4",
"@types/xml2js": "^0.4.14",
"@typescript-eslint/eslint-plugin": "^8.17.0",
"@typescript-eslint/parser": "^8.17.0",

791
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -59,7 +59,7 @@ app.get("/", (c) => c.html(<Home />));
app.notFound((c) => c.html(<NotFound />, 404));
// error
app.onError((err, c) => {
logger.error(`出现致命错误:${err}`);
logger.error(`❌ [ERROR] ${err?.message}`);
return c.html(<Error error={err?.message} />, 500);
});

View File

@@ -12,15 +12,12 @@ export type Config = {
ALLOWED_HOST: string;
USE_LOG_FILE: boolean;
RSS_MODE: boolean;
USE_PUPPETEER: boolean;
};
// 验证并提取环境变量
const getEnvVariable = (key: string): string | undefined => {
const value = process.env[key];
if (value === undefined) {
return undefined;
}
if (value === undefined) return undefined;
return value;
};
@@ -28,9 +25,7 @@ const getEnvVariable = (key: string): string | undefined => {
const getNumericEnvVariable = (key: string, defaultValue: number): number => {
const value = getEnvVariable(key) ?? String(defaultValue);
const parsedValue = parseInt(value, 10);
if (isNaN(parsedValue)) {
return defaultValue;
}
if (isNaN(parsedValue)) return defaultValue;
return parsedValue;
};
@@ -45,10 +40,9 @@ export const config: Config = {
PORT: getNumericEnvVariable("PORT", 6688),
DISALLOW_ROBOT: getBooleanEnvVariable("DISALLOW_ROBOT", true),
CACHE_TTL: getNumericEnvVariable("CACHE_TTL", 3600),
REQUEST_TIMEOUT: getNumericEnvVariable("CACHE_TTL", 6000),
REQUEST_TIMEOUT: getNumericEnvVariable("REQUEST_TIMEOUT", 6000),
ALLOWED_DOMAIN: getEnvVariable("ALLOWED_DOMAIN") || "*",
ALLOWED_HOST: getEnvVariable("ALLOWED_HOST") || "imsyy.top",
USE_LOG_FILE: getBooleanEnvVariable("USE_LOG_FILE", true),
RSS_MODE: getBooleanEnvVariable("RSS_MODE", false),
USE_PUPPETEER: getBooleanEnvVariable("USE_PUPPETEER", false),
};

View File

@@ -10,8 +10,7 @@ const serveHotApi: (port?: number) => void = (port: number = config.PORT) => {
fetch: app.fetch,
port,
});
logger.info(`🔥 DailyHot API 成功在端口 ${port} 上运行`);
logger.info(`💻 Puppeteer: ${config.USE_PUPPETEER}`);
logger.info(`🔥 DailyHot API successfully runs on port ${port}`);
logger.info(`🔗 Local: 👉 http://localhost:${port}`);
return apiServer;
} catch (error) {

33
src/router.types.d.ts vendored
View File

@@ -315,14 +315,31 @@ export type RouterType = {
message: string;
replynum: number;
};
nodeseek: {
guid: {
_: string;
}[];
xueqiu: {
pic: string;
tag: string;
id: number;
hot: number;
content: string;
};
guokr: {
id: number;
title: string;
description: string | string[];
"dc:creator": string;
pubDate: string[];
link: string[];
summary: string;
author: {
nickname: string;
};
date_modified: string;
small_image: string;
};
kuaishou: {
id: string;
name: string;
hotValue: string;
iconUrl: string;
poster:string;
photoIds: {
json: string[];
};
};
};

View File

@@ -1,24 +1,27 @@
import type { RouterData, ListContext, Options, RouterResType } from "../types.js";
import { web } from "../utils/getData.js";
import { extractRss, parseRSS } from "../utils/parseRSS.js";
import { get } from "../utils/getData.js";
import { getTime } from "../utils/getTime.js";
import { parseRSS } from "../utils/parseRSS.js";
import iconv from "iconv-lite";
const typeMap: Record<string, string> = {
digest: "最新精华",
hot: "最新热门",
new: "最新回复",
newthread: "最新发表",
};
export const handleRoute = async (c: ListContext, noCache: boolean) => {
const type = c.req.query("type") || "hot";
const type = c.req.query("type") || "digest";
const listData = await getList({ type }, noCache);
const routeData: RouterData = {
name: "52pojie",
title: "吾爱破解",
type: "榜单",
type: typeMap[type],
params: {
type: {
name: "榜单分类",
type: {
tech: "新鲜出炉",
newthread: "技术分享",
hot: "人气热门",
digest: "精华采撷",
},
type: typeMap,
},
},
link: "https://www.52pojie.cn/",
@@ -31,28 +34,27 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => {
const getList = async (options: Options, noCache: boolean): Promise<RouterResType> => {
const { type } = options;
const url = `https://www.52pojie.cn/forum.php?mod=guide&view=${type}&rss=1`;
const result = await web({
const result = await get({
url,
noCache,
userAgent:
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
responseType: "arraybuffer",
headers: {
userAgent:
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
},
});
const parseData = async () => {
if (typeof result?.data !== "string") return [];
const rssContent = extractRss(result.data);
if (!rssContent) return [];
return await parseRSS(rssContent);
};
const list = await parseData();
// 转码
const utf8Data = iconv.decode(result.data, "gbk");
const list = await parseRSS(utf8Data);
return {
...result,
data: list.map((v, i) => ({
id: v.guid || i,
title: v.title || "",
desc: v.content || "",
author: v.author || "",
desc: v.content?.trim() || "",
author: v.author,
timestamp: getTime(v.pubDate || 0),
hot: 0,
hot: undefined,
url: v.link || "",
mobileUrl: v.link || "",
})),

45
src/routes/guokr.ts Normal file
View File

@@ -0,0 +1,45 @@
import type { RouterData } from "../types.js";
import type { RouterType } from "../router.types.js";
import { get } from "../utils/getData.js";
import { getTime } from "../utils/getTime.js";
export const handleRoute = async (_: undefined, noCache: boolean) => {
const listData = await getList(noCache);
const routeData: RouterData = {
name: "guokr",
title: "果壳",
type: "热门文章",
description: "科技有意思",
link: "https://www.guokr.com/",
total: listData.data?.length || 0,
...listData,
};
return routeData;
};
const getList = async (noCache: boolean) => {
const url = `https://www.guokr.com/beta/proxy/science_api/articles?limit=30`;
const result = await get({
url,
noCache,
headers: {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
},
});
const list = result.data;
return {
...result,
data: list.map((v: RouterType["guokr"]) => ({
id: v.id,
title: v.title,
desc: v.summary,
cover: v.small_image,
author: v.author?.nickname,
hot: undefined,
timestamp: getTime(v.date_modified),
url: `https://www.guokr.com/article/${v.id}`,
mobileUrl: `https://m.guokr.com/article/${v.id}`,
})),
};
};

View File

@@ -1,6 +1,6 @@
import type { RouterData, ListContext, Options } from "../types.js";
import { web } from "../utils/getData.js";
import { extractRss, parseRSS } from "../utils/parseRSS.js";
import { get } from "../utils/getData.js";
import { parseRSS } from "../utils/parseRSS.js";
import { getTime } from "../utils/getTime.js";
const typeMap: Record<string, string> = {
@@ -33,19 +33,15 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => {
const getList = async (options: Options, noCache: boolean) => {
const { type } = options;
const url = `https://hostloc.com/forum.php?mod=guide&view=${type}&rss=1`;
const result = await web({
const result = await get({
url,
noCache,
userAgent:
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
headers: {
userAgent:
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
},
});
const parseData = async () => {
if (typeof result?.data !== "string") return [];
const rssContent = extractRss(result.data);
if (!rssContent) return [];
return await parseRSS(rssContent);
};
const list = await parseData();
const list = await parseRSS(result.data);
return {
...result,
data: list.map((v, i) => ({

View File

@@ -29,8 +29,8 @@ const getList = async (noCache: boolean) => {
desc: v.post_content,
timestamp: getTime(v.created_at),
hot: v.like_count || v.comment_count,
url: v.buzz_original_url || `https://www.ifanr.com/${v.id}`,
mobileUrl: v.buzz_original_url || `https://www.ifanr.com/digest/${v.id}`,
url: v.buzz_original_url || `https://www.ifanr.com/${v.post_id}`,
mobileUrl: v.buzz_original_url || `https://www.ifanr.com/digest/${v.post_id}`,
})),
};
};

59
src/routes/kuaishou.ts Normal file
View File

@@ -0,0 +1,59 @@
import type { RouterType } from "../router.types.js";
import type { ListItem, RouterData } from "../types.js";
import { get } from "../utils/getData.js";
import { parseChineseNumber } from "../utils/getNum.js";
import UserAgent from "user-agents";
export const handleRoute = async (_: undefined, noCache: boolean) => {
const listData = await getList(noCache);
const routeData: RouterData = {
name: "kuaishou",
title: "快手",
type: "热榜",
description: "快手,拥抱每一种生活",
link: "https://www.kuaishou.com/",
total: listData.data?.length || 0,
...listData,
};
return routeData;
};
const getList = async (noCache: boolean) => {
const url = `https://www.kuaishou.com/?isHome=1`;
const userAgent = new UserAgent({
deviceCategory: "desktop",
});
const result = await get({
url,
noCache,
headers: {
"User-Agent": userAgent.toString(),
},
});
const listData: ListItem[] = [];
// 获取主要内容
const pattern = /window.__APOLLO_STATE__=(.*);\(function\(\)/s;
const matchResult = result.data?.match(pattern);
const jsonObject = JSON.parse(matchResult[1])["defaultClient"];
// 获取所有分类
const allItems = jsonObject['$ROOT_QUERY.visionHotRank({"page":"home"})']["items"];
// 获取全部热榜
allItems?.forEach((item: { id: string }) => {
// 基础数据
const hotItem: RouterType["kuaishou"] = jsonObject[item.id];
const id = hotItem.photoIds?.json?.[0];
listData.push({
id: hotItem.id,
title: hotItem.name,
cover: decodeURIComponent(hotItem.poster),
hot: parseChineseNumber(hotItem.hotValue),
timestamp: undefined,
url: `https://www.kuaishou.com/short-video/${id}`,
mobileUrl: `https://www.kuaishou.com/short-video/${id}`,
});
});
return {
...result,
data: listData,
};
};

View File

@@ -1,8 +1,7 @@
import type { RouterData } from "../types.js";
import type { RouterType } from "../router.types.js";
import { get } from "../utils/getData.js";
import { parseStringPromise } from "xml2js";
import { getTime } from "../utils/getTime.js";
import { parseRSS } from "../utils/parseRSS.js";
export const handleRoute = async (_: undefined, noCache: boolean) => {
const listData = await getList(noCache);
@@ -28,19 +27,18 @@ export const handleRoute = async (_: undefined, noCache: boolean) => {
const getList = async (noCache: boolean) => {
const url = `https://rss.nodeseek.com/`;
const result = await get({ url, noCache });
const rssData = await parseStringPromise(result.data);
const list = rssData.rss.channel[0].item;
const list = await parseRSS(result.data);
return {
...result,
data: list.map((v: RouterType["nodeseek"]) => ({
id: v.guid[0]._,
title: v.title[0],
desc: v.description ? v.description[0] : "",
author: v["dc:creator"] ? v["dc:creator"][0] : "unknown",
timestamp: getTime(v.pubDate[0]),
data: list.map((v, i) => ({
id: v.guid || i,
title: v.title || "",
desc: v.content?.trim() || "",
author: v.author,
timestamp: getTime(v.pubDate || 0),
hot: undefined,
url: v.link[0],
mobileUrl: v.link[0],
url: v.link || "",
mobileUrl: v.link || "",
})),
};
};

44
src/routes/xueqiu.ts Normal file
View File

@@ -0,0 +1,44 @@
import type { RouterData } from "../types.js";
import type { RouterType } from "../router.types.js";
import { get } from "../utils/getData.js";
export const handleRoute = async (_: undefined, noCache: boolean) => {
const listData = await getList(noCache);
const routeData: RouterData = {
name: "xueqiu",
title: "雪球",
type: "热门话题",
description: "聪明的投资者都在这里",
link: "https://xueqiu.com/",
total: listData.data?.length || 0,
...listData,
};
return routeData;
};
const getList = async (noCache: boolean) => {
const url = `https://xueqiu.com/hot_event/list.json`;
const result = await get({
url,
noCache,
headers: {
"User-Agent":
"Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/1.0 Mobile/12F69 Safari/605.1.15",
},
});
const list = result.data.list;
return {
...result,
data: list.map((v: RouterType["xueqiu"]) => ({
id: v.id,
title: v.tag?.startsWith("#") && v.tag?.endsWith("#") ? v.tag.slice(1, -1) : v.tag,
desc: v.content,
cover: v.pic,
author: undefined,
hot: v.hot,
timestamp: undefined,
url: undefined,
mobileUrl: undefined,
})),
};
};

10
src/types.d.ts vendored
View File

@@ -1,4 +1,5 @@
import type { Context } from "hono";
import type { ResponseType } from "axios";
// Context
export type ListContext = Context;
@@ -44,6 +45,7 @@ export interface Get {
noCache?: boolean;
ttl?: number;
originaInfo?: boolean;
responseType?: ResponseType;
}
export interface Post {
@@ -56,14 +58,6 @@ export interface Post {
originaInfo?: boolean;
}
export interface Web {
url: string;
timeout?: number;
noCache?: boolean;
ttl?: number;
userAgent?: string;
}
// 参数类型
export interface Options {
[key: string]: string | number | undefined;

View File

@@ -27,7 +27,7 @@ export const getCache = <T>(key: string): GetCache<T> | undefined => {
// 将数据写入缓存
export const setCache = <T>(key: string, value: T, ttl: number = config.CACHE_TTL) => {
const success = cache.set(key, value, ttl);
if (logger) logger.info("数据缓存成功", { url: key });
if (logger) logger.info(`💾 [CHCHE] ${key} has been cached`);
return success;
};

View File

@@ -1,8 +1,6 @@
import type { Get, Post, Web } from "../types.js";
import type { Page } from "puppeteer";
import type { Get, Post } from "../types.js";
import { config } from "../config.js";
import { getCache, setCache, delCache } from "./cache.js";
import { Cluster } from "puppeteer-cluster";
import logger from "./logger.js";
import axios from "axios";
@@ -13,69 +11,6 @@ const request = axios.create({
withCredentials: true,
});
// Puppeteer Cluster
let cluster: Cluster | null = null;
/**
* 创建 Puppeteer Cluster
* @returns {Promise<Cluster|null>} Puppeteer Cluster
*/
export const createCluster = async (): Promise<Cluster | null> => {
if (cluster) return cluster;
try {
cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_BROWSER,
maxConcurrency: 5,
puppeteerOptions: {
headless: true,
args: [
"--no-sandbox",
"--disable-gpu",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
],
},
});
logger.info("Puppeteer Cluster 已成功初始化");
} catch (error) {
logger.warn("启动 Puppeteer Cluster 失败,可能是没有 Chromium 安装", error);
cluster = null;
}
return cluster;
};
/**
* 处理任务
* @param page 页面
* @param data 数据
* @returns
*/
const taskHandler = async ({
page,
data: { url, userAgent },
}: {
page: Page;
data: { url: string; userAgent?: string };
}): Promise<string> => {
// 用户代理
if (userAgent) await page.setUserAgent(userAgent);
// 请求拦截
await page.setRequestInterception(true);
// 拦截非必要资源
page.on("request", (request) => {
const type = request.resourceType();
if (type === "document" || type === "script") {
request.continue();
} else {
request.abort();
}
});
// 加载页面
await page.goto(url, { waitUntil: "networkidle0", timeout: config.REQUEST_TIMEOUT });
// 返回页面内容
return page.content();
};
// 请求拦截
request.interceptors.request.use(
(request) => {
@@ -84,7 +19,7 @@ request.interceptors.request.use(
return request;
},
(error) => {
logger.error("请求失败,请稍后重试");
logger.error("❌ [ERROR] request failed");
return Promise.reject(error);
},
);
@@ -102,31 +37,38 @@ request.interceptors.response.use(
// GET
export const get = async (options: Get) => {
const { url, headers, params, noCache, ttl = config.CACHE_TTL, originaInfo = false } = options;
logger.info("发起 GET 请求", options);
const {
url,
headers,
params,
noCache,
ttl = config.CACHE_TTL,
originaInfo = false,
responseType = "json",
} = options;
logger.info(`🌐 [GET] ${url}`);
try {
// 检查缓存
if (noCache) delCache(url);
else {
const cachedData = getCache(url);
if (cachedData) {
logger.info("采用缓存", { url });
logger.info("💾 [CHCHE] The request is cached");
return { fromCache: true, data: cachedData.data, updateTime: cachedData.updateTime };
}
}
// 缓存不存在时请求接口
logger.info("请求接口", { url });
const response = await request.get(url, { headers, params });
const response = await request.get(url, { headers, params, responseType });
const responseData = response?.data || response;
// 存储新获取的数据到缓存
const updateTime = new Date().toISOString();
const data = originaInfo ? response : responseData;
setCache(url, { data, updateTime }, ttl);
// 返回数据
logger.info("接口调用成功", { status: response?.statusText });
logger.info(`✅ [${response?.statusText}] request was successful`);
return { fromCache: false, data, updateTime };
} catch (error) {
logger.error("GET 请求出错", error);
logger.error("❌ [ERROR] request failed");
throw error;
}
};
@@ -134,19 +76,18 @@ export const get = async (options: Get) => {
// POST
export const post = async (options: Post) => {
const { url, headers, body, noCache, ttl = config.CACHE_TTL, originaInfo = false } = options;
logger.info("发起 POST 请求", options);
logger.info(`🌐 [POST] ${url}`);
try {
// 检查缓存
if (noCache) delCache(url);
else {
const cachedData = getCache(url);
if (cachedData) {
logger.info("采用缓存", { url });
logger.info("💾 [CHCHE] The request is cached");
return { fromCache: true, data: cachedData.data, updateTime: cachedData.updateTime };
}
}
// 缓存不存在时请求接口
logger.info("请求接口", { url });
const response = await request.post(url, body, { headers });
const responseData = response?.data || response;
// 存储新获取的数据到缓存
@@ -156,68 +97,10 @@ export const post = async (options: Post) => {
setCache(url, { data, updateTime }, ttl);
}
// 返回数据
logger.info("接口调用成功", { status: response?.statusText });
logger.info(`✅ [${response?.statusText}] request was successful`);
return { fromCache: false, data, updateTime };
} catch (error) {
logger.error("POST 请求出错", error);
logger.error("❌ [ERROR] request failed");
throw error;
}
};
// WEB - Puppeteer
export const web = async (options: Web) => {
const { url, noCache, ttl = config.CACHE_TTL, userAgent } = options;
logger.info("使用 Puppeteer 发起页面请求", options);
if (config.USE_PUPPETEER === false) {
return {
fromCache: false,
data: [],
message: "Puppeteer is not enabled, please set USE_PUPPETEER=true",
updateTime: 0,
};
}
// 初始化 Cluster
const clusterInstance = await createCluster();
if (!clusterInstance) {
logger.error("Cluster 初始化失败,无法继续");
throw new Error("Cluster 初始化失败");
}
try {
// 检查缓存
if (noCache) {
delCache(url);
} else {
const cachedData = getCache(url);
if (cachedData) {
logger.info("采用缓存", { url });
return { fromCache: true, data: cachedData.data, updateTime: cachedData.updateTime };
}
}
// 缓存不存在时使用 Puppeteer 请求页面
logger.info("启动浏览器请求页面", { url });
const pageContent = await clusterInstance.execute({ url, userAgent }, taskHandler);
// 存储新获取的数据到缓存
const updateTime = new Date().toISOString();
setCache(url, { data: pageContent, updateTime }, ttl);
// 返回数据
logger.info("页面内容获取成功");
return { fromCache: false, data: pageContent, updateTime };
} catch (error) {
logger.error("Puppeteer 请求出错", error);
throw error;
}
};
/**
* 关闭 Cluster
*/
export const closeCluster = async () => {
if (cluster) {
try {
await cluster.close();
logger.info("Puppeteer Cluster 已成功关闭");
} catch (error) {
logger.error("关闭 Puppeteer Cluster 时出错", error);
}
}
};

View File

@@ -35,7 +35,8 @@ const getRSS = (data: RouterData) => {
const rssData = feed.rss2();
return rssData;
} catch (error) {
logger.error("RSS 生成失败:", error);
logger.error("❌ [ERROR] getRSS failed");
throw error;
}
};

View File

@@ -47,7 +47,7 @@ export const parseRSS = async (rssContent: string) => {
// 返回解析数据
return items;
} catch (error) {
logger.error("解析 RSS 内容时出错:", error);
return [];
logger.error("❌ [ERROR] An error occurred while parsing RSS content");
throw error;
}
};