mirror of
https://github.com/imsyy/DailyHotApi.git
synced 2026-01-12 05:04:56 +08:00
✨ feat: 新增 吾爱破解、果壳、快手
This commit is contained in:
@@ -21,7 +21,4 @@ REQUEST_TIMEOUT=6000
|
||||
USE_LOG_FILE=true
|
||||
|
||||
# RSS Mode
|
||||
RSS_MODE=false
|
||||
|
||||
# Puppeteer
|
||||
USE_PUPPETEER=false
|
||||
RSS_MODE=false
|
||||
17
.github/workflows/docker.yml
vendored
17
.github/workflows/docker.yml
vendored
@@ -43,7 +43,7 @@ jobs:
|
||||
imsyy/dailyhot-api
|
||||
ghcr.io/${{ github.repository }}
|
||||
|
||||
- name: Build and push regular image (no Puppeteer)
|
||||
- name: Build and push regular image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
@@ -54,18 +54,3 @@ jobs:
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
||||
- name: Build and push Puppeteer image (with Puppeteer)
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64
|
||||
push: true
|
||||
tags: imsyy/dailyhot-api:web-latest
|
||||
build-args: |
|
||||
USE_PUPPETEER=true
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
||||
|
||||
24
Dockerfile
24
Dockerfile
@@ -2,24 +2,6 @@ FROM node:20-alpine AS base
|
||||
|
||||
ENV NODE_ENV=docker
|
||||
|
||||
# 安装 Puppeteer 所需的依赖库
|
||||
RUN apk add --no-cache \
|
||||
libc6-compat \
|
||||
nss \
|
||||
freetype \
|
||||
harfbuzz \
|
||||
ca-certificates
|
||||
|
||||
# 判断是否需要安装 Chromium
|
||||
ARG USE_PUPPETEER=false
|
||||
RUN if [ "$USE_PUPPETEER" = "true" ]; then \
|
||||
apk add --no-cache chromium; \
|
||||
fi
|
||||
|
||||
# 配置 Chromium
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
|
||||
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||
|
||||
# 清理缓存
|
||||
RUN rm -rf /var/cache/apk/*
|
||||
|
||||
@@ -33,12 +15,10 @@ COPY package*json tsconfig.json pnpm-lock.yaml .env.example ./
|
||||
COPY src ./src
|
||||
COPY public ./public
|
||||
|
||||
# add .env.example to .env
|
||||
# 复制环境变量
|
||||
RUN [ ! -e ".env" ] && cp .env.example .env || true
|
||||
RUN if [ "$USE_PUPPETEER" = "true" ]; then \
|
||||
sed -i 's/^USE_PUPPETEER=false/USE_PUPPETEER=true/' .env; \
|
||||
fi
|
||||
|
||||
# 安装依赖
|
||||
RUN pnpm install
|
||||
RUN pnpm build
|
||||
RUN pnpm prune --production
|
||||
|
||||
21
README.md
21
README.md
@@ -30,7 +30,7 @@
|
||||
<details>
|
||||
<summary>查看全部接口</summary>
|
||||
|
||||
> 实例站点运行于海外服务器,部分国内站点可能存在访问异常,请以实际情况为准
|
||||
> 示例站点运行于海外服务器,部分国内站点可能存在访问异常,请以实际情况为准
|
||||
|
||||
| **站点** | **类别** | **调用名称** | **状态** |
|
||||
| ---------------- | ------------ | -------------- | --------------------------------------------------------------------------------------------------------------------------- |
|
||||
@@ -41,6 +41,7 @@
|
||||
| 知乎日报 | 推荐榜 | zhihu-daily |  |
|
||||
| 百度 | 热搜榜 | baidu |  |
|
||||
| 抖音 | 热点榜 | douyin |  |
|
||||
| 快手 | 热点榜 | kuaishou |  |
|
||||
| 豆瓣电影 | 新片榜 | douban-movie |  |
|
||||
| 豆瓣讨论小组 | 讨论精选 | douban-group |  |
|
||||
| 百度贴吧 | 热议榜 | tieba |  |
|
||||
@@ -48,6 +49,7 @@
|
||||
| IT之家 | 热榜 | ithome |  |
|
||||
| IT之家「喜加一」 | 最新动态 | ithome-xijiayi |  |
|
||||
| 简书 | 热门推荐 | jianshu |  |
|
||||
| 果壳 | 热门文章 | guokr |  |
|
||||
| 澎湃新闻 | 热榜 | thepaper |  |
|
||||
| 今日头条 | 热榜 | toutiao |  |
|
||||
| 36 氪 | 热榜 | 36kr |  |
|
||||
@@ -66,7 +68,7 @@
|
||||
| 虎扑 | 步行街热帖 | hupu |  |
|
||||
| 爱范儿 | 快讯 | ifanr |  |
|
||||
| 英雄联盟 | 更新公告 | lol |  |
|
||||
| 米游社 | 最新消息 | miyoushe |  |
|
||||
| 米游社 | 最新消息 | miyoushe |  |
|
||||
| 原神 | 最新消息 | genshin |  |
|
||||
| 崩坏3 | 最新动态 | honkai |  |
|
||||
| 崩坏:星穹铁道 | 最新动态 | starrail |  |
|
||||
@@ -103,8 +105,6 @@ serveHotApi(3000);
|
||||
|
||||
## ⚙️ 部署
|
||||
|
||||
由于部分接口无法通过接口调用等方式获取数据,故采用 `Puppeteer` 来实现,但由于使用后会造成 **内存占用过大或镜像过大**,可选择性开启,详情请参考下方说明。
|
||||
|
||||
具体使用说明可参考 [我的博客](https://blog.imsyy.top/posts/2024/0408),下方仅讲解基础操作:
|
||||
|
||||
### Docker 部署
|
||||
@@ -116,8 +116,6 @@ serveHotApi(3000);
|
||||
```bash
|
||||
# 构建
|
||||
docker build -t dailyhot-api .
|
||||
# 构建 Puppeteer 版
|
||||
docker build --build-arg USE_PUPPETEER=true -t dailyhot-api .
|
||||
|
||||
# 运行
|
||||
docker run --restart always -p 6688:6688 -d dailyhot-api
|
||||
@@ -130,8 +128,6 @@ docker-compose up -d
|
||||
```bash
|
||||
# 拉取
|
||||
docker pull imsyy/dailyhot-api:latest
|
||||
# 拉取 Puppeteer 版
|
||||
docker pull imsyy/dailyhot-api:web-latest
|
||||
|
||||
# 运行
|
||||
docker run --restart always -p 6688:6688 -d imsyy/dailyhot-api:latest
|
||||
@@ -139,7 +135,7 @@ docker run --restart always -p 6688:6688 -d imsyy/dailyhot-api:latest
|
||||
|
||||
### 手动部署
|
||||
|
||||
最直接的方式,您可以按照以下步骤将 DailyHotApi 部署在您的电脑、服务器或者其他任何地方
|
||||
最直接的方式,您可以按照以下步骤将 `DailyHotApi` 部署在您的电脑、服务器或者其他任何地方
|
||||
|
||||
#### 安装
|
||||
|
||||
@@ -159,10 +155,7 @@ npm install
|
||||
#### 开发
|
||||
|
||||
```bash
|
||||
# 标准运行
|
||||
npm run dev
|
||||
# 采用 Puppeteer 运行
|
||||
npm run dev:web
|
||||
```
|
||||
|
||||
成功启动后程序会在控制台输出可访问的地址
|
||||
@@ -171,11 +164,7 @@ npm run dev:web
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
|
||||
# 标准运行
|
||||
npm run start
|
||||
# 采用 Puppeteer 运行
|
||||
npm run start:web
|
||||
```
|
||||
|
||||
成功启动后程序会在控制台输出可访问的地址
|
||||
|
||||
12
package.json
12
package.json
@@ -30,11 +30,9 @@
|
||||
"format": "prettier --write .",
|
||||
"lint": "eslint .",
|
||||
"dev": "cross-env NODE_ENV=development tsx watch --no-cache src/index.ts",
|
||||
"dev:web": "cross-env NODE_ENV=development USE_PUPPETEER=true tsx watch --no-cache src/index.ts",
|
||||
"dev:cache": "cross-env NODE_ENV=development tsx watch src/index.ts",
|
||||
"build": "tsc --project tsconfig.json",
|
||||
"start": "cross-env NODE_ENV=development tsx dist/index.js",
|
||||
"start:web": "cross-env NODE_ENV=development USE_PUPPETEER=true tsx dist/index.js"
|
||||
"start": "cross-env NODE_ENV=development tsx dist/index.js"
|
||||
},
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
@@ -46,18 +44,18 @@
|
||||
"dotenv": "^16.4.6",
|
||||
"feed": "^4.2.2",
|
||||
"hono": "^4.6.12",
|
||||
"iconv-lite": "^0.6.3",
|
||||
"md5": "^2.3.0",
|
||||
"node-cache": "^5.1.2",
|
||||
"puppeteer": "^23.10.0",
|
||||
"puppeteer-cluster": "^0.24.0",
|
||||
"rss-parser": "^3.13.0",
|
||||
"winston": "^3.17.0",
|
||||
"xml2js": "^0.6.2"
|
||||
"user-agents": "^1.1.379",
|
||||
"winston": "^3.17.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.16.0",
|
||||
"@types/md5": "^2.3.5",
|
||||
"@types/node": "^22.10.1",
|
||||
"@types/user-agents": "^1.0.4",
|
||||
"@types/xml2js": "^0.4.14",
|
||||
"@typescript-eslint/eslint-plugin": "^8.17.0",
|
||||
"@typescript-eslint/parser": "^8.17.0",
|
||||
|
||||
791
pnpm-lock.yaml
generated
791
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -59,7 +59,7 @@ app.get("/", (c) => c.html(<Home />));
|
||||
app.notFound((c) => c.html(<NotFound />, 404));
|
||||
// error
|
||||
app.onError((err, c) => {
|
||||
logger.error(`出现致命错误:${err}`);
|
||||
logger.error(`❌ [ERROR] ${err?.message}`);
|
||||
return c.html(<Error error={err?.message} />, 500);
|
||||
});
|
||||
|
||||
|
||||
@@ -12,15 +12,12 @@ export type Config = {
|
||||
ALLOWED_HOST: string;
|
||||
USE_LOG_FILE: boolean;
|
||||
RSS_MODE: boolean;
|
||||
USE_PUPPETEER: boolean;
|
||||
};
|
||||
|
||||
// 验证并提取环境变量
|
||||
const getEnvVariable = (key: string): string | undefined => {
|
||||
const value = process.env[key];
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
return value;
|
||||
};
|
||||
|
||||
@@ -28,9 +25,7 @@ const getEnvVariable = (key: string): string | undefined => {
|
||||
const getNumericEnvVariable = (key: string, defaultValue: number): number => {
|
||||
const value = getEnvVariable(key) ?? String(defaultValue);
|
||||
const parsedValue = parseInt(value, 10);
|
||||
if (isNaN(parsedValue)) {
|
||||
return defaultValue;
|
||||
}
|
||||
if (isNaN(parsedValue)) return defaultValue;
|
||||
return parsedValue;
|
||||
};
|
||||
|
||||
@@ -45,10 +40,9 @@ export const config: Config = {
|
||||
PORT: getNumericEnvVariable("PORT", 6688),
|
||||
DISALLOW_ROBOT: getBooleanEnvVariable("DISALLOW_ROBOT", true),
|
||||
CACHE_TTL: getNumericEnvVariable("CACHE_TTL", 3600),
|
||||
REQUEST_TIMEOUT: getNumericEnvVariable("CACHE_TTL", 6000),
|
||||
REQUEST_TIMEOUT: getNumericEnvVariable("REQUEST_TIMEOUT", 6000),
|
||||
ALLOWED_DOMAIN: getEnvVariable("ALLOWED_DOMAIN") || "*",
|
||||
ALLOWED_HOST: getEnvVariable("ALLOWED_HOST") || "imsyy.top",
|
||||
USE_LOG_FILE: getBooleanEnvVariable("USE_LOG_FILE", true),
|
||||
RSS_MODE: getBooleanEnvVariable("RSS_MODE", false),
|
||||
USE_PUPPETEER: getBooleanEnvVariable("USE_PUPPETEER", false),
|
||||
};
|
||||
|
||||
@@ -10,8 +10,7 @@ const serveHotApi: (port?: number) => void = (port: number = config.PORT) => {
|
||||
fetch: app.fetch,
|
||||
port,
|
||||
});
|
||||
logger.info(`🔥 DailyHot API 成功在端口 ${port} 上运行`);
|
||||
logger.info(`💻 Puppeteer: ${config.USE_PUPPETEER}`);
|
||||
logger.info(`🔥 DailyHot API successfully runs on port ${port}`);
|
||||
logger.info(`🔗 Local: 👉 http://localhost:${port}`);
|
||||
return apiServer;
|
||||
} catch (error) {
|
||||
|
||||
33
src/router.types.d.ts
vendored
33
src/router.types.d.ts
vendored
@@ -315,14 +315,31 @@ export type RouterType = {
|
||||
message: string;
|
||||
replynum: number;
|
||||
};
|
||||
nodeseek: {
|
||||
guid: {
|
||||
_: string;
|
||||
}[];
|
||||
xueqiu: {
|
||||
pic: string;
|
||||
tag: string;
|
||||
id: number;
|
||||
hot: number;
|
||||
content: string;
|
||||
};
|
||||
guokr: {
|
||||
id: number;
|
||||
title: string;
|
||||
description: string | string[];
|
||||
"dc:creator": string;
|
||||
pubDate: string[];
|
||||
link: string[];
|
||||
summary: string;
|
||||
author: {
|
||||
nickname: string;
|
||||
};
|
||||
date_modified: string;
|
||||
small_image: string;
|
||||
};
|
||||
kuaishou: {
|
||||
id: string;
|
||||
name: string;
|
||||
hotValue: string;
|
||||
iconUrl: string;
|
||||
poster:string;
|
||||
photoIds: {
|
||||
json: string[];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,24 +1,27 @@
|
||||
import type { RouterData, ListContext, Options, RouterResType } from "../types.js";
|
||||
import { web } from "../utils/getData.js";
|
||||
import { extractRss, parseRSS } from "../utils/parseRSS.js";
|
||||
import { get } from "../utils/getData.js";
|
||||
import { getTime } from "../utils/getTime.js";
|
||||
import { parseRSS } from "../utils/parseRSS.js";
|
||||
import iconv from "iconv-lite";
|
||||
|
||||
const typeMap: Record<string, string> = {
|
||||
digest: "最新精华",
|
||||
hot: "最新热门",
|
||||
new: "最新回复",
|
||||
newthread: "最新发表",
|
||||
};
|
||||
|
||||
export const handleRoute = async (c: ListContext, noCache: boolean) => {
|
||||
const type = c.req.query("type") || "hot";
|
||||
const type = c.req.query("type") || "digest";
|
||||
const listData = await getList({ type }, noCache);
|
||||
const routeData: RouterData = {
|
||||
name: "52pojie",
|
||||
title: "吾爱破解",
|
||||
type: "榜单",
|
||||
type: typeMap[type],
|
||||
params: {
|
||||
type: {
|
||||
name: "榜单分类",
|
||||
type: {
|
||||
tech: "新鲜出炉",
|
||||
newthread: "技术分享",
|
||||
hot: "人气热门",
|
||||
digest: "精华采撷",
|
||||
},
|
||||
type: typeMap,
|
||||
},
|
||||
},
|
||||
link: "https://www.52pojie.cn/",
|
||||
@@ -31,28 +34,27 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => {
|
||||
const getList = async (options: Options, noCache: boolean): Promise<RouterResType> => {
|
||||
const { type } = options;
|
||||
const url = `https://www.52pojie.cn/forum.php?mod=guide&view=${type}&rss=1`;
|
||||
const result = await web({
|
||||
const result = await get({
|
||||
url,
|
||||
noCache,
|
||||
userAgent:
|
||||
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
|
||||
responseType: "arraybuffer",
|
||||
headers: {
|
||||
userAgent:
|
||||
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
|
||||
},
|
||||
});
|
||||
const parseData = async () => {
|
||||
if (typeof result?.data !== "string") return [];
|
||||
const rssContent = extractRss(result.data);
|
||||
if (!rssContent) return [];
|
||||
return await parseRSS(rssContent);
|
||||
};
|
||||
const list = await parseData();
|
||||
// 转码
|
||||
const utf8Data = iconv.decode(result.data, "gbk");
|
||||
const list = await parseRSS(utf8Data);
|
||||
return {
|
||||
...result,
|
||||
data: list.map((v, i) => ({
|
||||
id: v.guid || i,
|
||||
title: v.title || "",
|
||||
desc: v.content || "",
|
||||
author: v.author || "",
|
||||
desc: v.content?.trim() || "",
|
||||
author: v.author,
|
||||
timestamp: getTime(v.pubDate || 0),
|
||||
hot: 0,
|
||||
hot: undefined,
|
||||
url: v.link || "",
|
||||
mobileUrl: v.link || "",
|
||||
})),
|
||||
|
||||
45
src/routes/guokr.ts
Normal file
45
src/routes/guokr.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import type { RouterData } from "../types.js";
|
||||
import type { RouterType } from "../router.types.js";
|
||||
import { get } from "../utils/getData.js";
|
||||
import { getTime } from "../utils/getTime.js";
|
||||
|
||||
export const handleRoute = async (_: undefined, noCache: boolean) => {
|
||||
const listData = await getList(noCache);
|
||||
const routeData: RouterData = {
|
||||
name: "guokr",
|
||||
title: "果壳",
|
||||
type: "热门文章",
|
||||
description: "科技有意思",
|
||||
link: "https://www.guokr.com/",
|
||||
total: listData.data?.length || 0,
|
||||
...listData,
|
||||
};
|
||||
return routeData;
|
||||
};
|
||||
|
||||
const getList = async (noCache: boolean) => {
|
||||
const url = `https://www.guokr.com/beta/proxy/science_api/articles?limit=30`;
|
||||
const result = await get({
|
||||
url,
|
||||
noCache,
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
},
|
||||
});
|
||||
const list = result.data;
|
||||
return {
|
||||
...result,
|
||||
data: list.map((v: RouterType["guokr"]) => ({
|
||||
id: v.id,
|
||||
title: v.title,
|
||||
desc: v.summary,
|
||||
cover: v.small_image,
|
||||
author: v.author?.nickname,
|
||||
hot: undefined,
|
||||
timestamp: getTime(v.date_modified),
|
||||
url: `https://www.guokr.com/article/${v.id}`,
|
||||
mobileUrl: `https://m.guokr.com/article/${v.id}`,
|
||||
})),
|
||||
};
|
||||
};
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { RouterData, ListContext, Options } from "../types.js";
|
||||
import { web } from "../utils/getData.js";
|
||||
import { extractRss, parseRSS } from "../utils/parseRSS.js";
|
||||
import { get } from "../utils/getData.js";
|
||||
import { parseRSS } from "../utils/parseRSS.js";
|
||||
import { getTime } from "../utils/getTime.js";
|
||||
|
||||
const typeMap: Record<string, string> = {
|
||||
@@ -33,19 +33,15 @@ export const handleRoute = async (c: ListContext, noCache: boolean) => {
|
||||
const getList = async (options: Options, noCache: boolean) => {
|
||||
const { type } = options;
|
||||
const url = `https://hostloc.com/forum.php?mod=guide&view=${type}&rss=1`;
|
||||
const result = await web({
|
||||
const result = await get({
|
||||
url,
|
||||
noCache,
|
||||
userAgent:
|
||||
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
|
||||
headers: {
|
||||
userAgent:
|
||||
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36",
|
||||
},
|
||||
});
|
||||
const parseData = async () => {
|
||||
if (typeof result?.data !== "string") return [];
|
||||
const rssContent = extractRss(result.data);
|
||||
if (!rssContent) return [];
|
||||
return await parseRSS(rssContent);
|
||||
};
|
||||
const list = await parseData();
|
||||
const list = await parseRSS(result.data);
|
||||
return {
|
||||
...result,
|
||||
data: list.map((v, i) => ({
|
||||
|
||||
@@ -29,8 +29,8 @@ const getList = async (noCache: boolean) => {
|
||||
desc: v.post_content,
|
||||
timestamp: getTime(v.created_at),
|
||||
hot: v.like_count || v.comment_count,
|
||||
url: v.buzz_original_url || `https://www.ifanr.com/${v.id}`,
|
||||
mobileUrl: v.buzz_original_url || `https://www.ifanr.com/digest/${v.id}`,
|
||||
url: v.buzz_original_url || `https://www.ifanr.com/${v.post_id}`,
|
||||
mobileUrl: v.buzz_original_url || `https://www.ifanr.com/digest/${v.post_id}`,
|
||||
})),
|
||||
};
|
||||
};
|
||||
|
||||
59
src/routes/kuaishou.ts
Normal file
59
src/routes/kuaishou.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import type { RouterType } from "../router.types.js";
|
||||
import type { ListItem, RouterData } from "../types.js";
|
||||
import { get } from "../utils/getData.js";
|
||||
import { parseChineseNumber } from "../utils/getNum.js";
|
||||
import UserAgent from "user-agents";
|
||||
|
||||
export const handleRoute = async (_: undefined, noCache: boolean) => {
|
||||
const listData = await getList(noCache);
|
||||
const routeData: RouterData = {
|
||||
name: "kuaishou",
|
||||
title: "快手",
|
||||
type: "热榜",
|
||||
description: "快手,拥抱每一种生活",
|
||||
link: "https://www.kuaishou.com/",
|
||||
total: listData.data?.length || 0,
|
||||
...listData,
|
||||
};
|
||||
return routeData;
|
||||
};
|
||||
|
||||
const getList = async (noCache: boolean) => {
|
||||
const url = `https://www.kuaishou.com/?isHome=1`;
|
||||
const userAgent = new UserAgent({
|
||||
deviceCategory: "desktop",
|
||||
});
|
||||
const result = await get({
|
||||
url,
|
||||
noCache,
|
||||
headers: {
|
||||
"User-Agent": userAgent.toString(),
|
||||
},
|
||||
});
|
||||
const listData: ListItem[] = [];
|
||||
// 获取主要内容
|
||||
const pattern = /window.__APOLLO_STATE__=(.*);\(function\(\)/s;
|
||||
const matchResult = result.data?.match(pattern);
|
||||
const jsonObject = JSON.parse(matchResult[1])["defaultClient"];
|
||||
// 获取所有分类
|
||||
const allItems = jsonObject['$ROOT_QUERY.visionHotRank({"page":"home"})']["items"];
|
||||
// 获取全部热榜
|
||||
allItems?.forEach((item: { id: string }) => {
|
||||
// 基础数据
|
||||
const hotItem: RouterType["kuaishou"] = jsonObject[item.id];
|
||||
const id = hotItem.photoIds?.json?.[0];
|
||||
listData.push({
|
||||
id: hotItem.id,
|
||||
title: hotItem.name,
|
||||
cover: decodeURIComponent(hotItem.poster),
|
||||
hot: parseChineseNumber(hotItem.hotValue),
|
||||
timestamp: undefined,
|
||||
url: `https://www.kuaishou.com/short-video/${id}`,
|
||||
mobileUrl: `https://www.kuaishou.com/short-video/${id}`,
|
||||
});
|
||||
});
|
||||
return {
|
||||
...result,
|
||||
data: listData,
|
||||
};
|
||||
};
|
||||
@@ -1,8 +1,7 @@
|
||||
import type { RouterData } from "../types.js";
|
||||
import type { RouterType } from "../router.types.js";
|
||||
import { get } from "../utils/getData.js";
|
||||
import { parseStringPromise } from "xml2js";
|
||||
import { getTime } from "../utils/getTime.js";
|
||||
import { parseRSS } from "../utils/parseRSS.js";
|
||||
|
||||
export const handleRoute = async (_: undefined, noCache: boolean) => {
|
||||
const listData = await getList(noCache);
|
||||
@@ -28,19 +27,18 @@ export const handleRoute = async (_: undefined, noCache: boolean) => {
|
||||
const getList = async (noCache: boolean) => {
|
||||
const url = `https://rss.nodeseek.com/`;
|
||||
const result = await get({ url, noCache });
|
||||
const rssData = await parseStringPromise(result.data);
|
||||
const list = rssData.rss.channel[0].item;
|
||||
const list = await parseRSS(result.data);
|
||||
return {
|
||||
...result,
|
||||
data: list.map((v: RouterType["nodeseek"]) => ({
|
||||
id: v.guid[0]._,
|
||||
title: v.title[0],
|
||||
desc: v.description ? v.description[0] : "",
|
||||
author: v["dc:creator"] ? v["dc:creator"][0] : "unknown",
|
||||
timestamp: getTime(v.pubDate[0]),
|
||||
data: list.map((v, i) => ({
|
||||
id: v.guid || i,
|
||||
title: v.title || "",
|
||||
desc: v.content?.trim() || "",
|
||||
author: v.author,
|
||||
timestamp: getTime(v.pubDate || 0),
|
||||
hot: undefined,
|
||||
url: v.link[0],
|
||||
mobileUrl: v.link[0],
|
||||
url: v.link || "",
|
||||
mobileUrl: v.link || "",
|
||||
})),
|
||||
};
|
||||
};
|
||||
|
||||
44
src/routes/xueqiu.ts
Normal file
44
src/routes/xueqiu.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import type { RouterData } from "../types.js";
|
||||
import type { RouterType } from "../router.types.js";
|
||||
import { get } from "../utils/getData.js";
|
||||
|
||||
export const handleRoute = async (_: undefined, noCache: boolean) => {
|
||||
const listData = await getList(noCache);
|
||||
const routeData: RouterData = {
|
||||
name: "xueqiu",
|
||||
title: "雪球",
|
||||
type: "热门话题",
|
||||
description: "聪明的投资者都在这里",
|
||||
link: "https://xueqiu.com/",
|
||||
total: listData.data?.length || 0,
|
||||
...listData,
|
||||
};
|
||||
return routeData;
|
||||
};
|
||||
|
||||
const getList = async (noCache: boolean) => {
|
||||
const url = `https://xueqiu.com/hot_event/list.json`;
|
||||
const result = await get({
|
||||
url,
|
||||
noCache,
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 14_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/1.0 Mobile/12F69 Safari/605.1.15",
|
||||
},
|
||||
});
|
||||
const list = result.data.list;
|
||||
return {
|
||||
...result,
|
||||
data: list.map((v: RouterType["xueqiu"]) => ({
|
||||
id: v.id,
|
||||
title: v.tag?.startsWith("#") && v.tag?.endsWith("#") ? v.tag.slice(1, -1) : v.tag,
|
||||
desc: v.content,
|
||||
cover: v.pic,
|
||||
author: undefined,
|
||||
hot: v.hot,
|
||||
timestamp: undefined,
|
||||
url: undefined,
|
||||
mobileUrl: undefined,
|
||||
})),
|
||||
};
|
||||
};
|
||||
10
src/types.d.ts
vendored
10
src/types.d.ts
vendored
@@ -1,4 +1,5 @@
|
||||
import type { Context } from "hono";
|
||||
import type { ResponseType } from "axios";
|
||||
|
||||
// Context
|
||||
export type ListContext = Context;
|
||||
@@ -44,6 +45,7 @@ export interface Get {
|
||||
noCache?: boolean;
|
||||
ttl?: number;
|
||||
originaInfo?: boolean;
|
||||
responseType?: ResponseType;
|
||||
}
|
||||
|
||||
export interface Post {
|
||||
@@ -56,14 +58,6 @@ export interface Post {
|
||||
originaInfo?: boolean;
|
||||
}
|
||||
|
||||
export interface Web {
|
||||
url: string;
|
||||
timeout?: number;
|
||||
noCache?: boolean;
|
||||
ttl?: number;
|
||||
userAgent?: string;
|
||||
}
|
||||
|
||||
// 参数类型
|
||||
export interface Options {
|
||||
[key: string]: string | number | undefined;
|
||||
|
||||
@@ -27,7 +27,7 @@ export const getCache = <T>(key: string): GetCache<T> | undefined => {
|
||||
// 将数据写入缓存
|
||||
export const setCache = <T>(key: string, value: T, ttl: number = config.CACHE_TTL) => {
|
||||
const success = cache.set(key, value, ttl);
|
||||
if (logger) logger.info("数据缓存成功", { url: key });
|
||||
if (logger) logger.info(`💾 [CHCHE] ${key} has been cached`);
|
||||
return success;
|
||||
};
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import type { Get, Post, Web } from "../types.js";
|
||||
import type { Page } from "puppeteer";
|
||||
import type { Get, Post } from "../types.js";
|
||||
import { config } from "../config.js";
|
||||
import { getCache, setCache, delCache } from "./cache.js";
|
||||
import { Cluster } from "puppeteer-cluster";
|
||||
import logger from "./logger.js";
|
||||
import axios from "axios";
|
||||
|
||||
@@ -13,69 +11,6 @@ const request = axios.create({
|
||||
withCredentials: true,
|
||||
});
|
||||
|
||||
// Puppeteer Cluster
|
||||
let cluster: Cluster | null = null;
|
||||
|
||||
/**
|
||||
* 创建 Puppeteer Cluster
|
||||
* @returns {Promise<Cluster|null>} Puppeteer Cluster
|
||||
*/
|
||||
export const createCluster = async (): Promise<Cluster | null> => {
|
||||
if (cluster) return cluster;
|
||||
try {
|
||||
cluster = await Cluster.launch({
|
||||
concurrency: Cluster.CONCURRENCY_BROWSER,
|
||||
maxConcurrency: 5,
|
||||
puppeteerOptions: {
|
||||
headless: true,
|
||||
args: [
|
||||
"--no-sandbox",
|
||||
"--disable-gpu",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
],
|
||||
},
|
||||
});
|
||||
logger.info("Puppeteer Cluster 已成功初始化");
|
||||
} catch (error) {
|
||||
logger.warn("启动 Puppeteer Cluster 失败,可能是没有 Chromium 安装", error);
|
||||
cluster = null;
|
||||
}
|
||||
return cluster;
|
||||
};
|
||||
|
||||
/**
|
||||
* 处理任务
|
||||
* @param page 页面
|
||||
* @param data 数据
|
||||
* @returns
|
||||
*/
|
||||
const taskHandler = async ({
|
||||
page,
|
||||
data: { url, userAgent },
|
||||
}: {
|
||||
page: Page;
|
||||
data: { url: string; userAgent?: string };
|
||||
}): Promise<string> => {
|
||||
// 用户代理
|
||||
if (userAgent) await page.setUserAgent(userAgent);
|
||||
// 请求拦截
|
||||
await page.setRequestInterception(true);
|
||||
// 拦截非必要资源
|
||||
page.on("request", (request) => {
|
||||
const type = request.resourceType();
|
||||
if (type === "document" || type === "script") {
|
||||
request.continue();
|
||||
} else {
|
||||
request.abort();
|
||||
}
|
||||
});
|
||||
// 加载页面
|
||||
await page.goto(url, { waitUntil: "networkidle0", timeout: config.REQUEST_TIMEOUT });
|
||||
// 返回页面内容
|
||||
return page.content();
|
||||
};
|
||||
|
||||
// 请求拦截
|
||||
request.interceptors.request.use(
|
||||
(request) => {
|
||||
@@ -84,7 +19,7 @@ request.interceptors.request.use(
|
||||
return request;
|
||||
},
|
||||
(error) => {
|
||||
logger.error("请求失败,请稍后重试");
|
||||
logger.error("❌ [ERROR] request failed");
|
||||
return Promise.reject(error);
|
||||
},
|
||||
);
|
||||
@@ -102,31 +37,38 @@ request.interceptors.response.use(
|
||||
|
||||
// GET
|
||||
export const get = async (options: Get) => {
|
||||
const { url, headers, params, noCache, ttl = config.CACHE_TTL, originaInfo = false } = options;
|
||||
logger.info("发起 GET 请求", options);
|
||||
const {
|
||||
url,
|
||||
headers,
|
||||
params,
|
||||
noCache,
|
||||
ttl = config.CACHE_TTL,
|
||||
originaInfo = false,
|
||||
responseType = "json",
|
||||
} = options;
|
||||
logger.info(`🌐 [GET] ${url}`);
|
||||
try {
|
||||
// 检查缓存
|
||||
if (noCache) delCache(url);
|
||||
else {
|
||||
const cachedData = getCache(url);
|
||||
if (cachedData) {
|
||||
logger.info("采用缓存", { url });
|
||||
logger.info("💾 [CHCHE] The request is cached");
|
||||
return { fromCache: true, data: cachedData.data, updateTime: cachedData.updateTime };
|
||||
}
|
||||
}
|
||||
// 缓存不存在时请求接口
|
||||
logger.info("请求接口", { url });
|
||||
const response = await request.get(url, { headers, params });
|
||||
const response = await request.get(url, { headers, params, responseType });
|
||||
const responseData = response?.data || response;
|
||||
// 存储新获取的数据到缓存
|
||||
const updateTime = new Date().toISOString();
|
||||
const data = originaInfo ? response : responseData;
|
||||
setCache(url, { data, updateTime }, ttl);
|
||||
// 返回数据
|
||||
logger.info("接口调用成功", { status: response?.statusText });
|
||||
logger.info(`✅ [${response?.statusText}] request was successful`);
|
||||
return { fromCache: false, data, updateTime };
|
||||
} catch (error) {
|
||||
logger.error("GET 请求出错", error);
|
||||
logger.error("❌ [ERROR] request failed");
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
@@ -134,19 +76,18 @@ export const get = async (options: Get) => {
|
||||
// POST
|
||||
export const post = async (options: Post) => {
|
||||
const { url, headers, body, noCache, ttl = config.CACHE_TTL, originaInfo = false } = options;
|
||||
logger.info("发起 POST 请求", options);
|
||||
logger.info(`🌐 [POST] ${url}`);
|
||||
try {
|
||||
// 检查缓存
|
||||
if (noCache) delCache(url);
|
||||
else {
|
||||
const cachedData = getCache(url);
|
||||
if (cachedData) {
|
||||
logger.info("采用缓存", { url });
|
||||
logger.info("💾 [CHCHE] The request is cached");
|
||||
return { fromCache: true, data: cachedData.data, updateTime: cachedData.updateTime };
|
||||
}
|
||||
}
|
||||
// 缓存不存在时请求接口
|
||||
logger.info("请求接口", { url });
|
||||
const response = await request.post(url, body, { headers });
|
||||
const responseData = response?.data || response;
|
||||
// 存储新获取的数据到缓存
|
||||
@@ -156,68 +97,10 @@ export const post = async (options: Post) => {
|
||||
setCache(url, { data, updateTime }, ttl);
|
||||
}
|
||||
// 返回数据
|
||||
logger.info("接口调用成功", { status: response?.statusText });
|
||||
logger.info(`✅ [${response?.statusText}] request was successful`);
|
||||
return { fromCache: false, data, updateTime };
|
||||
} catch (error) {
|
||||
logger.error("POST 请求出错", error);
|
||||
logger.error("❌ [ERROR] request failed");
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
// WEB - Puppeteer
|
||||
export const web = async (options: Web) => {
|
||||
const { url, noCache, ttl = config.CACHE_TTL, userAgent } = options;
|
||||
logger.info("使用 Puppeteer 发起页面请求", options);
|
||||
if (config.USE_PUPPETEER === false) {
|
||||
return {
|
||||
fromCache: false,
|
||||
data: [],
|
||||
message: "Puppeteer is not enabled, please set USE_PUPPETEER=true",
|
||||
updateTime: 0,
|
||||
};
|
||||
}
|
||||
// 初始化 Cluster
|
||||
const clusterInstance = await createCluster();
|
||||
if (!clusterInstance) {
|
||||
logger.error("Cluster 初始化失败,无法继续");
|
||||
throw new Error("Cluster 初始化失败");
|
||||
}
|
||||
try {
|
||||
// 检查缓存
|
||||
if (noCache) {
|
||||
delCache(url);
|
||||
} else {
|
||||
const cachedData = getCache(url);
|
||||
if (cachedData) {
|
||||
logger.info("采用缓存", { url });
|
||||
return { fromCache: true, data: cachedData.data, updateTime: cachedData.updateTime };
|
||||
}
|
||||
}
|
||||
// 缓存不存在时使用 Puppeteer 请求页面
|
||||
logger.info("启动浏览器请求页面", { url });
|
||||
const pageContent = await clusterInstance.execute({ url, userAgent }, taskHandler);
|
||||
// 存储新获取的数据到缓存
|
||||
const updateTime = new Date().toISOString();
|
||||
setCache(url, { data: pageContent, updateTime }, ttl);
|
||||
// 返回数据
|
||||
logger.info("页面内容获取成功");
|
||||
return { fromCache: false, data: pageContent, updateTime };
|
||||
} catch (error) {
|
||||
logger.error("Puppeteer 请求出错", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 关闭 Cluster
|
||||
*/
|
||||
export const closeCluster = async () => {
|
||||
if (cluster) {
|
||||
try {
|
||||
await cluster.close();
|
||||
logger.info("Puppeteer Cluster 已成功关闭");
|
||||
} catch (error) {
|
||||
logger.error("关闭 Puppeteer Cluster 时出错", error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -35,7 +35,8 @@ const getRSS = (data: RouterData) => {
|
||||
const rssData = feed.rss2();
|
||||
return rssData;
|
||||
} catch (error) {
|
||||
logger.error("RSS 生成失败:", error);
|
||||
logger.error("❌ [ERROR] getRSS failed");
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ export const parseRSS = async (rssContent: string) => {
|
||||
// 返回解析数据
|
||||
return items;
|
||||
} catch (error) {
|
||||
logger.error("解析 RSS 内容时出错:", error);
|
||||
return [];
|
||||
logger.error("❌ [ERROR] An error occurred while parsing RSS content");
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user