From 6086bd70860c74ad8501957a2f400d56d04fe912 Mon Sep 17 00:00:00 2001 From: putyy Date: Wed, 23 Apr 2025 16:55:24 +0800 Subject: [PATCH] feat: Added header cache, optional header settings... --- README-EN.md | 2 +- README.md | 4 +-- build/README.md | 1 - build/windows/installer/wails_tools.nsh | 2 +- core/config.go | 21 +++++++++++++-- core/downloader.go | 35 ++++++++++++++++--------- core/proxy.go | 6 +++++ core/resource.go | 27 ++++++++++++++++++- frontend/src/stores/index.ts | 1 + frontend/src/types/app.d.ts | 1 + frontend/src/views/setting.vue | 11 ++++++++ wails.json | 4 +-- 12 files changed, 92 insertions(+), 23 deletions(-) diff --git a/README-EN.md b/README-EN.md index 5226734..022d4b0 100644 --- a/README-EN.md +++ b/README-EN.md @@ -4,7 +4,7 @@ > A cross-platform resource downloader built with Go + [Wails](https://github.com/wailsapp/wails). Clean UI, easy to use, and supports a wide range of resource sniffing and downloading. -### 📖 [中文](./README.md) | English +### 📖 [中文](./README.md) | English --- diff --git a/README.md b/README.md index 5bb43cd..e440355 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ ## 📚 文档 & 版本 - 📘 [在线文档](https://res.putyy.com/) -- 🧩 [Mini版 UI使用默认浏览器展示](https://github.com/putyy/res-downloader) ｜ [Electron旧版 支持Win7](https://github.com/putyy/res-downloader/tree/old) +- 🧩 [Mini版 UI使用默认浏览器展示](https://github.com/putyy/resd-mini) ｜ [Electron旧版 支持Win7](https://github.com/putyy/res-downloader/tree/old) - 💬 [加入交流群](https://www.putyy.com/app/admin/upload/img/20250418/6801d9554dc7.webp) > *群满时可加微信 `AmorousWorld`，请备注“来源”* @@ -32,7 +32,7 @@ - 🆕 [蓝奏云下载（密码：9vs5）](https://wwjv.lanzoum.com/b04wgtfyb) - ⚠️ *Win7 用户请下载 `2.3.0` 版本* ---- +--- ## 🚀 使用方法 diff --git a/build/README.md b/build/README.md index 5ff65d0..3a32135 100644 --- a/build/README.md +++ b/build/README.md @@ -9,7 +9,6 @@ mv -f "build/bin/res-downloader $(jq -r '.info.productVersion' wails.json).dmg" ```bash wails build -f -nsis -platform "windows/amd64" -webview2 Embed -skipbindings && mv -f "build/bin/res-downloader-amd64-installer.exe" "build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_win_amd64.exe" wails build -f -nsis -platform "windows/arm64" -webview2 Embed -skipbindings && mv -f "build/bin/res-downloader-arm64-installer.exe" "build/bin/res-downloader_$(jq -r '.info.productVersion' wails.json)_win_arm64.exe" - ``` ## Linux diff --git a/build/windows/installer/wails_tools.nsh b/build/windows/installer/wails_tools.nsh index fd35a46..24eac16 100644 --- a/build/windows/installer/wails_tools.nsh +++ b/build/windows/installer/wails_tools.nsh @@ -14,7 +14,7 @@ !define INFO_PRODUCTNAME "res-downloader" !endif !ifndef INFO_PRODUCTVERSION - !define INFO_PRODUCTVERSION "3.0.4" + !define INFO_PRODUCTVERSION "3.0.5" !endif !ifndef INFO_COPYRIGHT !define INFO_COPYRIGHT "Copyright © 2023" diff --git a/core/config.go b/core/config.go index ca5f2ab..89c5a42 100644 --- a/core/config.go +++ b/core/config.go @@ -24,6 +24,7 @@ type Config struct { WxAction bool `json:"WxAction"` TaskNumber int `json:"TaskNumber"` UserAgent string `json:"UserAgent"` + UseHeaders string `json:"UseHeaders"` } func initConfig() *Config { @@ -43,7 +44,8 @@ func initConfig() *Config { "AutoProxy": true, "WxAction": true, "TaskNumber": __TaskNumber__, - "UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" + "UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36", + "UseHeaders": "User-Agent,Referer,Authorization,Cookie" } ` def = strings.ReplaceAll(def, "__TaskNumber__", strconv.Itoa(runtime.NumCPU()*2)) @@ -51,9 +53,23 @@ func initConfig() *Config { storage: NewStorage("config.json", []byte(def)), } + defaultMap := make(map[string]interface{}) + _ = json.Unmarshal([]byte(def), &defaultMap) + data, err := globalConfig.storage.Load() if err == nil { - _ = json.Unmarshal(data, &globalConfig) + var loadedMap map[string]interface{} + _ = json.Unmarshal(data, &loadedMap) + + for key, val := range defaultMap { + if _, ok := loadedMap[key]; !ok { + loadedMap[key] = val + } + } + + finalBytes, _ := json.Marshal(loadedMap) + _ = json.Unmarshal(finalBytes, &globalConfig) + } else { globalLogger.Esg(err, "load config err") } @@ -77,6 +93,7 @@ func (c *Config) setConfig(config Config) { c.AutoProxy = config.AutoProxy c.TaskNumber = config.TaskNumber c.WxAction = config.WxAction + c.UseHeaders = config.UseHeaders if oldProxy != c.UpstreamProxy { proxyOnce.setTransport() } diff --git a/core/downloader.go b/core/downloader.go index a910561..e7a013b 100644 --- a/core/downloader.go +++ b/core/downloader.go @@ -5,7 +5,6 @@ import ( "io" "log" "net/http" - "net/http/cookiejar" "net/url" "os" "path/filepath" @@ -32,17 +31,19 @@ type FileDownloader struct { totalTasks int TotalSize int64 IsMultiPart bool + Headers map[string]string DownloadTaskList []*DownloadTask progressCallback ProgressCallback } -func NewFileDownloader(url, filename string, totalTasks int) *FileDownloader { +func NewFileDownloader(url, filename string, totalTasks int, headers map[string]string) *FileDownloader { return &FileDownloader{ Url: url, FileName: filename, totalTasks: totalTasks, IsMultiPart: false, TotalSize: 0, + Headers: headers, DownloadTaskList: make([]*DownloadTask, 0), } } @@ -53,10 +54,16 @@ func (fd *FileDownloader) buildClient() *http.Client { transport.Proxy = http.ProxyURL(fd.ProxyUrl) } // Cookie handle - jar, _ := cookiejar.New(nil) return &http.Client{ Transport: transport, - Jar: jar, + } +} + +func (fd *FileDownloader) setHeaders(request *http.Request) { + for key, values := range fd.Headers { + if strings.Contains(globalConfig.UseHeaders, key) { + request.Header.Set(key, values) + } } } @@ -76,21 +83,22 @@ func (fd *FileDownloader) init() error { } } - req, err := http.NewRequest("HEAD", fd.Url, nil) + request, err := http.NewRequest("HEAD", fd.Url, nil) if err != nil { return fmt.Errorf("create request failed") } - // 设置请求头 - if globalConfig.UserAgent != "" { - req.Header.Set("User-Agent", globalConfig.UserAgent) + if _, ok := fd.Headers["User-Agent"]; !ok { + fd.Headers["User-Agent"] = globalConfig.UserAgent } - if fd.Referer != "" { - req.Header.Set("Referer", fd.Referer) + if _, ok := fd.Headers["Referer"]; !ok { + fd.Headers["Referer"] = fd.Referer } - resp, err := fd.buildClient().Do(req) + fd.setHeaders(request) + + resp, err := fd.buildClient().Do(request) if err != nil { return fmt.Errorf("request failed" + err.Error()) } @@ -182,8 +190,9 @@ func (fd *FileDownloader) startDownloadTask(waitGroup *sync.WaitGroup, progressC globalLogger.Error().Stack().Err(err).Msgf("任务%d创建请求出错", task.taskID) return } - request.Header.Set("User-Agent", globalConfig.UserAgent) - request.Header.Set("Referer", fd.Referer) + + fd.setHeaders(request) + if fd.IsMultiPart { request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", task.rangeStart, task.rangeEnd)) } diff --git a/core/proxy.go b/core/proxy.go index ed372c9..31bb4ac 100644 --- a/core/proxy.go +++ b/core/proxy.go @@ -336,6 +336,12 @@ func (p *Proxy) httpResponseEvent(resp *http.Response, ctx *goproxy.ProxyCtx) *h Description: "", ContentType: resp.Header.Get("Content-Type"), } + + // Store entire request headers as JSON + if headers, err := json.Marshal(resp.Request.Header); err == nil { + res.OtherData["headers"] = string(headers) + } + resourceOnce.markMedia(urlSign) httpServerOnce.send("newResources", res) } diff --git a/core/resource.go b/core/resource.go index 573ea20..de858c8 100644 --- a/core/resource.go +++ b/core/resource.go @@ -2,6 +2,8 @@ package core import ( "encoding/base64" + "encoding/json" + "fmt" "io" "net/url" "os" @@ -142,7 +144,9 @@ func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) { } } - downloader := NewFileDownloader(rawUrl, mediaInfo.SavePath, globalConfig.TaskNumber) + headers, _ := r.parseHeaders(mediaInfo) + + downloader := NewFileDownloader(rawUrl, mediaInfo.SavePath, globalConfig.TaskNumber, headers) downloader.progressCallback = func(totalDownloaded float64) { r.progressEventsEmit(mediaInfo, strconv.Itoa(int(totalDownloaded))+"%", DownloadStatusRunning) } @@ -162,6 +166,27 @@ func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) { }(mediaInfo) } +// 解析并组装 headers +func (r *Resource) parseHeaders(mediaInfo MediaInfo) (map[string]string, error) { + headers := make(map[string]string) + + if hh, ok := mediaInfo.OtherData["headers"]; ok { + var tempHeaders map[string][]string + // 解析 JSON 字符串为 map[string][]string + if err := json.Unmarshal([]byte(hh), &tempHeaders); err != nil { + return headers, fmt.Errorf("parse headers JSON err: %v", err) + } + + for key, values := range tempHeaders { + if len(values) > 0 { + headers[key] = values[0] + } + } + } + + return headers, nil +} + func (r *Resource) wxFileDecode(mediaInfo MediaInfo, fileName, decodeStr string) (string, error) { sourceFile, err := os.Open(fileName) if err != nil { diff --git a/frontend/src/stores/index.ts b/frontend/src/stores/index.ts index d88412f..fa5837e 100644 --- a/frontend/src/stores/index.ts +++ b/frontend/src/stores/index.ts @@ -27,6 +27,7 @@ export const useIndexStore = defineStore("index-store", () => { WxAction: false, TaskNumber: 8, UserAgent: "", + UseHeaders: "", }) const envInfo = ref({ diff --git a/frontend/src/types/app.d.ts b/frontend/src/types/app.d.ts index d9f15bf..14bee2c 100644 --- a/frontend/src/types/app.d.ts +++ b/frontend/src/types/app.d.ts @@ -21,6 +21,7 @@ export namespace appType { WxAction: boolean TaskNumber: number UserAgent: string + UseHeaders: string } interface MediaInfo { diff --git a/frontend/src/views/setting.vue b/frontend/src/views/setting.vue index 7a4efd6..d76bb60 100644 --- a/frontend/src/views/setting.vue +++ b/frontend/src/views/setting.vue @@ -138,6 +138,17 @@ 如不清楚请保持默认 + + + + + 3.0.4版本缓存了请求header信息，这个参数定义在下载时可使用的header参数: ,分割 + + diff --git a/wails.json b/wails.json index 538c86c..eb0339b 100644 --- a/wails.json +++ b/wails.json @@ -13,8 +13,8 @@ "info": { "companyName": "res-downloader", "productName": "res-downloader", - "productVersion": "3.0.4", + "productVersion": "3.0.5", "copyright": "Copyright © 2023", - "comments": "This is a high-value, high-performance, and diverse resource downloader called res-downloader." + "comments": "This is a high-value high-performance and diverse resource downloader called res-downloader." } }