perf: optimization download、Proxy settings, add batch export

This commit is contained in:
putyy
2025-04-29 11:09:42 +08:00
parent 6086bd7086
commit 85781a150a
14 changed files with 359 additions and 182 deletions

View File

@@ -166,30 +166,28 @@ func (a *App) installCert() {
}
}
func (a *App) OpenSystemProxy() bool {
func (a *App) OpenSystemProxy() error {
if a.IsProxy {
return true
return nil
}
err := systemOnce.setProxy()
if err == nil {
a.IsProxy = true
return true
return nil
}
DialogErr("设置失败:" + err.Error())
return false
return err
}
func (a *App) UnsetSystemProxy() bool {
func (a *App) UnsetSystemProxy() error {
if !a.IsProxy {
return true
return nil
}
err := systemOnce.unsetProxy()
if err == nil {
a.IsProxy = false
return true
return nil
}
DialogErr("设置失败:" + err.Error())
return false
return err
}
func (a *App) isInstall() bool {

View File

@@ -12,7 +12,7 @@ import (
"sync"
)
type ProgressCallback func(totalDownloaded float64)
type ProgressCallback func(totalDownloaded float64, totalSize float64)
type DownloadTask struct {
taskID int
@@ -53,16 +53,15 @@ func (fd *FileDownloader) buildClient() *http.Client {
if fd.ProxyUrl != nil {
transport.Proxy = http.ProxyURL(fd.ProxyUrl)
}
// Cookie handle
return &http.Client{
Transport: transport,
}
}
func (fd *FileDownloader) setHeaders(request *http.Request) {
for key, values := range fd.Headers {
for key, value := range fd.Headers {
if strings.Contains(globalConfig.UseHeaders, key) {
request.Header.Set(key, values)
request.Header.Set(key, value)
}
}
}
@@ -85,50 +84,42 @@ func (fd *FileDownloader) init() error {
request, err := http.NewRequest("HEAD", fd.Url, nil)
if err != nil {
return fmt.Errorf("create request failed")
return fmt.Errorf("create HEAD request failed: %w", err)
}
if _, ok := fd.Headers["User-Agent"]; !ok {
fd.Headers["User-Agent"] = globalConfig.UserAgent
}
if _, ok := fd.Headers["Referer"]; !ok {
fd.Headers["Referer"] = fd.Referer
}
fd.setHeaders(request)
resp, err := fd.buildClient().Do(request)
if err != nil {
return fmt.Errorf("request failed" + err.Error())
return fmt.Errorf("HEAD request failed: %w", err)
}
defer resp.Body.Close()
fd.TotalSize = resp.ContentLength
if fd.TotalSize <= 0 {
return fmt.Errorf("request init failed: size 0")
return fmt.Errorf("invalid file size")
}
if resp.Header.Get("Accept-Ranges") == "bytes" && fd.TotalSize > 10485760 {
if resp.Header.Get("Accept-Ranges") == "bytes" && fd.TotalSize > 10*1024*1024 {
fd.IsMultiPart = true
}
fd.FileName = filepath.Clean(fd.FileName)
dir := filepath.Dir(fd.FileName)
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return err
}
fd.File, err = os.OpenFile(fd.FileName, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return fmt.Errorf("文件初始化失败: %w", err)
return fmt.Errorf("file open failed: %w", err)
}
if err = fd.File.Truncate(fd.TotalSize); err != nil {
if err := fd.File.Truncate(fd.TotalSize); err != nil {
fd.File.Close()
return fmt.Errorf("文件大小设置失败: %w", err)
return fmt.Errorf("file truncate failed: %w", err)
}
return nil
}
@@ -139,97 +130,100 @@ func (fd *FileDownloader) createDownloadTasks() {
fd.totalTasks = int(fd.TotalSize)
}
eachSize := fd.TotalSize / int64(fd.totalTasks)
for i := 0; i < fd.totalTasks; i++ {
start := eachSize * int64(i)
end := eachSize*int64(i+1) - 1
if i == fd.totalTasks-1 {
end = fd.TotalSize - 1
}
fd.DownloadTaskList = append(fd.DownloadTaskList, &DownloadTask{
taskID: i,
rangeStart: eachSize * int64(i),
rangeEnd: eachSize*int64(i+1) - 1,
downloadedSize: 0,
isCompleted: false,
taskID: i,
rangeStart: start,
rangeEnd: end,
})
}
fd.DownloadTaskList[len(fd.DownloadTaskList)-1].rangeEnd = fd.TotalSize - 1
} else {
fd.DownloadTaskList = append(fd.DownloadTaskList, &DownloadTask{
taskID: 0,
rangeStart: 0,
rangeEnd: 0,
downloadedSize: 0,
isCompleted: false,
})
fd.DownloadTaskList = append(fd.DownloadTaskList, &DownloadTask{taskID: 0})
}
}
func (fd *FileDownloader) startDownload() {
waitGroup := &sync.WaitGroup{}
wg := &sync.WaitGroup{}
progressChan := make(chan int64)
for _, task := range fd.DownloadTaskList {
go fd.startDownloadTask(waitGroup, progressChan, task)
waitGroup.Add(1)
wg.Add(1)
go fd.startDownloadTask(wg, progressChan, task)
}
go func() {
waitGroup.Wait()
wg.Wait()
close(progressChan)
}()
if fd.progressCallback != nil {
totalDownloaded := int64(0)
for progress := range progressChan {
totalDownloaded += progress
fd.progressCallback(float64(totalDownloaded) * 100 / float64(fd.TotalSize))
for p := range progressChan {
totalDownloaded += p
fd.progressCallback(float64(totalDownloaded), float64(fd.TotalSize))
}
}
}
func (fd *FileDownloader) startDownloadTask(waitGroup *sync.WaitGroup, progressChan chan int64, task *DownloadTask) {
defer waitGroup.Done()
func (fd *FileDownloader) startDownloadTask(wg *sync.WaitGroup, progressChan chan int64, task *DownloadTask) {
defer wg.Done()
request, err := http.NewRequest("GET", fd.Url, nil)
if err != nil {
globalLogger.Error().Stack().Err(err).Msgf("任务%d创建请求出错", task.taskID)
return
}
fd.setHeaders(request)
if fd.IsMultiPart {
request.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", task.rangeStart, task.rangeEnd))
rangeHeader := fmt.Sprintf("bytes=%d-%d", task.rangeStart, task.rangeEnd)
request.Header.Set("Range", rangeHeader)
}
resp, err := fd.buildClient().Do(request)
client := fd.buildClient()
resp, err := client.Do(request)
if err != nil {
log.Printf("任务%d发送下载请求出错%s", task.taskID, err)
return
}
defer resp.Body.Close()
buf := make([]byte, 8192)
for {
n, err := resp.Body.Read(buf)
if n > 0 {
_, err := fd.File.WriteAt(buf[:n], task.rangeStart+task.downloadedSize)
if err != nil {
log.Printf("任务%d写入文件时出现错误位置:%d, err: %s\n", task.taskID, task.rangeStart+task.downloadedSize, err)
remain := task.rangeEnd - (task.rangeStart + task.downloadedSize) + 1
n64 := int64(n)
if n64 > remain {
n = int(remain)
}
_, writeErr := fd.File.WriteAt(buf[:n], task.rangeStart+task.downloadedSize)
if writeErr != nil {
log.Printf("任务%d写入文件时出现错误位置:%d, err: %s\n", task.taskID, task.rangeStart+task.downloadedSize, writeErr)
return
}
downSize := int64(n)
task.downloadedSize += downSize
progressChan <- downSize
task.downloadedSize += n64
progressChan <- n64
if task.rangeStart+task.downloadedSize-1 >= task.rangeEnd {
task.isCompleted = true
break
}
}
if err != nil {
if err == io.EOF {
task.isCompleted = true
break
}
log.Printf("任务%d读取响应错误%s", task.taskID, err)
return
break
}
}
}
func (fd *FileDownloader) Start() error {
err := fd.init()
if err != nil {
if err := fd.init(); err != nil {
return err
}
fd.createDownloadTasks()

View File

@@ -10,11 +10,15 @@ import (
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
sysRuntime "runtime"
"strings"
)
type respData map[string]interface{}
type ResponseData struct {
Code int `json:"code"`
Message string `json:"message"`
@@ -118,20 +122,54 @@ func (h *HttpServer) writeJson(w http.ResponseWriter, data ResponseData) {
}
}
func (h *HttpServer) error(w http.ResponseWriter, args ...interface{}) {
message := "ok"
var data interface{}
if len(args) > 0 {
message = args[0].(string)
}
if len(args) > 1 {
data = args[1]
}
h.writeJson(w, ResponseData{
Code: 0,
Message: message,
Data: data,
})
}
func (h *HttpServer) success(w http.ResponseWriter, args ...interface{}) {
message := "ok"
var data interface{}
if len(args) > 0 {
data = args[0]
}
if len(args) > 1 {
message = args[1].(string)
}
h.writeJson(w, ResponseData{
Code: 1,
Message: message,
Data: data,
})
}
func (h *HttpServer) openDirectoryDialog(w http.ResponseWriter, r *http.Request) {
folder, err := runtime.OpenDirectoryDialog(appOnce.ctx, runtime.OpenDialogOptions{
DefaultDirectory: "",
Title: "Select a folder",
})
if err != nil {
h.writeJson(w, ResponseData{Code: 0, Message: err.Error()})
h.error(w, err.Error())
return
}
h.writeJson(w, ResponseData{
Code: 1,
Data: map[string]interface{}{
"folder": folder,
},
h.success(w, respData{
"folder": folder,
})
}
@@ -146,14 +184,11 @@ func (h *HttpServer) openFileDialog(w http.ResponseWriter, r *http.Request) {
Title: "Select a file",
})
if err != nil {
h.writeJson(w, ResponseData{Code: 0, Message: err.Error()})
h.error(w, err.Error())
return
}
h.writeJson(w, ResponseData{
Code: 1,
Data: map[string]interface{}{
"file": filePath,
},
h.success(w, respData{
"file": filePath,
})
}
@@ -208,47 +243,57 @@ func (h *HttpServer) openFolder(w http.ResponseWriter, r *http.Request) {
h.writeJson(w, ResponseData{Code: 1})
}
func (h *HttpServer) setSystemPassword(w http.ResponseWriter, r *http.Request) {
var data struct {
Password string `json:"password"`
}
err := json.NewDecoder(r.Body).Decode(&data)
if err != nil {
h.error(w, err.Error())
return
}
systemOnce.SetPassword(data.Password)
h.success(w)
}
func (h *HttpServer) openSystemProxy(w http.ResponseWriter, r *http.Request) {
appOnce.OpenSystemProxy()
h.writeJson(w, ResponseData{
Code: 1,
Data: map[string]bool{
err := appOnce.OpenSystemProxy()
if err != nil {
h.error(w, err.Error(), respData{
"isProxy": appOnce.IsProxy,
},
})
return
}
h.success(w, respData{
"isProxy": appOnce.IsProxy,
})
}
func (h *HttpServer) unsetSystemProxy(w http.ResponseWriter, r *http.Request) {
appOnce.UnsetSystemProxy()
h.writeJson(w, ResponseData{
Code: 1,
Data: map[string]bool{
err := appOnce.UnsetSystemProxy()
if err != nil {
h.error(w, err.Error(), respData{
"isProxy": appOnce.IsProxy,
},
})
return
}
h.success(w, respData{
"isProxy": appOnce.IsProxy,
})
}
func (h *HttpServer) isProxy(w http.ResponseWriter, r *http.Request) {
h.writeJson(w, ResponseData{
Code: 1,
Data: map[string]interface{}{
"isProxy": appOnce.IsProxy,
},
h.success(w, respData{
"isProxy": appOnce.IsProxy,
})
}
func (h *HttpServer) appInfo(w http.ResponseWriter, r *http.Request) {
h.writeJson(w, ResponseData{
Code: 1,
Data: appOnce,
})
h.success(w, appOnce)
}
func (h *HttpServer) getConfig(w http.ResponseWriter, r *http.Request) {
h.writeJson(w, ResponseData{
Code: 1,
Data: globalConfig,
})
h.success(w, globalConfig)
}
func (h *HttpServer) setConfig(w http.ResponseWriter, r *http.Request) {
@@ -258,7 +303,7 @@ func (h *HttpServer) setConfig(w http.ResponseWriter, r *http.Request) {
return
}
globalConfig.setConfig(data)
h.writeJson(w, ResponseData{Code: 1})
h.success(w)
}
func (h *HttpServer) setType(w http.ResponseWriter, r *http.Request) {
@@ -274,12 +319,12 @@ func (h *HttpServer) setType(w http.ResponseWriter, r *http.Request) {
}
}
h.writeJson(w, ResponseData{Code: 1})
h.success(w)
}
func (h *HttpServer) clear(w http.ResponseWriter, r *http.Request) {
resourceOnce.clear()
h.writeJson(w, ResponseData{Code: 1})
h.success(w)
}
func (h *HttpServer) delete(w http.ResponseWriter, r *http.Request) {
@@ -290,7 +335,7 @@ func (h *HttpServer) delete(w http.ResponseWriter, r *http.Request) {
if err == nil && data.Sign != "" {
resourceOnce.delete(data.Sign)
}
h.writeJson(w, ResponseData{Code: 1})
h.success(w)
}
func (h *HttpServer) download(w http.ResponseWriter, r *http.Request) {
@@ -303,7 +348,7 @@ func (h *HttpServer) download(w http.ResponseWriter, r *http.Request) {
return
}
resourceOnce.download(data.MediaInfo, data.DecodeStr)
h.writeJson(w, ResponseData{Code: 1})
h.success(w)
}
func (h *HttpServer) wxFileDecode(w http.ResponseWriter, r *http.Request) {
@@ -313,18 +358,35 @@ func (h *HttpServer) wxFileDecode(w http.ResponseWriter, r *http.Request) {
DecodeStr string `json:"decodeStr"`
}
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
h.writeJson(w, ResponseData{Code: 0, Message: err.Error()})
h.error(w, err.Error())
return
}
savePath, err := resourceOnce.wxFileDecode(data.MediaInfo, data.Filename, data.DecodeStr)
if err != nil {
h.writeJson(w, ResponseData{Code: 0, Message: err.Error()})
h.error(w, err.Error())
return
}
h.writeJson(w, ResponseData{
Code: 1,
Data: map[string]string{
"save_path": savePath,
},
h.success(w, respData{
"save_path": savePath,
})
}
func (h *HttpServer) batchImport(w http.ResponseWriter, r *http.Request) {
var data struct {
Content string `json:"content"`
}
if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
h.error(w, err.Error())
return
}
fileName := filepath.Join(globalConfig.SaveDirectory, "res-downloader-"+GetCurrentDateTimeFormatted()+".txt")
// 0644 是文件权限:-rw-r--r--
err := os.WriteFile(fileName, []byte(data.Content), 0644)
if err != nil {
h.error(w, err.Error())
return
}
h.success(w, respData{
"file_name": fileName,
})
}

View File

@@ -26,6 +26,8 @@ func HandleApi(w http.ResponseWriter, r *http.Request) bool {
switch r.URL.Path {
case "/api/preview":
httpServerOnce.preview(w, r)
case "/api/set-system-password":
httpServerOnce.setSystemPassword(w, r)
case "/api/proxy-open":
httpServerOnce.openSystemProxy(w, r)
case "/api/proxy-unset":
@@ -54,6 +56,8 @@ func HandleApi(w http.ResponseWriter, r *http.Request) bool {
httpServerOnce.download(w, r)
case "/api/wx-file-decode":
httpServerOnce.wxFileDecode(w, r)
case "/api/batch-import":
httpServerOnce.batchImport(w, r)
}
return true
}

View File

@@ -147,8 +147,8 @@ func (r *Resource) download(mediaInfo MediaInfo, decodeStr string) {
headers, _ := r.parseHeaders(mediaInfo)
downloader := NewFileDownloader(rawUrl, mediaInfo.SavePath, globalConfig.TaskNumber, headers)
downloader.progressCallback = func(totalDownloaded float64) {
r.progressEventsEmit(mediaInfo, strconv.Itoa(int(totalDownloaded))+"%", DownloadStatusRunning)
downloader.progressCallback = func(totalDownloaded, totalSize float64) {
r.progressEventsEmit(mediaInfo, strconv.Itoa(int(totalDownloaded*100/totalSize))+"%", DownloadStatusRunning)
}
err := downloader.Start()
if err != nil {

View File

@@ -7,6 +7,7 @@ import (
type SystemSetup struct {
CertFile string
Password string
}
func initSystem() *SystemSetup {
@@ -33,3 +34,7 @@ func (s *SystemSetup) initCert() ([]byte, error) {
return nil, err
}
}
func (s *SystemSetup) SetPassword(password string) {
s.Password = password
}

View File

@@ -9,15 +9,30 @@ import (
"strings"
)
func (s *SystemSetup) runCommand(args []string) ([]byte, error) {
if len(args) == 0 {
return nil, fmt.Errorf("no command provided")
}
var cmd *exec.Cmd
if s.Password != "" {
cmd = exec.Command("sudo", append([]string{"-S"}, args...)...)
cmd.Stdin = bytes.NewReader([]byte(s.Password + "\n"))
} else {
cmd = exec.Command(args[0], args[1:]...)
}
output, err := cmd.CombinedOutput()
return output, err
}
func (s *SystemSetup) getNetworkServices() ([]string, error) {
cmd := exec.Command("networksetup", "-listallnetworkservices")
output, err := cmd.Output()
output, err := s.runCommand([]string{"networksetup", "-listallnetworkservices"})
if err != nil {
return nil, fmt.Errorf("failed to execute command: %v", err)
}
services := strings.Split(string(output), "\n")
var activeServices []string
for _, service := range services {
service = strings.TrimSpace(service)
@@ -25,15 +40,12 @@ func (s *SystemSetup) getNetworkServices() ([]string, error) {
continue
}
// 检查服务是否活动
infoCmd := exec.Command("networksetup", "-getinfo", service)
infoOutput, err := infoCmd.Output()
infoOutput, err := s.runCommand([]string{"networksetup", "-getinfo", service})
if err != nil {
fmt.Printf("failed to get info for service %s: %v\n", service, err)
continue
}
// 如果输出中包含 "IP address:",说明服务是活动的
if strings.Contains(string(infoOutput), "IP address:") {
activeServices = append(activeServices, service)
}
@@ -53,16 +65,19 @@ func (s *SystemSetup) setProxy() error {
}
is := false
errs := ""
for _, serviceName := range services {
if err := exec.Command("networksetup", "-setwebproxy", serviceName, "127.0.0.1", globalConfig.Port).Run(); err != nil {
fmt.Println(err)
} else {
is = true
cmds := [][]string{
{"networksetup", "-setwebproxy", serviceName, "127.0.0.1", globalConfig.Port},
{"networksetup", "-setsecurewebproxy", serviceName, "127.0.0.1", globalConfig.Port},
}
if err := exec.Command("networksetup", "-setsecurewebproxy", serviceName, "127.0.0.1", globalConfig.Port).Run(); err != nil {
fmt.Println(err)
} else {
is = true
for _, args := range cmds {
if output, err := s.runCommand(args); err != nil {
errs = errs + "\n output" + string(output) + "err:" + err.Error()
fmt.Println("setProxy:", output, " err:", err.Error())
} else {
is = true
}
}
}
@@ -70,7 +85,7 @@ func (s *SystemSetup) setProxy() error {
return nil
}
return fmt.Errorf("failed to set proxy for any active network service")
return fmt.Errorf("failed to set proxy for any active network service, errs: %s", errs)
}
func (s *SystemSetup) unsetProxy() error {
@@ -80,16 +95,19 @@ func (s *SystemSetup) unsetProxy() error {
}
is := false
errs := ""
for _, serviceName := range services {
if err := exec.Command("networksetup", "-setwebproxystate", serviceName, "off").Run(); err != nil {
fmt.Println(err)
} else {
is = true
cmds := [][]string{
{"networksetup", "-setwebproxystate", serviceName, "off"},
{"networksetup", "-setsecurewebproxystate", serviceName, "off"},
}
if err := exec.Command("networksetup", "-setsecurewebproxystate", serviceName, "off").Run(); err != nil {
fmt.Println(err)
} else {
is = true
for _, args := range cmds {
if output, err := s.runCommand(args); err != nil {
errs = errs + "\n output" + string(output) + "err:" + err.Error()
fmt.Println("unsetProxy:", output, " err:", err.Error())
} else {
is = true
}
}
}
@@ -97,7 +115,7 @@ func (s *SystemSetup) unsetProxy() error {
return nil
}
return fmt.Errorf("failed to set proxy for any active network service")
return fmt.Errorf("failed to unset proxy for any active network service, errs: %s", errs)
}
func (s *SystemSetup) installCert() (string, error) {
@@ -112,10 +130,11 @@ func (s *SystemSetup) installCert() (string, error) {
return string(passwordOutput), err
}
password := bytes.TrimSpace(passwordOutput)
cmd := exec.Command("sudo", "-S", "security", "add-trusted-cert", "-d", "-r", "trustRoot", "-k", "/Library/Keychains/System.keychain", s.CertFile)
password := strings.TrimSpace(string(passwordOutput))
s.SetPassword(password)
cmd.Stdin = bytes.NewReader(append(password, '\n'))
cmd := exec.Command("sudo", "-S", "security", "add-trusted-cert", "-d", "-r", "trustRoot", "-k", "/Library/Keychains/System.keychain", s.CertFile)
cmd.Stdin = bytes.NewReader([]byte(password + "\n"))
output, err := cmd.CombinedOutput()
if err != nil {
return string(output), err