Skip to content

Commit 1d071c0

Browse files
PIKACHUIMCopilotj2rong4cnSuyunmeng
authored
feat(func): support ed2k & magnet & torrent offline download (#2452)
Add end-to-end offline download support for torrent files, magnet links, and ed2k links, including torrent parse and generate APIs, CAS-based rapid upload for Cloud189, and protocol-aware tool routing with transfer flow improvements. --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Co-authored-by: j2rong4cn <j2rong@qq.com> Co-authored-by: Suyunjing <suyunmeng@oplist.org.cn>
1 parent 948e871 commit 1d071c0

15 files changed

Lines changed: 2202 additions & 29 deletions

File tree

‎drivers/189/torrent.go‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
package _189
2+
3+
import (
4+
"context"
5+
"crypto/sha1"
6+
"encoding/hex"
7+
"fmt"
8+
"io"
9+
"strings"
10+
11+
"github.com/OpenListTeam/OpenList/v4/internal/model"
12+
"github.com/OpenListTeam/OpenList/v4/pkg/torrent"
13+
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
14+
)
15+
16+
// GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件
17+
func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) {
18+
// 计算 sliceMD5
19+
sliceMD5 := fileMD5
20+
if len(sliceMD5s) > 1 {
21+
joined := strings.Join(sliceMD5s, "\n")
22+
sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined))
23+
}
24+
25+
t := torrent.NewTorrent(fileName, fileSize, fileMD5)
26+
t.Info.PieceLength = sliceSize
27+
t.SetPieces(pieceHashes)
28+
t.SetCASInfo(&torrent.CASInfo{
29+
FileMD5: fileMD5,
30+
SliceMD5: sliceMD5,
31+
SliceMD5s: sliceMD5s,
32+
SliceSize: sliceSize,
33+
Cloud: "189",
34+
})
35+
36+
return t.Encode()
37+
}
38+
39+
// RapidUploadFromTorrent 从 torrent 文件中提取 CAS 信息进行秒传
40+
func (d *Cloud189) RapidUploadFromTorrent(ctx context.Context, dstDir model.Obj, torrentData []byte) error {
41+
// 解析 torrent
42+
t, err := torrent.Decode(torrentData)
43+
if err != nil {
44+
return fmt.Errorf("解析 torrent 失败: %w", err)
45+
}
46+
47+
// 检查是否包含 CAS 扩展信息
48+
if !t.HasCASInfo() {
49+
return fmt.Errorf("torrent 不包含 CAS 扩展信息,无法秒传")
50+
}
51+
52+
cas := t.CAS
53+
fileName := t.Info.Name
54+
fileSize := t.GetTotalSize()
55+
56+
// 获取 sessionKey
57+
sessionKey, err := d.getSessionKey()
58+
if err != nil {
59+
return err
60+
}
61+
d.sessionKey = sessionKey
62+
63+
// 初始化上传
64+
res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{
65+
"parentFolderId": dstDir.GetID(),
66+
"fileName": encode(fileName),
67+
"fileSize": fmt.Sprint(fileSize),
68+
"sliceSize": fmt.Sprint(cas.SliceSize),
69+
"lazyCheck": "1",
70+
}, nil)
71+
if err != nil {
72+
return fmt.Errorf("初始化上传失败: %w", err)
73+
}
74+
75+
uploadFileId := utils.Json.Get(res, "data", "uploadFileId").ToString()
76+
77+
// 提交上传(使用 CAS 信息秒传)
78+
_, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{
79+
"uploadFileId": uploadFileId,
80+
"fileMd5": cas.FileMD5,
81+
"sliceMd5": cas.SliceMD5,
82+
"lazyCheck": "1",
83+
"opertype": "3",
84+
}, nil)
85+
if err != nil {
86+
return fmt.Errorf("秒传提交失败: %w", err)
87+
}
88+
89+
return nil
90+
}
91+
92+
// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件
93+
func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64, sliceSize int64) ([]byte, error) {
94+
if sliceSize <= 0 {
95+
sliceSize = torrent.DefaultPieceSize
96+
}
97+
98+
hw := torrent.NewHashWriter(sliceSize, sliceSize)
99+
100+
buf := make([]byte, 32*1024)
101+
for {
102+
n, err := reader.Read(buf)
103+
if n > 0 {
104+
hw.Write(buf[:n])
105+
}
106+
if err == io.EOF {
107+
break
108+
}
109+
if err != nil {
110+
return nil, err
111+
}
112+
}
113+
hw.Finish()
114+
115+
fileMD5 := hw.GetFileMD5()
116+
sliceMD5s := hw.GetSliceMD5s()
117+
pieceHashes := hw.GetPieceHashes()
118+
119+
return GenerateTorrent(fileName, fileSize, fileMD5, sliceMD5s, sliceSize, pieceHashes)
120+
}
121+
122+
// ComputePieceSHA1 计算单个分片的 SHA-1 哈希
123+
func ComputePieceSHA1(data []byte) []byte {
124+
h := sha1.Sum(data)
125+
return h[:]
126+
}
127+
128+
// ExtractCASFromTorrent 从 torrent 数据中提取 CAS 信息
129+
func ExtractCASFromTorrent(torrentData []byte) (*torrent.CASInfo, string, int64, error) {
130+
t, err := torrent.Decode(torrentData)
131+
if err != nil {
132+
return nil, "", 0, fmt.Errorf("解析 torrent 失败: %w", err)
133+
}
134+
135+
if !t.HasCASInfo() {
136+
return nil, "", 0, fmt.Errorf("torrent 不包含 CAS 扩展信息")
137+
}
138+
139+
return t.CAS, t.Info.Name, t.GetTotalSize(), nil
140+
}
141+
142+
// GetInfoHashHex 获取 torrent 的 info_hash(十六进制字符串)
143+
func GetInfoHashHex(torrentData []byte) (string, error) {
144+
t, err := torrent.Decode(torrentData)
145+
if err != nil {
146+
return "", err
147+
}
148+
return hex.EncodeToString(t.InfoHash), nil
149+
}

‎drivers/189/util.go‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"context"
66
"crypto/md5"
7+
sha1Pkg "crypto/sha1"
78
"encoding/base64"
89
"encoding/hex"
910
"errors"
@@ -18,6 +19,8 @@ import (
1819
"github.com/OpenListTeam/OpenList/v4/drivers/base"
1920
"github.com/OpenListTeam/OpenList/v4/internal/driver"
2021
"github.com/OpenListTeam/OpenList/v4/internal/model"
22+
"github.com/OpenListTeam/OpenList/v4/internal/op"
23+
"github.com/OpenListTeam/OpenList/v4/internal/stream"
2124
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
2225
myrand "github.com/OpenListTeam/OpenList/v4/pkg/utils/random"
2326
"github.com/go-resty/resty/v2"
@@ -388,6 +391,10 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F
388391
var finish int64 = 0
389392
var i int64
390393
var byteSize int64
394+
395+
// 额外计算 SHA-1 piece hash 用于生成 torrent
396+
pieceSHA1Hashes := make([]byte, 0, int(count)*20)
397+
391398
for i = 1; i <= count; i++ {
392399
if utils.IsCanceled(ctx) {
393400
return ctx.Err()
@@ -404,6 +411,10 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F
404411
finish += int64(n)
405412
md5Bytes := getMd5(byteData)
406413
md5Base64 := base64.StdEncoding.EncodeToString(md5Bytes)
414+
415+
// 计算 SHA-1 piece hash
416+
sha1Hash := sha1Pkg.Sum(byteData)
417+
pieceSHA1Hashes = append(pieceSHA1Hashes, sha1Hash[:]...)
407418
var resp UploadUrlsResp
408419
res, err = d.uploadRequest("/person/getMultiUploadUrls", map[string]string{
409420
"partInfo": fmt.Sprintf("%s-%s", strconv.FormatInt(i, 10), md5Base64),
@@ -439,7 +450,49 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F
439450
"lazyCheck": "1",
440451
"opertype": "3",
441452
}, nil)
442-
return err
453+
if err != nil {
454+
return err
455+
}
456+
457+
// 生成 torrent 文件(异步,不影响上传结果)
458+
capturedDstDir := dstDir
459+
capturedFileName := file.GetName()
460+
capturedFileSize := fileSize
461+
capturedFileMd5Hex := fileMd5Hex
462+
capturedMd5s := md5s
463+
go func() {
464+
fileMD5Upper := strings.ToUpper(capturedFileMd5Hex)
465+
torrentData, err := GenerateTorrent(capturedFileName, capturedFileSize, fileMD5Upper, capturedMd5s, DEFAULT, pieceSHA1Hashes)
466+
if err != nil {
467+
log.Warnf("生成 torrent 失败: %v", err)
468+
return
469+
}
470+
infoHash, _ := GetInfoHashHex(torrentData)
471+
torrentName := capturedFileName + ".cas.torrent"
472+
log.Infof("已生成 torrent: %s (info_hash: %s, size: %d bytes)",
473+
torrentName, infoHash, len(torrentData))
474+
475+
// 将 torrent 文件上传到同一目录
476+
torrentFileStream := &stream.FileStream{
477+
Ctx: context.Background(),
478+
Obj: &model.Object{
479+
Name: torrentName,
480+
Size: int64(len(torrentData)),
481+
IsFolder: false,
482+
},
483+
Reader: bytes.NewReader(torrentData),
484+
Mimetype: "application/x-bittorrent",
485+
}
486+
uploadErr := d.oldUpload(capturedDstDir, torrentFileStream)
487+
if uploadErr != nil {
488+
log.Warnf("上传 torrent 文件失败: %v", uploadErr)
489+
} else {
490+
log.Infof("torrent 文件已上传: %s", torrentName)
491+
op.Cache.DeleteDirectory(d, capturedDstDir.GetPath())
492+
}
493+
}()
494+
495+
return nil
443496
}
444497

445498
func (d *Cloud189) getCapacityInfo(ctx context.Context) (*CapacityResp, error) {

‎drivers/189pc/driver.go‎

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,7 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil
343343

344344
// 响应时间长,按需启用
345345
if y.Addition.RapidUpload && !stream.IsForceStreamUpload() {
346+
// 尝试妙传
346347
if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil {
347348
return newObj, nil
348349
}
@@ -351,10 +352,11 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil
351352
uploadMethod := y.UploadMethod
352353
if stream.IsForceStreamUpload() {
353354
uploadMethod = "stream"
354-
}
355-
356-
// 旧版上传家庭云也有限制
357-
if uploadMethod == "old" {
355+
} else if y.Addition.RapidUpload && stream.GetFile() != nil {
356+
// 文件流支持随机读取,走FastUpload计算MD5并尝试秒传
357+
uploadMethod = "rapid"
358+
} else if uploadMethod == "old" {
359+
// 旧版上传家庭云也有限制
358360
return y.OldUpload(ctx, dstDir, stream, up, isFamily, overwrite)
359361
}
360362

@@ -416,19 +418,6 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil
416418
if stream.GetSize() == 0 {
417419
return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite)
418420
}
419-
// 尝试秒传:如果已有MD5且启用了RapidUpload则用RapidUpload,否则走FastUpload(会计算MD5并尝试秒传)
420-
if !stream.IsForceStreamUpload() {
421-
fileMd5 := stream.GetHash().GetHash(utils.MD5)
422-
if len(fileMd5) >= utils.MD5.Width && y.Addition.RapidUpload {
423-
// 源文件已有MD5且启用了RapidUpload配置,尝试快速秒传
424-
if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil {
425-
return newObj, nil
426-
}
427-
} else if len(fileMd5) < utils.MD5.Width {
428-
// 源文件无MD5(如从本地复制),走FastUpload计算MD5并尝试秒传
429-
return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite)
430-
}
431-
}
432421
fallthrough
433422
default:
434423
return y.StreamUpload(ctx, dstDir, stream, up, isFamily, overwrite)

‎drivers/189pc/meta.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,16 @@ type Addition struct {
1313
AccessToken string `json:"access_token" required:"false"`
1414
RefreshToken string `json:"refresh_token" help:"To switch accounts, please clear this field"`
1515
driver.RootID
16-
OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"`
17-
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
18-
Type string `json:"type" type:"select" options:"personal,family" default:"personal"`
19-
FamilyID string `json:"family_id"`
20-
UploadMethod string `json:"upload_method" type:"select" options:"stream,rapid,old" default:"stream"`
21-
UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"`
22-
FamilyTransfer bool `json:"family_transfer"`
23-
RapidUpload bool `json:"rapid_upload"`
24-
NoUseOcr bool `json:"no_use_ocr"`
16+
OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"`
17+
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
18+
Type string `json:"type" type:"select" options:"personal,family" default:"personal"`
19+
FamilyID string `json:"family_id"`
20+
UploadMethod string `json:"upload_method" type:"select" options:"stream,rapid,old" default:"stream"`
21+
UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"`
22+
FamilyTransfer bool `json:"family_transfer"`
23+
RapidUpload bool `json:"rapid_upload"`
24+
NoUseOcr bool `json:"no_use_ocr"`
25+
GenerateTorrent bool `json:"generate_torrent" help:"Generate torrent file with CAS extension after upload"`
2526
}
2627

2728
var config = driver.Config{

0 commit comments

Comments
 (0)