Skip to content

Commit 893457c

Browse files
Nostalgianostalumecodex
authored
fix(aliyundrive): limit callback concurrency (#3071)
* fix(aliyundrive): limit callback concurrency - Share proxy callback admission by Aliyun user identity and hold permits for complete response-body lifetimes. - Retry only verified callback-capacity rejections while preserving direct redirects and server download limiting. - Map exhausted temporary capacity to S3 SlowDown through the merged OpenListTeam gofakes3 module. - Cover shared limits, lifecycle release, cancellation, retry classification, and the S3 HTTP response. Co-authored-by: Codex <267193182+codex@users.noreply.github.com> # Conflicts: # go.mod # go.sum # server/s3/pager.go * fix(op): separate redirect and proxy link cache entries - Include redirect mode in the link cache key for all drivers. - Cover both redirect-to-proxy and proxy-to-redirect cache reuse. Co-authored-by: Codex <267193182+codex@users.noreply.github.com> * fix(proxy): close range bodies before opening next - make ServeHTTP own each range body and preserve cleanup failures - remove the aggregate range closer and pass range readers directly - replace the obsolete callback transport test with focused lifecycle coverage Co-authored-by: Codex <267193182+codex@users.noreply.github.com> --------- Co-authored-by: nostalume <nostalucent@gmail.com> Co-authored-by: Codex <267193182+codex@users.noreply.github.com>
1 parent 90acfa1 commit 893457c

16 files changed

Lines changed: 1139 additions & 69 deletions

File tree

Lines changed: 295 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,295 @@
1+
package aliyundrive_open
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"math/rand/v2"
8+
"net/http"
9+
"strings"
10+
"sync"
11+
"time"
12+
13+
"github.com/OpenListTeam/OpenList/v4/internal/conf"
14+
"github.com/OpenListTeam/OpenList/v4/internal/errs"
15+
anet "github.com/OpenListTeam/OpenList/v4/internal/net"
16+
"github.com/OpenListTeam/OpenList/v4/internal/stream"
17+
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
18+
)
19+
20+
const (
21+
defaultCallbackConcurrency = 1
22+
callbackAcquireTimeout = time.Second
23+
callbackRequestAttempts = 3
24+
callbackRetryBaseDelay = 200 * time.Millisecond
25+
callbackErrorBodyLimit = 64 << 10
26+
)
27+
28+
var callbackLimiters = struct {
29+
sync.Mutex
30+
byUser map[string]*callbackLimiter
31+
}{byUser: make(map[string]*callbackLimiter)}
32+
33+
type callbackLimiter struct {
34+
userID string
35+
mu sync.Mutex
36+
active int
37+
nextID uint64
38+
registrations map[uint64]int
39+
changed chan struct{}
40+
}
41+
42+
type callbackRegistration struct {
43+
limiter *callbackLimiter
44+
id uint64
45+
once sync.Once
46+
}
47+
48+
type callbackPermit struct {
49+
limiter *callbackLimiter
50+
once sync.Once
51+
}
52+
53+
func normalizeCallbackConcurrency(limit int) int {
54+
if limit <= 0 {
55+
return defaultCallbackConcurrency
56+
}
57+
return limit
58+
}
59+
60+
func registerCallbackLimiter(userID string, limit int) *callbackRegistration {
61+
callbackLimiters.Lock()
62+
defer callbackLimiters.Unlock()
63+
64+
limiter := callbackLimiters.byUser[userID]
65+
if limiter == nil {
66+
limiter = &callbackLimiter{
67+
userID: userID,
68+
registrations: make(map[uint64]int),
69+
changed: make(chan struct{}),
70+
}
71+
callbackLimiters.byUser[userID] = limiter
72+
}
73+
limiter.mu.Lock()
74+
limiter.nextID++
75+
id := limiter.nextID
76+
limiter.registrations[id] = normalizeCallbackConcurrency(limit)
77+
limiter.signalLocked()
78+
limiter.mu.Unlock()
79+
return &callbackRegistration{limiter: limiter, id: id}
80+
}
81+
82+
func (r *callbackRegistration) unregister() {
83+
if r == nil || r.limiter == nil {
84+
return
85+
}
86+
r.once.Do(func() {
87+
callbackLimiters.Lock()
88+
defer callbackLimiters.Unlock()
89+
r.limiter.mu.Lock()
90+
delete(r.limiter.registrations, r.id)
91+
r.limiter.signalLocked()
92+
if len(r.limiter.registrations) == 0 && r.limiter.active == 0 {
93+
delete(callbackLimiters.byUser, r.limiter.userID)
94+
}
95+
r.limiter.mu.Unlock()
96+
})
97+
}
98+
99+
func (r *callbackRegistration) acquire(ctx context.Context) (*callbackPermit, error) {
100+
if r == nil || r.limiter == nil {
101+
return nil, errs.NewErr(errs.TemporaryCapacity, "callback limiter is unavailable")
102+
}
103+
if err := ctx.Err(); err != nil {
104+
return nil, err
105+
}
106+
waitCtx, cancel := context.WithTimeout(ctx, callbackAcquireTimeout)
107+
defer cancel()
108+
for {
109+
r.limiter.mu.Lock()
110+
if r.limiter.active < r.limiter.limitLocked() {
111+
r.limiter.active++
112+
r.limiter.mu.Unlock()
113+
return &callbackPermit{limiter: r.limiter}, nil
114+
}
115+
changed := r.limiter.changed
116+
r.limiter.mu.Unlock()
117+
select {
118+
case <-ctx.Done():
119+
return nil, ctx.Err()
120+
case <-waitCtx.Done():
121+
if err := ctx.Err(); err != nil {
122+
return nil, err
123+
}
124+
return nil, errs.NewErr(errs.TemporaryCapacity, "timed out waiting for callback admission")
125+
case <-changed:
126+
}
127+
}
128+
}
129+
130+
func (l *callbackLimiter) limitLocked() int {
131+
limit := 0
132+
for _, registered := range l.registrations {
133+
if limit == 0 || registered < limit {
134+
limit = registered
135+
}
136+
}
137+
return limit
138+
}
139+
140+
func (l *callbackLimiter) signalLocked() {
141+
close(l.changed)
142+
l.changed = make(chan struct{})
143+
}
144+
145+
func (p *callbackPermit) release() {
146+
if p == nil || p.limiter == nil {
147+
return
148+
}
149+
p.once.Do(func() {
150+
callbackLimiters.Lock()
151+
defer callbackLimiters.Unlock()
152+
p.limiter.mu.Lock()
153+
p.limiter.active--
154+
p.limiter.signalLocked()
155+
if len(p.limiter.registrations) == 0 && p.limiter.active == 0 {
156+
delete(callbackLimiters.byUser, p.limiter.userID)
157+
}
158+
p.limiter.mu.Unlock()
159+
})
160+
}
161+
162+
func (d *AliyundriveOpen) callbackRegistration() *callbackRegistration {
163+
if d.callback != nil {
164+
return d.callback
165+
}
166+
if d.ref != nil {
167+
return d.ref.callbackRegistration()
168+
}
169+
return nil
170+
}
171+
172+
func (d *AliyundriveOpen) callbackRangeReader(url string, size int64) stream.RangeReaderFunc {
173+
return func(ctx context.Context, requested http_range.Range) (io.ReadCloser, error) {
174+
if requested.Length < 0 || requested.Start+requested.Length > size {
175+
requested.Length = size - requested.Start
176+
}
177+
for attempt := 0; attempt < callbackRequestAttempts; attempt++ {
178+
permit, err := d.callbackRegistration().acquire(ctx)
179+
if err != nil {
180+
return nil, err
181+
}
182+
body, retry, err := openCallbackRange(ctx, url, size, requested)
183+
if !retry && err == nil {
184+
return newCallbackBody(ctx, body, permit.release), nil
185+
}
186+
permit.release()
187+
if !retry {
188+
return nil, err
189+
}
190+
if attempt+1 == callbackRequestAttempts {
191+
return nil, errs.NewErr(errs.TemporaryCapacity, "Aliyun callback concurrency limit rejected %d attempts", callbackRequestAttempts)
192+
}
193+
delay := callbackRetryBaseDelay << attempt
194+
delay += time.Duration(rand.Int64N(int64(delay / 2)))
195+
timer := time.NewTimer(delay)
196+
select {
197+
case <-ctx.Done():
198+
timer.Stop()
199+
return nil, ctx.Err()
200+
case <-timer.C:
201+
}
202+
}
203+
return nil, errs.NewErr(errs.TemporaryCapacity, "callback attempts exhausted")
204+
}
205+
}
206+
207+
func openCallbackRange(ctx context.Context, url string, size int64, requested http_range.Range) (io.ReadCloser, bool, error) {
208+
requestHeader, _ := ctx.Value(conf.RequestHeaderKey).(http.Header)
209+
header := anet.ProcessHeader(requestHeader, nil)
210+
header = http_range.ApplyRangeToHttpHeader(requested, header)
211+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
212+
if err != nil {
213+
return nil, false, fmt.Errorf("create Aliyun callback request: %w", err)
214+
}
215+
req.Header = header
216+
response, err := anet.HttpClient().Do(req)
217+
if err != nil {
218+
return nil, false, fmt.Errorf("Aliyun callback request failed: %w", err)
219+
}
220+
if response.StatusCode >= http.StatusBadRequest {
221+
defer response.Body.Close()
222+
body, readErr := io.ReadAll(io.LimitReader(response.Body, callbackErrorBodyLimit))
223+
if readErr != nil {
224+
return nil, false, fmt.Errorf("read Aliyun callback error response: %w", readErr)
225+
}
226+
if isCallbackCapacityRejection(response.StatusCode, body) {
227+
return nil, true, nil
228+
}
229+
message := strings.ReplaceAll(strings.TrimSpace(string(body)), url, "<redacted>")
230+
return nil, false, fmt.Errorf("Aliyun callback request failed: %w; response: %s", anet.HttpStatusCodeError(response.StatusCode), message)
231+
}
232+
if requested.Start == 0 && requested.Length == size || response.StatusCode == http.StatusPartialContent || callbackContentRangeStartsAt(response.Header, requested.Start) {
233+
return response.Body, false, nil
234+
}
235+
if response.StatusCode == http.StatusOK {
236+
body, rangeErr := anet.GetRangedHttpReader(response.Body, requested.Start, requested.Length)
237+
if rangeErr != nil {
238+
response.Body.Close()
239+
return nil, false, rangeErr
240+
}
241+
return body, false, nil
242+
}
243+
return response.Body, false, nil
244+
}
245+
246+
func isCallbackCapacityRejection(status int, body []byte) bool {
247+
return status == http.StatusForbidden &&
248+
strings.Contains(string(body), "RequestDeniedByCallback") &&
249+
strings.Contains(string(body), "ExceedMaxConcurrency")
250+
}
251+
252+
func callbackContentRangeStartsAt(header http.Header, offset int64) bool {
253+
start, _, err := http_range.ParseContentRange(header.Get("Content-Range"))
254+
return err == nil && start == offset
255+
}
256+
257+
type callbackBody struct {
258+
body io.ReadCloser
259+
release func()
260+
once sync.Once
261+
mu sync.Mutex
262+
stop func() bool
263+
}
264+
265+
func newCallbackBody(ctx context.Context, body io.ReadCloser, release func()) *callbackBody {
266+
b := &callbackBody{body: body, release: release}
267+
stop := context.AfterFunc(ctx, func() { _ = b.Close() })
268+
b.mu.Lock()
269+
b.stop = stop
270+
b.mu.Unlock()
271+
return b
272+
}
273+
274+
func (b *callbackBody) Read(p []byte) (int, error) {
275+
n, err := b.body.Read(p)
276+
if err != nil {
277+
_ = b.Close()
278+
}
279+
return n, err
280+
}
281+
282+
func (b *callbackBody) Close() error {
283+
var err error
284+
b.once.Do(func() {
285+
b.mu.Lock()
286+
stop := b.stop
287+
b.mu.Unlock()
288+
if stop != nil {
289+
stop()
290+
}
291+
err = b.body.Close()
292+
b.release()
293+
})
294+
return err
295+
}

0 commit comments

Comments
 (0)