Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/50153-cancel-batch-download-on-failure
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Fixed GitOps software batch uploads continuing to download remaining packages after one package failed.
10 changes: 9 additions & 1 deletion ee/server/service/software_installers.go
Original file line number Diff line number Diff line change
Expand Up @@ -3443,7 +3443,7 @@ func (svc *Service) softwareBatchUpload(
manualAgentInstall = team.Config.MDM.MacOSSetup.ManualAgentInstall.Value
}

var g errgroup.Group
g, gctx := errgroup.WithContext(ctx)
g.SetLimit(1) // TODO: consider whether we can increase this limit, see https://github.com/fleetdm/fleet/issues/22704#issuecomment-2397407837

// the reason for this struct with extra installers support is that:
Expand Down Expand Up @@ -3486,6 +3486,14 @@ func (svc *Service) softwareBatchUpload(
i, p := i, p

g.Go(func() error {
// A single failure fails the whole batch, so don't process (and download)
// the remaining payloads. With the limit of 1 above, gctx is always
// cancelled before the next goroutine starts; in-flight work isn't
// interrupted if the limit is raised.
if err := gctx.Err(); err != nil {
return err
}

// NOTE: cannot defer tfr.Close() here because the reader needs to be
// available after the goroutine completes. Instead, all temp file
// readers are collected in toBeClosedTFRs and will have their Close
Expand Down
52 changes: 52 additions & 0 deletions ee/server/service/software_installers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3201,6 +3201,58 @@ func TestSoftwareBatchUploadFMAVersionCache(t *testing.T) {
}
}

func TestBatchSetSoftwareInstallersStopsAfterFirstFailure(t *testing.T) {
dev_mode.SetOverride("FLEET_DEV_BATCH_RETRY_INTERVAL", "1ms", t)

kvs, getKey := inMemoryKeyValueStore()

var downloads [3]atomic.Int32
installerSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/a.pkg":
downloads[0].Add(1)
case "/b.pkg":
downloads[1].Add(1)
case "/c.pkg":
downloads[2].Add(1)
}
http.Error(w, "download failed", http.StatusInternalServerError)
}))
t.Cleanup(installerSrv.Close)

ds := new(mock.Store)
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
return &fleet.AppConfig{}, nil
}
ds.GetTeamsWithInstallerByHashFunc = func(ctx context.Context, sha256, url string) (map[uint][]*fleet.ExistingSoftwareInstaller, error) {
return nil, nil
}

svc := newTestService(t, ds)
svc.keyValueStore = kvs
svc.logger = slog.New(slog.NewTextHandler(io.Discard, nil))

ctx := viewer.NewContext(t.Context(), viewer.Viewer{
User: &fleet.User{GlobalRole: new(fleet.RoleAdmin)},
})

payloads := []*fleet.SoftwareInstallerPayload{
{URL: installerSrv.URL + "/a.pkg", AlwaysDownload: true},
{URL: installerSrv.URL + "/b.pkg", AlwaysDownload: true},
{URL: installerSrv.URL + "/c.pkg", AlwaysDownload: true},
}
requestUUID, err := svc.BatchSetSoftwareInstallers(ctx, "", payloads, true)
require.NoError(t, err)

require.Eventually(t, func() bool {
status := getKey(batchSoftwarePrefix + requestUUID)
return status != nil && strings.HasPrefix(*status, batchSetFailedPrefix)
}, 10*time.Second, 50*time.Millisecond, "batch never failed")
require.Positive(t, downloads[0].Load())
require.Zero(t, downloads[1].Load())
require.Zero(t, downloads[2].Load())
}

func TestGetBatchSetSoftwareInstallersResultMissingDeletedKey(t *testing.T) {
t.Parallel()

Expand Down
Loading