From 374f48e45d78252837720f03291c78b89b20afe8 Mon Sep 17 00:00:00 2001 From: Michael Alt Date: Wed, 14 Oct 2026 20:37:21 +0200 Subject: [PATCH 1/4] Add Azure Blob Storage support for software installers Add Azure Blob Storage as an alternative to S3 for storing software installers, bootstrap packages, software title icons and org logos, configured via the `cloud_storage.azure.files_*` keys (or the `FLEET_CLOUD_STORAGE_AZURE_FILES_*` environment variables). - New `server/datastore/azure` package with installer, bootstrap package, title icon and org logo stores, including optional SAS-signed download URLs. - Group the config under `FleetConfig.CloudStorage` so it round-trips through YAML and the environment. - Select the software installers backend through a single `FleetConfig.SoftwareInstallersBackend()` seam, with a check that only one of S3 or Azure is configured. S3 behavior and config are unchanged, and moving S3 under `cloud_storage` later only touches that seam. - Extract per-backend store constructors in `cmd/fleet/serve.go`. - Route signed-URL and staged-upload gating through config accessors so the Azure signed-URL option is honored. - Run the Azure store tests against Azurite in CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/test-go-suite.yaml | 11 +- .../azure-blob-storage-software-installers.md | 1 + cmd/fleet/serve.go | 238 ++++++++++++------ cmd/fleet/serve_test.go | 2 +- docker-compose.yml | 10 + ee/server/service/in_house_apps.go | 2 +- ee/server/service/software_installers.go | 4 +- go.mod | 25 +- go.sum | 65 +++-- server/config/config.go | 138 ++++++++++ server/config/config_test.go | 117 ++++++++- server/datastore/azure/azureblobstore.go | 113 +++++++++ server/datastore/azure/bootstrap_package.go | 25 ++ .../datastore/azure/bootstrap_package_test.go | 155 ++++++++++++ server/datastore/azure/common_file_store.go | 165 ++++++++++++ server/datastore/azure/org_logo.go | 57 +++++ server/datastore/azure/org_logo_test.go | 79 ++++++ server/datastore/azure/signed_url_test.go | 54 ++++ server/datastore/azure/software_installer.go | 26 ++ .../azure/software_installer_test.go | 155 ++++++++++++ server/datastore/azure/software_title_icon.go | 25 ++ .../azure/software_title_icon_test.go | 155 ++++++++++++ server/datastore/azure/testing_utils_test.go | 76 ++++++ server/service/appconfig.go | 2 +- tools/dibble/go.sum | 8 +- tools/gitops-auto-complete/go.mod | 4 +- tools/gitops-auto-complete/go.sum | 21 +- tools/upgrade/go.mod | 6 +- tools/upgrade/go.sum | 33 ++- 29 files changed, 1616 insertions(+), 156 deletions(-) create mode 100644 changes/azure-blob-storage-software-installers.md create mode 100644 server/datastore/azure/azureblobstore.go create mode 100644 server/datastore/azure/bootstrap_package.go create mode 100644 server/datastore/azure/bootstrap_package_test.go create mode 100644 server/datastore/azure/common_file_store.go create mode 100644 server/datastore/azure/org_logo.go create mode 100644 server/datastore/azure/org_logo_test.go create mode 100644 server/datastore/azure/signed_url_test.go create mode 100644 server/datastore/azure/software_installer.go create mode 100644 server/datastore/azure/software_installer_test.go create mode 100644 server/datastore/azure/software_title_icon.go create mode 100644 server/datastore/azure/software_title_icon_test.go create mode 100644 server/datastore/azure/testing_utils_test.go diff --git a/.github/workflows/test-go-suite.yaml b/.github/workflows/test-go-suite.yaml index 63568f9fdae..257d6fc07e4 100644 --- a/.github/workflows/test-go-suite.yaml +++ b/.github/workflows/test-go-suite.yaml @@ -74,7 +74,8 @@ jobs: if [[ "${{ inputs.suite }}" == "main" ]]; then echo "CI_TEST_PKG=main" >> $GITHUB_ENV echo "NEED_DOCKER=1" >> $GITHUB_ENV - echo "DOCKER_COMMAND=${{ env.DOCKER_COMMAND }} localstack" >> $GITHUB_ENV + echo "DOCKER_COMMAND=${{ env.DOCKER_COMMAND }} localstack azurite" >> $GITHUB_ENV + echo "AZURE_STORAGE_TEST=1" >> $GITHUB_ENV elif [[ "${{ inputs.suite }}" == "fast" ]]; then # DO NOT add any dependencies in this test suite. echo "CI_TEST_PKG=${{ inputs.suite }}" >> $GITHUB_ENV @@ -254,6 +255,14 @@ jobs: echo "Waiting for LocalStack..." timeout 60 bash -c 'until curl -sf http://localhost:4566/_localstack/health; do sleep 2; done' echo "LocalStack is ready" + + - name: Wait for Azurite + if: ${{ env.NEED_DOCKER && contains(env.DOCKER_COMMAND, 'azurite') }} + run: | + echo "Waiting for Azurite..." + # Azurite has no health endpoint; omitting -f means any HTTP response (even 400) counts as "up". + timeout 60 bash -c 'until curl -s -o /dev/null http://127.0.0.1:10000/devstoreaccount1; do sleep 2; done' + echo "Azurite is ready" - name: Generate test schema if: ${{ env.GENERATE_TEST_SCHEMA }} diff --git a/changes/azure-blob-storage-software-installers.md b/changes/azure-blob-storage-software-installers.md new file mode 100644 index 00000000000..db3e7989c4a --- /dev/null +++ b/changes/azure-blob-storage-software-installers.md @@ -0,0 +1 @@ +- Added Azure Blob Storage support (via new `FLEET_CLOUD_STORAGE_AZURE_FILES_*` environment variables) as an alternative to S3 for storing software installers, bootstrap packages, software title icons, and org logos. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 12b538312a9..1f506a01e87 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -54,6 +54,7 @@ import ( configpkg "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" licensectx "github.com/fleetdm/fleet/v4/server/contexts/license" + "github.com/fleetdm/fleet/v4/server/datastore/azure" "github.com/fleetdm/fleet/v4/server/datastore/cached_mysql" "github.com/fleetdm/fleet/v4/server/datastore/failing" "github.com/fleetdm/fleet/v4/server/datastore/filesystem" @@ -543,7 +544,11 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev initFatal(err, "initializing ee android service") } - orgLogoStore := initOrgLogoStore(ctx, config.S3, mds, logger) + // Must run before initOrgLogoStore, which picks the object store backend regardless of license tier. + config.ValidateSoftwareInstallersBackend(initFatal) + softwareInstallersBackend, _ := config.SoftwareInstallersBackend() + + orgLogoStore := initOrgLogoStore(ctx, softwareInstallersBackend, config.S3, config.CloudStorage.Azure, mds, logger) svc, err = service.NewService( ctx, @@ -589,84 +594,19 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev if license.IsPremium() { hydrantService := est.NewService(est.WithLogger(logger)) profileMatcher := apple_mdm.NewProfileMatcher(redisPool) - if config.S3.SoftwareInstallersBucket != "" { - if config.S3.BucketsAndPrefixesMatch() { - logger.WarnContext(ctx, - "the S3 buckets and prefixes for carves and software installers appear to be identical, this can cause issues") - } - // Extract the CloudFront URL signer before creating the S3 stores. - config.S3.ValidateCloudFrontURL(initFatal) - config.S3.ValidateSoftwareInstallersSignedURL(initFatal) - if config.S3.SoftwareInstallersCloudFrontURLSigningPrivateKey != "" { - // Strip newlines from private key - signingPrivateKey := strings.ReplaceAll(config.S3.SoftwareInstallersCloudFrontURLSigningPrivateKey, "\\n", "\n") - privateKey, err := cryptoutil.ParsePrivateKey([]byte(signingPrivateKey), - "CloudFront URL signing private key") - if err != nil { - initFatal(err, "parsing CloudFront URL signing private key") - } - var ok bool - config.S3.SoftwareInstallersCloudFrontSigner, ok = privateKey.(crypto.Signer) - if !ok { - initFatal(errors.New("CloudFront URL signing private key is not a crypto.Signer"), - "parsing CloudFront URL signing private key") - } - } - store, err := s3.NewSoftwareInstallerStore(config.S3) - if err != nil { - initFatal(err, "initializing S3 software installer store") - } - softwareInstallStore = store - logger.InfoContext(ctx, "using S3 software installer store", "bucket", config.S3.SoftwareInstallersBucket) - - bstore, err := s3.NewBootstrapPackageStore(config.S3) - if err != nil { - initFatal(err, "initializing S3 bootstrap package store") - } - bootstrapPackageStore = bstore - logger.InfoContext(ctx, "using S3 bootstrap package store", "bucket", config.S3.SoftwareInstallersBucket) - - softwareTitleIconStore, err = s3.NewSoftwareTitleIconStore(config.S3) - if err != nil { - initFatal(err, "initializing S3 software title icon store") - } - logger.InfoContext(ctx, "using S3 software title icon store", "bucket", config.S3.SoftwareInstallersBucket) - - stagedUploadStore, err = s3.NewStagedUploadStore(config.S3) - if err != nil { - initFatal(err, "initializing S3 staged upload store") - } - } else { - installerDir := os.TempDir() - if dir := os.Getenv("FLEET_SOFTWARE_INSTALLER_STORE_DIR"); dir != "" { - installerDir = dir - } - store, err := filesystem.NewSoftwareInstallerStore(installerDir) - if err != nil { - logger.ErrorContext(ctx, "failed to configure local filesystem software installer store", "err", err) - softwareInstallStore = failing.NewFailingSoftwareInstallerStore() - } else { - softwareInstallStore = store - logger.InfoContext(ctx, - "using local filesystem software installer store, this is not suitable for multi-container deployments", "directory", - installerDir) - } - - iconDir := os.TempDir() - if dir := os.Getenv("FLEET_SOFTWARE_TITLE_ICON_STORE_DIR"); dir != "" { - iconDir = dir - } - iconStore, err := filesystem.NewSoftwareTitleIconStore(iconDir) - if err != nil { - logger.ErrorContext(ctx, "failed to configure local filesystem software title icon store", "err", err) - softwareTitleIconStore = failing.NewFailingSoftwareTitleIconStore() - } else { - softwareTitleIconStore = iconStore - logger.WarnContext(ctx, - "using local filesystem software title icon store, this is not suitable for production use", "directory", - iconDir) - } + var stores softwareInstallerStores + switch softwareInstallersBackend { + case configpkg.StorageBackendAzure: + stores = newAzureSoftwareInstallerStores(ctx, config.CloudStorage.Azure, logger) + case configpkg.StorageBackendS3: + stores = newS3SoftwareInstallerStores(ctx, &config, logger) + default: + stores = newFilesystemSoftwareInstallerStores(ctx, logger) } + softwareInstallStore = stores.installer + bootstrapPackageStore = stores.bootstrap + softwareTitleIconStore = stores.icon + stagedUploadStore = stores.stagedUpload distributedLock = redis_lock.NewLock(redisPool) svc, err = eeservice.NewService( @@ -1320,10 +1260,18 @@ func createChartBoundedContext(dbConns *common_mysql.DBConnections, svc fleet.Se } // initOrgLogoStore builds the OrgLogoStore implementation appropriate for the deployment: -// - S3 when a software installers bucket is configured (shared bucket, distinct prefix) +// - the configured object store backend (S3 or Azure Blob) when one is selected for the software installers group // - otherwise a database-backed store, so custom org logos work without object storage or a writable filesystem. -func initOrgLogoStore(ctx context.Context, s3Config configpkg.S3Config, ds *mysql.Datastore, logger *slog.Logger) fleet.OrgLogoStore { - if s3Config.SoftwareInstallersBucket != "" { +func initOrgLogoStore(ctx context.Context, backend configpkg.StorageBackend, s3Config configpkg.S3Config, azureConfig configpkg.AzureConfig, ds *mysql.Datastore, logger *slog.Logger) fleet.OrgLogoStore { + switch backend { + case configpkg.StorageBackendAzure: + store, err := azure.NewOrgLogoStore(azureConfig) + if err != nil { + initFatal(err, "initializing Azure Blob org logo store") + } + logger.InfoContext(ctx, "using Azure Blob org logo store", "container", azureConfig.SoftwareInstallersContainer) + return store + case configpkg.StorageBackendS3: store, err := s3.NewOrgLogoStore(s3Config) if err != nil { initFatal(err, "initializing S3 org logo store") @@ -1335,6 +1283,134 @@ func initOrgLogoStore(ctx context.Context, s3Config configpkg.S3Config, ds *mysq return ds.NewOrgLogoStore() } +// softwareInstallerStores groups the stores backing the software installers +// group for a given storage backend. Stores a backend doesn't support are nil. +type softwareInstallerStores struct { + installer fleet.SoftwareInstallerStore + bootstrap fleet.MDMBootstrapPackageStore + icon fleet.SoftwareTitleIconStore + stagedUpload fleet.StagedUploadStore +} + +func newAzureSoftwareInstallerStores(ctx context.Context, cfg configpkg.AzureConfig, logger *slog.Logger) softwareInstallerStores { + var stores softwareInstallerStores + + installer, err := azure.NewSoftwareInstallerStore(cfg) + if err != nil { + initFatal(err, "initializing Azure Blob software installer store") + } + stores.installer = installer + logger.InfoContext(ctx, "using Azure Blob software installer store", "container", cfg.SoftwareInstallersContainer) + + bstore, err := azure.NewBootstrapPackageStore(cfg) + if err != nil { + initFatal(err, "initializing Azure Blob bootstrap package store") + } + stores.bootstrap = bstore + logger.InfoContext(ctx, "using Azure Blob bootstrap package store", "container", cfg.SoftwareInstallersContainer) + + stores.icon, err = azure.NewSoftwareTitleIconStore(cfg) + if err != nil { + initFatal(err, "initializing Azure Blob software title icon store") + } + logger.InfoContext(ctx, "using Azure Blob software title icon store", "container", cfg.SoftwareInstallersContainer) + + return stores +} + +// newS3SoftwareInstallerStores also stores the parsed CloudFront URL signer in +// the provided config, as the service layer reads it from there. +func newS3SoftwareInstallerStores(ctx context.Context, config *configpkg.FleetConfig, logger *slog.Logger) softwareInstallerStores { + var stores softwareInstallerStores + + if config.S3.BucketsAndPrefixesMatch() { + logger.WarnContext(ctx, + "the S3 buckets and prefixes for carves and software installers appear to be identical, this can cause issues") + } + // Extract the CloudFront URL signer before creating the S3 stores. + config.S3.ValidateCloudFrontURL(initFatal) + config.S3.ValidateSoftwareInstallersSignedURL(initFatal) + if config.S3.SoftwareInstallersCloudFrontURLSigningPrivateKey != "" { + // Strip newlines from private key + signingPrivateKey := strings.ReplaceAll(config.S3.SoftwareInstallersCloudFrontURLSigningPrivateKey, "\\n", "\n") + privateKey, err := cryptoutil.ParsePrivateKey([]byte(signingPrivateKey), + "CloudFront URL signing private key") + if err != nil { + initFatal(err, "parsing CloudFront URL signing private key") + } + var ok bool + config.S3.SoftwareInstallersCloudFrontSigner, ok = privateKey.(crypto.Signer) + if !ok { + initFatal(errors.New("CloudFront URL signing private key is not a crypto.Signer"), + "parsing CloudFront URL signing private key") + } + } + installer, err := s3.NewSoftwareInstallerStore(config.S3) + if err != nil { + initFatal(err, "initializing S3 software installer store") + } + stores.installer = installer + logger.InfoContext(ctx, "using S3 software installer store", "bucket", config.S3.SoftwareInstallersBucket) + + bstore, err := s3.NewBootstrapPackageStore(config.S3) + if err != nil { + initFatal(err, "initializing S3 bootstrap package store") + } + stores.bootstrap = bstore + logger.InfoContext(ctx, "using S3 bootstrap package store", "bucket", config.S3.SoftwareInstallersBucket) + + stores.icon, err = s3.NewSoftwareTitleIconStore(config.S3) + if err != nil { + initFatal(err, "initializing S3 software title icon store") + } + logger.InfoContext(ctx, "using S3 software title icon store", "bucket", config.S3.SoftwareInstallersBucket) + + stores.stagedUpload, err = s3.NewStagedUploadStore(config.S3) + if err != nil { + initFatal(err, "initializing S3 staged upload store") + } + + return stores +} + +// newFilesystemSoftwareInstallerStores is the fallback when no object store is +// configured. It is not suitable for multi-container deployments. +func newFilesystemSoftwareInstallerStores(ctx context.Context, logger *slog.Logger) softwareInstallerStores { + var stores softwareInstallerStores + + installerDir := os.TempDir() + if dir := os.Getenv("FLEET_SOFTWARE_INSTALLER_STORE_DIR"); dir != "" { + installerDir = dir + } + store, err := filesystem.NewSoftwareInstallerStore(installerDir) + if err != nil { + logger.ErrorContext(ctx, "failed to configure local filesystem software installer store", "err", err) + stores.installer = failing.NewFailingSoftwareInstallerStore() + } else { + stores.installer = store + logger.InfoContext(ctx, + "using local filesystem software installer store, this is not suitable for multi-container deployments", "directory", + installerDir) + } + + iconDir := os.TempDir() + if dir := os.Getenv("FLEET_SOFTWARE_TITLE_ICON_STORE_DIR"); dir != "" { + iconDir = dir + } + iconStore, err := filesystem.NewSoftwareTitleIconStore(iconDir) + if err != nil { + logger.ErrorContext(ctx, "failed to configure local filesystem software title icon store", "err", err) + stores.icon = failing.NewFailingSoftwareTitleIconStore() + } else { + stores.icon = iconStore + logger.WarnContext(ctx, + "using local filesystem software title icon store, this is not suitable for production use", "directory", + iconDir) + } + + return stores +} + func createActivityBoundedContext(svc fleet.Service, ds fleet.Datastore, dbConns *common_mysql.DBConnections, logger *slog.Logger) (activity_api.Service, endpointer.HandlerRoutesFunc) { legacyAuthorizer, err := authz.NewAuthorizer() if err != nil { diff --git a/cmd/fleet/serve_test.go b/cmd/fleet/serve_test.go index 433bed36d13..27b260b4241 100644 --- a/cmd/fleet/serve_test.go +++ b/cmd/fleet/serve_test.go @@ -1633,6 +1633,6 @@ func TestInitOrgLogoStore(t *testing.T) { // the database-backed implementation. NewOrgLogoStore does no DB work at // construction, so a zero-value Datastore is enough to verify selection. ds := &mysql.Datastore{} - store := initOrgLogoStore(t.Context(), config.S3Config{}, ds, logger) + store := initOrgLogoStore(t.Context(), config.StorageBackendLocal, config.S3Config{}, config.AzureConfig{}, ds, logger) require.IsType(t, ds.NewOrgLogoStore(), store) } diff --git a/docker-compose.yml b/docker-compose.yml index 275b5ce7390..86bd96969f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,6 +156,16 @@ services: volumes: - data-s3:/data:rw + # Azurite emulator for Azure Blob Storage (software installers/bootstrap packages/icons/org logos) + azurite: + image: mcr.microsoft.com/azure-storage/azurite:3.35.0 + command: ["azurite-blob", "--blobHost", "0.0.0.0", "--blobPort", "10000", "--skipApiVersionCheck"] + ports: + - "127.0.0.1:${FLEET_AZURITE_PORT:-10000}:10000" + volumes: + - data-azurite:/data:rw + volumes: mysql-persistent-volume: data-s3: + data-azurite: diff --git a/ee/server/service/in_house_apps.go b/ee/server/service/in_house_apps.go index 1f9ef839db9..15afa1f9838 100644 --- a/ee/server/service/in_house_apps.go +++ b/ee/server/service/in_house_apps.go @@ -210,7 +210,7 @@ func (svc *Service) GetInHouseAppManifest(ctx context.Context, titleID uint, tok appConfig.MDMUrl(), titleID, token, ) - if svc.config.S3.SoftwareInstallersCloudFrontSigner != nil || svc.config.S3.SoftwareInstallersSignedURL { + if svc.config.SoftwareInstallersSignedURLEnabled() { signedURL, err := svc.softwareInstallStore.Sign(ctx, meta.StorageID, fleet.InHouseAppSignedURLExpiry) if err != nil { // We log the error and continue to send the Fleet server URL for the in-house app diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index f993ee99a15..cf47043520c 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -1598,8 +1598,8 @@ func (svc *Service) GetSoftwareInstallDetails(ctx context.Context, installUUID s return nil, err } - // Sign the download URL when CloudFront signing (premium-only) or GCS presigning is configured. - if svc.config.S3.SoftwareInstallersCloudFrontSigner != nil || svc.config.S3.SoftwareInstallersSignedURL { + // Sign the download URL when CloudFront signing (premium-only), GCS presigning or Azure SAS signing is configured. + if svc.config.SoftwareInstallersSignedURLEnabled() { // Sign the URL for the installer installerURL, err := svc.getSoftwareInstallURL(ctx, details.InstallerID) if err != nil { diff --git a/go.mod b/go.mod index 55585358f39..b772d45b46f 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go/pubsub v1.50.1 fyne.io/systray v1.10.1-0.20240111184411-11c585fff98d github.com/AbGuthrie/goquery/v2 v2.0.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1 github.com/Azure/go-ntlmssp v0.1.1 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Masterminds/semver v1.5.0 @@ -46,7 +47,7 @@ require ( github.com/containerd/containerd v1.7.36 github.com/crewjam/saml v0.5.1 github.com/danieljoos/wincred v1.2.1 - github.com/davecgh/go-spew v1.1.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dgraph-io/badger/v2 v2.2007.4 github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e github.com/docker/go-units v0.5.0 @@ -71,7 +72,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 github.com/gofrs/flock v0.12.1 github.com/golang-jwt/jwt/v4 v4.5.2 - github.com/golang/snappy v0.0.4 + github.com/golang/snappy v1.0.0 github.com/gomodule/oauth1 v0.2.0 github.com/gomodule/redigo v1.8.9 github.com/google/go-cmp v0.7.0 @@ -91,7 +92,7 @@ require ( github.com/igm/sockjs-go/v3 v3.0.2 github.com/jmoiron/sqlx v1.3.5 github.com/josephspurrier/goversioninfo v1.4.0 - github.com/klauspost/compress v1.18.6 + github.com/klauspost/compress v1.19.2 github.com/kolide/launcher v1.0.12 github.com/lib/pq v1.10.9 github.com/macadmins/osquery-extension v1.5.4 @@ -118,7 +119,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pelletier/go-toml/v2 v2.2.4 github.com/pkg/errors v0.9.1 - github.com/pmezard/go-difflib v1.0.0 + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/prometheus/client_golang v1.21.1 github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/realclientip/realclientip-go v1.0.0 @@ -170,7 +171,7 @@ require ( go.opentelemetry.io/otel/trace v1.46.0 go.step.sm/crypto v0.77.1 golang.org/x/crypto v0.55.0 - golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f + golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 golang.org/x/image v0.45.0 golang.org/x/mod v0.40.0 golang.org/x/net v0.58.0 @@ -205,6 +206,8 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/AlekSi/pointer v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -213,6 +216,7 @@ require ( github.com/akavel/rsrc v0.10.2 // indirect github.com/antchfx/xpath v1.3.6 // indirect github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op // indirect + github.com/apache/arrow-go/v18 v18.7.0 // indirect github.com/apache/thrift v0.24.0 // indirect github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect github.com/armon/go-radix v1.0.0 // indirect @@ -238,8 +242,7 @@ require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect - github.com/clipperhouse/stringish v0.1.1 // indirect - github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cloudflare/circl v1.6.3 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/containerd/api v1.8.0 // indirect @@ -278,10 +281,12 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/gobwas/glob v0.2.3 // indirect + github.com/goccy/go-json v0.10.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.5 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/go-tpm-tools v0.4.7 // indirect github.com/google/rpmpack v0.7.1 // indirect @@ -304,14 +309,14 @@ require ( github.com/jonboulle/clockwork v0.5.0 // indirect github.com/kevinburke/go-bindata v3.24.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kolide/kit v0.0.0-20221107170827-fb85e3d59eab // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.20 // indirect github.com/mattn/go-tty v0.0.3 // indirect github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -333,6 +338,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opencontainers/selinux v1.13.1 // indirect github.com/oschwald/maxminddb-golang v1.10.0 // indirect + github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect @@ -364,6 +370,7 @@ require ( github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yashtewari/glob-intersection v0.2.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.elastic.co/fastjson v1.1.0 // indirect go.mozilla.org/pkcs7 v0.9.0 // indirect diff --git a/go.sum b/go.sum index f6fb2536eab..43f2496f771 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,21 @@ github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59M github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w= github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1 h1:gkBLVmB3Z/HnGP/Jo4o12/RDpi0agnKav6sCKsX5Vu0= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1/go.mod h1:e3/1P5K+jIUi9JevDRklq/tFeTvbBb75bNAjU4xd31w= github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw= github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= @@ -86,6 +99,8 @@ github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtC github.com/akavel/rsrc v0.10.2 h1:Zxm8V5eI1hW4gGaYsJQUhxpjkENuG91ki8B4zCrvEsw= github.com/akavel/rsrc v0.10.2/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= @@ -98,6 +113,8 @@ github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op h1:kpBdlEPbRvff0mDD1gk7o9BhI16b9p5yYAXRlidpqJE= github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= @@ -215,10 +232,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -260,8 +275,9 @@ github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1 github.com/danieljoos/wincred v1.2.1 h1:dl9cBrupW8+r5250DYkYxocLeZ1Y4vB1kxgtjxw8GQs= github.com/danieljoos/wincred v1.2.1/go.mod h1:uGaFL9fDn3OLTvzCGulzE+SzjEe5NGlh5FdCcyfPwps= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= @@ -410,6 +426,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055 h1:UfcDMw41lSx3XM7UvD1i7Fsu3rMgD55OU5LYwLoR/Yk= github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -420,6 +438,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= @@ -447,16 +467,16 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo= github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08= github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= -github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= -github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -578,10 +598,10 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kolide/kit v0.0.0-20221107170827-fb85e3d59eab h1:KVR7cs+oPyy85i+8t1ZaNSy1bymCy5FuWyt51pdrXu4= @@ -631,8 +651,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -733,10 +753,14 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= +github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -745,8 +769,9 @@ github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942 h1:A7GG7zcGjl3jqAqGPmcNjd github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= @@ -917,6 +942,10 @@ github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= @@ -1008,8 +1037,8 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/server/config/config.go b/server/config/config.go index 25484096be0..87ad433d00b 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -541,6 +541,11 @@ type LambdaConfig struct { AuditFunction string `yaml:"audit_function"` } +// CloudStorageConfig groups the configs of cloud object stores. +type CloudStorageConfig struct { + Azure AzureConfig `yaml:"azure"` +} + // S3Config defines config to enable file carving storage to an S3 bucket type S3Config struct { Bucket string `yaml:"bucket"` @@ -771,6 +776,124 @@ type S3CloudFrontConfig struct { Signer crypto.Signer } +// AzureConfig defines config to enable Azure Blob Storage as an alternative +// object store to S3 for the software installers group (installers, +// bootstrap packages, software title icons, org logos). It is purely +// additive: when unset, storage selection and behavior for S3/GCS deployments +// are unchanged. +type AzureConfig struct { + SoftwareInstallersAccountName string `yaml:"files_account_name"` + SoftwareInstallersAccountKey string `yaml:"files_account_key"` + SoftwareInstallersContainer string `yaml:"files_container"` + // SoftwareInstallersContainerURL overrides the default + // https://.blob.core.windows.net endpoint, for Azurite or other + // Azure-compatible emulators in tests. + SoftwareInstallersContainerURL string `yaml:"files_container_url"` + // SoftwareInstallersSignedURL, when true, makes Fleet hand out a SAS-signed + // GET URL (instead of proxying the bytes) for software installer, in-house + // app and bootstrap package downloads. This is the Azure counterpart to the + // S3 CloudFront/GCS signed URL configs. + SoftwareInstallersSignedURL bool `yaml:"files_signed_url"` +} + +// SoftwareInstallersToInternalCfg creates an internal Azure config struct from +// the ingested Azure config for the software installers group (installers, +// bootstrap packages, icons, org logos). +func (a AzureConfig) SoftwareInstallersToInternalCfg() AzureConfigInternal { + return AzureConfigInternal{ + AccountName: a.SoftwareInstallersAccountName, + AccountKey: a.SoftwareInstallersAccountKey, + Container: a.SoftwareInstallersContainer, + ContainerURL: a.SoftwareInstallersContainerURL, + SignedURL: a.SoftwareInstallersSignedURL, + } +} + +// AzureConfigInternal is used internally to configure an Azure Blob Storage +// client for a given purpose (e.g. software installers). +type AzureConfigInternal struct { + AccountName string + AccountKey string + Container string + ContainerURL string + SignedURL bool +} + +// StorageBackend identifies the object store backing the software installers +// group (installers, bootstrap packages, software title icons, org logos). +type StorageBackend string + +const ( + // StorageBackendLocal means no object store is configured; Fleet falls back + // to the local filesystem/database stores. + StorageBackendLocal StorageBackend = "" + StorageBackendS3 StorageBackend = "s3" + StorageBackendAzure StorageBackend = "azure" +) + +type storageBackendCandidate struct { + backend StorageBackend + configured bool + // key is the name of the config key that selects this backend, used in + // error messages. + key string +} + +// softwareInstallersBackendCandidates is the single place that knows where +// each backend's selecting config lives. To add a backend (or to move S3 under +// CloudStorage), only this list needs to change. +func (c FleetConfig) softwareInstallersBackendCandidates() []storageBackendCandidate { + return []storageBackendCandidate{ + {StorageBackendS3, c.S3.SoftwareInstallersBucket != "", "s3_software_installers_bucket"}, + {StorageBackendAzure, c.CloudStorage.Azure.SoftwareInstallersContainer != "", "cloud_storage_azure_files_container"}, + } +} + +// SoftwareInstallersBackend returns the object store configured for the +// software installers group, or an error if more than one is configured since +// only one backend can be selected. +func (c FleetConfig) SoftwareInstallersBackend() (StorageBackend, error) { + var keys []string + backend := StorageBackendLocal + for _, cand := range c.softwareInstallersBackendCandidates() { + if cand.configured { + backend = cand.backend + keys = append(keys, "`"+cand.key+"`") + } + } + switch { + case len(keys) > 2: + return StorageBackendLocal, fmt.Errorf("Couldn't configure. %s cannot all be set; choose one storage backend for software installers.", strings.Join(keys, ", ")) + case len(keys) == 2: + return StorageBackendLocal, fmt.Errorf("Couldn't configure. %s and %s cannot both be set; choose one storage backend for software installers.", keys[0], keys[1]) + } + return backend, nil +} + +// ValidateSoftwareInstallersBackend fails initialization if more than one +// object store is configured for the software installers group. +func (c FleetConfig) ValidateSoftwareInstallersBackend(initFatal func(err error, msg string)) { + if _, err := c.SoftwareInstallersBackend(); err != nil { + initFatal(err, "software installers storage") + } +} + +// SoftwareInstallersSignedURLEnabled reports whether downloads of software +// installers and in-house apps should be handed out as signed URLs (pointing +// directly at the object store) instead of proxying the bytes through Fleet. +func (c FleetConfig) SoftwareInstallersSignedURLEnabled() bool { + if c.S3.SoftwareInstallersCloudFrontSigner != nil || c.S3.SoftwareInstallersSignedURL { + return true + } + return c.CloudStorage.Azure.SoftwareInstallersContainer != "" && c.CloudStorage.Azure.SoftwareInstallersSignedURL +} + +// StagedUploadAvailable reports whether clients can upload packages straight +// to object storage. Only supported with S3-compatible storage for now. +func (c FleetConfig) StagedUploadAvailable() bool { + return c.S3.SoftwareInstallersSignedURL && c.S3.SoftwareInstallersBucket != "" +} + // PubSubConfig defines configs the for Google PubSub logging plugin type PubSubConfig struct { Project string `json:"project"` @@ -922,6 +1045,7 @@ type FleetConfig struct { Kinesis KinesisConfig Lambda LambdaConfig S3 S3Config + CloudStorage CloudStorageConfig `yaml:"cloud_storage"` Email EmailConfig SES SESConfig PubSub PubSubConfig @@ -2002,6 +2126,13 @@ func (man Manager) addConfigs() { man.addConfigBool("s3.software_installers_gcs_signed_url", false, "Hand out presigned GCS URLs for installer/in-house app/bootstrap uploads and downloads instead of proxying bytes (requires a storage.googleapis.com endpoint)") man.addConfigBool("s3.software_installers_signed_url", false, "Deprecated: use s3.software_installers_gcs_signed_url") + // Azure Blob Storage for software installers + man.addConfigString("cloud_storage.azure.files_account_name", "", "Azure Storage account name") + man.addConfigString("cloud_storage.azure.files_account_key", "", "Azure Storage account shared key") + man.addConfigString("cloud_storage.azure.files_container", "", "Azure Storage container where to store uploaded software installers") + man.addConfigString("cloud_storage.azure.files_container_url", "", "Azure Blob Storage endpoint override (leave blank for the default https://.blob.core.windows.net; used for Azurite/testing)") + man.addConfigBool("cloud_storage.azure.files_signed_url", false, "Hand out SAS-signed URLs for installer/in-house app/bootstrap downloads instead of proxying bytes") + // PubSub man.addConfigString("pubsub.project", "", "Google Cloud Project to use") man.addConfigString("pubsub.status_topic", "", "PubSub topic for status logs") @@ -2439,6 +2570,13 @@ func (man Manager) LoadConfig() FleetConfig { StsExternalID: man.getConfigString("lambda.sts_external_id"), }, S3: man.loadS3Config(), + CloudStorage: CloudStorageConfig{Azure: AzureConfig{ + SoftwareInstallersAccountName: man.getConfigString("cloud_storage.azure.files_account_name"), + SoftwareInstallersAccountKey: man.getConfigString("cloud_storage.azure.files_account_key"), + SoftwareInstallersContainer: man.getConfigString("cloud_storage.azure.files_container"), + SoftwareInstallersContainerURL: man.getConfigString("cloud_storage.azure.files_container_url"), + SoftwareInstallersSignedURL: man.getConfigBool("cloud_storage.azure.files_signed_url"), + }}, Email: EmailConfig{ EmailBackend: man.getConfigString("email.backend"), }, diff --git a/server/config/config_test.go b/server/config/config_test.go index 2bb41014ab9..4bffdc91e6f 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "bytes" "context" + "crypto" "crypto/tls" "fmt" "io" @@ -49,8 +50,8 @@ func TestConfigRoundtrip(t *testing.T) { // true. original := &FleetConfig{} v := reflect.ValueOf(original) - for conf_index := 0; conf_index < v.Elem().NumField(); conf_index++ { - conf_v := v.Elem().Field(conf_index) + var populate func(conf_v reflect.Value, name string, conf_index int) + populate = func(conf_v reflect.Value, name string, conf_index int) { conf_t := conf_v.Type() for key_index := 0; key_index < conf_v.NumField(); key_index++ { // ignore unexported fields @@ -61,7 +62,7 @@ func TestConfigRoundtrip(t *testing.T) { key_v := conf_v.Field(key_index) switch key_v.Interface().(type) { case string: - switch conf_v.Type().Field(key_index).Name { + switch conf_t.Field(key_index).Name { case "TLSProfile": // we have to explicitly set value for this key as it will only // accept intermediate or modern @@ -79,12 +80,12 @@ func TestConfigRoundtrip(t *testing.T) { case "GlobalEnrollSecret": key_v.SetString("") default: - key_v.SetString(v.Elem().Type().Field(conf_index).Name + "_" + conf_v.Type().Field(key_index).Name) + key_v.SetString(name + "_" + conf_t.Field(key_index).Name) } case int: key_v.SetInt(int64(conf_index*100 + key_index)) case bool: - switch conf_v.Type().Field(key_index).Name { + switch conf_t.Field(key_index).Name { // These are deprecated field names in the S3 config. Set them to zero value, which leads to the new fields being populated instead. case "DisableSSL", "ForceS3PathStyle": key_v.SetBool(false) @@ -94,9 +95,15 @@ func TestConfigRoundtrip(t *testing.T) { case time.Duration: d := time.Duration(conf_index*100 + key_index) key_v.Set(reflect.ValueOf(d)) + case AzureConfig: + // nested under CloudStorage + populate(key_v, name+"_"+conf_t.Field(key_index).Name, conf_index) } } } + for conf_index := 0; conf_index < v.Elem().NumField(); conf_index++ { + populate(v.Elem().Field(conf_index), v.Elem().Type().Field(conf_index).Name, conf_index) + } // Marshal the generated config buf, err := yaml.Marshal(original) @@ -1624,3 +1631,103 @@ func TestSoftwareInstallersSignedURLKeys(t *testing.T) { }) } } + +func TestSoftwareInstallersBackend(t *testing.T) { + s3Cfg := S3Config{SoftwareInstallersBucket: "bucket"} + azureCfg := AzureConfig{SoftwareInstallersContainer: "container"} + + cases := []struct { + name string + cfg FleetConfig + want StorageBackend + wantErr string + }{ + {name: "local", cfg: FleetConfig{}, want: StorageBackendLocal}, + {name: "s3 only", cfg: FleetConfig{S3: s3Cfg}, want: StorageBackendS3}, + {name: "azure only", cfg: FleetConfig{CloudStorage: CloudStorageConfig{Azure: azureCfg}}, want: StorageBackendAzure}, + { + name: "both", + cfg: FleetConfig{S3: s3Cfg, CloudStorage: CloudStorageConfig{Azure: azureCfg}}, + wantErr: "Couldn't configure. `s3_software_installers_bucket` and `cloud_storage_azure_files_container` cannot both be set; choose one storage backend for software installers.", + }, + { + // the carves bucket does not select the software installers backend + name: "s3 carves only", cfg: FleetConfig{S3: S3Config{CarvesBucket: "carves"}}, want: StorageBackendLocal, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := c.cfg.SoftwareInstallersBackend() + if c.wantErr != "" { + require.EqualError(t, err, c.wantErr) + require.Equal(t, StorageBackendLocal, got) + } else { + require.NoError(t, err) + require.Equal(t, c.want, got) + } + + var fatalErr error + c.cfg.ValidateSoftwareInstallersBackend(func(err error, _ string) { fatalErr = err }) + if c.wantErr != "" { + require.EqualError(t, fatalErr, c.wantErr) + } else { + require.NoError(t, fatalErr) + } + }) + } +} + +func TestSoftwareInstallersSignedURLAndStagedUpload(t *testing.T) { + cases := []struct { + name string + cfg FleetConfig + wantSigned bool + wantStagedUpload bool + }{ + {name: "default", cfg: FleetConfig{}}, + { + name: "s3 cloudfront signer", + cfg: FleetConfig{S3: S3Config{SoftwareInstallersBucket: "b", SoftwareInstallersCloudFrontSigner: &testSigner{}}}, + wantSigned: true, + }, + { + name: "s3 signed url with bucket", + cfg: FleetConfig{S3: S3Config{SoftwareInstallersBucket: "b", SoftwareInstallersSignedURL: true}}, + wantSigned: true, + wantStagedUpload: true, + }, + { + // matches the pre-existing behavior: staged upload requires a bucket, signing does not + name: "s3 signed url without bucket", + cfg: FleetConfig{S3: S3Config{SoftwareInstallersSignedURL: true}}, + wantSigned: true, + }, + { + name: "azure signed url", + cfg: FleetConfig{CloudStorage: CloudStorageConfig{Azure: AzureConfig{ + SoftwareInstallersContainer: "c", SoftwareInstallersSignedURL: true, + }}}, + wantSigned: true, + }, + { + name: "azure without signed url", + cfg: FleetConfig{CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersContainer: "c"}}}, + }, + { + name: "azure signed url without container", + cfg: FleetConfig{CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersSignedURL: true}}}, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + require.Equal(t, c.wantSigned, c.cfg.SoftwareInstallersSignedURLEnabled()) + require.Equal(t, c.wantStagedUpload, c.cfg.StagedUploadAvailable()) + }) + } +} + +type testSigner struct{} + +func (*testSigner) Public() crypto.PublicKey { return nil } + +func (*testSigner) Sign(io.Reader, []byte, crypto.SignerOpts) ([]byte, error) { return nil, nil } diff --git a/server/datastore/azure/azureblobstore.go b/server/datastore/azure/azureblobstore.go new file mode 100644 index 00000000000..ef2a1e91bac --- /dev/null +++ b/server/datastore/azure/azureblobstore.go @@ -0,0 +1,113 @@ +// Package azure implements fleet.SoftwareInstallerStore and friends backed by +// Azure Blob Storage. Azure Blob Storage has no S3-compatible API, so this is +// a separate implementation rather than another branch of the s3 package (see +// the s3 package's isGCS for how S3-compatible backends are handled instead). +package azure + +import ( + "context" + "errors" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/fleet" +) + +type azureBlobStore struct { + client *azblob.Client + container string + // signedURL, when true, makes Sign() return a SAS-signed GET URL instead of + // erroring out with fleet.ErrNotConfigured. + signedURL bool +} + +type fileNotFoundError struct{} + +var _ fleet.NotFoundError = (*fileNotFoundError)(nil) + +func (fileNotFoundError) Error() string { + return "file not found" +} + +func (fileNotFoundError) IsNotFound() bool { + return true +} + +// newAzureBlobStore initializes an Azure Blob Storage-backed store using +// shared-key authentication. +func newAzureBlobStore(cfg config.AzureConfigInternal) (*azureBlobStore, error) { + if cfg.AccountName == "" || cfg.AccountKey == "" { + return nil, errors.New("azure storage account name and account key are required") + } + if cfg.Container == "" { + return nil, errors.New("azure storage container is required") + } + + cred, err := azblob.NewSharedKeyCredential(cfg.AccountName, cfg.AccountKey) + if err != nil { + return nil, fmt.Errorf("creating azure shared key credential: %w", err) + } + + // ContainerURL overrides the default endpoint; used for Azurite/testing. + serviceURL := cfg.ContainerURL + if serviceURL == "" { + serviceURL = fmt.Sprintf("https://%s.blob.core.windows.net/", cfg.AccountName) + } + + client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) + if err != nil { + return nil, fmt.Errorf("creating azure blob client: %w", err) + } + + return &azureBlobStore{ + client: client, + container: cfg.Container, + signedURL: cfg.SignedURL, + }, nil +} + +// CreateTestContainer creates the container associated with this store. +// Only recommended for local testing. +func (s *azureBlobStore) CreateTestContainer(ctx context.Context, name string) error { + _, err := s.client.CreateContainer(ctx, name, nil) + if bloberror.HasCode(err, bloberror.ContainerAlreadyExists) { + return nil + } + return err +} + +// CleanupTestContainer empties and deletes the container associated with +// this store. Only recommended for local testing. If the container no +// longer exists, it returns nil. +func (s *azureBlobStore) CleanupTestContainer(ctx context.Context) error { + pager := s.client.NewListBlobsFlatPager(s.container, nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if bloberror.HasCode(err, bloberror.ContainerNotFound) { + return nil + } + if err != nil { + return err + } + + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + if _, err := s.client.DeleteBlob(ctx, s.container, *item.Name, nil); err != nil { + return err + } + } + } + + _, err := s.client.DeleteContainer(ctx, s.container, nil) + if bloberror.HasCode(err, bloberror.ContainerNotFound) { + return nil + } + return err +} diff --git a/server/datastore/azure/bootstrap_package.go b/server/datastore/azure/bootstrap_package.go new file mode 100644 index 00000000000..d103b3e2aff --- /dev/null +++ b/server/datastore/azure/bootstrap_package.go @@ -0,0 +1,25 @@ +package azure + +import "github.com/fleetdm/fleet/v4/server/config" + +const bootstrapPackagePrefix = "bootstrap-packages" + +type BootstrapPackageStore struct { + *commonFileStore +} + +// NewBootstrapPackageStore creates a new instance with the given Azure config. +func NewBootstrapPackageStore(cfg config.AzureConfig) (*BootstrapPackageStore, error) { + // bootstrap packages use the same Azure config as software installers + store, err := newAzureBlobStore(cfg.SoftwareInstallersToInternalCfg()) + if err != nil { + return nil, err + } + return &BootstrapPackageStore{ + &commonFileStore{ + azureBlobStore: store, + pathPrefix: bootstrapPackagePrefix, + fileLabel: "bootstrap package", + }, + }, nil +} diff --git a/server/datastore/azure/bootstrap_package_test.go b/server/datastore/azure/bootstrap_package_test.go new file mode 100644 index 00000000000..f26e1810af6 --- /dev/null +++ b/server/datastore/azure/bootstrap_package_test.go @@ -0,0 +1,155 @@ +package azure + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "path" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestBootstrapPackage(t *testing.T) { + ctx := context.Background() + store := setupTestBootstrapPackageStore(t, "bootstrap-packages-unit-test") + + // get a non-existing package + blob, length, err := store.Get(ctx, "no-such-package") + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, "no-such-package") + require.NoError(t, err) + require.False(t, exists) + + createPackageAndHash := func() ([]byte, string) { + b := make([]byte, 1024) + _, err = rand.Read(b) + require.NoError(t, err) + + h := sha256.New() + _, err = h.Write(b) + require.NoError(t, err) + fileID := hex.EncodeToString(h.Sum(nil)) + + return b, fileID + } + + getAndCheck := func(fileID string, expected []byte) { + rc, sz, err := store.Get(ctx, fileID) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, fileID) + require.NoError(t, err) + require.True(t, exists) + } + + // store a package + b0, id0 := createPackageAndHash() + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id0, b0) + + // store another one + b1, id1 := createPackageAndHash() + err = store.Put(ctx, id1, bytes.NewReader(b1)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id1, b1) + + // replace the first one + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should still match + getAndCheck(id0, b0) +} + +func TestBootstrapPackageCleanup(t *testing.T) { + ctx := context.Background() + store := setupTestBootstrapPackageStore(t, "bootstrap-packages-unit-test") + + assertExisting := func(want []string) { + prefix := bootstrapPackagePrefix + "/" + pager := store.client.NewListBlobsFlatPager(store.container, nil) + + var got []string + for pager.More() { + page, err := pager.NextPage(ctx) + require.NoError(t, err) + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + require.True(t, len(*item.Name) > len(prefix) && (*item.Name)[:len(prefix)] == prefix) + got = append(got, path.Base(*item.Name)) + } + } + require.ElementsMatch(t, want, got) + } + + // cleanup an empty store + n, err := store.Cleanup(ctx, nil, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + // put a package + ins0 := uuid.NewString() + err = store.Put(ctx, ins0, bytes.NewReader([]byte("package0"))) + require.NoError(t, err) + + // cleanup but mark it as used + n, err = store.Cleanup(ctx, []string{ins0}, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + assertExisting([]string{ins0}) + + // cleanup but mark it as unused + n, err = store.Cleanup(ctx, []string{}, time.Now()) + require.NoError(t, err) + require.Equal(t, 1, n) + + assertExisting(nil) + + // put a few packages + packages := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} + for i, ins := range packages { + err = store.Put(ctx, ins, bytes.NewReader([]byte("package"+fmt.Sprint(i)))) + require.NoError(t, err) + } + + // cleanup with a time in the past, nothing gets removed + n, err = store.Cleanup(ctx, []string{}, time.Now().Add(-time.Minute)) + require.NoError(t, err) + require.Equal(t, 0, n) + assertExisting([]string{packages[0], packages[1], packages[2], packages[3]}) + + // cleanup in the future, all unused get removed + n, err = store.Cleanup(ctx, []string{packages[0], packages[2]}, time.Now().Add(time.Minute)) + require.NoError(t, err) + require.Equal(t, 2, n) + assertExisting([]string{packages[0], packages[2]}) +} diff --git a/server/datastore/azure/common_file_store.go b/server/datastore/azure/common_file_store.go new file mode 100644 index 00000000000..7670b807aed --- /dev/null +++ b/server/datastore/azure/common_file_store.go @@ -0,0 +1,165 @@ +package azure + +import ( + "context" + "errors" + "io" + "path" + "sync/atomic" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "golang.org/x/sync/errgroup" +) + +// commonFileStore implements the common Get, Put, Exists, Sign and Cleanup +// operations typical for storage of files in an Azure Blob Storage container. +// It mirrors the s3 package's commonFileStore. The only variable thing is the +// path prefix inside the configured container, e.g. for software installers +// it is: +// +// /software-installers/ +type commonFileStore struct { + *azureBlobStore + pathPrefix string + fileLabel string // how to call the file in error messages +} + +// Get retrieves the requested file from Azure Blob Storage. +// It is important that the caller closes the reader when done. +func (s *commonFileStore) Get(ctx context.Context, fileID string) (io.ReadCloser, int64, error) { + key := s.keyForFile(fileID) + + resp, err := s.client.DownloadStream(ctx, s.container, key, nil) + if err != nil { + if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ContainerNotFound) { + return nil, int64(0), fileNotFoundError{} + } + return nil, int64(0), ctxerr.Wrapf(ctx, err, "retrieving %s from Azure Blob store", s.fileLabel) + } + + var contentLength int64 + if resp.ContentLength != nil { + contentLength = *resp.ContentLength + } + return resp.Body, contentLength, nil +} + +// Put uploads a file to Azure Blob Storage. +func (s *commonFileStore) Put(ctx context.Context, fileID string, content io.ReadSeeker) error { + if fileID == "" { + return errors.New("azure blob file identifier is empty") + } + + key := s.keyForFile(fileID) + + _, err := s.client.UploadStream(ctx, s.container, key, content, nil) + if err != nil { + return ctxerr.Wrapf(ctx, err, "uploading %s to Azure Blob store", s.fileLabel) + } + return nil +} + +// Exists checks if a file exists in the Azure Blob Storage container for the ID. +func (s *commonFileStore) Exists(ctx context.Context, fileID string) (bool, error) { + key := s.keyForFile(fileID) + + blobClient := s.client.ServiceClient().NewContainerClient(s.container).NewBlobClient(key) + _, err := blobClient.GetProperties(ctx, nil) + if err != nil { + if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ContainerNotFound) { + return false, nil + } + return false, ctxerr.Wrapf(ctx, err, "checking existence of %s in Azure Blob store", s.fileLabel) + } + return true, nil +} + +func (s *commonFileStore) Cleanup(ctx context.Context, usedFileIDs []string, removeCreatedBefore time.Time) (int, error) { + removeCreatedBefore = removeCreatedBefore.UTC() + + usedSet := make(map[string]struct{}, len(usedFileIDs)) + for _, id := range usedFileIDs { + usedSet[id] = struct{}{} + } + + // Iterate every page: unlike S3's ListObjectsV2 (explicitly capped at 1000 + // keys per call), the Azure pager's page size isn't guaranteed to cover all + // matching blobs in one call. + prefix := s.pathPrefix + "/" + pager := s.client.NewListBlobsFlatPager(s.container, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + }) + + var toDelete []string + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return 0, ctxerr.Wrapf(ctx, err, "listing %s in Azure Blob store", s.fileLabel) + } + + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + if _, ok := usedSet[path.Base(*item.Name)]; ok { + continue + } + // default to doing the cleanup if we don't have the timestamp information + if item.Properties == nil || item.Properties.LastModified == nil || !item.Properties.LastModified.UTC().After(removeCreatedBefore) { + toDelete = append(toDelete, *item.Name) + } + } + } + + if len(toDelete) == 0 { + return 0, nil + } + + var deleted atomic.Int32 + var g errgroup.Group + g.SetLimit(10) + + for _, key := range toDelete { + g.Go(func() error { + _, err := s.client.DeleteBlob(ctx, s.container, key, nil) + if err != nil { + return ctxerr.Wrapf(ctx, err, "deleting %s in Azure Blob store", s.fileLabel) + } + deleted.Add(1) + return nil + }) + } + + if err := g.Wait(); err != nil { + return int(deleted.Load()), ctxerr.Wrap(ctx, err, "errors occurred during Azure Blob deletion") + } + + return int(deleted.Load()), nil +} + +func (s *commonFileStore) Sign(ctx context.Context, fileID string, expiresIn time.Duration) (string, error) { + if !s.signedURL { + return "", ctxerr.Wrapf(ctx, fleet.ErrNotConfigured, "signing %s URL in Azure Blob store", s.fileLabel) + } + + key := s.keyForFile(fileID) + blobClient := s.client.ServiceClient().NewContainerClient(s.container).NewBlobClient(key) + signedURL, err := blobClient.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expiresIn), nil) + if err != nil { + return "", ctxerr.Wrapf(ctx, err, "signing %s URL in Azure Blob store", s.fileLabel) + } + return signedURL, nil +} + +// keyForFile builds an Azure blob name to identify the file. +func (s *commonFileStore) keyForFile(fileID string) string { + return path.Join(s.pathPrefix, fileID) +} diff --git a/server/datastore/azure/org_logo.go b/server/datastore/azure/org_logo.go new file mode 100644 index 00000000000..8a9106dedd7 --- /dev/null +++ b/server/datastore/azure/org_logo.go @@ -0,0 +1,57 @@ +package azure + +import ( + "context" + "io" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" +) + +type OrgLogoStore struct { + *commonFileStore +} + +// NewOrgLogoStore reuses the software-installers Azure config (same +// container, distinct prefix) — see azure.NewSoftwareTitleIconStore for the +// precedent. +func NewOrgLogoStore(cfg config.AzureConfig) (*OrgLogoStore, error) { + store, err := newAzureBlobStore(cfg.SoftwareInstallersToInternalCfg()) + if err != nil { + return nil, err + } + return &OrgLogoStore{ + &commonFileStore{ + azureBlobStore: store, + pathPrefix: "org-logos", + fileLabel: "org logo", + }, + }, nil +} + +// Put stores the logo bytes for mode under /org-logos/. +func (s *OrgLogoStore) Put(ctx context.Context, mode fleet.OrgLogoMode, content io.ReadSeeker) error { + return s.commonFileStore.Put(ctx, string(mode), content) +} + +func (s *OrgLogoStore) Get(ctx context.Context, mode fleet.OrgLogoMode) (io.ReadCloser, int64, error) { + return s.commonFileStore.Get(ctx, string(mode)) +} + +func (s *OrgLogoStore) Exists(ctx context.Context, mode fleet.OrgLogoMode) (bool, error) { + return s.commonFileStore.Exists(ctx, string(mode)) +} + +func (s *OrgLogoStore) Delete(ctx context.Context, mode fleet.OrgLogoMode) error { + key := s.keyForFile(string(mode)) + _, err := s.client.DeleteBlob(ctx, s.container, key, nil) + if err != nil { + if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ContainerNotFound) { + return nil + } + return ctxerr.Wrap(ctx, err, "deleting org logo from Azure Blob store") + } + return nil +} diff --git a/server/datastore/azure/org_logo_test.go b/server/datastore/azure/org_logo_test.go new file mode 100644 index 00000000000..abb52d8d545 --- /dev/null +++ b/server/datastore/azure/org_logo_test.go @@ -0,0 +1,79 @@ +package azure + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestOrgLogo(t *testing.T) { + ctx := context.Background() + store := setupTestOrgLogoStore(t, "org-logos-unit-test") + + getAndCheck := func(mode fleet.OrgLogoMode, expected []byte) { + rc, sz, err := store.Get(ctx, mode) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, mode) + require.NoError(t, err) + require.True(t, exists) + } + + // get a non-existing logo + blob, length, err := store.Get(ctx, fleet.OrgLogoModeLight) + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, fleet.OrgLogoModeLight) + require.NoError(t, err) + require.False(t, exists) + + // store the light logo + light := []byte("light-logo-content") + err = store.Put(ctx, fleet.OrgLogoModeLight, bytes.NewReader(light)) + require.NoError(t, err) + getAndCheck(fleet.OrgLogoModeLight, light) + + // the dark logo is still missing + exists, err = store.Exists(ctx, fleet.OrgLogoModeDark) + require.NoError(t, err) + require.False(t, exists) + + // store the dark logo + dark := []byte("dark-logo-content") + err = store.Put(ctx, fleet.OrgLogoModeDark, bytes.NewReader(dark)) + require.NoError(t, err) + getAndCheck(fleet.OrgLogoModeDark, dark) + + // replace the light logo + light2 := []byte("light-logo-content-v2") + err = store.Put(ctx, fleet.OrgLogoModeLight, bytes.NewReader(light2)) + require.NoError(t, err) + getAndCheck(fleet.OrgLogoModeLight, light2) + + // delete the light logo, the dark one is unaffected + err = store.Delete(ctx, fleet.OrgLogoModeLight) + require.NoError(t, err) + + exists, err = store.Exists(ctx, fleet.OrgLogoModeLight) + require.NoError(t, err) + require.False(t, exists) + + getAndCheck(fleet.OrgLogoModeDark, dark) + + // deleting again is a no-op + err = store.Delete(ctx, fleet.OrgLogoModeLight) + require.NoError(t, err) +} diff --git a/server/datastore/azure/signed_url_test.go b/server/datastore/azure/signed_url_test.go new file mode 100644 index 00000000000..85839efa19b --- /dev/null +++ b/server/datastore/azure/signed_url_test.go @@ -0,0 +1,54 @@ +package azure + +import ( + "context" + "net/url" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +// TestSignSASURL verifies the SAS-signed URL download path added on top of +// Azure Blob Storage. It runs fully offline: GetSASURL computes the URL +// locally from the shared key credential without contacting the container. +func TestSignSASURL(t *testing.T) { + baseCfg := func() config.AzureConfig { + return config.AzureConfig{ + SoftwareInstallersAccountName: testAccountName, + SoftwareInstallersAccountKey: testAccountKey, + SoftwareInstallersContainer: "test-container", + } + } + + t.Run("signed url enabled returns SAS URL", func(t *testing.T) { + cfg := baseCfg() + cfg.SoftwareInstallersSignedURL = true + store, err := NewSoftwareInstallerStore(cfg) + require.NoError(t, err) + + signed, err := store.Sign(context.Background(), "abc123", 15*time.Minute) + require.NoError(t, err) + + u, err := url.Parse(signed) + require.NoError(t, err) + require.Equal(t, "https", u.Scheme) + require.Contains(t, u.Host, testAccountName) + require.Contains(t, u.Path, "test-container") + require.Contains(t, u.Path, "abc123") + + q := u.Query() + require.NotEmpty(t, q.Get("sig")) + require.Equal(t, "r", q.Get("sp")) // read-only permission + }) + + t.Run("signed url disabled returns ErrNotConfigured", func(t *testing.T) { + store, err := NewSoftwareInstallerStore(baseCfg()) + require.NoError(t, err) + + _, err = store.Sign(context.Background(), "abc123", 15*time.Minute) + require.ErrorIs(t, err, fleet.ErrNotConfigured) + }) +} diff --git a/server/datastore/azure/software_installer.go b/server/datastore/azure/software_installer.go new file mode 100644 index 00000000000..e7275047953 --- /dev/null +++ b/server/datastore/azure/software_installer.go @@ -0,0 +1,26 @@ +package azure + +import ( + "github.com/fleetdm/fleet/v4/server/config" +) + +const softwareInstallersPrefix = "software-installers" + +type SoftwareInstallerStore struct { + *commonFileStore +} + +// NewSoftwareInstallerStore creates a new instance with the given Azure config. +func NewSoftwareInstallerStore(cfg config.AzureConfig) (*SoftwareInstallerStore, error) { + store, err := newAzureBlobStore(cfg.SoftwareInstallersToInternalCfg()) + if err != nil { + return nil, err + } + return &SoftwareInstallerStore{ + &commonFileStore{ + azureBlobStore: store, + pathPrefix: softwareInstallersPrefix, + fileLabel: "software installer", + }, + }, nil +} diff --git a/server/datastore/azure/software_installer_test.go b/server/datastore/azure/software_installer_test.go new file mode 100644 index 00000000000..962133fd1a8 --- /dev/null +++ b/server/datastore/azure/software_installer_test.go @@ -0,0 +1,155 @@ +package azure + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "path" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestSoftwareInstaller(t *testing.T) { + ctx := context.Background() + store := setupTestSoftwareInstallerStore(t, "software-installers-unit-test") + + // get a non-existing installer + blob, length, err := store.Get(ctx, "no-such-installer") + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, "no-such-installer") + require.NoError(t, err) + require.False(t, exists) + + createInstallerAndHash := func() ([]byte, string) { + b := make([]byte, 1024) + _, err = rand.Read(b) + require.NoError(t, err) + + h := sha256.New() + _, err = h.Write(b) + require.NoError(t, err) + installerID := hex.EncodeToString(h.Sum(nil)) + + return b, installerID + } + + getAndCheck := func(installerID string, expected []byte) { + rc, sz, err := store.Get(ctx, installerID) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, installerID) + require.NoError(t, err) + require.True(t, exists) + } + + // store an installer + b0, id0 := createInstallerAndHash() + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id0, b0) + + // store another one + b1, id1 := createInstallerAndHash() + err = store.Put(ctx, id1, bytes.NewReader(b1)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id1, b1) + + // replace the first one + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should still match + getAndCheck(id0, b0) +} + +func TestSoftwareInstallerCleanup(t *testing.T) { + ctx := context.Background() + store := setupTestSoftwareInstallerStore(t, "software-installers-unit-test") + + assertExisting := func(want []string) { + prefix := softwareInstallersPrefix + "/" + pager := store.client.NewListBlobsFlatPager(store.container, nil) + + var got []string + for pager.More() { + page, err := pager.NextPage(ctx) + require.NoError(t, err) + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + require.True(t, len(*item.Name) > len(prefix) && (*item.Name)[:len(prefix)] == prefix) + got = append(got, path.Base(*item.Name)) + } + } + require.ElementsMatch(t, want, got) + } + + // cleanup an empty store + n, err := store.Cleanup(ctx, nil, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + // put an installer + ins0 := uuid.NewString() + err = store.Put(ctx, ins0, bytes.NewReader([]byte("installer0"))) + require.NoError(t, err) + + // cleanup but mark it as used + n, err = store.Cleanup(ctx, []string{ins0}, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + assertExisting([]string{ins0}) + + // cleanup but mark it as unused + n, err = store.Cleanup(ctx, []string{}, time.Now()) + require.NoError(t, err) + require.Equal(t, 1, n) + + assertExisting(nil) + + // put a few installers + installers := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} + for i, ins := range installers { + err = store.Put(ctx, ins, bytes.NewReader([]byte("installer"+fmt.Sprint(i)))) + require.NoError(t, err) + } + + // cleanup with a time in the past, nothing gets removed + n, err = store.Cleanup(ctx, []string{}, time.Now().Add(-time.Minute)) + require.NoError(t, err) + require.Equal(t, 0, n) + assertExisting([]string{installers[0], installers[1], installers[2], installers[3]}) + + // cleanup in the future, all unused get removed + n, err = store.Cleanup(ctx, []string{installers[0], installers[2]}, time.Now().Add(time.Minute)) + require.NoError(t, err) + require.Equal(t, 2, n) + assertExisting([]string{installers[0], installers[2]}) +} diff --git a/server/datastore/azure/software_title_icon.go b/server/datastore/azure/software_title_icon.go new file mode 100644 index 00000000000..584d2678b92 --- /dev/null +++ b/server/datastore/azure/software_title_icon.go @@ -0,0 +1,25 @@ +package azure + +import ( + "github.com/fleetdm/fleet/v4/server/config" +) + +type SoftwareTitleIconStore struct { + *commonFileStore +} + +// NewSoftwareTitleIconStore creates a new instance with the given Azure config. +func NewSoftwareTitleIconStore(cfg config.AzureConfig) (*SoftwareTitleIconStore, error) { + // software title icons use the same Azure config as software installers + store, err := newAzureBlobStore(cfg.SoftwareInstallersToInternalCfg()) + if err != nil { + return nil, err + } + return &SoftwareTitleIconStore{ + &commonFileStore{ + azureBlobStore: store, + pathPrefix: "software-title-icons", + fileLabel: "software title icon", + }, + }, nil +} diff --git a/server/datastore/azure/software_title_icon_test.go b/server/datastore/azure/software_title_icon_test.go new file mode 100644 index 00000000000..13dc25b1eab --- /dev/null +++ b/server/datastore/azure/software_title_icon_test.go @@ -0,0 +1,155 @@ +package azure + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "path" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestSoftwareTitleIcon(t *testing.T) { + ctx := context.Background() + store := setupTestSoftwareTitleIconStore(t, "software-title-icons-unit-test") + + // get a non-existing icon + blob, length, err := store.Get(ctx, "no-such-icon") + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, "no-such-icon") + require.NoError(t, err) + require.False(t, exists) + + createIconAndHash := func() ([]byte, string) { + b := make([]byte, 1024) + _, err = rand.Read(b) + require.NoError(t, err) + + h := sha256.New() + _, err = h.Write(b) + require.NoError(t, err) + iconID := hex.EncodeToString(h.Sum(nil)) + + return b, iconID + } + + getAndCheck := func(iconID string, expected []byte) { + rc, sz, err := store.Get(ctx, iconID) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, iconID) + require.NoError(t, err) + require.True(t, exists) + } + + // store an icon + b0, id0 := createIconAndHash() + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id0, b0) + + // store another one + b1, id1 := createIconAndHash() + err = store.Put(ctx, id1, bytes.NewReader(b1)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id1, b1) + + // replace the first one + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should still match + getAndCheck(id0, b0) +} + +func TestSoftwareTitleIconCleanup(t *testing.T) { + ctx := context.Background() + store := setupTestSoftwareTitleIconStore(t, "software-title-icons-unit-test") + + assertExisting := func(want []string) { + prefix := "software-title-icons/" + pager := store.client.NewListBlobsFlatPager(store.container, nil) + + var got []string + for pager.More() { + page, err := pager.NextPage(ctx) + require.NoError(t, err) + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + require.True(t, len(*item.Name) > len(prefix) && (*item.Name)[:len(prefix)] == prefix) + got = append(got, path.Base(*item.Name)) + } + } + require.ElementsMatch(t, want, got) + } + + // cleanup an empty store + n, err := store.Cleanup(ctx, nil, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + // put an icon + ic0 := uuid.NewString() + err = store.Put(ctx, ic0, bytes.NewReader([]byte("icon0"))) + require.NoError(t, err) + + // cleanup but mark it as used + n, err = store.Cleanup(ctx, []string{ic0}, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + assertExisting([]string{ic0}) + + // cleanup but mark it as unused + n, err = store.Cleanup(ctx, []string{}, time.Now()) + require.NoError(t, err) + require.Equal(t, 1, n) + + assertExisting(nil) + + // put a few icons + icons := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} + for i, ic := range icons { + err = store.Put(ctx, ic, bytes.NewReader([]byte("icon"+fmt.Sprint(i)))) + require.NoError(t, err) + } + + // cleanup with a time in the past, nothing gets removed + n, err = store.Cleanup(ctx, []string{}, time.Now().Add(-time.Minute)) + require.NoError(t, err) + require.Equal(t, 0, n) + assertExisting([]string{icons[0], icons[1], icons[2], icons[3]}) + + // cleanup in the future, all unused get removed + n, err = store.Cleanup(ctx, []string{icons[0], icons[2]}, time.Now().Add(time.Minute)) + require.NoError(t, err) + require.Equal(t, 2, n) + assertExisting([]string{icons[0], icons[2]}) +} diff --git a/server/datastore/azure/testing_utils_test.go b/server/datastore/azure/testing_utils_test.go new file mode 100644 index 00000000000..5004f2fc510 --- /dev/null +++ b/server/datastore/azure/testing_utils_test.go @@ -0,0 +1,76 @@ +package azure + +import ( + "context" + "os" + "testing" + + "github.com/fleetdm/fleet/v4/server/config" + "github.com/stretchr/testify/require" +) + +// Well-known Azurite (the Azure Storage emulator) development account and +// key, documented at +// https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#well-known-storage-account-and-key. +const ( + testAccountName = "devstoreaccount1" + testAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" +) + +var testContainerURL = func() string { + port := "10000" + if p := os.Getenv("FLEET_AZURITE_PORT"); p != "" { + port = p + } + return "http://127.0.0.1:" + port + "/" + testAccountName +}() + +func setupTestSoftwareInstallerStore(tb testing.TB, container string) *SoftwareInstallerStore { + return setupTestStore(tb, container, NewSoftwareInstallerStore) +} + +func setupTestBootstrapPackageStore(tb testing.TB, container string) *BootstrapPackageStore { + return setupTestStore(tb, container, NewBootstrapPackageStore) +} + +func setupTestOrgLogoStore(tb testing.TB, container string) *OrgLogoStore { + return setupTestStore(tb, container, NewOrgLogoStore) +} + +func setupTestSoftwareTitleIconStore(tb testing.TB, container string) *SoftwareTitleIconStore { + return setupTestStore(tb, container, NewSoftwareTitleIconStore) +} + +type testStoreIface interface { + CreateTestContainer(ctx context.Context, name string) error + CleanupTestContainer(ctx context.Context) error +} + +func setupTestStore[T testStoreIface](tb testing.TB, container string, newFn func(config.AzureConfig) (T, error)) T { + checkTestEnv(tb) + + store, err := newFn(config.AzureConfig{ + SoftwareInstallersAccountName: testAccountName, + SoftwareInstallersAccountKey: testAccountKey, + SoftwareInstallersContainer: container, + SoftwareInstallersContainerURL: testContainerURL, + }) + require.NoError(tb, err) + + err = store.CreateTestContainer(context.Background(), container) + require.NoError(tb, err) + + tb.Cleanup(func() { + if err := store.CleanupTestContainer(context.Background()); err != nil { + tb.Errorf("cleanup azure container %q: %v", container, err) + } + }) + + return store +} + +func checkTestEnv(tb testing.TB) { + if _, ok := os.LookupEnv("AZURE_STORAGE_TEST"); !ok { + tb.Skip("set AZURE_STORAGE_TEST environment variable to run Azure-based tests") + } +} diff --git a/server/service/appconfig.go b/server/service/appconfig.go index 3e7bbe3fd05..95fa0256686 100644 --- a/server/service/appconfig.go +++ b/server/service/appconfig.go @@ -3372,5 +3372,5 @@ func (svc *Service) MaxInstallerSizeBytes() int64 { } func (svc *Service) StagedUploadAvailable(ctx context.Context) bool { - return svc.config.S3.SoftwareInstallersSignedURL && svc.config.S3.SoftwareInstallersBucket != "" && license.IsPremium(ctx) + return svc.config.StagedUploadAvailable() && license.IsPremium(ctx) } diff --git a/tools/dibble/go.sum b/tools/dibble/go.sum index 5d242f7b494..569fb25db3a 100644 --- a/tools/dibble/go.sum +++ b/tools/dibble/go.sum @@ -127,8 +127,8 @@ github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo= github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= @@ -171,8 +171,8 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/tools/gitops-auto-complete/go.mod b/tools/gitops-auto-complete/go.mod index 7efa2f7b516..faca05067d1 100644 --- a/tools/gitops-auto-complete/go.mod +++ b/tools/gitops-auto-complete/go.mod @@ -66,7 +66,7 @@ require ( github.com/go-sql-driver/mysql v1.9.3 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v1.0.0 // indirect github.com/gomodule/oauth1 v0.2.0 // indirect github.com/gomodule/redigo v1.8.9 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -83,7 +83,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/micromdm/micromdm v1.9.0 // indirect diff --git a/tools/gitops-auto-complete/go.sum b/tools/gitops-auto-complete/go.sum index 38af23d7ab1..3d272eae803 100644 --- a/tools/gitops-auto-complete/go.sum +++ b/tools/gitops-auto-complete/go.sum @@ -110,18 +110,17 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -221,8 +220,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo= github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08= github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= @@ -280,8 +279,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -298,8 +297,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= diff --git a/tools/upgrade/go.mod b/tools/upgrade/go.mod index c5fcdff9834..48aa82f1716 100644 --- a/tools/upgrade/go.mod +++ b/tools/upgrade/go.mod @@ -99,7 +99,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v1.0.0 // indirect github.com/gomodule/oauth1 v0.2.0 // indirect github.com/gomodule/redigo v1.8.9 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -118,7 +118,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -201,7 +201,7 @@ require ( go.opentelemetry.io/otel/trace v1.46.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.55.0 // indirect - golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect + golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect golang.org/x/image v0.45.0 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect diff --git a/tools/upgrade/go.sum b/tools/upgrade/go.sum index 20bdd54e898..fe29cb5a3b7 100644 --- a/tools/upgrade/go.sum +++ b/tools/upgrade/go.sum @@ -148,10 +148,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -168,8 +166,9 @@ github.com/crewjam/saml v0.5.1/go.mod h1:r0fDkmFe5URDgPrmtH0IYokva6fac3AUdstiPhy github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger/v4 v4.7.0 h1:Q+J8HApYAY7UMpL8d9owqiB+odzEc0zn/aqOD9jhc6Y= github.com/dgraph-io/badger/v4 v4.7.0/go.mod h1:He7TzG3YBy3j4f5baj5B7Zl2XyfNe5bl4Udl0aPemVA= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= @@ -307,16 +306,16 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo= github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08= github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= -github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= -github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -390,10 +389,10 @@ github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbd github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -418,8 +417,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= @@ -682,8 +681,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From 67d508dd68563ef7844dc2793d6076f8c7804fca Mon Sep 17 00:00:00 2001 From: Michael Alt Date: Wed, 14 Oct 2026 20:51:33 +0200 Subject: [PATCH 2/4] fixed linting --- server/config/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 87ad433d00b..d30832abbaa 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -842,20 +842,20 @@ type storageBackendCandidate struct { // softwareInstallersBackendCandidates is the single place that knows where // each backend's selecting config lives. To add a backend (or to move S3 under // CloudStorage), only this list needs to change. -func (c FleetConfig) softwareInstallersBackendCandidates() []storageBackendCandidate { +func (f FleetConfig) softwareInstallersBackendCandidates() []storageBackendCandidate { return []storageBackendCandidate{ - {StorageBackendS3, c.S3.SoftwareInstallersBucket != "", "s3_software_installers_bucket"}, - {StorageBackendAzure, c.CloudStorage.Azure.SoftwareInstallersContainer != "", "cloud_storage_azure_files_container"}, + {StorageBackendS3, f.S3.SoftwareInstallersBucket != "", "s3_software_installers_bucket"}, + {StorageBackendAzure, f.CloudStorage.Azure.SoftwareInstallersContainer != "", "cloud_storage_azure_files_container"}, } } // SoftwareInstallersBackend returns the object store configured for the // software installers group, or an error if more than one is configured since // only one backend can be selected. -func (c FleetConfig) SoftwareInstallersBackend() (StorageBackend, error) { +func (f FleetConfig) SoftwareInstallersBackend() (StorageBackend, error) { var keys []string backend := StorageBackendLocal - for _, cand := range c.softwareInstallersBackendCandidates() { + for _, cand := range f.softwareInstallersBackendCandidates() { if cand.configured { backend = cand.backend keys = append(keys, "`"+cand.key+"`") @@ -872,8 +872,8 @@ func (c FleetConfig) SoftwareInstallersBackend() (StorageBackend, error) { // ValidateSoftwareInstallersBackend fails initialization if more than one // object store is configured for the software installers group. -func (c FleetConfig) ValidateSoftwareInstallersBackend(initFatal func(err error, msg string)) { - if _, err := c.SoftwareInstallersBackend(); err != nil { +func (f FleetConfig) ValidateSoftwareInstallersBackend(initFatal func(err error, msg string)) { + if _, err := f.SoftwareInstallersBackend(); err != nil { initFatal(err, "software installers storage") } } @@ -881,17 +881,17 @@ func (c FleetConfig) ValidateSoftwareInstallersBackend(initFatal func(err error, // SoftwareInstallersSignedURLEnabled reports whether downloads of software // installers and in-house apps should be handed out as signed URLs (pointing // directly at the object store) instead of proxying the bytes through Fleet. -func (c FleetConfig) SoftwareInstallersSignedURLEnabled() bool { - if c.S3.SoftwareInstallersCloudFrontSigner != nil || c.S3.SoftwareInstallersSignedURL { +func (f FleetConfig) SoftwareInstallersSignedURLEnabled() bool { + if f.S3.SoftwareInstallersCloudFrontSigner != nil || f.S3.SoftwareInstallersSignedURL { return true } - return c.CloudStorage.Azure.SoftwareInstallersContainer != "" && c.CloudStorage.Azure.SoftwareInstallersSignedURL + return f.CloudStorage.Azure.SoftwareInstallersContainer != "" && f.CloudStorage.Azure.SoftwareInstallersSignedURL } // StagedUploadAvailable reports whether clients can upload packages straight // to object storage. Only supported with S3-compatible storage for now. -func (c FleetConfig) StagedUploadAvailable() bool { - return c.S3.SoftwareInstallersSignedURL && c.S3.SoftwareInstallersBucket != "" +func (f FleetConfig) StagedUploadAvailable() bool { + return f.S3.SoftwareInstallersSignedURL && f.S3.SoftwareInstallersBucket != "" } // PubSubConfig defines configs the for Google PubSub logging plugin From b2be604d32a4fee0fbe9c871a8233993f62c024c Mon Sep 17 00:00:00 2001 From: Michael Alt Date: Wed, 14 Oct 2026 21:22:00 +0200 Subject: [PATCH 3/4] review fixes --- server/config/config.go | 17 +++++++++++++---- server/config/config_test.go | 36 +++++++++++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index d30832abbaa..f5b17f02256 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -845,7 +845,7 @@ type storageBackendCandidate struct { func (f FleetConfig) softwareInstallersBackendCandidates() []storageBackendCandidate { return []storageBackendCandidate{ {StorageBackendS3, f.S3.SoftwareInstallersBucket != "", "s3_software_installers_bucket"}, - {StorageBackendAzure, f.CloudStorage.Azure.SoftwareInstallersContainer != "", "cloud_storage_azure_files_container"}, + {StorageBackendAzure, f.CloudStorage.Azure.SoftwareInstallersContainer != "", "cloud_storage.azure.files_container"}, } } @@ -881,11 +881,20 @@ func (f FleetConfig) ValidateSoftwareInstallersBackend(initFatal func(err error, // SoftwareInstallersSignedURLEnabled reports whether downloads of software // installers and in-house apps should be handed out as signed URLs (pointing // directly at the object store) instead of proxying the bytes through Fleet. +// Only the signing settings of the selected storage backend are considered. func (f FleetConfig) SoftwareInstallersSignedURLEnabled() bool { - if f.S3.SoftwareInstallersCloudFrontSigner != nil || f.S3.SoftwareInstallersSignedURL { - return true + backend, err := f.SoftwareInstallersBackend() + if err != nil { + return false + } + switch backend { + case StorageBackendS3: + return f.S3.SoftwareInstallersCloudFrontSigner != nil || f.S3.SoftwareInstallersSignedURL + case StorageBackendAzure: + return f.CloudStorage.Azure.SoftwareInstallersSignedURL + default: + return false } - return f.CloudStorage.Azure.SoftwareInstallersContainer != "" && f.CloudStorage.Azure.SoftwareInstallersSignedURL } // StagedUploadAvailable reports whether clients can upload packages straight diff --git a/server/config/config_test.go b/server/config/config_test.go index 4bffdc91e6f..9c4f1c3f25f 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -1648,7 +1648,7 @@ func TestSoftwareInstallersBackend(t *testing.T) { { name: "both", cfg: FleetConfig{S3: s3Cfg, CloudStorage: CloudStorageConfig{Azure: azureCfg}}, - wantErr: "Couldn't configure. `s3_software_installers_bucket` and `cloud_storage_azure_files_container` cannot both be set; choose one storage backend for software installers.", + wantErr: "Couldn't configure. `s3_software_installers_bucket` and `cloud_storage.azure.files_container` cannot both be set; choose one storage backend for software installers.", }, { // the carves bucket does not select the software installers backend @@ -1697,10 +1697,36 @@ func TestSoftwareInstallersSignedURLAndStagedUpload(t *testing.T) { wantStagedUpload: true, }, { - // matches the pre-existing behavior: staged upload requires a bucket, signing does not - name: "s3 signed url without bucket", - cfg: FleetConfig{S3: S3Config{SoftwareInstallersSignedURL: true}}, - wantSigned: true, + // S3 is only selected when a bucket is set, so its signing settings are ignored otherwise + name: "s3 signed url without bucket", + cfg: FleetConfig{S3: S3Config{SoftwareInstallersSignedURL: true}}, + }, + { + name: "s3 cloudfront signer without bucket", + cfg: FleetConfig{S3: S3Config{SoftwareInstallersCloudFrontSigner: &testSigner{}}}, + }, + { + name: "s3 signed url ignored when azure is selected", + cfg: FleetConfig{ + S3: S3Config{SoftwareInstallersSignedURL: true}, + CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersContainer: "c"}}, + }, + }, + { + name: "azure signed url ignored when s3 is selected", + cfg: FleetConfig{ + S3: S3Config{SoftwareInstallersBucket: "b"}, + CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersSignedURL: true}}, + }, + wantStagedUpload: false, + }, + { + name: "both backends configured", + cfg: FleetConfig{ + S3: S3Config{SoftwareInstallersBucket: "b", SoftwareInstallersSignedURL: true}, + CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersContainer: "c", SoftwareInstallersSignedURL: true}}, + }, + wantStagedUpload: true, }, { name: "azure signed url", From 93acbf38b96d0e87057fb6049afaabc9f2febad9 Mon Sep 17 00:00:00 2001 From: Michael Alt Date: Wed, 14 Oct 2026 21:42:27 +0200 Subject: [PATCH 4/4] fixe tests --- ee/server/service/software_installers_test.go | 1 + server/service/integration_install_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/ee/server/service/software_installers_test.go b/ee/server/service/software_installers_test.go index 53d849907dd..00886d2679c 100644 --- a/ee/server/service/software_installers_test.go +++ b/ee/server/service/software_installers_test.go @@ -857,6 +857,7 @@ func TestGetInHouseAppManifest(t *testing.T) { require.ErrorAs(t, err, &permErr) signer, _ := rsa.GenerateKey(rand.Reader, 2048) + svc.config.S3.SoftwareInstallersBucket = "test-bucket" svc.config.S3.SoftwareInstallersCloudFrontSigner = signer signerURL := "https://example.cloudfront.net" diff --git a/server/service/integration_install_test.go b/server/service/integration_install_test.go index deffee04a1c..1da88fd54b0 100644 --- a/server/service/integration_install_test.go +++ b/server/service/integration_install_test.go @@ -50,6 +50,8 @@ func (s *integrationInstallTestSuite) SetupSuite() { fleetConfig := config.TestConfig() signer, _ := rsa.GenerateKey(rand.Reader, 2048) + // Signing settings are only honored when S3 is the selected storage backend (bucket set). + fleetConfig.S3.SoftwareInstallersBucket = "test-bucket" fleetConfig.S3.SoftwareInstallersCloudFrontSigner = signer installConfig := TestServerOpts{ License: &fleet.LicenseInfo{