diff --git a/.github/workflows/test-go-suite.yaml b/.github/workflows/test-go-suite.yaml index 63568f9fdae..257d6fc07e4 100644 --- a/.github/workflows/test-go-suite.yaml +++ b/.github/workflows/test-go-suite.yaml @@ -74,7 +74,8 @@ jobs: if [[ "${{ inputs.suite }}" == "main" ]]; then echo "CI_TEST_PKG=main" >> $GITHUB_ENV echo "NEED_DOCKER=1" >> $GITHUB_ENV - echo "DOCKER_COMMAND=${{ env.DOCKER_COMMAND }} localstack" >> $GITHUB_ENV + echo "DOCKER_COMMAND=${{ env.DOCKER_COMMAND }} localstack azurite" >> $GITHUB_ENV + echo "AZURE_STORAGE_TEST=1" >> $GITHUB_ENV elif [[ "${{ inputs.suite }}" == "fast" ]]; then # DO NOT add any dependencies in this test suite. echo "CI_TEST_PKG=${{ inputs.suite }}" >> $GITHUB_ENV @@ -254,6 +255,14 @@ jobs: echo "Waiting for LocalStack..." timeout 60 bash -c 'until curl -sf http://localhost:4566/_localstack/health; do sleep 2; done' echo "LocalStack is ready" + + - name: Wait for Azurite + if: ${{ env.NEED_DOCKER && contains(env.DOCKER_COMMAND, 'azurite') }} + run: | + echo "Waiting for Azurite..." + # Azurite has no health endpoint; omitting -f means any HTTP response (even 400) counts as "up". + timeout 60 bash -c 'until curl -s -o /dev/null http://127.0.0.1:10000/devstoreaccount1; do sleep 2; done' + echo "Azurite is ready" - name: Generate test schema if: ${{ env.GENERATE_TEST_SCHEMA }} diff --git a/changes/azure-blob-storage-software-installers.md b/changes/azure-blob-storage-software-installers.md new file mode 100644 index 00000000000..db3e7989c4a --- /dev/null +++ b/changes/azure-blob-storage-software-installers.md @@ -0,0 +1 @@ +- Added Azure Blob Storage support (via new `FLEET_CLOUD_STORAGE_AZURE_FILES_*` environment variables) as an alternative to S3 for storing software installers, bootstrap packages, software title icons, and org logos. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 12b538312a9..1f506a01e87 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -54,6 +54,7 @@ import ( configpkg "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" licensectx "github.com/fleetdm/fleet/v4/server/contexts/license" + "github.com/fleetdm/fleet/v4/server/datastore/azure" "github.com/fleetdm/fleet/v4/server/datastore/cached_mysql" "github.com/fleetdm/fleet/v4/server/datastore/failing" "github.com/fleetdm/fleet/v4/server/datastore/filesystem" @@ -543,7 +544,11 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev initFatal(err, "initializing ee android service") } - orgLogoStore := initOrgLogoStore(ctx, config.S3, mds, logger) + // Must run before initOrgLogoStore, which picks the object store backend regardless of license tier. + config.ValidateSoftwareInstallersBackend(initFatal) + softwareInstallersBackend, _ := config.SoftwareInstallersBackend() + + orgLogoStore := initOrgLogoStore(ctx, softwareInstallersBackend, config.S3, config.CloudStorage.Azure, mds, logger) svc, err = service.NewService( ctx, @@ -589,84 +594,19 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev if license.IsPremium() { hydrantService := est.NewService(est.WithLogger(logger)) profileMatcher := apple_mdm.NewProfileMatcher(redisPool) - if config.S3.SoftwareInstallersBucket != "" { - if config.S3.BucketsAndPrefixesMatch() { - logger.WarnContext(ctx, - "the S3 buckets and prefixes for carves and software installers appear to be identical, this can cause issues") - } - // Extract the CloudFront URL signer before creating the S3 stores. - config.S3.ValidateCloudFrontURL(initFatal) - config.S3.ValidateSoftwareInstallersSignedURL(initFatal) - if config.S3.SoftwareInstallersCloudFrontURLSigningPrivateKey != "" { - // Strip newlines from private key - signingPrivateKey := strings.ReplaceAll(config.S3.SoftwareInstallersCloudFrontURLSigningPrivateKey, "\\n", "\n") - privateKey, err := cryptoutil.ParsePrivateKey([]byte(signingPrivateKey), - "CloudFront URL signing private key") - if err != nil { - initFatal(err, "parsing CloudFront URL signing private key") - } - var ok bool - config.S3.SoftwareInstallersCloudFrontSigner, ok = privateKey.(crypto.Signer) - if !ok { - initFatal(errors.New("CloudFront URL signing private key is not a crypto.Signer"), - "parsing CloudFront URL signing private key") - } - } - store, err := s3.NewSoftwareInstallerStore(config.S3) - if err != nil { - initFatal(err, "initializing S3 software installer store") - } - softwareInstallStore = store - logger.InfoContext(ctx, "using S3 software installer store", "bucket", config.S3.SoftwareInstallersBucket) - - bstore, err := s3.NewBootstrapPackageStore(config.S3) - if err != nil { - initFatal(err, "initializing S3 bootstrap package store") - } - bootstrapPackageStore = bstore - logger.InfoContext(ctx, "using S3 bootstrap package store", "bucket", config.S3.SoftwareInstallersBucket) - - softwareTitleIconStore, err = s3.NewSoftwareTitleIconStore(config.S3) - if err != nil { - initFatal(err, "initializing S3 software title icon store") - } - logger.InfoContext(ctx, "using S3 software title icon store", "bucket", config.S3.SoftwareInstallersBucket) - - stagedUploadStore, err = s3.NewStagedUploadStore(config.S3) - if err != nil { - initFatal(err, "initializing S3 staged upload store") - } - } else { - installerDir := os.TempDir() - if dir := os.Getenv("FLEET_SOFTWARE_INSTALLER_STORE_DIR"); dir != "" { - installerDir = dir - } - store, err := filesystem.NewSoftwareInstallerStore(installerDir) - if err != nil { - logger.ErrorContext(ctx, "failed to configure local filesystem software installer store", "err", err) - softwareInstallStore = failing.NewFailingSoftwareInstallerStore() - } else { - softwareInstallStore = store - logger.InfoContext(ctx, - "using local filesystem software installer store, this is not suitable for multi-container deployments", "directory", - installerDir) - } - - iconDir := os.TempDir() - if dir := os.Getenv("FLEET_SOFTWARE_TITLE_ICON_STORE_DIR"); dir != "" { - iconDir = dir - } - iconStore, err := filesystem.NewSoftwareTitleIconStore(iconDir) - if err != nil { - logger.ErrorContext(ctx, "failed to configure local filesystem software title icon store", "err", err) - softwareTitleIconStore = failing.NewFailingSoftwareTitleIconStore() - } else { - softwareTitleIconStore = iconStore - logger.WarnContext(ctx, - "using local filesystem software title icon store, this is not suitable for production use", "directory", - iconDir) - } + var stores softwareInstallerStores + switch softwareInstallersBackend { + case configpkg.StorageBackendAzure: + stores = newAzureSoftwareInstallerStores(ctx, config.CloudStorage.Azure, logger) + case configpkg.StorageBackendS3: + stores = newS3SoftwareInstallerStores(ctx, &config, logger) + default: + stores = newFilesystemSoftwareInstallerStores(ctx, logger) } + softwareInstallStore = stores.installer + bootstrapPackageStore = stores.bootstrap + softwareTitleIconStore = stores.icon + stagedUploadStore = stores.stagedUpload distributedLock = redis_lock.NewLock(redisPool) svc, err = eeservice.NewService( @@ -1320,10 +1260,18 @@ func createChartBoundedContext(dbConns *common_mysql.DBConnections, svc fleet.Se } // initOrgLogoStore builds the OrgLogoStore implementation appropriate for the deployment: -// - S3 when a software installers bucket is configured (shared bucket, distinct prefix) +// - the configured object store backend (S3 or Azure Blob) when one is selected for the software installers group // - otherwise a database-backed store, so custom org logos work without object storage or a writable filesystem. -func initOrgLogoStore(ctx context.Context, s3Config configpkg.S3Config, ds *mysql.Datastore, logger *slog.Logger) fleet.OrgLogoStore { - if s3Config.SoftwareInstallersBucket != "" { +func initOrgLogoStore(ctx context.Context, backend configpkg.StorageBackend, s3Config configpkg.S3Config, azureConfig configpkg.AzureConfig, ds *mysql.Datastore, logger *slog.Logger) fleet.OrgLogoStore { + switch backend { + case configpkg.StorageBackendAzure: + store, err := azure.NewOrgLogoStore(azureConfig) + if err != nil { + initFatal(err, "initializing Azure Blob org logo store") + } + logger.InfoContext(ctx, "using Azure Blob org logo store", "container", azureConfig.SoftwareInstallersContainer) + return store + case configpkg.StorageBackendS3: store, err := s3.NewOrgLogoStore(s3Config) if err != nil { initFatal(err, "initializing S3 org logo store") @@ -1335,6 +1283,134 @@ func initOrgLogoStore(ctx context.Context, s3Config configpkg.S3Config, ds *mysq return ds.NewOrgLogoStore() } +// softwareInstallerStores groups the stores backing the software installers +// group for a given storage backend. Stores a backend doesn't support are nil. +type softwareInstallerStores struct { + installer fleet.SoftwareInstallerStore + bootstrap fleet.MDMBootstrapPackageStore + icon fleet.SoftwareTitleIconStore + stagedUpload fleet.StagedUploadStore +} + +func newAzureSoftwareInstallerStores(ctx context.Context, cfg configpkg.AzureConfig, logger *slog.Logger) softwareInstallerStores { + var stores softwareInstallerStores + + installer, err := azure.NewSoftwareInstallerStore(cfg) + if err != nil { + initFatal(err, "initializing Azure Blob software installer store") + } + stores.installer = installer + logger.InfoContext(ctx, "using Azure Blob software installer store", "container", cfg.SoftwareInstallersContainer) + + bstore, err := azure.NewBootstrapPackageStore(cfg) + if err != nil { + initFatal(err, "initializing Azure Blob bootstrap package store") + } + stores.bootstrap = bstore + logger.InfoContext(ctx, "using Azure Blob bootstrap package store", "container", cfg.SoftwareInstallersContainer) + + stores.icon, err = azure.NewSoftwareTitleIconStore(cfg) + if err != nil { + initFatal(err, "initializing Azure Blob software title icon store") + } + logger.InfoContext(ctx, "using Azure Blob software title icon store", "container", cfg.SoftwareInstallersContainer) + + return stores +} + +// newS3SoftwareInstallerStores also stores the parsed CloudFront URL signer in +// the provided config, as the service layer reads it from there. +func newS3SoftwareInstallerStores(ctx context.Context, config *configpkg.FleetConfig, logger *slog.Logger) softwareInstallerStores { + var stores softwareInstallerStores + + if config.S3.BucketsAndPrefixesMatch() { + logger.WarnContext(ctx, + "the S3 buckets and prefixes for carves and software installers appear to be identical, this can cause issues") + } + // Extract the CloudFront URL signer before creating the S3 stores. + config.S3.ValidateCloudFrontURL(initFatal) + config.S3.ValidateSoftwareInstallersSignedURL(initFatal) + if config.S3.SoftwareInstallersCloudFrontURLSigningPrivateKey != "" { + // Strip newlines from private key + signingPrivateKey := strings.ReplaceAll(config.S3.SoftwareInstallersCloudFrontURLSigningPrivateKey, "\\n", "\n") + privateKey, err := cryptoutil.ParsePrivateKey([]byte(signingPrivateKey), + "CloudFront URL signing private key") + if err != nil { + initFatal(err, "parsing CloudFront URL signing private key") + } + var ok bool + config.S3.SoftwareInstallersCloudFrontSigner, ok = privateKey.(crypto.Signer) + if !ok { + initFatal(errors.New("CloudFront URL signing private key is not a crypto.Signer"), + "parsing CloudFront URL signing private key") + } + } + installer, err := s3.NewSoftwareInstallerStore(config.S3) + if err != nil { + initFatal(err, "initializing S3 software installer store") + } + stores.installer = installer + logger.InfoContext(ctx, "using S3 software installer store", "bucket", config.S3.SoftwareInstallersBucket) + + bstore, err := s3.NewBootstrapPackageStore(config.S3) + if err != nil { + initFatal(err, "initializing S3 bootstrap package store") + } + stores.bootstrap = bstore + logger.InfoContext(ctx, "using S3 bootstrap package store", "bucket", config.S3.SoftwareInstallersBucket) + + stores.icon, err = s3.NewSoftwareTitleIconStore(config.S3) + if err != nil { + initFatal(err, "initializing S3 software title icon store") + } + logger.InfoContext(ctx, "using S3 software title icon store", "bucket", config.S3.SoftwareInstallersBucket) + + stores.stagedUpload, err = s3.NewStagedUploadStore(config.S3) + if err != nil { + initFatal(err, "initializing S3 staged upload store") + } + + return stores +} + +// newFilesystemSoftwareInstallerStores is the fallback when no object store is +// configured. It is not suitable for multi-container deployments. +func newFilesystemSoftwareInstallerStores(ctx context.Context, logger *slog.Logger) softwareInstallerStores { + var stores softwareInstallerStores + + installerDir := os.TempDir() + if dir := os.Getenv("FLEET_SOFTWARE_INSTALLER_STORE_DIR"); dir != "" { + installerDir = dir + } + store, err := filesystem.NewSoftwareInstallerStore(installerDir) + if err != nil { + logger.ErrorContext(ctx, "failed to configure local filesystem software installer store", "err", err) + stores.installer = failing.NewFailingSoftwareInstallerStore() + } else { + stores.installer = store + logger.InfoContext(ctx, + "using local filesystem software installer store, this is not suitable for multi-container deployments", "directory", + installerDir) + } + + iconDir := os.TempDir() + if dir := os.Getenv("FLEET_SOFTWARE_TITLE_ICON_STORE_DIR"); dir != "" { + iconDir = dir + } + iconStore, err := filesystem.NewSoftwareTitleIconStore(iconDir) + if err != nil { + logger.ErrorContext(ctx, "failed to configure local filesystem software title icon store", "err", err) + stores.icon = failing.NewFailingSoftwareTitleIconStore() + } else { + stores.icon = iconStore + logger.WarnContext(ctx, + "using local filesystem software title icon store, this is not suitable for production use", "directory", + iconDir) + } + + return stores +} + func createActivityBoundedContext(svc fleet.Service, ds fleet.Datastore, dbConns *common_mysql.DBConnections, logger *slog.Logger) (activity_api.Service, endpointer.HandlerRoutesFunc) { legacyAuthorizer, err := authz.NewAuthorizer() if err != nil { diff --git a/cmd/fleet/serve_test.go b/cmd/fleet/serve_test.go index 433bed36d13..27b260b4241 100644 --- a/cmd/fleet/serve_test.go +++ b/cmd/fleet/serve_test.go @@ -1633,6 +1633,6 @@ func TestInitOrgLogoStore(t *testing.T) { // the database-backed implementation. NewOrgLogoStore does no DB work at // construction, so a zero-value Datastore is enough to verify selection. ds := &mysql.Datastore{} - store := initOrgLogoStore(t.Context(), config.S3Config{}, ds, logger) + store := initOrgLogoStore(t.Context(), config.StorageBackendLocal, config.S3Config{}, config.AzureConfig{}, ds, logger) require.IsType(t, ds.NewOrgLogoStore(), store) } diff --git a/docker-compose.yml b/docker-compose.yml index 275b5ce7390..86bd96969f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,6 +156,16 @@ services: volumes: - data-s3:/data:rw + # Azurite emulator for Azure Blob Storage (software installers/bootstrap packages/icons/org logos) + azurite: + image: mcr.microsoft.com/azure-storage/azurite:3.35.0 + command: ["azurite-blob", "--blobHost", "0.0.0.0", "--blobPort", "10000", "--skipApiVersionCheck"] + ports: + - "127.0.0.1:${FLEET_AZURITE_PORT:-10000}:10000" + volumes: + - data-azurite:/data:rw + volumes: mysql-persistent-volume: data-s3: + data-azurite: diff --git a/ee/server/service/in_house_apps.go b/ee/server/service/in_house_apps.go index 1f9ef839db9..15afa1f9838 100644 --- a/ee/server/service/in_house_apps.go +++ b/ee/server/service/in_house_apps.go @@ -210,7 +210,7 @@ func (svc *Service) GetInHouseAppManifest(ctx context.Context, titleID uint, tok appConfig.MDMUrl(), titleID, token, ) - if svc.config.S3.SoftwareInstallersCloudFrontSigner != nil || svc.config.S3.SoftwareInstallersSignedURL { + if svc.config.SoftwareInstallersSignedURLEnabled() { signedURL, err := svc.softwareInstallStore.Sign(ctx, meta.StorageID, fleet.InHouseAppSignedURLExpiry) if err != nil { // We log the error and continue to send the Fleet server URL for the in-house app diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index f993ee99a15..cf47043520c 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -1598,8 +1598,8 @@ func (svc *Service) GetSoftwareInstallDetails(ctx context.Context, installUUID s return nil, err } - // Sign the download URL when CloudFront signing (premium-only) or GCS presigning is configured. - if svc.config.S3.SoftwareInstallersCloudFrontSigner != nil || svc.config.S3.SoftwareInstallersSignedURL { + // Sign the download URL when CloudFront signing (premium-only), GCS presigning or Azure SAS signing is configured. + if svc.config.SoftwareInstallersSignedURLEnabled() { // Sign the URL for the installer installerURL, err := svc.getSoftwareInstallURL(ctx, details.InstallerID) if err != nil { diff --git a/ee/server/service/software_installers_test.go b/ee/server/service/software_installers_test.go index 53d849907dd..00886d2679c 100644 --- a/ee/server/service/software_installers_test.go +++ b/ee/server/service/software_installers_test.go @@ -857,6 +857,7 @@ func TestGetInHouseAppManifest(t *testing.T) { require.ErrorAs(t, err, &permErr) signer, _ := rsa.GenerateKey(rand.Reader, 2048) + svc.config.S3.SoftwareInstallersBucket = "test-bucket" svc.config.S3.SoftwareInstallersCloudFrontSigner = signer signerURL := "https://example.cloudfront.net" diff --git a/go.mod b/go.mod index 55585358f39..b772d45b46f 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go/pubsub v1.50.1 fyne.io/systray v1.10.1-0.20240111184411-11c585fff98d github.com/AbGuthrie/goquery/v2 v2.0.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1 github.com/Azure/go-ntlmssp v0.1.1 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Masterminds/semver v1.5.0 @@ -46,7 +47,7 @@ require ( github.com/containerd/containerd v1.7.36 github.com/crewjam/saml v0.5.1 github.com/danieljoos/wincred v1.2.1 - github.com/davecgh/go-spew v1.1.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dgraph-io/badger/v2 v2.2007.4 github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e github.com/docker/go-units v0.5.0 @@ -71,7 +72,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 github.com/gofrs/flock v0.12.1 github.com/golang-jwt/jwt/v4 v4.5.2 - github.com/golang/snappy v0.0.4 + github.com/golang/snappy v1.0.0 github.com/gomodule/oauth1 v0.2.0 github.com/gomodule/redigo v1.8.9 github.com/google/go-cmp v0.7.0 @@ -91,7 +92,7 @@ require ( github.com/igm/sockjs-go/v3 v3.0.2 github.com/jmoiron/sqlx v1.3.5 github.com/josephspurrier/goversioninfo v1.4.0 - github.com/klauspost/compress v1.18.6 + github.com/klauspost/compress v1.19.2 github.com/kolide/launcher v1.0.12 github.com/lib/pq v1.10.9 github.com/macadmins/osquery-extension v1.5.4 @@ -118,7 +119,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pelletier/go-toml/v2 v2.2.4 github.com/pkg/errors v0.9.1 - github.com/pmezard/go-difflib v1.0.0 + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/prometheus/client_golang v1.21.1 github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/realclientip/realclientip-go v1.0.0 @@ -170,7 +171,7 @@ require ( go.opentelemetry.io/otel/trace v1.46.0 go.step.sm/crypto v0.77.1 golang.org/x/crypto v0.55.0 - golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f + golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 golang.org/x/image v0.45.0 golang.org/x/mod v0.40.0 golang.org/x/net v0.58.0 @@ -205,6 +206,8 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/AlekSi/pointer v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -213,6 +216,7 @@ require ( github.com/akavel/rsrc v0.10.2 // indirect github.com/antchfx/xpath v1.3.6 // indirect github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op // indirect + github.com/apache/arrow-go/v18 v18.7.0 // indirect github.com/apache/thrift v0.24.0 // indirect github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect github.com/armon/go-radix v1.0.0 // indirect @@ -238,8 +242,7 @@ require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect - github.com/clipperhouse/stringish v0.1.1 // indirect - github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cloudflare/circl v1.6.3 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/containerd/api v1.8.0 // indirect @@ -278,10 +281,12 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/gobwas/glob v0.2.3 // indirect + github.com/goccy/go-json v0.10.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.5 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/go-tpm-tools v0.4.7 // indirect github.com/google/rpmpack v0.7.1 // indirect @@ -304,14 +309,14 @@ require ( github.com/jonboulle/clockwork v0.5.0 // indirect github.com/kevinburke/go-bindata v3.24.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kolide/kit v0.0.0-20221107170827-fb85e3d59eab // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.20 // indirect github.com/mattn/go-tty v0.0.3 // indirect github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -333,6 +338,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opencontainers/selinux v1.13.1 // indirect github.com/oschwald/maxminddb-golang v1.10.0 // indirect + github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect @@ -364,6 +370,7 @@ require ( github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yashtewari/glob-intersection v0.2.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.elastic.co/fastjson v1.1.0 // indirect go.mozilla.org/pkcs7 v0.9.0 // indirect diff --git a/go.sum b/go.sum index f6fb2536eab..43f2496f771 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,21 @@ github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59M github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w= github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1 h1:gkBLVmB3Z/HnGP/Jo4o12/RDpi0agnKav6sCKsX5Vu0= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1/go.mod h1:e3/1P5K+jIUi9JevDRklq/tFeTvbBb75bNAjU4xd31w= github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw= github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= @@ -86,6 +99,8 @@ github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtC github.com/akavel/rsrc v0.10.2 h1:Zxm8V5eI1hW4gGaYsJQUhxpjkENuG91ki8B4zCrvEsw= github.com/akavel/rsrc v0.10.2/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= @@ -98,6 +113,8 @@ github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op h1:kpBdlEPbRvff0mDD1gk7o9BhI16b9p5yYAXRlidpqJE= github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= @@ -215,10 +232,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -260,8 +275,9 @@ github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1 github.com/danieljoos/wincred v1.2.1 h1:dl9cBrupW8+r5250DYkYxocLeZ1Y4vB1kxgtjxw8GQs= github.com/danieljoos/wincred v1.2.1/go.mod h1:uGaFL9fDn3OLTvzCGulzE+SzjEe5NGlh5FdCcyfPwps= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= @@ -410,6 +426,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055 h1:UfcDMw41lSx3XM7UvD1i7Fsu3rMgD55OU5LYwLoR/Yk= github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -420,6 +438,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= @@ -447,16 +467,16 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo= github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08= github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= -github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= -github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -578,10 +598,10 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kolide/kit v0.0.0-20221107170827-fb85e3d59eab h1:KVR7cs+oPyy85i+8t1ZaNSy1bymCy5FuWyt51pdrXu4= @@ -631,8 +651,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -733,10 +753,14 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= +github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -745,8 +769,9 @@ github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942 h1:A7GG7zcGjl3jqAqGPmcNjd github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= @@ -917,6 +942,10 @@ github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= @@ -1008,8 +1037,8 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/server/config/config.go b/server/config/config.go index 25484096be0..f5b17f02256 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -541,6 +541,11 @@ type LambdaConfig struct { AuditFunction string `yaml:"audit_function"` } +// CloudStorageConfig groups the configs of cloud object stores. +type CloudStorageConfig struct { + Azure AzureConfig `yaml:"azure"` +} + // S3Config defines config to enable file carving storage to an S3 bucket type S3Config struct { Bucket string `yaml:"bucket"` @@ -771,6 +776,133 @@ type S3CloudFrontConfig struct { Signer crypto.Signer } +// AzureConfig defines config to enable Azure Blob Storage as an alternative +// object store to S3 for the software installers group (installers, +// bootstrap packages, software title icons, org logos). It is purely +// additive: when unset, storage selection and behavior for S3/GCS deployments +// are unchanged. +type AzureConfig struct { + SoftwareInstallersAccountName string `yaml:"files_account_name"` + SoftwareInstallersAccountKey string `yaml:"files_account_key"` + SoftwareInstallersContainer string `yaml:"files_container"` + // SoftwareInstallersContainerURL overrides the default + // https://.blob.core.windows.net endpoint, for Azurite or other + // Azure-compatible emulators in tests. + SoftwareInstallersContainerURL string `yaml:"files_container_url"` + // SoftwareInstallersSignedURL, when true, makes Fleet hand out a SAS-signed + // GET URL (instead of proxying the bytes) for software installer, in-house + // app and bootstrap package downloads. This is the Azure counterpart to the + // S3 CloudFront/GCS signed URL configs. + SoftwareInstallersSignedURL bool `yaml:"files_signed_url"` +} + +// SoftwareInstallersToInternalCfg creates an internal Azure config struct from +// the ingested Azure config for the software installers group (installers, +// bootstrap packages, icons, org logos). +func (a AzureConfig) SoftwareInstallersToInternalCfg() AzureConfigInternal { + return AzureConfigInternal{ + AccountName: a.SoftwareInstallersAccountName, + AccountKey: a.SoftwareInstallersAccountKey, + Container: a.SoftwareInstallersContainer, + ContainerURL: a.SoftwareInstallersContainerURL, + SignedURL: a.SoftwareInstallersSignedURL, + } +} + +// AzureConfigInternal is used internally to configure an Azure Blob Storage +// client for a given purpose (e.g. software installers). +type AzureConfigInternal struct { + AccountName string + AccountKey string + Container string + ContainerURL string + SignedURL bool +} + +// StorageBackend identifies the object store backing the software installers +// group (installers, bootstrap packages, software title icons, org logos). +type StorageBackend string + +const ( + // StorageBackendLocal means no object store is configured; Fleet falls back + // to the local filesystem/database stores. + StorageBackendLocal StorageBackend = "" + StorageBackendS3 StorageBackend = "s3" + StorageBackendAzure StorageBackend = "azure" +) + +type storageBackendCandidate struct { + backend StorageBackend + configured bool + // key is the name of the config key that selects this backend, used in + // error messages. + key string +} + +// softwareInstallersBackendCandidates is the single place that knows where +// each backend's selecting config lives. To add a backend (or to move S3 under +// CloudStorage), only this list needs to change. +func (f FleetConfig) softwareInstallersBackendCandidates() []storageBackendCandidate { + return []storageBackendCandidate{ + {StorageBackendS3, f.S3.SoftwareInstallersBucket != "", "s3_software_installers_bucket"}, + {StorageBackendAzure, f.CloudStorage.Azure.SoftwareInstallersContainer != "", "cloud_storage.azure.files_container"}, + } +} + +// SoftwareInstallersBackend returns the object store configured for the +// software installers group, or an error if more than one is configured since +// only one backend can be selected. +func (f FleetConfig) SoftwareInstallersBackend() (StorageBackend, error) { + var keys []string + backend := StorageBackendLocal + for _, cand := range f.softwareInstallersBackendCandidates() { + if cand.configured { + backend = cand.backend + keys = append(keys, "`"+cand.key+"`") + } + } + switch { + case len(keys) > 2: + return StorageBackendLocal, fmt.Errorf("Couldn't configure. %s cannot all be set; choose one storage backend for software installers.", strings.Join(keys, ", ")) + case len(keys) == 2: + return StorageBackendLocal, fmt.Errorf("Couldn't configure. %s and %s cannot both be set; choose one storage backend for software installers.", keys[0], keys[1]) + } + return backend, nil +} + +// ValidateSoftwareInstallersBackend fails initialization if more than one +// object store is configured for the software installers group. +func (f FleetConfig) ValidateSoftwareInstallersBackend(initFatal func(err error, msg string)) { + if _, err := f.SoftwareInstallersBackend(); err != nil { + initFatal(err, "software installers storage") + } +} + +// SoftwareInstallersSignedURLEnabled reports whether downloads of software +// installers and in-house apps should be handed out as signed URLs (pointing +// directly at the object store) instead of proxying the bytes through Fleet. +// Only the signing settings of the selected storage backend are considered. +func (f FleetConfig) SoftwareInstallersSignedURLEnabled() bool { + backend, err := f.SoftwareInstallersBackend() + if err != nil { + return false + } + switch backend { + case StorageBackendS3: + return f.S3.SoftwareInstallersCloudFrontSigner != nil || f.S3.SoftwareInstallersSignedURL + case StorageBackendAzure: + return f.CloudStorage.Azure.SoftwareInstallersSignedURL + default: + return false + } +} + +// StagedUploadAvailable reports whether clients can upload packages straight +// to object storage. Only supported with S3-compatible storage for now. +func (f FleetConfig) StagedUploadAvailable() bool { + return f.S3.SoftwareInstallersSignedURL && f.S3.SoftwareInstallersBucket != "" +} + // PubSubConfig defines configs the for Google PubSub logging plugin type PubSubConfig struct { Project string `json:"project"` @@ -922,6 +1054,7 @@ type FleetConfig struct { Kinesis KinesisConfig Lambda LambdaConfig S3 S3Config + CloudStorage CloudStorageConfig `yaml:"cloud_storage"` Email EmailConfig SES SESConfig PubSub PubSubConfig @@ -2002,6 +2135,13 @@ func (man Manager) addConfigs() { man.addConfigBool("s3.software_installers_gcs_signed_url", false, "Hand out presigned GCS URLs for installer/in-house app/bootstrap uploads and downloads instead of proxying bytes (requires a storage.googleapis.com endpoint)") man.addConfigBool("s3.software_installers_signed_url", false, "Deprecated: use s3.software_installers_gcs_signed_url") + // Azure Blob Storage for software installers + man.addConfigString("cloud_storage.azure.files_account_name", "", "Azure Storage account name") + man.addConfigString("cloud_storage.azure.files_account_key", "", "Azure Storage account shared key") + man.addConfigString("cloud_storage.azure.files_container", "", "Azure Storage container where to store uploaded software installers") + man.addConfigString("cloud_storage.azure.files_container_url", "", "Azure Blob Storage endpoint override (leave blank for the default https://.blob.core.windows.net; used for Azurite/testing)") + man.addConfigBool("cloud_storage.azure.files_signed_url", false, "Hand out SAS-signed URLs for installer/in-house app/bootstrap downloads instead of proxying bytes") + // PubSub man.addConfigString("pubsub.project", "", "Google Cloud Project to use") man.addConfigString("pubsub.status_topic", "", "PubSub topic for status logs") @@ -2439,6 +2579,13 @@ func (man Manager) LoadConfig() FleetConfig { StsExternalID: man.getConfigString("lambda.sts_external_id"), }, S3: man.loadS3Config(), + CloudStorage: CloudStorageConfig{Azure: AzureConfig{ + SoftwareInstallersAccountName: man.getConfigString("cloud_storage.azure.files_account_name"), + SoftwareInstallersAccountKey: man.getConfigString("cloud_storage.azure.files_account_key"), + SoftwareInstallersContainer: man.getConfigString("cloud_storage.azure.files_container"), + SoftwareInstallersContainerURL: man.getConfigString("cloud_storage.azure.files_container_url"), + SoftwareInstallersSignedURL: man.getConfigBool("cloud_storage.azure.files_signed_url"), + }}, Email: EmailConfig{ EmailBackend: man.getConfigString("email.backend"), }, diff --git a/server/config/config_test.go b/server/config/config_test.go index 2bb41014ab9..9c4f1c3f25f 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "bytes" "context" + "crypto" "crypto/tls" "fmt" "io" @@ -49,8 +50,8 @@ func TestConfigRoundtrip(t *testing.T) { // true. original := &FleetConfig{} v := reflect.ValueOf(original) - for conf_index := 0; conf_index < v.Elem().NumField(); conf_index++ { - conf_v := v.Elem().Field(conf_index) + var populate func(conf_v reflect.Value, name string, conf_index int) + populate = func(conf_v reflect.Value, name string, conf_index int) { conf_t := conf_v.Type() for key_index := 0; key_index < conf_v.NumField(); key_index++ { // ignore unexported fields @@ -61,7 +62,7 @@ func TestConfigRoundtrip(t *testing.T) { key_v := conf_v.Field(key_index) switch key_v.Interface().(type) { case string: - switch conf_v.Type().Field(key_index).Name { + switch conf_t.Field(key_index).Name { case "TLSProfile": // we have to explicitly set value for this key as it will only // accept intermediate or modern @@ -79,12 +80,12 @@ func TestConfigRoundtrip(t *testing.T) { case "GlobalEnrollSecret": key_v.SetString("") default: - key_v.SetString(v.Elem().Type().Field(conf_index).Name + "_" + conf_v.Type().Field(key_index).Name) + key_v.SetString(name + "_" + conf_t.Field(key_index).Name) } case int: key_v.SetInt(int64(conf_index*100 + key_index)) case bool: - switch conf_v.Type().Field(key_index).Name { + switch conf_t.Field(key_index).Name { // These are deprecated field names in the S3 config. Set them to zero value, which leads to the new fields being populated instead. case "DisableSSL", "ForceS3PathStyle": key_v.SetBool(false) @@ -94,9 +95,15 @@ func TestConfigRoundtrip(t *testing.T) { case time.Duration: d := time.Duration(conf_index*100 + key_index) key_v.Set(reflect.ValueOf(d)) + case AzureConfig: + // nested under CloudStorage + populate(key_v, name+"_"+conf_t.Field(key_index).Name, conf_index) } } } + for conf_index := 0; conf_index < v.Elem().NumField(); conf_index++ { + populate(v.Elem().Field(conf_index), v.Elem().Type().Field(conf_index).Name, conf_index) + } // Marshal the generated config buf, err := yaml.Marshal(original) @@ -1624,3 +1631,129 @@ func TestSoftwareInstallersSignedURLKeys(t *testing.T) { }) } } + +func TestSoftwareInstallersBackend(t *testing.T) { + s3Cfg := S3Config{SoftwareInstallersBucket: "bucket"} + azureCfg := AzureConfig{SoftwareInstallersContainer: "container"} + + cases := []struct { + name string + cfg FleetConfig + want StorageBackend + wantErr string + }{ + {name: "local", cfg: FleetConfig{}, want: StorageBackendLocal}, + {name: "s3 only", cfg: FleetConfig{S3: s3Cfg}, want: StorageBackendS3}, + {name: "azure only", cfg: FleetConfig{CloudStorage: CloudStorageConfig{Azure: azureCfg}}, want: StorageBackendAzure}, + { + name: "both", + cfg: FleetConfig{S3: s3Cfg, CloudStorage: CloudStorageConfig{Azure: azureCfg}}, + wantErr: "Couldn't configure. `s3_software_installers_bucket` and `cloud_storage.azure.files_container` cannot both be set; choose one storage backend for software installers.", + }, + { + // the carves bucket does not select the software installers backend + name: "s3 carves only", cfg: FleetConfig{S3: S3Config{CarvesBucket: "carves"}}, want: StorageBackendLocal, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := c.cfg.SoftwareInstallersBackend() + if c.wantErr != "" { + require.EqualError(t, err, c.wantErr) + require.Equal(t, StorageBackendLocal, got) + } else { + require.NoError(t, err) + require.Equal(t, c.want, got) + } + + var fatalErr error + c.cfg.ValidateSoftwareInstallersBackend(func(err error, _ string) { fatalErr = err }) + if c.wantErr != "" { + require.EqualError(t, fatalErr, c.wantErr) + } else { + require.NoError(t, fatalErr) + } + }) + } +} + +func TestSoftwareInstallersSignedURLAndStagedUpload(t *testing.T) { + cases := []struct { + name string + cfg FleetConfig + wantSigned bool + wantStagedUpload bool + }{ + {name: "default", cfg: FleetConfig{}}, + { + name: "s3 cloudfront signer", + cfg: FleetConfig{S3: S3Config{SoftwareInstallersBucket: "b", SoftwareInstallersCloudFrontSigner: &testSigner{}}}, + wantSigned: true, + }, + { + name: "s3 signed url with bucket", + cfg: FleetConfig{S3: S3Config{SoftwareInstallersBucket: "b", SoftwareInstallersSignedURL: true}}, + wantSigned: true, + wantStagedUpload: true, + }, + { + // S3 is only selected when a bucket is set, so its signing settings are ignored otherwise + name: "s3 signed url without bucket", + cfg: FleetConfig{S3: S3Config{SoftwareInstallersSignedURL: true}}, + }, + { + name: "s3 cloudfront signer without bucket", + cfg: FleetConfig{S3: S3Config{SoftwareInstallersCloudFrontSigner: &testSigner{}}}, + }, + { + name: "s3 signed url ignored when azure is selected", + cfg: FleetConfig{ + S3: S3Config{SoftwareInstallersSignedURL: true}, + CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersContainer: "c"}}, + }, + }, + { + name: "azure signed url ignored when s3 is selected", + cfg: FleetConfig{ + S3: S3Config{SoftwareInstallersBucket: "b"}, + CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersSignedURL: true}}, + }, + wantStagedUpload: false, + }, + { + name: "both backends configured", + cfg: FleetConfig{ + S3: S3Config{SoftwareInstallersBucket: "b", SoftwareInstallersSignedURL: true}, + CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersContainer: "c", SoftwareInstallersSignedURL: true}}, + }, + wantStagedUpload: true, + }, + { + name: "azure signed url", + cfg: FleetConfig{CloudStorage: CloudStorageConfig{Azure: AzureConfig{ + SoftwareInstallersContainer: "c", SoftwareInstallersSignedURL: true, + }}}, + wantSigned: true, + }, + { + name: "azure without signed url", + cfg: FleetConfig{CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersContainer: "c"}}}, + }, + { + name: "azure signed url without container", + cfg: FleetConfig{CloudStorage: CloudStorageConfig{Azure: AzureConfig{SoftwareInstallersSignedURL: true}}}, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + require.Equal(t, c.wantSigned, c.cfg.SoftwareInstallersSignedURLEnabled()) + require.Equal(t, c.wantStagedUpload, c.cfg.StagedUploadAvailable()) + }) + } +} + +type testSigner struct{} + +func (*testSigner) Public() crypto.PublicKey { return nil } + +func (*testSigner) Sign(io.Reader, []byte, crypto.SignerOpts) ([]byte, error) { return nil, nil } diff --git a/server/datastore/azure/azureblobstore.go b/server/datastore/azure/azureblobstore.go new file mode 100644 index 00000000000..ef2a1e91bac --- /dev/null +++ b/server/datastore/azure/azureblobstore.go @@ -0,0 +1,113 @@ +// Package azure implements fleet.SoftwareInstallerStore and friends backed by +// Azure Blob Storage. Azure Blob Storage has no S3-compatible API, so this is +// a separate implementation rather than another branch of the s3 package (see +// the s3 package's isGCS for how S3-compatible backends are handled instead). +package azure + +import ( + "context" + "errors" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/fleet" +) + +type azureBlobStore struct { + client *azblob.Client + container string + // signedURL, when true, makes Sign() return a SAS-signed GET URL instead of + // erroring out with fleet.ErrNotConfigured. + signedURL bool +} + +type fileNotFoundError struct{} + +var _ fleet.NotFoundError = (*fileNotFoundError)(nil) + +func (fileNotFoundError) Error() string { + return "file not found" +} + +func (fileNotFoundError) IsNotFound() bool { + return true +} + +// newAzureBlobStore initializes an Azure Blob Storage-backed store using +// shared-key authentication. +func newAzureBlobStore(cfg config.AzureConfigInternal) (*azureBlobStore, error) { + if cfg.AccountName == "" || cfg.AccountKey == "" { + return nil, errors.New("azure storage account name and account key are required") + } + if cfg.Container == "" { + return nil, errors.New("azure storage container is required") + } + + cred, err := azblob.NewSharedKeyCredential(cfg.AccountName, cfg.AccountKey) + if err != nil { + return nil, fmt.Errorf("creating azure shared key credential: %w", err) + } + + // ContainerURL overrides the default endpoint; used for Azurite/testing. + serviceURL := cfg.ContainerURL + if serviceURL == "" { + serviceURL = fmt.Sprintf("https://%s.blob.core.windows.net/", cfg.AccountName) + } + + client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) + if err != nil { + return nil, fmt.Errorf("creating azure blob client: %w", err) + } + + return &azureBlobStore{ + client: client, + container: cfg.Container, + signedURL: cfg.SignedURL, + }, nil +} + +// CreateTestContainer creates the container associated with this store. +// Only recommended for local testing. +func (s *azureBlobStore) CreateTestContainer(ctx context.Context, name string) error { + _, err := s.client.CreateContainer(ctx, name, nil) + if bloberror.HasCode(err, bloberror.ContainerAlreadyExists) { + return nil + } + return err +} + +// CleanupTestContainer empties and deletes the container associated with +// this store. Only recommended for local testing. If the container no +// longer exists, it returns nil. +func (s *azureBlobStore) CleanupTestContainer(ctx context.Context) error { + pager := s.client.NewListBlobsFlatPager(s.container, nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if bloberror.HasCode(err, bloberror.ContainerNotFound) { + return nil + } + if err != nil { + return err + } + + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + if _, err := s.client.DeleteBlob(ctx, s.container, *item.Name, nil); err != nil { + return err + } + } + } + + _, err := s.client.DeleteContainer(ctx, s.container, nil) + if bloberror.HasCode(err, bloberror.ContainerNotFound) { + return nil + } + return err +} diff --git a/server/datastore/azure/bootstrap_package.go b/server/datastore/azure/bootstrap_package.go new file mode 100644 index 00000000000..d103b3e2aff --- /dev/null +++ b/server/datastore/azure/bootstrap_package.go @@ -0,0 +1,25 @@ +package azure + +import "github.com/fleetdm/fleet/v4/server/config" + +const bootstrapPackagePrefix = "bootstrap-packages" + +type BootstrapPackageStore struct { + *commonFileStore +} + +// NewBootstrapPackageStore creates a new instance with the given Azure config. +func NewBootstrapPackageStore(cfg config.AzureConfig) (*BootstrapPackageStore, error) { + // bootstrap packages use the same Azure config as software installers + store, err := newAzureBlobStore(cfg.SoftwareInstallersToInternalCfg()) + if err != nil { + return nil, err + } + return &BootstrapPackageStore{ + &commonFileStore{ + azureBlobStore: store, + pathPrefix: bootstrapPackagePrefix, + fileLabel: "bootstrap package", + }, + }, nil +} diff --git a/server/datastore/azure/bootstrap_package_test.go b/server/datastore/azure/bootstrap_package_test.go new file mode 100644 index 00000000000..f26e1810af6 --- /dev/null +++ b/server/datastore/azure/bootstrap_package_test.go @@ -0,0 +1,155 @@ +package azure + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "path" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestBootstrapPackage(t *testing.T) { + ctx := context.Background() + store := setupTestBootstrapPackageStore(t, "bootstrap-packages-unit-test") + + // get a non-existing package + blob, length, err := store.Get(ctx, "no-such-package") + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, "no-such-package") + require.NoError(t, err) + require.False(t, exists) + + createPackageAndHash := func() ([]byte, string) { + b := make([]byte, 1024) + _, err = rand.Read(b) + require.NoError(t, err) + + h := sha256.New() + _, err = h.Write(b) + require.NoError(t, err) + fileID := hex.EncodeToString(h.Sum(nil)) + + return b, fileID + } + + getAndCheck := func(fileID string, expected []byte) { + rc, sz, err := store.Get(ctx, fileID) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, fileID) + require.NoError(t, err) + require.True(t, exists) + } + + // store a package + b0, id0 := createPackageAndHash() + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id0, b0) + + // store another one + b1, id1 := createPackageAndHash() + err = store.Put(ctx, id1, bytes.NewReader(b1)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id1, b1) + + // replace the first one + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should still match + getAndCheck(id0, b0) +} + +func TestBootstrapPackageCleanup(t *testing.T) { + ctx := context.Background() + store := setupTestBootstrapPackageStore(t, "bootstrap-packages-unit-test") + + assertExisting := func(want []string) { + prefix := bootstrapPackagePrefix + "/" + pager := store.client.NewListBlobsFlatPager(store.container, nil) + + var got []string + for pager.More() { + page, err := pager.NextPage(ctx) + require.NoError(t, err) + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + require.True(t, len(*item.Name) > len(prefix) && (*item.Name)[:len(prefix)] == prefix) + got = append(got, path.Base(*item.Name)) + } + } + require.ElementsMatch(t, want, got) + } + + // cleanup an empty store + n, err := store.Cleanup(ctx, nil, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + // put a package + ins0 := uuid.NewString() + err = store.Put(ctx, ins0, bytes.NewReader([]byte("package0"))) + require.NoError(t, err) + + // cleanup but mark it as used + n, err = store.Cleanup(ctx, []string{ins0}, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + assertExisting([]string{ins0}) + + // cleanup but mark it as unused + n, err = store.Cleanup(ctx, []string{}, time.Now()) + require.NoError(t, err) + require.Equal(t, 1, n) + + assertExisting(nil) + + // put a few packages + packages := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} + for i, ins := range packages { + err = store.Put(ctx, ins, bytes.NewReader([]byte("package"+fmt.Sprint(i)))) + require.NoError(t, err) + } + + // cleanup with a time in the past, nothing gets removed + n, err = store.Cleanup(ctx, []string{}, time.Now().Add(-time.Minute)) + require.NoError(t, err) + require.Equal(t, 0, n) + assertExisting([]string{packages[0], packages[1], packages[2], packages[3]}) + + // cleanup in the future, all unused get removed + n, err = store.Cleanup(ctx, []string{packages[0], packages[2]}, time.Now().Add(time.Minute)) + require.NoError(t, err) + require.Equal(t, 2, n) + assertExisting([]string{packages[0], packages[2]}) +} diff --git a/server/datastore/azure/common_file_store.go b/server/datastore/azure/common_file_store.go new file mode 100644 index 00000000000..7670b807aed --- /dev/null +++ b/server/datastore/azure/common_file_store.go @@ -0,0 +1,165 @@ +package azure + +import ( + "context" + "errors" + "io" + "path" + "sync/atomic" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "golang.org/x/sync/errgroup" +) + +// commonFileStore implements the common Get, Put, Exists, Sign and Cleanup +// operations typical for storage of files in an Azure Blob Storage container. +// It mirrors the s3 package's commonFileStore. The only variable thing is the +// path prefix inside the configured container, e.g. for software installers +// it is: +// +// /software-installers/ +type commonFileStore struct { + *azureBlobStore + pathPrefix string + fileLabel string // how to call the file in error messages +} + +// Get retrieves the requested file from Azure Blob Storage. +// It is important that the caller closes the reader when done. +func (s *commonFileStore) Get(ctx context.Context, fileID string) (io.ReadCloser, int64, error) { + key := s.keyForFile(fileID) + + resp, err := s.client.DownloadStream(ctx, s.container, key, nil) + if err != nil { + if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ContainerNotFound) { + return nil, int64(0), fileNotFoundError{} + } + return nil, int64(0), ctxerr.Wrapf(ctx, err, "retrieving %s from Azure Blob store", s.fileLabel) + } + + var contentLength int64 + if resp.ContentLength != nil { + contentLength = *resp.ContentLength + } + return resp.Body, contentLength, nil +} + +// Put uploads a file to Azure Blob Storage. +func (s *commonFileStore) Put(ctx context.Context, fileID string, content io.ReadSeeker) error { + if fileID == "" { + return errors.New("azure blob file identifier is empty") + } + + key := s.keyForFile(fileID) + + _, err := s.client.UploadStream(ctx, s.container, key, content, nil) + if err != nil { + return ctxerr.Wrapf(ctx, err, "uploading %s to Azure Blob store", s.fileLabel) + } + return nil +} + +// Exists checks if a file exists in the Azure Blob Storage container for the ID. +func (s *commonFileStore) Exists(ctx context.Context, fileID string) (bool, error) { + key := s.keyForFile(fileID) + + blobClient := s.client.ServiceClient().NewContainerClient(s.container).NewBlobClient(key) + _, err := blobClient.GetProperties(ctx, nil) + if err != nil { + if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ContainerNotFound) { + return false, nil + } + return false, ctxerr.Wrapf(ctx, err, "checking existence of %s in Azure Blob store", s.fileLabel) + } + return true, nil +} + +func (s *commonFileStore) Cleanup(ctx context.Context, usedFileIDs []string, removeCreatedBefore time.Time) (int, error) { + removeCreatedBefore = removeCreatedBefore.UTC() + + usedSet := make(map[string]struct{}, len(usedFileIDs)) + for _, id := range usedFileIDs { + usedSet[id] = struct{}{} + } + + // Iterate every page: unlike S3's ListObjectsV2 (explicitly capped at 1000 + // keys per call), the Azure pager's page size isn't guaranteed to cover all + // matching blobs in one call. + prefix := s.pathPrefix + "/" + pager := s.client.NewListBlobsFlatPager(s.container, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + }) + + var toDelete []string + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return 0, ctxerr.Wrapf(ctx, err, "listing %s in Azure Blob store", s.fileLabel) + } + + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + if _, ok := usedSet[path.Base(*item.Name)]; ok { + continue + } + // default to doing the cleanup if we don't have the timestamp information + if item.Properties == nil || item.Properties.LastModified == nil || !item.Properties.LastModified.UTC().After(removeCreatedBefore) { + toDelete = append(toDelete, *item.Name) + } + } + } + + if len(toDelete) == 0 { + return 0, nil + } + + var deleted atomic.Int32 + var g errgroup.Group + g.SetLimit(10) + + for _, key := range toDelete { + g.Go(func() error { + _, err := s.client.DeleteBlob(ctx, s.container, key, nil) + if err != nil { + return ctxerr.Wrapf(ctx, err, "deleting %s in Azure Blob store", s.fileLabel) + } + deleted.Add(1) + return nil + }) + } + + if err := g.Wait(); err != nil { + return int(deleted.Load()), ctxerr.Wrap(ctx, err, "errors occurred during Azure Blob deletion") + } + + return int(deleted.Load()), nil +} + +func (s *commonFileStore) Sign(ctx context.Context, fileID string, expiresIn time.Duration) (string, error) { + if !s.signedURL { + return "", ctxerr.Wrapf(ctx, fleet.ErrNotConfigured, "signing %s URL in Azure Blob store", s.fileLabel) + } + + key := s.keyForFile(fileID) + blobClient := s.client.ServiceClient().NewContainerClient(s.container).NewBlobClient(key) + signedURL, err := blobClient.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expiresIn), nil) + if err != nil { + return "", ctxerr.Wrapf(ctx, err, "signing %s URL in Azure Blob store", s.fileLabel) + } + return signedURL, nil +} + +// keyForFile builds an Azure blob name to identify the file. +func (s *commonFileStore) keyForFile(fileID string) string { + return path.Join(s.pathPrefix, fileID) +} diff --git a/server/datastore/azure/org_logo.go b/server/datastore/azure/org_logo.go new file mode 100644 index 00000000000..8a9106dedd7 --- /dev/null +++ b/server/datastore/azure/org_logo.go @@ -0,0 +1,57 @@ +package azure + +import ( + "context" + "io" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" +) + +type OrgLogoStore struct { + *commonFileStore +} + +// NewOrgLogoStore reuses the software-installers Azure config (same +// container, distinct prefix) — see azure.NewSoftwareTitleIconStore for the +// precedent. +func NewOrgLogoStore(cfg config.AzureConfig) (*OrgLogoStore, error) { + store, err := newAzureBlobStore(cfg.SoftwareInstallersToInternalCfg()) + if err != nil { + return nil, err + } + return &OrgLogoStore{ + &commonFileStore{ + azureBlobStore: store, + pathPrefix: "org-logos", + fileLabel: "org logo", + }, + }, nil +} + +// Put stores the logo bytes for mode under /org-logos/. +func (s *OrgLogoStore) Put(ctx context.Context, mode fleet.OrgLogoMode, content io.ReadSeeker) error { + return s.commonFileStore.Put(ctx, string(mode), content) +} + +func (s *OrgLogoStore) Get(ctx context.Context, mode fleet.OrgLogoMode) (io.ReadCloser, int64, error) { + return s.commonFileStore.Get(ctx, string(mode)) +} + +func (s *OrgLogoStore) Exists(ctx context.Context, mode fleet.OrgLogoMode) (bool, error) { + return s.commonFileStore.Exists(ctx, string(mode)) +} + +func (s *OrgLogoStore) Delete(ctx context.Context, mode fleet.OrgLogoMode) error { + key := s.keyForFile(string(mode)) + _, err := s.client.DeleteBlob(ctx, s.container, key, nil) + if err != nil { + if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ContainerNotFound) { + return nil + } + return ctxerr.Wrap(ctx, err, "deleting org logo from Azure Blob store") + } + return nil +} diff --git a/server/datastore/azure/org_logo_test.go b/server/datastore/azure/org_logo_test.go new file mode 100644 index 00000000000..abb52d8d545 --- /dev/null +++ b/server/datastore/azure/org_logo_test.go @@ -0,0 +1,79 @@ +package azure + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestOrgLogo(t *testing.T) { + ctx := context.Background() + store := setupTestOrgLogoStore(t, "org-logos-unit-test") + + getAndCheck := func(mode fleet.OrgLogoMode, expected []byte) { + rc, sz, err := store.Get(ctx, mode) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, mode) + require.NoError(t, err) + require.True(t, exists) + } + + // get a non-existing logo + blob, length, err := store.Get(ctx, fleet.OrgLogoModeLight) + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, fleet.OrgLogoModeLight) + require.NoError(t, err) + require.False(t, exists) + + // store the light logo + light := []byte("light-logo-content") + err = store.Put(ctx, fleet.OrgLogoModeLight, bytes.NewReader(light)) + require.NoError(t, err) + getAndCheck(fleet.OrgLogoModeLight, light) + + // the dark logo is still missing + exists, err = store.Exists(ctx, fleet.OrgLogoModeDark) + require.NoError(t, err) + require.False(t, exists) + + // store the dark logo + dark := []byte("dark-logo-content") + err = store.Put(ctx, fleet.OrgLogoModeDark, bytes.NewReader(dark)) + require.NoError(t, err) + getAndCheck(fleet.OrgLogoModeDark, dark) + + // replace the light logo + light2 := []byte("light-logo-content-v2") + err = store.Put(ctx, fleet.OrgLogoModeLight, bytes.NewReader(light2)) + require.NoError(t, err) + getAndCheck(fleet.OrgLogoModeLight, light2) + + // delete the light logo, the dark one is unaffected + err = store.Delete(ctx, fleet.OrgLogoModeLight) + require.NoError(t, err) + + exists, err = store.Exists(ctx, fleet.OrgLogoModeLight) + require.NoError(t, err) + require.False(t, exists) + + getAndCheck(fleet.OrgLogoModeDark, dark) + + // deleting again is a no-op + err = store.Delete(ctx, fleet.OrgLogoModeLight) + require.NoError(t, err) +} diff --git a/server/datastore/azure/signed_url_test.go b/server/datastore/azure/signed_url_test.go new file mode 100644 index 00000000000..85839efa19b --- /dev/null +++ b/server/datastore/azure/signed_url_test.go @@ -0,0 +1,54 @@ +package azure + +import ( + "context" + "net/url" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +// TestSignSASURL verifies the SAS-signed URL download path added on top of +// Azure Blob Storage. It runs fully offline: GetSASURL computes the URL +// locally from the shared key credential without contacting the container. +func TestSignSASURL(t *testing.T) { + baseCfg := func() config.AzureConfig { + return config.AzureConfig{ + SoftwareInstallersAccountName: testAccountName, + SoftwareInstallersAccountKey: testAccountKey, + SoftwareInstallersContainer: "test-container", + } + } + + t.Run("signed url enabled returns SAS URL", func(t *testing.T) { + cfg := baseCfg() + cfg.SoftwareInstallersSignedURL = true + store, err := NewSoftwareInstallerStore(cfg) + require.NoError(t, err) + + signed, err := store.Sign(context.Background(), "abc123", 15*time.Minute) + require.NoError(t, err) + + u, err := url.Parse(signed) + require.NoError(t, err) + require.Equal(t, "https", u.Scheme) + require.Contains(t, u.Host, testAccountName) + require.Contains(t, u.Path, "test-container") + require.Contains(t, u.Path, "abc123") + + q := u.Query() + require.NotEmpty(t, q.Get("sig")) + require.Equal(t, "r", q.Get("sp")) // read-only permission + }) + + t.Run("signed url disabled returns ErrNotConfigured", func(t *testing.T) { + store, err := NewSoftwareInstallerStore(baseCfg()) + require.NoError(t, err) + + _, err = store.Sign(context.Background(), "abc123", 15*time.Minute) + require.ErrorIs(t, err, fleet.ErrNotConfigured) + }) +} diff --git a/server/datastore/azure/software_installer.go b/server/datastore/azure/software_installer.go new file mode 100644 index 00000000000..e7275047953 --- /dev/null +++ b/server/datastore/azure/software_installer.go @@ -0,0 +1,26 @@ +package azure + +import ( + "github.com/fleetdm/fleet/v4/server/config" +) + +const softwareInstallersPrefix = "software-installers" + +type SoftwareInstallerStore struct { + *commonFileStore +} + +// NewSoftwareInstallerStore creates a new instance with the given Azure config. +func NewSoftwareInstallerStore(cfg config.AzureConfig) (*SoftwareInstallerStore, error) { + store, err := newAzureBlobStore(cfg.SoftwareInstallersToInternalCfg()) + if err != nil { + return nil, err + } + return &SoftwareInstallerStore{ + &commonFileStore{ + azureBlobStore: store, + pathPrefix: softwareInstallersPrefix, + fileLabel: "software installer", + }, + }, nil +} diff --git a/server/datastore/azure/software_installer_test.go b/server/datastore/azure/software_installer_test.go new file mode 100644 index 00000000000..962133fd1a8 --- /dev/null +++ b/server/datastore/azure/software_installer_test.go @@ -0,0 +1,155 @@ +package azure + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "path" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestSoftwareInstaller(t *testing.T) { + ctx := context.Background() + store := setupTestSoftwareInstallerStore(t, "software-installers-unit-test") + + // get a non-existing installer + blob, length, err := store.Get(ctx, "no-such-installer") + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, "no-such-installer") + require.NoError(t, err) + require.False(t, exists) + + createInstallerAndHash := func() ([]byte, string) { + b := make([]byte, 1024) + _, err = rand.Read(b) + require.NoError(t, err) + + h := sha256.New() + _, err = h.Write(b) + require.NoError(t, err) + installerID := hex.EncodeToString(h.Sum(nil)) + + return b, installerID + } + + getAndCheck := func(installerID string, expected []byte) { + rc, sz, err := store.Get(ctx, installerID) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, installerID) + require.NoError(t, err) + require.True(t, exists) + } + + // store an installer + b0, id0 := createInstallerAndHash() + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id0, b0) + + // store another one + b1, id1 := createInstallerAndHash() + err = store.Put(ctx, id1, bytes.NewReader(b1)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id1, b1) + + // replace the first one + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should still match + getAndCheck(id0, b0) +} + +func TestSoftwareInstallerCleanup(t *testing.T) { + ctx := context.Background() + store := setupTestSoftwareInstallerStore(t, "software-installers-unit-test") + + assertExisting := func(want []string) { + prefix := softwareInstallersPrefix + "/" + pager := store.client.NewListBlobsFlatPager(store.container, nil) + + var got []string + for pager.More() { + page, err := pager.NextPage(ctx) + require.NoError(t, err) + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + require.True(t, len(*item.Name) > len(prefix) && (*item.Name)[:len(prefix)] == prefix) + got = append(got, path.Base(*item.Name)) + } + } + require.ElementsMatch(t, want, got) + } + + // cleanup an empty store + n, err := store.Cleanup(ctx, nil, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + // put an installer + ins0 := uuid.NewString() + err = store.Put(ctx, ins0, bytes.NewReader([]byte("installer0"))) + require.NoError(t, err) + + // cleanup but mark it as used + n, err = store.Cleanup(ctx, []string{ins0}, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + assertExisting([]string{ins0}) + + // cleanup but mark it as unused + n, err = store.Cleanup(ctx, []string{}, time.Now()) + require.NoError(t, err) + require.Equal(t, 1, n) + + assertExisting(nil) + + // put a few installers + installers := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} + for i, ins := range installers { + err = store.Put(ctx, ins, bytes.NewReader([]byte("installer"+fmt.Sprint(i)))) + require.NoError(t, err) + } + + // cleanup with a time in the past, nothing gets removed + n, err = store.Cleanup(ctx, []string{}, time.Now().Add(-time.Minute)) + require.NoError(t, err) + require.Equal(t, 0, n) + assertExisting([]string{installers[0], installers[1], installers[2], installers[3]}) + + // cleanup in the future, all unused get removed + n, err = store.Cleanup(ctx, []string{installers[0], installers[2]}, time.Now().Add(time.Minute)) + require.NoError(t, err) + require.Equal(t, 2, n) + assertExisting([]string{installers[0], installers[2]}) +} diff --git a/server/datastore/azure/software_title_icon.go b/server/datastore/azure/software_title_icon.go new file mode 100644 index 00000000000..584d2678b92 --- /dev/null +++ b/server/datastore/azure/software_title_icon.go @@ -0,0 +1,25 @@ +package azure + +import ( + "github.com/fleetdm/fleet/v4/server/config" +) + +type SoftwareTitleIconStore struct { + *commonFileStore +} + +// NewSoftwareTitleIconStore creates a new instance with the given Azure config. +func NewSoftwareTitleIconStore(cfg config.AzureConfig) (*SoftwareTitleIconStore, error) { + // software title icons use the same Azure config as software installers + store, err := newAzureBlobStore(cfg.SoftwareInstallersToInternalCfg()) + if err != nil { + return nil, err + } + return &SoftwareTitleIconStore{ + &commonFileStore{ + azureBlobStore: store, + pathPrefix: "software-title-icons", + fileLabel: "software title icon", + }, + }, nil +} diff --git a/server/datastore/azure/software_title_icon_test.go b/server/datastore/azure/software_title_icon_test.go new file mode 100644 index 00000000000..13dc25b1eab --- /dev/null +++ b/server/datastore/azure/software_title_icon_test.go @@ -0,0 +1,155 @@ +package azure + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "path" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestSoftwareTitleIcon(t *testing.T) { + ctx := context.Background() + store := setupTestSoftwareTitleIconStore(t, "software-title-icons-unit-test") + + // get a non-existing icon + blob, length, err := store.Get(ctx, "no-such-icon") + require.Error(t, err) + require.True(t, fleet.IsNotFound(err)) + require.Nil(t, blob) + require.Zero(t, length) + + exists, err := store.Exists(ctx, "no-such-icon") + require.NoError(t, err) + require.False(t, exists) + + createIconAndHash := func() ([]byte, string) { + b := make([]byte, 1024) + _, err = rand.Read(b) + require.NoError(t, err) + + h := sha256.New() + _, err = h.Write(b) + require.NoError(t, err) + iconID := hex.EncodeToString(h.Sum(nil)) + + return b, iconID + } + + getAndCheck := func(iconID string, expected []byte) { + rc, sz, err := store.Get(ctx, iconID) + require.NoError(t, err) + require.EqualValues(t, len(expected), sz) + defer rc.Close() + + got, err := io.ReadAll(rc) + require.NoError(t, err) + require.Equal(t, expected, got) + + exists, err := store.Exists(ctx, iconID) + require.NoError(t, err) + require.True(t, exists) + } + + // store an icon + b0, id0 := createIconAndHash() + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id0, b0) + + // store another one + b1, id1 := createIconAndHash() + err = store.Put(ctx, id1, bytes.NewReader(b1)) + require.NoError(t, err) + + // read it back, it should match + getAndCheck(id1, b1) + + // replace the first one + err = store.Put(ctx, id0, bytes.NewReader(b0)) + require.NoError(t, err) + + // read it back, it should still match + getAndCheck(id0, b0) +} + +func TestSoftwareTitleIconCleanup(t *testing.T) { + ctx := context.Background() + store := setupTestSoftwareTitleIconStore(t, "software-title-icons-unit-test") + + assertExisting := func(want []string) { + prefix := "software-title-icons/" + pager := store.client.NewListBlobsFlatPager(store.container, nil) + + var got []string + for pager.More() { + page, err := pager.NextPage(ctx) + require.NoError(t, err) + if page.Segment == nil { + continue + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + require.True(t, len(*item.Name) > len(prefix) && (*item.Name)[:len(prefix)] == prefix) + got = append(got, path.Base(*item.Name)) + } + } + require.ElementsMatch(t, want, got) + } + + // cleanup an empty store + n, err := store.Cleanup(ctx, nil, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + // put an icon + ic0 := uuid.NewString() + err = store.Put(ctx, ic0, bytes.NewReader([]byte("icon0"))) + require.NoError(t, err) + + // cleanup but mark it as used + n, err = store.Cleanup(ctx, []string{ic0}, time.Now()) + require.NoError(t, err) + require.Equal(t, 0, n) + + assertExisting([]string{ic0}) + + // cleanup but mark it as unused + n, err = store.Cleanup(ctx, []string{}, time.Now()) + require.NoError(t, err) + require.Equal(t, 1, n) + + assertExisting(nil) + + // put a few icons + icons := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} + for i, ic := range icons { + err = store.Put(ctx, ic, bytes.NewReader([]byte("icon"+fmt.Sprint(i)))) + require.NoError(t, err) + } + + // cleanup with a time in the past, nothing gets removed + n, err = store.Cleanup(ctx, []string{}, time.Now().Add(-time.Minute)) + require.NoError(t, err) + require.Equal(t, 0, n) + assertExisting([]string{icons[0], icons[1], icons[2], icons[3]}) + + // cleanup in the future, all unused get removed + n, err = store.Cleanup(ctx, []string{icons[0], icons[2]}, time.Now().Add(time.Minute)) + require.NoError(t, err) + require.Equal(t, 2, n) + assertExisting([]string{icons[0], icons[2]}) +} diff --git a/server/datastore/azure/testing_utils_test.go b/server/datastore/azure/testing_utils_test.go new file mode 100644 index 00000000000..5004f2fc510 --- /dev/null +++ b/server/datastore/azure/testing_utils_test.go @@ -0,0 +1,76 @@ +package azure + +import ( + "context" + "os" + "testing" + + "github.com/fleetdm/fleet/v4/server/config" + "github.com/stretchr/testify/require" +) + +// Well-known Azurite (the Azure Storage emulator) development account and +// key, documented at +// https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#well-known-storage-account-and-key. +const ( + testAccountName = "devstoreaccount1" + testAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" +) + +var testContainerURL = func() string { + port := "10000" + if p := os.Getenv("FLEET_AZURITE_PORT"); p != "" { + port = p + } + return "http://127.0.0.1:" + port + "/" + testAccountName +}() + +func setupTestSoftwareInstallerStore(tb testing.TB, container string) *SoftwareInstallerStore { + return setupTestStore(tb, container, NewSoftwareInstallerStore) +} + +func setupTestBootstrapPackageStore(tb testing.TB, container string) *BootstrapPackageStore { + return setupTestStore(tb, container, NewBootstrapPackageStore) +} + +func setupTestOrgLogoStore(tb testing.TB, container string) *OrgLogoStore { + return setupTestStore(tb, container, NewOrgLogoStore) +} + +func setupTestSoftwareTitleIconStore(tb testing.TB, container string) *SoftwareTitleIconStore { + return setupTestStore(tb, container, NewSoftwareTitleIconStore) +} + +type testStoreIface interface { + CreateTestContainer(ctx context.Context, name string) error + CleanupTestContainer(ctx context.Context) error +} + +func setupTestStore[T testStoreIface](tb testing.TB, container string, newFn func(config.AzureConfig) (T, error)) T { + checkTestEnv(tb) + + store, err := newFn(config.AzureConfig{ + SoftwareInstallersAccountName: testAccountName, + SoftwareInstallersAccountKey: testAccountKey, + SoftwareInstallersContainer: container, + SoftwareInstallersContainerURL: testContainerURL, + }) + require.NoError(tb, err) + + err = store.CreateTestContainer(context.Background(), container) + require.NoError(tb, err) + + tb.Cleanup(func() { + if err := store.CleanupTestContainer(context.Background()); err != nil { + tb.Errorf("cleanup azure container %q: %v", container, err) + } + }) + + return store +} + +func checkTestEnv(tb testing.TB) { + if _, ok := os.LookupEnv("AZURE_STORAGE_TEST"); !ok { + tb.Skip("set AZURE_STORAGE_TEST environment variable to run Azure-based tests") + } +} diff --git a/server/service/appconfig.go b/server/service/appconfig.go index 3e7bbe3fd05..95fa0256686 100644 --- a/server/service/appconfig.go +++ b/server/service/appconfig.go @@ -3372,5 +3372,5 @@ func (svc *Service) MaxInstallerSizeBytes() int64 { } func (svc *Service) StagedUploadAvailable(ctx context.Context) bool { - return svc.config.S3.SoftwareInstallersSignedURL && svc.config.S3.SoftwareInstallersBucket != "" && license.IsPremium(ctx) + return svc.config.StagedUploadAvailable() && license.IsPremium(ctx) } diff --git a/server/service/integration_install_test.go b/server/service/integration_install_test.go index deffee04a1c..1da88fd54b0 100644 --- a/server/service/integration_install_test.go +++ b/server/service/integration_install_test.go @@ -50,6 +50,8 @@ func (s *integrationInstallTestSuite) SetupSuite() { fleetConfig := config.TestConfig() signer, _ := rsa.GenerateKey(rand.Reader, 2048) + // Signing settings are only honored when S3 is the selected storage backend (bucket set). + fleetConfig.S3.SoftwareInstallersBucket = "test-bucket" fleetConfig.S3.SoftwareInstallersCloudFrontSigner = signer installConfig := TestServerOpts{ License: &fleet.LicenseInfo{ diff --git a/tools/dibble/go.sum b/tools/dibble/go.sum index 5d242f7b494..569fb25db3a 100644 --- a/tools/dibble/go.sum +++ b/tools/dibble/go.sum @@ -127,8 +127,8 @@ github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo= github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= @@ -171,8 +171,8 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/tools/gitops-auto-complete/go.mod b/tools/gitops-auto-complete/go.mod index 7efa2f7b516..faca05067d1 100644 --- a/tools/gitops-auto-complete/go.mod +++ b/tools/gitops-auto-complete/go.mod @@ -66,7 +66,7 @@ require ( github.com/go-sql-driver/mysql v1.9.3 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v1.0.0 // indirect github.com/gomodule/oauth1 v0.2.0 // indirect github.com/gomodule/redigo v1.8.9 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -83,7 +83,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/micromdm/micromdm v1.9.0 // indirect diff --git a/tools/gitops-auto-complete/go.sum b/tools/gitops-auto-complete/go.sum index 38af23d7ab1..3d272eae803 100644 --- a/tools/gitops-auto-complete/go.sum +++ b/tools/gitops-auto-complete/go.sum @@ -110,18 +110,17 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -221,8 +220,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo= github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08= github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= @@ -280,8 +279,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -298,8 +297,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= diff --git a/tools/upgrade/go.mod b/tools/upgrade/go.mod index c5fcdff9834..48aa82f1716 100644 --- a/tools/upgrade/go.mod +++ b/tools/upgrade/go.mod @@ -99,7 +99,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/gocarina/gocsv v0.0.0-20220310154401-d4df709ca055 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v1.0.0 // indirect github.com/gomodule/oauth1 v0.2.0 // indirect github.com/gomodule/redigo v1.8.9 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -118,7 +118,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -201,7 +201,7 @@ require ( go.opentelemetry.io/otel/trace v1.46.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.55.0 // indirect - golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect + golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect golang.org/x/image v0.45.0 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect diff --git a/tools/upgrade/go.sum b/tools/upgrade/go.sum index 20bdd54e898..fe29cb5a3b7 100644 --- a/tools/upgrade/go.sum +++ b/tools/upgrade/go.sum @@ -148,10 +148,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -168,8 +166,9 @@ github.com/crewjam/saml v0.5.1/go.mod h1:r0fDkmFe5URDgPrmtH0IYokva6fac3AUdstiPhy github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger/v4 v4.7.0 h1:Q+J8HApYAY7UMpL8d9owqiB+odzEc0zn/aqOD9jhc6Y= github.com/dgraph-io/badger/v4 v4.7.0/go.mod h1:He7TzG3YBy3j4f5baj5B7Zl2XyfNe5bl4Udl0aPemVA= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= @@ -307,16 +306,16 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/oauth1 v0.2.0 h1:/nNHAD99yipOEspQFbAnNmwGTZ1UNXiD/+JLxwx79fo= github.com/gomodule/oauth1 v0.2.0/go.mod h1:4r/a8/3RkhMBxJQWL5qzbOEcaQmNPIkNoI7P8sXeI08= github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= -github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= -github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -390,10 +389,10 @@ github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbd github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -418,8 +417,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= @@ -682,8 +681,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=