diff --git a/cmd/prow-job-dispatcher/main.go b/cmd/prow-job-dispatcher/main.go index e6429c8b314..11bae5f21ef 100644 --- a/cmd/prow-job-dispatcher/main.go +++ b/cmd/prow-job-dispatcher/main.go @@ -208,9 +208,11 @@ type clusterVolume struct { clusterMap dispatcher.ClusterMap } +func clusterStressScore(volume float64, info dispatcher.ClusterInfo, maxIP int) float64 { + return volume / dispatcher.LoadWeight(info, maxIP) +} + // findClusterForJobConfig finds a cluster running on a preferred cloud provider for the jobs in a Prow job config. -// The chosen cluster will be the one with minimal workload with the given cloud provider. -// If the cluster provider is empty string, it will choose the one with minimal workload across all cloud providers. func (cv *clusterVolume) findClusterForJobConfig(cloudProvider string, jc *prowconfig.JobConfig, path string, config *dispatcher.Config, jobVolumes map[string]float64) (string, error) { if _, ok := cv.clusterVolumeMap[cloudProvider]; !ok { cloudProvider = "" @@ -224,26 +226,38 @@ func (cv *clusterVolume) findClusterForJobConfig(cloudProvider string, jc *prowc mostUsedCluster := dispatcher.FindMostUsedCluster(jc) // TODO: 75% as we still have manual assignments and these are affecting even distribution, re-evaluate when manual assignments are gone if determinedCloudProvider := config.IsInBuildFarm(api.Cluster(mostUsedCluster)); determinedCloudProvider != "" && + !cv.blocked.Has(mostUsedCluster) && cv.clusterVolumeMap[string(determinedCloudProvider)][mostUsedCluster] < cv.volumeDistribution[mostUsedCluster]*0.75 { cluster = mostUsedCluster } else { - min := float64(-1) + maxIP := dispatcher.MaxIPCapacity(cv.clusterMap) + minScore := float64(-1) for _, cp := range sets.List(cv.cloudProviders) { m := cv.clusterVolumeMap[cp] for c, v := range m { - if cv.clusterMap[c].Capacity != 100 { + if cloudProvider != "" && cloudProvider != cp { continue } - if cloudProvider == "" || cloudProvider == cp { - if min < 0 || min > v { - min = v - cluster = c - } + if cv.blocked.Has(c) { + continue + } + info, ok := cv.clusterMap[c] + if !ok || info.Capacity <= 0 { + continue + } + score := clusterStressScore(v, info, maxIP) + if minScore < 0 || score < minScore { + minScore = score + cluster = c } } } } + if cluster == "" { + return "", fmt.Errorf("no eligible cluster found for job config %q (cloud provider: %q)", path, cloudProvider) + } + var errs []error for k := range jc.PresubmitsStatic { for _, job := range jc.PresubmitsStatic[k] { diff --git a/cmd/prow-job-dispatcher/main_test.go b/cmd/prow-job-dispatcher/main_test.go index 2d784079e3e..8eb5b32c008 100644 --- a/cmd/prow-job-dispatcher/main_test.go +++ b/cmd/prow-job-dispatcher/main_test.go @@ -243,6 +243,159 @@ func TestDispatchJobConfig(t *testing.T) { }, expected: "build02", }, + { + name: "capacity-weighted: prefers lower volume/capacity even when capacity is not 100", + cv: &clusterVolume{ + clusterVolumeMap: map[string]map[string]float64{ + "aws": { + "build01": 90, + "build09": 20, + }, + }, + cloudProviders: sets.New[string]("aws"), + pjs: map[string]dispatcher.ProwJobData{}, + blocked: sets.New[string](), + specialClusters: map[string]float64{}, + volumeDistribution: map[string]float64{ + "build01": 50, + "build09": 30, + }, + clusterMap: dispatcher.ClusterMap{ + "build01": dispatcher.ClusterInfo{Capacity: 100}, + "build09": dispatcher.ClusterInfo{Capacity: 75}, + }, + }, + config: &c, + jc: &prowconfig.JobConfig{ + PresubmitsStatic: map[string][]prowconfig.Presubmit{ + "repo": {{JobBase: prowconfig.JobBase{Name: "job", + Spec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Env: []corev1.EnvVar{{Name: "CLUSTER_TYPE", Value: "openstack"}}}, + }, + }}}}, + }, + }, + path: "repo-presubmits.yaml", + jobVolumes: map[string]float64{ + "job": 1, + }, + expected: "build09", + }, + { + name: "capacity-weighted: skips blocked clusters", + cv: &clusterVolume{ + clusterVolumeMap: map[string]map[string]float64{ + "aws": { + "build01": 10, + "build09": 0, + }, + }, + cloudProviders: sets.New[string]("aws"), + pjs: map[string]dispatcher.ProwJobData{}, + blocked: sets.New[string]("build09"), + specialClusters: map[string]float64{}, + volumeDistribution: map[string]float64{ + "build01": 50, + "build09": 30, + }, + clusterMap: dispatcher.ClusterMap{ + "build01": dispatcher.ClusterInfo{Capacity: 100}, + "build09": dispatcher.ClusterInfo{Capacity: 75}, + }, + }, + config: &c, + jc: &prowconfig.JobConfig{ + PresubmitsStatic: map[string][]prowconfig.Presubmit{ + "repo": {{JobBase: prowconfig.JobBase{Name: "job", + Spec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Env: []corev1.EnvVar{{Name: "CLUSTER_TYPE", Value: "openstack"}}}, + }, + }}}}, + }, + }, + path: "repo-presubmits.yaml", + jobVolumes: map[string]float64{ + "job": 1, + }, + expected: "build01", + }, + { + name: "ipCapacity: prefers roomy /24 cluster over tight /26 at equal load capacity", + cv: &clusterVolume{ + clusterVolumeMap: map[string]map[string]float64{ + "aws": { + "build05": 40, + "build09": 40, + }, + }, + cloudProviders: sets.New[string]("aws"), + pjs: map[string]dispatcher.ProwJobData{}, + blocked: sets.New[string](), + specialClusters: map[string]float64{}, + volumeDistribution: map[string]float64{ + "build05": 50, + "build09": 50, + }, + clusterMap: dispatcher.ClusterMap{ + "build05": dispatcher.ClusterInfo{Capacity: 50, IPCapacity: 743}, + "build09": dispatcher.ClusterInfo{Capacity: 50, IPCapacity: 167}, + }, + }, + config: &c, + jc: &prowconfig.JobConfig{ + PresubmitsStatic: map[string][]prowconfig.Presubmit{ + "repo": {{JobBase: prowconfig.JobBase{Name: "job", + Spec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Env: []corev1.EnvVar{{Name: "CLUSTER_TYPE", Value: "openstack"}}}, + }, + }}}}, + }, + }, + path: "repo-presubmits.yaml", + jobVolumes: map[string]float64{ + "job": 1, + }, + expected: "build05", + }, + { + name: "no eligible cluster returns error", + cv: &clusterVolume{ + clusterVolumeMap: map[string]map[string]float64{ + "aws": { + "build01": 0, + }, + }, + cloudProviders: sets.New[string]("aws"), + pjs: map[string]dispatcher.ProwJobData{}, + blocked: sets.New[string]("build01"), + specialClusters: map[string]float64{}, + volumeDistribution: map[string]float64{ + "build01": 50, + }, + clusterMap: dispatcher.ClusterMap{ + "build01": dispatcher.ClusterInfo{Capacity: 100}, + }, + }, + config: &c, + jc: &prowconfig.JobConfig{ + PresubmitsStatic: map[string][]prowconfig.Presubmit{ + "repo": {{JobBase: prowconfig.JobBase{Name: "job", + Spec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Env: []corev1.EnvVar{{Name: "CLUSTER_TYPE", Value: "openstack"}}}, + }, + }}}}, + }, + }, + path: "repo-presubmits.yaml", + jobVolumes: map[string]float64{ + "job": 1, + }, + expectedErr: fmt.Errorf("fail to find cluster for job config: no eligible cluster found for job config %q (cloud provider: %q)", "repo-presubmits.yaml", ""), + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -257,6 +410,53 @@ func TestDispatchJobConfig(t *testing.T) { } } +func TestClusterStressScore(t *testing.T) { + testCases := []struct { + name string + volume float64 + info dispatcher.ClusterInfo + maxIP int + expected float64 + }{ + { + name: "capacity only when ipCapacity omitted", + volume: 75, + info: dispatcher.ClusterInfo{Capacity: 100}, + maxIP: 743, + expected: 0.75, + }, + { + name: "capacity 100 at farm max IP keeps full weight", + volume: 50, + info: dispatcher.ClusterInfo{Capacity: 100, IPCapacity: 743}, + maxIP: 743, + expected: 0.5, + }, + { + name: "capacity 50 halves load weight", + volume: 50, + info: dispatcher.ClusterInfo{Capacity: 50, IPCapacity: 743}, + maxIP: 743, + expected: 1.0, + }, + { + name: "tight CIDR reduces weight vs roomy maxIP", + volume: 50, + info: dispatcher.ClusterInfo{Capacity: 100, IPCapacity: 167}, + maxIP: 743, + expected: 50.0 / (100.0 * 167.0 / 743.0), + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := clusterStressScore(tc.volume, tc.info, tc.maxIP) + if diff := cmp.Diff(tc.expected, actual); diff != "" { + t.Errorf("%s: actual does not match expected, diff: %s", tc.name, diff) + } + }) + } +} + func TestGetCloudProvidersForE2ETests(t *testing.T) { testCases := []struct { name string diff --git a/pkg/dispatcher/config.go b/pkg/dispatcher/config.go index 4d784700077..c4b40505f5b 100644 --- a/pkg/dispatcher/config.go +++ b/pkg/dispatcher/config.go @@ -22,12 +22,38 @@ import ( type ClusterInfo struct { Provider string Capacity int + IPCapacity int // max usable node IPs (0=omit); with Capacity forms load weight Capabilities []string } // ClusterMap maps a cluster name to its corresponding ClusterInfo. type ClusterMap map[string]ClusterInfo +// MaxIPCapacity returns the largest IPCapacity in m. +func MaxIPCapacity(m ClusterMap) int { + max := 0 + for _, info := range m { + if info.IPCapacity > max { + max = info.IPCapacity + } + } + return max +} + +// LoadWeight is Capacity scaled by ipCapacity/maxIP when any cluster sets ipCapacity. +// Omitted ipCapacity uses maxIP as baseline so weights stay stable across partial config. +func LoadWeight(info ClusterInfo, maxIP int) float64 { + w := float64(info.Capacity) + if maxIP <= 0 { + return w + } + ip := info.IPCapacity + if ip <= 0 { + ip = maxIP + } + return w * float64(ip) / float64(maxIP) +} + // Config is the configuration file of this tools, which defines the cluster parameter for each Prow job, i.e., where it runs type Config struct { // the job will be run on the same cloud as the one for the e2e test diff --git a/pkg/dispatcher/helpers.go b/pkg/dispatcher/helpers.go index 0dc73856015..f785f57b562 100644 --- a/pkg/dispatcher/helpers.go +++ b/pkg/dispatcher/helpers.go @@ -1,9 +1,11 @@ package dispatcher import ( + "fmt" "os" "reflect" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" prowconfig "sigs.k8s.io/prow/pkg/config" "sigs.k8s.io/yaml" @@ -13,6 +15,7 @@ func loadClusterConfigFromBytes(data []byte) (ClusterMap, sets.Set[string], erro var clusters map[string][]struct { Name string `yaml:"name"` Capacity int `yaml:"capacity"` + IPCapacity int `yaml:"ipCapacity,omitempty"` Capabilities []string `yaml:"capabilities"` Blocked bool `yaml:"blocked"` } @@ -36,14 +39,29 @@ func loadClusterConfigFromBytes(data []byte) (ClusterMap, sets.Set[string], erro clusterMap[cluster.Name] = ClusterInfo{ Provider: provider, Capacity: cluster.Capacity, + IPCapacity: cluster.IPCapacity, Capabilities: cluster.Capabilities, } } } + if err := validateClusterMapIPCapacity(clusterMap); err != nil { + return nil, nil, err + } + return clusterMap, blockedClusters, nil } +func validateClusterMapIPCapacity(clusterMap ClusterMap) error { + var errs []error + for name, info := range clusterMap { + if info.IPCapacity < 0 { + errs = append(errs, fmt.Errorf("cluster %q has negative ipCapacity: %d", name, info.IPCapacity)) + } + } + return utilerrors.NewAggregate(errs) +} + // LoadClusterConfig loads cluster configuration from a YAML file, returning a ClusterMap and a set of blocked clusters. func LoadClusterConfig(filePath string) (ClusterMap, sets.Set[string], error) { data, err := os.ReadFile(filePath) @@ -109,6 +127,9 @@ func HasCapacityOrCapabilitiesChanged(prev, next ClusterMap) bool { if info1.Capacity != info2.Capacity { return true } + if info1.IPCapacity != info2.IPCapacity { + return true + } if !reflect.DeepEqual(info1.Capabilities, info2.Capabilities) { return true } diff --git a/pkg/dispatcher/helpers_test.go b/pkg/dispatcher/helpers_test.go index 8b2b6780131..b07bfb1e4f8 100644 --- a/pkg/dispatcher/helpers_test.go +++ b/pkg/dispatcher/helpers_test.go @@ -1,10 +1,15 @@ package dispatcher import ( + "fmt" "testing" + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/util/sets" prowconfig "sigs.k8s.io/prow/pkg/config" + + "github.com/openshift/ci-tools/pkg/testhelper" ) const build01 = "build01" @@ -81,9 +86,9 @@ func TestFindMostUsedCluster(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FindMostUsedCluster(&tt.jobConfig) - if result != tt.expected { - t.Errorf("expected %s, got %s", tt.expected, result) + actual := FindMostUsedCluster(&tt.jobConfig) + if diff := cmp.Diff(tt.expected, actual); diff != "" { + t.Errorf("%s: actual does not match expected, diff: %s", tt.name, diff) } }) } @@ -173,8 +178,9 @@ func TestDetermineTargetCluster(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := DetermineTargetCluster(tt.args.cluster, tt.args.determinedCluster, tt.args.defaultCluster, tt.args.canBeRelocated, tt.fields.blocked); got != tt.want { - t.Errorf("clusterVolume.determineCluster() = %v, want %v", got, tt.want) + actual := DetermineTargetCluster(tt.args.cluster, tt.args.determinedCluster, tt.args.defaultCluster, tt.args.canBeRelocated, tt.fields.blocked) + if diff := cmp.Diff(tt.want, actual); diff != "" { + t.Errorf("%s: actual does not match expected, diff: %s", tt.name, diff) } }) } @@ -186,6 +192,7 @@ func TestLoadClusterConfigFromBytes(t *testing.T) { yamlData string expectedCluster ClusterMap expectedBlocked sets.Set[string] + expectedError error }{ { name: "Valid config with AWS and GCP", @@ -193,11 +200,13 @@ func TestLoadClusterConfigFromBytes(t *testing.T) { aws: - name: build01 capacity: 80 + ipCapacity: 59 capabilities: - aarch64 - amd64 - intranet - name: build03 + ipCapacity: 59 - name: build09 blocked: true - name: build99 @@ -205,6 +214,7 @@ aws: gcp: - name: build02 capacity: 60 + ipCapacity: 100 capabilities: - intranet `, @@ -212,16 +222,19 @@ gcp: "build01": { Provider: "aws", Capacity: 80, + IPCapacity: 59, Capabilities: []string{"aarch64", "amd64", "intranet"}, }, "build03": { Provider: "aws", Capacity: 100, + IPCapacity: 59, Capabilities: nil, }, "build02": { Provider: "gcp", Capacity: 60, + IPCapacity: 100, Capabilities: []string{"intranet"}, }, }, @@ -244,16 +257,74 @@ gcp: "build01": { Provider: "aws", Capacity: 100, + IPCapacity: 0, Capabilities: nil, }, "build02": { Provider: "gcp", Capacity: 100, + IPCapacity: 0, Capabilities: []string{"intranet"}, }, }, expectedBlocked: sets.New[string]("build03"), }, + { + name: "omitted ipCapacity is zero and does not fail when no cluster sets it", + yamlData: ` +aws: + - name: build01 + capacity: 50 +gcp: + - name: build02 + capacity: 100 +`, + expectedCluster: ClusterMap{ + "build01": { + Provider: "aws", + Capacity: 50, + }, + "build02": { + Provider: "gcp", + Capacity: 100, + }, + }, + expectedBlocked: sets.New[string](), + }, + { + name: "mixed ipCapacity loads when only some clusters set it", + yamlData: ` +aws: + - name: build01 + capacity: 50 + ipCapacity: 100 +gcp: + - name: build02 + capacity: 100 +`, + expectedCluster: ClusterMap{ + "build01": { + Provider: "aws", + Capacity: 50, + IPCapacity: 100, + }, + "build02": { + Provider: "gcp", + Capacity: 100, + }, + }, + expectedBlocked: sets.New[string](), + }, + { + name: "negative ipCapacity fails", + yamlData: ` +aws: + - name: build01 + capacity: 50 + ipCapacity: -1 +`, + expectedError: fmt.Errorf(`cluster "build01" has negative ipCapacity: -1`), + }, { name: "Empty config", yamlData: ` @@ -267,35 +338,18 @@ gcp: [] for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - data := []byte(tt.yamlData) - - clusterMap, blockedClusters, err := loadClusterConfigFromBytes(data) + clusterMap, blockedClusters, err := loadClusterConfigFromBytes([]byte(tt.yamlData)) + if diff := cmp.Diff(tt.expectedError, err, testhelper.EquateErrorMessage); diff != "" { + t.Errorf("%s: actual does not match expected, diff: %s", tt.name, diff) + } if err != nil { - t.Fatalf("Failed to load cluster config: %v", err) + return } - - for clusterName, expectedInfo := range tt.expectedCluster { - if info, exists := clusterMap[clusterName]; !exists { - t.Errorf("Expected cluster %s to be in clusterMap", clusterName) - } else { - if info.Provider != expectedInfo.Provider { - t.Errorf("Expected provider for %s: %s, got: %s", clusterName, expectedInfo.Provider, info.Provider) - } - if info.Capacity != expectedInfo.Capacity { - t.Errorf("Expected capacity for %s: %d, got: %d", clusterName, expectedInfo.Capacity, info.Capacity) - } - if len(info.Capabilities) != len(expectedInfo.Capabilities) { - t.Errorf("Expected capabilities length for %s: %d, got: %d", clusterName, len(expectedInfo.Capabilities), len(info.Capabilities)) - } - for i, capability := range info.Capabilities { - if capability != expectedInfo.Capabilities[i] { - t.Errorf("Expected capability %d for %s: %s, got: %s", i, clusterName, expectedInfo.Capabilities[i], capability) - } - } - } + if diff := cmp.Diff(tt.expectedCluster, clusterMap); diff != "" { + t.Errorf("%s: actual does not match expected, diff: %s", tt.name, diff) } - if !blockedClusters.Equal(tt.expectedBlocked) { - t.Errorf("Expected blocked clusters: %v, got: %v", tt.expectedBlocked.UnsortedList(), blockedClusters.UnsortedList()) + if diff := cmp.Diff(sets.List(tt.expectedBlocked), sets.List(blockedClusters)); diff != "" { + t.Errorf("%s: actual does not match expected, diff: %s", tt.name, diff) } }) } @@ -344,6 +398,18 @@ func TestHasCapacityOrCapabilitiesChanged(t *testing.T) { }, expected: true, }, + { + name: "Change in ipCapacity for build01", + prev: ClusterMap{ + "build01": {Provider: "AWS", Capacity: 10, IPCapacity: 40, Capabilities: []string{"aarch64"}}, + "build02": {Provider: "GCP", Capacity: 20, IPCapacity: 100, Capabilities: []string{"amd64"}}, + }, + next: ClusterMap{ + "build01": {Provider: "AWS", Capacity: 10, IPCapacity: 80, Capabilities: []string{"aarch64"}}, + "build02": {Provider: "GCP", Capacity: 20, IPCapacity: 100, Capabilities: []string{"amd64"}}, + }, + expected: true, + }, { name: "No corresponding clusters in next map", prev: ClusterMap{ @@ -359,9 +425,45 @@ func TestHasCapacityOrCapabilitiesChanged(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := HasCapacityOrCapabilitiesChanged(tt.prev, tt.next) - if result != tt.expected { - t.Errorf("Test %s failed: expected %v, got %v", tt.name, tt.expected, result) + actual := HasCapacityOrCapabilitiesChanged(tt.prev, tt.next) + if diff := cmp.Diff(tt.expected, actual); diff != "" { + t.Errorf("%s: actual does not match expected, diff: %s", tt.name, diff) + } + }) + } +} + +func TestLoadWeight(t *testing.T) { + tests := []struct { + name string + info ClusterInfo + maxIP int + expected float64 + }{ + { + name: "capacity only when no cluster sets ipCapacity", + info: ClusterInfo{Capacity: 100}, + maxIP: 0, + expected: 100, + }, + { + name: "omitted ipCapacity uses maxIP baseline", + info: ClusterInfo{Capacity: 100}, + maxIP: 743, + expected: 100, + }, + { + name: "configured ipCapacity scales against maxIP", + info: ClusterInfo{Capacity: 100, IPCapacity: 167}, + maxIP: 743, + expected: 100 * 167.0 / 743.0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := LoadWeight(tt.info, tt.maxIP) + if diff := cmp.Diff(tt.expected, actual); diff != "" { + t.Errorf("%s: actual does not match expected, diff: %s", tt.name, diff) } }) } diff --git a/pkg/dispatcher/prometheus_volumes.go b/pkg/dispatcher/prometheus_volumes.go index a49a95250c9..e1886377c53 100644 --- a/pkg/dispatcher/prometheus_volumes.go +++ b/pkg/dispatcher/prometheus_volumes.go @@ -62,16 +62,22 @@ func (pv *prometheusVolumes) getTotalVolume() float64 { } func (pv *prometheusVolumes) CalculateVolumeDistribution(clusterMap ClusterMap) map[string]float64 { - totalCapacity := 0 - for _, cluster := range clusterMap { - totalCapacity += cluster.Capacity + maxIP := MaxIPCapacity(clusterMap) + var totalWeight float64 + weights := make(map[string]float64, len(clusterMap)) + for name, cluster := range clusterMap { + w := LoadWeight(cluster, maxIP) + weights[name] = w + totalWeight += w } totalVolume := pv.getTotalVolume() - volumeDistribution := make(map[string]float64) - for clusterName, cluster := range clusterMap { - volumeShare := (float64(cluster.Capacity) / float64(totalCapacity)) * totalVolume - volumeDistribution[clusterName] = volumeShare + volumeDistribution := make(map[string]float64, len(clusterMap)) + for name, w := range weights { + if totalWeight == 0 { + volumeDistribution[name] = 0 + continue + } + volumeDistribution[name] = (w / totalWeight) * totalVolume } - return volumeDistribution } diff --git a/pkg/dispatcher/prometheus_volumes_test.go b/pkg/dispatcher/prometheus_volumes_test.go index 2db8c4db32b..3d4a960ad38 100644 --- a/pkg/dispatcher/prometheus_volumes_test.go +++ b/pkg/dispatcher/prometheus_volumes_test.go @@ -142,6 +142,30 @@ func TestCalculateVolumeDistribution(t *testing.T) { "clusterB": 1000, }, }, + { + name: "ipCapacity scales capacity share", + jobVolumes: map[string]float64{"jobA": 1500}, + clusterMap: ClusterMap{ + "build05": {Provider: "AWS", Capacity: 100, IPCapacity: 200}, + "build09": {Provider: "AWS", Capacity: 100, IPCapacity: 100}, + }, + expected: map[string]float64{ + "build05": 1000, + "build09": 500, + }, + }, + { + name: "capacity half cuts share with same ipCapacity", + jobVolumes: map[string]float64{"jobA": 1500}, + clusterMap: ClusterMap{ + "build05": {Provider: "AWS", Capacity: 50, IPCapacity: 200}, + "build09": {Provider: "AWS", Capacity: 100, IPCapacity: 200}, + }, + expected: map[string]float64{ + "build05": 500, + "build09": 1000, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {