Skip to content

Fix container.cpu.time on Linux: report it as a counter in seconds #7840

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
evgenyfedorov2 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fix container.cpu.time on Linux: report it as a counter in seconds #7840

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5535d85
Fix container.cpu.time on Linux to report container CPU time in seconds
Oct 14, 2026
0b7bd02
Restore container.cpu.time on Linux as a counter in seconds
Oct 14, 2026
af86617
Simplify container.cpu.time test
Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions ...ies/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxUtilizationProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,10 +105,11 @@ public LinuxUtilizationProvider(
observeValues: () => GetMeasurementWithRetry(() => CpuUtilizationRequest(_cpuRequest)),
unit: "1");

_ = meter.CreateObservableGauge(
_ = meter.CreateObservableCounter(

@amadeuszl amadeuszl Oct 15, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It depends on backend but its a breaking a change, are we good as its supposed to be released in .net 11?

@evgenyfedorov2 evgenyfedorov2 Oct 15, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do you remember or understand how we could have missed that? I personally don't remember and the incorrect instrument type looks like a really silly mistake. I couldn't find any previous discussions about instrument types here either.

I don't have a strong preference as to when to merge and release it. Can very well wait and release with 11.1, for instance.

name: ResourceUtilizationInstruments.ContainerCpuTime,
observeValues: GetCpuTime,
unit: "1");
unit: "s",
description: "CPU time used by the container.");
}
else
{
Expand Down Expand Up @@ -353,9 +354,9 @@ private IEnumerable<Measurement<double>> GetCpuTime()
yield return new Measurement<double>(systemCpuTime / NanosecondsInSecond, [new KeyValuePair<string, object?>("cpu.mode", "system")]);
}

if (TryGetValueWithRetry(CpuUtilizationV2, out double userCpuTime))
if (TryGetValueWithRetry(() => _parser.GetCgroupCpuUsageInNanosecondsAndCpuPeriodsV2().cpuUsageNanoseconds, out long userCpuTime))
{
yield return new Measurement<double>(userCpuTime, [new KeyValuePair<string, object?>("cpu.mode", "user")]);
yield return new Measurement<double>(userCpuTime / NanosecondsInSecond, [new KeyValuePair<string, object?>("cpu.mode", "user")]);
}
}
}
2 changes: 1 addition & 1 deletion ...braries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/AcceptanceTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ public Task ResourceUtilizationTracker_And_Metrics_Report_Same_Values_With_Cgrou

Assert.Equal(42, Math.Round(cpuLimitFromGauge * 100));
Assert.Equal(84, Math.Round(cpuRequestFromGauge * 100));
Assert.Equal(168, Math.Round(cpuUserTime * 100));
Assert.Equal(112, Math.Round(cpuUserTime * 100));
Assert.Equal(81, Math.Round(cpuKernelTime * 100));

return Task.CompletedTask;
Expand Down
90 changes: 89 additions & 1 deletion ...ft.Extensions.Diagnostics.ResourceMonitoring.Tests/Linux/LinuxUtilizationProviderTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using Microsoft.Extensions.Diagnostics.Metrics.Testing;
using Microsoft.Extensions.Diagnostics.ResourceMonitoring.Test.Helpers;
using Microsoft.Extensions.Logging.Testing;
using Microsoft.Extensions.Time.Testing;
Expand Down Expand Up @@ -292,7 +293,7 @@ public void Provider_Registers_Instruments_CgroupV2_WithoutHostCpu()
Assert.True(double.IsNaN(samples.Single(i => i.instrument.Name == ResourceUtilizationInstruments.ContainerCpuRequestUtilization).value));

Assert.Contains(samples, x => x.instrument.Name == ResourceUtilizationInstruments.ContainerCpuTime);
Assert.All(samples.Where(i => i.instrument.Name == ResourceUtilizationInstruments.ContainerCpuTime), item => double.IsNaN(item.value));
Assert.Equal(new[] { 0.8, 0.102312 }, samples.Where(i => i.instrument.Name == ResourceUtilizationInstruments.ContainerCpuTime).Select(i => i.value));

Assert.Contains(samples, x => x.instrument.Name == ResourceUtilizationInstruments.ContainerMemoryLimitUtilization);
Assert.Equal(1, samples.Single(i => i.instrument.Name == ResourceUtilizationInstruments.ContainerMemoryLimitUtilization).value);
Expand Down Expand Up @@ -736,4 +737,91 @@ public void Provider_WithZeroToOneRangeTrue_AndCalculationV2_ReturnsNormalizedVa
Assert.True(cpuLimitSamples.First().value <= 1.0);
}
}

[Fact]
public void Provider_WithCalculationV2_ReportsContainerCpuTimeInSeconds()
{
var options = Options.Options.Create(new ResourceMonitoringOptions { UseLinuxCalculationV2 = true });
using var meter = new Meter(nameof(Provider_WithCalculationV2_ReportsContainerCpuTimeInSeconds));
var meterFactoryMock = new Mock<IMeterFactory>();
meterFactoryMock.Setup(x => x.Create(It.IsAny<MeterOptions>())).Returns(meter);

var fileSystem = new HardcodedValueFileSystem(new Dictionary<FileInfo, string>
{
{ new FileInfo("/proc/self/cgroup"), "0::/fakeslice" },
{ new FileInfo("/sys/fs/cgroup/memory.max"), "2097152" },
{ new FileInfo("/proc/stat"), "cpu 1000 0 500 8000 0 0 0 0 0 0"},
{ new FileInfo("/sys/fs/cgroup/fakeslice/cpu.stat"), "usage_usec 1500000\nnr_periods 10"},
{ new FileInfo("/proc/meminfo"), "MemTotal: 2048 kB"},
{ new FileInfo("/sys/fs/cgroup/cpuset.cpus.effective"), "0-3"},
{ new FileInfo("/sys/fs/cgroup/fakeslice/cpu.max"), "200000 100000"},
{ new FileInfo("/sys/fs/cgroup/memory.stat"), "inactive_file 0"},
{ new FileInfo("/sys/fs/cgroup/memory.current"), "1048576"},
{ new FileInfo("/sys/fs/cgroup/fakeslice/cpu.weight"), "100"},
});

var parser = new LinuxUtilizationParserCgroupV2(fileSystem: fileSystem, new FakeUserHz(100));
var resourceQuotaProvider = new LinuxResourceQuotaProvider(parser, options);
var fakeTime = new FakeTimeProvider(DateTimeOffset.UtcNow);
_ = new LinuxUtilizationProvider(options, parser, meterFactoryMock.Object, resourceQuotaProvider, new FakeLogger<LinuxUtilizationProvider>(), fakeTime);
using var cpuTime = new MetricCollector<double>(meter, ResourceUtilizationInstruments.ContainerCpuTime, fakeTime);

cpuTime.RecordObservableInstruments();
Assert.Equal(new[] { 95, 1.5 }, cpuTime.GetMeasurementSnapshot(clear: true).Select(m => m.Value));

fileSystem.ReplaceFileContent(new FileInfo("/proc/stat"), "cpu 1100 0 600 8300 0 0 0 0 0 0");
fileSystem.ReplaceFileContent(new FileInfo("/sys/fs/cgroup/fakeslice/cpu.stat"), "usage_usec 2250000\nnr_periods 20");
fakeTime.Advance(TimeSpan.FromSeconds(10));
cpuTime.RecordObservableInstruments();
Assert.Equal(new[] { 100, 2.25 }, cpuTime.GetMeasurementSnapshot(clear: true).Select(m => m.Value));
}

[Fact]
public void Provider_WithCalculationV2_ObservingContainerCpuTime_DoesNotAffectCpuUtilization()
{
var options = Options.Options.Create(new ResourceMonitoringOptions { UseLinuxCalculationV2 = true });
using var meter = new Meter(nameof(Provider_WithCalculationV2_ObservingContainerCpuTime_DoesNotAffectCpuUtilization));
var meterFactoryMock = new Mock<IMeterFactory>();
meterFactoryMock.Setup(x => x.Create(It.IsAny<MeterOptions>())).Returns(meter);

var cpuStat = new FileInfo("/sys/fs/cgroup/fakeslice/cpu.stat");
var fileSystem = new HardcodedValueFileSystem(new Dictionary<FileInfo, string>
{
{ new FileInfo("/proc/self/cgroup"), "0::/fakeslice" },
{ new FileInfo("/sys/fs/cgroup/memory.max"), "2097152" },
{ new FileInfo("/proc/stat"), "cpu 100 10 10 10 10 10 10 10 10 10"},
{ cpuStat, "usage_usec 1000000\nnr_periods 10"},
{ new FileInfo("/proc/meminfo"), "MemTotal: 2048 kB"},
{ new FileInfo("/sys/fs/cgroup/cpuset.cpus.effective"), "0-3"},
{ new FileInfo("/sys/fs/cgroup/fakeslice/cpu.max"), "200000 100000"},
{ new FileInfo("/sys/fs/cgroup/memory.stat"), "inactive_file 0"},
{ new FileInfo("/sys/fs/cgroup/memory.current"), "1048576"},
{ new FileInfo("/sys/fs/cgroup/fakeslice/cpu.weight"), "100"},
});

var parser = new LinuxUtilizationParserCgroupV2(fileSystem: fileSystem, new FakeUserHz(100));
var resourceQuotaProvider = new LinuxResourceQuotaProvider(parser, options);
var fakeTime = new FakeTimeProvider(DateTimeOffset.UtcNow);
var provider = new LinuxUtilizationProvider(options, parser, meterFactoryMock.Object, resourceQuotaProvider, new FakeLogger<LinuxUtilizationProvider>(), fakeTime);
using var cpuTime = new MetricCollector<double>(meter, ResourceUtilizationInstruments.ContainerCpuTime, fakeTime);
using var cpuLimitUtilization = new MetricCollector<double>(meter, ResourceUtilizationInstruments.ContainerCpuLimitUtilization, fakeTime);
using var cpuRequestUtilization = new MetricCollector<double>(meter, ResourceUtilizationInstruments.ContainerCpuRequestUtilization, fakeTime);

// Only container.cpu.time is observed while the container uses 1 core (100 ms in 1 period of 100 ms):
fileSystem.ReplaceFileContent(cpuStat, "usage_usec 1100000\nnr_periods 11");
fakeTime.Advance(TimeSpan.FromSeconds(10));
cpuTime.RecordObservableInstruments();

// Then the container uses 0.25 cores (100 ms in 4 periods of 100 ms):
fileSystem.ReplaceFileContent(cpuStat, "usage_usec 1200000\nnr_periods 15");
fakeTime.Advance(TimeSpan.FromSeconds(10));
cpuLimitUtilization.RecordObservableInstruments();
cpuRequestUtilization.RecordObservableInstruments();

// The utilization covers the whole time since the provider was created (200 ms in 5 periods of 100 ms = 0.4 cores),
// because observing container.cpu.time doesn't change the state used to calculate it:
const double CoresUsed = 0.4;
Assert.Equal(CoresUsed / provider.Resources.MaximumCpuUnits, Assert.Single(cpuLimitUtilization.GetMeasurementSnapshot()).Value);
Assert.Equal(CoresUsed / provider.Resources.GuaranteedCpuUnits, Assert.Single(cpuRequestUtilization.GetMeasurementSnapshot()).Value);
}
}
Loading