AddRange() .NET 8 performance improvements with and without Dynamic Profile Guided Optimization (PGO)
Date Added (UTC):
08 Apr 2024 @ 01:51
Date Updated (UTC):08 Apr 2024 @ 01:57
.NET Version(s): Tag(s):
#.Net8PerfImprovement #Collections #Dynamic PGO
Added By:
.NET Developer and tech lead from Ireland!
Benchmark Results:
Benchmark Code:
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using System.Collections.Generic;
[Config(typeof(Config))]
[HideColumns("Error", "StdDev", "Median", "RatioSD", "EnvironmentVariables", "Runtime")]
public class AddRangeBenchmark
{
private readonly IEnumerable<int> _source = GetItems(1024);
private readonly List<int> _list = new();
[Benchmark]
public void AddRange()
{
_list.Clear();
_list.AddRange(_source);
}
private static IEnumerable<int> GetItems(int count)
{
for (int i = 0; i < count; i++) yield return i;
}
private class Config : ManualConfig
{
public Config()
{
AddJob(Job.Default.WithId(".NET 7").WithRuntime(CoreRuntime.Core70).AsBaseline());
AddJob(Job.Default.WithId(".NET 8 w/o PGO").WithRuntime(CoreRuntime.Core80).WithEnvironmentVariable("DOTNET_TieredPGO", "0"));
AddJob(Job.Default.WithId(".NET 8").WithRuntime(CoreRuntime.Core80));
SummaryStyle =
SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage);
}
}
}
Powered by SharpLab
// .NET 7
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Security;
using System.Security.Permissions;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using Microsoft.CodeAnalysis;
[assembly: CompilationRelaxations(8)]
[assembly: RuntimeCompatibility(WrapNonExceptionThrows = true)]
[assembly: Debuggable(DebuggableAttribute.DebuggingModes.IgnoreSymbolStoreSequencePoints)]
[assembly: SecurityPermission(SecurityAction.RequestMinimum, SkipVerification = true)]
[assembly: AssemblyVersion("0.0.0.0")]
[module: UnverifiableCode]
namespace Microsoft.CodeAnalysis
{
[CompilerGenerated]
[Embedded]
internal sealed class EmbeddedAttribute : Attribute
{
}
}
namespace System.Runtime.CompilerServices
{
[CompilerGenerated]
[Microsoft.CodeAnalysis.Embedded]
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Event | AttributeTargets.Parameter | AttributeTargets.ReturnValue | AttributeTargets.GenericParameter, AllowMultiple = false, Inherited = false)]
internal sealed class NullableAttribute : Attribute
{
public readonly byte[] NullableFlags;
public NullableAttribute(byte P_0)
{
byte[] array = new byte[1];
array[0] = P_0;
NullableFlags = array;
}
public NullableAttribute(byte[] P_0)
{
NullableFlags = P_0;
}
}
[CompilerGenerated]
[Microsoft.CodeAnalysis.Embedded]
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct | AttributeTargets.Method | AttributeTargets.Interface | AttributeTargets.Delegate, AllowMultiple = false, Inherited = false)]
internal sealed class NullableContextAttribute : Attribute
{
public readonly byte Flag;
public NullableContextAttribute(byte P_0)
{
Flag = P_0;
}
}
}
[System.Runtime.CompilerServices.NullableContext(1)]
[System.Runtime.CompilerServices.Nullable(0)]
[Config(typeof(Config))]
[HideColumns(new string[] { "Error", "StdDev", "Median", "RatioSD", "EnvironmentVariables", "Runtime" })]
public class AddRangeBenchmark
{
[System.Runtime.CompilerServices.NullableContext(0)]
private class Config : ManualConfig
{
public Config()
{
Job[] array = new Job[1];
array[0] = JobExtensions.AsBaseline(JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 7"), CoreRuntime.Core70));
AddJob(array);
Job[] array2 = new Job[1];
array2[0] = JobExtensions.WithEnvironmentVariable(JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 8 w/o PGO"), CoreRuntime.Core80), "DOTNET_TieredPGO", "0");
AddJob(array2);
Job[] array3 = new Job[1];
array3[0] = JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 8"), CoreRuntime.Core80);
AddJob(array3);
base.SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage);
}
}
[CompilerGenerated]
private sealed class <GetItems>d__3 : IEnumerable<int>, IEnumerable, IEnumerator<int>, IEnumerator, IDisposable
{
private int <>1__state;
private int <>2__current;
private int <>l__initialThreadId;
private int count;
public int <>3__count;
private int <i>5__2;
int IEnumerator<int>.Current
{
[DebuggerHidden]
get
{
return <>2__current;
}
}
object IEnumerator.Current
{
[DebuggerHidden]
[return: System.Runtime.CompilerServices.Nullable(0)]
get
{
return <>2__current;
}
}
[DebuggerHidden]
public <GetItems>d__3(int <>1__state)
{
this.<>1__state = <>1__state;
<>l__initialThreadId = Environment.CurrentManagedThreadId;
}
[DebuggerHidden]
void IDisposable.Dispose()
{
}
private bool MoveNext()
{
int num = <>1__state;
if (num != 0)
{
if (num != 1)
{
return false;
}
<>1__state = -1;
<i>5__2++;
}
else
{
<>1__state = -1;
<i>5__2 = 0;
}
if (<i>5__2 < count)
{
<>2__current = <i>5__2;
<>1__state = 1;
return true;
}
return false;
}
bool IEnumerator.MoveNext()
{
//ILSpy generated this explicit interface implementation from .override directive in MoveNext
return this.MoveNext();
}
[DebuggerHidden]
void IEnumerator.Reset()
{
throw new NotSupportedException();
}
[DebuggerHidden]
IEnumerator<int> IEnumerable<int>.GetEnumerator()
{
<GetItems>d__3 <GetItems>d__;
if (<>1__state == -2 && <>l__initialThreadId == Environment.CurrentManagedThreadId)
{
<>1__state = 0;
<GetItems>d__ = this;
}
else
{
<GetItems>d__ = new <GetItems>d__3(0);
}
<GetItems>d__.count = <>3__count;
return <GetItems>d__;
}
[DebuggerHidden]
IEnumerator IEnumerable.GetEnumerator()
{
return ((IEnumerable<int>)this).GetEnumerator();
}
}
private readonly IEnumerable<int> _source = GetItems(1024);
private readonly List<int> _list = new List<int>();
[Benchmark(27, "_")]
public void AddRange()
{
_list.Clear();
_list.AddRange(_source);
}
[IteratorStateMachine(typeof(<GetItems>d__3))]
private static IEnumerable<int> GetItems(int count)
{
<GetItems>d__3 <GetItems>d__ = new <GetItems>d__3(-2);
<GetItems>d__.<>3__count = count;
return <GetItems>d__;
}
}
// .NET 8
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Security;
using System.Security.Permissions;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
[assembly: CompilationRelaxations(8)]
[assembly: RuntimeCompatibility(WrapNonExceptionThrows = true)]
[assembly: Debuggable(DebuggableAttribute.DebuggingModes.IgnoreSymbolStoreSequencePoints)]
[assembly: SecurityPermission(SecurityAction.RequestMinimum, SkipVerification = true)]
[assembly: AssemblyVersion("0.0.0.0")]
[module: UnverifiableCode]
[module: RefSafetyRules(11)]
[NullableContext(1)]
[Nullable(0)]
[Config(typeof(Config))]
[HideColumns(new string[] { "Error", "StdDev", "Median", "RatioSD", "EnvironmentVariables", "Runtime" })]
public class AddRangeBenchmark
{
[NullableContext(0)]
private class Config : ManualConfig
{
public Config()
{
Job[] array = new Job[1];
array[0] = JobExtensions.AsBaseline(JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 7"), CoreRuntime.Core70));
AddJob(array);
Job[] array2 = new Job[1];
array2[0] = JobExtensions.WithEnvironmentVariable(JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 8 w/o PGO"), CoreRuntime.Core80), "DOTNET_TieredPGO", "0");
AddJob(array2);
Job[] array3 = new Job[1];
array3[0] = JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 8"), CoreRuntime.Core80);
AddJob(array3);
base.SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage);
}
}
[CompilerGenerated]
private sealed class <GetItems>d__3 : IEnumerable<int>, IEnumerable, IEnumerator<int>, IEnumerator, IDisposable
{
private int <>1__state;
private int <>2__current;
private int <>l__initialThreadId;
private int count;
public int <>3__count;
private int <i>5__2;
int IEnumerator<int>.Current
{
[DebuggerHidden]
get
{
return <>2__current;
}
}
object IEnumerator.Current
{
[DebuggerHidden]
[return: Nullable(0)]
get
{
return <>2__current;
}
}
[DebuggerHidden]
public <GetItems>d__3(int <>1__state)
{
this.<>1__state = <>1__state;
<>l__initialThreadId = Environment.CurrentManagedThreadId;
}
[DebuggerHidden]
void IDisposable.Dispose()
{
}
private bool MoveNext()
{
int num = <>1__state;
if (num != 0)
{
if (num != 1)
{
return false;
}
<>1__state = -1;
<i>5__2++;
}
else
{
<>1__state = -1;
<i>5__2 = 0;
}
if (<i>5__2 < count)
{
<>2__current = <i>5__2;
<>1__state = 1;
return true;
}
return false;
}
bool IEnumerator.MoveNext()
{
//ILSpy generated this explicit interface implementation from .override directive in MoveNext
return this.MoveNext();
}
[DebuggerHidden]
void IEnumerator.Reset()
{
throw new NotSupportedException();
}
[DebuggerHidden]
IEnumerator<int> IEnumerable<int>.GetEnumerator()
{
<GetItems>d__3 <GetItems>d__;
if (<>1__state == -2 && <>l__initialThreadId == Environment.CurrentManagedThreadId)
{
<>1__state = 0;
<GetItems>d__ = this;
}
else
{
<GetItems>d__ = new <GetItems>d__3(0);
}
<GetItems>d__.count = <>3__count;
return <GetItems>d__;
}
[DebuggerHidden]
IEnumerator IEnumerable.GetEnumerator()
{
return ((IEnumerable<int>)this).GetEnumerator();
}
}
private readonly IEnumerable<int> _source = GetItems(1024);
private readonly List<int> _list = new List<int>();
[Benchmark(27, "_")]
public void AddRange()
{
_list.Clear();
_list.AddRange(_source);
}
[IteratorStateMachine(typeof(<GetItems>d__3))]
private static IEnumerable<int> GetItems(int count)
{
<GetItems>d__3 <GetItems>d__ = new <GetItems>d__3(-2);
<GetItems>d__.<>3__count = count;
return <GetItems>d__;
}
}
Powered by SharpLab
// .NET 7
.method public hidebysig
instance void AddRange () cil managed
{
.custom instance void [BenchmarkDotNet.Annotations]BenchmarkDotNet.Attributes.BenchmarkAttribute::.ctor(int32, string) = (
01 00 1b 00 00 00 01 5f 00 00
)
// Method begins at RVA 0x208e
// Code size 29 (0x1d)
.maxstack 8
// sequence point: (line 30, col 9) to (line 30, col 23) in _
IL_0000: ldarg.0
IL_0001: ldfld class [System.Collections]System.Collections.Generic.List`1<int32> AddRangeBenchmark::_list
IL_0006: callvirt instance void class [System.Collections]System.Collections.Generic.List`1<int32>::Clear()
// sequence point: (line 31, col 9) to (line 31, col 33) in _
IL_000b: ldarg.0
IL_000c: ldfld class [System.Collections]System.Collections.Generic.List`1<int32> AddRangeBenchmark::_list
IL_0011: ldarg.0
IL_0012: ldfld class [System.Runtime]System.Collections.Generic.IEnumerable`1<int32> AddRangeBenchmark::_source
IL_0017: callvirt instance void class [System.Collections]System.Collections.Generic.List`1<int32>::AddRange(class [System.Runtime]System.Collections.Generic.IEnumerable`1<!0>)
// sequence point: (line 32, col 5) to (line 32, col 6) in _
IL_001c: ret
}
// .NET 8
.method public hidebysig
instance void AddRange () cil managed
{
.custom instance void [BenchmarkDotNet.Annotations]BenchmarkDotNet.Attributes.BenchmarkAttribute::.ctor(int32, string) = (
01 00 1b 00 00 00 01 5f 00 00
)
// Method begins at RVA 0x2050
// Code size 29 (0x1d)
.maxstack 8
// sequence point: (line 30, col 9) to (line 30, col 23) in _
IL_0000: ldarg.0
IL_0001: ldfld class [System.Collections]System.Collections.Generic.List`1<int32> AddRangeBenchmark::_list
IL_0006: callvirt instance void class [System.Collections]System.Collections.Generic.List`1<int32>::Clear()
// sequence point: (line 31, col 9) to (line 31, col 33) in _
IL_000b: ldarg.0
IL_000c: ldfld class [System.Collections]System.Collections.Generic.List`1<int32> AddRangeBenchmark::_list
IL_0011: ldarg.0
IL_0012: ldfld class [System.Runtime]System.Collections.Generic.IEnumerable`1<int32> AddRangeBenchmark::_source
IL_0017: callvirt instance void class [System.Collections]System.Collections.Generic.List`1<int32>::AddRange(class [System.Runtime]System.Collections.Generic.IEnumerable`1<!0>)
// sequence point: (line 32, col 5) to (line 32, col 6) in _
IL_001c: ret
}
Powered by SharpLab
|
|
Benchmark Description:
AddRange is was previously implemented as delegating to InsertRange, and InsertRange in turn has a more complicated inner loop as part of adding each item from a source enumerable into the list. By just copying InsertRange's source into AddRange, deleting all the irrelevant stuff, and changing the Insert call to Add, throughput improves measurably.
We can see this that Dynamic PGO which is on by default in .NET 8 makes up a huge part of the performance difference. We can turn off Dynamic PGO by setting an environment variable as shown below ...
private class Config : ManualConfig
{
public Config()
{
AddJob(Job.Default.WithId(".NET 7").WithRuntime(CoreRuntime.Core70).AsBaseline());
AddJob(Job.Default.WithId(".NET 8 w/o PGO").WithRuntime(CoreRuntime.Core80).WithEnvironmentVariable("DOTNET_TieredPGO", "0"));
AddJob(Job.Default.WithId(".NET 8").WithRuntime(CoreRuntime.Core80));
SummaryStyle =
SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage);
}
}
### General Setup Overview
The benchmark setup described uses BenchmarkDotNet, a powerful .NET library for benchmarking, to measure the performance of a specific method, `AddRange`, across different .NET runtime versions. The configuration specifies that the benchmarks will run on .NET 7 as the baseline and compare it against two configurations of .NET 8, one with Profile-Guided Optimization (PGO) disabled.
- **.NET Versions**: The benchmarks are specifically targeting .NET 7 and .NET 8. .NET 7 is set as the baseline for comparison.
- **Configuration**: It uses a custom `Config` class derived from `ManualConfig` to specify the jobs (benchmark runs) and the summary style. The configuration hides certain columns in the output to focus on the most relevant data.
- **Jobs**: Three jobs are defined:
- `.NET 7` as the baseline.
- `.NET 8 w/o PGO` with an environment variable `DOTNET_TieredPGO` set to "0" to disable Profile-Guided Optimization.
- `.NET 8` with default settings, presumably with PGO enabled.
- **Summary Style**: The summary output is customized to use percentage ratios for easier comparison between different runs.
### Benchmark Method: `AddRange`
#### Purpose
The `AddRange` method is designed to test the performance of adding a range of items to a `List<int>` in .NET. This operation is common in many applications, making it a valuable scenario to benchmark.
#### Performance Aspect
This benchmark specifically measures how efficiently the `List<T>.AddRange` method can add elements from an `IEnumerable<int>` to a `List<int>`. It tests the allocation and iteration performance over a collection, as well as the efficiency of the internal data structures used by `List<T>` when expanding to accommodate new elements.
#### What It Measures
- **Allocation Efficiency**: How well the `List<T>` manages memory allocations when adding a large number of elements.
- **Iteration Performance**: The speed at which the `List<T>` can iterate through the `IEnumerable<int>` and add each element.
- **Data Structure Efficiency**: How the internal structure of `List<T>` handles the addition of many elements, including any resizing operations that may be necessary.
#### Importance
Understanding the performance of `AddRange` is crucial for developers who rely on collections in performance-critical applications. It can help identify potential bottlenecks and guide optimization efforts, especially when working with large datasets.
#### Expected Results or Insights
- **.NET Version Comparison**: Insights into how different .NET versions handle `List<T>.AddRange` operations, specifically the impact of new optimizations or runtime improvements in .NET 8.
- **PGO Impact**: Understanding the effect of Profile-Guided Optimization on the performance of `AddRange`. PGO can potentially improve performance by optimizing hot paths and frequently executed code, and this benchmark will reveal if such optimizations are beneficial for collection operations.
- **Performance Trends**: General trends in allocation and iteration efficiency across different runtime versions and configurations, helping to identify best practices and potential areas for improvement in code that heavily uses `List<T>.AddRange`.
By running these benchmarks, developers can gain valuable insights into the performance characteristics of collection operations across different .NET runtime versions, guiding optimization and development decisions for high-performance applications.