AddRange() .NET 8 performance improvements with and without Dynamic Profile Guided Optimization (PGO)
Date Added (UTC):
08 Apr 2024 @ 01:51
Date Updated (UTC):08 Apr 2024 @ 01:57
.NET Version(s): Tag(s):
#.Net8PerfImprovement #Collections #Dynamic PGO
Added By:
.NET Developer and tech lead from Ireland!
Benchmark Results:
Benchmark Code:
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using System.Collections.Generic;
[Config(typeof(Config))]
[HideColumns("Error", "StdDev", "Median", "RatioSD", "EnvironmentVariables", "Runtime")]
public class AddRangeBenchmark
{
private readonly IEnumerable<int> _source = GetItems(1024);
private readonly List<int> _list = new();
[Benchmark]
public void AddRange()
{
_list.Clear();
_list.AddRange(_source);
}
private static IEnumerable<int> GetItems(int count)
{
for (int i = 0; i < count; i++) yield return i;
}
private class Config : ManualConfig
{
public Config()
{
AddJob(Job.Default.WithId(".NET 7").WithRuntime(CoreRuntime.Core70).AsBaseline());
AddJob(Job.Default.WithId(".NET 8 w/o PGO").WithRuntime(CoreRuntime.Core80).WithEnvironmentVariable("DOTNET_TieredPGO", "0"));
AddJob(Job.Default.WithId(".NET 8").WithRuntime(CoreRuntime.Core80));
SummaryStyle =
SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage);
}
}
}
Powered by SharpLab
// .NET 7
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Security;
using System.Security.Permissions;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using Microsoft.CodeAnalysis;
[assembly: CompilationRelaxations(8)]
[assembly: RuntimeCompatibility(WrapNonExceptionThrows = true)]
[assembly: Debuggable(DebuggableAttribute.DebuggingModes.IgnoreSymbolStoreSequencePoints)]
[assembly: SecurityPermission(SecurityAction.RequestMinimum, SkipVerification = true)]
[assembly: AssemblyVersion("0.0.0.0")]
[module: UnverifiableCode]
namespace Microsoft.CodeAnalysis
{
[CompilerGenerated]
[Embedded]
internal sealed class EmbeddedAttribute : Attribute
{
}
}
namespace System.Runtime.CompilerServices
{
[CompilerGenerated]
[Microsoft.CodeAnalysis.Embedded]
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Event | AttributeTargets.Parameter | AttributeTargets.ReturnValue | AttributeTargets.GenericParameter, AllowMultiple = false, Inherited = false)]
internal sealed class NullableAttribute : Attribute
{
public readonly byte[] NullableFlags;
public NullableAttribute(byte P_0)
{
byte[] array = new byte[1];
array[0] = P_0;
NullableFlags = array;
}
public NullableAttribute(byte[] P_0)
{
NullableFlags = P_0;
}
}
[CompilerGenerated]
[Microsoft.CodeAnalysis.Embedded]
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct | AttributeTargets.Method | AttributeTargets.Interface | AttributeTargets.Delegate, AllowMultiple = false, Inherited = false)]
internal sealed class NullableContextAttribute : Attribute
{
public readonly byte Flag;
public NullableContextAttribute(byte P_0)
{
Flag = P_0;
}
}
}
[System.Runtime.CompilerServices.NullableContext(1)]
[System.Runtime.CompilerServices.Nullable(0)]
[Config(typeof(Config))]
[HideColumns(new string[] { "Error", "StdDev", "Median", "RatioSD", "EnvironmentVariables", "Runtime" })]
public class AddRangeBenchmark
{
[System.Runtime.CompilerServices.NullableContext(0)]
private class Config : ManualConfig
{
public Config()
{
Job[] array = new Job[1];
array[0] = JobExtensions.AsBaseline(JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 7"), CoreRuntime.Core70));
AddJob(array);
Job[] array2 = new Job[1];
array2[0] = JobExtensions.WithEnvironmentVariable(JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 8 w/o PGO"), CoreRuntime.Core80), "DOTNET_TieredPGO", "0");
AddJob(array2);
Job[] array3 = new Job[1];
array3[0] = JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 8"), CoreRuntime.Core80);
AddJob(array3);
base.SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage);
}
}
[CompilerGenerated]
private sealed class <GetItems>d__3 : IEnumerable<int>, IEnumerable, IEnumerator<int>, IEnumerator, IDisposable
{
private int <>1__state;
private int <>2__current;
private int <>l__initialThreadId;
private int count;
public int <>3__count;
private int <i>5__2;
int IEnumerator<int>.Current
{
[DebuggerHidden]
get
{
return <>2__current;
}
}
object IEnumerator.Current
{
[DebuggerHidden]
[return: System.Runtime.CompilerServices.Nullable(0)]
get
{
return <>2__current;
}
}
[DebuggerHidden]
public <GetItems>d__3(int <>1__state)
{
this.<>1__state = <>1__state;
<>l__initialThreadId = Environment.CurrentManagedThreadId;
}
[DebuggerHidden]
void IDisposable.Dispose()
{
}
private bool MoveNext()
{
int num = <>1__state;
if (num != 0)
{
if (num != 1)
{
return false;
}
<>1__state = -1;
<i>5__2++;
}
else
{
<>1__state = -1;
<i>5__2 = 0;
}
if (<i>5__2 < count)
{
<>2__current = <i>5__2;
<>1__state = 1;
return true;
}
return false;
}
bool IEnumerator.MoveNext()
{
//ILSpy generated this explicit interface implementation from .override directive in MoveNext
return this.MoveNext();
}
[DebuggerHidden]
void IEnumerator.Reset()
{
throw new NotSupportedException();
}
[DebuggerHidden]
IEnumerator<int> IEnumerable<int>.GetEnumerator()
{
<GetItems>d__3 <GetItems>d__;
if (<>1__state == -2 && <>l__initialThreadId == Environment.CurrentManagedThreadId)
{
<>1__state = 0;
<GetItems>d__ = this;
}
else
{
<GetItems>d__ = new <GetItems>d__3(0);
}
<GetItems>d__.count = <>3__count;
return <GetItems>d__;
}
[DebuggerHidden]
IEnumerator IEnumerable.GetEnumerator()
{
return ((IEnumerable<int>)this).GetEnumerator();
}
}
private readonly IEnumerable<int> _source = GetItems(1024);
private readonly List<int> _list = new List<int>();
[Benchmark(27, "_")]
public void AddRange()
{
_list.Clear();
_list.AddRange(_source);
}
[IteratorStateMachine(typeof(<GetItems>d__3))]
private static IEnumerable<int> GetItems(int count)
{
<GetItems>d__3 <GetItems>d__ = new <GetItems>d__3(-2);
<GetItems>d__.<>3__count = count;
return <GetItems>d__;
}
}
// .NET 8
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Security;
using System.Security.Permissions;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
[assembly: CompilationRelaxations(8)]
[assembly: RuntimeCompatibility(WrapNonExceptionThrows = true)]
[assembly: Debuggable(DebuggableAttribute.DebuggingModes.IgnoreSymbolStoreSequencePoints)]
[assembly: SecurityPermission(SecurityAction.RequestMinimum, SkipVerification = true)]
[assembly: AssemblyVersion("0.0.0.0")]
[module: UnverifiableCode]
[module: RefSafetyRules(11)]
[NullableContext(1)]
[Nullable(0)]
[Config(typeof(Config))]
[HideColumns(new string[] { "Error", "StdDev", "Median", "RatioSD", "EnvironmentVariables", "Runtime" })]
public class AddRangeBenchmark
{
[NullableContext(0)]
private class Config : ManualConfig
{
public Config()
{
Job[] array = new Job[1];
array[0] = JobExtensions.AsBaseline(JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 7"), CoreRuntime.Core70));
AddJob(array);
Job[] array2 = new Job[1];
array2[0] = JobExtensions.WithEnvironmentVariable(JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 8 w/o PGO"), CoreRuntime.Core80), "DOTNET_TieredPGO", "0");
AddJob(array2);
Job[] array3 = new Job[1];
array3[0] = JobExtensions.WithRuntime(JobExtensions.WithId(JobMode<Job>.Default, ".NET 8"), CoreRuntime.Core80);
AddJob(array3);
base.SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage);
}
}
[CompilerGenerated]
private sealed class <GetItems>d__3 : IEnumerable<int>, IEnumerable, IEnumerator<int>, IEnumerator, IDisposable
{
private int <>1__state;
private int <>2__current;
private int <>l__initialThreadId;
private int count;
public int <>3__count;
private int <i>5__2;
int IEnumerator<int>.Current
{
[DebuggerHidden]
get
{
return <>2__current;
}
}
object IEnumerator.Current
{
[DebuggerHidden]
[return: Nullable(0)]
get
{
return <>2__current;
}
}
[DebuggerHidden]
public <GetItems>d__3(int <>1__state)
{
this.<>1__state = <>1__state;
<>l__initialThreadId = Environment.CurrentManagedThreadId;
}
[DebuggerHidden]
void IDisposable.Dispose()
{
}
private bool MoveNext()
{
int num = <>1__state;
if (num != 0)
{
if (num != 1)
{
return false;
}
<>1__state = -1;
<i>5__2++;
}
else
{
<>1__state = -1;
<i>5__2 = 0;
}
if (<i>5__2 < count)
{
<>2__current = <i>5__2;
<>1__state = 1;
return true;
}
return false;
}
bool IEnumerator.MoveNext()
{
//ILSpy generated this explicit interface implementation from .override directive in MoveNext
return this.MoveNext();
}
[DebuggerHidden]
void IEnumerator.Reset()
{
throw new NotSupportedException();
}
[DebuggerHidden]
IEnumerator<int> IEnumerable<int>.GetEnumerator()
{
<GetItems>d__3 <GetItems>d__;
if (<>1__state == -2 && <>l__initialThreadId == Environment.CurrentManagedThreadId)
{
<>1__state = 0;
<GetItems>d__ = this;
}
else
{
<GetItems>d__ = new <GetItems>d__3(0);
}
<GetItems>d__.count = <>3__count;
return <GetItems>d__;
}
[DebuggerHidden]
IEnumerator IEnumerable.GetEnumerator()
{
return ((IEnumerable<int>)this).GetEnumerator();
}
}
private readonly IEnumerable<int> _source = GetItems(1024);
private readonly List<int> _list = new List<int>();
[Benchmark(27, "_")]
public void AddRange()
{
_list.Clear();
_list.AddRange(_source);
}
[IteratorStateMachine(typeof(<GetItems>d__3))]
private static IEnumerable<int> GetItems(int count)
{
<GetItems>d__3 <GetItems>d__ = new <GetItems>d__3(-2);
<GetItems>d__.<>3__count = count;
return <GetItems>d__;
}
}
Powered by SharpLab
// .NET 7
.method public hidebysig
instance void AddRange () cil managed
{
.custom instance void [BenchmarkDotNet.Annotations]BenchmarkDotNet.Attributes.BenchmarkAttribute::.ctor(int32, string) = (
01 00 1b 00 00 00 01 5f 00 00
)
// Method begins at RVA 0x208e
// Code size 29 (0x1d)
.maxstack 8
// sequence point: (line 30, col 9) to (line 30, col 23) in _
IL_0000: ldarg.0
IL_0001: ldfld class [System.Collections]System.Collections.Generic.List`1<int32> AddRangeBenchmark::_list
IL_0006: callvirt instance void class [System.Collections]System.Collections.Generic.List`1<int32>::Clear()
// sequence point: (line 31, col 9) to (line 31, col 33) in _
IL_000b: ldarg.0
IL_000c: ldfld class [System.Collections]System.Collections.Generic.List`1<int32> AddRangeBenchmark::_list
IL_0011: ldarg.0
IL_0012: ldfld class [System.Runtime]System.Collections.Generic.IEnumerable`1<int32> AddRangeBenchmark::_source
IL_0017: callvirt instance void class [System.Collections]System.Collections.Generic.List`1<int32>::AddRange(class [System.Runtime]System.Collections.Generic.IEnumerable`1<!0>)
// sequence point: (line 32, col 5) to (line 32, col 6) in _
IL_001c: ret
}
// .NET 8
.method public hidebysig
instance void AddRange () cil managed
{
.custom instance void [BenchmarkDotNet.Annotations]BenchmarkDotNet.Attributes.BenchmarkAttribute::.ctor(int32, string) = (
01 00 1b 00 00 00 01 5f 00 00
)
// Method begins at RVA 0x2050
// Code size 29 (0x1d)
.maxstack 8
// sequence point: (line 30, col 9) to (line 30, col 23) in _
IL_0000: ldarg.0
IL_0001: ldfld class [System.Collections]System.Collections.Generic.List`1<int32> AddRangeBenchmark::_list
IL_0006: callvirt instance void class [System.Collections]System.Collections.Generic.List`1<int32>::Clear()
// sequence point: (line 31, col 9) to (line 31, col 33) in _
IL_000b: ldarg.0
IL_000c: ldfld class [System.Collections]System.Collections.Generic.List`1<int32> AddRangeBenchmark::_list
IL_0011: ldarg.0
IL_0012: ldfld class [System.Runtime]System.Collections.Generic.IEnumerable`1<int32> AddRangeBenchmark::_source
IL_0017: callvirt instance void class [System.Collections]System.Collections.Generic.List`1<int32>::AddRange(class [System.Runtime]System.Collections.Generic.IEnumerable`1<!0>)
// sequence point: (line 32, col 5) to (line 32, col 6) in _
IL_001c: ret
}
Powered by SharpLab
|
|
Benchmark Description:
AddRange is was previously implemented as delegating to InsertRange, and InsertRange in turn has a more complicated inner loop as part of adding each item from a source enumerable into the list. By just copying InsertRange's source into AddRange, ***deleting all the irrelevant stuff***, and changing the Insert call to Add, throughput improves measurably.
[PR](https://github.com/dotnet/runtime/pull/76043)
We can see this that Dynamic PGO which is on by default in .NET 8 makes up a huge part of the performance difference.
We can turn off Dynamic PGO by setting an environment variable as shown below ...
```
private class Config : ManualConfig
{
public Config()
{
AddJob(Job.Default.WithId(".NET 7").WithRuntime(CoreRuntime.Core70).AsBaseline());
AddJob(Job.Default.WithId(".NET 8 w/o PGO").WithRuntime(CoreRuntime.Core80).WithEnvironmentVariable("DOTNET_TieredPGO", "0"));
AddJob(Job.Default.WithId(".NET 8").WithRuntime(CoreRuntime.Core80));
SummaryStyle =
SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage);
}
}
```
### General Setup Overview
The benchmark setup described uses BenchmarkDotNet, a powerful .NET library for benchmarking, to measure the performance of a specific method, `AddRange`, across different .NET runtime versions. The configuration specifies that the benchmarks will run on .NET 7 as the baseline and compare it against two configurations of .NET 8, one with Profile-Guided Optimization (PGO) disabled.
- **.NET Versions**: The benchmarks are specifically targeting .NET 7 and .NET 8. .NET 7 is set as the baseline for comparison.
- **Configuration**: It uses a custom `Config` class derived from `ManualConfig` to specify the jobs (benchmark runs) and the summary style. The configuration hides certain columns in the output to focus on the most relevant data.
- **Jobs**: Three jobs are defined:
- `.NET 7` as the baseline.
- `.NET 8 w/o PGO` with an environment variable `DOTNET_TieredPGO` set to "0" to disable Profile-Guided Optimization.
- `.NET 8` with default settings, presumably with PGO enabled.
- **Summary Style**: The summary output is customized to use percentage ratios for easier comparison between different runs.
### Benchmark Method: `AddRange`
#### Purpose
The `AddRange` method is designed to test the performance of adding a range of items to a `List<int>` in .NET. This operation is common in many applications, making it a valuable scenario to benchmark.
#### Performance Aspect
This benchmark specifically measures how efficiently the `List<T>.AddRange` method can add elements from an `IEnumerable<int>` to a `List<int>`. It tests the allocation and iteration performance over a collection, as well as the efficiency of the internal data structures used by `List<T>` when expanding to accommodate new elements.
#### What It Measures
- **Allocation Efficiency**: How well the `List<T>` manages memory allocations when adding a large number of elements.
- **Iteration Performance**: The speed at which the `List<T>` can iterate through the `IEnumerable<int>` and add each element.
- **Data Structure Efficiency**: How the internal structure of `List<T>` handles the addition of many elements, including any resizing operations that may be necessary.
#### Importance
Understanding the performance of `AddRange` is crucial for developers who rely on collections in performance-critical applications. It can help identify potential bottlenecks and guide optimization efforts, especially when working with large datasets.
#### Expected Results or Insights
- **.NET Version Comparison**: Insights into how different .NET versions handle `List<T>.AddRange` operations, specifically the impact of new optimizations or runtime improvements in .NET 8.
- **PGO Impact**: Understanding the effect of Profile-Guided Optimization on the performance of `AddRange`. PGO can potentially improve performance by optimizing hot paths and frequently executed code, and this benchmark will reveal if such optimizations are beneficial for collection operations.
- **Performance Trends**: General trends in allocation and iteration efficiency across different runtime versions and configurations, helping to identify best practices and potential areas for improvement in code that heavily uses `List<T>.AddRange`.
By running these benchmarks, developers can gain valuable insights into the performance characteristics of collection operations across different .NET runtime versions, guiding optimization and development decisions for high-performance applications.