Compiled lambda expressions used as property getters and setters: wrong benchmarking method or wrong lambda expression construction?

Question

Hi I was trying out using (cached) compiled lambda expressions for properties access, for sure I got a result much better (i.e. faster) than when using PropertyInfo.GetValue()/SetValue() method calls. However, I feel it's still really far getting close to the "native" properties speed. Is it the benchmarking method that makes the results so different from what others could get?

Here is below the result I got after running my piece of code below:

Native: Elapsed = 00:00:00.0995876 (99.5876 ms); Step = 1.992E-005 ms
Lambda Expression: Elapsed = 00:00:00.5369273 (536.9273 ms); Step = 1.074E-004 ms
Property Info: Elapsed = 00:00:01.9187312 (1918.7312 ms); Step = 3.837E-004 ms

1.000 < 5.392 < 19.267

Honestly I feel that based on others benchmarks, the compiled lambda expressions should be twice slower than using regular properties, not like between 5 - 6 times slower.

Any thought? The bench-marking method? The way the compiled lambda expression is computed?

public static class Program
{
    public static void Main(params string[] args)
    {
        var stepCount = 5000000UL;

        var dummy = new Dummy();

        const string propertyName = "Soother";

        const bool propertyValue = true;

        var propertyInfo = typeof(Dummy).GetProperty(propertyName);

        var nativeBenchmark = Benchmark.Run("Native", stepCount, () => dummy.Soother = propertyValue);
        var lambdaExpressionBenchmark = Benchmark.Run("Lambda Expression", stepCount, () => dummy.Set(propertyName, propertyValue));
        var propertyInfoBenchmark = Benchmark.Run("Property Info", stepCount, () => propertyInfo.SetValue(dummy, propertyValue, null));

        var benchmarkReports = new[] { nativeBenchmark, lambdaExpressionBenchmark, propertyInfoBenchmark }.OrderBy(item => item.ElapsedMilliseconds);

        benchmarkReports.Join(Environment.NewLine).WriteLineToConsole();

        var fastest = benchmarkReports.First().ElapsedMilliseconds;

        benchmarkReports.Select(report => (report.ElapsedMilliseconds / fastest).ToString("0.000")).Join(" < ").WriteLineToConsole();

        Console.ReadKey();
    }
}

public class Dummy
{
    public bool? Soother { get; set; } = true;
}

public class BenchMarkReport
{
    #region Fields & Properties

    public string Name { get; }
    public TimeSpan ElapsedTime { get; }
    public double ElapsedMilliseconds
    {
        get
        {
            return ElapsedTime.TotalMilliseconds;
        }
    }
    public ulong StepCount { get; }
    public double StepElapsedMilliseconds
    {
        get
        {
            return ElapsedMilliseconds / StepCount;
        }
    }

    #endregion

    #region Constructors

    internal BenchMarkReport(string name, TimeSpan elapsedTime, ulong stepCount)
    {
        Name = name;
        ElapsedTime = elapsedTime;
        StepCount = stepCount;
    }

    #endregion

    #region Methods

    public override string ToString()
    {
        return $"{Name}: Elapsed = {ElapsedTime} ({ElapsedMilliseconds} ms); Step = {StepElapsedMilliseconds:0.###E+000} ms";
    }

    #endregion
}

public class Benchmark
{
    #region Fields & Properties

    private readonly Action _stepAction;

    public string Name { get; }

    public ulong StepCount { get; }

    public Benchmark(string name, ulong stepCount, Action stepAction)
    {
        Name = name;
        StepCount = stepCount;
        _stepAction = stepAction;
    }

    #endregion

    #region Constructors

    #endregion

    #region Methods

    public static BenchMarkReport Run(string name, ulong stepCount, Action stepAction)
    {
        var benchmark = new Benchmark(name, stepCount, stepAction);

        var benchmarkReport = benchmark.Run();

        return benchmarkReport;
    }

    public BenchMarkReport Run()
    {
        return Run(StepCount);
    }

    public BenchMarkReport Run(ulong stepCountOverride)
    {
        var stopwatch = Stopwatch.StartNew();

        for (ulong i = 0; i < StepCount; i++)
        {
            _stepAction();
        }

        stopwatch.Stop();

        var benchmarkReport = new BenchMarkReport(Name, stopwatch.Elapsed, stepCountOverride);

        return benchmarkReport;
    }

    #endregion
}

public static class ObjectExtensions
{
    public static void WriteToConsole<TInstance>(this TInstance instance)
    {
        Console.Write(instance);
    }

    public static void WriteLineToConsole<TInstance>(this TInstance instance)
    {
        Console.WriteLine(instance);
    }

    // Goodies: add name inference from property lambda expression
    // e.g. "instance => instance.PropertyName" redirected using "PropertyName"

    public static TProperty Get<TInstance, TProperty>(this TInstance instance, string propertyName)
    {
        return FastPropertyRepository<TInstance, TProperty>.GetGetter(propertyName)(instance);
    }

    public static void Set<TInstance, TProperty>(this TInstance instance, string propertyName, TProperty propertyValue)
    {
        FastPropertyRepository<TInstance, TProperty>.GetSetter(propertyName)(instance, propertyValue);
    }
}

public static class EnumerableExtensions
{
    public static string Join<TSource>(this IEnumerable<TSource> source, string separator = ", ")
    {
        return string.Join(separator, source);
    }
}

internal static class FastPropertyRepository<TInstance, TProperty>
{
    private static readonly IDictionary<string, Action<TInstance, TProperty>> Setters;
    private static readonly IDictionary<string, Func<TInstance, TProperty>> Getters;

    static FastPropertyRepository()
    {
        Getters = new ConcurrentDictionary<string, Func<TInstance, TProperty>>();
        Setters = new ConcurrentDictionary<string, Action<TInstance, TProperty>>();
    }

    public static Func<TInstance, TProperty> GetGetter(string propertyName)
    {
        Func<TInstance, TProperty> getter;
        if (!Getters.TryGetValue(propertyName, out getter))
        {
            getter = FastPropertyFactory.GeneratePropertyGetter<TInstance, TProperty>(propertyName);
            Getters[propertyName] = getter;
        }

        return getter;
    }

    public static Action<TInstance, TProperty> GetSetter(string propertyName)
    {
        Action<TInstance, TProperty> setter;
        if (!Setters.TryGetValue(propertyName, out setter))
        {
            setter = FastPropertyFactory.GeneratePropertySetter<TInstance, TProperty>(propertyName);
            Setters[propertyName] = setter;
        }

        return setter;
    }
}

internal static class FastPropertyFactory
{
    public static Func<TInstance, TProperty> GeneratePropertyGetter<TInstance, TProperty>(string propertyName)
    {
        var parameterExpression = Expression.Parameter(typeof(TInstance), "value");

        var propertyValueExpression = Expression.Property(parameterExpression, propertyName);

        var expression = propertyValueExpression.Type == typeof(TProperty) ? propertyValueExpression : (Expression)Expression.Convert(propertyValueExpression, typeof(TProperty));

        var propertyGetter = Expression.Lambda<Func<TInstance, TProperty>>(expression, parameterExpression).Compile();

        return propertyGetter;
    }

    public static Action<TInstance, TProperty> GeneratePropertySetter<TInstance, TProperty>(string propertyName)
    {
        var instanceParameterExpression = Expression.Parameter(typeof(TInstance));

        var parameterExpression = Expression.Parameter(typeof(TProperty), propertyName);

        var propertyValueExpression = Expression.Property(instanceParameterExpression, propertyName);

        var conversionExpression = propertyValueExpression.Type == typeof(TProperty) ? parameterExpression : (Expression)Expression.Convert(parameterExpression, propertyValueExpression.Type);

        var propertySetter = Expression.Lambda<Action<TInstance, TProperty>>(Expression.Assign(propertyValueExpression, conversionExpression), instanceParameterExpression, parameterExpression).Compile();

        return propertySetter;
    }
}

Did you run your results using Release? The results might surprise you. — Svek
I also think you might want to simplify the benchmark into individual methods rather than passing the action around. I'm not an expert of the JIT, but I would suspect that would have something to do with it. — Svek
@Svek well actually I tried using the Release build but turns out it's pretty counter-intuitive, the ratio is now about 15 times slower, seems the average time spent on executing a step for the native access is much faster in release while the time waster on the cached compiled lambda expressions is a bit slower. About about wrapping up with a delegate, it does slow things a bit down, but in terms of ratio it should be consistent with or without going this way. — Natalie Perret
Well, to your question it's probably not your benchmark method. In response to your statement of "counter-intuitive".... It's most likely a misunderstanding of what's going on under the hood. Maybe you might want to explain where you got your expectation from and maybe form your question differently? — Svek
@Svek well actually I managed to find the bottleneck... the dictionary access is far more time-consuming than what I was expecting. — Natalie Perret

Svek Svek · Accepted Answer · 2017-02-14T09:53:28

I simplified your work into smaller methods. It increased the performance overall, but it also widened the gap.

Native              : 00:00:00.0029713 (    2.9713ms) 5.9426E-07
Lambda Expression   : 00:00:00.4356385 (  435.6385ms) 8.71277E-05
Property Info       : 00:00:01.3436626 ( 1343.6626ms) 0.00026873252

Here are the methods used

public class Dummy
{
    public bool? Soother { get; set; } = true;
}

public class Lab
{
    Dummy _dummy = new Dummy();
    ulong _iterations = 5000000UL;
    const bool _propertyValue = true;
    const string _propertyName = "Soother";

    public BenchmarkReport RunNative()
    {
        Stopwatch stopwatch = Stopwatch.StartNew();
        for (ulong i = 0; i < _iterations; i++)
        {
            _dummy.Soother = _propertyValue;
        }
        stopwatch.Stop();

        return new BenchmarkReport("Native", stopwatch.Elapsed, _iterations);
    }

    public BenchmarkReport RunLambdaExpression()
    {
        Stopwatch stopwatch = Stopwatch.StartNew();
        for (ulong i = 0; i < _iterations; i++)
        {
            _dummy.Set(_propertyName, _propertyValue);
        }
        stopwatch.Stop();

        return new BenchmarkReport("Lambda Expression", stopwatch.Elapsed, _iterations);
    }

    public BenchmarkReport RunPropertyInfo()
    {
        PropertyInfo propertyInfo = typeof(Dummy).GetProperty(_propertyName);

        Stopwatch stopwatch = Stopwatch.StartNew();
        for (ulong i = 0; i < _iterations; i++)
        {
            propertyInfo.SetValue(_dummy, _propertyValue);
        }
        stopwatch.Stop();

        return new BenchmarkReport("Property Info", stopwatch.Elapsed, _iterations);
    }
}

public class BenchmarkReport
{
    public string Name { get; set; }
    public TimeSpan ElapsedTime { get; set; }
    public ulong Iterations { get; set; }

    public BenchmarkReport(string name, TimeSpan elapsedTime, ulong iterations)
    {
        Name = name;
        ElapsedTime = elapsedTime;
        Iterations = iterations;
    }
}

and the program to run it

public static class Program
{
    public static void Main(params string[] args)
    {

        Lab lab = new Lab();
        List<BenchmarkReport> benchmarkReports = new List<BenchmarkReport>()
        {
            lab.RunNative(),
            lab.RunLambdaExpression(),
            lab.RunPropertyInfo()
        };

        foreach (var report in benchmarkReports)
        {
            Console.WriteLine("{0}: {1} ({2}ms) {3}",
                report.Name.PadRight(20),
                report.ElapsedTime,
                report.ElapsedTime.TotalMilliseconds.ToString().PadLeft(10),
                (double)report.ElapsedTime.TotalMilliseconds / report.Iterations);
        }

        Console.ReadKey();
    }
}

Compiled lambda expressions used as property getters and setters: wrong benchmarking method or wrong lambda expression construction?

2 Answers