Add fast path for Overlaps when other is ISet with fast lookup and same equality comparer #132763

[Advitalitum]

Similar to #132763

In FrozenSet.Overlaps(IEnumerable other) when other is ISet with fast lookup and same comparer, we should iterate through the smaller set and search within the larger one.

[azure-pipelines]

Azure Pipelines:
Successfully started running 3 pipeline(s).
13 pipeline(s) were filtered out due to trigger conditions.
There may be pipelines that require an authorized user to comment /azp run to run.

[dotnet-policy-service]

Tagging subscribers to this area: @dotnet/area-system-collections
See info in area-owners.md if you want to be subscribed.

[Advitalitum]

Maybe lookup in SortedSet is not fast enough... Need some benchmarking

[Advitalitum]

Let’s consider the computational complexity:

HashSet/FrozenSet/ImmutableHashSet size = hs
SortedSet size = ss

main:

hs >= ss, iterate SortedSet ~ O(ss)
hs < ss, iterate SortedSet ~ O(ss)

pr:

hs >= ss, iterate SortedSet ~ O(ss) // no diff from baseline
hs < ss, iterate HashSet ~ O(hs * log(ss)) // is better than O(ss) when ss >> hs

I don't want to make assumptions about O(hs * log(ss)) vs O(ss) performance in generic case, so I will use old logic when other is SortedSet/ImmutableSortedSet.

[Advitalitum]

@EgorBot -linux_amd -osx_arm64

using System.Collections.Frozen;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;

BenchmarkSwitcher.FromAssembly(typeof(Benchmarks).Assembly).Run(args);

[MemoryDiagnoser]
public class Benchmarks
{
    private FrozenSet<int> _frozenSet = null!;
    
    private IEnumerable<int> _largerDisjointFrozenSet = null!;
    private IEnumerable<int> _largerFrozenSetWithLateMatch = null!;
    
    private IEnumerable<int> _largerDisjointHashSet = null!;
    private IEnumerable<int> _largerHashSetWithLateMatch = null!;
    
    private IEnumerable<int> _largerDisjointImmutableHashSet = null!;
    private IEnumerable<int> _largerImmutableHashSetWithLateMatch = null!;
    
    private IEnumerable<int> _listWithImmediateMatch = null!;
    private IEnumerable<int> _largerDisjointList = null!;

    [GlobalSetup]
    public void Setup()
    {
        _frozenSet = Enumerable.Range(0, 100).ToFrozenSet();
        
        _largerDisjointFrozenSet = Enumerable.Range(100, 10_000).ToFrozenSet();
        _largerDisjointHashSet = new HashSet<int>(Enumerable.Range(100, 10_000));
        _largerDisjointImmutableHashSet = Enumerable.Range(100, 10_000).ToImmutableHashSet();

        _largerFrozenSetWithLateMatch = Enumerable.Range(100, 9_999).Concat([0]).ToFrozenSet();

        HashSet<int> largerHashSetWithLateMatch = new(Enumerable.Range(100, 9_999));
        largerHashSetWithLateMatch.Add(0);
        _largerHashSetWithLateMatch = largerHashSetWithLateMatch;

        _largerImmutableHashSetWithLateMatch = Enumerable.Range(100, 9_999).Concat([0]).ToImmutableHashSet();
        
        _listWithImmediateMatch = new List<int> { 0 };
        _largerDisjointList = new List<int>(Enumerable.Range(100, 10_000));
    }

    [Benchmark]
    public bool LargerFrozenSet_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointFrozenSet);
    }

    [Benchmark]
    public bool LargerFrozenSet_MatchLastInOther()
    {
        return _frozenSet.Overlaps(_largerFrozenSetWithLateMatch);
    }

    [Benchmark]
    public bool LargerHashSet_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointHashSet);
    }

    [Benchmark]
    public bool LargerHashSet_MatchLastInOther()
    {
        return _frozenSet.Overlaps(_largerHashSetWithLateMatch);
    }

    [Benchmark]
    public bool LargerImmutableHashSet_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointImmutableHashSet);
    }

    [Benchmark]
    public bool LargerImmutableHashSet_MatchLastInOther()
    {
        return _frozenSet.Overlaps(_largerImmutableHashSetWithLateMatch);
    }

    [Benchmark]
    public bool List_MatchFirst()
    {
        return _frozenSet.Overlaps(_listWithImmediateMatch);
    }

    [Benchmark]
    public bool LargerList_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointList);
    }
}

[Advitalitum]

@EgorBot -linux_amd -osx_arm64

using System.Collections.Frozen;
using System.Collections.Generic;
using System.Linq;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;

BenchmarkSwitcher.FromAssembly(typeof(Benchmarks).Assembly).Run(args);

[MemoryDiagnoser]
public class Benchmarks
{
    private FrozenSet<int> _frozenSet = null!;
    
    private IEnumerable<int> _largerDisjointFrozenSet = null!;
    private IEnumerable<int> _largerFrozenSetWithLateMatch = null!;
    
    private IEnumerable<int> _largerDisjointHashSet = null!;
    private IEnumerable<int> _largerHashSetWithLateMatch = null!;
    
    private IEnumerable<int> _listWithImmediateMatch = null!;
    private IEnumerable<int> _largerDisjointList = null!;

    [GlobalSetup]
    public void Setup()
    {
        _frozenSet = Enumerable.Range(0, 100).ToFrozenSet();
        
        _largerDisjointFrozenSet = Enumerable.Range(100, 10_000).ToFrozenSet();
        _largerDisjointHashSet = new HashSet<int>(Enumerable.Range(100, 10_000));

        _largerFrozenSetWithLateMatch = Enumerable.Range(100, 9_999).Concat([99]).ToFrozenSet();

        HashSet<int> largerHashSetWithLateMatch = new(Enumerable.Range(100, 9_999));
        largerHashSetWithLateMatch.Add(99);
        _largerHashSetWithLateMatch = largerHashSetWithLateMatch;

        _listWithImmediateMatch = new List<int> { 0 };
        _largerDisjointList = new List<int>(Enumerable.Range(100, 10_000));
    }

    [Benchmark]
    public bool LargerFrozenSet_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointFrozenSet);
    }

    [Benchmark]
    public bool LargerFrozenSet_MatchLastInOther()
    {
        return _frozenSet.Overlaps(_largerFrozenSetWithLateMatch);
    }

    [Benchmark]
    public bool LargerHashSet_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointHashSet);
    }

    [Benchmark]
    public bool LargerHashSet_MatchLastInOther()
    {
        return _frozenSet.Overlaps(_largerHashSetWithLateMatch);
    }

    [Benchmark]
    public bool List_MatchFirst()
    {
        return _frozenSet.Overlaps(_listWithImmediateMatch);
    }

    [Benchmark]
    public bool LargerList_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointList);
    }
}

[Advitalitum]

Hmm. It's complicated to generate case for _largerFrozenSetWithLateMatch...

[Advitalitum]

@EgorBot -linux_amd -osx_arm64

using System.Collections.Frozen;
using System.Collections.Generic;
using System.Linq;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;

BenchmarkSwitcher.FromAssembly(typeof(Benchmarks).Assembly).Run(args);

[MemoryDiagnoser]
public class Benchmarks
{
    private FrozenSet<int> _frozenSet = null!;
    
    private IEnumerable<int> _largerDisjointFrozenSet = null!;
    private IEnumerable<int> _largerFrozenSetWithLateMatch = null!;
    
    private IEnumerable<int> _largerDisjointHashSet = null!;
    private IEnumerable<int> _largerHashSetWithLateMatch = null!;
    
    private IEnumerable<int> _listWithImmediateMatch = null!;
    private IEnumerable<int> _largerDisjointList = null!;

    [GlobalSetup]
    public void Setup()
    {
        _frozenSet = Enumerable.Range(9_999, 100).ToFrozenSet();
        
        _largerDisjointFrozenSet = Enumerable.Range(0, 9_999).ToFrozenSet();
        _largerDisjointHashSet = new HashSet<int>(Enumerable.Range(0, 9_999));

        _largerFrozenSetWithLateMatch = Enumerable.Range(0, 9_999).Concat([9_999]).ToFrozenSet();

        HashSet<int> largerHashSetWithLateMatch = new(Enumerable.Range(0, 9_999));
        largerHashSetWithLateMatch.Add(9_999);
        _largerHashSetWithLateMatch = largerHashSetWithLateMatch;

        _listWithImmediateMatch = new List<int> { 9_999 };
        _largerDisjointList = new List<int>(Enumerable.Range(0, 9_999));
    }

    [Benchmark]
    public bool LargerFrozenSet_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointFrozenSet);
    }

    [Benchmark]
    public bool LargerFrozenSet_MatchLastInOther()
    {
        return _frozenSet.Overlaps(_largerFrozenSetWithLateMatch);
    }

    [Benchmark]
    public bool LargerHashSet_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointHashSet);
    }

    [Benchmark]
    public bool LargerHashSet_MatchLastInOther()
    {
        return _frozenSet.Overlaps(_largerHashSetWithLateMatch);
    }

    [Benchmark]
    public bool List_MatchFirst()
    {
        return _frozenSet.Overlaps(_listWithImmediateMatch);
    }

    [Benchmark]
    public bool LargerList_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointList);
    }
}

[Advitalitum]

@dotnet/area-system-collections, please review.

[Advitalitum]

Trying to remove allocations here #135189

[Advitalitum]

@EgorBot -macos_arm

using System.Collections.Frozen;
using System.Collections.Generic;
using System.Linq;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;

BenchmarkSwitcher.FromAssembly(typeof(Benchmarks).Assembly).Run(args);

[MemoryDiagnoser]
public class Benchmarks
{
    private FrozenSet<int> _frozenSet = null!;
    
    private IEnumerable<int> _largerDisjointFrozenSet = null!;
    private IEnumerable<int> _largerFrozenSetWithLateMatch = null!;
    
    private IEnumerable<int> _largerDisjointHashSet = null!;
    private IEnumerable<int> _largerHashSetWithLateMatch = null!;
    
    private IEnumerable<int> _listWithImmediateMatch = null!;
    private IEnumerable<int> _largerDisjointList = null!;

    [GlobalSetup]
    public void Setup()
    {
        _frozenSet = Enumerable.Range(9_999, 100).ToFrozenSet();
        
        _largerDisjointFrozenSet = Enumerable.Range(0, 9_999).ToFrozenSet();
        _largerDisjointHashSet = new HashSet<int>(Enumerable.Range(0, 9_999));

        _largerFrozenSetWithLateMatch = Enumerable.Range(0, 9_999).Concat([9_999]).ToFrozenSet();

        HashSet<int> largerHashSetWithLateMatch = new(Enumerable.Range(0, 9_999));
        largerHashSetWithLateMatch.Add(9_999);
        _largerHashSetWithLateMatch = largerHashSetWithLateMatch;

        _listWithImmediateMatch = new List<int> { 9_999 };
        _largerDisjointList = new List<int>(Enumerable.Range(0, 9_999));
    }

    [Benchmark]
    public bool LargerFrozenSet_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointFrozenSet);
    }

    [Benchmark]
    public bool LargerFrozenSet_MatchLastInOther()
    {
        return _frozenSet.Overlaps(_largerFrozenSetWithLateMatch);
    }

    [Benchmark]
    public bool LargerHashSet_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointHashSet);
    }

    [Benchmark]
    public bool LargerHashSet_MatchLastInOther()
    {
        return _frozenSet.Overlaps(_largerHashSetWithLateMatch);
    }

    [Benchmark]
    public bool List_MatchFirst()
    {
        return _frozenSet.Overlaps(_listWithImmediateMatch);
    }

    [Benchmark]
    public bool LargerList_Disjoint()
    {
        return _frozenSet.Overlaps(_largerDisjointList);
    }
}