Skip to content

Use compile time constants over method for all MMShuffle operations. #2365

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JimBobSquarePants merged 4 commits into from
Feb 22, 2023
Merged

Use compile time constants over method for all MMShuffle operations. #2365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9e7cf16
Use MMShuffle consts everywhere and refactor explicit shuffle compone…
JimBobSquarePants Feb 21, 2023
b9e4810
Refactor DefaultShuffleX to use constants.
JimBobSquarePants Feb 21, 2023
eda0869
Update benchmarks and add more tests
JimBobSquarePants Feb 21, 2023
51eb978
Merge branch 'main' into js/2121
JimBobSquarePants Feb 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 29 additions & 47 deletions src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static SixLabors.ImageSharp.SimdUtils;

// The JIT can detect and optimize rotation idioms ROTL (Rotate Left)
// and ROTR (Rotate Right) emitting efficient CPU instructions:
Expand All @@ -18,9 +19,12 @@ namespace SixLabors.ImageSharp;
internal interface IComponentShuffle
{
/// <summary>
/// Gets the shuffle control.
/// Shuffles then slices 8-bit integers within 128-bit lanes in <paramref name="source"/>
/// using the control and store the results in <paramref name="dest"/>.
/// </summary>
byte Control { get; }
/// <param name="source">The source span of bytes.</param>
/// <param name="dest">The destination span of bytes.</param>
void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest);

/// <summary>
/// Shuffle 8-bit integers within 128-bit lanes in <paramref name="source"/>
Expand All @@ -42,37 +46,25 @@ internal interface IShuffle4 : IComponentShuffle

internal readonly struct DefaultShuffle4 : IShuffle4
{
private readonly byte p3;
private readonly byte p2;
private readonly byte p1;
private readonly byte p0;

public DefaultShuffle4(byte p3, byte p2, byte p1, byte p0)
public DefaultShuffle4(byte control)
{
DebugGuard.MustBeBetweenOrEqualTo<byte>(p3, 0, 3, nameof(p3));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p2, 0, 3, nameof(p2));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p1, 0, 3, nameof(p1));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p0, 0, 3, nameof(p0));

this.p3 = p3;
this.p2 = p2;
this.p1 = p1;
this.p0 = p0;
this.Control = SimdUtils.Shuffle.MmShuffle(p3, p2, p1, p0);
DebugGuard.MustBeBetweenOrEqualTo<byte>(control, 0, 3, nameof(control));
this.Control = control;
}

public byte Control { get; }

[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, this.Control);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
{
ref byte sBase = ref MemoryMarshal.GetReference(source);
ref byte dBase = ref MemoryMarshal.GetReference(dest);

int p3 = this.p3;
int p2 = this.p2;
int p1 = this.p1;
int p0 = this.p0;
Shuffle.InverseMMShuffle(this.Control, out int p3, out int p2, out int p1, out int p0);

for (int i = 0; i < source.Length; i += 4)
{
Expand All @@ -86,11 +78,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

internal readonly struct WXYZShuffle4 : IShuffle4
{
public byte Control
{
[MethodImpl(InliningOptions.ShortMethod)]
get => SimdUtils.Shuffle.MmShuffle(2, 1, 0, 3);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle2103);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
Expand All @@ -112,11 +102,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

internal readonly struct WZYXShuffle4 : IShuffle4
{
public byte Control
{
[MethodImpl(InliningOptions.ShortMethod)]
get => SimdUtils.Shuffle.MmShuffle(0, 1, 2, 3);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle0123);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
Expand All @@ -138,11 +126,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

internal readonly struct YZWXShuffle4 : IShuffle4
{
public byte Control
{
[MethodImpl(InliningOptions.ShortMethod)]
get => SimdUtils.Shuffle.MmShuffle(0, 3, 2, 1);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle0321);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
Expand All @@ -164,11 +150,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

internal readonly struct ZYXWShuffle4 : IShuffle4
{
public byte Control
{
[MethodImpl(InliningOptions.ShortMethod)]
get => SimdUtils.Shuffle.MmShuffle(3, 0, 1, 2);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle3012);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
Expand Down Expand Up @@ -197,11 +181,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

internal readonly struct XWZYShuffle4 : IShuffle4
{
public byte Control
{
[MethodImpl(InliningOptions.ShortMethod)]
get => SimdUtils.Shuffle.MmShuffle(1, 2, 3, 0);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle1230);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
Expand Down
39 changes: 13 additions & 26 deletions src/ImageSharp/Common/Helpers/Shuffle/IPad3Shuffle4.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static SixLabors.ImageSharp.SimdUtils;

namespace SixLabors.ImageSharp;

Expand All @@ -13,45 +14,33 @@ internal interface IPad3Shuffle4 : IComponentShuffle

internal readonly struct DefaultPad3Shuffle4 : IPad3Shuffle4
{
private readonly byte p3;
private readonly byte p2;
private readonly byte p1;
private readonly byte p0;

public DefaultPad3Shuffle4(byte p3, byte p2, byte p1, byte p0)
public DefaultPad3Shuffle4(byte control)
{
DebugGuard.MustBeBetweenOrEqualTo<byte>(p3, 0, 3, nameof(p3));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p2, 0, 3, nameof(p2));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p1, 0, 3, nameof(p1));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p0, 0, 3, nameof(p0));

this.p3 = p3;
this.p2 = p2;
this.p1 = p1;
this.p0 = p0;
this.Control = SimdUtils.Shuffle.MmShuffle(p3, p2, p1, p0);
DebugGuard.MustBeBetweenOrEqualTo<byte>(control, 0, 3, nameof(control));
this.Control = control;
}

public byte Control { get; }

[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref dest, this.Control);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
{
ref byte sBase = ref MemoryMarshal.GetReference(source);
ref byte dBase = ref MemoryMarshal.GetReference(dest);

int p3 = this.p3;
int p2 = this.p2;
int p1 = this.p1;
int p0 = this.p0;
Shuffle.InverseMMShuffle(this.Control, out int p3, out int p2, out int p1, out int p0);

Span<byte> temp = stackalloc byte[4];
ref byte t = ref MemoryMarshal.GetReference(temp);
ref uint tu = ref Unsafe.As<byte, uint>(ref t);

for (int i = 0, j = 0; i < source.Length; i += 3, j += 4)
{
ref var s = ref Unsafe.Add(ref sBase, i);
ref byte s = ref Unsafe.Add(ref sBase, i);
tu = Unsafe.As<byte, uint>(ref s) | 0xFF000000;

Unsafe.Add(ref dBase, j) = Unsafe.Add(ref t, p0);
Expand All @@ -64,11 +53,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

internal readonly struct XYZWPad3Shuffle4 : IPad3Shuffle4
{
public byte Control
{
[MethodImpl(InliningOptions.ShortMethod)]
get => SimdUtils.Shuffle.MmShuffle(3, 2, 1, 0);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Pad3Shuffle4Reduce(ref source, ref dest, Shuffle.MMShuffle3210);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
Expand Down
25 changes: 9 additions & 16 deletions src/ImageSharp/Common/Helpers/Shuffle/IShuffle3.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static SixLabors.ImageSharp.SimdUtils;

namespace SixLabors.ImageSharp;

Expand All @@ -13,33 +14,25 @@ internal interface IShuffle3 : IComponentShuffle

internal readonly struct DefaultShuffle3 : IShuffle3
{
private readonly byte p2;
private readonly byte p1;
private readonly byte p0;

public DefaultShuffle3(byte p2, byte p1, byte p0)
public DefaultShuffle3(byte control)
{
DebugGuard.MustBeBetweenOrEqualTo<byte>(p2, 0, 2, nameof(p2));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p1, 0, 2, nameof(p1));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p0, 0, 2, nameof(p0));

this.p2 = p2;
this.p1 = p1;
this.p0 = p0;
this.Control = SimdUtils.Shuffle.MmShuffle(3, p2, p1, p0);
DebugGuard.MustBeBetweenOrEqualTo<byte>(control, 0, 3, nameof(control));
this.Control = control;
}

public byte Control { get; }

[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Shuffle3Reduce(ref source, ref dest, this.Control);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
{
ref byte sBase = ref MemoryMarshal.GetReference(source);
ref byte dBase = ref MemoryMarshal.GetReference(dest);

int p2 = this.p2;
int p1 = this.p1;
int p0 = this.p0;
Shuffle.InverseMMShuffle(this.Control, out _, out int p2, out int p1, out int p0);

for (int i = 0; i < source.Length; i += 3)
{
Expand Down
34 changes: 12 additions & 22 deletions src/ImageSharp/Common/Helpers/Shuffle/IShuffle4Slice3.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static SixLabors.ImageSharp.SimdUtils;

namespace SixLabors.ImageSharp;

Expand All @@ -13,34 +14,25 @@ internal interface IShuffle4Slice3 : IComponentShuffle

internal readonly struct DefaultShuffle4Slice3 : IShuffle4Slice3
{
private readonly byte p2;
private readonly byte p1;
private readonly byte p0;

public DefaultShuffle4Slice3(byte p3, byte p2, byte p1, byte p0)
public DefaultShuffle4Slice3(byte control)
{
DebugGuard.MustBeBetweenOrEqualTo<byte>(p3, 0, 3, nameof(p3));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p2, 0, 3, nameof(p2));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p1, 0, 3, nameof(p1));
DebugGuard.MustBeBetweenOrEqualTo<byte>(p0, 0, 3, nameof(p0));

this.p2 = p2;
this.p1 = p1;
this.p0 = p0;
this.Control = SimdUtils.Shuffle.MmShuffle(p3, p2, p1, p0);
DebugGuard.MustBeBetweenOrEqualTo<byte>(control, 0, 3, nameof(control));
this.Control = control;
}

public byte Control { get; }

[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref dest, this.Control);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
{
ref byte sBase = ref MemoryMarshal.GetReference(source);
ref byte dBase = ref MemoryMarshal.GetReference(dest);

int p2 = this.p2;
int p1 = this.p1;
int p0 = this.p0;
Shuffle.InverseMMShuffle(this.Control, out _, out int p2, out int p1, out int p0);

for (int i = 0, j = 0; i < dest.Length; i += 3, j += 4)
{
Expand All @@ -53,11 +45,9 @@ public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)

internal readonly struct XYZWShuffle4Slice3 : IShuffle4Slice3
{
public byte Control
{
[MethodImpl(InliningOptions.ShortMethod)]
get => SimdUtils.Shuffle.MmShuffle(3, 2, 1, 0);
}
[MethodImpl(InliningOptions.ShortMethod)]
public void ShuffleReduce(ref ReadOnlySpan<byte> source, ref Span<byte> dest)
=> HwIntrinsics.Shuffle4Slice3Reduce(ref source, ref dest, Shuffle.MMShuffle3210);

[MethodImpl(InliningOptions.ShortMethod)]
public void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest)
Expand Down
10 changes: 5 additions & 5 deletions src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ private static void Shuffle4(
// shuffle controls to add to the library.
// We can add static ROS instances if need be in the future.
Span<byte> bytes = stackalloc byte[Vector256<byte>.Count];
Shuffle.MmShuffleSpan(ref bytes, control);
Shuffle.MMShuffleSpan(ref bytes, control);
Vector256<byte> vshuffle = Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(bytes));

ref Vector256<byte> sourceBase =
Expand Down Expand Up @@ -333,7 +333,7 @@ private static void Shuffle4(
{
// Ssse3
Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Shuffle.MmShuffleSpan(ref bytes, control);
Shuffle.MMShuffleSpan(ref bytes, control);
Vector128<byte> vshuffle = Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(bytes));

ref Vector128<byte> sourceBase =
Expand Down Expand Up @@ -382,7 +382,7 @@ private static void Shuffle3(
Vector128<byte> vmaske = Ssse3.AlignRight(vmasko, vmasko, 12);

Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Shuffle.MmShuffleSpan(ref bytes, control);
Shuffle.MMShuffleSpan(ref bytes, control);
Vector128<byte> vshuffle = Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(bytes));

ref Vector128<byte> sourceBase =
Expand Down Expand Up @@ -445,7 +445,7 @@ private static void Pad3Shuffle4(
Vector128<byte> vfill = Vector128.Create(0xff000000ff000000ul).AsByte();

Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Shuffle.MmShuffleSpan(ref bytes, control);
Shuffle.MMShuffleSpan(ref bytes, control);
Vector128<byte> vshuffle = Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(bytes));

ref Vector128<byte> sourceBase =
Expand Down Expand Up @@ -489,7 +489,7 @@ private static void Shuffle4Slice3(
Vector128<byte> vmaske = Ssse3.AlignRight(vmasko, vmasko, 12);

Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Shuffle.MmShuffleSpan(ref bytes, control);
Shuffle.MMShuffleSpan(ref bytes, control);
Vector128<byte> vshuffle = Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(bytes));

ref Vector128<byte> sourceBase =
Expand Down
Loading