Skip to content

Handle Unicode scalars in UTF-8 parsing and formatting #132473

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tannergooding merged 3 commits into from
Aug 28, 2026
Merged

Handle Unicode scalars in UTF-8 parsing and formatting #132473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e9cbef9
Handle Unicode scalars in UTF-8 parsing and formatting
tannergooding Aug 18, 2026
fb88775
Make special-value hyphen tests culture-independent
tannergooding Aug 18, 2026
e3ff0c6
Merge dotnet/main into tannergooding-fix-unicode-parsing-formatting
tannergooding Aug 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 47 additions & 7 deletions src/libraries/Common/src/System/Number.Formatting.Common.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -842,7 +842,14 @@ internal static void NumberToStringFormat<TChar>(ref ValueListBuilder<TChar> vlb
break;
}
src++;
AppendUnknownChar(ref vlb, quoted);
if (char.IsHighSurrogate(quoted) && src < format.Length && char.IsLowSurrogate(format[src]))
{
AppendSurrogatePair(ref vlb, quoted, format[src++]);
}
else
{
AppendUnknownChar(ref vlb, quoted);
}
}

if (src < format.Length && format[src] != 0)
Expand All @@ -854,7 +861,15 @@ internal static void NumberToStringFormat<TChar>(ref ValueListBuilder<TChar> vlb
case '\\':
if (src < format.Length && format[src] != 0)
{
AppendUnknownChar(ref vlb, format[src++]);
char literal = format[src++];
if (char.IsHighSurrogate(literal) && src < format.Length && char.IsLowSurrogate(format[src]))
{
AppendSurrogatePair(ref vlb, literal, format[src++]);
}
else
{
AppendUnknownChar(ref vlb, literal);
}
}
break;

Expand Down Expand Up @@ -905,20 +920,27 @@ internal static void NumberToStringFormat<TChar>(ref ValueListBuilder<TChar> vlb
{
if (format[src] is '+' or '-')
{
AppendUnknownChar(ref vlb, format[src++]);
vlb.Append(TChar.CastFrom(format[src++]));
}

while (src < format.Length && format[src] == '0')
{
AppendUnknownChar(ref vlb, format[src++]);
vlb.Append(TChar.CastFrom(format[src++]));
}
}
}
break;
}

default:
AppendUnknownChar(ref vlb, ch);
if (char.IsHighSurrogate(ch) && src < format.Length && char.IsLowSurrogate(format[src]))
{
AppendSurrogatePair(ref vlb, ch, format[src++]);
}
else
{
AppendUnknownChar(ref vlb, ch);
}
break;
}
}
Expand Down Expand Up @@ -1123,8 +1145,26 @@ private static void AppendUnknownChar<TChar>(ref ValueListBuilder<TChar> vlb, ch
[MethodImpl(MethodImplOptions.NoInlining)]
static void AppendNonAsciiBytes(ref ValueListBuilder<TChar> vlb, char ch)
{
var r = new Rune(ch);
r.EncodeToUtf8(Unsafe.BitCast<Span<TChar>, Span<byte>>(vlb.AppendSpan(r.Utf8SequenceLength)));
Rune rune = Rune.TryCreate(ch, out Rune result) ? result : Rune.ReplacementChar;
rune.EncodeToUtf8(MemoryMarshal.AsBytes(vlb.AppendSpan(rune.Utf8SequenceLength)));
}
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static void AppendSurrogatePair<TChar>(ref ValueListBuilder<TChar> vlb, char highSurrogate, char lowSurrogate) where TChar : unmanaged, IUtfChar<TChar>
{
Debug.Assert(sizeof(TChar) is sizeof(char) or sizeof(byte));
Debug.Assert(char.IsSurrogatePair(highSurrogate, lowSurrogate));

if (sizeof(TChar) == sizeof(char))
{
vlb.Append(TChar.CastFrom(highSurrogate));
vlb.Append(TChar.CastFrom(lowSurrogate));
}
else
{
Rune rune = new(highSurrogate, lowSurrogate);
rune.EncodeToUtf8(MemoryMarshal.AsBytes(vlb.AppendSpan(rune.Utf8SequenceLength)));
}
}

Expand Down
93 changes: 83 additions & 10 deletions src/libraries/Common/src/System/Number.Parsing.Common.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,30 +375,103 @@ private static int MatchNegativeSignChars<TChar>(ReadOnlySpan<TChar> value, int
private static int MatchChars<TChar>(ReadOnlySpan<TChar> source, int index, ReadOnlySpan<TChar> value)
where TChar : unmanaged, IUtfChar<TChar>
{
// An empty pattern never matches, and one longer than the remaining input cannot match, so
// the loop only has to bound itself by the pattern.
if (value.IsEmpty || (value.Length > (source.Length - index)))
if (value.IsEmpty)
{
return -1;
}

if (value.Length > (source.Length - index))
{
if (!TChar.IsUtf8)
{
return -1;
}

ReadOnlySpan<byte> input = Unsafe.BitCast<ReadOnlySpan<TChar>, ReadOnlySpan<byte>>(source.Slice(index));
ReadOnlySpan<byte> utf8Value = Unsafe.BitCast<ReadOnlySpan<TChar>, ReadOnlySpan<byte>>(value);
int matchedLength = MatchUtf8SpaceReplacingChars(input, utf8Value);

return matchedLength >= 0 ? index + matchedLength : -1;
}

for (int i = 0; i < value.Length; i++)
{
uint cp = TChar.CastToUInt32(source[index + i]);
uint val = TChar.CastToUInt32(value[i]);

// We only hurt the failure case
// This fix is for cultures that use NBSP (U+00A0) or narrow NBSP (U+202F) as group/decimal separators
// (e.g., French, Kazakh, Ukrainian). Since a user cannot easily type these characters,
// we accept regular space (U+0020) as equivalent.
// For UTF-16, we also handle the reverse case where the input has NBSP and the format string has space.
if (cp != val && (TChar.IsUtf8 || NormalizeSpaceReplacingChar(cp) != NormalizeSpaceReplacingChar(val)))
if (cp != val)
{
return -1;
if (TChar.IsUtf8)
{
ReadOnlySpan<byte> input = Unsafe.BitCast<ReadOnlySpan<TChar>, ReadOnlySpan<byte>>(source.Slice(index));
ReadOnlySpan<byte> utf8Value = Unsafe.BitCast<ReadOnlySpan<TChar>, ReadOnlySpan<byte>>(value);
int matchedLength = MatchUtf8SpaceReplacingChars(input, utf8Value);

return matchedLength >= 0 ? index + matchedLength : -1;
}

if (NormalizeSpaceReplacingChar(cp) != NormalizeSpaceReplacingChar(val))
{
return -1;
}
}
}

return index + value.Length;
}

private static int MatchUtf8SpaceReplacingChars(ReadOnlySpan<byte> input, ReadOnlySpan<byte> value)
{
int inputIndex = 0;
int valueIndex = 0;

while (valueIndex < value.Length)
{
if (inputIndex >= input.Length)
{
return -1;
}

if (input[inputIndex] == value[valueIndex])
{
inputIndex++;
valueIndex++;
continue;
}

int inputLength = GetUtf8SpaceCharLength(input.Slice(inputIndex));
int valueLength = GetUtf8SpaceCharLength(value.Slice(valueIndex));

if (inputLength == 0 || valueLength == 0)
{
return -1;
}

inputIndex += inputLength;
valueIndex += valueLength;
}

return inputIndex;
}

private static int GetUtf8SpaceCharLength(ReadOnlySpan<byte> value)
{
if (value[0] == ' ')
{
return 1;
}

if (value.StartsWith("\u00A0"u8))
{
return 2;
}

if (value.StartsWith("\u202F"u8))
{
return 3;
}

return 0;
}
}
}
7 changes: 5 additions & 2 deletions src/libraries/System.Private.CoreLib/src/System/Char.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,11 @@ bool ISpanFormattable.TryFormat(Span<char> destination, out int charsWritten, Re
}

/// <inheritdoc cref="IUtf8SpanFormattable.TryFormat" />
bool IUtf8SpanFormattable.TryFormat(Span<byte> utf8Destination, out int bytesWritten, ReadOnlySpan<char> format, IFormatProvider? provider) =>
new Rune(this).TryEncodeToUtf8(utf8Destination, out bytesWritten);
bool IUtf8SpanFormattable.TryFormat(Span<byte> utf8Destination, out int bytesWritten, ReadOnlySpan<char> format, IFormatProvider? provider)
{
Rune rune = Rune.TryCreate(this, out Rune value) ? value : Rune.ReplacementChar;
return rune.TryEncodeToUtf8(utf8Destination, out bytesWritten);
}

string IFormattable.ToString(string? format, IFormatProvider? formatProvider) => ToString(m_value);

Expand Down
91 changes: 77 additions & 14 deletions src/libraries/System.Private.CoreLib/src/System/Globalization/DateTimeFormat.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,9 +289,10 @@ internal static int ParseQuoteString<TChar>(scoped ReadOnlySpan<char> format, in
bool foundQuote = false;
while (pos < formatLen)
{
char ch = format[pos++];
char ch = format[pos];
if (ch == quoteChar)
{
pos++;
foundQuote = true;
break;
}
Expand All @@ -302,9 +303,19 @@ internal static int ParseQuoteString<TChar>(scoped ReadOnlySpan<char> format, in
// Therefore, someone can use a format like "'minute:' mm\"" to display:
// minute: 45"
// because the second double quote is escaped.
pos++;
if (pos < formatLen)
{
result.Append(TChar.CastFrom(format[pos++]));
char escapedChar = format[pos];
if (char.IsHighSurrogate(escapedChar) && pos + 1 < formatLen && char.IsLowSurrogate(format[pos + 1]))
{
pos += AppendChar(ref result, format[pos..]);
}
else
{
AppendChar(ref result, escapedChar);
pos++;
}
}
else
{
Expand All @@ -316,7 +327,15 @@ internal static int ParseQuoteString<TChar>(scoped ReadOnlySpan<char> format, in
}
else
{
AppendChar(ref result, ch);
if (char.IsHighSurrogate(ch) && pos + 1 < formatLen && char.IsLowSurrogate(format[pos + 1]))
{
pos += AppendChar(ref result, format[pos..]);
}
else
{
AppendChar(ref result, ch);
pos++;
}
}
}

Expand Down Expand Up @@ -526,9 +545,9 @@ private static void FormatCustomized<TChar>(
if (tokenLen == 1)
{
string designator = dateTime.Hour < 12 ? dtfi.AMDesignator : dtfi.PMDesignator;
if (designator.Length >= 1)
if (designator.Length != 0)
{
AppendChar(ref result, designator[0]);
AppendChar(ref result, designator);
}
}
else
Expand Down Expand Up @@ -689,9 +708,17 @@ private static void FormatCustomized<TChar>(
// Besides, we will not allow "%%" to appear in the pattern.
if (nextChar >= 0 && nextChar != '%')
{
char nextCharChar = (char)nextChar;
FormatCustomized(dateTime, new ReadOnlySpan<char>(in nextCharChar), dtfi, offset, ref result);
tokenLen = 2;
if (char.IsHighSurrogate((char)nextChar) && i + 2 < format.Length && char.IsLowSurrogate(format[i + 2]))
{
FormatCustomized(dateTime, format.Slice(i + 1, 2), dtfi, offset, ref result);
tokenLen = 3;
}
else
{
char nextCharChar = (char)nextChar;
FormatCustomized(dateTime, new ReadOnlySpan<char>(in nextCharChar), dtfi, offset, ref result);
tokenLen = 2;
}
}
else
{
Expand All @@ -715,8 +742,16 @@ private static void FormatCustomized<TChar>(
nextChar = ParseNextChar(format, i);
if (nextChar >= 0)
{
result.Append(TChar.CastFrom(nextChar));
tokenLen = 2;
char escapedChar = (char)nextChar;
if (char.IsHighSurrogate(escapedChar) && i + 2 < format.Length && char.IsLowSurrogate(format[i + 2]))
{
tokenLen = 1 + AppendChar(ref result, format[(i + 1)..]);
}
else
{
AppendChar(ref result, escapedChar);
tokenLen = 2;
}
}
else
{
Expand All @@ -731,8 +766,15 @@ private static void FormatCustomized<TChar>(
// NOTENOTE : we can remove this rule if we enforce the enforced quote character rule.
// That is, if we ask everyone to use single quote or double quote to insert characters,
// then we can remove this default block.
AppendChar(ref result, ch);
tokenLen = 1;
if (char.IsHighSurrogate(ch) && i + 1 < format.Length && char.IsLowSurrogate(format[i + 1]))
{
tokenLen = AppendChar(ref result, format[i..]);
}
else
{
AppendChar(ref result, ch);
tokenLen = 1;
}
break;
}
i += tokenLen;
Expand All @@ -749,9 +791,30 @@ internal static void AppendChar<TChar>(ref ValueListBuilder<TChar> result, char
else
{
Debug.Assert(typeof(TChar) == typeof(byte));
var r = new Rune(ch);
r.EncodeToUtf8(MemoryMarshal.AsBytes(result.AppendSpan(r.Utf8SequenceLength)));
AppendNonAsciiByte(ref result, ch);
}

[MethodImpl(MethodImplOptions.NoInlining)]
static void AppendNonAsciiByte(ref ValueListBuilder<TChar> result, char ch)
{
Rune rune = Rune.TryCreate(ch, out Rune value) ? value : Rune.ReplacementChar;
rune.EncodeToUtf8(MemoryMarshal.AsBytes(result.AppendSpan(rune.Utf8SequenceLength)));
}
}

internal static int AppendChar<TChar>(ref ValueListBuilder<TChar> result, scoped ReadOnlySpan<char> value) where TChar : unmanaged, IUtfChar<TChar>
{
Debug.Assert(!value.IsEmpty);

char ch = value[0];
if (!char.IsHighSurrogate(ch) || value.Length == 1 || !char.IsLowSurrogate(value[1]))
{
AppendChar(ref result, ch);
return 1;
}

AppendString(ref result, value[..2]);
return 2;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
Expand Down
Loading
Loading