diff --git a/src/Microsoft.Data.SqlClient.sln b/src/Microsoft.Data.SqlClient.sln index 178d71db10..585e58a4f1 100644 --- a/src/Microsoft.Data.SqlClient.sln +++ b/src/Microsoft.Data.SqlClient.sln @@ -304,6 +304,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "steps", "steps", "{AD738BD4 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Data.SqlClient.UnitTests", "Microsoft.Data.SqlClient\tests\UnitTests\Microsoft.Data.SqlClient.UnitTests.csproj", "{4461063D-2F2B-274C-7E6F-F235119D258E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Common", "Microsoft.Data.SqlClient\tests\Common\Common.csproj", "{67128EC0-30F5-6A98-448B-55F88A1DE707}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -582,6 +584,18 @@ Global {4461063D-2F2B-274C-7E6F-F235119D258E}.Release|x64.Build.0 = Release|x64 {4461063D-2F2B-274C-7E6F-F235119D258E}.Release|x86.ActiveCfg = Release|x86 {4461063D-2F2B-274C-7E6F-F235119D258E}.Release|x86.Build.0 = Release|x86 + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Debug|Any CPU.Build.0 = Debug|Any CPU + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Debug|x64.ActiveCfg = Debug|x64 + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Debug|x64.Build.0 = Debug|x64 + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Debug|x86.ActiveCfg = Debug|x86 + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Debug|x86.Build.0 = Debug|x86 + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Release|Any CPU.ActiveCfg = Release|Any CPU + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Release|Any CPU.Build.0 = Release|Any CPU + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Release|x64.ActiveCfg = Release|x64 + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Release|x64.Build.0 = Release|x64 + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Release|x86.ActiveCfg = Release|x86 + {67128EC0-30F5-6A98-448B-55F88A1DE707}.Release|x86.Build.0 = Release|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -632,6 +646,7 @@ Global {09352F1D-878F-4F55-8AA2-6E47F1AD37D5} = {4CAE9195-4F1A-4D48-854C-1C9FBC512C66} {AD738BD4-6A02-4B88-8F93-FBBBA49A74C8} = {4CAE9195-4F1A-4D48-854C-1C9FBC512C66} {4461063D-2F2B-274C-7E6F-F235119D258E} = {0CC4817A-12F3-4357-912C-09315FAAD008} + {67128EC0-30F5-6A98-448B-55F88A1DE707} = {0CC4817A-12F3-4357-912C-09315FAAD008} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {01D48116-37A2-4D33-B9EC-94793C702431} diff --git a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs index 6cb154f764..5320d1e51f 100644 --- a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -2813,7 +2813,7 @@ private TdsOperationStatus TryProcessEnvChange(int tokenLength, TdsParserStateOb // UTF8 collation if (env._newCollation.IsUTF8) { - _defaultEncoding = Encoding.UTF8; + _defaultEncoding = s_utf8EncodingWithoutBom; } else { @@ -4324,7 +4324,7 @@ internal TdsOperationStatus TryProcessReturnValue(int length, if (rec.collation.IsUTF8) { // UTF8 collation - rec.encoding = Encoding.UTF8; + rec.encoding = s_utf8EncodingWithoutBom; } else { @@ -5181,7 +5181,7 @@ private TdsOperationStatus TryProcessTypeInfo(TdsParserStateObject stateObj, Sql if (col.collation.IsUTF8) { // UTF8 collation - col.encoding = Encoding.UTF8; + col.encoding = s_utf8EncodingWithoutBom; } else { @@ -5986,7 +5986,7 @@ private TdsOperationStatus TryReadSqlStringValue(SqlBuffer value, byte type, int break; case TdsEnums.SQLJSON: - encoding = Encoding.UTF8; + encoding = s_utf8EncodingWithoutBom; string jsonStringValue; result = stateObj.TryReadStringWithEncoding(length, encoding, isPlp, out jsonStringValue); if (result != TdsOperationStatus.Done) @@ -11052,7 +11052,7 @@ internal Task WriteBulkCopyValue(object value, SqlMetaDataPriv metadata, TdsPars // Replace encoding if it is UTF8 if (metadata.collation.IsUTF8) { - _defaultEncoding = Encoding.UTF8; + _defaultEncoding = s_utf8EncodingWithoutBom; } _defaultCollation = metadata.collation; diff --git a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs index c2049bb996..46f5748dc9 100644 --- a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -2865,7 +2865,7 @@ private TdsOperationStatus TryProcessEnvChange(int tokenLength, TdsParserStateOb // UTF8 collation if (env._newCollation.IsUTF8) { - _defaultEncoding = Encoding.UTF8; + _defaultEncoding = s_utf8EncodingWithoutBom; } else { @@ -4376,7 +4376,7 @@ internal TdsOperationStatus TryProcessReturnValue(int length, if (rec.collation.IsUTF8) { // UTF8 collation - rec.encoding = Encoding.UTF8; + rec.encoding = s_utf8EncodingWithoutBom; } else { @@ -5297,7 +5297,7 @@ private TdsOperationStatus TryProcessTypeInfo(TdsParserStateObject stateObj, Sql if (col.collation.IsUTF8) { // UTF8 collation - col.encoding = Encoding.UTF8; + col.encoding = s_utf8EncodingWithoutBom; } else { @@ -6183,7 +6183,7 @@ private TdsOperationStatus TryReadSqlStringValue(SqlBuffer value, byte type, int break; case TdsEnums.SQLJSON: - encoding = Encoding.UTF8; + encoding = s_utf8EncodingWithoutBom; string jsonStringValue; result = stateObj.TryReadStringWithEncoding(length, encoding, isPlp, out jsonStringValue); if (result != TdsOperationStatus.Done) @@ -11240,7 +11240,7 @@ internal Task WriteBulkCopyValue(object value, SqlMetaDataPriv metadata, TdsPars // Replace encoding if it is UTF8 if (metadata.collation.IsUTF8) { - _defaultEncoding = Encoding.UTF8; + _defaultEncoding = s_utf8EncodingWithoutBom; } _defaultCollation = metadata.collation; diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs index ad8226c7fe..7e0fd18503 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -1,14 +1,18 @@ using System; using System.Buffers; using System.Diagnostics; +using System.Text; using Microsoft.Data.SqlClient.Utilities; #nullable enable namespace Microsoft.Data.SqlClient { + internal partial class TdsParser { + private static readonly Encoding s_utf8EncodingWithoutBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + internal void ProcessSSPI(int receivedLength) { Debug.Assert(_authenticationProvider is not null); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj index c724d5dbea..2c2a6dcc8f 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj @@ -148,6 +148,7 @@ + diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/TestBulkCopyWithUTF8.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/TestBulkCopyWithUTF8.cs new file mode 100644 index 0000000000..dc3779b4dc --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlBulkCopyTest/TestBulkCopyWithUTF8.cs @@ -0,0 +1,200 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests +{ + /// + /// Validates SqlBulkCopy functionality when working with UTF-8 encoded data. + /// Ensures that data copied from a UTF-8 source table to a destination table retains its encoding and content integrity. + /// + public sealed class TestBulkCopyWithUtf8 : IDisposable + { + private static string s_sourceTable = DataTestUtility.GetUniqueName("SourceTableForUTF8Data"); + private static string s_destinationTable = DataTestUtility.GetUniqueName("DestinationTableForUTF8Data"); + private static string s_testValue = "test"; + private static byte[] s_testValueInUtf8Bytes = new byte[] { 0x74, 0x65, 0x73, 0x74 }; + private static readonly string s_insertQuery = $"INSERT INTO {s_sourceTable} VALUES('{s_testValue}')"; + + /// + /// Constructor: Initializes and populates source and destination tables required for the tests. + /// + public TestBulkCopyWithUtf8() + { + using SqlConnection sourceConnection = new SqlConnection(GetConnectionString(true)); + sourceConnection.Open(); + SetupTables(sourceConnection, s_sourceTable, s_destinationTable, s_insertQuery); + } + + /// + /// Cleanup method to drop tables after test completion. + /// + public void Dispose() + { + using SqlConnection connection = new SqlConnection(GetConnectionString(true)); + connection.Open(); + DataTestUtility.DropTable(connection, s_sourceTable); + DataTestUtility.DropTable(connection, s_destinationTable); + connection.Close(); + } + + /// + /// Builds a connection string with or without Multiple Active Result Sets (MARS) property. + /// + private string GetConnectionString(bool enableMars) + { + return new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString) + { + MultipleActiveResultSets = enableMars + }.ConnectionString; + } + + /// + /// Creates source and destination tables with a varchar(max) column with a collation setting + /// that stores the data in UTF8 encoding and inserts the data in the source table. + /// + private void SetupTables(SqlConnection connection, string sourceTable, string destinationTable, string insertQuery) + { + string columnDefinition = "(str_col varchar(max) COLLATE Latin1_General_100_CS_AS_KS_WS_SC_UTF8)"; + DataTestUtility.CreateTable(connection, sourceTable, columnDefinition); + DataTestUtility.CreateTable(connection, destinationTable, columnDefinition); + using SqlCommand insertCommand = connection.CreateCommand(); + insertCommand.CommandText = insertQuery; + Helpers.TryExecute(insertCommand, insertQuery); + } + + /// + /// Synchronous test case: Validates that data copied using SqlBulkCopy matches UTF-8 byte sequence for test value. + /// Tested with MARS enabled and disabled, and with streaming enabled and disabled. + /// + [ConditionalTheory(typeof(DataTestUtility), + nameof(DataTestUtility.AreConnStringsSetup), + nameof(DataTestUtility.IsNotAzureServer), + nameof(DataTestUtility.IsNotAzureSynapse))] + [InlineData(true, true)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(false, false)] + public void BulkCopy_Utf8Data_ShouldMatchSource(bool isMarsEnabled, bool enableStreaming) + { + // Setup connections for source and destination tables + string connectionString = GetConnectionString(isMarsEnabled); + using SqlConnection sourceConnection = new SqlConnection(connectionString); + sourceConnection.Open(); + using SqlConnection destinationConnection = new SqlConnection(connectionString); + destinationConnection.Open(); + + // Read data from source table + using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection); + using SqlDataReader reader = sourceDataCommand.ExecuteReader(CommandBehavior.SequentialAccess); + + // Verify that the destination table is empty before bulk copy + using SqlCommand countCommand = new SqlCommand($"SELECT COUNT(*) FROM {s_destinationTable}", destinationConnection); + Assert.Equal(0, Convert.ToInt16(countCommand.ExecuteScalar())); + + // Initialize bulk copy configuration + using SqlBulkCopy bulkCopy = new SqlBulkCopy(destinationConnection) + { + EnableStreaming = enableStreaming, + DestinationTableName = s_destinationTable + }; + + try + { + // Perform bulk copy from source to destination table + bulkCopy.WriteToServer(reader); + } + catch (Exception ex) + { + // If bulk copy fails, fail the test with the exception message + Assert.Fail($"Bulk copy failed: {ex.Message}"); + } + + // Verify that the 1 row from the source table has been copied into our destination table. + Assert.Equal(1, Convert.ToInt16(countCommand.ExecuteScalar())); + + // Read the data from destination table as varbinary to verify the UTF-8 byte sequence + using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection); + using SqlDataReader verifyReader = verifyCommand.ExecuteReader(CommandBehavior.SequentialAccess); + + // Verify that we have data in the destination table + Assert.True(verifyReader.Read(), "No data found in destination table after bulk copy."); + + // Read the value of the column as SqlBinary. + byte[] actualBytes = verifyReader.GetSqlBinary(0).Value; + + // Verify that the byte array matches the expected UTF-8 byte sequence + Assert.Equal(s_testValueInUtf8Bytes.Length, actualBytes.Length); + Assert.Equal(s_testValueInUtf8Bytes, actualBytes); + } + + /// + /// Asynchronous version of the testcase BulkCopy_Utf8Data_ShouldMatchSource + /// + [ConditionalTheory(typeof(DataTestUtility), + nameof(DataTestUtility.AreConnStringsSetup), + nameof(DataTestUtility.IsNotAzureServer), + nameof(DataTestUtility.IsNotAzureSynapse))] + [InlineData(true, true)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(false, false)] + public async Task BulkCopy_Utf8Data_ShouldMatchSource_Async(bool isMarsEnabled, bool enableStreaming) + { + // Setup connections for source and destination tables + string connectionString = GetConnectionString(isMarsEnabled); + using SqlConnection sourceConnection = new SqlConnection(connectionString); + await sourceConnection.OpenAsync(); + using SqlConnection destinationConnection = new SqlConnection(connectionString); + await destinationConnection.OpenAsync(); + + // Read data from source table + using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection); + using SqlDataReader reader = await sourceDataCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); + + // Verify that the destination table is empty before bulk copy + using SqlCommand countCommand = new SqlCommand($"SELECT COUNT(*) FROM {s_destinationTable}", destinationConnection); + Assert.Equal(0, Convert.ToInt16(await countCommand.ExecuteScalarAsync())); + + // Initialize bulk copy configuration + using SqlBulkCopy bulkCopy = new SqlBulkCopy(destinationConnection) + { + EnableStreaming = enableStreaming, + DestinationTableName = s_destinationTable + }; + + try + { + // Perform bulk copy from source to destination table + await bulkCopy.WriteToServerAsync(reader); + } + catch (Exception ex) + { + // If bulk copy fails, fail the test with the exception message + Assert.Fail($"Bulk copy failed: {ex.Message}"); + } + + // Verify that the 1 row from the source table has been copied into our destination table. + Assert.Equal(1, Convert.ToInt16(await countCommand.ExecuteScalarAsync())); + + // Read the data from destination table as varbinary to verify the UTF-8 byte sequence + using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection); + using SqlDataReader verifyReader = await verifyCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); + + // Verify that we have data in the destination table + Assert.True(await verifyReader.ReadAsync(), "No data found in destination table after bulk copy."); + + // Read the value of the column as SqlBinary. + byte[] actualBytes = verifyReader.GetSqlBinary(0).Value; + + // Verify that the byte array matches the expected UTF-8 byte sequence + Assert.Equal(s_testValueInUtf8Bytes.Length, actualBytes.Length); + Assert.Equal(s_testValueInUtf8Bytes, actualBytes); + } + } +}