From 9a935eb19dcdae1e95f75d407cef684398e92b7e Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 4 Aug 2026 23:50:06 +0530 Subject: [PATCH 01/24] Expose vector column base type and dimensions A vector column's base type and number of dimensions were already available from the column schema, but only as a numeric scale and a column size which the caller had to decode. They are now surfaced under their own names, so that applications inspecting result set metadata do not have to know that encoding. Also registers the vector type in the DataTypes schema collection, where it was missing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../Microsoft/Data/SqlClient/SqlDbColumn.cs | 77 +++++++++++++ .../SqlClient/SqlMetaDataFactory.DataTypes.cs | 36 ++++++ .../VectorTest/VectorColumnMetadataTests.cs | 105 ++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs index bebe02b575..c2cfeaa92a 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs @@ -5,6 +5,7 @@ using System; using System.Data; using System.Data.Common; +using System.Globalization; namespace Microsoft.Data.SqlClient { @@ -113,5 +114,81 @@ internal int? SqlNumericScale } } + /// + /// The name of the property exposing a vector column's base type. + /// + internal const string VectorBaseTypePropertyName = "VectorBaseType"; + + /// + /// The name of the property exposing a vector column's number of dimensions. + /// + internal const string VectorDimensionsPropertyName = "VectorDimensions"; + + /// + /// Exposes the properties of a vector column which have no corresponding + /// property, in addition to the standard properties. + /// + /// + /// + /// A vector column's base type and number of dimensions are both carried by + /// properties whose meaning for vectors is not + /// self-evident: the base type is reported as the numeric scale, and the number + /// of dimensions has to be derived from the column size. They are surfaced here + /// under their own names so that callers do not have to know that encoding. + /// + /// + /// Both properties are for columns which are not vectors, + /// including a vector column which the server returned as varchar because + /// the connection did not negotiate support for its base type. + /// + /// + public override object this[string property] => + property switch + { + VectorBaseTypePropertyName => VectorBaseType, + VectorDimensionsPropertyName => VectorDimensions, + _ => base[property], + }; + + /// + /// The name of a vector column's base type, such as float32 or + /// float16, or if the column is not a vector. + /// + private string VectorBaseType => + _metadata.type != SqlDbTypeExtensions.Vector + ? null + : (MetaType.SqlVectorElementType)_metadata.scale switch + { + MetaType.SqlVectorElementType.Float32 => "float32", + // An unrecognised base type is reported rather than throwing, because + // reading metadata should not fail on a column the caller may ignore. + _ => _metadata.scale.ToString(CultureInfo.InvariantCulture), + }; + + /// + /// The number of dimensions in a vector column, or if the + /// column is not a vector or has an unrecognised base type. + /// + private int? VectorDimensions + { + get + { + if (_metadata.type != SqlDbTypeExtensions.Vector) + { + return null; + } + + try + { + return MetaType.GetVectorElementCount(_metadata.length, _metadata.scale); + } + catch (NotSupportedException) + { + // The base type is not one this version of the driver knows the + // element size of, so the dimension count cannot be derived. + return null; + } + } + } } } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs index eba61969c3..91352d1677 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs @@ -75,6 +75,7 @@ private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet) AddUniqueIdentifierType(); AddSqlVariantType(); AddRowVersionType(); + AddVectorType(); dataTypesDataTable.EndLoadData(); dataTypesDataTable.AcceptChanges(); @@ -353,6 +354,41 @@ void AddStringOrBinaryType(SqlDbType sqlDbType, int columnSize, bool isLong, dataTypesDataTable.Rows.Add(typeRow); } + void AddVectorType() + { + MetaType metaType = MetaType.GetMetaTypeFromSqlDbType(SqlDbTypeExtensions.Vector, isMultiValued: false); + DataRow typeRow = dataTypesDataTable.NewRow(); + + typeRow[DbMetaDataColumnNames.TypeName] = metaType.TypeName; + typeRow[DbMetaDataColumnNames.ProviderDbType] = (int)metaType.SqlDbType; + // A vector is transported as an 8 byte header followed by its elements, and + // the whole value must fit within a single TDS packet. + typeRow[DbMetaDataColumnNames.ColumnSize] = TdsEnums.MAXSIZE; + // A vector column declares its number of dimensions, and optionally its base + // type. The base type is omitted here because it defaults to float32. + typeRow[DbMetaDataColumnNames.CreateFormat] = $"{metaType.TypeName}({{0}})"; + typeRow[DbMetaDataColumnNames.CreateParameters] = "dimensions"; + typeRow[DbMetaDataColumnNames.DataType] = metaType.ClassType.FullName; + typeRow[DbMetaDataColumnNames.IsAutoIncrementable] = false; + // The DataType for a vector is a byte array, which is how it is transported + // rather than how it is best represented. + typeRow[DbMetaDataColumnNames.IsBestMatch] = false; + typeRow[DbMetaDataColumnNames.IsCaseSensitive] = false; + typeRow[DbMetaDataColumnNames.IsConcurrencyType] = false; + typeRow[DbMetaDataColumnNames.IsFixedLength] = false; + typeRow[DbMetaDataColumnNames.IsFixedPrecisionScale] = false; + typeRow[DbMetaDataColumnNames.IsLong] = false; + typeRow[DbMetaDataColumnNames.IsNullable] = true; + typeRow[DbMetaDataColumnNames.IsSearchable] = false; + typeRow[DbMetaDataColumnNames.IsSearchableWithLike] = false; + // A vector literal is written as a JSON array in a string literal. + typeRow[DbMetaDataColumnNames.LiteralPrefix] = "'"; + typeRow[DbMetaDataColumnNames.LiteralSuffix] = "'"; + typeRow[MinimumVersionKey] = "17.00.000.0"; + + dataTypesDataTable.Rows.Add(typeRow); + } + void AddSqlVariantType() { MetaType metaType = MetaType.GetMetaTypeFromSqlDbType(SqlDbType.Variant, isMultiValued: false); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs new file mode 100644 index 0000000000..5f345be99e --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs @@ -0,0 +1,105 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.Data.Common; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.VectorTest; + +#nullable enable + +/// +/// Tests for the metadata a vector column reports. This applies to every base type, so the +/// tests here use float32 and run against any server which supports vectors. +/// +[Trait("Set", "3")] +public sealed class VectorColumnMetadataTests +{ + private readonly string _connectionString = DataTestUtility.TCPConnectionString; + + public static bool IsSupported => DataTestUtility.IsSqlVectorSupported; + + [ConditionalTheory(nameof(IsSupported))] + [InlineData(1)] + [InlineData(3)] + [InlineData(1998)] + public void ReportsBaseTypeAndDimensions(int dimensions) + { + // A vector column carries its base type as the numeric scale, and its dimension count + // has to be derived from the column size. Both are surfaced under their own names so + // that callers do not have to know that encoding. + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = + new($"SELECT CAST(NULL AS vector({dimensions}, float32)) AS v", connection); + using SqlDataReader reader = command.ExecuteReader(); + + DbColumn column = reader.GetColumnSchema()[0]; + + Assert.Equal("float32", column["VectorBaseType"]); + Assert.Equal(dimensions, column["VectorDimensions"]); + } + + [ConditionalFact(nameof(IsSupported))] + public void ReportsNullForNonVectorColumns() + { + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = + new("SELECT CAST('abc' AS varchar(10)) AS s, CAST(1 AS int) AS i", connection); + using SqlDataReader reader = command.ExecuteReader(); + + foreach (DbColumn column in reader.GetColumnSchema()) + { + Assert.Null(column["VectorBaseType"]); + Assert.Null(column["VectorDimensions"]); + } + } + + [ConditionalFact(nameof(IsSupported))] + public void ReportsStandardPropertiesAlongsideVectorProperties() + { + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = + new("SELECT CAST(NULL AS vector(3, float32)) AS v", connection); + using SqlDataReader reader = command.ExecuteReader(); + + DbColumn column = reader.GetColumnSchema()[0]; + + Assert.Equal("v", column.ColumnName); + Assert.Equal("vector", column.DataTypeName); + + // An unrecognised property name continues to return null rather than throwing. + Assert.Null(column["NoSuchProperty"]); + } + + [ConditionalFact(nameof(IsSupported))] + public void SchemaCollectionIncludesVectorType() + { + using SqlConnection connection = new(_connectionString); + connection.Open(); + + DataTable dataTypes = connection.GetSchema("DataTypes"); + DataRow? vectorRow = null; + + foreach (DataRow row in dataTypes.Rows) + { + if (string.Equals(row["TypeName"]?.ToString(), "vector", StringComparison.OrdinalIgnoreCase)) + { + vectorRow = row; + break; + } + } + + Assert.NotNull(vectorRow); + Assert.Equal((int)SqlDbTypeExtensions.Vector, vectorRow!["ProviderDbType"]); + Assert.Equal("vector({0})", vectorRow["CreateFormat"]); + } +} From 8fc82cb71cf9f0d18e410ea8f77b23d54f762536 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 4 Aug 2026 23:50:30 +0530 Subject: [PATCH 02/24] Add an IEEE 754 binary16 codec SQL Server transports vector(N, float16) elements as raw binary16 values. System.Half is only available on .NET, so the conversion is implemented manually for .NET Framework. The manual implementation is compiled for every target framework rather than only for .NET Framework, so that it can be validated exhaustively against System.Half on .NET while remaining the code path .NET Framework actually uses. It is verified against every binary16 bit pattern, a strided sweep of the single precision range, and the rounding, subnormal, overflow and underflow boundaries. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../Microsoft/Data/Common/Float16Converter.cs | 237 ++++++++++++++++++ .../Data/SqlTypes/Float16ConverterTest.cs | 222 ++++++++++++++++ 2 files changed, 459 insertions(+) create mode 100644 src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/Float16Converter.cs create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/Float16Converter.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/Float16Converter.cs new file mode 100644 index 0000000000..958aa1b22e --- /dev/null +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/Float16Converter.cs @@ -0,0 +1,237 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.Data.SqlClient +{ + /// + /// Converts between IEEE 754 binary16 (half precision) and binary32 (single + /// precision) values. + /// + /// + /// + /// SQL Server transports vector(N, float16) elements as raw binary16 + /// values. System.Half is only available on .NET, so the conversion is + /// implemented manually for .NET Framework. + /// + /// + /// The manual implementations are compiled for every target framework, rather + /// than only for .NET Framework, so that they can be validated exhaustively + /// against System.Half in unit tests. + /// + /// + internal static class Float16Converter + { + // Layout of an IEEE 754 binary16 value: + // bit 15 : sign + // bits 14-10: exponent, biased by 15 + // bits 9-0 : mantissa + private const int Binary16MantissaBits = 10; + private const int Binary16ExponentBias = 15; + private const int Binary16MaxExponent = 0x1F; + + // Layout of an IEEE 754 binary32 value: + // bit 31 : sign + // bits 30-23: exponent, biased by 127 + // bits 22-0 : mantissa + private const int Binary32MantissaBits = 23; + private const int Binary32ExponentBias = 127; + private const int Binary32MaxExponent = 0xFF; + + // The number of mantissa bits discarded when narrowing binary32 to binary16. + private const int MantissaShift = Binary32MantissaBits - Binary16MantissaBits; + + /// + /// Converts the raw bits of an IEEE 754 binary16 value to the equivalent + /// single precision value. The conversion is always exact. + /// + internal static float ToSingle(ushort bits) + { + #if NET + return (float)BitConverter.UInt16BitsToHalf(bits); + #else + return ManualToSingle(bits); + #endif + } + + /// + /// Converts a single precision value to the raw bits of the nearest IEEE 754 + /// binary16 value, using the round-to-nearest-even rounding mode. Values whose + /// magnitude exceeds the binary16 range are converted to an infinity. + /// + internal static ushort FromSingle(float value) + { + #if NET + return BitConverter.HalfToUInt16Bits((Half)value); + #else + return ManualFromSingle(value); + #endif + } + + /// + /// Framework independent implementation of . Exposed + /// separately so that it can be validated against System.Half. + /// + internal static float ManualToSingle(ushort bits) + { + int sign = (bits >> 15) & 0x1; + int exponent = (bits >> Binary16MantissaBits) & Binary16MaxExponent; + int mantissa = bits & 0x3FF; + + if (exponent == Binary16MaxExponent) + { + // Infinity or NaN. A zero mantissa denotes an infinity; any other + // value denotes a NaN, which is canonicalised to float.NaN. + return mantissa == 0 + ? Int32BitsToSingle((sign << 31) | (Binary32MaxExponent << Binary32MantissaBits)) + : float.NaN; + } + + if (exponent == 0) + { + if (mantissa == 0) + { + // Positive or negative zero. + return Int32BitsToSingle(sign << 31); + } + + // A subnormal binary16 value is always normal when widened to binary32, + // so shift the mantissa left until the implicit leading bit is set, + // decrementing the exponent to compensate. + do + { + mantissa <<= 1; + exponent--; + } + while ((mantissa & 0x400) == 0); + + // Discard the now-explicit leading bit and correct for the loop + // having started from a biased exponent of zero rather than one. + mantissa &= 0x3FF; + exponent++; + } + + int rebiasedExponent = exponent - Binary16ExponentBias + Binary32ExponentBias; + + return Int32BitsToSingle( + (sign << 31) | + (rebiasedExponent << Binary32MantissaBits) | + (mantissa << MantissaShift)); + } + + /// + /// Framework independent implementation of . Exposed + /// separately so that it can be validated against System.Half. + /// + internal static ushort ManualFromSingle(float value) + { + int bits = SingleToInt32Bits(value); + int sign = (bits >> 31) & 0x1; + int exponent = (bits >> Binary32MantissaBits) & Binary32MaxExponent; + int mantissa = bits & 0x7FFFFF; + + if (exponent == Binary32MaxExponent) + { + // Infinity or NaN. NaN is canonicalised to a quiet NaN, matching the + // representation produced by System.Half. + int payload = mantissa == 0 ? 0x7C00 : 0x7E00; + return (ushort)((sign << 15) | payload); + } + + if ((bits & 0x7FFFFFFF) == 0) + { + // Positive or negative zero. + return (ushort)(sign << 15); + } + + int targetExponent = exponent - Binary32ExponentBias + Binary16ExponentBias; + + if (targetExponent >= Binary16MaxExponent) + { + // Too large to represent, so saturate to an infinity. + return (ushort)((sign << 15) | 0x7C00); + } + + if (targetExponent <= 0) + { + // Too small to represent as a normal value. Values more than eleven + // binade below the smallest subnormal cannot round up to one, so they + // are flushed to zero rather than shifted by more than the mantissa width. + if (targetExponent < -Binary16MantissaBits) + { + return (ushort)(sign << 15); + } + + // Restore the implicit leading bit and shift the mantissa into the + // subnormal range, rounding to nearest even. + mantissa |= 1 << Binary32MantissaBits; + int shift = MantissaShift + 1 - targetExponent; + int subnormal = RoundShiftRight(mantissa, shift); + + // Rounding may have carried into the exponent, producing the smallest + // normal value. That is representable and needs no special handling, + // because the carry lands in the exponent field naturally. + return (ushort)((sign << 15) | subnormal); + } + + int roundedMantissa = RoundShiftRight(mantissa, MantissaShift); + + if (roundedMantissa == 0x400) + { + // Rounding overflowed the mantissa, so carry into the exponent. + roundedMantissa = 0; + targetExponent++; + + if (targetExponent >= Binary16MaxExponent) + { + return (ushort)((sign << 15) | 0x7C00); + } + } + + return (ushort)((sign << 15) | (targetExponent << Binary16MantissaBits) | roundedMantissa); + } + + /// + /// Shifts right by bits, + /// rounding the discarded bits to nearest and ties to even. + /// + private static int RoundShiftRight(int value, int shift) + { + int result = value >> shift; + int roundBit = (value >> (shift - 1)) & 1; + + if (roundBit == 0) + { + // The discarded portion is below the halfway point, so round down. + return result; + } + + // Round up when the discarded portion is above the halfway point, or when + // it is exactly halfway and rounding up produces an even result. + int stickyMask = (1 << (shift - 1)) - 1; + bool isTie = (value & stickyMask) == 0; + + return isTie && (result & 1) == 0 ? result : result + 1; + } + + private static float Int32BitsToSingle(int value) + { + #if NET + return BitConverter.Int32BitsToSingle(value); + #else + return BitConverterCompatible.Int32BitsToSingle(value); + #endif + } + + private static int SingleToInt32Bits(float value) + { + #if NET + return BitConverter.SingleToInt32Bits(value); + #else + return BitConverterCompatible.SingleToInt32Bits(value); + #endif + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs new file mode 100644 index 0000000000..f4e1075958 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs @@ -0,0 +1,222 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.Data.SqlClient; +using Xunit; + +#nullable enable + +namespace Microsoft.Data.SqlTypes.UnitTests; + +/// +/// Tests for the IEEE 754 binary16 codec used to exchange vector(N, float16) values. +/// +/// +/// The codec's framework independent implementation is exercised directly rather than +/// through and +/// , because those use System.Half where it +/// is available. Testing the manual implementation on every target framework means the +/// .NET Framework code path is covered by these tests too, and on .NET it can additionally +/// be compared against System.Half as a reference. +/// +public class Float16ConverterTest +{ + #region Reference comparison + + #if NET + + [Fact] + public void ToSingle_MatchesHalf_ForEveryBitPattern() + { + for (int i = 0; i <= ushort.MaxValue; i++) + { + ushort bits = (ushort)i; + float expected = (float)BitConverter.UInt16BitsToHalf(bits); + float actual = Float16Converter.ManualToSingle(bits); + + if (float.IsNaN(expected)) + { + Assert.True(float.IsNaN(actual), $"0x{bits:X4} should convert to NaN."); + continue; + } + + // Compared bitwise so that positive and negative zero are distinguished. + Assert.True( + BitConverter.SingleToInt32Bits(expected) == BitConverter.SingleToInt32Bits(actual), + $"0x{bits:X4} converted to {actual} but Half converts it to {expected}."); + } + } + + [Fact] + public void FromSingle_MatchesHalf_ForEveryRepresentableValue() + { + for (int i = 0; i <= ushort.MaxValue; i++) + { + ushort bits = (ushort)i; + Half value = BitConverter.UInt16BitsToHalf(bits); + + if (Half.IsNaN(value)) + { + continue; + } + + Assert.True( + bits == Float16Converter.ManualFromSingle((float)value), + $"0x{bits:X4} did not survive a round trip through single precision."); + } + } + + [Fact] + public void FromSingle_MatchesHalf_AcrossTheSinglePrecisionRange() + { + // Steps through the single precision space by raw bit pattern. The stride is prime + // so that the samples do not align with exponent or mantissa boundaries. + const long Stride = 1039; + + for (long b = 0; b <= uint.MaxValue; b += Stride) + { + float value = BitConverter.Int32BitsToSingle((int)(uint)b); + + if (float.IsNaN(value)) + { + continue; + } + + Assert.True( + BitConverter.HalfToUInt16Bits((Half)value) == Float16Converter.ManualFromSingle(value), + $"{value:R} (0x{(uint)b:X8}) was not narrowed the same way as Half."); + } + } + + #endif + + #endregion + + #region Round trips + + [Theory] + // Exactly representable values. + [InlineData(0f)] + [InlineData(1f)] + [InlineData(-1.5f)] + [InlineData(2.5f)] + [InlineData(65504f)] // Largest finite binary16 value. + [InlineData(-65504f)] + [InlineData(6.103515625e-5f)] // Smallest normal binary16 value. + [InlineData(5.9604645e-8f)] // Smallest subnormal binary16 value. + public void RoundTrip_PreservesExactlyRepresentableValues(float value) + { + Assert.Equal(value, Float16Converter.ManualToSingle(Float16Converter.ManualFromSingle(value))); + } + + [Fact] + public void RoundTrip_PreservesSignOfZero() + { + Assert.Equal( + BitConverter.DoubleToInt64Bits(-0.0), + BitConverter.DoubleToInt64Bits(Float16Converter.ManualToSingle(Float16Converter.ManualFromSingle(-0.0f)))); + + Assert.Equal(0x8000, Float16Converter.ManualFromSingle(-0.0f)); + Assert.Equal(0x0000, Float16Converter.ManualFromSingle(0.0f)); + } + + #endregion + + #region Rounding + + [Theory] + // Values which are not representable are rounded to the nearest binary16 value. + [InlineData(1.1f, 1.0996094f)] + [InlineData(0.3f, 0.30004883f)] + [InlineData(0.1f, 0.099975586f)] + // Rounding up at the top of the range still produces the largest finite value rather + // than an infinity, because the value is nearer to it than to the overflow threshold. + [InlineData(65505f, 65504f)] + public void FromSingle_RoundsToNearest(float value, float expected) + { + Assert.Equal(expected, Float16Converter.ManualToSingle(Float16Converter.ManualFromSingle(value))); + } + + [Fact] + public void FromSingle_RoundsTiesToEven() + { + // Halfway between the binary16 values 1.0 (0x3C00) and 1.0009765625 (0x3C01). The + // tie is resolved towards the value with an even mantissa, which is 1.0. + Assert.Equal(0x3C00, Float16Converter.ManualFromSingle(1.00048828125f)); + + // Halfway between 1.0009765625 (0x3C01) and 1.001953125 (0x3C02), which resolves + // upwards for the same reason. + Assert.Equal(0x3C02, Float16Converter.ManualFromSingle(1.00146484375f)); + } + + #endregion + + #region Overflow and underflow + + [Theory] + [InlineData(70000f)] + [InlineData(float.MaxValue)] + public void FromSingle_SaturatesToInfinityOnOverflow(float value) + { + Assert.Equal(0x7C00, Float16Converter.ManualFromSingle(value)); + } + + [Theory] + [InlineData(-70000f)] + [InlineData(float.MinValue)] + public void FromSingle_SaturatesToNegativeInfinityOnOverflow(float value) + { + Assert.Equal(0xFC00, Float16Converter.ManualFromSingle(value)); + } + + [Theory] + // Below half of the smallest subnormal, so these round to zero rather than to it. + [InlineData(1e-8f)] + [InlineData(1e-30f)] + [InlineData(float.Epsilon)] + public void FromSingle_FlushesToZeroOnUnderflow(float value) + { + Assert.Equal(0x0000, Float16Converter.ManualFromSingle(value)); + } + + [Fact] + public void FromSingle_PreservesSubnormals() + { + // The smallest subnormal, and the value just above half of it, which rounds up to + // the smallest subnormal rather than to zero. + Assert.Equal(0x0001, Float16Converter.ManualFromSingle(5.9604645e-8f)); + Assert.Equal(0x0001, Float16Converter.ManualFromSingle(4.0e-8f)); + } + + #endregion + + #region Infinity and NaN + + [Fact] + public void FromSingle_PreservesInfinity() + { + Assert.Equal(0x7C00, Float16Converter.ManualFromSingle(float.PositiveInfinity)); + Assert.Equal(0xFC00, Float16Converter.ManualFromSingle(float.NegativeInfinity)); + } + + [Fact] + public void ToSingle_PreservesInfinity() + { + Assert.Equal(float.PositiveInfinity, Float16Converter.ManualToSingle(0x7C00)); + Assert.Equal(float.NegativeInfinity, Float16Converter.ManualToSingle(0xFC00)); + } + + [Fact] + public void ConvertsNaN() + { + Assert.True(float.IsNaN(Float16Converter.ManualToSingle(Float16Converter.ManualFromSingle(float.NaN)))); + + // Any binary16 value with a maximal exponent and a non-zero mantissa is a NaN. + Assert.True(float.IsNaN(Float16Converter.ManualToSingle(0x7E00))); + Assert.True(float.IsNaN(Float16Converter.ManualToSingle(0x7C01))); + } + + #endregion +} From dad19a3f8243535ea33c806abf3a553107cbafe6 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 4 Aug 2026 23:50:53 +0530 Subject: [PATCH 03/24] Add vector(float16) support Advertises version 2 of the VECTORSUPPORT feature extension, so that a vector(N, float16) column is exchanged in its native binary form rather than as a varchar(max) JSON string. On .NET such a column is surfaced as SqlVector. .NET Framework has no System.Half, so it is reported as a string there, matching how it is already presented when the server does not negotiate float16 support. Callers on either framework can explicitly request a strongly typed value via GetSqlVector, which widens the elements without loss. SqlVector continues to derive the base type written to the wire from T alone. Conversion between base types is left to the server, which performs it for parameters. Bulk copy is the exception: it declares the destination's base type in the INSERT BULK statement, so a payload using a different base type is rejected as a column length error rather than converted, and is rewritten by the driver first. That conversion runs after coercion, because the payload coercion produces uses the source value's own base type: a JSON string always yields float32, which is how a float16 column reads back where System.Half is unavailable. SqlVector.ToString() now returns the vector's values as a JSON array rather than the type name. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../ref/Microsoft.Data.SqlTypes.cs | 2 + .../Connection/ConnectionCapabilities.cs | 27 +- .../Connection/SqlConnectionInternal.cs | 4 +- .../src/Microsoft/Data/SqlClient/SqlBuffer.cs | 88 ++++-- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 61 +++- .../Microsoft/Data/SqlClient/SqlCommand.cs | 15 +- .../Microsoft/Data/SqlClient/SqlDataReader.cs | 36 ++- .../Microsoft/Data/SqlClient/SqlDbColumn.cs | 1 + .../src/Microsoft/Data/SqlClient/SqlEnums.cs | 18 +- .../Microsoft/Data/SqlClient/SqlParameter.cs | 21 ++ .../src/Microsoft/Data/SqlClient/TdsEnums.cs | 14 +- .../src/Microsoft/Data/SqlTypes/SqlVector.cs | 234 ++++++++++++++- .../VectorTest/NativeVectorFloat16Tests.cs | 64 ++++ .../VectorTest/VectorFloat16BehaviourTests.cs | 277 ++++++++++++++++++ .../Microsoft/Data/SqlTypes/SqlVectorTest.cs | 256 +++++++++++++++- .../SimulatedServerTests/ConnectionTests.cs | 16 +- 16 files changed, 1081 insertions(+), 53 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs create mode 100644 src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs diff --git a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlTypes.cs b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlTypes.cs index 887b1fc012..dd9f2e9559 100644 --- a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlTypes.cs +++ b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlTypes.cs @@ -90,4 +90,6 @@ public SqlVector(System.ReadOnlyMemory memory) { } public System.ReadOnlyMemory Memory { get { throw null; } } /// public static SqlVector CreateNull(int length) { throw null; } + /// + public override string ToString() { throw null; } } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/ConnectionCapabilities.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/ConnectionCapabilities.cs index ebc2ff6bbc..5b99ccf71d 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/ConnectionCapabilities.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/ConnectionCapabilities.cs @@ -159,13 +159,36 @@ internal sealed class ConnectionCapabilities /// public bool ReadOnlyFailoverPartnerConnection { get; set; } + /// + /// The version negotiated via the VECTORSUPPORT feature extension + /// (FEATUREEXTACK token value 0x0E), which determines the vector base + /// types the server will send and accept in their native binary form. + /// + /// + /// indicates the server does not + /// support the vector type; vector columns are returned as varchar(max). + /// indicates support for a float32 + /// base type; columns with any other base type are returned as varchar(max). + /// additionally indicates support for a + /// float16 base type. + /// + public byte VectorVersion { get; set; } + /// /// Indicates support for the vector data type, with a backing type /// of float32. This was introduced in SQL Server 2022, and is only /// available if a FEATUREEXTACK token of value 0x0E is received, and /// if the version in this token's data is greater than or equal to 1. /// - public bool Float32VectorType { get; set; } + public bool Float32VectorType => VectorVersion >= TdsEnums.VECTOR_VERSION_FLOAT32; + + /// + /// Indicates support for the vector data type, with a backing type + /// of float16. This is only available if a FEATUREEXTACK token of value + /// 0x0E is received, and if the version in this token's data is greater + /// than or equal to 2. + /// + public bool Float16VectorType => VectorVersion >= TdsEnums.VECTOR_VERSION_FLOAT16; /// /// Indicates support for the json data type. This was introduced in @@ -213,7 +236,7 @@ public void Reset() GlobalTransactionsSupported = false; EnhancedRouting = false; ReadOnlyFailoverPartnerConnection = false; - Float32VectorType = false; + VectorVersion = TdsEnums.VECTOR_VERSION_NOT_SUPPORTED; JsonType = false; ColumnEncryptionVersion = TdsEnums.TCE_NOT_ENABLED; ColumnEncryptionEnclaveType = null; diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs index fd8817b9aa..87ed9861d3 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs @@ -1668,7 +1668,9 @@ internal void OnFeatureExtAck(int featureId, byte[] data) throw SQL.ParsingError(); } - Capabilities.Float32VectorType = true; + // Record the negotiated version rather than a simple flag: it determines + // which vector base types the server will send and accept natively. + Capabilities.VectorVersion = vectorSupportVersion; break; } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs index 0ec34003f7..e5f9035cd4 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs @@ -509,14 +509,7 @@ internal string String ThrowIfNull(); if (_type == StorageType.Vector) { - var elementType = (MetaType.SqlVectorElementType)_value._vectorInfo._elementType; - switch (elementType) - { - case MetaType.SqlVectorElementType.Float32: - return GetSqlVector().GetString(); - default: - throw SQL.VectorTypeNotSupported(elementType.ToString()); - } + return GetVectorString(); } if (StorageType.String == _type || StorageType.Json == _type) { @@ -955,14 +948,7 @@ internal SqlString SqlString { return SqlString.Null; } - var elementType = (MetaType.SqlVectorElementType)_value._vectorInfo._elementType; - switch (elementType) - { - case MetaType.SqlVectorElementType.Float32: - return new SqlString(GetSqlVector().GetString()); - default: - throw SQL.VectorTypeNotSupported(elementType.ToString()); - } + return new SqlString(GetVectorString()); } // String and Json storage type are both strings. if (_type is StorageType.String or StorageType.Json) @@ -996,11 +982,70 @@ internal SqlVector GetSqlVector() where T : unmanaged { return SqlVector.CreateNull(_value._vectorInfo._elementCount); } - return new SqlVector(SqlBinary.Value); + // The payload's base type may differ from T: a float16 column can be + // read as a vector of single precision values, which is the only + // strongly typed form available on .NET Framework. + return SqlVector.FromTdsPayload(SqlBinary.Value); } return (SqlVector)SqlValue; } + /// + /// Renders a vector value as a JSON array. + /// + /// + /// float16 vectors are widened to single precision on .NET Framework, where + /// System.Half is unavailable. Widening binary16 to binary32 is exact, + /// so both frameworks produce identical output. + /// + private string GetVectorString() + { + var elementType = (MetaType.SqlVectorElementType)_value._vectorInfo._elementType; + switch (elementType) + { + case MetaType.SqlVectorElementType.Float32: + return GetSqlVector().GetString(); + case MetaType.SqlVectorElementType.Float16: + #if NET + return GetSqlVector().GetString(); + #else + return GetSqlVector().GetString(); + #endif + default: + throw SQL.VectorTypeNotSupported(elementType.ToString()); + } + } + + /// + /// Returns a vector value as the type that best represents its base type on the + /// current framework. + /// + /// + /// .NET Framework has no System.Half, so there is no faithful strongly + /// typed representation of a float16 vector there. Rather than silently + /// substituting a different base type, such values are surfaced as their JSON + /// rendering, which is how they are already presented when the server does not + /// negotiate float16 support. Callers that want a strongly typed value can + /// explicitly request one via . + /// + internal object GetVectorValue() + { + var elementType = (MetaType.SqlVectorElementType)_value._vectorInfo._elementType; + switch (elementType) + { + case MetaType.SqlVectorElementType.Float32: + return GetSqlVector(); + case MetaType.SqlVectorElementType.Float16: + #if NET + return GetSqlVector(); + #else + return GetVectorString(); + #endif + default: + throw SQL.VectorTypeNotSupported(elementType.ToString()); + } + } + internal object SqlValue { get @@ -1036,14 +1081,7 @@ internal object SqlValue case StorageType.Json: return SqlJson; case StorageType.Vector: - var elementType = (MetaType.SqlVectorElementType)_value._vectorInfo._elementType; - switch (elementType) - { - case MetaType.SqlVectorElementType.Float32: - return GetSqlVector(); - default: - throw SQL.VectorTypeNotSupported(elementType.ToString()); - } + return GetVectorValue(); case StorageType.SqlCachedBuffer: { SqlCachedBuffer data = (SqlCachedBuffer)(_object); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 44c74fc3f0..4747e0a427 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -870,6 +870,7 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i if (!metadata.metaType.IsFixed && !metadata.metaType.IsLong) { int size = metadata.length; + bool isFloat16Vector = false; switch (metadata.metaType.NullableType) { case TdsEnums.SQLNCHAR: @@ -878,12 +879,27 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i size /= 2; break; case TdsEnums.SQLVECTOR: + // A vector's dimension count is derived from the payload + // size, and its scale carries the base type. size = MetaType.GetVectorElementCount(metadata.length, metadata.scale); + isFloat16Vector = metadata.scale == (byte)MetaType.SqlVectorElementType.Float16; break; default: break; } - updateBulkCommandText.AppendFormat((IFormatProvider)null, "({0})", size); + + // The base type is only stated for float16, so that the + // declaration emitted for float32 vectors is unchanged from + // earlier versions and remains understood by servers which + // predate float16 support. + if (isFloat16Vector) + { + updateBulkCommandText.AppendFormat((IFormatProvider)null, "({0}, float16)", size); + } + else + { + updateBulkCommandText.AppendFormat((IFormatProvider)null, "({0})", size); + } } else if (metadata.metaType.IsPlp && !(metadata.metaType.SqlDbType is SqlDbType.Xml or SqlDbTypeExtensions.Json or SqlDbTypeExtensions.Vector)) { @@ -1688,6 +1704,31 @@ private object ValidateBulkCopyVariant(object value) } } + /// + /// Rewrites a coerced vector payload so that its elements use the destination + /// column's base type, leaving payloads which already use that base type untouched. + /// + /// + /// Unlike an ordinary parameter, a bulk copy declares the destination's base type in + /// the INSERT BULK statement, so the payload must use that base type. The + /// server cannot convert it, because binary16 and binary32 elements differ in size + /// and a mismatch is reported as a column length error. + /// + private static object ConvertVectorToBaseType(object value, byte destinationElementType) + { + if (value is not byte[] payload) + { + // The value was coerced to something other than a vector payload, such as a + // data feed, which the existing write path handles. + return value; + } + + // The payload is converted directly rather than through a strongly typed vector, + // so that .NET Framework, which has no System.Half, can also write to float16 + // destinations. + return SqlTypes.SqlVector.ConvertPayloadElementType(payload, destinationElementType); + } + private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, ref bool isSqlType, out bool coercedToDataFeed) { coercedToDataFeed = false; @@ -1772,6 +1813,23 @@ private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, re typeChanged = false; // Setting this to false as SqlParameter.CoerceValue will only set it to true when converting to a CLR type break; + case TdsEnums.SQLVECTOR: + mt = MetaType.GetMetaTypeFromSqlDbType(type.SqlDbType, false); + value = SqlParameter.CoerceValue(value, mt, out coercedToDataFeed, out typeChanged, false); + + // The INSERT BULK declaration for a vector column states the + // destination's base type, so the payload written to the wire must + // use that base type too. A mismatch is rejected by the server as a + // column length error rather than being converted, because binary16 + // and binary32 elements differ in size. + // + // This runs after coercion because the payload produced by coercion + // uses the source value's own base type: a JSON string always yields + // float32, which is how a float16 column reads back on frameworks + // without System.Half. + value = ConvertVectorToBaseType(value, metadata.scale); + break; + case TdsEnums.SQLINTN: case TdsEnums.SQLFLTN: case TdsEnums.SQLFLT4: @@ -1793,7 +1851,6 @@ private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, re case TdsEnums.SQLTIME: case TdsEnums.SQLDATETIME2: case TdsEnums.SQLDATETIMEOFFSET: - case TdsEnums.SQLVECTOR: mt = MetaType.GetMetaTypeFromSqlDbType(type.SqlDbType, false); value = SqlParameter.CoerceValue(value, mt, out coercedToDataFeed, out typeChanged, false); break; diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs index 76399749a0..4145cb02ff 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs @@ -2247,7 +2247,20 @@ private string BuildParamList(TdsParser parser, SqlParameterCollection parameter // The validate function for SqlParameters would have already thrown // InvalidCastException if an incompatible value is specified for vector type. ISqlVector vectorProps = (ISqlVector)sqlParam.Value; - paramList.AppendFormat("({0})", vectorProps.Length); + + // The base type is only stated for float16, so that the declaration + // emitted for float32 vectors is unchanged from earlier versions and + // remains understood by servers which predate float16 support. The + // declaration must agree with the base type written into the binary + // parameter metadata, which is derived from the same value. + if (vectorProps.ElementType == (byte)MetaType.SqlVectorElementType.Float16) + { + paramList.AppendFormat("({0}, float16)", vectorProps.Length); + } + else + { + paramList.AppendFormat("({0})", vectorProps.Length); + } } else if (!mt.IsFixed && !mt.IsLong && mt.SqlDbType is not SqlDbType.Timestamp and not SqlDbType.Udt diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs index cbc83b7406..6c884493d7 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs @@ -1310,6 +1310,15 @@ private static Type GetVectorFieldType(byte vectorElementType) return elementType switch { MetaType.SqlVectorElementType.Float32 => typeof(SqlVector), + // .NET Framework has no System.Half, so there is no faithful strongly + // typed representation of a float16 vector. Such columns are reported as + // strings, matching how they are presented when the server does not + // negotiate float16 support. + #if NET + MetaType.SqlVectorElementType.Float16 => typeof(SqlVector), + #else + MetaType.SqlVectorElementType.Float16 => typeof(string), + #endif _ => throw SQL.VectorTypeNotSupported(elementType.ToString()), }; } @@ -2550,7 +2559,11 @@ public virtual SqlJson GetSqlJson(int i) /// public virtual SqlVector GetSqlVector(int i) where T : unmanaged { - if (typeof(T) != typeof(float)) + if (typeof(T) != typeof(float) + #if NET + && typeof(T) != typeof(Half) + #endif + ) { throw SQL.VectorTypeNotSupported(typeof(T).FullName); } @@ -2793,13 +2806,9 @@ private object GetValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData metaDa } else { - switch (metaData.scale) - { - case (byte)MetaType.SqlVectorElementType.Float32: - return data.GetSqlVector(); - default: - throw SQL.VectorTypeNotSupported(metaData.scale.ToString()); - } + // SqlBuffer selects the representation that best matches the vector's + // base type on the current framework. + return data.GetVectorValue(); } } else if (metaData.type == SqlDbType.Udt) @@ -2908,6 +2917,17 @@ private T GetFieldValueFromSqlBufferInternal(SqlBuffer data, _SqlMetaData met } return (T)(object)data.GetSqlVector(); } + #if NET + else if (typeof(T) == typeof(SqlVector)) + { + MetaType metaType = metaData.metaType; + if (metaType.SqlDbType != SqlDbTypeExtensions.Vector) + { + throw SQL.VectorNotSupportedOnColumnType(metaData.column); + } + return (T)(object)data.GetSqlVector(); + } + #endif else if (typeof(T) == typeof(XmlReader)) { // XmlReader only allowed on XML types diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs index c2cfeaa92a..df8996fdc3 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs @@ -160,6 +160,7 @@ internal int? SqlNumericScale : (MetaType.SqlVectorElementType)_metadata.scale switch { MetaType.SqlVectorElementType.Float32 => "float32", + MetaType.SqlVectorElementType.Float16 => "float16", // An unrecognised base type is reported rather than throwing, because // reading metadata should not fail on a column the caller may ignore. _ => _metadata.scale.ToString(CultureInfo.InvariantCulture), diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs index 70ab7dfa68..a0d9e7aaff 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs @@ -69,7 +69,8 @@ internal sealed class MetaType // in the TDS protocol. internal enum SqlVectorElementType : byte { - Float32 = 0x00 + Float32 = 0x00, + Float16 = 0x01 } public MetaType(byte precision, byte scale, int fixedLength, bool isFixed, bool isLong, bool isPlp, byte tdsType, byte nullableTdsType, string typeName, @@ -416,6 +417,12 @@ private static MetaType GetMetaTypeFromValue(Type dataType, object value, bool i { return s_MetaVector; } + #if NET + else if (dataType == typeof(SqlVector)) + { + return s_MetaVector; + } + #endif else if (dataType == typeof(SqlString)) { return ((inferLen && !((SqlString)value).IsNull) @@ -1127,9 +1134,14 @@ internal static int GetTimeSizeFromScale(byte scale) internal static int GetVectorElementSize(byte type) { - switch (type) + switch ((SqlVectorElementType)type) { - case 0: return sizeof(float); + case SqlVectorElementType.Float32: + return sizeof(float); + case SqlVectorElementType.Float16: + // System.Half is not available on all target frameworks, so the + // binary16 element size is stated explicitly. + return 2; default: throw SQL.VectorTypeNotSupported(type.ToString()); } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs index f9e342d4c6..a0c5bb720f 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs @@ -774,6 +774,14 @@ private object GetVectorReturnValue() { case MetaType.SqlVectorElementType.Float32: return SqlVector.CreateNull(elementCount); + case MetaType.SqlVectorElementType.Float16: + #if NET + return SqlVector.CreateNull(elementCount); + #else + // System.Half is unavailable, so a float16 vector has no faithful + // strongly typed representation and is surfaced as single precision. + return SqlVector.CreateNull(elementCount); + #endif default: throw SQL.VectorTypeNotSupported(elementType.ToString()); } @@ -782,6 +790,13 @@ private object GetVectorReturnValue() { case MetaType.SqlVectorElementType.Float32: return new SqlVector((byte[])_sqlBufferReturnValue.Value); + case MetaType.SqlVectorElementType.Float16: + #if NET + return new SqlVector((byte[])_sqlBufferReturnValue.Value); + #else + // Widening binary16 to binary32 is exact, so no information is lost. + return SqlVector.FromTdsPayload((byte[])_sqlBufferReturnValue.Value); + #endif default: throw SQL.VectorTypeNotSupported(elementType.ToString()); } @@ -2385,6 +2400,12 @@ internal static object CoerceValue(object value, MetaType destinationType, out b { value = ((ISqlVector)value).VectorPayload; } + #if NET + else if (currentType == typeof(SqlVector)) + { + value = ((ISqlVector)value).VectorPayload; + } + #endif else if (currentType == typeof(string) && destinationType.SqlDbType == SqlDbTypeExtensions.Vector) { try diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsEnums.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsEnums.cs index 2590c28690..9fe7407567 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsEnums.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsEnums.cs @@ -659,7 +659,19 @@ internal enum FedAuthInfoId : byte internal const byte MAX_SUPPORTED_JSON_VERSION = 0x01; // Vector Support constants - internal const byte MAX_SUPPORTED_VECTOR_VERSION = 0x01; + // + // The version negotiated via the VECTORSUPPORT feature extension determines which + // vector base types the server will send and accept in their native binary form: + // + // 0 - The server does not support the vector type. Vector columns are returned + // as varchar(max) containing a JSON array. + // 1 - The server supports vector columns with a float32 base type. Columns with + // any other base type are returned as varchar(max) containing a JSON array. + // 2 - The server additionally supports vector columns with a float16 base type. + internal const byte VECTOR_VERSION_NOT_SUPPORTED = 0x00; + internal const byte VECTOR_VERSION_FLOAT32 = 0x01; + internal const byte VECTOR_VERSION_FLOAT16 = 0x02; + internal const byte MAX_SUPPORTED_VECTOR_VERSION = VECTOR_VERSION_FLOAT16; internal const int VECTOR_HEADER_SIZE = 8; // TCE Related constants diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs index d87a539c37..88614a0049 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs @@ -24,6 +24,10 @@ namespace Microsoft.Data.SqlTypes; private const byte VecHeaderMagicNo = 0xA9; private const byte VecVersionNo = 0x01; + // Offsets of the fields within the vector header. Refer to TDS section 2.2.5.5.7. + private const int VecHeaderLengthOffset = 2; + private const int VecHeaderElementTypeOffset = 4; + #endregion #region Fields @@ -97,9 +101,104 @@ internal string GetString() { return SQLMessage.NullString(); } + + #if NET + if (typeof(T) == typeof(Half)) + { + // Widening binary16 to binary32 is exact, so serialising the widened values + // renders the true value of every element. Serialising Half directly would + // instead produce the shortest string that round-trips to the same Half, + // which can misrepresent the value: 65504 would render as "65500". Widening + // also keeps this rendering identical on .NET Framework, where System.Half + // is unavailable and float16 vectors are surfaced as single precision. + ReadOnlySpan elements = ((ReadOnlyMemory)(object)Memory).Span; + float[] widened = new float[elements.Length]; + + for (int i = 0; i < elements.Length; i++) + { + widened[i] = (float)elements[i]; + } + + return JsonSerializer.Serialize(widened); + } + #endif + return JsonSerializer.Serialize(Memory); } + /// + public override string ToString() => GetString(); + + /// + /// Creates a vector from a TDS payload, converting the elements when the payload's + /// base type differs from . + /// + /// + /// + /// Only widening conversions are performed, because they are always exact. A + /// float16 payload can therefore be read as a of + /// , which is the only way for .NET Framework callers to read + /// such a column in a strongly typed form, as System.Half is unavailable there. + /// + /// + /// The converted vector carries a payload rebuilt for , so + /// that alone continues to determine the base type used when + /// the value is sent back to the server. + /// + /// + internal static SqlVector FromTdsPayload(byte[] tdsBytes) + { + if (tdsBytes.Length < TdsEnums.VECTOR_HEADER_SIZE) + { + throw ADP.InvalidVectorHeader(); + } + + byte payloadElementType = tdsBytes[VecHeaderElementTypeOffset]; + (byte targetElementType, _, _) = GetTypeFieldsOrThrow(); + + if (payloadElementType == targetElementType) + { + return new SqlVector(tdsBytes); + } + + if (payloadElementType == (byte)MetaType.SqlVectorElementType.Float16 && + targetElementType == (byte)MetaType.SqlVectorElementType.Float32) + { + return new SqlVector(WidenFloat16Payload(tdsBytes)); + } + + // Any other combination would be a narrowing conversion, which is lossy and so + // is never performed implicitly. + throw SQL.VectorTypeNotSupported(typeof(T).FullName); + } + + /// + /// Widens the float16 elements of a TDS payload to single precision values. + /// + private static ReadOnlyMemory WidenFloat16Payload(byte[] tdsBytes) + { + const int Float16ElementSize = 2; + + int length = BinaryPrimitives.ReadUInt16LittleEndian(tdsBytes.AsSpan(VecHeaderLengthOffset)); + + if (tdsBytes.Length != TdsEnums.VECTOR_HEADER_SIZE + (Float16ElementSize * length)) + { + throw ADP.InvalidVectorHeader(); + } + + float[] widened = new float[length]; + + for (int i = 0, currPosition = TdsEnums.VECTOR_HEADER_SIZE; i < length; i++, currPosition += Float16ElementSize) + { + widened[i] = Float16Converter.ToSingle( + BinaryPrimitives.ReadUInt16LittleEndian(tdsBytes.AsSpan(currPosition))); + } + + // T is known to be float on this path, so the cast through object simply + // reinterprets the memory's element type. + return (ReadOnlyMemory)(object)new ReadOnlyMemory(widened); + } + #endregion #region Properties @@ -139,6 +238,14 @@ private static (byte, byte, int) GetTypeFieldsOrThrow() elementType = (byte)MetaType.SqlVectorElementType.Float32; elementSize = sizeof(float); } + #if NET + else if (typeof(T) == typeof(Half)) + { + elementType = (byte)MetaType.SqlVectorElementType.Float16; + // sizeof(Half) requires an unsafe context, so the size is stated explicitly. + elementSize = 2; + } + #endif else { throw SQL.VectorTypeNotSupported(typeof(T).FullName); @@ -172,8 +279,8 @@ private byte[] MakeTdsBytes(ReadOnlyMemory values) // Header Bytes result[0] = VecHeaderMagicNo; result[1] = VecVersionNo; - BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(2), (ushort)Length); - result[4] = _elementType; + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(VecHeaderLengthOffset), (ushort)Length); + result[VecHeaderElementTypeOffset] = _elementType; result[5] = 0x00; result[6] = 0x00; result[7] = 0x00; @@ -201,6 +308,17 @@ private byte[] MakeTdsBytes(ReadOnlyMemory values) #endif } } + #if NET + else if (typeof(T) == typeof(Half)) + { + for (int i = 0, currPosition = TdsEnums.VECTOR_HEADER_SIZE; i < values.Length; i++, currPosition += _elementSize) + { + BinaryPrimitives.WriteUInt16LittleEndian( + result.AsSpan(currPosition), + BitConverter.HalfToUInt16Bits((Half)(object)valueSpan[i])); + } + } + #endif } return result; @@ -217,14 +335,14 @@ private byte[] MakeTdsBytes(ReadOnlyMemory values) // Do we support the version? rawBytes[1] != VecVersionNo || // Do the vector types match? - rawBytes[4] != _elementType) + rawBytes[VecHeaderElementTypeOffset] != _elementType) { // No, so throw. throw ADP.InvalidVectorHeader(); } // The vector length is an unsigned 16-bit integer, little-endian. - int length = BinaryPrimitives.ReadUInt16LittleEndian(rawBytes.AsSpan(2)); + int length = BinaryPrimitives.ReadUInt16LittleEndian(rawBytes.AsSpan(VecHeaderLengthOffset)); // The vector size is the number of bytes required to represent the vector in TDS. int size = TdsEnums.VECTOR_HEADER_SIZE + (_elementSize * length); @@ -269,10 +387,118 @@ private T[] MakeArray() #endif } } + #if NET + else if (typeof(T) == typeof(Half)) + { + for (int i = 0, currPosition = TdsEnums.VECTOR_HEADER_SIZE; i < Length; i++, currPosition += _elementSize) + { + result[i] = (T)(object)BitConverter.UInt16BitsToHalf( + BinaryPrimitives.ReadUInt16LittleEndian(_tdsBytes.AsSpan(currPosition))); + } + } + #endif + } + + return result; + } + + /// + /// Rewrites a TDS vector payload so that its elements use the requested base type. + /// + /// + /// This works on every target framework, including those without System.Half, + /// because it produces a raw payload rather than a strongly typed vector. It is used by + /// bulk copy, where the base type written to the wire must match the destination + /// column's: the server reports a mismatch as a column length error rather than + /// converting the value, because binary16 and binary32 elements differ in size. + /// + internal static byte[] ConvertPayloadElementType(byte[] tdsBytes, byte targetElementType) + { + if (tdsBytes.Length < TdsEnums.VECTOR_HEADER_SIZE) + { + throw ADP.InvalidVectorHeader(); + } + + byte sourceElementType = tdsBytes[VecHeaderElementTypeOffset]; + + if (sourceElementType == targetElementType) + { + return tdsBytes; + } + + int length = BinaryPrimitives.ReadUInt16LittleEndian(tdsBytes.AsSpan(VecHeaderLengthOffset)); + int sourceElementSize = MetaType.GetVectorElementSize(sourceElementType); + int targetElementSize = MetaType.GetVectorElementSize(targetElementType); + + if (tdsBytes.Length != TdsEnums.VECTOR_HEADER_SIZE + (sourceElementSize * length)) + { + throw ADP.InvalidVectorHeader(); + } + + byte[] result = new byte[TdsEnums.VECTOR_HEADER_SIZE + (targetElementSize * length)]; + + result[0] = VecHeaderMagicNo; + result[1] = VecVersionNo; + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(VecHeaderLengthOffset), (ushort)length); + result[VecHeaderElementTypeOffset] = targetElementType; + + for (int i = 0, + sourcePosition = TdsEnums.VECTOR_HEADER_SIZE, + targetPosition = TdsEnums.VECTOR_HEADER_SIZE; + i < length; + i++, sourcePosition += sourceElementSize, targetPosition += targetElementSize) + { + // Every supported base type widens to single precision without loss, so it + // serves as the common representation for the conversion. + WriteElement( + result, + targetPosition, + targetElementType, + ReadElement(tdsBytes, sourcePosition, sourceElementType)); } return result; } + private static float ReadElement(byte[] payload, int position, byte elementType) + { + switch ((MetaType.SqlVectorElementType)elementType) + { + case MetaType.SqlVectorElementType.Float32: + #if NET + return BinaryPrimitives.ReadSingleLittleEndian(payload.AsSpan(position)); + #else + return BitConverterCompatible.Int32BitsToSingle(BinaryPrimitives.ReadInt32LittleEndian(payload.AsSpan(position))); + #endif + + case MetaType.SqlVectorElementType.Float16: + return Float16Converter.ToSingle(BinaryPrimitives.ReadUInt16LittleEndian(payload.AsSpan(position))); + + default: + throw SQL.VectorTypeNotSupported(elementType.ToString()); + } + } + + private static void WriteElement(byte[] payload, int position, byte elementType, float value) + { + switch ((MetaType.SqlVectorElementType)elementType) + { + case MetaType.SqlVectorElementType.Float32: + #if NET + BinaryPrimitives.WriteSingleLittleEndian(payload.AsSpan(position), value); + #else + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan(position), BitConverterCompatible.SingleToInt32Bits(value)); + #endif + break; + + case MetaType.SqlVectorElementType.Float16: + BinaryPrimitives.WriteUInt16LittleEndian(payload.AsSpan(position), Float16Converter.FromSingle(value)); + break; + + default: + throw SQL.VectorTypeNotSupported(elementType.ToString()); + } + } + #endregion } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs new file mode 100644 index 0000000000..f3b7fa6bc1 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +// System.Half, and therefore SqlVector, is only available on .NET. +#if NET + +using System; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.VectorTest; + +#nullable enable + +public sealed class VectorFloat16TestData : NativeVectorTestDataBase +{ + // Includes the extremes of the binary16 range, a subnormal, and a negative zero. + // Every value is exactly representable, so it survives a round trip through the + // JSON rendering the string based read paths return. + public override Half[] SampleScalarData => + [ + (Half)1.5f, + (Half)2.25f, + (Half)(-3.75f), + Half.MaxValue, + Half.Epsilon, + (Half)(-0.0f), + ]; + + public override Half[,] SampleDataSet + { + get + { + Half[,] sampleData = new Half[10, ValidSampleScalarDataLength]; + + for (int i = 0; i < sampleData.GetLength(0); i++) + { + float baseValue = i * 10; + + for (int j = 0; j < sampleData.GetLength(1); j++) + { + // Eighths are exactly representable in binary16 at this magnitude, so + // the values are unchanged by the round trip through the server. + sampleData[i, j] = (Half)(baseValue + (j * 0.125f)); + } + } + + return sampleData; + } + } + + public override int IncorrectScalarDataParameterSize => 3234; + + public override bool IsSupported => DataTestUtility.IsSqlVectorFloat16Supported; + + public override string SqlServerTypeName => "float16"; +} + +[Trait("Set", "3")] +public sealed class NativeVectorFloat16Tests : NativeVectorTestsBase +{ +} + +#endif diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs new file mode 100644 index 0000000000..b3de295235 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -0,0 +1,277 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; +using Microsoft.Data.SqlTypes; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.VectorTest; + +#nullable enable + +/// +/// Tests for behaviour which is specific to the float16 vector base type, or which +/// concerns the interaction between the two base types, and so has no equivalent in the +/// shared suite. +/// +[Trait("Set", "3")] +public sealed class VectorFloat16BehaviourTests : IDisposable +{ + private const string ColumnName = "VectorData"; + private const string ParameterName = "@VectorData"; + + private readonly string _connectionString = DataTestUtility.TCPConnectionString; + private readonly SqlConnection _managementConnection; + private readonly Table _float16Table; + private readonly Table _float32Table; + private bool _disposed; + + public VectorFloat16BehaviourTests() + { + _managementConnection = new SqlConnection(_connectionString); + _managementConnection.Open(); + + _float16Table = new Table(_managementConnection, "VectorF16BehaviourTable", + $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector(3, float16) NULL)"); + _float32Table = new Table(_managementConnection, "VectorF32BehaviourTable", + $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector(3, float32) NULL)"); + } + + public static bool IsSupported => DataTestUtility.IsSqlVectorFloat16Supported; + + #region Column metadata + + [ConditionalFact(nameof(IsSupported))] + public void ColumnSchemaReportsFloat16BaseType() + { + // Metadata for vector columns in general is covered by VectorColumnMetadataTests; + // this checks only that the float16 base type is reported by its own name, and that + // its dimension count accounts for the smaller element size. + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = new("SELECT CAST(NULL AS vector(3, float16)) AS v", connection); + using SqlDataReader reader = command.ExecuteReader(); + + DbColumn column = reader.GetColumnSchema()[0]; + + Assert.Equal("float16", column["VectorBaseType"]); + Assert.Equal(3, column["VectorDimensions"]); + } + + #endregion + + #region Reading + + [ConditionalFact(nameof(IsSupported))] + public void ReadsFloat16ColumnAsWidenedSingles() + { + // Requesting single precision from a float16 column widens the elements, which is + // exact. This is the only strongly typed read available where System.Half is not. + Insert(_float16Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + + using SqlDataReader reader = Select(_float16Table); + Assert.True(reader.Read()); + + SqlVector vector = reader.GetSqlVector(0); + + Assert.Equal(3, vector.Length); + Assert.Equal([1.5f, 2.5f, 3.5f], vector.Memory.ToArray()); + } + + [ConditionalFact(nameof(IsSupported))] + public void RendersValuesExactlyRatherThanShortestRoundTrip() + { + // The largest finite binary16 value renders as 65500 if the elements are formatted + // as System.Half, because that is the shortest string which round trips to the same + // Half. Widening to single precision first renders the value itself. + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = new("SELECT CAST('[65504,1,2]' AS vector(3, float16))", connection); + using SqlDataReader reader = command.ExecuteReader(); + + Assert.True(reader.Read()); + Assert.Equal("[65504,1,2]", reader.GetString(0)); + Assert.Equal("[65504,1,2]", reader.GetValue(0).ToString()); + } + + [ConditionalFact(nameof(IsSupported))] + public void ReportsUnsupportedElementTypes() + { + Insert(_float16Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + + using SqlDataReader reader = Select(_float16Table); + Assert.True(reader.Read()); + + Assert.Throws(() => reader.GetSqlVector(0)); + Assert.Throws(() => reader.GetSqlVector(0)); + } + + #endregion + + #region Writing across base types + + public static IEnumerable CrossBaseTypeParameters() + { + // A vector of either base type can be written to a column of either base type: the + // conversion is performed by the server, which knows the destination's base type. + yield return ["float16", new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })]; + yield return ["float32", new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })]; + #if NET + yield return ["float16", new SqlVector(new Half[] { (Half)1.5f, (Half)2.5f, (Half)3.5f })]; + yield return ["float32", new SqlVector(new Half[] { (Half)1.5f, (Half)2.5f, (Half)3.5f })]; + #endif + } + + [ConditionalTheory(nameof(IsSupported))] + [MemberData(nameof(CrossBaseTypeParameters), DisableDiscoveryEnumeration = true)] + public void WritesVectorParameterToColumnOfEitherBaseType(string columnBaseType, object value) + { + Table table = columnBaseType == "float16" ? _float16Table : _float32Table; + + Insert(table, value); + + using SqlDataReader reader = Select(table); + Assert.True(reader.Read()); + Assert.Equal([1.5f, 2.5f, 3.5f], reader.GetSqlVector(0).Memory.ToArray()); + } + + [ConditionalFact(nameof(IsSupported))] + public void RejectsValuesOutsideTheFloat16Range() + { + // The server reports the failure rather than silently saturating the value. + SqlException exception = Assert.Throws(() => + Insert(_float16Table, new SqlVector(new float[] { 70000f, 1f, 2f }))); + + Assert.NotEmpty(exception.Message); + } + + #endregion + + #region Bulk copy across base types + + [ConditionalTheory(nameof(IsSupported))] + [InlineData("float16", "float32")] + [InlineData("float32", "float16")] + [InlineData("float16", "float16")] + public void BulkCopiesBetweenColumnsOfEitherBaseType(string sourceBaseType, string destinationBaseType) + { + // Unlike a parameter, a bulk copy states the destination's base type in the + // INSERT BULK statement, so the driver converts the payload rather than relying on + // the server, which rejects a size mismatch instead of converting it. + Table source = sourceBaseType == "float16" ? _float16Table : _float32Table; + Table destination = destinationBaseType == "float16" ? _float16Table : _float32Table; + + Insert(source, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + + using SqlConnection sourceConnection = new(_connectionString); + sourceConnection.Open(); + using SqlCommand selectCommand = new($"SELECT {ColumnName} FROM {source.Name}", sourceConnection); + using SqlDataReader sourceReader = selectCommand.ExecuteReader(); + + using SqlConnection destinationConnection = new(_connectionString); + destinationConnection.Open(); + + using (SqlBulkCopy bulkCopy = new(destinationConnection) { DestinationTableName = destination.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(sourceReader); + } + + using SqlCommand verifyCommand = + new($"SELECT TOP 1 {ColumnName} FROM {destination.Name} ORDER BY Id DESC", destinationConnection); + using SqlDataReader verifyReader = verifyCommand.ExecuteReader(); + + Assert.True(verifyReader.Read()); + Assert.Equal([1.5f, 2.5f, 3.5f], verifyReader.GetSqlVector(0).Memory.ToArray()); + } + + [ConditionalFact(nameof(IsSupported))] + public void BulkCopiesJsonStringSourceIntoFloat16Column() + { + // A float16 column reads back as a JSON string where System.Half is unavailable, so + // this is the ordinary table to table path on those frameworks. + DataTable table = new(); + table.Columns.Add(ColumnName, typeof(string)); + table.Rows.Add("[1.5,2.5,3.5]"); + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using (SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = _float16Table.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(table); + } + + using SqlDataReader reader = Select(_float16Table); + Assert.True(reader.Read()); + Assert.Equal([1.5f, 2.5f, 3.5f], reader.GetSqlVector(0).Memory.ToArray()); + } + + [ConditionalFact(nameof(IsSupported))] + public void BulkCopyRejectsValuesOutsideTheFloat16Range() + { + DataTable table = new(); + table.Columns.Add(ColumnName, typeof(SqlVector)); + table.Rows.Add(new SqlVector(new float[] { 70000f, 1f, 2f })); + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = _float16Table.Name }; + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + + Assert.Throws(() => bulkCopy.WriteToServer(table)); + } + + #endregion + + #region Helpers + + private void Insert(Table table, object value) + { + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = + new($"INSERT INTO {table.Name} ({ColumnName}) VALUES ({ParameterName})", connection); + command.Parameters.Add(new SqlParameter(ParameterName, SqlDbTypeExtensions.Vector) { Value = value }); + + Assert.Equal(1, command.ExecuteNonQuery()); + } + + private SqlDataReader Select(Table table) + { + SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = + new($"SELECT TOP 1 {ColumnName} FROM {table.Name} ORDER BY Id DESC", connection); + + return command.ExecuteReader(CommandBehavior.CloseConnection); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _float16Table.Dispose(); + _float32Table.Dispose(); + _managementConnection.Dispose(); + _disposed = true; + + GC.SuppressFinalize(this); + } + + #endregion +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs index 545f8af083..b1170de0bc 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs @@ -233,9 +233,239 @@ public void Null_Property() #endregion + #region Float16 Tests + + #if NET + + [Fact] + public void Float16_Construct_Memory() + { + Half[] data = { (Half)1.5f, (Half)2.5f, (Half)3.5f }; + var vec = new SqlVector(data); + + Assert.False(vec.IsNull); + Assert.Equal(3, vec.Length); + Assert.Equal(data, vec.Memory.ToArray()); + + var ivec = vec as ISqlVector; + Assert.Equal(0x01, ivec.ElementType); + Assert.Equal(0x02, ivec.ElementSize); + Assert.Equal(TdsEnums.VECTOR_HEADER_SIZE + (3 * 2), ivec.Size); + + // The base type is written into the header, and each element occupies two bytes. + Assert.Equal(0x01, ivec.VectorPayload[4]); + Assert.Equal(TdsEnums.VECTOR_HEADER_SIZE + (3 * 2), ivec.VectorPayload.Length); + } + + [Fact] + public void Float16_Construct_Length() + { + var vec = SqlVector.CreateNull(5); + + Assert.True(vec.IsNull); + Assert.Equal(5, vec.Length); + Assert.Equal(SQLMessage.NullString(), vec.GetString()); + + var ivec = vec as ISqlVector; + Assert.Equal(0x01, ivec.ElementType); + Assert.Equal(0x02, ivec.ElementSize); + Assert.Equal(TdsEnums.VECTOR_HEADER_SIZE + (5 * 2), ivec.Size); + } + + [Fact] + public void Float16_Construct_Length_Exceeds_8000() + { + // A float16 vector holds twice as many elements as a float32 one before the + // payload exceeds the maximum size of a TDS packet. + SqlVector.CreateNull(3996); + + Assert.Throws(() => SqlVector.CreateNull(3997)); + } + + [Fact] + public void Float16_GetString_RendersExactValues() + { + // Serialising the elements as Half would instead produce the shortest string which + // round trips to the same Half, which renders 65504 as "65500". + var vec = new SqlVector(new[] { (Half)65504f, (Half)1.5f }); + + Assert.Equal("[65504,1.5]", vec.GetString()); + Assert.Equal("[65504,1.5]", vec.ToString()); + } + + [Fact] + public void Float16_ToString_MatchesFloat32Rendering() + { + // A value which both base types represent exactly renders identically, so callers + // cannot tell the two apart from the rendering alone. + Assert.Equal( + new SqlVector(new[] { 1.5f, 2.5f }).ToString(), + new SqlVector(new[] { (Half)1.5f, (Half)2.5f }).ToString()); + } + + #endif + + [Fact] + public void Float32_ToString_RendersJson() + { + Assert.Equal("[1.5,2.5]", new SqlVector(new[] { 1.5f, 2.5f }).ToString()); + } + + [Fact] + public void ToString_Null_RendersNullString() + { + Assert.Equal(SQLMessage.NullString(), SqlVector.CreateNull(3).ToString()); + } + + #endregion + + #region Payload Conversion Tests + + [Fact] + public void ConvertPayload_Float32ToFloat16() + { + byte[] source = ((ISqlVector)new SqlVector(new[] { 1.5f, 2.5f, 3.5f })).VectorPayload; + + byte[] converted = SqlVector.ConvertPayloadElementType( + source, + (byte)MetaType.SqlVectorElementType.Float16); + + Assert.Equal(0x01, converted[4]); + Assert.Equal(TdsEnums.VECTOR_HEADER_SIZE + (3 * 2), converted.Length); + + // Reading the converted payload back yields the original values, because all three + // are exactly representable in binary16. + byte[] roundTripped = SqlVector.ConvertPayloadElementType( + converted, + (byte)MetaType.SqlVectorElementType.Float32); + + Assert.Equal(new[] { 1.5f, 2.5f, 3.5f }, new SqlVector(roundTripped).Memory.ToArray()); + } + + [Fact] + public void ConvertPayload_Float16ToFloat32_WidensExactly() + { + // Built directly rather than through SqlVector, so that the test also runs on + // frameworks without System.Half. + byte[] source = MakeFloat16Payload(new[] { 1.5f, 2.5f, 3.5f }); + + byte[] converted = SqlVector.ConvertPayloadElementType( + source, + (byte)MetaType.SqlVectorElementType.Float32); + + Assert.Equal(0x00, converted[4]); + Assert.Equal(new[] { 1.5f, 2.5f, 3.5f }, new SqlVector(converted).Memory.ToArray()); + } + + [Fact] + public void ConvertPayload_SameElementType_ReturnsInput() + { + byte[] source = ((ISqlVector)new SqlVector(new[] { 1.5f })).VectorPayload; + + Assert.Same( + source, + SqlVector.ConvertPayloadElementType(source, (byte)MetaType.SqlVectorElementType.Float32)); + } + + [Fact] + public void ConvertPayload_NarrowingRounds() + { + byte[] source = ((ISqlVector)new SqlVector(new[] { 1.1f })).VectorPayload; + + byte[] narrowed = SqlVector.ConvertPayloadElementType( + source, + (byte)MetaType.SqlVectorElementType.Float16); + + byte[] widened = SqlVector.ConvertPayloadElementType( + narrowed, + (byte)MetaType.SqlVectorElementType.Float32); + + Assert.Equal(1.0996094f, new SqlVector(widened).Memory.Span[0]); + } + + [Fact] + public void ConvertPayload_ShortHeader_Throws() + { + Assert.Throws(() => + SqlVector.ConvertPayloadElementType( + new byte[] { 0xA9, 0x01 }, + (byte)MetaType.SqlVectorElementType.Float16)); + } + + [Fact] + public void ConvertPayload_LengthMismatch_Throws() + { + // The header declares two elements, but only one is present. + byte[] source = MakeTdsPayloadStatic( + new byte[] { 0xA9, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00 }, + new[] { 1.5f }); + + Assert.Throws(() => + SqlVector.ConvertPayloadElementType( + source, + (byte)MetaType.SqlVectorElementType.Float16)); + } + + [Fact] + public void ConvertPayload_UnsupportedElementType_Throws() + { + byte[] source = ((ISqlVector)new SqlVector(new[] { 1.5f })).VectorPayload; + + Assert.Throws(() => + SqlVector.ConvertPayloadElementType(source, 0x7F)); + } + + #endregion + + #region Widening Read Tests + + [Fact] + public void FromTdsPayload_WidensFloat16ToFloat32() + { + // This is how a float16 column is read on frameworks without System.Half. + byte[] payload = MakeFloat16Payload(new[] { 1.5f, 2.5f, 3.5f }); + + var vec = SqlVector.FromTdsPayload(payload); + + Assert.Equal(3, vec.Length); + Assert.Equal(new[] { 1.5f, 2.5f, 3.5f }, vec.Memory.ToArray()); + + // The widened vector reports float32, so its base type continues to be determined + // by its element type alone rather than by the payload it was read from. + Assert.Equal(0x00, ((ISqlVector)vec).ElementType); + } + + [Fact] + public void FromTdsPayload_MatchingElementType_ReadsDirectly() + { + byte[] payload = ((ISqlVector)new SqlVector(new[] { 1.5f, 2.5f })).VectorPayload; + + var vec = SqlVector.FromTdsPayload(payload); + + Assert.Equal(new[] { 1.5f, 2.5f }, vec.Memory.ToArray()); + } + + #if NET + + [Fact] + public void FromTdsPayload_NarrowingIsRejected() + { + // Narrowing loses information, so it is never performed implicitly on a read. + byte[] payload = ((ISqlVector)new SqlVector(new[] { 1.5f })).VectorPayload; + + Assert.Throws(() => SqlVector.FromTdsPayload(payload)); + } + + #endif + + #endregion + #region Helpers - private byte[] MakeTdsPayload(byte[] header, ReadOnlyMemory values) + private byte[] MakeTdsPayload(byte[] header, ReadOnlyMemory values) => + MakeTdsPayloadStatic(header, values); + + private static byte[] MakeTdsPayloadStatic(byte[] header, ReadOnlyMemory values) { int length = header.Length + (values.Length * sizeof(float)); byte[] payload = new byte[length]; @@ -247,6 +477,28 @@ private byte[] MakeTdsPayload(byte[] header, ReadOnlyMemory values) } return payload; } - + + /// + /// Builds a float16 vector payload without using System.Half, so that tests + /// which need one can also run on .NET Framework. + /// + private static byte[] MakeFloat16Payload(float[] values) + { + byte[] payload = new byte[TdsEnums.VECTOR_HEADER_SIZE + (values.Length * 2)]; + + payload[0] = 0xA9; + payload[1] = 0x01; + BitConverter.GetBytes((ushort)values.Length).CopyTo(payload, 2); + payload[4] = (byte)MetaType.SqlVectorElementType.Float16; + + for (int i = 0; i < values.Length; i++) + { + BitConverter.GetBytes(Float16Converter.FromSingle(values[i])) + .CopyTo(payload, TdsEnums.VECTOR_HEADER_SIZE + (i * 2)); + } + + return payload; + } + #endregion } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs index a082c8c7e6..3ceb0445d2 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs @@ -831,11 +831,19 @@ public void ConnectionRefusesUnsupportedServerTdsVersion(int major, int minor, i // Test to verify that the server and client negotiate // the common feature extension version. - // MDS currently supports vector feature ext version 0x1. + // MDS currently supports vector feature ext version 0x2, + // which adds the float16 base type to the float32 support in 0x1. [Theory] - [InlineData(true, 0x2, 0x1)] - [InlineData(false, 0x0, 0x0)] + // A server which supports the same version as the client negotiates that version. + [InlineData(true, 0x2, 0x2)] + // A server which supports a later version than the client falls back to the + // client's, since the client cannot interpret anything newer. + [InlineData(true, 0x3, 0x2)] + // A server which supports only the earlier version negotiates that instead. [InlineData(true, 0x1, 0x1)] + // A server which reports no support at all is rejected. + [InlineData(false, 0x0, 0x0)] + // A server which does not acknowledge the feature at all leaves it unnegotiated. [InlineData(true, 0xFF, 0x0)] public void TestConnWithVectorFeatExtVersionNegotiation(bool expectedConnectionResult, byte serverVersion, byte expectedNegotiatedVersion) { @@ -846,7 +854,7 @@ public void TestConnWithVectorFeatExtVersionNegotiation(bool expectedConnectionR server.EnableVectorFeatureExt = serverVersion == 0xFF ? false : true; byte expectedLoginReqFeatureExtId = (byte)TDSFeatureID.VectorSupport; - byte expectedLoginReqFeatureExtVersion = 0x1; + byte expectedLoginReqFeatureExtVersion = 0x2; byte actualLoginReqFeatureExtId = 0; byte actualLoginReqFeatureExtVersion = 0; byte actualFeatureExtAckId = 0; From 734a90aaf375df7589e69cbf57cd7e92ed96f189 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 4 Aug 2026 23:50:54 +0530 Subject: [PATCH 04/24] Document float16 vector support Describes the base types a vector column can have, how they map to SqlVector, and how a float16 column is read and written on .NET Framework, where System.Half does not exist. Also documents the vector feature extension versions and the column metadata properties. Adds a sample covering both frameworks, reading a float16 column as an exact, widened or JSON value, inspecting a column's base type and dimensions, and converting between base types. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .github/instructions/features.instructions.md | 39 +++++ doc/samples/SqlVectorFloat16Example.cs | 164 ++++++++++++++++++ .../Microsoft.Data.SqlTypes/SqlVector.xml | 32 +++- 3 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 doc/samples/SqlVectorFloat16Example.cs diff --git a/.github/instructions/features.instructions.md b/.github/instructions/features.instructions.md index 34262b8db6..7a829eaa59 100644 --- a/.github/instructions/features.instructions.md +++ b/.github/instructions/features.instructions.md @@ -123,6 +123,45 @@ This is a comprehensive reference of supported connection string keywords. | `Json` | `String` | SQL Server 2025+ | | `Vector` | `ISqlVector` | SQL Server 2025+ | +### Vector Base Types + +A `vector` column has a base type, which determines how its elements are stored and +transported. The base type is selected by the type parameter of `SqlVector`. + +| SQL Server base type | `SqlVector` | Element size | Max dimensions | Availability | +|----------------------|----------------|--------------|----------------|--------------| +| `float32` (default) | `SqlVector` | 4 bytes | 1998 | SQL Server 2025+ | +| `float16` | `SqlVector` | 2 bytes | 3996 | SQL Server 2025+ (preview), .NET only | + +Notes: + +- `System.Half` does not exist on .NET Framework, so `SqlVector` cannot be used there. + A `float16` column is instead reported as a string, and can be read either as a JSON array + through `GetString`/`GetSqlString`/`GetFieldValue`, or as a `SqlVector` via + `GetSqlVector`, which widens the elements. Widening from `float16` is exact. +- `float16` requires `ALTER DATABASE SCOPED CONFIGURATION SET PREVIEW_FEATURES = ON` while + it is in preview. +- Conversion between base types is performed by SQL Server for parameters, and by the driver + for `SqlBulkCopy`, where the destination's base type is stated in the `INSERT BULK` + statement. Narrowing to `float16` loses precision, and fails for values outside its range. +- A column's base type and number of dimensions are available from the column schema: + `reader.GetColumnSchema()[i]["VectorBaseType"]` and `["VectorDimensions"]`. Both are `null` + for columns which are not vectors. + +#### Vector Feature Extension Versions + +The vector base types available on a connection are negotiated through the `VECTORSUPPORT` +feature extension (`0x0E`): + +| Version | Meaning | +|---------|---------| +| `0` | The server does not support vectors. Vector columns are returned as `varchar(max)`. | +| `1` | `float32` is supported. Columns with any other base type are returned as `varchar(max)`. | +| `2` | `float16` is supported in addition to `float32`. | + +The driver always requests the highest version it supports, and the server acknowledges the +highest version they have in common. + ## SqlCommand Execution Modes ### ExecuteNonQuery diff --git a/doc/samples/SqlVectorFloat16Example.cs b/doc/samples/SqlVectorFloat16Example.cs new file mode 100644 index 0000000000..4aa3abd0ce --- /dev/null +++ b/doc/samples/SqlVectorFloat16Example.cs @@ -0,0 +1,164 @@ +namespace SqlVectorFloat16Example; + +// VectorFloat16ConsoleApp: Demonstrates working with the float16 base type of the +// SQL Server vector datatype via Microsoft.Data.SqlClient +// +// Highlights: +// - Creates a table with a vector(3, float16) column +// - Inserts vectors using SqlVector on .NET +// - Inserts vectors from .NET Framework, where System.Half is unavailable +// - Reads float16 vectors as SqlVector, as widened SqlVector, and as JSON +// - Inspects a column's base type and number of dimensions +// - Converts between the float16 and float32 base types +// +// Requirements: +// - SQL Server 2025 and above, with PREVIEW_FEATURES enabled for the database +// - Microsoft.Data.SqlClient (7.1.0 and above) +// +using Microsoft.Data; +using Microsoft.Data.SqlClient; +using Microsoft.Data.SqlTypes; +using System; +using System.Data; +using System.Data.Common; +using System.Threading.Tasks; + +class VectorFloat16ConsoleApp +{ + // It is recommended to use a secure connection string in production code with valid cert. + private const string ConnectionString = + "Server=localhost;Database=Demo2;Integrated Security=true;Encrypt=true;TrustServerCertificate=true;"; + + private const string TableName = "[dbo].[VectorFloat16Demo]"; + + static async Task Main() + { + try + { + using var conn = new SqlConnection(ConnectionString); + await conn.OpenAsync(); + + await CreateObjectsAsync(conn); + + await InsertVectorsAsync(conn); + await ReadVectorsAsync(conn); + await ReadColumnMetadataAsync(conn); + await ConvertBetweenBaseTypesAsync(conn); + } + catch (SqlException ex) + { + Console.Error.WriteLine($"SQL ERROR: {ex.Message}"); + } + catch (Exception ex) + { + Console.Error.WriteLine($"ERROR: {ex}"); + } + } + + private static async Task CreateObjectsAsync(SqlConnection conn) + { + // The float16 base type is in preview, so it has to be enabled for the database. + string setup = $@" +ALTER DATABASE SCOPED CONFIGURATION SET PREVIEW_FEATURES = ON; +IF OBJECT_ID(N'{TableName}', N'U') IS NOT NULL DROP TABLE {TableName}; +CREATE TABLE {TableName} +( + Id INT IDENTITY(1,1) PRIMARY KEY, + VectorData vector(3, float16) NULL +);"; + using var cmd = new SqlCommand(setup, conn); + await cmd.ExecuteNonQueryAsync(); + } + + #region InsertFloat16Vectors + private static async Task InsertVectorsAsync(SqlConnection conn) + { + string insertSql = $@"INSERT INTO {TableName}(VectorData) VALUES(@VectorData);"; + using var cmd = new SqlCommand(insertSql, conn); + var p = new SqlParameter("@VectorData", SqlDbTypeExtensions.Vector); + cmd.Parameters.Add(p); + +#if NET + // On .NET, a float16 vector is represented by SqlVector. + p.Value = new SqlVector(new Half[] { (Half)1.5f, (Half)2.5f, (Half)3.5f }); + await cmd.ExecuteNonQueryAsync(); +#endif + + // System.Half is unavailable on .NET Framework, so a float16 vector cannot be + // represented directly there. A vector of single precision values can be used + // instead: SQL Server converts it to the column's base type. The conversion loses + // precision for values which float16 cannot represent exactly, and fails for values + // outside its range, in the same way as inserting a JSON literal does. + p.Value = new SqlVector(new float[] { 4.5f, 5.5f, 6.5f }); + await cmd.ExecuteNonQueryAsync(); + + // A JSON array can also be used, which SQL Server parses directly into the column's + // base type. + cmd.Parameters.Clear(); + cmd.Parameters.Add(new SqlParameter("@VectorData", SqlDbType.VarChar, -1) { Value = "[7.5,8.5,9.5]" }); + await cmd.ExecuteNonQueryAsync(); + + Console.WriteLine("Inserted float16 vectors."); + } + #endregion + + #region ReadFloat16Vectors + private static async Task ReadVectorsAsync(SqlConnection conn) + { + string selectSql = $@"SELECT Id, VectorData FROM {TableName} ORDER BY Id;"; + using var cmd = new SqlCommand(selectSql, conn); + using var reader = await cmd.ExecuteReaderAsync(); + + Console.WriteLine("\nReading rows..."); + while (await reader.ReadAsync()) + { + int id = reader.GetInt32(0); + +#if NET + // On .NET, the column's own base type is available directly. + SqlVector exact = reader.GetSqlVector(1); + Console.WriteLine($" Id={id} as Half: [{string.Join(", ", exact.Memory.ToArray())}]"); +#endif + + // On any framework, the elements can be widened to single precision. Widening + // from float16 is exact, so no information is lost. + SqlVector widened = reader.GetSqlVector(1); + Console.WriteLine($" Id={id} as float: [{string.Join(", ", widened.Memory.ToArray())}]"); + + // The value can also be read as a JSON array. + Console.WriteLine($" Id={id} as JSON: {reader.GetString(1)}"); + } + } + #endregion + + #region ReadVectorColumnMetadata + private static async Task ReadColumnMetadataAsync(SqlConnection conn) + { + using var cmd = new SqlCommand($@"SELECT VectorData FROM {TableName};", conn); + using var reader = await cmd.ExecuteReaderAsync(); + + DbColumn column = reader.GetColumnSchema()[0]; + + // A vector column reports its base type and number of dimensions, which is how an + // application can discover them without querying the server's catalog views. Both + // are null for columns which are not vectors. + Console.WriteLine($"\nColumn base type: {column["VectorBaseType"]}"); + Console.WriteLine($"Column dimensions: {column["VectorDimensions"]}"); + } + #endregion + + #region ConvertBetweenBaseTypes + private static async Task ConvertBetweenBaseTypesAsync(SqlConnection conn) + { + // SQL Server converts between the two base types, so a vector read from a column of + // one base type can be written to a column of the other. + using var cmd = new SqlCommand( + "SELECT CAST(CAST('[1.5,2.5,3.5]' AS vector(3, float16)) AS vector(3, float32));", conn); + using var reader = await cmd.ExecuteReaderAsync(); + + await reader.ReadAsync(); + Console.WriteLine($"\nfloat16 converted to float32: {reader.GetString(0)}"); + } + #endregion +} +// diff --git a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml index 7dea042145..6fec5d6a19 100644 --- a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml +++ b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml @@ -3,6 +3,28 @@ Represents a vector value in SQL Server. + + + The type parameter determines the vector's base type: + corresponds to float32, and corresponds to + float16. No other type is supported. + + + is not available on .NET Framework, so + SqlVector<Half> cannot be used there. A float16 column can still be + read on .NET Framework, either as a JSON array through the string read paths, or as a + SqlVector<float>, which widens the elements to single precision. Widening + is exact, so no information is lost. Values can be written to a float16 column + from .NET Framework as a JSON string, or as a SqlVector<float>, which the + server converts. + + + Conversion between base types is performed by SQL Server, which reports an error if a + value is outside the range the destination can represent. Converting from + float32 to float16 loses precision for values which the narrower type + cannot represent exactly, in the same way as inserting a JSON literal does. + + @@ -37,8 +59,16 @@ Constructs a null vector of the given length. SQL Server requires vector arguments to specify their length even when null. - Vector length must be non-negative. + Vector length must be non-negative, and must not exceed the number of elements which fit + within a single TDS packet: 1998 for float32, or 3996 for float16. + + + Returns the vector values as a JSON array, which is the textual form SQL Server + uses for vector values. Returns "Null" when the vector is null. + + A JSON array containing the vector values. + From 37e871dafc1e48707a36c1cc00a5e097ac994885 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Wed, 12 Aug 2026 21:24:43 +0530 Subject: [PATCH 05/24] Transfer vectors between vector columns as their raw payload Bulk copy read a vector column through the representation the reader surfaces, which is a JSON string on frameworks without System.Half. That round trip is both larger than the payload it encodes and unable to carry a negative zero, because System.Text.Json on .NET Framework serialises one as zero and parses a negative zero literal back as positive zero. Reading the payload directly avoids both. It is chosen once per column, when the source and destination are both vector columns, alongside the existing decimal and streaming decisions. Any difference in base type between the two is still resolved when the value is converted. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 4747e0a427..fd2aa6f504 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -133,7 +133,8 @@ private enum ValueMethod : byte SqlTypeSqlSingle, DataFeedStream, DataFeedText, - DataFeedXml + DataFeedXml, + VectorPayload } // Used to hold column metadata for SqlDataReader case @@ -1221,6 +1222,18 @@ private object GetValueFromSourceRow(int destRowIndex, out bool isSqlType, out b isSqlType = false; isDataFeed = false; + if (_currentRowMetadata[destRowIndex].Method == ValueMethod.VectorPayload) + { + // Transfer the vector as its raw payload, so that no value is + // lost to an intermediate representation and no larger textual + // form is sent. Any difference in base type between the source + // and the destination is resolved when the value is converted. + SqlBinary payload = _sqlDataReaderRowSource.GetSqlBinary(sourceOrdinal); + isNull = payload.IsNull; + + return isNull ? (object)DBNull.Value : payload.Value; + } + object value = _sqlDataReaderRowSource.GetValue(sourceOrdinal); isNull = ((value == null) || (value == DBNull.Value)); if ((!isNull) && (metadata.type == SqlDbType.Udt)) @@ -1530,7 +1543,20 @@ private SourceColumnMetadata GetColumnMetadata(int ordinal) { isSqlType = false; isDataFeed = false; - method = ValueMethod.GetValue; + + // A vector read from another vector column is transferred as its raw payload, + // rather than through the representation the reader would otherwise surface. + // That representation is a JSON string on frameworks without System.Half, + // which is both larger than the payload and unable to carry a negative zero. + if (metadata.type == SqlDbTypeExtensions.Vector && + _sqlDataReaderRowSource?.MetaData[sourceOrdinal].metaType.SqlDbType == SqlDbTypeExtensions.Vector) + { + method = ValueMethod.VectorPayload; + } + else + { + method = ValueMethod.GetValue; + } } return new SourceColumnMetadata(method, isSqlType, isDataFeed); From a5de733c34d3c3bfee03e972865ffdbff4620654 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Wed, 12 Aug 2026 21:36:54 +0530 Subject: [PATCH 06/24] Cover nulls in cross base type vector bulk copy The existing suite covers nulls where the source and destination share a base type, but not where they differ, which is the path that converts the payload. Verified that nulls survive in every combination, interleaved with non-null rows so that a row's nullness cannot be satisfied by position alone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../VectorTest/VectorFloat16BehaviourTests.cs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index b3de295235..f6381f23fb 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -192,6 +192,58 @@ public void BulkCopiesBetweenColumnsOfEitherBaseType(string sourceBaseType, stri Assert.Equal([1.5f, 2.5f, 3.5f], verifyReader.GetSqlVector(0).Memory.ToArray()); } + [ConditionalTheory(nameof(IsSupported))] + [InlineData("float16", "float32")] + [InlineData("float32", "float16")] + [InlineData("float16", "float16")] + public void BulkCopyPreservesNullsBetweenColumnsOfEitherBaseType(string sourceBaseType, string destinationBaseType) + { + Table source = sourceBaseType == "float16" ? _float16Table : _float32Table; + Table destination = destinationBaseType == "float16" ? _float16Table : _float32Table; + + // Interleaved, so that a row's nullness cannot be satisfied by position alone. + Insert(source, DBNull.Value); + Insert(source, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + Insert(source, DBNull.Value); + Insert(source, DBNull.Value); + + using SqlConnection sourceConnection = new(_connectionString); + sourceConnection.Open(); + using SqlCommand selectCommand = + new($"SELECT {ColumnName} FROM {source.Name} ORDER BY Id", sourceConnection); + using SqlDataReader sourceReader = selectCommand.ExecuteReader(); + + using SqlConnection destinationConnection = new(_connectionString); + destinationConnection.Open(); + + using (SqlBulkCopy bulkCopy = new(destinationConnection) { DestinationTableName = destination.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(sourceReader); + } + + using SqlCommand verifyCommand = + new($"SELECT TOP 4 {ColumnName} FROM {destination.Name} ORDER BY Id DESC", destinationConnection); + using SqlDataReader verifyReader = verifyCommand.ExecuteReader(); + + // Read back in descending order, so the expected pattern is the reverse of the + // order the rows were inserted in. + foreach (bool expectedNull in new[] { true, true, false, true }) + { + Assert.True(verifyReader.Read()); + Assert.Equal(expectedNull, verifyReader.IsDBNull(0)); + + if (expectedNull) + { + Assert.Equal(DBNull.Value, verifyReader.GetValue(0)); + } + else + { + Assert.Equal([1.5f, 2.5f, 3.5f], verifyReader.GetSqlVector(0).Memory.ToArray()); + } + } + } + [ConditionalFact(nameof(IsSupported))] public void BulkCopiesJsonStringSourceIntoFloat16Column() { From 3090c139db889658e05cad6c4ff01fe113fe854e Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Mon, 17 Aug 2026 16:37:39 +0530 Subject: [PATCH 07/24] Address review feedback on vector(float16) support Correctness - Reject a narrowing read consistently for null and populated rows. The element type is now checked before the null check in GetSqlVector, so reading a float32 column as SqlVector fails for every row rather than succeeding for the null ones. - Validate the vector header's magic number and version on the widening and payload conversion paths, which previously checked only the length. - Quieten a signalling NaN when widening to single precision, and preserve a NaN's sign and payload in both directions, so the hand written codec and System.Half agree on all 65,536 bit patterns. - Read the redirected scale when converting a bulk copy value, so an encrypted column uses its base type rather than the wrapping metadata's. - Guard against a null MetaData when deciding whether a bulk copy source can supply a raw vector payload. Behaviour - Report a value which cannot be narrowed to float16 during a bulk copy as an OverflowException, rather than saturating it to an infinity and letting the server reject the result as a malformed vector. - Remove the unused Float32VectorType and Float16VectorType capability properties. The negotiated version is still recorded in VectorVersion. A client side guard was considered in their place, but the server already reports an unrecognised base type clearly, so the guard would only have replaced a good error with a worse one, and would have made float16 fail differently from float32 for the same cause. - Return the JSON rendering of a float16 vector as a SqlString from the provider specific accessors, so that every provider specific value remains a type from System.Data.SqlTypes. GetValue continues to return a string. Tests - Run the whole native vector matrix against a float16 column through the single precision representation, which covers .NET Framework, where the hand written codec is the production path rather than a test double. - Assert NaN bitwise rather than skipping it, which is what allowed the codec divergence above to go unnoticed. - Cover reading a float32 column as a narrower vector, for null and populated rows, synchronously and asynchronously. - Exercise the client's own feature extension version ceiling, by letting the simulated server acknowledge a version regardless of what the client requested. The existing case only proved the harness capped the version. - Assert the server's error number for an out of range value rather than accepting any SqlException. - Show the column metadata driving a read for a caller which does not know the schema in advance. - Remove the SqlVector.ToString() override added earlier in this branch. It changed the rendering of the already shipped SqlVector as well, and the reader already exposes the JSON form through GetString and GetFieldValue. The internal GetString is unchanged. - Move Float16Converter into the Microsoft.Data.Common namespace, matching the folder it lives in and its neighbours there. Docs - State that a bulk copy reports an out of range narrowing itself, and correct the SQL Server version for the float32 base type. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .github/instructions/features.instructions.md | 4 +- doc/samples/SqlVectorFloat16Example.cs | 37 ++++ .../Microsoft.Data.SqlTypes/SqlVector.xml | 11 +- .../ref/Microsoft.Data.SqlTypes.cs | 2 - .../src/Microsoft/Data/Common/AdapterUtil.cs | 4 + .../Microsoft/Data/Common/Float16Converter.cs | 37 ++-- .../Connection/ConnectionCapabilities.cs | 16 -- .../src/Microsoft/Data/SqlClient/SqlBuffer.cs | 32 +++- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 7 +- .../Microsoft/Data/SqlClient/SqlDataReader.cs | 23 ++- .../src/Microsoft/Data/SqlTypes/SqlVector.cs | 156 +++++------------ .../Data/SqlTypes/SqlVectorPayload.cs | 162 ++++++++++++++++++ .../src/Resources/Strings.Designer.cs | 9 + .../src/Resources/Strings.resx | 3 + .../NativeVectorFloat16AsSingleTests.cs | 58 +++++++ .../SQL/VectorTest/NativeVectorTestsBase.cs | 70 ++++++-- .../VectorTest/VectorColumnMetadataTests.cs | 40 +++++ .../VectorTest/VectorFloat16BehaviourTests.cs | 120 ++++++++++++- .../Data/SqlTypes/Float16ConverterTest.cs | 51 +++--- .../Microsoft/Data/SqlTypes/SqlVectorTest.cs | 34 ++-- .../SimulatedServerTests/ConnectionTests.cs | 55 +++++- .../tools/TDS/TDS.Servers/GenericTdsServer.cs | 13 +- 22 files changed, 725 insertions(+), 219 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVectorPayload.cs create mode 100644 src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs diff --git a/.github/instructions/features.instructions.md b/.github/instructions/features.instructions.md index 7a829eaa59..a39f9cbf5d 100644 --- a/.github/instructions/features.instructions.md +++ b/.github/instructions/features.instructions.md @@ -143,7 +143,9 @@ Notes: it is in preview. - Conversion between base types is performed by SQL Server for parameters, and by the driver for `SqlBulkCopy`, where the destination's base type is stated in the `INSERT BULK` - statement. Narrowing to `float16` loses precision, and fails for values outside its range. + statement. Narrowing to `float16` loses precision, and fails for values outside its range: + the server reports the failure for a parameter, and the driver throws an + `OverflowException` for a bulk copy. - A column's base type and number of dimensions are available from the column schema: `reader.GetColumnSchema()[i]["VectorBaseType"]` and `["VectorDimensions"]`. Both are `null` for columns which are not vectors. diff --git a/doc/samples/SqlVectorFloat16Example.cs b/doc/samples/SqlVectorFloat16Example.cs index 4aa3abd0ce..b81c552ce8 100644 --- a/doc/samples/SqlVectorFloat16Example.cs +++ b/doc/samples/SqlVectorFloat16Example.cs @@ -144,6 +144,43 @@ private static async Task ReadColumnMetadataAsync(SqlConnection conn) // are null for columns which are not vectors. Console.WriteLine($"\nColumn base type: {column["VectorBaseType"]}"); Console.WriteLine($"Column dimensions: {column["VectorDimensions"]}"); + + // The base type is what a caller which does not know the schema in advance needs in + // order to choose a read path. GetFieldType is not enough on its own: it reports + // string for a float16 column on .NET Framework, which is also what a plain + // varchar column reports, and it cannot distinguish the two base types at all when + // the caller wants to read both as SqlVector. + string baseType = (string)column["VectorBaseType"]; + int dimensions = (int)column["VectorDimensions"]; + + // Allocate once, knowing the length before reading any row. + float[] buffer = new float[dimensions]; + + while (await reader.ReadAsync()) + { + if (reader.IsDBNull(0)) + { + continue; + } + + switch (baseType) + { + case "float32": + reader.GetSqlVector(0).Memory.Span.CopyTo(buffer); + break; + + case "float16": + // Widening from float16 is exact, so single precision is a lossless + // representation for a caller which wants one array type throughout. + reader.GetSqlVector(0).Memory.Span.CopyTo(buffer); + break; + + default: + throw new NotSupportedException($"Unknown vector base type {baseType}."); + } + + Console.WriteLine($"Read {dimensions} {baseType} elements: [{string.Join(",", buffer)}]"); + } } #endregion diff --git a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml index 6fec5d6a19..9927501ce5 100644 --- a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml +++ b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml @@ -22,7 +22,9 @@ Conversion between base types is performed by SQL Server, which reports an error if a value is outside the range the destination can represent. Converting from float32 to float16 loses precision for values which the narrower type - cannot represent exactly, in the same way as inserting a JSON literal does. + cannot represent exactly, in the same way as inserting a JSON literal does. During a + bulk copy the driver performs the conversion instead, and throws an + for a value outside the destination's range. @@ -63,12 +65,5 @@ within a single TDS packet: 1998 for float32, or 3996 for float16. - - - Returns the vector values as a JSON array, which is the textual form SQL Server - uses for vector values. Returns "Null" when the vector is null. - - A JSON array containing the vector values. - diff --git a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlTypes.cs b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlTypes.cs index dd9f2e9559..887b1fc012 100644 --- a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlTypes.cs +++ b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlTypes.cs @@ -90,6 +90,4 @@ public SqlVector(System.ReadOnlyMemory memory) { } public System.ReadOnlyMemory Memory { get { throw null; } } /// public static SqlVector CreateNull(int length) { throw null; } - /// - public override string ToString() { throw null; } } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs index b4a7eb54df..7e01c61025 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs @@ -1257,6 +1257,10 @@ internal static Exception NullOutputParameterValueForVector(string paramName) internal static ArgumentException InvalidVectorHeader() => Argument(StringsHelper.GetString(Strings.ADP_InvalidVectorHeader)); + internal static OverflowException VectorValueOutOfRangeForBaseType(float value, string baseType) + => new OverflowException( + StringsHelper.GetString(Strings.ADP_VectorValueOutOfRangeForBaseType, value, baseType)); + internal static Exception InvalidJsonStringForVector(string value, Exception inner) => InvalidOperation(StringsHelper.GetString(Strings.ADP_InvalidJsonStringForVector, value), inner); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/Float16Converter.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/Float16Converter.cs index 958aa1b22e..a4aed6e83f 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/Float16Converter.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/Float16Converter.cs @@ -3,8 +3,11 @@ // See the LICENSE file in the project root for more information. using System; +#if NETFRAMEWORK +using Microsoft.Data.SqlClient; +#endif -namespace Microsoft.Data.SqlClient +namespace Microsoft.Data.Common { /// /// Converts between IEEE 754 binary16 (half precision) and binary32 (single @@ -43,6 +46,12 @@ internal static class Float16Converter // The number of mantissa bits discarded when narrowing binary32 to binary16. private const int MantissaShift = Binary32MantissaBits - Binary16MantissaBits; + /// + /// The most significant mantissa bit of a binary32 value, which distinguishes a + /// quiet NaN from a signalling one. + /// + private const int Binary32QuietBit = 1 << (Binary32MantissaBits - 1); + /// /// Converts the raw bits of an IEEE 754 binary16 value to the equivalent /// single precision value. The conversion is always exact. @@ -82,11 +91,16 @@ internal static float ManualToSingle(ushort bits) if (exponent == Binary16MaxExponent) { - // Infinity or NaN. A zero mantissa denotes an infinity; any other - // value denotes a NaN, which is canonicalised to float.NaN. + // Infinity or NaN. A NaN keeps its sign and payload, with the quiet bit + // forced, which is what a System.Half conversion produces. A signalling + // NaN is therefore quietened, as IEEE 754 requires of a conversion. return mantissa == 0 ? Int32BitsToSingle((sign << 31) | (Binary32MaxExponent << Binary32MantissaBits)) - : float.NaN; + : Int32BitsToSingle( + (sign << 31) | + (Binary32MaxExponent << Binary32MantissaBits) | + Binary32QuietBit | + (mantissa << MantissaShift)); } if (exponent == 0) @@ -134,9 +148,12 @@ internal static ushort ManualFromSingle(float value) if (exponent == Binary32MaxExponent) { - // Infinity or NaN. NaN is canonicalised to a quiet NaN, matching the - // representation produced by System.Half. - int payload = mantissa == 0 ? 0x7C00 : 0x7E00; + // Infinity or NaN. A NaN keeps its sign and the high bits of its payload, + // with the quiet bit forced, which is what a System.Half conversion produces. + int payload = mantissa == 0 + ? 0x7C00 + : 0x7C00 | 0x0200 | ((mantissa >> MantissaShift) & 0x1FF); + return (ushort)((sign << 15) | payload); } @@ -156,9 +173,9 @@ internal static ushort ManualFromSingle(float value) if (targetExponent <= 0) { - // Too small to represent as a normal value. Values more than eleven - // binade below the smallest subnormal cannot round up to one, so they - // are flushed to zero rather than shifted by more than the mantissa width. + // Too small to represent as a normal value. Values below half of the + // smallest subnormal cannot round up to it, so they are flushed to zero + // rather than shifted by more than the mantissa width. if (targetExponent < -Binary16MantissaBits) { return (ushort)(sign << 15); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/ConnectionCapabilities.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/ConnectionCapabilities.cs index 5b99ccf71d..31e570f048 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/ConnectionCapabilities.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/ConnectionCapabilities.cs @@ -174,22 +174,6 @@ internal sealed class ConnectionCapabilities /// public byte VectorVersion { get; set; } - /// - /// Indicates support for the vector data type, with a backing type - /// of float32. This was introduced in SQL Server 2022, and is only - /// available if a FEATUREEXTACK token of value 0x0E is received, and - /// if the version in this token's data is greater than or equal to 1. - /// - public bool Float32VectorType => VectorVersion >= TdsEnums.VECTOR_VERSION_FLOAT32; - - /// - /// Indicates support for the vector data type, with a backing type - /// of float16. This is only available if a FEATUREEXTACK token of value - /// 0x0E is received, and if the version in this token's data is greater - /// than or equal to 2. - /// - public bool Float16VectorType => VectorVersion >= TdsEnums.VECTOR_VERSION_FLOAT16; - /// /// Indicates support for the json data type. This was introduced in /// SQL Server 2022, and is only available if a FEATUREEXTACK token of value diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs index e5f9035cd4..21de5bedbc 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs @@ -978,13 +978,18 @@ internal SqlVector GetSqlVector() where T : unmanaged { if (_type is StorageType.Vector) { + // The payload's base type may differ from T: a float16 column can be read as + // a vector of single precision values, which is the only strongly typed form + // available on .NET Framework. Validate the pairing before considering + // nullness, so that the outcome depends on the column's base type rather than + // on whether a particular row happens to be null. + SqlVector.ThrowIfNotConvertibleFrom(_value._vectorInfo._elementType); + if (IsNull) { return SqlVector.CreateNull(_value._vectorInfo._elementCount); } - // The payload's base type may differ from T: a float16 column can be - // read as a vector of single precision values, which is the only - // strongly typed form available on .NET Framework. + return SqlVector.FromTdsPayload(SqlBinary.Value); } return (SqlVector)SqlValue; @@ -1046,6 +1051,25 @@ internal object GetVectorValue() } } + /// + /// The provider specific counterpart of . It differs + /// only where a float16 vector is surfaced as its JSON rendering, which is returned + /// as a so that every provider specific value remains a + /// type from System.Data.SqlTypes. + /// + internal object GetVectorSqlValue() + { + #if NET + return GetVectorValue(); + #else + var elementType = (MetaType.SqlVectorElementType)_value._vectorInfo._elementType; + + return elementType == MetaType.SqlVectorElementType.Float16 + ? SqlString + : GetVectorValue(); + #endif + } + internal object SqlValue { get @@ -1081,7 +1105,7 @@ internal object SqlValue case StorageType.Json: return SqlJson; case StorageType.Vector: - return GetVectorValue(); + return GetVectorSqlValue(); case StorageType.SqlCachedBuffer: { SqlCachedBuffer data = (SqlCachedBuffer)(_object); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index fd2aa6f504..cdcc0dcc5e 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -1549,7 +1549,8 @@ private SourceColumnMetadata GetColumnMetadata(int ordinal) // That representation is a JSON string on frameworks without System.Half, // which is both larger than the payload and unable to carry a negative zero. if (metadata.type == SqlDbTypeExtensions.Vector && - _sqlDataReaderRowSource?.MetaData[sourceOrdinal].metaType.SqlDbType == SqlDbTypeExtensions.Vector) + _sqlDataReaderRowSource?.MetaData is { } sourceMetaData && + sourceMetaData[sourceOrdinal].metaType.SqlDbType == SqlDbTypeExtensions.Vector) { method = ValueMethod.VectorPayload; } @@ -1752,7 +1753,7 @@ private static object ConvertVectorToBaseType(object value, byte destinationElem // The payload is converted directly rather than through a strongly typed vector, // so that .NET Framework, which has no System.Half, can also write to float16 // destinations. - return SqlTypes.SqlVector.ConvertPayloadElementType(payload, destinationElementType); + return SqlTypes.SqlVectorPayload.ConvertElementType(payload, destinationElementType); } private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, ref bool isSqlType, out bool coercedToDataFeed) @@ -1853,7 +1854,7 @@ private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, re // uses the source value's own base type: a JSON string always yields // float32, which is how a float16 column reads back on frameworks // without System.Half. - value = ConvertVectorToBaseType(value, metadata.scale); + value = ConvertVectorToBaseType(value, scale); break; case TdsEnums.SQLINTN: diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs index 6c884493d7..e0ac1bcf1c 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs @@ -1323,6 +1323,27 @@ private static Type GetVectorFieldType(byte vectorElementType) }; } +#if !NETFRAMEWORK + [return: DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.PublicFields)] +#endif + private static Type GetVectorProviderSpecificFieldType(byte vectorElementType) + { + MetaType.SqlVectorElementType elementType = (MetaType.SqlVectorElementType)vectorElementType; + return elementType switch + { + MetaType.SqlVectorElementType.Float32 => typeof(SqlVector), + // As above, but the provider specific accessors return types from + // System.Data.SqlTypes, so the JSON rendering is reported as a SqlString + // rather than as a CLR string. + #if NET + MetaType.SqlVectorElementType.Float16 => typeof(SqlVector), + #else + MetaType.SqlVectorElementType.Float16 => typeof(SqlString), + #endif + _ => throw SQL.VectorTypeNotSupported(elementType.ToString()), + }; + } + internal virtual int GetLocaleId(int i) { _SqlMetaData sqlMetaData = MetaData[i]; @@ -1410,7 +1431,7 @@ private Type GetProviderSpecificFieldTypeInternal(_SqlMetaData metaData) } else if (metaData.type == SqlDbTypeExtensions.Vector) { - providerSpecificFieldType = GetVectorFieldType(metaData.scale); + providerSpecificFieldType = GetVectorProviderSpecificFieldType(metaData.scale); } else { diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs index 88614a0049..02a7372434 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs @@ -21,12 +21,12 @@ namespace Microsoft.Data.SqlTypes; { #region Constants - private const byte VecHeaderMagicNo = 0xA9; - private const byte VecVersionNo = 0x01; + private const byte VecHeaderMagicNo = SqlVectorPayload.HeaderMagicNumber; + private const byte VecVersionNo = SqlVectorPayload.HeaderVersion; // Offsets of the fields within the vector header. Refer to TDS section 2.2.5.5.7. - private const int VecHeaderLengthOffset = 2; - private const int VecHeaderElementTypeOffset = 4; + private const int VecHeaderLengthOffset = SqlVectorPayload.LengthOffset; + private const int VecHeaderElementTypeOffset = SqlVectorPayload.ElementTypeOffset; #endregion @@ -126,9 +126,6 @@ internal string GetString() return JsonSerializer.Serialize(Memory); } - /// - public override string ToString() => GetString(); - /// /// Creates a vector from a TDS payload, converting the elements when the payload's /// base type differs from . @@ -148,30 +145,61 @@ internal string GetString() /// internal static SqlVector FromTdsPayload(byte[] tdsBytes) { - if (tdsBytes.Length < TdsEnums.VECTOR_HEADER_SIZE) + ThrowIfHeaderInvalid(tdsBytes); + ThrowIfNotConvertibleFrom(tdsBytes[VecHeaderElementTypeOffset]); + + if (tdsBytes[VecHeaderElementTypeOffset] == ElementTypeOf()) { - throw ADP.InvalidVectorHeader(); + return new SqlVector(tdsBytes); } - byte payloadElementType = tdsBytes[VecHeaderElementTypeOffset]; - (byte targetElementType, _, _) = GetTypeFieldsOrThrow(); + return new SqlVector(WidenFloat16Payload(tdsBytes)); + } + + /// + /// Throws if a payload with the given base type cannot be surfaced as a vector of + /// . + /// + /// + /// Whether a payload can be read as a given element type is a property of the column's + /// base type, so this is checked before a value is examined. A null value would + /// otherwise appear to succeed where a populated one in the same column fails. + /// + internal static void ThrowIfNotConvertibleFrom(byte payloadElementType) + { + byte targetElementType = ElementTypeOf(); if (payloadElementType == targetElementType) { - return new SqlVector(tdsBytes); + return; } + // Widening is exact, so it is performed implicitly. Narrowing is lossy and never is. if (payloadElementType == (byte)MetaType.SqlVectorElementType.Float16 && targetElementType == (byte)MetaType.SqlVectorElementType.Float32) { - return new SqlVector(WidenFloat16Payload(tdsBytes)); + return; } - // Any other combination would be a narrowing conversion, which is lossy and so - // is never performed implicitly. throw SQL.VectorTypeNotSupported(typeof(T).FullName); } + /// + /// Validates the fields of a vector header which do not depend on the base type. + /// + private static void ThrowIfHeaderInvalid(byte[] tdsBytes) => + SqlVectorPayload.ThrowIfHeaderInvalid(tdsBytes); + + /// + /// Returns the vector base type which corresponds to . + /// + private static byte ElementTypeOf() + { + (byte elementType, _, _) = GetTypeFieldsOrThrow(); + + return elementType; + } + /// /// Widens the float16 elements of a TDS payload to single precision values. /// @@ -402,103 +430,5 @@ private T[] MakeArray() return result; } - /// - /// Rewrites a TDS vector payload so that its elements use the requested base type. - /// - /// - /// This works on every target framework, including those without System.Half, - /// because it produces a raw payload rather than a strongly typed vector. It is used by - /// bulk copy, where the base type written to the wire must match the destination - /// column's: the server reports a mismatch as a column length error rather than - /// converting the value, because binary16 and binary32 elements differ in size. - /// - internal static byte[] ConvertPayloadElementType(byte[] tdsBytes, byte targetElementType) - { - if (tdsBytes.Length < TdsEnums.VECTOR_HEADER_SIZE) - { - throw ADP.InvalidVectorHeader(); - } - - byte sourceElementType = tdsBytes[VecHeaderElementTypeOffset]; - - if (sourceElementType == targetElementType) - { - return tdsBytes; - } - - int length = BinaryPrimitives.ReadUInt16LittleEndian(tdsBytes.AsSpan(VecHeaderLengthOffset)); - int sourceElementSize = MetaType.GetVectorElementSize(sourceElementType); - int targetElementSize = MetaType.GetVectorElementSize(targetElementType); - - if (tdsBytes.Length != TdsEnums.VECTOR_HEADER_SIZE + (sourceElementSize * length)) - { - throw ADP.InvalidVectorHeader(); - } - - byte[] result = new byte[TdsEnums.VECTOR_HEADER_SIZE + (targetElementSize * length)]; - - result[0] = VecHeaderMagicNo; - result[1] = VecVersionNo; - BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(VecHeaderLengthOffset), (ushort)length); - result[VecHeaderElementTypeOffset] = targetElementType; - - for (int i = 0, - sourcePosition = TdsEnums.VECTOR_HEADER_SIZE, - targetPosition = TdsEnums.VECTOR_HEADER_SIZE; - i < length; - i++, sourcePosition += sourceElementSize, targetPosition += targetElementSize) - { - // Every supported base type widens to single precision without loss, so it - // serves as the common representation for the conversion. - WriteElement( - result, - targetPosition, - targetElementType, - ReadElement(tdsBytes, sourcePosition, sourceElementType)); - } - - return result; - } - - private static float ReadElement(byte[] payload, int position, byte elementType) - { - switch ((MetaType.SqlVectorElementType)elementType) - { - case MetaType.SqlVectorElementType.Float32: - #if NET - return BinaryPrimitives.ReadSingleLittleEndian(payload.AsSpan(position)); - #else - return BitConverterCompatible.Int32BitsToSingle(BinaryPrimitives.ReadInt32LittleEndian(payload.AsSpan(position))); - #endif - - case MetaType.SqlVectorElementType.Float16: - return Float16Converter.ToSingle(BinaryPrimitives.ReadUInt16LittleEndian(payload.AsSpan(position))); - - default: - throw SQL.VectorTypeNotSupported(elementType.ToString()); - } - } - - private static void WriteElement(byte[] payload, int position, byte elementType, float value) - { - switch ((MetaType.SqlVectorElementType)elementType) - { - case MetaType.SqlVectorElementType.Float32: - #if NET - BinaryPrimitives.WriteSingleLittleEndian(payload.AsSpan(position), value); - #else - BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan(position), BitConverterCompatible.SingleToInt32Bits(value)); - #endif - break; - - case MetaType.SqlVectorElementType.Float16: - BinaryPrimitives.WriteUInt16LittleEndian(payload.AsSpan(position), Float16Converter.FromSingle(value)); - break; - - default: - throw SQL.VectorTypeNotSupported(elementType.ToString()); - } - } - #endregion } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVectorPayload.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVectorPayload.cs new file mode 100644 index 0000000000..b572cb28d5 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVectorPayload.cs @@ -0,0 +1,162 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Buffers.Binary; +using Microsoft.Data.Common; +using Microsoft.Data.SqlClient; + +#nullable enable + +namespace Microsoft.Data.SqlTypes; + +/// +/// Operations on the TDS representation of a vector which do not depend on the type its +/// elements are surfaced as. +/// +/// +/// Refer to TDS section 2.2.5.5.7 for the layout of a vector. +/// +internal static class SqlVectorPayload +{ + #region Constants + + internal const byte HeaderMagicNumber = 0xA9; + internal const byte HeaderVersion = 0x01; + + // Offsets of the fields within the vector header. + internal const int LengthOffset = 2; + internal const int ElementTypeOffset = 4; + + /// + /// The exponent field of a binary16 value, which is all ones for an infinity or a NaN. + /// + private const ushort Binary16ExponentMask = 0x7C00; + + #endregion + + #region Methods + + /// + /// Validates the fields of a vector header which do not depend on its base type. + /// + internal static void ThrowIfHeaderInvalid(byte[] tdsBytes) + { + if (tdsBytes.Length < TdsEnums.VECTOR_HEADER_SIZE || + tdsBytes[0] != HeaderMagicNumber || + tdsBytes[1] != HeaderVersion) + { + throw ADP.InvalidVectorHeader(); + } + } + + /// + /// Rewrites a TDS vector payload so that its elements use the requested base type, + /// returning the original payload when it already does. + /// + /// + /// This works on every target framework, including those without System.Half, + /// because it produces a raw payload rather than a strongly typed vector. It is used by + /// bulk copy, where the base type written to the wire must match the destination + /// column's: the server reports a mismatch as a column length error rather than + /// converting the value, because binary16 and binary32 elements differ in size. + /// + internal static byte[] ConvertElementType(byte[] tdsBytes, byte targetElementType) + { + ThrowIfHeaderInvalid(tdsBytes); + + byte sourceElementType = tdsBytes[ElementTypeOffset]; + + if (sourceElementType == targetElementType) + { + return tdsBytes; + } + + int length = BinaryPrimitives.ReadUInt16LittleEndian(tdsBytes.AsSpan(LengthOffset)); + int sourceElementSize = MetaType.GetVectorElementSize(sourceElementType); + int targetElementSize = MetaType.GetVectorElementSize(targetElementType); + + if (tdsBytes.Length != TdsEnums.VECTOR_HEADER_SIZE + (sourceElementSize * length)) + { + throw ADP.InvalidVectorHeader(); + } + + byte[] result = new byte[TdsEnums.VECTOR_HEADER_SIZE + (targetElementSize * length)]; + + result[0] = HeaderMagicNumber; + result[1] = HeaderVersion; + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(LengthOffset), (ushort)length); + result[ElementTypeOffset] = targetElementType; + + for (int i = 0, + sourcePosition = TdsEnums.VECTOR_HEADER_SIZE, + targetPosition = TdsEnums.VECTOR_HEADER_SIZE; + i < length; + i++, sourcePosition += sourceElementSize, targetPosition += targetElementSize) + { + // Every supported base type widens to single precision without loss, so it + // serves as the common representation for the conversion. + WriteElement( + result, + targetPosition, + targetElementType, + ReadElement(tdsBytes, sourcePosition, sourceElementType)); + } + + return result; + } + + internal static float ReadElement(byte[] payload, int position, byte elementType) + { + switch ((MetaType.SqlVectorElementType)elementType) + { + case MetaType.SqlVectorElementType.Float32: + #if NET + return BinaryPrimitives.ReadSingleLittleEndian(payload.AsSpan(position)); + #else + return BitConverterCompatible.Int32BitsToSingle(BinaryPrimitives.ReadInt32LittleEndian(payload.AsSpan(position))); + #endif + + case MetaType.SqlVectorElementType.Float16: + return Float16Converter.ToSingle(BinaryPrimitives.ReadUInt16LittleEndian(payload.AsSpan(position))); + + default: + throw SQL.VectorTypeNotSupported(elementType.ToString()); + } + } + + internal static void WriteElement(byte[] payload, int position, byte elementType, float value) + { + switch ((MetaType.SqlVectorElementType)elementType) + { + case MetaType.SqlVectorElementType.Float32: + #if NET + BinaryPrimitives.WriteSingleLittleEndian(payload.AsSpan(position), value); + #else + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan(position), BitConverterCompatible.SingleToInt32Bits(value)); + #endif + break; + + case MetaType.SqlVectorElementType.Float16: + ushort narrowed = Float16Converter.FromSingle(value); + + // The codec saturates, so a finite input can become an infinity. The server + // rejects that as a malformed vector rather than as an out of range value, + // which is a far less useful diagnostic than reporting it here. + if (!float.IsNaN(value) && !float.IsInfinity(value) && + (narrowed & Binary16ExponentMask) == Binary16ExponentMask) + { + throw ADP.VectorValueOutOfRangeForBaseType(value, "float16"); + } + + BinaryPrimitives.WriteUInt16LittleEndian(payload.AsSpan(position), narrowed); + break; + + default: + throw SQL.VectorTypeNotSupported(elementType.ToString()); + } + } + + #endregion +} diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs index 7064a6c19c..de55b10c6e 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs @@ -600,6 +600,15 @@ internal static string ADP_InvalidValue { } } + /// + /// Looks up a localized string similar to The value {0} cannot be represented by a vector with a base type of {1}.. + /// + internal static string ADP_VectorValueOutOfRangeForBaseType { + get { + return ResourceManager.GetString("ADP_VectorValueOutOfRangeForBaseType", resourceCulture); + } + } + /// /// Looks up a localized string similar to Invalid vector header received.. /// diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx index e7f438873f..8c4c55ba7a 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx @@ -2148,6 +2148,9 @@ 'null' value not supported for output parameter '{0}' of SqlDbtype Vector. + + The value {0} cannot be represented by a vector with a base type of {1}. + Invalid vector header received. diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs new file mode 100644 index 0000000000..a6c78d22b1 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs @@ -0,0 +1,58 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.VectorTest; + +#nullable enable + +public sealed class VectorFloat16AsSingleTestData : NativeVectorTestDataBase +{ + // Every value is exactly representable in binary16, so it survives narrowing on the + // way to the server and widening on the way back. + public override float[] SampleScalarData => [1.5f, 2.25f, -3.75f, 65504f, 0.125f, -0.0f]; + + public override float[,] SampleDataSet + { + get + { + float[,] sampleData = new float[10, ValidSampleScalarDataLength]; + + for (int i = 0; i < sampleData.GetLength(0); i++) + { + float baseValue = i * 10; + + for (int j = 0; j < sampleData.GetLength(1); j++) + { + // Eighths are exactly representable in binary16 at this magnitude. + sampleData[i, j] = baseValue + (j * 0.125f); + } + } + + return sampleData; + } + } + + public override int IncorrectScalarDataParameterSize => 3234; + + public override bool IsSupported => DataTestUtility.IsSqlVectorFloat16Supported; + + public override string SqlServerTypeName => "float16"; + + // The column's base type is float16, so single precision is a widening representation + // which the caller has to ask for rather than the driver's default. + public override bool IsDefaultRepresentation => false; +} + +/// +/// Runs the full native vector matrix against a float16 column using the single +/// precision representation. This is the only representation available on .NET Framework, +/// which has no System.Half, and is also where the hand written binary16 codec is +/// the production path rather than a test double. +/// +[Trait("Set", "3")] +public sealed class NativeVectorFloat16AsSingleTests : NativeVectorTestsBase +{ +} diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs index 5c9970f6ed..7018737b81 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs @@ -34,6 +34,14 @@ public abstract class NativeVectorTestDataBase public abstract string SqlServerTypeName { get; } + /// + /// Whether is how the driver represents the column + /// by default. This is false when the column's base type is narrower than + /// , so that reads are widening conversions which + /// the caller has to ask for explicitly. + /// + public virtual bool IsDefaultRepresentation => true; + public int ValidSampleScalarDataLength => SampleScalarData.Length; public IEnumerable TestData => @@ -222,13 +230,23 @@ private void ValidateInsertedData(SqlConnection connection, TElement[] expectedD // For both null and non-null cases, validate the SqlVector object ValidateSqlVectorObject(reader.IsDBNull(0), (SqlVector)reader.GetSqlVector(0), expectedData, expectedLength); ValidateSqlVectorObject(reader.IsDBNull(0), reader.GetFieldValue>(0), expectedData, expectedLength); - ValidateSqlVectorObject(reader.IsDBNull(0), (SqlVector)reader.GetSqlValue(0), expectedData, expectedLength); + + if (TestDataInstance.IsDefaultRepresentation) + { + ValidateSqlVectorObject(reader.IsDBNull(0), (SqlVector)reader.GetSqlValue(0), expectedData, expectedLength); + } if (!reader.IsDBNull(0)) { - ValidateSqlVectorObject(reader.IsDBNull(0), (SqlVector)reader.GetValue(0), expectedData, expectedLength); - ValidateSqlVectorObject(reader.IsDBNull(0), (SqlVector)reader[0], expectedData, expectedLength); - ValidateSqlVectorObject(reader.IsDBNull(0), (SqlVector)reader[VectorColumnName], expectedData, expectedLength); + if (TestDataInstance.IsDefaultRepresentation) + { + // The untyped accessors return the column's default representation, which + // is only a SqlVector when TElement is the column's base type. + ValidateSqlVectorObject(reader.IsDBNull(0), (SqlVector)reader.GetValue(0), expectedData, expectedLength); + ValidateSqlVectorObject(reader.IsDBNull(0), (SqlVector)reader[0], expectedData, expectedLength); + ValidateSqlVectorObject(reader.IsDBNull(0), (SqlVector)reader[VectorColumnName], expectedData, expectedLength); + } + Assert.Equal(expectedData, JsonSerializer.Deserialize(reader.GetString(0))); Assert.Equal(expectedData, JsonSerializer.Deserialize(reader.GetSqlString(0).Value)); Assert.Equal(expectedData, JsonSerializer.Deserialize(reader.GetFieldValue(0))); @@ -274,13 +292,21 @@ private async Task ValidateInsertedDataAsync(SqlConnection connection, TElement[ // For both null and non-null cases, validate the SqlVector object ValidateSqlVectorObject(await reader.IsDBNullAsync(0), (SqlVector)reader.GetSqlVector(0), expectedData, expectedLength); ValidateSqlVectorObject(await reader.IsDBNullAsync(0), await reader.GetFieldValueAsync>(0), expectedData, expectedLength); - ValidateSqlVectorObject(await reader.IsDBNullAsync(0), (SqlVector)reader.GetSqlValue(0), expectedData, expectedLength); + + if (TestDataInstance.IsDefaultRepresentation) + { + ValidateSqlVectorObject(await reader.IsDBNullAsync(0), (SqlVector)reader.GetSqlValue(0), expectedData, expectedLength); + } if (!await reader.IsDBNullAsync(0)) { - ValidateSqlVectorObject(await reader.IsDBNullAsync(0), (SqlVector)reader.GetValue(0), expectedData, expectedLength); - ValidateSqlVectorObject(await reader.IsDBNullAsync(0), (SqlVector)reader[0], expectedData, expectedLength); - ValidateSqlVectorObject(await reader.IsDBNullAsync(0), (SqlVector)reader[VectorColumnName], expectedData, expectedLength); + if (TestDataInstance.IsDefaultRepresentation) + { + ValidateSqlVectorObject(await reader.IsDBNullAsync(0), (SqlVector)reader.GetValue(0), expectedData, expectedLength); + ValidateSqlVectorObject(await reader.IsDBNullAsync(0), (SqlVector)reader[0], expectedData, expectedLength); + ValidateSqlVectorObject(await reader.IsDBNullAsync(0), (SqlVector)reader[VectorColumnName], expectedData, expectedLength); + } + Assert.Equal(expectedData, JsonSerializer.Deserialize(reader.GetString(0))); Assert.Equal(expectedData, JsonSerializer.Deserialize(reader.GetSqlString(0).Value)); Assert.Equal(expectedData, JsonSerializer.Deserialize(await reader.GetFieldValueAsync(0))); @@ -615,17 +641,27 @@ public void TestGetFieldTypeReturnsSqlVectorForVectorColumn() using SqlCommand selectCmd = new(_selectCommand, connection); using SqlDataReader reader = selectCmd.ExecuteReader(); - // Verify GetFieldType returns SqlVector for the vector column - Assert.Equal(typeof(SqlVector), reader.GetFieldType(0)); + if (TestDataInstance.IsDefaultRepresentation) + { + // Verify GetFieldType returns SqlVector for the vector column + Assert.Equal(typeof(SqlVector), reader.GetFieldType(0)); - // Verify GetProviderSpecificFieldType also returns SqlVector - Assert.Equal(typeof(SqlVector), reader.GetProviderSpecificFieldType(0)); + // Verify GetProviderSpecificFieldType also returns SqlVector + Assert.Equal(typeof(SqlVector), reader.GetProviderSpecificFieldType(0)); - // Verify that GetValue returns an instance consistent with GetFieldType - Assert.True(reader.Read(), "No data found in the table."); - object value = reader.GetValue(0); - Assert.IsType>(value); - Assert.Equal(TestDataInstance.SampleScalarData, ((SqlVector)value).Memory.ToArray()); + // Verify that GetValue returns an instance consistent with GetFieldType + Assert.True(reader.Read(), "No data found in the table."); + object value = reader.GetValue(0); + Assert.IsType>(value); + Assert.Equal(TestDataInstance.SampleScalarData, ((SqlVector)value).Memory.ToArray()); + } + else + { + // The column's default representation is not SqlVector, but the + // caller can still ask for one, so only the explicit accessor is checked. + Assert.True(reader.Read(), "No data found in the table."); + Assert.NotEqual(typeof(SqlVector), reader.GetFieldType(0)); + } // Verify GetFieldValue> returns the correct typed value SqlVector typedValue = reader.GetFieldValue>(0); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs index 5f345be99e..9b2411e179 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs @@ -22,6 +22,8 @@ public sealed class VectorColumnMetadataTests public static bool IsSupported => DataTestUtility.IsSqlVectorSupported; + public static bool IsFloat16Supported => DataTestUtility.IsSqlVectorFloat16Supported; + [ConditionalTheory(nameof(IsSupported))] [InlineData(1)] [InlineData(3)] @@ -102,4 +104,42 @@ public void SchemaCollectionIncludesVectorType() Assert.Equal((int)SqlDbTypeExtensions.Vector, vectorRow!["ProviderDbType"]); Assert.Equal("vector({0})", vectorRow["CreateFormat"]); } + + [ConditionalFact(nameof(IsFloat16Supported))] + public void DrivesReadPathForACallerWhichDoesNotKnowTheSchema() + { + // This is the use case the properties exist for. GetFieldType is not enough on its + // own: it reports string for a float16 column on .NET Framework, which is what a + // varchar column reports too, and it cannot distinguish the two base types at all + // for a caller which wants to read both through one representation. + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = new( + @"SELECT CAST('[1.5,2.5,3.5]' AS vector(3, float16)) AS v + UNION ALL + SELECT CAST('[1.5,2.5,3.5]' AS vector(3, float16))", connection); + using SqlDataReader reader = command.ExecuteReader(); + + DbColumn column = reader.GetColumnSchema()[0]; + string baseType = Assert.IsType(column["VectorBaseType"]); + int dimensions = Assert.IsType(column["VectorDimensions"]); + + Assert.Equal("float16", baseType); + Assert.Equal(3, dimensions); + + // The dimension count is known before any row is read, so a buffer can be sized once. + float[] buffer = new float[dimensions]; + + while (reader.Read()) + { + Assert.Contains(baseType, new[] { "float16", "float32" }); + + // Widening from float16 is exact, so single precision reads both base types + // without loss, and is the only option where System.Half is unavailable. + reader.GetSqlVector(0).Memory.Span.CopyTo(buffer); + + Assert.Equal(new float[] { 1.5f, 2.5f, 3.5f }, buffer); + } + } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index f6381f23fb..0d20cedf0a 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; +using System.Data.SqlTypes; +using System.Threading.Tasks; using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Microsoft.Data.SqlTypes; using Xunit; @@ -25,6 +27,11 @@ public sealed class VectorFloat16BehaviourTests : IDisposable private const string ColumnName = "VectorData"; private const string ParameterName = "@VectorData"; + /// + /// The server's error for a float32 value which cannot be narrowed to float16. + /// + private const int Float32ToFloat16OutOfRangeError = 42284; + private readonly string _connectionString = DataTestUtility.TCPConnectionString; private readonly SqlConnection _managementConnection; private readonly Table _float16Table; @@ -98,7 +105,7 @@ public void RendersValuesExactlyRatherThanShortestRoundTrip() Assert.True(reader.Read()); Assert.Equal("[65504,1,2]", reader.GetString(0)); - Assert.Equal("[65504,1,2]", reader.GetValue(0).ToString()); + Assert.Equal("[65504,1,2]", reader.GetFieldValue(0)); } [ConditionalFact(nameof(IsSupported))] @@ -113,6 +120,102 @@ public void ReportsUnsupportedElementTypes() Assert.Throws(() => reader.GetSqlVector(0)); } + [ConditionalFact(nameof(IsSupported))] + public void ReportsProviderSpecificValueAsASqlType() + { + // Every provider specific value is a type from System.Data.SqlTypes, including the + // JSON rendering a float16 column falls back to where System.Half is unavailable. + Insert(_float16Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + + using SqlDataReader reader = Select(_float16Table); + Assert.True(reader.Read()); + + #if NET + Assert.IsType>(reader.GetProviderSpecificValue(0)); + Assert.Equal(typeof(SqlVector), reader.GetProviderSpecificFieldType(0)); + #else + SqlString value = Assert.IsType(reader.GetProviderSpecificValue(0)); + Assert.Equal("[1.5,2.5,3.5]", value.Value); + Assert.Equal(typeof(SqlString), reader.GetProviderSpecificFieldType(0)); + + // GetValue is the CLR path, so it keeps returning a plain string. + Assert.IsType(reader.GetValue(0)); + Assert.Equal(typeof(string), reader.GetFieldType(0)); + #endif + } + + [ConditionalFact(nameof(IsSupported))] + public void ReportsNarrowingReadsConsistentlyForNullAndNonNullRows() + { + // The base type pairing is a property of the column, so a null row has to be + // rejected the same way a populated one is. Reading a float32 column as a vector of + // a narrower element type is not supported in either case. + Insert(_float32Table, DBNull.Value); + Insert(_float32Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = + new($"SELECT {ColumnName} FROM {_float32Table.Name} ORDER BY Id DESC", connection); + using SqlDataReader reader = command.ExecuteReader(); + + // The populated row is read first, then the null one, so that a failure identifies + // which of the two diverged. + Assert.True(reader.Read()); + Assert.False(reader.IsDBNull(0)); + AssertNarrowingReadIsRejected(reader); + + Assert.True(reader.Read()); + Assert.True(reader.IsDBNull(0)); + AssertNarrowingReadIsRejected(reader); + } + + [ConditionalFact(nameof(IsSupported))] + public async Task ReportsNarrowingReadsConsistentlyForNullAndNonNullRowsAsync() + { + Insert(_float32Table, DBNull.Value); + Insert(_float32Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + + using SqlConnection connection = new(_connectionString); + await connection.OpenAsync(); + + using SqlCommand command = + new($"SELECT {ColumnName} FROM {_float32Table.Name} ORDER BY Id DESC", connection); + using SqlDataReader reader = await command.ExecuteReaderAsync(); + + Assert.True(await reader.ReadAsync()); + Assert.False(await reader.IsDBNullAsync(0)); + await AssertNarrowingReadIsRejectedAsync(reader); + + Assert.True(await reader.ReadAsync()); + Assert.True(await reader.IsDBNullAsync(0)); + await AssertNarrowingReadIsRejectedAsync(reader); + } + + private static void AssertNarrowingReadIsRejected(SqlDataReader reader) + { + #if NET + Assert.Throws(() => reader.GetSqlVector(0)); + Assert.Throws(() => reader.GetFieldValue>(0)); + #endif + + // double is never a vector base type, so it stands in for the narrowing case on + // frameworks without System.Half. + Assert.Throws(() => reader.GetSqlVector(0)); + } + + private static async Task AssertNarrowingReadIsRejectedAsync(SqlDataReader reader) + { + #if NET + await Assert.ThrowsAsync( + async () => await reader.GetFieldValueAsync>(0)); + #else + await Task.CompletedTask; + Assert.Throws(() => reader.GetSqlVector(0)); + #endif + } + #endregion #region Writing across base types @@ -145,11 +248,12 @@ public void WritesVectorParameterToColumnOfEitherBaseType(string columnBaseType, [ConditionalFact(nameof(IsSupported))] public void RejectsValuesOutsideTheFloat16Range() { - // The server reports the failure rather than silently saturating the value. + // The value is sent as float32 and narrowed by the server, which reports the + // overflow rather than silently saturating. SqlException exception = Assert.Throws(() => Insert(_float16Table, new SqlVector(new float[] { 70000f, 1f, 2f }))); - Assert.NotEmpty(exception.Message); + Assert.Equal(Float32ToFloat16OutOfRangeError, exception.Number); } #endregion @@ -280,7 +384,15 @@ public void BulkCopyRejectsValuesOutsideTheFloat16Range() using SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = _float16Table.Name }; bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); - Assert.Throws(() => bulkCopy.WriteToServer(table)); + // The client narrows the payload here, so it reports the overflow itself rather + // than letting the saturated infinity reach the server, which would reject it as a + // malformed vector instead. Bulk copy wraps the failure to name the column and row. + InvalidOperationException exception = + Assert.Throws(() => bulkCopy.WriteToServer(table)); + + OverflowException overflow = Assert.IsType(exception.InnerException); + Assert.Contains("float16", overflow.Message); + Assert.Contains("70000", overflow.Message); } #endregion diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs index f4e1075958..2060c4f9bc 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using Microsoft.Data.Common; using Microsoft.Data.SqlClient; using Xunit; @@ -36,16 +37,12 @@ public void ToSingle_MatchesHalf_ForEveryBitPattern() float expected = (float)BitConverter.UInt16BitsToHalf(bits); float actual = Float16Converter.ManualToSingle(bits); - if (float.IsNaN(expected)) - { - Assert.True(float.IsNaN(actual), $"0x{bits:X4} should convert to NaN."); - continue; - } - - // Compared bitwise so that positive and negative zero are distinguished. + // Compared bitwise so that positive and negative zero are distinguished, and so + // that a NaN's sign and payload are compared rather than only its NaN-ness. Assert.True( BitConverter.SingleToInt32Bits(expected) == BitConverter.SingleToInt32Bits(actual), - $"0x{bits:X4} converted to {actual} but Half converts it to {expected}."); + $"0x{bits:X4} converted to 0x{BitConverter.SingleToInt32Bits(actual):X8} " + + $"but Half converts it to 0x{BitConverter.SingleToInt32Bits(expected):X8}."); } } @@ -57,13 +54,10 @@ public void FromSingle_MatchesHalf_ForEveryRepresentableValue() ushort bits = (ushort)i; Half value = BitConverter.UInt16BitsToHalf(bits); - if (Half.IsNaN(value)) - { - continue; - } - + // Compared against a round trip through Half rather than against the original + // bits, because widening quietens a signalling NaN and so cannot be reversed. Assert.True( - bits == Float16Converter.ManualFromSingle((float)value), + BitConverter.HalfToUInt16Bits((Half)(float)value) == Float16Converter.ManualFromSingle((float)value), $"0x{bits:X4} did not survive a round trip through single precision."); } } @@ -79,14 +73,31 @@ public void FromSingle_MatchesHalf_AcrossTheSinglePrecisionRange() { float value = BitConverter.Int32BitsToSingle((int)(uint)b); - if (float.IsNaN(value)) - { - continue; - } - Assert.True( BitConverter.HalfToUInt16Bits((Half)value) == Float16Converter.ManualFromSingle(value), - $"{value:R} (0x{(uint)b:X8}) was not narrowed the same way as Half."); + $"0x{(uint)b:X8} was not narrowed the same way as Half."); + } + } + + [Fact] + public void ConvertsNaN_PreservingSignAndPayload() + { + // A NaN's sign and payload are carried through rather than canonicalised, so that + // the manual implementation and System.Half cannot diverge. + foreach (ushort bits in new ushort[] { 0x7E00, 0xFE00, 0x7C01, 0x7DFF, 0xFFFF }) + { + Assert.Equal( + BitConverter.SingleToInt32Bits((float)BitConverter.UInt16BitsToHalf(bits)), + BitConverter.SingleToInt32Bits(Float16Converter.ManualToSingle(bits))); + } + + foreach (uint bits in new uint[] { 0x7FC00000, 0xFFC00000, 0x7FFFFFFF, 0x7F800001 }) + { + float value = BitConverter.Int32BitsToSingle((int)bits); + + Assert.Equal( + BitConverter.HalfToUInt16Bits((Half)value), + Float16Converter.ManualFromSingle(value)); } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs index b1170de0bc..e1fe0229c4 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using Microsoft.Data.Common; using Microsoft.Data.SqlClient; using Xunit; @@ -290,31 +291,30 @@ public void Float16_GetString_RendersExactValues() var vec = new SqlVector(new[] { (Half)65504f, (Half)1.5f }); Assert.Equal("[65504,1.5]", vec.GetString()); - Assert.Equal("[65504,1.5]", vec.ToString()); } [Fact] - public void Float16_ToString_MatchesFloat32Rendering() + public void Float16_GetString_MatchesFloat32Rendering() { // A value which both base types represent exactly renders identically, so callers // cannot tell the two apart from the rendering alone. Assert.Equal( - new SqlVector(new[] { 1.5f, 2.5f }).ToString(), - new SqlVector(new[] { (Half)1.5f, (Half)2.5f }).ToString()); + new SqlVector(new[] { 1.5f, 2.5f }).GetString(), + new SqlVector(new[] { (Half)1.5f, (Half)2.5f }).GetString()); } #endif [Fact] - public void Float32_ToString_RendersJson() + public void Float32_GetString_RendersJson() { - Assert.Equal("[1.5,2.5]", new SqlVector(new[] { 1.5f, 2.5f }).ToString()); + Assert.Equal("[1.5,2.5]", new SqlVector(new[] { 1.5f, 2.5f }).GetString()); } [Fact] - public void ToString_Null_RendersNullString() + public void GetString_Null_RendersNullString() { - Assert.Equal(SQLMessage.NullString(), SqlVector.CreateNull(3).ToString()); + Assert.Equal(SQLMessage.NullString(), SqlVector.CreateNull(3).GetString()); } #endregion @@ -326,7 +326,7 @@ public void ConvertPayload_Float32ToFloat16() { byte[] source = ((ISqlVector)new SqlVector(new[] { 1.5f, 2.5f, 3.5f })).VectorPayload; - byte[] converted = SqlVector.ConvertPayloadElementType( + byte[] converted = SqlVectorPayload.ConvertElementType( source, (byte)MetaType.SqlVectorElementType.Float16); @@ -335,7 +335,7 @@ public void ConvertPayload_Float32ToFloat16() // Reading the converted payload back yields the original values, because all three // are exactly representable in binary16. - byte[] roundTripped = SqlVector.ConvertPayloadElementType( + byte[] roundTripped = SqlVectorPayload.ConvertElementType( converted, (byte)MetaType.SqlVectorElementType.Float32); @@ -349,7 +349,7 @@ public void ConvertPayload_Float16ToFloat32_WidensExactly() // frameworks without System.Half. byte[] source = MakeFloat16Payload(new[] { 1.5f, 2.5f, 3.5f }); - byte[] converted = SqlVector.ConvertPayloadElementType( + byte[] converted = SqlVectorPayload.ConvertElementType( source, (byte)MetaType.SqlVectorElementType.Float32); @@ -364,7 +364,7 @@ public void ConvertPayload_SameElementType_ReturnsInput() Assert.Same( source, - SqlVector.ConvertPayloadElementType(source, (byte)MetaType.SqlVectorElementType.Float32)); + SqlVectorPayload.ConvertElementType(source, (byte)MetaType.SqlVectorElementType.Float32)); } [Fact] @@ -372,11 +372,11 @@ public void ConvertPayload_NarrowingRounds() { byte[] source = ((ISqlVector)new SqlVector(new[] { 1.1f })).VectorPayload; - byte[] narrowed = SqlVector.ConvertPayloadElementType( + byte[] narrowed = SqlVectorPayload.ConvertElementType( source, (byte)MetaType.SqlVectorElementType.Float16); - byte[] widened = SqlVector.ConvertPayloadElementType( + byte[] widened = SqlVectorPayload.ConvertElementType( narrowed, (byte)MetaType.SqlVectorElementType.Float32); @@ -387,7 +387,7 @@ public void ConvertPayload_NarrowingRounds() public void ConvertPayload_ShortHeader_Throws() { Assert.Throws(() => - SqlVector.ConvertPayloadElementType( + SqlVectorPayload.ConvertElementType( new byte[] { 0xA9, 0x01 }, (byte)MetaType.SqlVectorElementType.Float16)); } @@ -401,7 +401,7 @@ public void ConvertPayload_LengthMismatch_Throws() new[] { 1.5f }); Assert.Throws(() => - SqlVector.ConvertPayloadElementType( + SqlVectorPayload.ConvertElementType( source, (byte)MetaType.SqlVectorElementType.Float16)); } @@ -412,7 +412,7 @@ public void ConvertPayload_UnsupportedElementType_Throws() byte[] source = ((ISqlVector)new SqlVector(new[] { 1.5f })).VectorPayload; Assert.Throws(() => - SqlVector.ConvertPayloadElementType(source, 0x7F)); + SqlVectorPayload.ConvertElementType(source, 0x7F)); } #endregion diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs index 3ceb0445d2..e1b0a781b4 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs @@ -836,8 +836,9 @@ public void ConnectionRefusesUnsupportedServerTdsVersion(int major, int minor, i [Theory] // A server which supports the same version as the client negotiates that version. [InlineData(true, 0x2, 0x2)] - // A server which supports a later version than the client falls back to the - // client's, since the client cannot interpret anything newer. + // A server which supports a later version than the client is capped by the test + // harness, so the client still sees its own version on the wire. The client's own + // ceiling is exercised by TestConnRejectsVectorFeatExtVersionAboveClientCeiling. [InlineData(true, 0x3, 0x2)] // A server which supports only the earlier version negotiates that instead. [InlineData(true, 0x1, 0x1)] @@ -934,6 +935,56 @@ public void TestConnWithVectorFeatExtVersionNegotiation(bool expectedConnectionR } } + // Test that the driver refuses a vector feature extension ack whose version it + // cannot interpret. The server here acknowledges its own version rather than + // capping it to the client's, which is what a future server supporting a later + // layout would do. + [Theory] + // One past the client's ceiling. + [InlineData(0x3)] + // Well past it. + [InlineData(0xF)] + public void TestConnRejectsVectorFeatExtVersionAboveClientCeiling(byte serverVersion) + { + using TdsServer server = new(); + server.Start(); + server.EnableVectorFeatureExt = true; + server.ServerSupportedVectorFeatureExtVersion = serverVersion; + server.AcknowledgeRawVectorFeatureExtVersion = true; + + byte acknowledgedVersion = 0; + + server.OnAuthenticationResponseCompleted = response => + { + TDSFeatureExtAckGenericOption option = response + .OfType() + .FirstOrDefault()? + .Options + .OfType() + .FirstOrDefault(o => o.FeatureID == TDSFeatureID.VectorSupport)!; + + if (option != null) + { + acknowledgedVersion = option.FeatureAckData[0]; + } + }; + + string connStr = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + Pooling = false, // Disable pooling so this expected failure does not poison a shared pool + }.ConnectionString; + + using SqlConnection connection = new(connStr); + + Assert.Throws(() => connection.Open()); + + // Confirms the harness really did send the unsupported version, so the failure + // above is the client's ceiling rather than an unrelated connection problem. + Assert.Equal(serverVersion, acknowledgedVersion); + } + // Test that the driver sends the UserAgent feature extension when // the context switch is enabled, and that the presence or absence of // an ack from the server has no effect. diff --git a/src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.Servers/GenericTdsServer.cs b/src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.Servers/GenericTdsServer.cs index c3fadc3e0a..3cde1a0e7a 100644 --- a/src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.Servers/GenericTdsServer.cs +++ b/src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.Servers/GenericTdsServer.cs @@ -70,6 +70,15 @@ public delegate void OnAuthenticationCompletedDelegate( /// public byte ServerSupportedVectorFeatureExtVersion { get; set; } = DefaultSupportedVectorFeatureExtVersion; + /// + /// When true, the vector feature extension is acknowledged with + /// as-is, rather than capped to + /// the version the client requested. This models a server which acknowledges a + /// version the client cannot interpret, so that the client's own ceiling can be + /// exercised. + /// + public bool AcknowledgeRawVectorFeatureExtVersion { get; set; } + /// /// Property for setting server version for user agent feature extension. /// @@ -785,7 +794,9 @@ protected void CheckVectorSupport(ITDSServerSession session, TDSMessage response { // Create ack data (1 byte: Version number) byte[] data = new byte[1]; - data[0] = ServerSupportedVectorFeatureExtVersion > _clientSupportedVectorFeatureExtVersion ? _clientSupportedVectorFeatureExtVersion : ServerSupportedVectorFeatureExtVersion; + data[0] = AcknowledgeRawVectorFeatureExtVersion || ServerSupportedVectorFeatureExtVersion <= _clientSupportedVectorFeatureExtVersion + ? ServerSupportedVectorFeatureExtVersion + : _clientSupportedVectorFeatureExtVersion; // Create vector support as a generic feature extension option TDSFeatureExtAckGenericOption vectorSupportOption = new TDSFeatureExtAckGenericOption(TDSFeatureID.VectorSupport, (uint)data.Length, data); From ae88d3519c353ab8ced0293fe1d990b647039c30 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 25 Aug 2026 09:30:07 +0530 Subject: [PATCH 08/24] Negotiate the vector feature extension version by connection string The vector feature extension version requested at login was previously the highest this client understands, so upgrading the driver changed how a float16 column is presented. It is now chosen by the application. Connection string - Add the "Vector Type Support" keyword, with values off, v1 and v2, and the matching SqlConnectionStringBuilder.VectorTypeSupport property of the new SqlVectorTypeSupport type. This mirrors the vectorTypeSupport property in the JDBC driver, including its default. - Default to v1, so float32 vectors are exchanged in their binary form as before and a float16 column continues to be returned as a varchar(max) containing a JSON array unless an application opts in. - Request the configured version at login, and omit the feature request entirely when the keyword asks for no vector support. Bulk copy - Parse a textual source into the destination column's base type. A JSON string is always coerced to a float32 payload, which does not match the INSERT BULK declaration for a float16 column. - Do not convert a payload read from another vector column. Copying between columns of different base types is now reported by the server rather than being silently narrowed, which matches the JDBC driver. Both follow from the same constraint: the INSERT BULK statement states the destination's base type, and the server then requires a binary payload of exactly that width. Measured, at every negotiated version: a text declaration for a vector column is refused with "Invalid column type from bcp client", and text sent under a vector declaration is refused with a length mismatch. Only a client which never negotiates the feature extension may send text. Tests - Cover the version requested for each keyword value, including that no request is written when the keyword asks for none. - Run the float16 suites over a connection which opts in to v2, and cover a textual source into a float16 column at both v1 and v2. - Replace the cross base type bulk copy cases with ones which assert the behaviour above, and skip the source mode which supplies a typed vector through a DataTable where the suite reads a widened representation. Documentation - Describe the keyword, and show how the column metadata reports a vector's base type and dimension count, which is the only way to tell the two base types apart when a float16 column is surfaced as a JSON string. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .github/instructions/features.instructions.md | 57 +++++- doc/samples/SqlVectorFloat16Example.cs | 5 +- .../SqlConnectionStringBuilder.xml | 17 ++ .../SqlVectorTypeSupport.xml | 45 +++++ .../Microsoft.Data.SqlTypes/SqlVector.xml | 18 +- .../ref/Microsoft.Data.SqlClient.cs | 15 ++ .../DbConnectionStringDefaults.cs | 1 + .../DbConnectionStringKeywords.cs | 1 + .../VectorTypeSupportUtilities.cs | 135 ++++++++++++++ .../Connection/SqlConnectionInternal.cs | 17 +- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 81 ++++++--- .../Data/SqlClient/SqlConnectionOptions.cs | 22 +++ .../SqlClient/SqlConnectionStringBuilder.cs | 41 +++++ .../Data/SqlClient/SqlVectorTypeSupport.cs | 22 +++ .../src/Microsoft/Data/SqlClient/TdsParser.cs | 18 +- .../src/Resources/Strings.Designer.cs | 9 + .../src/Resources/Strings.resx | 3 + .../ManualTests/DataCommon/DataTestUtility.cs | 15 ++ .../NativeVectorFloat16AsSingleTests.cs | 4 + .../VectorTest/NativeVectorFloat16Tests.cs | 4 + .../SQL/VectorTest/NativeVectorTestsBase.cs | 33 +++- .../VectorTest/VectorColumnMetadataTests.cs | 2 +- .../VectorTest/VectorFloat16BehaviourTests.cs | 170 +++++++++++++++--- .../Microsoft/Data/SqlTypes/SqlVectorTest.cs | 98 ---------- .../SimulatedServerTests/ConnectionTests.cs | 66 ++++++- 25 files changed, 727 insertions(+), 172 deletions(-) create mode 100644 doc/snippets/Microsoft.Data.SqlClient/SqlVectorTypeSupport.xml create mode 100644 src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/VectorTypeSupportUtilities.cs create mode 100644 src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlVectorTypeSupport.cs diff --git a/.github/instructions/features.instructions.md b/.github/instructions/features.instructions.md index a39f9cbf5d..4ca29da34d 100644 --- a/.github/instructions/features.instructions.md +++ b/.github/instructions/features.instructions.md @@ -75,6 +75,7 @@ This is a comprehensive reference of supported connection string keywords. | `Column Encryption Setting` | Disabled | Always Encrypted mode | | `Enclave Attestation Url` | | Enclave attestation URL | | `Type System Version` | Latest | Type system version | +| `Vector Type Support` | v1 | Vector base types exchanged in binary form: `off`, `v1`, `v2` | | `Replication` | False | Replication support | | `User Instance` | False | SQL Express user instance | | `ConnectRetryCount` | 1 | Connection retry count | @@ -135,20 +136,43 @@ transported. The base type is selected by the type parameter of `SqlVector`. Notes: +- The `float16` base type is only exchanged in its binary form when the connection asks for + it through the `Vector Type Support` keyword. See below. - `System.Half` does not exist on .NET Framework, so `SqlVector` cannot be used there. A `float16` column is instead reported as a string, and can be read either as a JSON array through `GetString`/`GetSqlString`/`GetFieldValue`, or as a `SqlVector` via `GetSqlVector`, which widens the elements. Widening from `float16` is exact. - `float16` requires `ALTER DATABASE SCOPED CONFIGURATION SET PREVIEW_FEATURES = ON` while it is in preview. -- Conversion between base types is performed by SQL Server for parameters, and by the driver - for `SqlBulkCopy`, where the destination's base type is stated in the `INSERT BULK` - statement. Narrowing to `float16` loses precision, and fails for values outside its range: - the server reports the failure for a parameter, and the driver throws an - `OverflowException` for a bulk copy. +- SQL Server converts between base types for a parameter. A bulk copy is different: the + `INSERT BULK` statement states the destination's base type, and the server then requires a + binary payload of exactly that width, so it performs no conversion within the data stream. + A textual source is therefore parsed into the destination's base type by the driver, and + narrowing to `float16` throws an `OverflowException` for a value outside its range. A + payload read from another vector column keeps its own base type, so copying between + columns of different base types is reported by the server. - A column's base type and number of dimensions are available from the column schema: `reader.GetColumnSchema()[i]["VectorBaseType"]` and `["VectorDimensions"]`. Both are `null` - for columns which are not vectors. + for columns which are not vectors. This is the only way to tell the two base types apart + when a `float16` column is surfaced as a JSON string, because `GetFieldType` reports + `string` for such a column just as it does for a `varchar` one: + + ```csharp + DbColumn column = reader.GetColumnSchema()[0]; + + string baseType = (string)column["VectorBaseType"]; // "float16" | "float32" + int dimensions = (int) column["VectorDimensions"]; // element count + + // The dimension count is known before any row is read, so a buffer can be sized once. + float[] buffer = new float[dimensions]; + + while (reader.Read()) + { + // Widening from float16 is exact, so single precision reads either base type + // losslessly, and is the only strongly typed option on .NET Framework. + reader.GetSqlVector(0).Memory.Span.CopyTo(buffer); + } + ``` #### Vector Feature Extension Versions @@ -161,8 +185,25 @@ feature extension (`0x0E`): | `1` | `float32` is supported. Columns with any other base type are returned as `varchar(max)`. | | `2` | `float16` is supported in addition to `float32`. | -The driver always requests the highest version it supports, and the server acknowledges the -highest version they have in common. +The version requested at login is chosen by the `Vector Type Support` connection string +keyword, and the server acknowledges the highest version they have in common: + +| Keyword value | Requested version | +|---------------|-------------------| +| `off` | The feature extension is not requested at all. | +| `v1` | `1` — this is the **default**. | +| `v2` | `2` | + +The default is `v1`, so an application opts in to the `float16` representation rather than +receiving it when it upgrades the driver. The equivalent property is +`SqlConnectionStringBuilder.VectorTypeSupport`, of type `SqlVectorTypeSupport`. + +```csharp +var builder = new SqlConnectionStringBuilder(connectionString) +{ + VectorTypeSupport = SqlVectorTypeSupport.V2 +}; +``` ## SqlCommand Execution Modes diff --git a/doc/samples/SqlVectorFloat16Example.cs b/doc/samples/SqlVectorFloat16Example.cs index b81c552ce8..ae6266d778 100644 --- a/doc/samples/SqlVectorFloat16Example.cs +++ b/doc/samples/SqlVectorFloat16Example.cs @@ -26,8 +26,11 @@ namespace SqlVectorFloat16Example; class VectorFloat16ConsoleApp { // It is recommended to use a secure connection string in production code with valid cert. + // + // "Vector Type Support=v2" opts in to the float16 base type. It defaults to v1, under + // which a float16 column is returned as a varchar(max) containing a JSON array. private const string ConnectionString = - "Server=localhost;Database=Demo2;Integrated Security=true;Encrypt=true;TrustServerCertificate=true;"; + "Server=localhost;Database=Demo2;Integrated Security=true;Encrypt=true;TrustServerCertificate=true;Vector Type Support=v2;"; private const string TableName = "[dbo].[VectorFloat16Demo]"; diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml index 4014bd1450..163496b8da 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml @@ -1222,6 +1222,23 @@ The following example converts an existing connection string from using SQL Serv + + + Gets or sets the level of support for the vector data type which the driver + negotiates with the server. + + + A . The default is + . + + + Corresponds to the Vector Type Support connection string keyword, whose + values are off, v1 and v2. + + + The value is not a member of . + + Gets or sets a Boolean value that indicates whether the connection will be pooled or explicitly opened every time that the connection is requested. diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlVectorTypeSupport.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlVectorTypeSupport.xml new file mode 100644 index 0000000000..98196d314c --- /dev/null +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlVectorTypeSupport.xml @@ -0,0 +1,45 @@ + + + + + + Specifies the level of support for the vector data type which the driver + negotiates with the server. + + + + A vector column is only exchanged in its binary form when the server acknowledges + a version of the vector feature extension which covers the column's base type. A + column whose base type is not covered is returned as a varchar(max) + containing a JSON array. + + + The default is , + so an application opts in to a newer representation rather than receiving it when + it upgrades the driver. + + + + + + The vector feature extension is not requested. Every vector column is returned as a + varchar(max) containing a JSON array, as it was before the type existed. + + + + + Vectors with a float32 base type are exchanged in their binary form and are + surfaced as SqlVector<float>. Columns with any other base type are + returned as a varchar(max) containing a JSON array. This is the default. + + + + + Adds float16 to the base types exchanged in their binary form. On .NET such a + column is surfaced as SqlVector<System.Half>; on .NET Framework, where + System.Half is unavailable, it is surfaced as a JSON array string and can be + read as a SqlVector<float> on request. + + + + diff --git a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml index 9927501ce5..a95ddea4af 100644 --- a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml +++ b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml @@ -18,13 +18,25 @@ from .NET Framework as a JSON string, or as a SqlVector<float>, which the server converts. + + A float16 column is only exchanged in its binary form when the connection asks + for it through the Vector Type Support keyword, which defaults to v1. + Otherwise such a column is returned as a varchar(max) containing a JSON array. + Conversion between base types is performed by SQL Server, which reports an error if a value is outside the range the destination can represent. Converting from float32 to float16 loses precision for values which the narrower type - cannot represent exactly, in the same way as inserting a JSON literal does. During a - bulk copy the driver performs the conversion instead, and throws an - for a value outside the destination's range. + cannot represent exactly, in the same way as inserting a JSON literal does. + + + A bulk copy is different, because the INSERT BULK statement states the + destination column's base type and the server then requires a binary payload of + exactly that width. A textual source is therefore parsed into the destination's base + type by the driver, which throws an + for a value outside that range. A payload + read from another vector column keeps its own base type, so copying between columns + of different base types is reported by the server. diff --git a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs index a5e3f1a473..d24ef2c5c2 100644 --- a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs +++ b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs @@ -27,6 +27,17 @@ public enum PoolBlockingPeriod NeverBlock = 2, } +/// +public enum SqlVectorTypeSupport +{ + /// + Off = 0, + /// + V1 = 1, + /// + V2 = 2, +} + /// public enum SortOrder { @@ -1425,6 +1436,10 @@ public SqlConnectionStringBuilder(string connectionString) { } [System.ComponentModel.DisplayNameAttribute("Pool Blocking Period")] [System.ComponentModel.RefreshPropertiesAttribute(System.ComponentModel.RefreshProperties.All)] public Microsoft.Data.SqlClient.PoolBlockingPeriod PoolBlockingPeriod { get { throw null; } set { } } + /// + [System.ComponentModel.DisplayNameAttribute("Vector Type Support")] + [System.ComponentModel.RefreshPropertiesAttribute(System.ComponentModel.RefreshProperties.All)] + public Microsoft.Data.SqlClient.SqlVectorTypeSupport VectorTypeSupport { get { throw null; } set { } } /// [System.ComponentModel.DisplayNameAttribute("Pooling")] [System.ComponentModel.RefreshPropertiesAttribute(System.ComponentModel.RefreshProperties.All)] diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringDefaults.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringDefaults.cs index bf700c0b55..894786ae5f 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringDefaults.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringDefaults.cs @@ -56,6 +56,7 @@ internal static class DbConnectionStringDefaults internal const string TypeSystemVersion = "Latest"; internal const string UserId = ""; internal const bool UserInstance = false; + internal const SqlVectorTypeSupport VectorTypeSupport = SqlClient.SqlVectorTypeSupport.V1; internal const string WorkstationId = ""; #if NETFRAMEWORK diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringKeywords.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringKeywords.cs index b163ec2536..c0d7db8811 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringKeywords.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringKeywords.cs @@ -51,6 +51,7 @@ internal static class DbConnectionStringKeywords internal const string TypeSystemVersion = "Type System Version"; internal const string UserId = "User ID"; internal const string UserInstance = "User Instance"; + internal const string VectorTypeSupport = "Vector Type Support"; internal const string WorkstationId = "Workstation ID"; #if NETFRAMEWORK diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/VectorTypeSupportUtilities.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/VectorTypeSupportUtilities.cs new file mode 100644 index 0000000000..cf30ff42d1 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/VectorTypeSupportUtilities.cs @@ -0,0 +1,135 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Diagnostics; +using Microsoft.Data.SqlClient; + +namespace Microsoft.Data.Common.ConnectionString +{ + internal static class VectorTypeSupportUtilities + { + internal static bool TryConvertToVectorTypeSupport(string value, out SqlVectorTypeSupport result) + { + Debug.Assert(Enum.GetNames(typeof(SqlVectorTypeSupport)).Length == 3, "SqlVectorTypeSupport enum has changed, update needed"); + Debug.Assert(value != null, "TryConvertToVectorTypeSupport(null,...)"); + + if (StringComparer.OrdinalIgnoreCase.Equals(value, nameof(SqlVectorTypeSupport.Off))) + { + result = SqlVectorTypeSupport.Off; + return true; + } + else if (StringComparer.OrdinalIgnoreCase.Equals(value, nameof(SqlVectorTypeSupport.V1))) + { + result = SqlVectorTypeSupport.V1; + return true; + } + else if (StringComparer.OrdinalIgnoreCase.Equals(value, nameof(SqlVectorTypeSupport.V2))) + { + result = SqlVectorTypeSupport.V2; + return true; + } + else + { + result = DbConnectionStringDefaults.VectorTypeSupport; + return false; + } + } + + internal static bool IsValidVectorTypeSupportValue(SqlVectorTypeSupport value) + { + Debug.Assert(Enum.GetNames(typeof(SqlVectorTypeSupport)).Length == 3, "SqlVectorTypeSupport enum has changed, update needed"); + return value == SqlVectorTypeSupport.Off || value == SqlVectorTypeSupport.V1 || value == SqlVectorTypeSupport.V2; + } + + internal static string VectorTypeSupportToString(SqlVectorTypeSupport value) + { + Debug.Assert(IsValidVectorTypeSupportValue(value)); + + return value switch + { + SqlVectorTypeSupport.Off => nameof(SqlVectorTypeSupport.Off), + SqlVectorTypeSupport.V2 => nameof(SqlVectorTypeSupport.V2), + _ => nameof(SqlVectorTypeSupport.V1), + }; + } + + /// + /// Converts the given value to a , following the same + /// rules as the other enumerated connection string values: a string is matched against + /// the enum names using an ordinal, case insensitive comparison; a value of the enum + /// type is used as is; an integral value is converted; and anything else is rejected. + /// + internal static SqlVectorTypeSupport ConvertToVectorTypeSupport(string keyword, object value) + { + Debug.Assert(value != null, "ConvertToVectorTypeSupport(null)"); + + if (value is string sValue) + { + if (TryConvertToVectorTypeSupport(sValue, out SqlVectorTypeSupport result)) + { + return result; + } + + // Try again without any leading or trailing whitespace. + sValue = sValue.Trim(); + if (TryConvertToVectorTypeSupport(sValue, out result)) + { + return result; + } + + throw ADP.InvalidConnectionOptionValue(keyword); + } + else + { + SqlVectorTypeSupport eValue; + + if (value is SqlVectorTypeSupport support) + { + // Quick path for the most common case. + eValue = support; + } + else if (value.GetType().IsEnum) + { + // Block the use of an unrelated enum type, which would otherwise be + // converted through its underlying integral value. + throw ADP.ConvertFailed(value.GetType(), typeof(SqlVectorTypeSupport), null); + } + else + { + try + { + eValue = (SqlVectorTypeSupport)Enum.ToObject(typeof(SqlVectorTypeSupport), value); + } + catch (ArgumentException e) + { + throw ADP.ConvertFailed(value.GetType(), typeof(SqlVectorTypeSupport), e); + } + } + + if (IsValidVectorTypeSupportValue(eValue)) + { + return eValue; + } + + throw ADP.InvalidEnumerationValue(typeof(SqlVectorTypeSupport), (int)eValue); + } + } + + /// + /// The vector feature extension version which corresponds to the given setting. + /// + internal static byte ToFeatureExtensionVersion(SqlVectorTypeSupport value) + { + Debug.Assert(IsValidVectorTypeSupportValue(value)); + + return value switch + { + SqlVectorTypeSupport.Off => TdsEnums.VECTOR_VERSION_NOT_SUPPORTED, + SqlVectorTypeSupport.V2 => TdsEnums.VECTOR_VERSION_FLOAT16, + _ => TdsEnums.VECTOR_VERSION_FLOAT32, + }; + } + } +} diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs index 87ed9861d3..f36028e321 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs @@ -13,16 +13,13 @@ using System.Threading.Tasks; using System.Transactions; using Microsoft.Data.Common; +using Microsoft.Data.Common.ConnectionString; using Microsoft.Data.ProviderBase; using Microsoft.Data.SqlClient.ConnectionPool; using Microsoft.Data.SqlClient.Internal; using Microsoft.Data.SqlClient.Utilities; using IsolationLevel = System.Data.IsolationLevel; -#if NETFRAMEWORK -using Microsoft.Data.Common.ConnectionString; -#endif - namespace Microsoft.Data.SqlClient.Connection { internal class SqlConnectionInternal : DbConnectionInternal, IDisposable @@ -3055,7 +3052,17 @@ private void Login( // @TODO: Request all the implicit features in one place (probably at the very top) requestedFeatures |= TdsEnums.FeatureExtension.SQLDNSCaching; requestedFeatures |= TdsEnums.FeatureExtension.JsonSupport; - requestedFeatures |= TdsEnums.FeatureExtension.VectorSupport; + + // The vector feature extension is only requested when the connection asks for a + // version of it. Omitting the request leaves vector columns as varchar(max) + // containing a JSON array, which is how they were presented before the type + // was introduced. + if (VectorTypeSupportUtilities.ToFeatureExtensionVersion(ConnectionOptions.VectorTypeSupport) + != TdsEnums.VECTOR_VERSION_NOT_SUPPORTED) + { + requestedFeatures |= TdsEnums.FeatureExtension.VectorSupport; + } + requestedFeatures |= TdsEnums.FeatureExtension.EnhancedRoutingSupport; requestedFeatures |= TdsEnums.FeatureExtension.UserAgent; diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index cdcc0dcc5e..850402ef80 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -839,7 +839,8 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i AppendColumnNameAndTypeName(updateBulkCommandText, metadata.column, metadata.type.ToString()); } - switch (metadata.metaType.NullableType) + { + switch (metadata.metaType.NullableType) { case TdsEnums.SQLNUMERICN: case TdsEnums.SQLDECIMALN: @@ -909,6 +910,7 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i } break; } + } } // Get collation for column i @@ -1430,11 +1432,46 @@ private bool ReadFromRowSource() } } + /// + /// The CLR type of a source column, or when the source cannot + /// report one. + /// + private Type GetSourceColumnType(int sourceOrdinal) + { + switch (_rowSourceType) + { + case ValueSourceType.DbDataReader: + case ValueSourceType.IDataReader: + return _sqlDataReaderRowSource?.GetFieldType(sourceOrdinal); + + case ValueSourceType.DataTable: + case ValueSourceType.RowArray: + return _dataTableSource?.Columns[sourceOrdinal].DataType; + + default: + return null; + } + } + + /// + /// Whether a vector destination column is supplied from a textual source, in which + /// case the value is transferred as text and converted by the server. + /// + /// + /// A vector has no textual form on the wire, so such a column is declared as a + /// varchar(max) in the INSERT BULK statement. The server then parses + /// the JSON array into the destination's base type, which is the only way to write a + /// base type the client cannot represent, and avoids a conversion whose result could + /// differ from the server's. + /// + private bool IsTextSourcedVectorColumn(int sourceOrdinal, _SqlMetaData metadata) => + metadata.type == SqlDbTypeExtensions.Vector && + GetSourceColumnType(sourceOrdinal) == typeof(string); + private SourceColumnMetadata GetColumnMetadata(int ordinal) { int sourceOrdinal = _sortedColumnMappings[ordinal]._sourceColumnOrdinal; _SqlMetaData metadata = _sortedColumnMappings[ordinal]._metadata; - // Handle special Sql data types for SqlDataReader and DataTables ValueMethod method; bool isSqlType; @@ -1736,10 +1773,10 @@ private object ValidateBulkCopyVariant(object value) /// column's base type, leaving payloads which already use that base type untouched. /// /// - /// Unlike an ordinary parameter, a bulk copy declares the destination's base type in - /// the INSERT BULK statement, so the payload must use that base type. The - /// server cannot convert it, because binary16 and binary32 elements differ in size - /// and a mismatch is reported as a column length error. + /// Used only for a textual source, whose JSON array is always coerced to a float32 + /// payload regardless of the destination's base type. The server does not convert + /// within the bulk copy data stream, because binary16 and binary32 elements differ + /// in size and the INSERT BULK declaration fixes the element width. /// private static object ConvertVectorToBaseType(object value, byte destinationElementType) { @@ -1750,14 +1787,12 @@ private static object ConvertVectorToBaseType(object value, byte destinationElem return value; } - // The payload is converted directly rather than through a strongly typed vector, - // so that .NET Framework, which has no System.Half, can also write to float16 - // destinations. return SqlTypes.SqlVectorPayload.ConvertElementType(payload, destinationElementType); } - private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, ref bool isSqlType, out bool coercedToDataFeed) + private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, ref bool isSqlType, out bool coercedToDataFeed, int sourceOrdinal) { + bool isTextSourcedVector = IsTextSourcedVectorColumn(sourceOrdinal, metadata); coercedToDataFeed = false; if (isNull) @@ -1844,17 +1879,20 @@ private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, re mt = MetaType.GetMetaTypeFromSqlDbType(type.SqlDbType, false); value = SqlParameter.CoerceValue(value, mt, out coercedToDataFeed, out typeChanged, false); - // The INSERT BULK declaration for a vector column states the - // destination's base type, so the payload written to the wire must - // use that base type too. A mismatch is rejected by the server as a - // column length error rather than being converted, because binary16 - // and binary32 elements differ in size. + // A JSON string is always coerced to a float32 payload, so a textual + // source bound for a column with a different base type has to be + // rewritten to that base type. The INSERT BULK declaration states the + // destination's base type, and the server does not convert within the + // data stream, because binary16 and binary32 elements differ in size. // - // This runs after coercion because the payload produced by coercion - // uses the source value's own base type: a JSON string always yields - // float32, which is how a float16 column reads back on frameworks - // without System.Half. - value = ConvertVectorToBaseType(value, scale); + // Only a textual source is converted. A payload read from another + // vector column keeps its own base type, so copying between columns + // of different base types is reported by the server rather than being + // silently narrowed. + if (isTextSourcedVector) + { + value = ConvertVectorToBaseType(value, scale); + } break; case TdsEnums.SQLINTN: @@ -2517,7 +2555,8 @@ private Task ReadWriteColumnValueAsync(int col) _SqlMetaData metadata = _sortedColumnMappings[col]._metadata; if (!isDataFeed) { - value = ConvertValue(value, metadata, isNull, ref isSqlType, out isDataFeed); + value = ConvertValue(value, metadata, isNull, ref isSqlType, out isDataFeed, + _sortedColumnMappings[col]._sourceColumnOrdinal); // If column encryption is requested via connection string option, perform encryption here if (!isNull && // if value is not NULL diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs index 42c562a4d1..fd03412c18 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs @@ -97,6 +97,7 @@ internal static class TRANSACTIONBINDING private readonly bool _mars; private readonly bool _persistSecurityInfo; private readonly PoolBlockingPeriod _poolBlockingPeriod; + private readonly SqlVectorTypeSupport _vectorTypeSupport; private readonly bool _pooling; private readonly bool _replication; private readonly bool _userInstance; @@ -230,6 +231,7 @@ static SqlConnectionOptions() DbConnectionStringSynonyms.Uid, DbConnectionStringSynonyms.User); AddKeywordToMap(DbConnectionStringKeywords.UserInstance); + AddKeywordToMap(DbConnectionStringKeywords.VectorTypeSupport); AddKeywordToMap(DbConnectionStringKeywords.WorkstationId, DbConnectionStringSynonyms.WorkstationId, DbConnectionStringSynonyms.WsId); @@ -269,6 +271,7 @@ internal SqlConnectionOptions(string connectionString) _integratedSecurity = ConvertValueToIntegratedSecurity(); _poolBlockingPeriod = ConvertValueToPoolBlockingPeriod(); + _vectorTypeSupport = ConvertValueToVectorTypeSupport(); _encrypt = ConvertValueToSqlConnectionEncrypt(); _enlist = ConvertValueToBoolean(DbConnectionStringKeywords.Enlist, DbConnectionStringDefaults.Enlist); _mars = ConvertValueToBoolean(DbConnectionStringKeywords.MultipleActiveResultSets, DbConnectionStringDefaults.MultipleActiveResultSets); @@ -686,6 +689,8 @@ internal SqlConnectionOptions(SqlConnectionOptions connectionOptions, string dat internal string UserID => _userID; internal string WorkstationId => _workstationId; internal PoolBlockingPeriod PoolBlockingPeriod => _poolBlockingPeriod; + + internal SqlVectorTypeSupport VectorTypeSupport => _vectorTypeSupport; internal string ServerSPN => _serverSPN; internal string FailoverPartnerSPN => _failoverPartnerSPN; @@ -963,6 +968,23 @@ internal PoolBlockingPeriod ConvertValueToPoolBlockingPeriod() } } + internal SqlVectorTypeSupport ConvertValueToVectorTypeSupport() + { + if (!TryGetParsetableValue(DbConnectionStringKeywords.VectorTypeSupport, out string value)) + { + return DbConnectionStringDefaults.VectorTypeSupport; + } + + try + { + return VectorTypeSupportUtilities.ConvertToVectorTypeSupport(DbConnectionStringKeywords.VectorTypeSupport, value); + } + catch (Exception e) when (e is FormatException || e is OverflowException) + { + throw ADP.InvalidConnectionOptionValue(DbConnectionStringKeywords.VectorTypeSupport, e); + } + } + internal SqlConnectionEncryptOption ConvertValueToSqlConnectionEncrypt() { if (!TryGetParsetableValue(DbConnectionStringKeywords.Encrypt, out string value)) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs index 4038dbb295..ea6fe49e24 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs @@ -71,6 +71,7 @@ private enum Keywords ServerSPN, FailoverPartnerSPN, ContextConnection, + VectorTypeSupport, #if NETFRAMEWORK ConnectionReset, NetworkLibrary, @@ -119,6 +120,7 @@ private enum Keywords private bool _persistSecurityInfo = DbConnectionStringDefaults.PersistSecurityInfo; private PoolBlockingPeriod _poolBlockingPeriod = DbConnectionStringDefaults.PoolBlockingPeriod; + private SqlVectorTypeSupport _vectorTypeSupport = DbConnectionStringDefaults.VectorTypeSupport; private bool _pooling = DbConnectionStringDefaults.Pooling; private bool _replication = DbConnectionStringDefaults.Replication; private bool _userInstance = DbConnectionStringDefaults.UserInstance; @@ -172,6 +174,7 @@ private static string[] CreateValidKeywords() validKeywords[(int)Keywords.TypeSystemVersion] = DbConnectionStringKeywords.TypeSystemVersion; validKeywords[(int)Keywords.UserID] = DbConnectionStringKeywords.UserId; validKeywords[(int)Keywords.UserInstance] = DbConnectionStringKeywords.UserInstance; + validKeywords[(int)Keywords.VectorTypeSupport] = DbConnectionStringKeywords.VectorTypeSupport; validKeywords[(int)Keywords.WorkstationID] = DbConnectionStringKeywords.WorkstationId; validKeywords[(int)Keywords.ConnectRetryCount] = DbConnectionStringKeywords.ConnectRetryCount; validKeywords[(int)Keywords.ConnectRetryInterval] = DbConnectionStringKeywords.ConnectRetryInterval; @@ -230,6 +233,7 @@ private static Dictionary CreateKeywordsDictionary() { DbConnectionStringKeywords.TypeSystemVersion, Keywords.TypeSystemVersion }, { DbConnectionStringKeywords.UserId, Keywords.UserID }, { DbConnectionStringKeywords.UserInstance, Keywords.UserInstance }, + { DbConnectionStringKeywords.VectorTypeSupport, Keywords.VectorTypeSupport }, { DbConnectionStringKeywords.WorkstationId, Keywords.WorkstationID }, { DbConnectionStringKeywords.ConnectRetryCount, Keywords.ConnectRetryCount }, { DbConnectionStringKeywords.ConnectRetryInterval, Keywords.ConnectRetryInterval }, @@ -317,6 +321,9 @@ private static SqlConnectionIPAddressPreference ConvertToIPAddressPreference(str private static PoolBlockingPeriod ConvertToPoolBlockingPeriod(string keyword, object value) => PoolBlockingUtilities.ConvertToPoolBlockingPeriod(keyword, value); + private static SqlVectorTypeSupport ConvertToVectorTypeSupport(string keyword, object value) + => VectorTypeSupportUtilities.ConvertToVectorTypeSupport(keyword, value); + private object GetAt(Keywords index) { switch (index) @@ -329,6 +336,8 @@ private object GetAt(Keywords index) return AttachDBFilename; case Keywords.PoolBlockingPeriod: return PoolBlockingPeriod; + case Keywords.VectorTypeSupport: + return VectorTypeSupport; case Keywords.CommandTimeout: return CommandTimeout; case Keywords.ConnectTimeout: @@ -451,6 +460,9 @@ private void Reset(Keywords index) case Keywords.PoolBlockingPeriod: _poolBlockingPeriod = DbConnectionStringDefaults.PoolBlockingPeriod; break; + case Keywords.VectorTypeSupport: + _vectorTypeSupport = DbConnectionStringDefaults.VectorTypeSupport; + break; case Keywords.CommandTimeout: _commandTimeout = DbConnectionStringDefaults.CommandTimeout; break; @@ -631,6 +643,12 @@ private void SetPoolBlockingPeriodValue(PoolBlockingPeriod value) base[DbConnectionStringKeywords.PoolBlockingPeriod] = PoolBlockingUtilities.PoolBlockingPeriodToString(value); } + private void SetVectorTypeSupportValue(SqlVectorTypeSupport value) + { + Debug.Assert(VectorTypeSupportUtilities.IsValidVectorTypeSupportValue(value), "Invalid value for VectorTypeSupport"); + base[DbConnectionStringKeywords.VectorTypeSupport] = VectorTypeSupportUtilities.VectorTypeSupportToString(value); + } + private Exception UnsupportedKeyword(string keyword) { #if NET @@ -1022,6 +1040,9 @@ public override object this[string keyword] case Keywords.PoolBlockingPeriod: PoolBlockingPeriod = ConvertToPoolBlockingPeriod(keyword, value); break; + case Keywords.VectorTypeSupport: + VectorTypeSupport = ConvertToVectorTypeSupport(keyword, value); + break; case Keywords.Encrypt: Encrypt = ConvertToSqlConnectionEncryptOption(keyword, value); break; @@ -1682,6 +1703,26 @@ public PoolBlockingPeriod PoolBlockingPeriod } } + /// + [DisplayName(DbConnectionStringKeywords.VectorTypeSupport)] + [ResCategory(nameof(Strings.DataCategory_Advanced))] + [ResDescription(nameof(Strings.DbConnectionString_VectorTypeSupport))] + [RefreshProperties(RefreshProperties.All)] + public SqlVectorTypeSupport VectorTypeSupport + { + get => _vectorTypeSupport; + set + { + if (!VectorTypeSupportUtilities.IsValidVectorTypeSupportValue(value)) + { + throw ADP.InvalidEnumerationValue(typeof(SqlVectorTypeSupport), (int)value); + } + + SetVectorTypeSupportValue(value); + _vectorTypeSupport = value; + } + } + /// [DisplayName(DbConnectionStringKeywords.Pooling)] [ResCategory(nameof(Strings.DataCategory_Pooling))] diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlVectorTypeSupport.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlVectorTypeSupport.cs new file mode 100644 index 0000000000..2e71dde86a --- /dev/null +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlVectorTypeSupport.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Data.SqlClient +{ + /// +#if NETFRAMEWORK + [System.Serializable] +#endif + public enum SqlVectorTypeSupport + { + /// + Off = 0, // Vector columns are returned as varchar(max) containing a JSON array. + + /// + V1 = 1, // Vectors with a float32 base type are exchanged in their binary form. + + /// + V2 = 2, // Adds float16 to the base types exchanged in their binary form. + } +} diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs index 8a4186e299..518e1b0b8a 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -84,6 +84,17 @@ internal sealed partial class TdsParser private SqlCollation _defaultCollation; // default collation from the server + /// + /// The default collation reported by the server, used for a column whose destination + /// type carries no collation of its own but which is sent as character data. + /// + internal SqlCollation DefaultCollation => _defaultCollation; + + /// + /// The code page which corresponds to . + /// + internal int DefaultCodePage => _defaultCodePage; + private int _defaultCodePage; private int _defaultLCID; @@ -9179,7 +9190,12 @@ internal int WriteVectorSupportFeatureRequest(bool write) // Feature Data Length WriteInt(1, _physicalStateObj); - _physicalStateObj.WriteByte(TdsEnums.MAX_SUPPORTED_VECTOR_VERSION); + // The version the connection asks for, rather than the highest this client + // understands, so that an application opts in to a newer representation + // rather than receiving it on upgrade. + _physicalStateObj.WriteByte( + VectorTypeSupportUtilities.ToFeatureExtensionVersion( + _connHandler.ConnectionOptions.VectorTypeSupport)); } return len; diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs index de55b10c6e..d130b6ee52 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs @@ -1518,6 +1518,15 @@ internal static string DbConnectionString_PersistSecurityInfo { } } + /// + /// Looks up a localized string similar to The level of vector type support to negotiate with the server.. + /// + internal static string DbConnectionString_VectorTypeSupport { + get { + return ResourceManager.GetString("DbConnectionString_VectorTypeSupport", resourceCulture); + } + } + /// /// Looks up a localized string similar to Defines the blocking period behavior for a connection pool.. /// diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx index 8c4c55ba7a..c96a4deade 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx @@ -1899,6 +1899,9 @@ The server did not acknowledge a recovery attempt, connection recovery is not possible. + + The level of vector type support to negotiate with the server. + Defines the blocking period behavior for a connection pool. diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs index c465235078..ca49a67a20 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs @@ -192,6 +192,21 @@ public static string SQLServerVersion IsSqlVectorSupported && CheckVectorFloat16Supported(); + /// + /// A TCP connection string which opts in to the vector feature extension version that + /// covers the float16 base type. + /// + /// + /// The Vector Type Support keyword defaults to v1, so a float16 column is + /// returned as a varchar(max) containing a JSON array unless a connection asks + /// for v2. Tests which exercise the native float16 representation must use this. + /// + public static string VectorFloat16ConnectionString => + new SqlConnectionStringBuilder(TCPConnectionString) + { + VectorTypeSupport = SqlVectorTypeSupport.V2 + }.ConnectionString; + public static bool IsDebugBuild { get diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs index a6c78d22b1..2720386130 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs @@ -44,6 +44,10 @@ public sealed class VectorFloat16AsSingleTestData : NativeVectorTestDataBase false; + + // float16 is only exchanged in its binary form when the connection asks for the + // feature extension version which covers it. + public override string ConnectionString => DataTestUtility.VectorFloat16ConnectionString; } /// diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs index f3b7fa6bc1..86f4bb9913 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs @@ -54,6 +54,10 @@ public sealed class VectorFloat16TestData : NativeVectorTestDataBase public override bool IsSupported => DataTestUtility.IsSqlVectorFloat16Supported; public override string SqlServerTypeName => "float16"; + + // float16 is only exchanged in its binary form when the connection asks for the + // feature extension version which covers it. + public override string ConnectionString => DataTestUtility.VectorFloat16ConnectionString; } [Trait("Set", "3")] diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs index 7018737b81..4b28b422de 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs @@ -42,6 +42,13 @@ public abstract class NativeVectorTestDataBase /// public virtual bool IsDefaultRepresentation => true; + /// + /// The connection string the suite runs against. A float16 column is only exchanged + /// in its binary form when the connection opts in to the feature extension version + /// which covers that base type, so a suite for such a column overrides this. + /// + public virtual string ConnectionString => DataTestUtility.TCPConnectionString; + public int ValidSampleScalarDataLength => SampleScalarData.Length; public IEnumerable TestData => @@ -109,6 +116,24 @@ public abstract class NativeVectorTestsBase : IDisposable public static bool IsSupported => TestDataInstance.IsSupported; + /// + /// The bulk copy source modes which apply to this suite. Mode 2 supplies the value as + /// a through a , which carries its + /// own base type; it is therefore only valid when that base type is the column's. + /// + public static IEnumerable BulkCopySourceModes + { + get + { + yield return new object[] { 1 }; + + if (TestDataInstance.IsDefaultRepresentation) + { + yield return new object[] { 2 }; + } + } + } + public static IEnumerable TestData => TestDataInstance.TestData; public NativeVectorTestsBase() @@ -116,7 +141,7 @@ public NativeVectorTestsBase() int vectorDimensions = TestDataInstance.ValidSampleScalarDataLength; string tableDefinition = $@"(Id INT PRIMARY KEY IDENTITY, {VectorColumnName} vector({vectorDimensions}, {TestDataInstance.SqlServerTypeName}) NULL)"; - _connectionString = DataTestUtility.TCPConnectionString; + _connectionString = TestDataInstance.ConnectionString; _managementConnection = new SqlConnection(_connectionString); _vectorTable = new Table(_managementConnection, "VectorTestTable", tableDefinition); _bulkCopySourceTable = new Table(_managementConnection, "VectorBulkCopyTestTable", tableDefinition); @@ -430,8 +455,7 @@ public async Task TestStoredProcParamsForVectorAsync( } [ConditionalTheory(nameof(IsSupported))] - [InlineData(1)] - [InlineData(2)] + [MemberData(nameof(BulkCopySourceModes))] public void TestBulkCopyFromSqlTable(int bulkCopySourceMode) { // Setup source with test data and create destination table for bulkcopy. @@ -526,8 +550,7 @@ public void TestBulkCopyFromSqlTable(int bulkCopySourceMode) } [ConditionalTheory(nameof(IsSupported))] - [InlineData(1)] - [InlineData(2)] + [MemberData(nameof(BulkCopySourceModes))] public async Task TestBulkCopyFromSqlTableAsync(int bulkCopySourceMode) { // Setup source with test data and create destination table for bulk copy. diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs index 9b2411e179..5701bab2dd 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs @@ -112,7 +112,7 @@ public void DrivesReadPathForACallerWhichDoesNotKnowTheSchema() // own: it reports string for a float16 column on .NET Framework, which is what a // varchar column reports too, and it cannot distinguish the two base types at all // for a caller which wants to read both through one representation. - using SqlConnection connection = new(_connectionString); + using SqlConnection connection = new(DataTestUtility.VectorFloat16ConnectionString); connection.Open(); using SqlCommand command = new( diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index 0d20cedf0a..ede14c36a8 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -32,7 +32,7 @@ public sealed class VectorFloat16BehaviourTests : IDisposable /// private const int Float32ToFloat16OutOfRangeError = 42284; - private readonly string _connectionString = DataTestUtility.TCPConnectionString; + private readonly string _connectionString = DataTestUtility.VectorFloat16ConnectionString; private readonly SqlConnection _managementConnection; private readonly Table _float16Table; private readonly Table _float32Table; @@ -261,14 +261,51 @@ public void RejectsValuesOutsideTheFloat16Range() #region Bulk copy across base types [ConditionalTheory(nameof(IsSupported))] - [InlineData("float16", "float32")] + [InlineData("float16")] + [InlineData("float32")] + public void BulkCopiesBetweenColumnsOfTheSameBaseType(string baseType) + { + // A payload read from a vector column is transferred to a column of the same base + // type as-is, with no conversion and no intermediate representation. + Table table = baseType == "float16" ? _float16Table : _float32Table; + + Insert(table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + + using SqlConnection sourceConnection = new(_connectionString); + sourceConnection.Open(); + using SqlCommand selectCommand = new($"SELECT {ColumnName} FROM {table.Name}", sourceConnection); + using SqlDataReader sourceReader = selectCommand.ExecuteReader(); + + using SqlConnection destinationConnection = new(_connectionString); + destinationConnection.Open(); + + using (SqlBulkCopy bulkCopy = new(destinationConnection) { DestinationTableName = table.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(sourceReader); + } + + using SqlCommand verifyCommand = + new($"SELECT TOP 1 {ColumnName} FROM {table.Name} ORDER BY Id DESC", destinationConnection); + using SqlDataReader verifyReader = verifyCommand.ExecuteReader(); + + Assert.True(verifyReader.Read()); + Assert.Equal([1.5f, 2.5f, 3.5f], verifyReader.GetSqlVector(0).Memory.ToArray()); + } + + [ConditionalTheory(nameof(IsSupported))] [InlineData("float32", "float16")] - [InlineData("float16", "float16")] - public void BulkCopiesBetweenColumnsOfEitherBaseType(string sourceBaseType, string destinationBaseType) + #if NET + // On .NET Framework a float16 column reads as text, so this pairing takes the textual + // path and the value is converted rather than rejected. + [InlineData("float16", "float32")] + #endif + public void BulkCopyRejectsColumnsOfDifferentBaseTypes(string sourceBaseType, string destinationBaseType) { - // Unlike a parameter, a bulk copy states the destination's base type in the - // INSERT BULK statement, so the driver converts the payload rather than relying on - // the server, which rejects a size mismatch instead of converting it. + // A payload read from a vector column keeps its own base type, and the INSERT BULK + // declaration states the destination's, so the server reports the mismatch. The + // driver does not silently rewrite the payload: a caller which wants the conversion + // reads the source column as text, which the server converts. Table source = sourceBaseType == "float16" ? _float16Table : _float32Table; Table destination = destinationBaseType == "float16" ? _float16Table : _float32Table; @@ -282,52 +319,74 @@ public void BulkCopiesBetweenColumnsOfEitherBaseType(string sourceBaseType, stri using SqlConnection destinationConnection = new(_connectionString); destinationConnection.Open(); - using (SqlBulkCopy bulkCopy = new(destinationConnection) { DestinationTableName = destination.Name }) + using SqlBulkCopy bulkCopy = new(destinationConnection) { DestinationTableName = destination.Name }; + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + + Assert.Throws(() => bulkCopy.WriteToServer(sourceReader)); + } + + #if !NET + [ConditionalFact(nameof(IsSupported))] + public void BulkCopiesFloat16ToFloat32ThroughTheTextualRepresentation() + { + // On .NET Framework a float16 column reads as a JSON string, so a copy into a + // float32 column takes the textual path and the value is converted rather than + // rejected. This is the counterpart of the .NET case above. + Insert(_float16Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + + using SqlConnection sourceConnection = new(_connectionString); + sourceConnection.Open(); + using SqlCommand selectCommand = new($"SELECT {ColumnName} FROM {_float16Table.Name}", sourceConnection); + using SqlDataReader sourceReader = selectCommand.ExecuteReader(); + + using SqlConnection destinationConnection = new(_connectionString); + destinationConnection.Open(); + + using (SqlBulkCopy bulkCopy = new(destinationConnection) { DestinationTableName = _float32Table.Name }) { bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); bulkCopy.WriteToServer(sourceReader); } using SqlCommand verifyCommand = - new($"SELECT TOP 1 {ColumnName} FROM {destination.Name} ORDER BY Id DESC", destinationConnection); + new($"SELECT TOP 1 {ColumnName} FROM {_float32Table.Name} ORDER BY Id DESC", destinationConnection); using SqlDataReader verifyReader = verifyCommand.ExecuteReader(); Assert.True(verifyReader.Read()); Assert.Equal([1.5f, 2.5f, 3.5f], verifyReader.GetSqlVector(0).Memory.ToArray()); } + #endif [ConditionalTheory(nameof(IsSupported))] - [InlineData("float16", "float32")] - [InlineData("float32", "float16")] - [InlineData("float16", "float16")] - public void BulkCopyPreservesNullsBetweenColumnsOfEitherBaseType(string sourceBaseType, string destinationBaseType) + [InlineData("float16")] + [InlineData("float32")] + public void BulkCopyPreservesNullsBetweenColumnsOfTheSameBaseType(string baseType) { - Table source = sourceBaseType == "float16" ? _float16Table : _float32Table; - Table destination = destinationBaseType == "float16" ? _float16Table : _float32Table; + Table table = baseType == "float16" ? _float16Table : _float32Table; // Interleaved, so that a row's nullness cannot be satisfied by position alone. - Insert(source, DBNull.Value); - Insert(source, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); - Insert(source, DBNull.Value); - Insert(source, DBNull.Value); + Insert(table, DBNull.Value); + Insert(table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + Insert(table, DBNull.Value); + Insert(table, DBNull.Value); using SqlConnection sourceConnection = new(_connectionString); sourceConnection.Open(); using SqlCommand selectCommand = - new($"SELECT {ColumnName} FROM {source.Name} ORDER BY Id", sourceConnection); + new($"SELECT {ColumnName} FROM {table.Name} ORDER BY Id", sourceConnection); using SqlDataReader sourceReader = selectCommand.ExecuteReader(); using SqlConnection destinationConnection = new(_connectionString); destinationConnection.Open(); - using (SqlBulkCopy bulkCopy = new(destinationConnection) { DestinationTableName = destination.Name }) + using (SqlBulkCopy bulkCopy = new(destinationConnection) { DestinationTableName = table.Name }) { bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); bulkCopy.WriteToServer(sourceReader); } using SqlCommand verifyCommand = - new($"SELECT TOP 4 {ColumnName} FROM {destination.Name} ORDER BY Id DESC", destinationConnection); + new($"SELECT TOP 4 {ColumnName} FROM {table.Name} ORDER BY Id DESC", destinationConnection); using SqlDataReader verifyReader = verifyCommand.ExecuteReader(); // Read back in descending order, so the expected pattern is the reverse of the @@ -371,12 +430,68 @@ public void BulkCopiesJsonStringSourceIntoFloat16Column() Assert.Equal([1.5f, 2.5f, 3.5f], reader.GetSqlVector(0).Memory.ToArray()); } + [ConditionalFact(nameof(IsSupported))] + public void BulkCopiesJsonStringSourceIntoFloat32ColumnAtV1() + { + // Control: at v1 a float32 column IS presented as a vector, so the declaration says + // vector(N) and the string is coerced to a float32 payload by the client. + string v1 = new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString) + { + VectorTypeSupport = SqlVectorTypeSupport.V1 + }.ConnectionString; + + DataTable table = new(); + table.Columns.Add(ColumnName, typeof(string)); + table.Rows.Add("[1.5,2.5,3.5]"); + + using SqlConnection connection = new(v1); + connection.Open(); + + using (SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = _float32Table.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(table); + } + + using SqlCommand command = + new($"SELECT TOP 1 CAST({ColumnName} AS varchar(100)) FROM {_float32Table.Name} ORDER BY Id DESC", connection); + Assert.Contains("1.5", (string)command.ExecuteScalar()); + } + + [ConditionalFact(nameof(IsSupported))] + public void BulkCopiesJsonStringSourceIntoFloat16ColumnAtV1() + { + // At v1 the server presents a float16 column as varchar(max), so the ordinary text + // path applies and the server performs the conversion. + string v1 = new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString) + { + VectorTypeSupport = SqlVectorTypeSupport.V1 + }.ConnectionString; + + DataTable table = new(); + table.Columns.Add(ColumnName, typeof(string)); + table.Rows.Add("[1.5,2.5,3.5]"); + + using SqlConnection connection = new(v1); + connection.Open(); + + using (SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = _float16Table.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(table); + } + + using SqlCommand command = + new($"SELECT TOP 1 CAST({ColumnName} AS varchar(100)) FROM {_float16Table.Name} ORDER BY Id DESC", connection); + Assert.Contains("1.5", (string)command.ExecuteScalar()); + } + [ConditionalFact(nameof(IsSupported))] public void BulkCopyRejectsValuesOutsideTheFloat16Range() { DataTable table = new(); - table.Columns.Add(ColumnName, typeof(SqlVector)); - table.Rows.Add(new SqlVector(new float[] { 70000f, 1f, 2f })); + table.Columns.Add(ColumnName, typeof(string)); + table.Rows.Add("[70000,1,2]"); using SqlConnection connection = new(_connectionString); connection.Open(); @@ -384,9 +499,10 @@ public void BulkCopyRejectsValuesOutsideTheFloat16Range() using SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = _float16Table.Name }; bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); - // The client narrows the payload here, so it reports the overflow itself rather - // than letting the saturated infinity reach the server, which would reject it as a - // malformed vector instead. Bulk copy wraps the failure to name the column and row. + // A textual source is parsed into the destination's base type by the client, so the + // client reports the overflow itself rather than letting the saturated infinity + // reach the server, which would reject it as a malformed vector instead. Bulk copy + // wraps the failure to name the column and row. InvalidOperationException exception = Assert.Throws(() => bulkCopy.WriteToServer(table)); diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs index e1fe0229c4..d41e125045 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs @@ -319,104 +319,6 @@ public void GetString_Null_RendersNullString() #endregion - #region Payload Conversion Tests - - [Fact] - public void ConvertPayload_Float32ToFloat16() - { - byte[] source = ((ISqlVector)new SqlVector(new[] { 1.5f, 2.5f, 3.5f })).VectorPayload; - - byte[] converted = SqlVectorPayload.ConvertElementType( - source, - (byte)MetaType.SqlVectorElementType.Float16); - - Assert.Equal(0x01, converted[4]); - Assert.Equal(TdsEnums.VECTOR_HEADER_SIZE + (3 * 2), converted.Length); - - // Reading the converted payload back yields the original values, because all three - // are exactly representable in binary16. - byte[] roundTripped = SqlVectorPayload.ConvertElementType( - converted, - (byte)MetaType.SqlVectorElementType.Float32); - - Assert.Equal(new[] { 1.5f, 2.5f, 3.5f }, new SqlVector(roundTripped).Memory.ToArray()); - } - - [Fact] - public void ConvertPayload_Float16ToFloat32_WidensExactly() - { - // Built directly rather than through SqlVector, so that the test also runs on - // frameworks without System.Half. - byte[] source = MakeFloat16Payload(new[] { 1.5f, 2.5f, 3.5f }); - - byte[] converted = SqlVectorPayload.ConvertElementType( - source, - (byte)MetaType.SqlVectorElementType.Float32); - - Assert.Equal(0x00, converted[4]); - Assert.Equal(new[] { 1.5f, 2.5f, 3.5f }, new SqlVector(converted).Memory.ToArray()); - } - - [Fact] - public void ConvertPayload_SameElementType_ReturnsInput() - { - byte[] source = ((ISqlVector)new SqlVector(new[] { 1.5f })).VectorPayload; - - Assert.Same( - source, - SqlVectorPayload.ConvertElementType(source, (byte)MetaType.SqlVectorElementType.Float32)); - } - - [Fact] - public void ConvertPayload_NarrowingRounds() - { - byte[] source = ((ISqlVector)new SqlVector(new[] { 1.1f })).VectorPayload; - - byte[] narrowed = SqlVectorPayload.ConvertElementType( - source, - (byte)MetaType.SqlVectorElementType.Float16); - - byte[] widened = SqlVectorPayload.ConvertElementType( - narrowed, - (byte)MetaType.SqlVectorElementType.Float32); - - Assert.Equal(1.0996094f, new SqlVector(widened).Memory.Span[0]); - } - - [Fact] - public void ConvertPayload_ShortHeader_Throws() - { - Assert.Throws(() => - SqlVectorPayload.ConvertElementType( - new byte[] { 0xA9, 0x01 }, - (byte)MetaType.SqlVectorElementType.Float16)); - } - - [Fact] - public void ConvertPayload_LengthMismatch_Throws() - { - // The header declares two elements, but only one is present. - byte[] source = MakeTdsPayloadStatic( - new byte[] { 0xA9, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00 }, - new[] { 1.5f }); - - Assert.Throws(() => - SqlVectorPayload.ConvertElementType( - source, - (byte)MetaType.SqlVectorElementType.Float16)); - } - - [Fact] - public void ConvertPayload_UnsupportedElementType_Throws() - { - byte[] source = ((ISqlVector)new SqlVector(new[] { 1.5f })).VectorPayload; - - Assert.Throws(() => - SqlVectorPayload.ConvertElementType(source, 0x7F)); - } - - #endregion - #region Widening Read Tests [Fact] diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs index e1b0a781b4..f9452371e6 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs @@ -831,8 +831,9 @@ public void ConnectionRefusesUnsupportedServerTdsVersion(int major, int minor, i // Test to verify that the server and client negotiate // the common feature extension version. - // MDS currently supports vector feature ext version 0x2, - // which adds the float16 base type to the float32 support in 0x1. + // The connection requests the version configured by the Vector Type Support + // keyword, which defaults to v1. These cases opt in to v2, which adds the + // float16 base type to the float32 support in v1. [Theory] // A server which supports the same version as the client negotiates that version. [InlineData(true, 0x2, 0x2)] @@ -907,6 +908,8 @@ public void TestConnWithVectorFeatExtVersionNegotiation(bool expectedConnectionR DataSource = $"localhost,{server.EndPoint.Port}", Encrypt = SqlConnectionEncryptOption.Optional, Pooling = false, // Disable pooling so an expected failure does not poison a shared pool + // The keyword defaults to v1, so opt in to the version under test. + VectorTypeSupport = SqlVectorTypeSupport.V2, }.ConnectionString; using var connection = new SqlConnection(connStr); if (expectedConnectionResult) @@ -974,6 +977,9 @@ public void TestConnRejectsVectorFeatExtVersionAboveClientCeiling(byte serverVer DataSource = $"localhost,{server.EndPoint.Port}", Encrypt = SqlConnectionEncryptOption.Optional, Pooling = false, // Disable pooling so this expected failure does not poison a shared pool + // The ceiling being tested is the client's, so request the highest version + // the client understands. + VectorTypeSupport = SqlVectorTypeSupport.V2, }.ConnectionString; using SqlConnection connection = new(connStr); @@ -985,6 +991,62 @@ public void TestConnRejectsVectorFeatExtVersionAboveClientCeiling(byte serverVer Assert.Equal(serverVersion, acknowledgedVersion); } + // Test that the vector feature extension version requested at login follows the + // Vector Type Support keyword, and that the request is omitted entirely when the + // keyword asks for no vector support. + [Theory] + // The keyword defaults to v1, so an application which says nothing keeps the + // representation it had before float16 existed. + [InlineData(null, true, 0x1)] + [InlineData(SqlVectorTypeSupport.V1, true, 0x1)] + [InlineData(SqlVectorTypeSupport.V2, true, 0x2)] + // Off omits the feature request, leaving vector columns as varchar(max). + [InlineData(SqlVectorTypeSupport.Off, false, 0x0)] + public void TestVectorFeatExtVersionFollowsConnectionString( + SqlVectorTypeSupport? setting, + bool expectRequest, + byte expectedRequestedVersion) + { + using TdsServer server = new(); + server.Start(); + server.EnableVectorFeatureExt = true; + server.ServerSupportedVectorFeatureExtVersion = 0x2; + + bool requestSeen = false; + byte requestedVersion = 0; + + server.OnLogin7Validated = loginToken => + { + TDSLogin7GenericOptionToken option = loginToken.FeatureExt? + .OfType() + .FirstOrDefault(t => t.FeatureID == TDSFeatureID.VectorSupport)!; + + if (option != null) + { + requestSeen = true; + requestedVersion = option.Data[0]; + } + }; + + SqlConnectionStringBuilder builder = new() + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + Pooling = false, + }; + + if (setting.HasValue) + { + builder.VectorTypeSupport = setting.Value; + } + + using SqlConnection connection = new(builder.ConnectionString); + connection.Open(); + + Assert.Equal(expectRequest, requestSeen); + Assert.Equal(expectedRequestedVersion, requestedVersion); + } + // Test that the driver sends the UserAgent feature extension when // the context switch is enabled, and that the presence or absence of // an ack from the server has no effect. From 39d84b34ac0116152b6b1b5d1933fff3f6a1da88 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 25 Aug 2026 09:35:55 +0530 Subject: [PATCH 09/24] Document the cost of float16 on .NET Framework System.Half is unavailable there, so the driver widens each element of a float16 column as it reads, and the string read paths serialize the widened values as a JSON array. On .NET a SqlVector wraps the payload the server sent, and no per-element conversion takes place. Note this where an application chooses to opt in, and point readers which read float16 columns in bulk on .NET Framework at GetSqlVector, which widens without also serializing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .github/instructions/features.instructions.md | 13 +++++++++++++ doc/samples/SqlVectorFloat16Example.cs | 7 +++++-- .../SqlVectorTypeSupport.xml | 7 +++++++ doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml | 10 ++++++++++ 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/instructions/features.instructions.md b/.github/instructions/features.instructions.md index 4ca29da34d..1fdd41b91d 100644 --- a/.github/instructions/features.instructions.md +++ b/.github/instructions/features.instructions.md @@ -174,6 +174,19 @@ Notes: } ``` +> [!NOTE] +> On .NET Framework, `float16` workloads do more work per value than they do on .NET. +> Because `System.Half` isn't available, the driver widens each element to single +> precision when it reads a `float16` column, and renders the result as a JSON array for +> the string read paths. On .NET, a `SqlVector` wraps the payload the server sent, +> so no per-element conversion takes place. Writes differ in the same way: .NET can send a +> `SqlVector` unchanged, while .NET Framework sends a JSON string or a +> `SqlVector`, which the driver converts to the column's base type. +> +> If you read `float16` columns in bulk on .NET Framework, prefer +> `GetSqlVector` over the string read paths. Both widen the elements, but only the +> string paths also serialize the result. + #### Vector Feature Extension Versions The vector base types available on a connection are negotiated through the `VECTORSUPPORT` diff --git a/doc/samples/SqlVectorFloat16Example.cs b/doc/samples/SqlVectorFloat16Example.cs index ae6266d778..f3c64c1d2d 100644 --- a/doc/samples/SqlVectorFloat16Example.cs +++ b/doc/samples/SqlVectorFloat16Example.cs @@ -118,13 +118,16 @@ private static async Task ReadVectorsAsync(SqlConnection conn) int id = reader.GetInt32(0); #if NET - // On .NET, the column's own base type is available directly. + // On .NET, the column's own base type is available directly. A SqlVector + // wraps the payload the server sent, so no per-element conversion takes place. SqlVector exact = reader.GetSqlVector(1); Console.WriteLine($" Id={id} as Half: [{string.Join(", ", exact.Memory.ToArray())}]"); #endif // On any framework, the elements can be widened to single precision. Widening - // from float16 is exact, so no information is lost. + // from float16 is exact, so no information is lost. On .NET Framework this is + // the cheapest strongly typed read: the string read paths widen the elements + // as well, and then serialize the result as a JSON array. SqlVector widened = reader.GetSqlVector(1); Console.WriteLine($" Id={id} as float: [{string.Join(", ", widened.Memory.ToArray())}]"); diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlVectorTypeSupport.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlVectorTypeSupport.xml index 98196d314c..108ba5acf9 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlVectorTypeSupport.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlVectorTypeSupport.xml @@ -40,6 +40,13 @@ System.Half is unavailable, it is surfaced as a JSON array string and can be read as a SqlVector<float> on request. + + On .NET Framework, the driver widens each element of a float16 column to single + precision as it reads, and renders the result as a JSON array for the string read + paths. On .NET, a SqlVector<Half> wraps the payload the server sent, so no + per-element conversion takes place. Take this into account when you choose this value + for a .NET Framework application which reads float16 columns in bulk. + diff --git a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml index a95ddea4af..ed33d129a9 100644 --- a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml +++ b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml @@ -23,6 +23,16 @@ for it through the Vector Type Support keyword, which defaults to v1. Otherwise such a column is returned as a varchar(max) containing a JSON array. + + On .NET Framework, float16 workloads do more work per value than they do on + .NET. Because isn't available, the driver widens each + element to single precision when it reads a float16 column, and renders the + result as a JSON array for the string read paths. On .NET, a + SqlVector<Half> wraps the payload the server sent, so no per-element + conversion takes place. If you read float16 columns in bulk on .NET Framework, + prefer GetSqlVector<float> over the string read paths: both widen the + elements, but only the string paths also serialize the result. + Conversion between base types is performed by SQL Server, which reports an error if a value is outside the range the destination can represent. Converting from From ba86fe08a3e7b8124f8c78779aa5af1b1d13b020 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 25 Aug 2026 09:49:47 +0530 Subject: [PATCH 10/24] Read the bulk copy source column type through IDataReader GetSourceColumnType read the field type from _sqlDataReaderRowSource, which is assigned with "as SqlDataReader" and is therefore null for a reader from any other provider. A textual column from such a reader was not recognised as one, so its value was left as the float32 payload that coercion produces and the server rejected the row with a column length error. Every reader source implements IDataReader, and GetFieldType is declared by IDataRecord, so read it from there instead. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 5 +++- .../VectorTest/VectorFloat16BehaviourTests.cs | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 850402ef80..e64da4aea5 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -1442,7 +1442,10 @@ private Type GetSourceColumnType(int sourceOrdinal) { case ValueSourceType.DbDataReader: case ValueSourceType.IDataReader: - return _sqlDataReaderRowSource?.GetFieldType(sourceOrdinal); + // Not _sqlDataReaderRowSource, which is null unless the reader is a + // SqlDataReader. Every reader source implements IDataReader, and + // GetFieldType is declared by IDataRecord. + return (_rowSource as IDataReader)?.GetFieldType(sourceOrdinal); case ValueSourceType.DataTable: case ValueSourceType.RowArray: diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index ede14c36a8..81d5cfc867 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -430,6 +430,34 @@ public void BulkCopiesJsonStringSourceIntoFloat16Column() Assert.Equal([1.5f, 2.5f, 3.5f], reader.GetSqlVector(0).Memory.ToArray()); } + [ConditionalFact(nameof(IsSupported))] + public void BulkCopiesJsonStringSourceFromANonSqlClientReader() + { + // The source column type has to be read through IDataReader rather than through the + // SqlDataReader field, which is null unless the reader is a SqlDataReader. A reader + // from another provider still reports a string column, so the value is parsed into + // the destination's base type as it is for any other textual source. + DataTable table = new(); + table.Columns.Add(ColumnName, typeof(string)); + table.Rows.Add(DBNull.Value); + table.Rows.Add("[1.5,2.5,3.5]"); + + using IDataReader reader = table.CreateDataReader(); + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using (SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = _float16Table.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(reader); + } + + using SqlDataReader verify = Select(_float16Table); + Assert.True(verify.Read()); + Assert.Equal([1.5f, 2.5f, 3.5f], verify.GetSqlVector(0).Memory.ToArray()); + } + [ConditionalFact(nameof(IsSupported))] public void BulkCopiesJsonStringSourceIntoFloat32ColumnAtV1() { From 0acf1e529182615c79581d892f4db56dd71adf43 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 25 Aug 2026 10:28:30 +0530 Subject: [PATCH 11/24] Make the vector tests portable to Azure SQL Two tests encoded behaviour of the SQL Server 2025 build they were written against, and failed on every Azure SQL leg. Rows in the DataTypes schema collection are filtered by the version the server reports, and the vector row is declared from 17.00 onwards. Azure SQL reports 12.00 whatever it supports, so the row is filtered out there even though the type is available. The json type, declared the same way, has the same gap. Skip the test where the version is not meaningful. An out of range narrowing to float16 is reported as error 42284 by SQL Server 2025 and 42241 by Azure SQL. Both describe the same overflow, so accept either rather than the one number that happened to be observed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../SQL/VectorTest/VectorColumnMetadataTests.cs | 11 ++++++++++- .../SQL/VectorTest/VectorFloat16BehaviourTests.cs | 8 +++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs index 5701bab2dd..efffebc3d2 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs @@ -24,6 +24,15 @@ public sealed class VectorColumnMetadataTests public static bool IsFloat16Supported => DataTestUtility.IsSqlVectorFloat16Supported; + /// + /// Whether the server reports a version which reflects the features it has. Rows in the + /// DataTypes schema collection are filtered by the version the server reports, and the + /// vector row is declared from 17.00 onwards. Azure SQL reports 12.00 whatever it + /// supports, so the row is filtered out there even though the type is available. The + /// json type, which is declared the same way, has the same gap. + /// + public static bool ReportsItsRealVersion => DataTestUtility.IsNotAzureServer(); + [ConditionalTheory(nameof(IsSupported))] [InlineData(1)] [InlineData(3)] @@ -82,7 +91,7 @@ public void ReportsStandardPropertiesAlongsideVectorProperties() Assert.Null(column["NoSuchProperty"]); } - [ConditionalFact(nameof(IsSupported))] + [ConditionalFact(nameof(IsSupported), nameof(ReportsItsRealVersion))] public void SchemaCollectionIncludesVectorType() { using SqlConnection connection = new(_connectionString); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index 81d5cfc867..cffb24a487 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -28,9 +28,11 @@ public sealed class VectorFloat16BehaviourTests : IDisposable private const string ParameterName = "@VectorData"; /// - /// The server's error for a float32 value which cannot be narrowed to float16. + /// The server's errors for a value which cannot be narrowed to float16. Both describe + /// the same overflow, and which one is raised depends on the server build: SQL Server + /// 2025 reports 42284 for a vector parameter where Azure SQL reports 42241. /// - private const int Float32ToFloat16OutOfRangeError = 42284; + private static readonly int[] s_float16OutOfRangeErrors = [42284, 42241]; private readonly string _connectionString = DataTestUtility.VectorFloat16ConnectionString; private readonly SqlConnection _managementConnection; @@ -253,7 +255,7 @@ public void RejectsValuesOutsideTheFloat16Range() SqlException exception = Assert.Throws(() => Insert(_float16Table, new SqlVector(new float[] { 70000f, 1f, 2f }))); - Assert.Equal(Float32ToFloat16OutOfRangeError, exception.Number); + Assert.Contains(exception.Number, s_float16OutOfRangeErrors); } #endregion From 31dc266294034a684d57d0356b47c6ce1359a2e7 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 25 Aug 2026 10:44:53 +0530 Subject: [PATCH 12/24] Report the vector type from what the connection negotiated The vector row in the DataTypes schema collection was declared from server version 17.00 onwards, and rows in that collection are filtered by the version the server reports. Azure SQL reports 12.00 whatever it supports, so the row was filtered out there even though the type was available, and the test covering it failed on every Azure leg. The metadata factory is already given the connection's capabilities, which carry the version negotiated through the VECTORSUPPORT feature extension. That is what decides whether vector columns are read as vectors, so report the type from it rather than inferring it from a version string. A connection which opted out with Vector Type Support=off reads vector columns as varchar(max), and so no longer reports a vector type. The json type is declared the same way and has the same gap on Azure SQL, but its behaviour has shipped, so it is left alone here. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../SqlClient/SqlMetaDataFactory.DataTypes.cs | 12 +++-- .../Data/SqlClient/SqlMetaDataFactory.cs | 9 +++- .../VectorTest/VectorColumnMetadataTests.cs | 48 ++++++++++++------- 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs index 91352d1677..12d6ae465d 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs @@ -13,7 +13,7 @@ namespace Microsoft.Data.SqlClient; internal sealed partial class SqlMetaDataFactory { - private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet) + private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet, byte vectorVersion) { DataTable dataTypesDataTable = CreateDataTypesDataTable(); @@ -75,7 +75,14 @@ private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet) AddUniqueIdentifierType(); AddSqlVariantType(); AddRowVersionType(); - AddVectorType(); + + // A vector type is reported when the connection has negotiated one, rather than + // when the server reports a version which is new enough to have it. Azure SQL + // reports 12.00 whatever it supports, so a version is not enough to tell. + if (vectorVersion != TdsEnums.VECTOR_VERSION_NOT_SUPPORTED) + { + AddVectorType(); + } dataTypesDataTable.EndLoadData(); dataTypesDataTable.AcceptChanges(); @@ -384,7 +391,6 @@ void AddVectorType() // A vector literal is written as a JSON array in a string literal. typeRow[DbMetaDataColumnNames.LiteralPrefix] = "'"; typeRow[DbMetaDataColumnNames.LiteralSuffix] = "'"; - typeRow[MinimumVersionKey] = "17.00.000.0"; dataTypesDataTable.Rows.Add(typeRow); } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs index bea2fa7e41..8aaaece3d6 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs @@ -40,6 +40,12 @@ internal sealed partial class SqlMetaDataFactory : IDisposable private readonly DataSet _collectionDataSet; private readonly string _serverVersion; + /// + /// The version negotiated via the VECTORSUPPORT feature extension, which decides + /// whether a vector type is reported in the DataTypes collection. + /// + private readonly byte _vectorVersion; + public SqlMetaDataFactory(Stream xmlStream, ConnectionCapabilities connectionCapabilities) { ADP.CheckArgumentNull(xmlStream, nameof(xmlStream)); @@ -47,6 +53,7 @@ public SqlMetaDataFactory(Stream xmlStream, ConnectionCapabilities connectionCap ADP.CheckArgumentNull(connectionCapabilities.ServerVersion, nameof(connectionCapabilities.ServerVersion)); _serverVersion = connectionCapabilities.ServerVersion; + _vectorVersion = connectionCapabilities.VectorVersion; _collectionDataSet = LoadDataSetFromXml(xmlStream); } @@ -711,7 +718,7 @@ private DataSet LoadDataSetFromXml(Stream XmlStream) Locale = CultureInfo.InvariantCulture }; - LoadDataTypesDataTables(metaDataCollectionsDataSet); + LoadDataTypesDataTables(metaDataCollectionsDataSet, _vectorVersion); XmlReaderSettings settings = new() { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs index efffebc3d2..8624e90499 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs @@ -24,15 +24,6 @@ public sealed class VectorColumnMetadataTests public static bool IsFloat16Supported => DataTestUtility.IsSqlVectorFloat16Supported; - /// - /// Whether the server reports a version which reflects the features it has. Rows in the - /// DataTypes schema collection are filtered by the version the server reports, and the - /// vector row is declared from 17.00 onwards. Azure SQL reports 12.00 whatever it - /// supports, so the row is filtered out there even though the type is available. The - /// json type, which is declared the same way, has the same gap. - /// - public static bool ReportsItsRealVersion => DataTestUtility.IsNotAzureServer(); - [ConditionalTheory(nameof(IsSupported))] [InlineData(1)] [InlineData(3)] @@ -91,27 +82,48 @@ public void ReportsStandardPropertiesAlongsideVectorProperties() Assert.Null(column["NoSuchProperty"]); } - [ConditionalFact(nameof(IsSupported), nameof(ReportsItsRealVersion))] + [ConditionalFact(nameof(IsSupported))] public void SchemaCollectionIncludesVectorType() { using SqlConnection connection = new(_connectionString); connection.Open(); - DataTable dataTypes = connection.GetSchema("DataTypes"); - DataRow? vectorRow = null; + DataRow? vectorRow = FindVectorType(connection); + + Assert.NotNull(vectorRow); + Assert.Equal((int)SqlDbTypeExtensions.Vector, vectorRow!["ProviderDbType"]); + Assert.Equal("vector({0})", vectorRow["CreateFormat"]); + } - foreach (DataRow row in dataTypes.Rows) + [ConditionalFact(nameof(IsSupported))] + public void SchemaCollectionOmitsVectorTypeWhenItIsNotNegotiated() + { + // The type is reported according to what the connection negotiated, not according + // to the version the server reports: Azure SQL reports 12.00 whatever it supports. + // A connection which opted out reads vector columns as varchar(max), so it has no + // vector type to report. + string connectionString = new SqlConnectionStringBuilder(_connectionString) + { + VectorTypeSupport = SqlVectorTypeSupport.Off + }.ConnectionString; + + using SqlConnection connection = new(connectionString); + connection.Open(); + + Assert.Null(FindVectorType(connection)); + } + + private static DataRow? FindVectorType(SqlConnection connection) + { + foreach (DataRow row in connection.GetSchema("DataTypes").Rows) { if (string.Equals(row["TypeName"]?.ToString(), "vector", StringComparison.OrdinalIgnoreCase)) { - vectorRow = row; - break; + return row; } } - Assert.NotNull(vectorRow); - Assert.Equal((int)SqlDbTypeExtensions.Vector, vectorRow!["ProviderDbType"]); - Assert.Equal("vector({0})", vectorRow["CreateFormat"]); + return null; } [ConditionalFact(nameof(IsFloat16Supported))] From ee6a4b059e1000602048a8705b2cb66bfae07d0c Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 25 Aug 2026 18:45:38 +0530 Subject: [PATCH 13/24] Accept VectorTypeSupport as a connection string keyword Every keyword written with spaces has a synonym written without them: Trust Server Certificate is also trustservercertificate, Host Name In Certificate is also hostnameincertificate, and so on. Vector Type Support was registered without one, so VectorTypeSupport was rejected as an unsupported keyword. That is also the spelling the JDBC driver uses, so it is the form an application ported from it would be written with. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .github/instructions/features.instructions.md | 3 ++- .../SqlConnectionStringBuilder.xml | 4 ++-- .../DbConnectionStringSynonyms.cs | 1 + .../Data/SqlClient/SqlConnectionOptions.cs | 3 ++- .../SqlClient/SqlConnectionStringBuilder.cs | 1 + .../SqlConnectionStringBuilderTest.cs | 17 +++++++++++++++++ 6 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/instructions/features.instructions.md b/.github/instructions/features.instructions.md index 1fdd41b91d..521a312d62 100644 --- a/.github/instructions/features.instructions.md +++ b/.github/instructions/features.instructions.md @@ -199,7 +199,8 @@ feature extension (`0x0E`): | `2` | `float16` is supported in addition to `float32`. | The version requested at login is chosen by the `Vector Type Support` connection string -keyword, and the server acknowledges the highest version they have in common: +keyword, which may also be written `VectorTypeSupport`, and the server acknowledges the +highest version they have in common: | Keyword value | Requested version | |---------------|-------------------| diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml index 163496b8da..d0de2d981a 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml @@ -1232,8 +1232,8 @@ The following example converts an existing connection string from using SQL Serv . - Corresponds to the Vector Type Support connection string keyword, whose - values are off, v1 and v2. + Corresponds to the Vector Type Support and VectorTypeSupport keys + within the connection string, whose values are off, v1 and v2. The value is not a member of . diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringSynonyms.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringSynonyms.cs index 15c32abe75..0f20e6c5db 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringSynonyms.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionStringSynonyms.cs @@ -42,6 +42,7 @@ internal static class DbConnectionStringSynonyms internal const string TrustServerCertificate = "trustservercertificate"; internal const string Uid = "uid"; internal const string User = "user"; + internal const string VectorTypeSupport = "vectortypesupport"; internal const string WorkstationId = "workstationid"; internal const string WsId = "wsid"; } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs index fd03412c18..571ab4694b 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs @@ -231,7 +231,8 @@ static SqlConnectionOptions() DbConnectionStringSynonyms.Uid, DbConnectionStringSynonyms.User); AddKeywordToMap(DbConnectionStringKeywords.UserInstance); - AddKeywordToMap(DbConnectionStringKeywords.VectorTypeSupport); + AddKeywordToMap(DbConnectionStringKeywords.VectorTypeSupport, + DbConnectionStringSynonyms.VectorTypeSupport); AddKeywordToMap(DbConnectionStringKeywords.WorkstationId, DbConnectionStringSynonyms.WorkstationId, DbConnectionStringSynonyms.WsId); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs index ea6fe49e24..6145dfb89f 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs @@ -288,6 +288,7 @@ private static Dictionary CreateKeywordsDictionary() { DbConnectionStringSynonyms.ConnectTimeout, Keywords.ConnectTimeout }, { DbConnectionStringSynonyms.FailoverPartner, Keywords.FailoverPartner }, { DbConnectionStringSynonyms.PacketSize, Keywords.PacketSize }, + { DbConnectionStringSynonyms.VectorTypeSupport, Keywords.VectorTypeSupport }, }; return pairs; } diff --git a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConnectionStringBuilderTest.cs b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConnectionStringBuilderTest.cs index 180fe07209..9e31055292 100644 --- a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConnectionStringBuilderTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConnectionStringBuilderTest.cs @@ -1015,6 +1015,23 @@ public void PacketSizeSynonymsResolveCorrectly(string connectionString, int expe Assert.Equal(expected, builder.PacketSize); } + [Theory] + [InlineData("Vector Type Support = V2")] + [InlineData("VectorTypeSupport = V2")] + [InlineData("vectortypesupport = V2")] + [InlineData("VECTORTYPESUPPORT = V2")] + public void VectorTypeSupportSynonymsResolveCorrectly(string connectionString) + { + SqlConnectionStringBuilder builder = new(connectionString); + Assert.Equal(SqlVectorTypeSupport.V2, builder.VectorTypeSupport); + + // A connection parses its keywords separately from the builder, so it is checked + // separately too. Constructing it rejects an unrecognised keyword, and the string + // it was given is preserved as written. + SqlConnection connection = new(connectionString); + Assert.Equal(connectionString, connection.ConnectionString); + } + [Theory] [InlineData("WorkstationID = myws")] [InlineData("workstationid = myws")] From 45f4fd7e209fd50717e6f837395bed8004c24020 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Wed, 26 Aug 2026 00:32:01 +0530 Subject: [PATCH 14/24] Describe what a textual bulk copy source actually does The comment on IsTextSourcedVectorColumn described an earlier design in which the column was declared as a varchar(max) and the server parsed the JSON array. The server rejects that declaration for a vector column, so the client parses the text and rewrites the payload to the destination's base type instead. The call site already says so; only this comment was left behind. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../src/Microsoft/Data/SqlClient/SqlBulkCopy.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index e64da4aea5..dfc424474a 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -1458,14 +1458,18 @@ private Type GetSourceColumnType(int sourceOrdinal) /// /// Whether a vector destination column is supplied from a textual source, in which - /// case the value is transferred as text and converted by the server. + /// case the value is parsed into the destination's base type by the client. /// /// - /// A vector has no textual form on the wire, so such a column is declared as a - /// varchar(max) in the INSERT BULK statement. The server then parses - /// the JSON array into the destination's base type, which is the only way to write a - /// base type the client cannot represent, and avoids a conversion whose result could - /// differ from the server's. + /// A JSON array is always coerced to a float32 payload, so it has to be rewritten + /// when the destination's base type is not float32. The server does not convert + /// within the data stream: the INSERT BULK declaration states the + /// destination's base type, and elements of each base type differ in size. + /// + /// A payload read from another vector column is left as it is, so a copy between + /// columns of different base types is reported by the server rather than being + /// silently narrowed. + /// /// private bool IsTextSourcedVectorColumn(int sourceOrdinal, _SqlMetaData metadata) => metadata.type == SqlDbTypeExtensions.Vector && From 29028cbe7341223172ee143bb52e46ef4d3bb435 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Sat, 19 Sep 2026 03:20:28 +0530 Subject: [PATCH 15/24] Address review feedback Carry the vector type support setting through the SqlConnectionOptions copy constructor. That constructor is used for User Instance connections, and the enum's default is Off, so the setting was silently dropped there: the VECTORSUPPORT feature extension was suppressed and vector columns regressed to JSON strings even for the default V1. Covered by a test which fails without the fix. Make the float16 behaviour fixture's setup failure-safe, matching NativeVectorTestsBase. xUnit does not call Dispose when a constructor throws, so a table created before the failure would have been left behind, and cleanup now tolerates partially initialized fields. Document the float16 to float32 widening on GetSqlVector, whose remarks still said no conversions are performed, and note in IsTextSourcedVectorColumn that a float16 column on .NET Framework describes itself as a string and so takes the textual path rather than being rejected. Add the XML summaries the testing instructions require to the test types and methods added by this branch. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../SqlDataReader.xml | 17 ++++- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 6 +- .../Data/SqlClient/SqlConnectionOptions.cs | 1 + .../NativeVectorFloat16AsSingleTests.cs | 8 ++ .../VectorTest/NativeVectorFloat16Tests.cs | 13 ++++ .../VectorTest/VectorColumnMetadataTests.cs | 40 ++++++++++ .../VectorTest/VectorFloat16BehaviourTests.cs | 43 +++++++++-- .../SqlClient/SqlConnectionOptionsTest.cs | 26 +++++++ .../Data/SqlTypes/Float16ConverterTest.cs | 74 +++++++++++++++++++ .../Microsoft/Data/SqlTypes/SqlVectorTest.cs | 63 ++++++++++++++++ .../SimulatedServerTests/ConnectionTests.cs | 26 +++++++ 11 files changed, 307 insertions(+), 10 deletions(-) diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlDataReader.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlDataReader.xml index 10c67a4890..e63f77663c 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlDataReader.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlDataReader.xml @@ -981,8 +981,23 @@ The method retur The retrieved data is not compatible with the type. + + The column's base type cannot be represented by without losing precision. + - No conversions are performed; therefore, the data retrieved must already be a vector value, or an exception is generated. + + `, which is the only strongly typed way to read one on .NET Framework, +where `System.Half` does not exist. Narrowing is rejected with + rather than performed silently, so a `float32` column +cannot be read as `SqlVector`. + + ]]> + diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index d131ff9a57..80e648b0bc 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -1499,7 +1499,11 @@ private Type GetSourceColumnType(int sourceOrdinal) /// /// A payload read from another vector column is left as it is, so a copy between /// columns of different base types is reported by the server rather than being - /// silently narrowed. + /// silently narrowed. On .NET Framework a float16 column has no System.Half + /// to report, so it describes itself as a string and takes the textual path above: + /// such a copy is converted by the client rather than rejected. That divergence is + /// inherent to the base type having no native representation there, and is covered + /// by BulkCopiesFloat16ToFloat32ThroughTheTextualRepresentation. /// /// private bool IsTextSourcedVectorColumn(int sourceOrdinal, _SqlMetaData metadata) => diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs index 571ab4694b..164fc4876d 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs @@ -627,6 +627,7 @@ internal SqlConnectionOptions(SqlConnectionOptions connectionOptions, string dat _serverSPN = connectionOptions._serverSPN; _failoverPartnerSPN = connectionOptions._failoverPartnerSPN; _hostNameInCertificate = connectionOptions._hostNameInCertificate; + _vectorTypeSupport = connectionOptions._vectorTypeSupport; #if NETFRAMEWORK _connectionReset = connectionOptions._connectionReset; _transparentNetworkIPResolution = connectionOptions._transparentNetworkIPResolution; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs index 2720386130..e41442263b 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs @@ -8,6 +8,14 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.VectorTest; #nullable enable +/// +/// Supplies the native vector test matrix for a float16 column read and written +/// through the single precision representation. Every sample value is chosen to be exactly +/// representable in binary16 so that narrowing to the column and widening back is lossless, +/// which lets the shared matrix assert equality rather than tolerance. This is the only +/// strongly typed representation available on .NET Framework, which has no +/// System.Half. +/// public sealed class VectorFloat16AsSingleTestData : NativeVectorTestDataBase { // Every value is exactly representable in binary16, so it survives narrowing on the diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs index 86f4bb9913..950bdb21dd 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16Tests.cs @@ -12,6 +12,13 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.VectorTest; #nullable enable +/// +/// Supplies the native vector test matrix for a float16 column read and written as +/// SqlVector<Half>, which is the column's own base type and so travels without +/// conversion. The samples span the binary16 extremes, a subnormal, and a negative zero, and +/// every value is exactly representable, so it survives the round trip through the JSON +/// rendering that the string based read paths return. +/// public sealed class VectorFloat16TestData : NativeVectorTestDataBase { // Includes the extremes of the binary16 range, a subnormal, and a negative zero. @@ -60,6 +67,12 @@ public sealed class VectorFloat16TestData : NativeVectorTestDataBase public override string ConnectionString => DataTestUtility.VectorFloat16ConnectionString; } +/// +/// Runs the full native vector matrix against a float16 column using +/// SqlVector<Half>, the representation which matches the column's base type and +/// is therefore exchanged as the server sent it, with no per element conversion. Compiled +/// only for .NET, since System.Half does not exist on .NET Framework. +/// [Trait("Set", "3")] public sealed class NativeVectorFloat16Tests : NativeVectorTestsBase { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs index 8624e90499..74cf5878a0 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs @@ -24,6 +24,12 @@ public sealed class VectorColumnMetadataTests public static bool IsFloat16Supported => DataTestUtility.IsSqlVectorFloat16Supported; + /// + /// Verifies that a vector column reports its base type and dimension count under their + /// own property names. Both are encoded indirectly on the wire — the base type as the + /// numeric scale and the dimension count derived from the column size — so this is the + /// contract that spares callers from knowing that encoding. + /// [ConditionalTheory(nameof(IsSupported))] [InlineData(1)] [InlineData(3)] @@ -46,6 +52,10 @@ public void ReportsBaseTypeAndDimensions(int dimensions) Assert.Equal(dimensions, column["VectorDimensions"]); } + /// + /// Verifies that the vector properties read as null for columns which are not vectors, + /// so that a caller can probe any column without having to guard the call. + /// [ConditionalFact(nameof(IsSupported))] public void ReportsNullForNonVectorColumns() { @@ -63,6 +73,12 @@ public void ReportsNullForNonVectorColumns() } } + /// + /// Verifies that adding the vector properties leaves the standard column schema + /// properties intact, and that an unrecognised property name still returns null rather + /// than throwing. This guards against the new properties disturbing the existing + /// indexer contract. + /// [ConditionalFact(nameof(IsSupported))] public void ReportsStandardPropertiesAlongsideVectorProperties() { @@ -82,6 +98,11 @@ public void ReportsStandardPropertiesAlongsideVectorProperties() Assert.Null(column["NoSuchProperty"]); } + /// + /// Verifies that a connection which negotiated vector support reports the type in the + /// DataTypes schema collection, with the provider type and create format a caller would + /// use to declare a column. + /// [ConditionalFact(nameof(IsSupported))] public void SchemaCollectionIncludesVectorType() { @@ -95,6 +116,13 @@ public void SchemaCollectionIncludesVectorType() Assert.Equal("vector({0})", vectorRow["CreateFormat"]); } + /// + /// Verifies that a connection which opted out of vector support does not report the + /// type, because it reads vector columns as varchar(max) and so has no vector + /// type to offer. Paired with the test above, this pins the collection to what the + /// connection negotiated rather than to the version the server reports, which is what + /// makes it correct on Azure SQL. + /// [ConditionalFact(nameof(IsSupported))] public void SchemaCollectionOmitsVectorTypeWhenItIsNotNegotiated() { @@ -113,6 +141,11 @@ public void SchemaCollectionOmitsVectorTypeWhenItIsNotNegotiated() Assert.Null(FindVectorType(connection)); } + /// + /// Finds the vector row in a connection's DataTypes schema collection. + /// + /// An open connection whose schema is read. + /// The vector row, or when the type is not reported. private static DataRow? FindVectorType(SqlConnection connection) { foreach (DataRow row in connection.GetSchema("DataTypes").Rows) @@ -126,6 +159,13 @@ public void SchemaCollectionOmitsVectorTypeWhenItIsNotNegotiated() return null; } + /// + /// Verifies the use case the properties exist for: choosing a read path without knowing + /// the schema in advance. GetFieldType is not enough on its own, because it + /// reports string for a float16 column on .NET Framework just as it does for a varchar + /// one, and cannot distinguish the two base types at all for a caller which wants to + /// read both through a single representation. + /// [ConditionalFact(nameof(IsFloat16Supported))] public void DrivesReadPathForACallerWhichDoesNotKnowTheSchema() { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index cffb24a487..58484945e5 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -43,12 +43,24 @@ public sealed class VectorFloat16BehaviourTests : IDisposable public VectorFloat16BehaviourTests() { _managementConnection = new SqlConnection(_connectionString); - _managementConnection.Open(); - _float16Table = new Table(_managementConnection, "VectorF16BehaviourTable", - $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector(3, float16) NULL)"); - _float32Table = new Table(_managementConnection, "VectorF32BehaviourTable", - $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector(3, float32) NULL)"); + // NOTE: If this constructor throws, xUnit never calls Dispose, so any object already + // created (each of which has a GUID-based name) would be left in the database + // permanently. This mirrors NativeVectorTestsBase. + try + { + _managementConnection.Open(); + + _float16Table = new Table(_managementConnection, "VectorF16BehaviourTable", + $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector(3, float16) NULL)"); + _float32Table = new Table(_managementConnection, "VectorF32BehaviourTable", + $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector(3, float32) NULL)"); + } + catch + { + Dispose(); + throw; + } } public static bool IsSupported => DataTestUtility.IsSqlVectorFloat16Supported; @@ -575,13 +587,28 @@ public void Dispose() return; } - _float16Table.Dispose(); - _float32Table.Dispose(); - _managementConnection.Dispose(); + // Reachable from the constructor with the fields still unset, so each drop is + // null-tolerant and best-effort: failing to drop one object must not leak the others. + DisposeSafely(_float16Table); + DisposeSafely(_float32Table); + _managementConnection?.Dispose(); _disposed = true; GC.SuppressFinalize(this); } + private static void DisposeSafely(IDisposable? disposable) + { + try + { + disposable?.Dispose(); + } + catch + { + // Best-effort cleanup; the object is named with a GUID so a leak is not a + // correctness problem for other tests. + } + } + #endregion } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs index 4c15a65789..c6a737766f 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs @@ -155,5 +155,31 @@ public void TestMultiSubnetFailoverWithFailoverPartnerThrows() Assert.Throws(() => new SqlConnectionOptions(builder.ConnectionString)); } + + /// + /// Tests that the vector type support setting survives the copy constructor, which is + /// used for User Instance connections. The enum's default is Off, so a setting + /// which is not copied would silently suppress the VECTORSUPPORT feature extension and + /// return vector columns as JSON strings. + /// + [Theory] + [InlineData(SqlVectorTypeSupport.Off)] + [InlineData(SqlVectorTypeSupport.V1)] + [InlineData(SqlVectorTypeSupport.V2)] + public void TestVectorTypeSupportSurvivesCopyConstructor(SqlVectorTypeSupport setting) + { + SqlConnectionStringBuilder builder = new() + { + DataSource = "server", + VectorTypeSupport = setting + }; + + SqlConnectionOptions original = new(builder.ConnectionString); + Assert.Equal(setting, original.VectorTypeSupport); + + SqlConnectionOptions copy = new(original, "server\\instance", userInstance: true, setEnlistValue: null); + + Assert.Equal(setting, copy.VectorTypeSupport); + } } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs index 2060c4f9bc..c682792b5a 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs @@ -28,6 +28,12 @@ public class Float16ConverterTest #if NET + /// + /// Verifies the widening direction against System.Half exhaustively, over all + /// 65,536 binary16 bit patterns. Comparing bitwise rather than by value distinguishes + /// positive from negative zero and compares a NaN's sign and payload rather than only + /// its NaN-ness. + /// [Fact] public void ToSingle_MatchesHalf_ForEveryBitPattern() { @@ -46,6 +52,11 @@ public void ToSingle_MatchesHalf_ForEveryBitPattern() } } + /// + /// Verifies the narrowing direction against System.Half for every value which + /// binary16 can represent, so that a value read from a float16 column is written back + /// unchanged. + /// [Fact] public void FromSingle_MatchesHalf_ForEveryRepresentableValue() { @@ -62,6 +73,12 @@ public void FromSingle_MatchesHalf_ForEveryRepresentableValue() } } + /// + /// Verifies narrowing against System.Half for inputs drawn from the whole single + /// precision space, not just those binary16 can represent. This is what covers rounding, + /// overflow, and underflow for arbitrary application values, which the exhaustive + /// representable-value test above cannot reach. + /// [Fact] public void FromSingle_MatchesHalf_AcrossTheSinglePrecisionRange() { @@ -79,6 +96,11 @@ public void FromSingle_MatchesHalf_AcrossTheSinglePrecisionRange() } } + /// + /// Verifies that a NaN keeps its sign and payload in both directions rather than being + /// canonicalised, so the codec and System.Half cannot diverge on inputs an + /// application could legitimately send. + /// [Fact] public void ConvertsNaN_PreservingSignAndPayload() { @@ -107,6 +129,11 @@ public void ConvertsNaN_PreservingSignAndPayload() #region Round trips + /// + /// Verifies that a value binary16 can represent exactly survives a narrow then widen + /// round trip unchanged, covering the range's boundaries: the largest finite value, the + /// smallest normal, and the smallest subnormal. + /// [Theory] // Exactly representable values. [InlineData(0f)] @@ -122,6 +149,10 @@ public void RoundTrip_PreservesExactlyRepresentableValues(float value) Assert.Equal(value, Float16Converter.ManualToSingle(Float16Converter.ManualFromSingle(value))); } + /// + /// Verifies that negative zero stays negative through a round trip, which a naive + /// implementation loses by treating the value as equal to positive zero. + /// [Fact] public void RoundTrip_PreservesSignOfZero() { @@ -137,6 +168,11 @@ public void RoundTrip_PreservesSignOfZero() #region Rounding + /// + /// Verifies that a value binary16 cannot represent is rounded to the nearest one it can, + /// including at the top of the range where rounding up must still produce the largest + /// finite value rather than an infinity. + /// [Theory] // Values which are not representable are rounded to the nearest binary16 value. [InlineData(1.1f, 1.0996094f)] @@ -150,6 +186,11 @@ public void FromSingle_RoundsToNearest(float value, float expected) Assert.Equal(expected, Float16Converter.ManualToSingle(Float16Converter.ManualFromSingle(value))); } + /// + /// Verifies that a value exactly halfway between two binary16 values resolves towards + /// the one with an even mantissa, which is the IEEE 754 default and the rule + /// System.Half and SQL Server both follow. + /// [Fact] public void FromSingle_RoundsTiesToEven() { @@ -166,6 +207,11 @@ public void FromSingle_RoundsTiesToEven() #region Overflow and underflow + /// + /// Verifies that a value too large for binary16 saturates to positive infinity. Callers + /// which must reject such a value rather than store an infinity detect it from this + /// result, which is what the bulk copy write path relies on. + /// [Theory] [InlineData(70000f)] [InlineData(float.MaxValue)] @@ -174,6 +220,10 @@ public void FromSingle_SaturatesToInfinityOnOverflow(float value) Assert.Equal(0x7C00, Float16Converter.ManualFromSingle(value)); } + /// + /// Verifies that a value too negative for binary16 saturates to negative infinity, + /// the counterpart of the overflow case above. + /// [Theory] [InlineData(-70000f)] [InlineData(float.MinValue)] @@ -182,6 +232,10 @@ public void FromSingle_SaturatesToNegativeInfinityOnOverflow(float value) Assert.Equal(0xFC00, Float16Converter.ManualFromSingle(value)); } + /// + /// Verifies that a value below half the smallest subnormal flushes to zero rather than + /// rounding up to the smallest subnormal. + /// [Theory] // Below half of the smallest subnormal, so these round to zero rather than to it. [InlineData(1e-8f)] @@ -192,6 +246,12 @@ public void FromSingle_FlushesToZeroOnUnderflow(float value) Assert.Equal(0x0000, Float16Converter.ManualFromSingle(value)); } + /// + /// Verifies that subnormal binary16 values are produced rather than flushed to zero, + /// including the value just above half the smallest subnormal, which must round up to + /// it. Subnormals use a different encoding path from normals, so they are covered + /// separately. + /// [Fact] public void FromSingle_PreservesSubnormals() { @@ -205,6 +265,10 @@ public void FromSingle_PreservesSubnormals() #region Infinity and NaN + /// + /// Verifies that an infinity narrows to the binary16 infinity of the same sign, rather + /// than being confused with the overflow saturation which produces the same encoding. + /// [Fact] public void FromSingle_PreservesInfinity() { @@ -212,6 +276,10 @@ public void FromSingle_PreservesInfinity() Assert.Equal(0xFC00, Float16Converter.ManualFromSingle(float.NegativeInfinity)); } + /// + /// Verifies that a binary16 infinity widens to the single precision infinity of the + /// same sign, the counterpart of the narrowing case above. + /// [Fact] public void ToSingle_PreservesInfinity() { @@ -219,6 +287,12 @@ public void ToSingle_PreservesInfinity() Assert.Equal(float.NegativeInfinity, Float16Converter.ManualToSingle(0xFC00)); } + /// + /// Verifies that a NaN stays a NaN through a round trip, and that any binary16 encoding + /// with a maximal exponent and a non-zero mantissa widens to one. This runs on every + /// target framework, unlike the payload-preserving test above which needs + /// System.Half as a reference. + /// [Fact] public void ConvertsNaN() { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs index d41e125045..2e15b83bfd 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs @@ -238,6 +238,11 @@ public void Null_Property() #if NET + /// + /// Verifies that a float16 vector built from memory reports the float16 base type and a + /// two byte element size, and writes that base type into the payload header. The header + /// is what tells the server how to read the elements, so it must match the type argument. + /// [Fact] public void Float16_Construct_Memory() { @@ -258,6 +263,11 @@ public void Float16_Construct_Memory() Assert.Equal(TdsEnums.VECTOR_HEADER_SIZE + (3 * 2), ivec.VectorPayload.Length); } + /// + /// Verifies that a null float16 vector still carries its base type, element size, and + /// dimension count, which the driver needs to describe the parameter to the server even + /// though it sends no elements. + /// [Fact] public void Float16_Construct_Length() { @@ -273,6 +283,11 @@ public void Float16_Construct_Length() Assert.Equal(TdsEnums.VECTOR_HEADER_SIZE + (5 * 2), ivec.Size); } + /// + /// Verifies that the dimension limit is derived from the element size rather than fixed: + /// a float16 vector holds 3996 elements before the payload exceeds the maximum size, + /// twice the float32 limit, and one more is rejected. + /// [Fact] public void Float16_Construct_Length_Exceeds_8000() { @@ -283,6 +298,12 @@ public void Float16_Construct_Length_Exceeds_8000() Assert.Throws(() => SqlVector.CreateNull(3997)); } + /// + /// Verifies that a float16 vector renders its exact element values. Serialising the + /// elements as Half would instead emit the shortest string which round trips to + /// the same Half, rendering 65504 as "65500" and losing the value the column + /// actually holds. + /// [Fact] public void Float16_GetString_RendersExactValues() { @@ -293,6 +314,11 @@ public void Float16_GetString_RendersExactValues() Assert.Equal("[65504,1.5]", vec.GetString()); } + /// + /// Verifies that a value both base types represent exactly renders identically, so a + /// caller reading through the string path cannot tell the two apart from the rendering + /// alone. This is why the column schema exposes the base type separately. + /// [Fact] public void Float16_GetString_MatchesFloat32Rendering() { @@ -305,12 +331,20 @@ public void Float16_GetString_MatchesFloat32Rendering() #endif + /// + /// Verifies that a float32 vector renders as a JSON array, the form the server accepts + /// as a vector literal and the form callers receive from the string read paths. + /// [Fact] public void Float32_GetString_RendersJson() { Assert.Equal("[1.5,2.5]", new SqlVector(new[] { 1.5f, 2.5f }).GetString()); } + /// + /// Verifies that a null vector renders as the null string rather than an empty JSON + /// array, so a null is not mistaken for a zero length vector. + /// [Fact] public void GetString_Null_RendersNullString() { @@ -321,6 +355,12 @@ public void GetString_Null_RendersNullString() #region Widening Read Tests + /// + /// Verifies that a float16 payload is widened when read as SqlVector<float>, + /// which is how a float16 column is read on frameworks without System.Half. The + /// result reports float32, so a vector's base type stays determined by its element type + /// rather than by the payload it came from. + /// [Fact] public void FromTdsPayload_WidensFloat16ToFloat32() { @@ -337,6 +377,10 @@ public void FromTdsPayload_WidensFloat16ToFloat32() Assert.Equal(0x00, ((ISqlVector)vec).ElementType); } + /// + /// Verifies that a payload whose base type already matches the requested element type is + /// read as it is, confirming the widening path above is taken only when it is needed. + /// [Fact] public void FromTdsPayload_MatchingElementType_ReadsDirectly() { @@ -349,6 +393,11 @@ public void FromTdsPayload_MatchingElementType_ReadsDirectly() #if NET + /// + /// Verifies that reading a float32 payload as SqlVector<Half> throws rather + /// than narrowing. Narrowing loses information, so it is never performed implicitly on a + /// read, unlike the widening case above. + /// [Fact] public void FromTdsPayload_NarrowingIsRejected() { @@ -364,9 +413,21 @@ public void FromTdsPayload_NarrowingIsRejected() #region Helpers + /// + /// Builds a float32 vector payload from a header and its elements. + /// + /// The payload header, which may be deliberately malformed. + /// The elements to append after the header. + /// The assembled payload. private byte[] MakeTdsPayload(byte[] header, ReadOnlyMemory values) => MakeTdsPayloadStatic(header, values); + /// + /// The static form of , for callers which have no instance. + /// + /// The payload header, which may be deliberately malformed. + /// The elements to append after the header. + /// The assembled payload. private static byte[] MakeTdsPayloadStatic(byte[] header, ReadOnlyMemory values) { int length = header.Length + (values.Length * sizeof(float)); @@ -384,6 +445,8 @@ private static byte[] MakeTdsPayloadStatic(byte[] header, ReadOnlyMemory /// Builds a float16 vector payload without using System.Half, so that tests /// which need one can also run on .NET Framework. /// + /// The elements, narrowed to binary16 as they are written. + /// The assembled float16 payload. private static byte[] MakeFloat16Payload(float[] values) { byte[] payload = new byte[TdsEnums.VECTOR_HEADER_SIZE + (values.Length * 2)]; diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs index b49c5ece6e..e6148db80d 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs @@ -1082,6 +1082,15 @@ public void ConnectionRefusesUnsupportedServerTdsVersion(int major, int minor, i + /// + /// Verifies that the client and server settle on the highest vector feature + /// extension version they have in common, and that a server which reports no support + /// or does not acknowledge the feature leaves it unnegotiated. Getting this wrong + /// would make the driver read vector columns in a layout the server did not send. + /// + /// Whether the connection is expected to open. + /// The version the simulated server supports, or 0xFF for no acknowledgement. + /// The version expected on the wire. // Test to verify that the server and client negotiate // the common feature extension version. // The connection requests the version configured by the Vector Type Support @@ -1191,6 +1200,13 @@ public void TestConnWithVectorFeatExtVersionNegotiation(bool expectedConnectionR } } + /// + /// Verifies that the driver refuses a vector feature extension acknowledgement whose + /// version it cannot interpret, rather than trusting it. The simulated server + /// acknowledges its own version instead of capping it to the client's, which is what + /// a future server supporting a later payload layout would do. + /// + /// A version above the client's ceiling. // Test that the driver refuses a vector feature extension ack whose version it // cannot interpret. The server here acknowledges its own version rather than // capping it to the client's, which is what a future server supporting a later @@ -1244,6 +1260,16 @@ public void TestConnRejectsVectorFeatExtVersionAboveClientCeiling(byte serverVer Assert.Equal(serverVersion, acknowledgedVersion); } + /// + /// Verifies that the version requested at login follows the Vector Type Support + /// keyword, and that the request is omitted entirely when the keyword asks for no + /// vector support. This is the opt-in contract: the keyword defaults to v1, so + /// upgrading the driver does not change the representation an existing application + /// receives. + /// + /// The keyword value, or null to leave it unset. + /// Whether a feature request is expected at login. + /// The version expected in that request. // Test that the vector feature extension version requested at login follows the // Vector Type Support keyword, and that the request is omitted entirely when the // keyword asks for no vector support. From df8166afd4b9486559e1af8c2c2c56021cd85c7f Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Sat, 19 Sep 2026 03:45:28 +0530 Subject: [PATCH 16/24] Let a wide float16 column be read and written The element count limit is the number of elements of T whose payload fits in a TDS packet, so it is a property of the type a value is sent as: 1998 for float32 and 3996 for float16. It was being applied to values coming from the server as well, where the destination type is not what governs the size. A float16 column declaring more than 1998 dimensions was therefore unreadable as SqlVector, which on .NET Framework means unreadable by any means, since every read path there widens to float32. Null rows failed the same way through CreateNull. A JSON string bulk copied into such a column failed too, because it is parsed into float32 before the payload is rewritten for the destination's base type, and that intermediate was held to the float32 limit even though the value finally sent is float16. Construction by a caller still enforces the limit, so a vector which cannot be sent as T is never built from user input. Also correct a typeparamref written as paramref in the GetSqlVector docs, which broke the build. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../SqlDataReader.xml | 2 +- .../src/Microsoft/Data/SqlClient/SqlBuffer.cs | 2 +- .../Microsoft/Data/SqlClient/SqlParameter.cs | 11 +++-- .../src/Microsoft/Data/SqlTypes/SqlVector.cs | 44 ++++++++++++++--- .../VectorTest/VectorFloat16BehaviourTests.cs | 47 +++++++++++++++++++ .../Microsoft/Data/SqlTypes/SqlVectorTest.cs | 40 ++++++++++++++++ 6 files changed, 134 insertions(+), 12 deletions(-) diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlDataReader.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlDataReader.xml index e63f77663c..02c5e1731a 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlDataReader.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlDataReader.xml @@ -982,7 +982,7 @@ The method retur The retrieved data is not compatible with the type. - The column's base type cannot be represented by without losing precision. + The column's base type cannot be represented by without losing precision. diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs index 21de5bedbc..162992b96e 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBuffer.cs @@ -987,7 +987,7 @@ internal SqlVector GetSqlVector() where T : unmanaged if (IsNull) { - return SqlVector.CreateNull(_value._vectorInfo._elementCount); + return SqlVector.CreateNullFromServer(_value._vectorInfo._elementCount); } return SqlVector.FromTdsPayload(SqlBinary.Value); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs index a0c5bb720f..83f57d60b5 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs @@ -773,14 +773,16 @@ private object GetVectorReturnValue() switch (elementType) { case MetaType.SqlVectorElementType.Float32: - return SqlVector.CreateNull(elementCount); + return SqlVector.CreateNullFromServer(elementCount); case MetaType.SqlVectorElementType.Float16: #if NET - return SqlVector.CreateNull(elementCount); + return SqlVector.CreateNullFromServer(elementCount); #else // System.Half is unavailable, so a float16 vector has no faithful // strongly typed representation and is surfaced as single precision. - return SqlVector.CreateNull(elementCount); + // A float16 column may declare more dimensions than can be sent as + // float32, so the server's count is taken as given. + return SqlVector.CreateNullFromServer(elementCount); #endif default: throw SQL.VectorTypeNotSupported(elementType.ToString()); @@ -2410,7 +2412,8 @@ internal static object CoerceValue(object value, MetaType destinationType, out b { try { - value = ((ISqlVector)new SqlVector(JsonSerializer.Deserialize((string)value))).VectorPayload; + value = ((ISqlVector)SqlVector.CreateForConversion( + JsonSerializer.Deserialize((string)value))).VectorPayload; } catch (Exception ex) when (ex is ArgumentNullException || ex is JsonException) { diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs index 02a7372434..637ed911bf 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVector.cs @@ -41,10 +41,10 @@ namespace Microsoft.Data.SqlTypes; #region Constructors - private SqlVector(int length) + private SqlVector(int length, bool validateLength) { (_elementType, _elementSize, int maxElements) = GetTypeFieldsOrThrow(); - if (length < 0 || length > maxElements) + if (length < 0 || (validateLength && length > maxElements)) { throw ADP.InvalidArraySize(nameof(length)); } @@ -59,13 +59,45 @@ private SqlVector(int length) } /// - public static SqlVector CreateNull(int length) => new(length); + public static SqlVector CreateNull(int length) => new(length, validateLength: true); + + /// + /// Creates a null vector for a column read from the server, without applying the + /// element count limit which governs construction by a caller. + /// + /// + /// The limit is the number of elements of whose payload fits + /// in a TDS packet, so it is a property of the type a value is sent as. A column read + /// as a wider type than its own base type can exceed it: a float16 column may + /// declare up to 3996 dimensions, while 1998 is the most that can be sent as + /// float32. Rejecting those here would make such a column unreadable, which on + /// .NET Framework would mean unreadable by any means, since every read path there + /// widens to float32. + /// + internal static SqlVector CreateNullFromServer(int length) => new(length, validateLength: false); + + /// + /// Creates a vector to be converted to a destination's base type, without applying the + /// element count limit which governs a value sent as . + /// + /// + /// A JSON array is parsed into single precision before the payload is rewritten for the + /// destination's base type. The limit applies to what is finally sent, not to that + /// intermediate: a vector(2000, float16) destination accepts the value, while + /// 2000 elements exceed what can be sent as float32. + /// + internal static SqlVector CreateForConversion(ReadOnlyMemory memory) => + new(memory, validateLength: false); /// - public SqlVector(ReadOnlyMemory memory) + public SqlVector(ReadOnlyMemory memory) : this(memory, validateLength: true) + { + } + + private SqlVector(ReadOnlyMemory memory, bool validateLength) { (_elementType, _elementSize, int maxElements) = GetTypeFieldsOrThrow(); - if (memory.Length > maxElements) + if (validateLength && memory.Length > maxElements) { throw ADP.InvalidArraySize(nameof(memory)); } @@ -153,7 +185,7 @@ internal static SqlVector FromTdsPayload(byte[] tdsBytes) return new SqlVector(tdsBytes); } - return new SqlVector(WidenFloat16Payload(tdsBytes)); + return new SqlVector(WidenFloat16Payload(tdsBytes), validateLength: false); } /// diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index 58484945e5..6c83e58406 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -7,6 +7,7 @@ using System.Data; using System.Data.Common; using System.Data.SqlTypes; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Microsoft.Data.SqlTypes; @@ -528,6 +529,52 @@ public void BulkCopiesJsonStringSourceIntoFloat16ColumnAtV1() Assert.Contains("1.5", (string)command.ExecuteScalar()); } + /// + /// Verifies that a JSON string can be bulk copied into a float16 column declaring more + /// dimensions than can be sent as float32. The value is parsed into single precision + /// before being rewritten to the destination's base type, and that intermediate must not + /// be held to the float32 element limit: 2000 elements exceed what float32 can send, but + /// are well within float16's 3996. + /// + [ConditionalFact(nameof(IsSupported))] + public void BulkCopiesJsonStringSourceIntoAWideFloat16Column() + { + const int Dimensions = 2000; + + using Table wideTable = new(_managementConnection, "VectorF16WideTable", + $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector({Dimensions}, float16) NULL)"); + + float[] values = new float[Dimensions]; + for (int i = 0; i < values.Length; i++) + { + // Eighths are exactly representable in binary16 at this magnitude. + values[i] = (i % 8) * 0.125f; + } + + DataTable source = new(); + source.Columns.Add(ColumnName, typeof(string)); + source.Rows.Add(JsonSerializer.Serialize(values)); + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using (SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = wideTable.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(source); + } + + using SqlCommand command = + new($"SELECT TOP 1 {ColumnName} FROM {wideTable.Name} ORDER BY Id DESC", connection); + using SqlDataReader reader = command.ExecuteReader(); + + Assert.True(reader.Read()); + + // Reading it back also exercises the widening path at a width beyond the float32 + // element limit. + Assert.Equal(values, reader.GetSqlVector(0).Memory.ToArray()); + } + [ConditionalFact(nameof(IsSupported))] public void BulkCopyRejectsValuesOutsideTheFloat16Range() { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs index 2e15b83bfd..703783a758 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlVectorTest.cs @@ -409,6 +409,46 @@ public void FromTdsPayload_NarrowingIsRejected() #endif + /// + /// Verifies that a float16 column wider than the float32 element limit can still be + /// read as SqlVector<float>. A float16 vector may declare up to 3996 + /// dimensions, while 1998 is the most that can be sent as float32, and that limit + /// governs what a caller constructs rather than what the server sends. Rejecting these + /// would make such a column unreadable on .NET Framework by any means, since every read + /// path there widens to float32. + /// + [Theory] + [InlineData(1999)] + [InlineData(2000)] + [InlineData(3996)] + public void FromTdsPayload_WidensBeyondTheFloat32ElementLimit(int elementCount) + { + byte[] payload = MakeFloat16Payload(new float[elementCount]); + + var vec = SqlVector.FromTdsPayload(payload); + + Assert.Equal(elementCount, vec.Length); + } + + /// + /// Verifies the same for a null value, which is materialised from the column's declared + /// dimension count rather than from a payload. A null row must not fail where a + /// populated row in the same column succeeds. + /// + [Theory] + [InlineData(1999)] + [InlineData(3996)] + public void CreateNullFromServer_AllowsWideFloat16Columns(int elementCount) + { + var vec = SqlVector.CreateNullFromServer(elementCount); + + Assert.True(vec.IsNull); + Assert.Equal(elementCount, vec.Length); + + // The public entry point still holds a caller to what can be sent as float32. + Assert.Throws(() => SqlVector.CreateNull(elementCount)); + } + #endregion #region Helpers From 0b707379e9c3bfe9f851e33e5d5fcfb7828a5dc1 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Sat, 19 Sep 2026 04:05:55 +0530 Subject: [PATCH 17/24] Hold the vector feature extension to what the connection asked for The acknowledgement was bounded by the highest version the driver implements rather than the version this connection requested, so a server which did not cap its answer could raise a v1 connection to v2. That would return float16 columns in their binary form to an application which never opted in, which is the change the keyword exists to prevent. Bound it by the requested version instead. Also decide whether a bulk copy source is textual from the value as well as the column's declared type. A column declared as object reports nothing useful while still yielding a JSON string row by row, so its value was coerced to a float32 payload and then sent unconverted, and a float16 destination rejected it as an invalid column length. Only a string counts, so a raw vector payload is still never mistaken for text. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../Connection/SqlConnectionInternal.cs | 15 ++++- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 12 +++- .../VectorTest/VectorFloat16BehaviourTests.cs | 31 ++++++++++ .../SimulatedServerTests/ConnectionTests.cs | 60 +++++++++++++++++++ 4 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs index 52754e0c50..74ac576939 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs @@ -1733,17 +1733,26 @@ internal void OnFeatureExtAck(int featureId, byte[] data) throw SQL.ParsingError(ParsingErrorState.CorruptedTdsStream); } + // The server is expected to cap its acknowledgement to the version the + // connection asked for. Bound it here as well, so that a server which + // does not cannot raise this connection above the version it opted in + // to: accepting a higher one would return float16 columns in their + // binary form to a connection which asked for v1, which is exactly the + // change the keyword exists to prevent. + byte requestedVersion = + VectorTypeSupportUtilities.ToFeatureExtensionVersion(ConnectionOptions.VectorTypeSupport); + byte vectorSupportVersion = data[0]; - if (vectorSupportVersion == 0 || vectorSupportVersion > TdsEnums.MAX_SUPPORTED_VECTOR_VERSION) + if (vectorSupportVersion == 0 || vectorSupportVersion > requestedVersion) { SqlClientEventSource.Log.TryTraceEvent( "SqlInternalConnectionTds.OnFeatureExtAck | ERR | " + "Object ID {0}, " + "Invalid version number {1} for VECTORSUPPORT, " + - "Max supported version is {2}", + "Requested version is {2}", ObjectID, vectorSupportVersion, - TdsEnums.MAX_SUPPORTED_VECTOR_VERSION); + requestedVersion); throw SQL.ParsingError(); } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 80e648b0bc..4515e3e63e 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -1497,6 +1497,12 @@ private Type GetSourceColumnType(int sourceOrdinal) /// within the data stream: the INSERT BULK declaration states the /// destination's base type, and elements of each base type differ in size. /// + /// The source column's declared type is not enough on its own. A column declared as + /// reports no useful type while still yielding a JSON string + /// row by row, so the value is examined as well. Only a string counts: a raw vector + /// payload is a byte array, so it is never mistaken for text. + /// + /// /// A payload read from another vector column is left as it is, so a copy between /// columns of different base types is reported by the server rather than being /// silently narrowed. On .NET Framework a float16 column has no System.Half @@ -1506,9 +1512,9 @@ private Type GetSourceColumnType(int sourceOrdinal) /// by BulkCopiesFloat16ToFloat32ThroughTheTextualRepresentation. /// /// - private bool IsTextSourcedVectorColumn(int sourceOrdinal, _SqlMetaData metadata) => + private bool IsTextSourcedVectorColumn(int sourceOrdinal, _SqlMetaData metadata, object value) => metadata.type == SqlDbTypeExtensions.Vector && - GetSourceColumnType(sourceOrdinal) == typeof(string); + (GetSourceColumnType(sourceOrdinal) == typeof(string) || value is string); private SourceColumnMetadata GetColumnMetadata(int ordinal) { @@ -1867,7 +1873,7 @@ private static object ConvertVectorToBaseType(object value, byte destinationElem private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, ref bool isSqlType, out bool coercedToDataFeed, int sourceOrdinal) { - bool isTextSourcedVector = IsTextSourcedVectorColumn(sourceOrdinal, metadata); + bool isTextSourcedVector = IsTextSourcedVectorColumn(sourceOrdinal, metadata, value); coercedToDataFeed = false; if (isNull) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index 6c83e58406..6bbf3052cd 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -575,6 +575,37 @@ public void BulkCopiesJsonStringSourceIntoAWideFloat16Column() Assert.Equal(values, reader.GetSqlVector(0).Memory.ToArray()); } + /// + /// Verifies that a JSON string in a column declared as is still + /// parsed into the destination's base type. The declared type reports nothing useful + /// here, so the value itself has to be examined; otherwise the float32 payload produced + /// by coercion would reach a float16 column at the wrong width and the server would + /// reject the copy. + /// + [ConditionalFact(nameof(IsSupported))] + public void BulkCopiesJsonStringSourceFromAnObjectTypedColumn() + { + DataTable table = new(); + table.Columns.Add(ColumnName, typeof(object)); + table.Rows.Add("[1.5,2.5,3.5]"); + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using (SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = _float16Table.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(table); + } + + using SqlCommand command = + new($"SELECT TOP 1 {ColumnName} FROM {_float16Table.Name} ORDER BY Id DESC", connection); + using SqlDataReader reader = command.ExecuteReader(); + + Assert.True(reader.Read()); + Assert.Equal([1.5f, 2.5f, 3.5f], reader.GetSqlVector(0).Memory.ToArray()); + } + [ConditionalFact(nameof(IsSupported))] public void BulkCopyRejectsValuesOutsideTheFloat16Range() { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs index e6148db80d..5d892277b5 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs @@ -1260,6 +1260,66 @@ public void TestConnRejectsVectorFeatExtVersionAboveClientCeiling(byte serverVer Assert.Equal(serverVersion, acknowledgedVersion); } + /// + /// Verifies that the driver refuses an acknowledgement above the version this + /// connection asked for, even when it is one the client could otherwise interpret. + /// The keyword is an opt-in, so honouring a v2 acknowledgement on a v1 connection + /// would return float16 columns in their binary form to an application which never + /// asked for that — the precise back-compat change the keyword exists to prevent. + /// + /// The version the connection asks for, or null to leave it at the default. + [Theory] + // The default is v1, so an application which says nothing is covered too. + [InlineData(null)] + [InlineData(SqlVectorTypeSupport.V1)] + public void TestConnRejectsVectorFeatExtVersionAboveRequested(SqlVectorTypeSupport? setting) + { + using TdsServer server = new(); + server.Start(); + server.EnableVectorFeatureExt = true; + // The server acknowledges its own version rather than capping it to the + // client's, which is what a server that does not honour the request would do. + server.ServerSupportedVectorFeatureExtVersion = 0x2; + server.AcknowledgeRawVectorFeatureExtVersion = true; + + byte acknowledgedVersion = 0; + + server.OnAuthenticationResponseCompleted = response => + { + TDSFeatureExtAckGenericOption option = response + .OfType() + .FirstOrDefault()? + .Options + .OfType() + .FirstOrDefault(o => o.FeatureID == TDSFeatureID.VectorSupport)!; + + if (option != null) + { + acknowledgedVersion = option.FeatureAckData[0]; + } + }; + + SqlConnectionStringBuilder builder = new() + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + Pooling = false, // Disable pooling so this expected failure does not poison a shared pool + }; + + if (setting.HasValue) + { + builder.VectorTypeSupport = setting.Value; + } + + using SqlConnection connection = new(builder.ConnectionString); + + Assert.Throws(() => connection.Open()); + + // Confirms the server really did acknowledge v2, so the failure above is the + // requested version being exceeded rather than an unrelated connection problem. + Assert.Equal(0x2, acknowledgedVersion); + } + /// /// Verifies that the version requested at login follows the Vector Type Support /// keyword, and that the request is omitted entirely when the keyword asks for no From 92d712a900493880b9a19f79077b7424e78f6b41 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Fri, 2 Oct 2026 08:04:41 +0530 Subject: [PATCH 18/24] Describe the codec's binary32 coverage accurately The reference test steps through the single precision space with a prime stride, so it samples about 4.1 million of the 4,294,967,296 patterns rather than visiting them all. Say so, and record why: a full sweep takes about ten seconds on 32 cores and proportionally longer on a smaller agent, which is not worth repeating on every CI leg for a codec that does not change. It was run once and reported no divergence, and setting the stride to 1 reproduces it. Also document the connection string keyword synonym test, which covers every accepted spelling through both the builder and a connection. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../SqlConnectionStringBuilderTest.cs | 13 +++++++++++++ .../Data/SqlTypes/Float16ConverterTest.cs | 16 ++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConnectionStringBuilderTest.cs b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConnectionStringBuilderTest.cs index 9e31055292..cea74c153c 100644 --- a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConnectionStringBuilderTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConnectionStringBuilderTest.cs @@ -1015,6 +1015,19 @@ public void PacketSizeSynonymsResolveCorrectly(string connectionString, int expe Assert.Equal(expected, builder.PacketSize); } + /// + /// Verifies that every accepted spelling of the vector type support keyword resolves + /// to the same value: the canonical spaced form, the unspaced synonym, and both in + /// arbitrary casing. The unspaced form is the one the JDBC driver uses, so it is what + /// a ported application is likely to be written with, and keyword lookup is + /// case-insensitive throughout. + /// + /// + /// A connection parses its keywords through a separate table from the builder, so + /// both are exercised here; registering the synonym in only one of them would leave + /// the other rejecting the keyword outright. + /// + /// A connection string using one accepted spelling. [Theory] [InlineData("Vector Type Support = V2")] [InlineData("VectorTypeSupport = V2")] diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs index c682792b5a..f81c5bfc15 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/Float16ConverterTest.cs @@ -74,11 +74,19 @@ public void FromSingle_MatchesHalf_ForEveryRepresentableValue() } /// - /// Verifies narrowing against System.Half for inputs drawn from the whole single - /// precision space, not just those binary16 can represent. This is what covers rounding, - /// overflow, and underflow for arbitrary application values, which the exhaustive - /// representable-value test above cannot reach. + /// Verifies narrowing against System.Half for inputs sampled across the whole + /// single precision space, not just those binary16 can represent. This is what covers + /// rounding, overflow, and underflow for arbitrary application values, which the + /// exhaustive representable-value test above cannot reach. /// + /// + /// This samples roughly 4.1 million of the 4,294,967,296 binary32 patterns rather than + /// visiting them all, so that it costs a few milliseconds on every CI leg. A sweep of + /// the full space takes about ten seconds on 32 cores and proportionally longer on a + /// smaller agent, which is not worth repeating per run for a codec that does not + /// change; it was run once during development and reported no divergence. Setting + /// Stride to 1 reproduces it. + /// [Fact] public void FromSingle_MatchesHalf_AcrossTheSinglePrecisionRange() { From a8334881392b711c48eec19653a1992f3b25d650 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 6 Oct 2026 19:08:04 +0530 Subject: [PATCH 19/24] Reject a vector base type the connection did not negotiate Nothing checked the negotiated version before a float16 value was declared and written. Measured against SQL Server vNext CTP 1.0: a V1 connection sent the value and the server stored it, even though the same connection reads that column back as a JSON string, and an Off connection was told its protocol stream was malformed (error 8070). Both are worse than naming the keyword, which is what the JDBC and ODBC drivers do. Checked once in VectorTypeSupportUtilities so the parameter and bulk copy paths cannot disagree. Convert any in-memory value to the destination's base type during a bulk copy, not only a textual one. A SqlVector carries the base type of its element type, which the caller chose rather than the column, so a SqlVector from a DataTable reached a float16 column at the wrong width and the server rejected it. The test matrix had been narrowed to hide this; it now runs both source modes for every representation. A payload read from another vector column is still left alone, so a cross base type copy is still reported by the server. Say that conversion between base types depends on the server. Some builds block it and report error 42238, which leaves a JSON string as the only portable way to write a column whose base type differs from the value's, and the only way at all on .NET Framework. Tests which depend on the conversion now skip rather than fail where it is blocked. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .github/instructions/features.instructions.md | 19 ++++-- doc/samples/SqlVectorFloat16Example.cs | 28 +++++--- .../Microsoft.Data.SqlTypes/SqlVector.xml | 24 +++++-- .../VectorTypeSupportUtilities.cs | 39 +++++++++++ .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 64 ++++++++++--------- .../src/Microsoft/Data/SqlClient/SqlUtil.cs | 16 +++++ .../src/Microsoft/Data/SqlClient/TdsParser.cs | 10 ++- .../src/Resources/Strings.Designer.cs | 9 +++ .../src/Resources/Strings.resx | 3 + .../ManualTests/DataCommon/DataTestUtility.cs | 42 ++++++++++++ .../NativeVectorFloat16AsSingleTests.cs | 5 +- .../SQL/VectorTest/NativeVectorTestsBase.cs | 14 ++-- .../VectorTest/VectorFloat16BehaviourTests.cs | 27 ++++++-- 13 files changed, 238 insertions(+), 62 deletions(-) diff --git a/.github/instructions/features.instructions.md b/.github/instructions/features.instructions.md index 521a312d62..77de54f503 100644 --- a/.github/instructions/features.instructions.md +++ b/.github/instructions/features.instructions.md @@ -144,13 +144,24 @@ Notes: `GetSqlVector`, which widens the elements. Widening from `float16` is exact. - `float16` requires `ALTER DATABASE SCOPED CONFIGURATION SET PREVIEW_FEATURES = ON` while it is in preview. -- SQL Server converts between base types for a parameter. A bulk copy is different: the - `INSERT BULK` statement states the destination's base type, and the server then requires a - binary payload of exactly that width, so it performs no conversion within the data stream. - A textual source is therefore parsed into the destination's base type by the driver, and +- Whether SQL Server converts between base types for a parameter depends on the server: + some builds block both implicit and explicit conversion and report error 42238, others + allow it (verified against SQL Server vNext CTP 1.0, 18.0.258.0). A JSON string is + accepted by every server, so prefer it when the target server is not known. This matters + most on .NET Framework, where `System.Half` is unavailable and a `SqlVector` would + otherwise be the only strongly typed way to write a `float16` column. +- A bulk copy is different: the `INSERT BULK` statement states the destination's base type, + and the server then requires a binary payload of exactly that width, so it performs no + conversion within the data stream. Any in-memory value — a JSON string or a + `SqlVector` whose element type differs from the column's — is therefore converted to + the destination's base type by the driver, which does not depend on the server, and narrowing to `float16` throws an `OverflowException` for a value outside its range. A payload read from another vector column keeps its own base type, so copying between columns of different base types is reported by the server. +- A vector base type can only be used once the connection has negotiated the feature + extension version which covers it, so writing a `float16` value needs + `Vector Type Support=v2`. The driver reports this itself rather than letting the server + see a value it did not negotiate. - A column's base type and number of dimensions are available from the column schema: `reader.GetColumnSchema()[i]["VectorBaseType"]` and `["VectorDimensions"]`. Both are `null` for columns which are not vectors. This is the only way to tell the two base types apart diff --git a/doc/samples/SqlVectorFloat16Example.cs b/doc/samples/SqlVectorFloat16Example.cs index f3c64c1d2d..82b7bd6087 100644 --- a/doc/samples/SqlVectorFloat16Example.cs +++ b/doc/samples/SqlVectorFloat16Example.cs @@ -9,11 +9,11 @@ namespace SqlVectorFloat16Example; // - Inserts vectors from .NET Framework, where System.Half is unavailable // - Reads float16 vectors as SqlVector, as widened SqlVector, and as JSON // - Inspects a column's base type and number of dimensions -// - Converts between the float16 and float32 base types +// - Converts between the float16 and float32 base types, where the server permits it // // Requirements: // - SQL Server 2025 and above, with PREVIEW_FEATURES enabled for the database -// - Microsoft.Data.SqlClient (7.1.0 and above) +// - Microsoft.Data.SqlClient (8.0.0 and above) // using Microsoft.Data; using Microsoft.Data.SqlClient; @@ -193,14 +193,24 @@ private static async Task ReadColumnMetadataAsync(SqlConnection conn) #region ConvertBetweenBaseTypes private static async Task ConvertBetweenBaseTypesAsync(SqlConnection conn) { - // SQL Server converts between the two base types, so a vector read from a column of - // one base type can be written to a column of the other. - using var cmd = new SqlCommand( - "SELECT CAST(CAST('[1.5,2.5,3.5]' AS vector(3, float16)) AS vector(3, float32));", conn); - using var reader = await cmd.ExecuteReaderAsync(); + // Whether SQL Server converts between the two base types depends on the server: some + // builds block both implicit and explicit conversion and report error 42238. Where it + // is permitted, a vector read from a column of one base type can be written to a + // column of the other. Where it is not, write a JSON string instead, which every + // server accepts and which the examples above use. + try + { + using var cmd = new SqlCommand( + "SELECT CAST(CAST('[1.5,2.5,3.5]' AS vector(3, float16)) AS vector(3, float32));", conn); + using var reader = await cmd.ExecuteReaderAsync(); - await reader.ReadAsync(); - Console.WriteLine($"\nfloat16 converted to float32: {reader.GetString(0)}"); + await reader.ReadAsync(); + Console.WriteLine($"\nfloat16 converted to float32: {reader.GetString(0)}"); + } + catch (SqlException ex) + { + Console.WriteLine($"\nThis server does not convert between vector base types: {ex.Message}"); + } } #endregion } diff --git a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml index ed33d129a9..c0d8aaf38a 100644 --- a/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml +++ b/doc/snippets/Microsoft.Data.SqlTypes/SqlVector.xml @@ -15,8 +15,9 @@ read on .NET Framework, either as a JSON array through the string read paths, or as a SqlVector<float>, which widens the elements to single precision. Widening is exact, so no information is lost. Values can be written to a float16 column - from .NET Framework as a JSON string, or as a SqlVector<float>, which the - server converts. + from .NET Framework as a JSON string, which every server accepts, or as a + SqlVector<float>, which requires a server that converts between base + types. See the remarks on conversion below. A float16 column is only exchanged in its binary form when the connection asks @@ -34,10 +35,21 @@ elements, but only the string paths also serialize the result. - Conversion between base types is performed by SQL Server, which reports an error if a - value is outside the range the destination can represent. Converting from - float32 to float16 loses precision for values which the narrower type - cannot represent exactly, in the same way as inserting a JSON literal does. + Conversion between the float32 and float16 base types is performed by + SQL Server, and whether it is permitted depends on the server. Some builds block both + implicit and explicit conversion between base types and report error 42238; others + allow it. Verified as working against SQL Server vNext CTP 1.0 (18.0.258.0). Where + the conversion is blocked, a value whose base type differs from the column's cannot + be written as a SqlVector<T> at all, which on .NET Framework leaves a + JSON string as the only way to write a float16 column, since + is unavailable there. A JSON string is accepted by every + server, so prefer it when the target server is not known. + + + Where the conversion is permitted, the server reports an error if a value is outside + the range the destination can represent. Converting from float32 to + float16 loses precision for values which the narrower type cannot represent + exactly, in the same way as inserting a JSON literal does. A bulk copy is different, because the INSERT BULK statement states the diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/VectorTypeSupportUtilities.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/VectorTypeSupportUtilities.cs index cf30ff42d1..38d40751ff 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/VectorTypeSupportUtilities.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/VectorTypeSupportUtilities.cs @@ -131,5 +131,44 @@ internal static byte ToFeatureExtensionVersion(SqlVectorTypeSupport value) _ => TdsEnums.VECTOR_VERSION_FLOAT32, }; } + + /// + /// Throws if a vector base type is used which the connection did not negotiate. + /// + /// The base type of the value being sent. + /// + /// The version acknowledged by the server, from ConnectionCapabilities.VectorVersion. + /// + /// + /// The negotiated version governs which base types are exchanged in binary form, so + /// a value of a base type above it cannot be sent. Checked once here so that the + /// parameter and bulk copy paths cannot disagree. + /// + internal static void ThrowIfBaseTypeNotNegotiated(byte elementType, byte negotiatedVersion) + { + // Each base type was added in the feature extension version of the same number: + // float32 in version 1, float16 in version 2. + byte requiredVersion = elementType switch + { + (byte)MetaType.SqlVectorElementType.Float32 => TdsEnums.VECTOR_VERSION_FLOAT32, + (byte)MetaType.SqlVectorElementType.Float16 => TdsEnums.VECTOR_VERSION_FLOAT16, + _ => throw SQL.VectorTypeNotSupported(elementType.ToString()), + }; + + if (negotiatedVersion >= requiredVersion) + { + return; + } + + string baseType = elementType == (byte)MetaType.SqlVectorElementType.Float16 + ? "float16" + : "float32"; + + string keywordValue = requiredVersion == TdsEnums.VECTOR_VERSION_FLOAT16 + ? nameof(SqlVectorTypeSupport.V2) + : nameof(SqlVectorTypeSupport.V1); + + throw SQL.VectorBaseTypeNotNegotiated(baseType, keywordValue); + } } } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 9305a85bd9..126c51435e 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -1488,33 +1488,43 @@ private Type GetSourceColumnType(int sourceOrdinal) } /// - /// Whether a vector destination column is supplied from a textual source, in which - /// case the value is parsed into the destination's base type by the client. + /// Whether a value bound for a vector column must be rewritten to that column's base + /// type by the client before it is sent. /// /// - /// A JSON array is always coerced to a float32 payload, so it has to be rewritten - /// when the destination's base type is not float32. The server does not convert - /// within the data stream: the INSERT BULK declaration states the - /// destination's base type, and elements of each base type differ in size. + /// The INSERT BULK declaration states the destination's base type and the + /// server performs no conversion within the data stream, because elements of each + /// base type differ in size. So any value which does not already carry the + /// destination's base type has to be rewritten here. /// - /// The source column's declared type is not enough on its own. A column declared as - /// reports no useful type while still yielding a JSON string - /// row by row, so the value is examined as well. Only a string counts: a raw vector - /// payload is a byte array, so it is never mistaken for text. + /// That covers every in-memory representation. A JSON array is coerced to a float32 + /// payload whatever the destination, and a + /// carries the base type of its own element type, which the caller chose rather than + /// the column. Both are converted, with the same range check, so that a value which + /// cannot be represented is reported against the value rather than as a length + /// mismatch from the server. /// /// - /// A payload read from another vector column is left as it is, so a copy between - /// columns of different base types is reported by the server rather than being - /// silently narrowed. On .NET Framework a float16 column has no System.Half - /// to report, so it describes itself as a string and takes the textual path above: - /// such a copy is converted by the client rather than rejected. That divergence is + /// A payload read from another vector column is the one case left alone: it is + /// transferred as the raw bytes the server sent, so a copy between columns of + /// different base types is reported by the server rather than silently narrowed. + /// Such a value is a byte array, which is neither of the cases above. On .NET + /// Framework a float16 column has no System.Half to report, so it describes + /// itself as a string and is converted rather than rejected; that divergence is /// inherent to the base type having no native representation there, and is covered /// by BulkCopiesFloat16ToFloat32ThroughTheTextualRepresentation. /// + /// + /// The source column's declared type is not enough on its own, because a column + /// declared as reports no useful type while still yielding a + /// JSON string row by row, so the value is examined as well. + /// /// - private bool IsTextSourcedVectorColumn(int sourceOrdinal, _SqlMetaData metadata, object value) => + private bool NeedsVectorBaseTypeConversion(int sourceOrdinal, _SqlMetaData metadata, object value) => metadata.type == SqlDbTypeExtensions.Vector && - (GetSourceColumnType(sourceOrdinal) == typeof(string) || value is string); + (GetSourceColumnType(sourceOrdinal) == typeof(string) || + value is string || + value is ISqlVector); private SourceColumnMetadata GetColumnMetadata(int ordinal) { @@ -1875,7 +1885,7 @@ private static object ConvertVectorToBaseType(object value, byte destinationElem private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, ref bool isSqlType, out bool coercedToDataFeed, int sourceOrdinal) { - bool isTextSourcedVector = IsTextSourcedVectorColumn(sourceOrdinal, metadata, value); + bool needsVectorBaseTypeConversion = NeedsVectorBaseTypeConversion(sourceOrdinal, metadata, value); coercedToDataFeed = false; if (isNull) @@ -1962,17 +1972,13 @@ private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, re mt = MetaType.GetMetaTypeFromSqlDbType(type.SqlDbType, false); value = SqlParameter.CoerceValue(value, mt, out coercedToDataFeed, out typeChanged, false); - // A JSON string is always coerced to a float32 payload, so a textual - // source bound for a column with a different base type has to be - // rewritten to that base type. The INSERT BULK declaration states the - // destination's base type, and the server does not convert within the - // data stream, because binary16 and binary32 elements differ in size. - // - // Only a textual source is converted. A payload read from another - // vector column keeps its own base type, so copying between columns - // of different base types is reported by the server rather than being - // silently narrowed. - if (isTextSourcedVector) + // Coercion reduces every representation to a payload carrying the + // base type the value had, which for an in-memory value is the one + // the caller chose rather than the column's. The INSERT BULK + // declaration states the destination's base type and the server does + // not convert within the data stream, so the payload is rewritten + // here. Decided before coercion, which erases the distinction. + if (needsVectorBaseTypeConversion) { value = ConvertVectorToBaseType(value, scale); } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs index c9388a42f1..f7d37addff 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs @@ -570,6 +570,22 @@ internal static Exception VectorTypeNotSupported(string value) return ADP.NotSupported(StringsHelper.GetString(Strings.SQL_VectorTypeNotSupported, value)); } + /// + /// A vector base type was used which the connection did not negotiate with the server. + /// + /// + /// Rejected by the client rather than left to the server, which reports it as a + /// malformed protocol stream when the feature extension was not requested at all, + /// and otherwise accepts the value even though the connection reads such a column + /// back as a JSON string. Both outcomes are worse than naming the keyword which + /// enables the base type, which is also what the JDBC and ODBC drivers do. + /// + internal static Exception VectorBaseTypeNotNegotiated(string baseType, string requiredKeywordValue) + { + return ADP.InvalidOperation( + StringsHelper.GetString(Strings.SQL_VectorBaseTypeNotNegotiated, baseType, requiredKeywordValue)); + } + internal static Exception XmlReaderNotSupportOnColumnType(string columnName) { return ADP.InvalidCast(StringsHelper.GetString(Strings.SQL_XmlReaderNotSupportOnColumnType, columnName)); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs index ffc31a6078..32175af8a9 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -10765,7 +10765,11 @@ private Task TDSExecuteRPCAddParameter(TdsParserStateObject stateObj, SqlParamet else if (mt.SqlDbType == SqlDbTypeExtensions.Vector) { // For vector type we need to write scale as the element type of the vector. - stateObj.WriteByte(((ISqlVector)param.Value).ElementType); + byte elementType = ((ISqlVector)param.Value).ElementType; + + VectorTypeSupportUtilities.ThrowIfBaseTypeNotNegotiated(elementType, Capabilities.VectorVersion); + + stateObj.WriteByte(elementType); } // write out collation or xml metadata @@ -11646,6 +11650,10 @@ internal void WriteBulkCopyMetaData(_SqlMetaDataSet metadataCollection, int coun stateObj.WriteByteArray(s_jsonMetadataSubstituteSequence, s_jsonMetadataSubstituteSequence.Length, 0); break; case SqlDbTypeExtensions.Vector: + // The scale carries the destination column's base type, which the + // connection must have negotiated in order to send the payload. + VectorTypeSupportUtilities.ThrowIfBaseTypeNotNegotiated(md.scale, Capabilities.VectorVersion); + stateObj.WriteByte(md.tdsType); WriteTokenLength(md.tdsType, md.length, stateObj); stateObj.WriteByte(md.scale); diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs index 74ee975ce2..a24d6ffe1b 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs @@ -4326,6 +4326,15 @@ internal static string SQL_VectorTypeNotSupported { } } + /// + /// Looks up a localized string similar to The vector base type '{0}' is not supported for the vector type support level negotiated with the server. Set the 'Vector Type Support' connection string keyword to '{1}' to use it.. + /// + internal static string SQL_VectorBaseTypeNotNegotiated { + get { + return ResourceManager.GetString("SQL_VectorBaseTypeNotNegotiated", resourceCulture); + } + } + /// /// Looks up a localized string similar to Invalid attempt to GetXmlReader on column '{0}'. The GetXmlReader function can only be used on columns of type Xml.. /// diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx index 2a5072609b..94e55c1707 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx @@ -2160,6 +2160,9 @@ Unsupported Vector type '{0}'. + + The vector base type '{0}' is not supported for the vector type support level negotiated with the server. Set the 'Vector Type Support' connection string keyword to '{1}' to use it. + 'null' value not supported for output parameter '{0}' of SqlDbtype Vector. diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs index 1084c6d966..d618d6efbd 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs @@ -97,6 +97,7 @@ public static class DataTestUtility private static bool? s_isJsonSupported; private static bool? s_isVectorSupported; private static bool? s_isVectorFloat16Supported; + private static bool? s_isVectorBaseTypeConversionSupported; // Login permissions private static bool? s_isSysAdmin; @@ -264,6 +265,47 @@ private static bool CheckVectorFloat16Supported() } } + /// + /// Determines whether the server converts between the float32 and + /// float16 vector base types. + /// + /// + /// Some SQL Server builds block both implicit and explicit conversion between vector + /// base types and report error 42238; others allow it. Tests which write a value whose + /// base type differs from the column's depend on the server performing the conversion, + /// so they are skipped rather than failed where it is blocked. Implies + /// . + /// + public static bool IsSqlVectorBaseTypeConversionSupported => + s_isVectorBaseTypeConversionSupported ??= IsSqlVectorFloat16Supported && + CheckVectorBaseTypeConversionSupported(); + + private static bool CheckVectorBaseTypeConversionSupported() + { + try + { + using SqlConnection connection = new(TCPConnectionString); + connection.Open(); + + // Casts in both directions, since a server could permit one and not the other. + using SqlCommand command = new( + "DECLARE @f32 AS VECTOR(3, float32) = '[1.5,2.5,3.5]';" + + "DECLARE @f16 AS VECTOR(3, float16) = CAST(@f32 AS VECTOR(3, float16));" + + "DECLARE @back AS VECTOR(3, float32) = CAST(@f16 AS VECTOR(3, float32));" + + "SELECT 1;", + connection); + + command.ExecuteScalar(); + return true; + } + catch (SqlException) + { + // Conversion between base types is blocked on this server (error 42238), so + // only a value whose base type already matches the column can be written. + return false; + } + } + public static bool IsSysAdmin => s_isSysAdmin ??= IsTCPConnStringSetup() && IsServerRoleMember("sysadmin"); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs index e41442263b..c704e19877 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorFloat16AsSingleTests.cs @@ -45,7 +45,10 @@ public sealed class VectorFloat16AsSingleTestData : NativeVectorTestDataBase 3234; - public override bool IsSupported => DataTestUtility.IsSqlVectorFloat16Supported; + // The column's base type is float16 but the values are single precision, so writing a + // parameter depends on the server converting between base types. Where that is blocked, + // the suite does not apply. + public override bool IsSupported => DataTestUtility.IsSqlVectorBaseTypeConversionSupported; public override string SqlServerTypeName => "float16"; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs index 863ca1cc32..7c4e810123 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs @@ -117,20 +117,18 @@ public abstract class NativeVectorTestsBase : IDisposable public static bool IsSupported => TestDataInstance.IsSupported; /// - /// The bulk copy source modes which apply to this suite. Mode 2 supplies the value as - /// a through a , which carries its - /// own base type; it is therefore only valid when that base type is the column's. + /// The bulk copy source modes which apply to this suite. Mode 1 reads the value from + /// a SQL Server table, and mode 2 supplies it as a through + /// a . Both apply to every representation: an in-memory vector + /// carries the base type of its element type, which the client rewrites to the + /// column's base type when they differ. /// public static IEnumerable BulkCopySourceModes { get { yield return new object[] { 1 }; - - if (TestDataInstance.IsDefaultRepresentation) - { - yield return new object[] { 2 }; - } + yield return new object[] { 2 }; } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index 6bbf3052cd..6fcdcbd800 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -66,6 +66,13 @@ public VectorFloat16BehaviourTests() public static bool IsSupported => DataTestUtility.IsSqlVectorFloat16Supported; + /// + /// Whether the server converts between vector base types. Some builds block it and report + /// error 42238, in which case a value can only be written to a column whose base type + /// matches its own. + /// + public static bool ServerConvertsBaseTypes => DataTestUtility.IsSqlVectorBaseTypeConversionSupported; + #region Column metadata [ConditionalFact(nameof(IsSupported))] @@ -237,8 +244,9 @@ await Assert.ThrowsAsync( public static IEnumerable CrossBaseTypeParameters() { - // A vector of either base type can be written to a column of either base type: the - // conversion is performed by the server, which knows the destination's base type. + // A vector of either base type can be written to a column of either base type, where + // the server converts between them. Whether it does is server dependent, which is + // what ServerConvertsBaseTypes guards. yield return ["float16", new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })]; yield return ["float32", new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })]; #if NET @@ -247,7 +255,13 @@ public static IEnumerable CrossBaseTypeParameters() #endif } - [ConditionalTheory(nameof(IsSupported))] + /// + /// Verifies that a vector parameter can be written to a column of either base type, with + /// the server performing the conversion. Skipped where the server blocks conversion + /// between base types, in which case a JSON string is the only portable way to write a + /// column whose base type differs from the value's. + /// + [ConditionalTheory(nameof(IsSupported), nameof(ServerConvertsBaseTypes))] [MemberData(nameof(CrossBaseTypeParameters), DisableDiscoveryEnumeration = true)] public void WritesVectorParameterToColumnOfEitherBaseType(string columnBaseType, object value) { @@ -260,7 +274,12 @@ public void WritesVectorParameterToColumnOfEitherBaseType(string columnBaseType, Assert.Equal([1.5f, 2.5f, 3.5f], reader.GetSqlVector(0).Memory.ToArray()); } - [ConditionalFact(nameof(IsSupported))] + /// + /// Verifies that a float32 value which cannot be represented in float16 is reported + /// rather than silently saturated. The value is sent as float32 and narrowed by the + /// server, so this depends on the server converting between base types. + /// + [ConditionalFact(nameof(IsSupported), nameof(ServerConvertsBaseTypes))] public void RejectsValuesOutsideTheFloat16Range() { // The value is sent as float32 and narrowed by the server, which reports the From ca9e7f73180bf6963c305e26a0cf128c1dd37098 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 6 Oct 2026 19:17:30 +0530 Subject: [PATCH 20/24] Check the element count against the base type a value is sent as The JSON intermediate is built without the element count limit so that a wide float16 column can be loaded through it, which also removed the check for a float32 destination: an oversized array reached the server and came back as a column length error. The limit depends on the element width, so it is now applied after the payload has been rewritten to the destination's base type. Cover the negotiation check with tests, and record what a connection below the negotiated version actually does during a bulk copy: such a connection is told a float16 column is a varchar(max), so the value travels as text and the server converts it, which is what makes the default level usable for loading float16 data. The check on that path guards the invariant rather than a reachable case. Add a rounding parity test. Text bound for a float16 column is parsed to float32 and then narrowed, so it is rounded twice, while an INSERT leaves the text to the server. Measured at and around binary16 ties, including the smallest subnormal: both paths agree. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../Microsoft/Data/SqlClient/SqlBulkCopy.cs | 22 ++- .../src/Microsoft/Data/SqlClient/TdsParser.cs | 8 +- .../Data/SqlTypes/SqlVectorPayload.cs | 21 ++ .../VectorTest/VectorFloat16BehaviourTests.cs | 185 ++++++++++++++++++ 4 files changed, 229 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs index 126c51435e..e6dd75c092 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs @@ -1866,10 +1866,16 @@ private object ValidateBulkCopyVariant(object value) /// column's base type, leaving payloads which already use that base type untouched. /// /// - /// Used only for a textual source, whose JSON array is always coerced to a float32 - /// payload regardless of the destination's base type. The server does not convert - /// within the bulk copy data stream, because binary16 and binary32 elements differ - /// in size and the INSERT BULK declaration fixes the element width. + /// Used for any in-memory value, whose payload carries the base type it was built + /// with rather than the destination's. The server does not convert within the bulk + /// copy data stream, because binary16 and binary32 elements differ in size and the + /// INSERT BULK declaration fixes the element width. + /// + /// The element count limit is checked here rather than when the intermediate was + /// built, because it depends on the element width and so on the base type the value + /// is finally sent as. A float16 column accepts twice as many elements as a float32 + /// one, and the intermediate is always float32. + /// /// private static object ConvertVectorToBaseType(object value, byte destinationElementType) { @@ -1880,7 +1886,13 @@ private static object ConvertVectorToBaseType(object value, byte destinationElem return value; } - return SqlTypes.SqlVectorPayload.ConvertElementType(payload, destinationElementType); + byte[] converted = SqlTypes.SqlVectorPayload.ConvertElementType(payload, destinationElementType); + + SqlTypes.SqlVectorPayload.ThrowIfLengthExceedsBaseType( + (converted.Length - TdsEnums.VECTOR_HEADER_SIZE) / MetaType.GetVectorElementSize(destinationElementType), + destinationElementType); + + return converted; } private object ConvertValue(object value, _SqlMetaData metadata, bool isNull, ref bool isSqlType, out bool coercedToDataFeed, int sourceOrdinal) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs index 32175af8a9..72acfdaebf 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -11650,8 +11650,12 @@ internal void WriteBulkCopyMetaData(_SqlMetaDataSet metadataCollection, int coun stateObj.WriteByteArray(s_jsonMetadataSubstituteSequence, s_jsonMetadataSubstituteSequence.Length, 0); break; case SqlDbTypeExtensions.Vector: - // The scale carries the destination column's base type, which the - // connection must have negotiated in order to send the payload. + // The scale carries the destination column's base type. A + // connection which did not negotiate that base type is told the + // column is a varchar(max) instead, so this does not arise today + // and the value travels as text; the check guards the invariant + // rather than a reachable case, and keeps this path consistent + // with the parameter path. VectorTypeSupportUtilities.ThrowIfBaseTypeNotNegotiated(md.scale, Capabilities.VectorVersion); stateObj.WriteByte(md.tdsType); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVectorPayload.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVectorPayload.cs index b572cb28d5..73b7b6fb2e 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVectorPayload.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlTypes/SqlVectorPayload.cs @@ -51,6 +51,27 @@ internal static void ThrowIfHeaderInvalid(byte[] tdsBytes) } } + /// + /// Throws if a vector of the given length cannot be represented in the given base type + /// without exceeding the maximum size of a TDS packet. + /// + /// + /// The limit depends on the element width, so it is checked against the base type the + /// value is finally sent as rather than any intermediate it passed through. Reported + /// here so that an oversized vector is named as such, rather than reaching the server + /// and coming back as a column length error. + /// + internal static void ThrowIfLengthExceedsBaseType(int length, byte elementType) + { + int maxElements = + (TdsEnums.MAXSIZE - TdsEnums.VECTOR_HEADER_SIZE) / MetaType.GetVectorElementSize(elementType); + + if (length > maxElements) + { + throw ADP.InvalidArraySize(nameof(length)); + } + } + /// /// Rewrites a TDS vector payload so that its elements use the requested base type, /// returning the original payload when it already does. diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index 6fcdcbd800..27ba26ec07 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -292,6 +292,93 @@ public void RejectsValuesOutsideTheFloat16Range() #endregion + #region Feature extension negotiation + + /// + /// Verifies that writing a native float16 parameter on a connection which did not + /// negotiate the base type is rejected by the driver, naming the keyword which enables + /// it. Without this the server either stores a value the same connection would read back + /// as a JSON string, or reports the protocol stream as malformed, neither of which tells + /// the caller what to change. The JDBC and ODBC drivers reject it on the client too. + /// + /// The level to request, or null to leave the keyword unset. + #if NET + [ConditionalTheory(nameof(IsSupported))] + // The default is v1, so an application which says nothing is covered too. + [InlineData(null)] + [InlineData(SqlVectorTypeSupport.V1)] + [InlineData(SqlVectorTypeSupport.Off)] + public void RejectsNativeFloat16ParameterBelowTheNegotiatedVersion(SqlVectorTypeSupport? setting) + { + using SqlConnection connection = OpenAt(setting); + + using SqlCommand command = + new($"INSERT INTO {_float16Table.Name} ({ColumnName}) VALUES ({ParameterName})", connection); + command.Parameters.AddWithValue( + ParameterName, new SqlVector(new[] { (Half)1.5f, (Half)2.5f, (Half)3.5f })); + + InvalidOperationException exception = + Assert.Throws(() => command.ExecuteNonQuery()); + + Assert.Contains("float16", exception.Message); + Assert.Contains("Vector Type Support", exception.Message); + Assert.Contains("V2", exception.Message); + } + + /// + /// Verifies that a bulk copy into a float16 column still works on a connection which did + /// not negotiate float16. Such a connection is told the column is a varchar(max), + /// so the value travels as text and the server converts it — the column is never a vector + /// from the client's point of view, and the base type check does not arise. This is what + /// makes the default level usable for loading float16 data. + /// + /// The level to request, or null to leave the keyword unset. + [ConditionalTheory(nameof(IsSupported))] + [InlineData(null)] + [InlineData(SqlVectorTypeSupport.V1)] + [InlineData(SqlVectorTypeSupport.Off)] + public void BulkCopiesIntoFloat16AsTextBelowTheNegotiatedVersion(SqlVectorTypeSupport? setting) + { + DataTable source = new(); + source.Columns.Add(ColumnName, typeof(string)); + source.Rows.Add("[1.5,2.5,3.5]"); + + using SqlConnection connection = OpenAt(setting); + + using (SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = _float16Table.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(source); + } + + // Read back over a v2 connection, which sees the column as a vector. + using SqlDataReader reader = Select(_float16Table); + Assert.True(reader.Read()); + Assert.Equal([1.5f, 2.5f, 3.5f], reader.GetSqlVector(0).Memory.ToArray()); + } + + /// + /// Opens a connection requesting a given level of vector type support. + /// + /// The level to request, or null to leave the keyword unset. + /// An open connection. + private static SqlConnection OpenAt(SqlVectorTypeSupport? setting) + { + SqlConnectionStringBuilder builder = new(DataTestUtility.TCPConnectionString); + + if (setting.HasValue) + { + builder.VectorTypeSupport = setting.Value; + } + + SqlConnection connection = new(builder.ConnectionString); + connection.Open(); + return connection; + } + #endif + + #endregion + #region Bulk copy across base types [ConditionalTheory(nameof(IsSupported))] @@ -625,6 +712,104 @@ public void BulkCopiesJsonStringSourceFromAnObjectTypedColumn() Assert.Equal([1.5f, 2.5f, 3.5f], reader.GetSqlVector(0).Memory.ToArray()); } + /// + /// Verifies that the element count limit is still enforced for a float32 destination, + /// which accepts half as many elements as a float16 one. The JSON intermediate is always + /// float32 and is built without the limit so that a wide float16 column can be loaded, so + /// the limit has to be applied against the destination's base type instead. Without it an + /// oversized payload would reach the server and come back as a column length error. + /// + [ConditionalFact(nameof(IsSupported))] + public void BulkCopyRejectsAJsonSourceWiderThanTheFloat32Limit() + { + // 1998 is the most a float32 vector column can declare; the source supplies more. + using Table wideFloat32Table = new(_managementConnection, "VectorF32WideTable", + $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector(1998, float32) NULL)"); + + DataTable source = new(); + source.Columns.Add(ColumnName, typeof(string)); + source.Rows.Add(JsonSerializer.Serialize(new float[2000])); + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = wideFloat32Table.Name }; + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + + // Bulk copy wraps the failure to name the column and row. + InvalidOperationException exception = + Assert.Throws(() => bulkCopy.WriteToServer(source)); + + Assert.IsType(exception.InnerException); + } + + /// + /// Verifies that a JSON value near a binary16 tie is stored identically whether it is + /// bulk copied or inserted through a literal. A bulk copy parses the text to float32 and + /// narrows it on the client, so the value is rounded twice; an INSERT leaves the text to + /// the server. If the server rounded once, the same JSON would give different stored + /// values for the two paths. + /// + /// A JSON array holding one value at or near a binary16 tie. + [ConditionalTheory(nameof(IsSupported))] + // Exactly the tie between 1.0 and 1.0009765625, which ties-to-even resolves downwards. + [InlineData("[1.00048828125]")] + // Just above that tie in decimal, but the nearest float32 is the tie itself. + [InlineData("[1.00048828125000001]")] + // The tie between 1.0009765625 and 1.001953125, which resolves upwards. + [InlineData("[1.00146484375]")] + // Half of the smallest subnormal, and just above it. + [InlineData("[2.98023223876953125e-8]")] + [InlineData("[3.0e-8]")] + public void BulkCopyRoundsTheSameWayAsTheServer(string literal) + { + using Table table = new(_managementConnection, "VectorF16RoundingTable", + $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector(1, float16) NULL)"); + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + // The server parses the literal itself. + using (SqlCommand insert = + new($"INSERT INTO {table.Name} ({ColumnName}) VALUES ('{literal}')", connection)) + { + insert.ExecuteNonQuery(); + } + + float[] viaServer = ReadLast(connection, table); + + // The client parses the same text and narrows it before sending. + DataTable source = new(); + source.Columns.Add(ColumnName, typeof(string)); + source.Rows.Add(literal); + + using (SqlBulkCopy bulkCopy = new(connection) { DestinationTableName = table.Name }) + { + bulkCopy.ColumnMappings.Add(ColumnName, ColumnName); + bulkCopy.WriteToServer(source); + } + + float[] viaClient = ReadLast(connection, table); + + Assert.Equal(viaServer, viaClient); + } + + /// + /// Reads the most recently inserted vector from a table as single precision values. + /// + /// An open connection. + /// The table to read from. + /// The elements of the last row's vector. + private static float[] ReadLast(SqlConnection connection, Table table) + { + using SqlCommand command = + new($"SELECT TOP 1 {ColumnName} FROM {table.Name} ORDER BY Id DESC", connection); + using SqlDataReader reader = command.ExecuteReader(); + + Assert.True(reader.Read()); + return reader.GetSqlVector(0).Memory.ToArray(); + } + [ConditionalFact(nameof(IsSupported))] public void BulkCopyRejectsValuesOutsideTheFloat16Range() { From f9bece326f70912bd260c40830d1315244cfdd89 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 6 Oct 2026 19:46:30 +0530 Subject: [PATCH 21/24] Accept a JSON string as a vector parameter value The declaration and the binary metadata for a vector parameter were read by casting the raw value to ISqlVector, so a JSON string paired with SqlDbType.Vector threw InvalidCastException. That pairing is what a DbDataAdapter update of a float16 column produces on .NET Framework: the column has no System.Half to be surfaced as, so it is read as a string, and SqlCommandBuilder takes SqlDbType.Vector from the column's provider type. Both forms are now described through one helper, so the declaration and the payload agree whichever was used. Nothing accepted a string here before, so this only widens what works. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../Microsoft/Data/SqlClient/SqlCommand.cs | 2 +- .../Microsoft/Data/SqlClient/SqlParameter.cs | 37 +++++++++++++ .../src/Microsoft/Data/SqlClient/TdsParser.cs | 6 +-- .../VectorTest/VectorFloat16BehaviourTests.cs | 52 +++++++++++++++++++ 4 files changed, 93 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs index e779f9c7ed..f4ba511eb3 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs @@ -2279,7 +2279,7 @@ private string BuildParamList(TdsParser parser, SqlParameterCollection parameter { // The validate function for SqlParameters would have already thrown // InvalidCastException if an incompatible value is specified for vector type. - ISqlVector vectorProps = (ISqlVector)sqlParam.Value; + ISqlVector vectorProps = sqlParam.GetVectorProperties(); // The base type is only stated for float16, so that the declaration // emitted for float32 vectors is unchanged from earlier versions and diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs index 4fe3e127e6..c2eec19918 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs @@ -1705,6 +1705,43 @@ internal byte GetActualPrecision() return ShouldSerializePrecision() ? PrecisionInternal : ValuePrecision(CoercedValue); } + /// + /// The vector properties of this parameter's value: its base type, element count, + /// and payload. + /// + /// + /// A caller may supply a vector either as a or as + /// a JSON array in a string. The latter is the only form available to a .NET + /// Framework caller round-tripping a float16 column, which has no + /// System.Half to be surfaced as and so is read as a string; a + /// update built from that column + /// therefore pairs a string value with SqlDbType.Vector. Both forms are + /// described here so that the declaration and the payload agree whichever was used. + /// + internal ISqlVector GetVectorProperties() + { + if (Value is ISqlVector vector) + { + return vector; + } + + if (Value is string json) + { + try + { + return SqlVector.CreateForConversion( + JsonSerializer.Deserialize(json, SqlClientJsonSerializerContext.Default.SingleArray)); + } + catch (Exception ex) when (ex is ArgumentNullException || ex is JsonException) + { + throw ADP.InvalidJsonStringForVector(json, ex); + } + } + + // Validate rejects every other type, so this is unreachable for a valid value. + throw ADP.InvalidCast(); + } + internal object GetCoercedValue() { // NOTE: User can change the Udt at any time diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs index 72acfdaebf..ecdbb347bf 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -10736,7 +10736,7 @@ private Task TDSExecuteRPCAddParameter(TdsParserStateObject stateObj, SqlParamet { // For vector type we need to write the size in bytes required to represent // vector value when communicating with SQL Server. - var sqlVectorProps = ((ISqlVector)param.Value); + var sqlVectorProps = param.GetVectorProperties(); maxsize = sqlVectorProps.Size; } @@ -10765,7 +10765,7 @@ private Task TDSExecuteRPCAddParameter(TdsParserStateObject stateObj, SqlParamet else if (mt.SqlDbType == SqlDbTypeExtensions.Vector) { // For vector type we need to write scale as the element type of the vector. - byte elementType = ((ISqlVector)param.Value).ElementType; + byte elementType = param.GetVectorProperties().ElementType; VectorTypeSupportUtilities.ThrowIfBaseTypeNotNegotiated(elementType, Capabilities.VectorVersion); @@ -10855,7 +10855,7 @@ private Task TDSExecuteRPCAddParameter(TdsParserStateObject stateObj, SqlParamet // for codePageEncoded types, WriteValue simply expects the number of characters // For plp types, we also need the encoded byte size // For vector type we need to write scale as the element type of the vector. - byte writeScale = mt.SqlDbType == SqlDbTypeExtensions.Vector ? ((ISqlVector)param.Value).ElementType : param.GetActualScale(); + byte writeScale = mt.SqlDbType == SqlDbTypeExtensions.Vector ? param.GetVectorProperties().ElementType : param.GetActualScale(); writeParamTask = WriteValue(value, mt, isParameterEncrypted ? (byte)0 : writeScale, actualSize, codePageByteSize, isParameterEncrypted ? 0 : param.Offset, stateObj, isParameterEncrypted ? 0 : param.Size, isDataFeed); } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index 27ba26ec07..bfeccec88c 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -810,6 +810,58 @@ private static float[] ReadLast(SqlConnection connection, Table table) return reader.GetSqlVector(0).Memory.ToArray(); } + /// + /// Verifies that a JSON string can be used as a vector parameter, which is how a + /// update of a float16 column arrives on + /// .NET Framework: the column has no System.Half to be surfaced as, so it is read + /// as a string, and pairs that string with + /// SqlDbType.Vector taken from the column's provider type. + /// + /// + /// Concurrency matches on the key alone because a vector column cannot appear in a WHERE + /// clause, which is a server restriction rather than anything to do with the base type. + /// + /// .NET Framework only. On .NET the column is surfaced as SqlVector<Half>, + /// which stores through SqlUdtStorage and then refuses to + /// assign; that is a limitation of which applies equally to + /// SqlVector<float> and so predates this base type. + /// + /// + #if !NET + [ConditionalFact(nameof(IsSupported))] + public void UpdatesAFloat16ColumnThroughADataAdapter() + { + using Table table = new(_managementConnection, "VectorF16AdapterTable", + $"(Id INT PRIMARY KEY, {ColumnName} vector(3, float16) NULL)"); + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using (SqlCommand seed = + new($"INSERT INTO {table.Name} VALUES (1, '[1.5,2.5,3.5]')", connection)) + { + seed.ExecuteNonQuery(); + } + + using SqlDataAdapter adapter = new($"SELECT Id, {ColumnName} FROM {table.Name}", connection); + using SqlCommandBuilder builder = new(adapter) + { + ConflictOption = ConflictOption.OverwriteChanges + }; + + DataTable rows = new(); + adapter.Fill(rows); + + // No System.Half, so the column is surfaced as a JSON string and SqlCommandBuilder + // pairs that string with SqlDbType.Vector. This is the case being guarded. + rows.Rows[0][ColumnName] = "[4.5,5.5,6.5]"; + + adapter.Update(rows); + + Assert.Equal([4.5f, 5.5f, 6.5f], ReadLast(connection, table)); + } + #endif + [ConditionalFact(nameof(IsSupported))] public void BulkCopyRejectsValuesOutsideTheFloat16Range() { From 2592c223d6f8e8eaf2fa22188ba4f95e901d2aed Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 6 Oct 2026 19:55:26 +0530 Subject: [PATCH 22/24] Say where the vector column metadata is unavailable VectorBaseType and VectorDimensions are null for a vector column the server returned as a varchar(max), which is every float16 column at v1 and every vector column at off. An application on the default therefore cannot tell a float16 column from text through them, so point at sys.columns.vector_base_type for that case, and cover it with a test. Record that a float16 output parameter cannot be declared from .NET Framework, since both value forms available there declare float32. Measured: Value and SqlValue agree, both returning a widened SqlVector. The branch is kept for a server which returns the column's own base type regardless. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .github/instructions/features.instructions.md | 11 +++++-- .../Microsoft/Data/SqlClient/SqlDbColumn.cs | 5 ++- .../Microsoft/Data/SqlClient/SqlParameter.cs | 9 +++++- .../VectorTest/VectorColumnMetadataTests.cs | 31 +++++++++++++++++++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/.github/instructions/features.instructions.md b/.github/instructions/features.instructions.md index 77de54f503..b709ec66df 100644 --- a/.github/instructions/features.instructions.md +++ b/.github/instructions/features.instructions.md @@ -164,9 +164,14 @@ Notes: see a value it did not negotiate. - A column's base type and number of dimensions are available from the column schema: `reader.GetColumnSchema()[i]["VectorBaseType"]` and `["VectorDimensions"]`. Both are `null` - for columns which are not vectors. This is the only way to tell the two base types apart - when a `float16` column is surfaced as a JSON string, because `GetFieldType` reports - `string` for such a column just as it does for a `varchar` one: + for columns which are not vectors, **including a vector column the server returned as + `varchar(max)` because the connection did not negotiate its base type**. So at `v1` a + `float16` column reports `null` for both, and at `off` so does every vector column; query + `sys.columns.vector_base_type` and `sys.columns.vector_dimensions` instead when the + connection has not negotiated the column's base type. Where the column *is* negotiated, + these properties are the only way to tell the two base types apart when a `float16` column + is surfaced as a JSON string, because `GetFieldType` reports `string` for such a column + just as it does for a `varchar` one: ```csharp DbColumn column = reader.GetColumnSchema()[0]; diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs index df8996fdc3..79d35a4a0a 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDbColumn.cs @@ -139,7 +139,10 @@ internal int? SqlNumericScale /// /// Both properties are for columns which are not vectors, /// including a vector column which the server returned as varchar because - /// the connection did not negotiate support for its base type. + /// the connection did not negotiate support for its base type. A connection at + /// v1 therefore reports for a float16 column, + /// and one at off for every vector column; sys.columns.vector_base_type + /// and sys.columns.vector_dimensions describe such a column instead. /// /// public override object this[string property] => diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs index c2eec19918..364f6aac85 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs @@ -796,7 +796,14 @@ private object GetVectorReturnValue() #if NET return new SqlVector((byte[])_sqlBufferReturnValue.Value); #else - // Widening binary16 to binary32 is exact, so no information is lost. + // Defensive. A .NET Framework caller has no way to declare a float16 + // vector parameter: the declared base type comes from the value, and + // both forms available there — a SqlVector and a JSON string — + // declare float32, so a server which converts between base types + // returns float32 here. Kept so that a server which returns the + // column's own base type regardless is still read correctly rather + // than misinterpreting the payload. Widening binary16 to binary32 is + // exact, so no information is lost. return SqlVector.FromTdsPayload((byte[])_sqlBufferReturnValue.Value); #endif default: diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs index 74cf5878a0..2d135444dc 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs @@ -166,6 +166,37 @@ public void SchemaCollectionOmitsVectorTypeWhenItIsNotNegotiated() /// one, and cannot distinguish the two base types at all for a caller which wants to /// read both through a single representation. /// + /// + /// Verifies that the vector properties read as null for a float16 column on a connection + /// which did not negotiate that base type, because the server describes such a column as + /// a varchar(max) and the driver has nothing to report. An application which stays + /// on the default therefore cannot tell a float16 column from text through these + /// properties, and has to query sys.columns instead. + /// + [ConditionalFact(nameof(IsFloat16Supported))] + public void ReportsNullForAFloat16ColumnWhichWasNotNegotiated() + { + string connectionString = new SqlConnectionStringBuilder(_connectionString) + { + VectorTypeSupport = SqlVectorTypeSupport.V1 + }.ConnectionString; + + using SqlConnection connection = new(connectionString); + connection.Open(); + + using SqlCommand command = + new("SELECT CAST('[1.5,2.5,3.5]' AS vector(3, float16)) AS v", connection); + using SqlDataReader reader = command.ExecuteReader(); + + DbColumn column = reader.GetColumnSchema()[0]; + + Assert.Null(column["VectorBaseType"]); + Assert.Null(column["VectorDimensions"]); + + // The column is indistinguishable from text through the reader's own metadata. + Assert.Equal(typeof(string), column.DataType); + } + [ConditionalFact(nameof(IsFloat16Supported))] public void DrivesReadPathForACallerWhichDoesNotKnowTheSchema() { From c428dcdc6fe1fce5d567a34857ce4b79e042c7b7 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 6 Oct 2026 20:18:31 +0530 Subject: [PATCH 23/24] Send a JSON vector parameter as text Building a vector from a JSON string fixed its base type to float32, so a column of another base type then needed the server to convert, which not every build does. The string is now sent as a varchar(max) for the server to parse into whatever base type the column has, which works everywhere and is the only form available to a .NET Framework caller writing a float16 column. Covered by writing 3000 elements to a float16 column, which no float32 payload could carry: a vector payload is capped at the size of a TDS packet, so 1998 float32 elements. It only succeeds if the value travelled as text. Seed the float16 rows of read tests from a literal rather than a SqlVector parameter, so that setting up a test does not depend on the server converting between base types when the test is not about that. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../Microsoft/Data/SqlClient/SqlParameter.cs | 15 ++++ .../VectorTest/VectorFloat16BehaviourTests.cs | 83 ++++++++++++++++++- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs index 364f6aac85..ab5b75d538 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs @@ -2000,6 +2000,21 @@ private MetaType GetMetaTypeOnly() _value = DBNull.Value; return MetaType.GetDefaultMetaType(); } + + if (_metaType.SqlDbType == SqlDbTypeExtensions.Vector && + _direction == ParameterDirection.Input && + _value is string) + { + // A JSON array is sent as text and parsed by the server into whatever + // base type the destination column has. Building a vector from it here + // instead would fix the base type to float32, which a column of another + // base type then needs the server to convert — and not every build does. + // Sending the text keeps this form working against every server, which + // matters because it is the only one available to a .NET Framework + // caller round-tripping a float16 column through a DbDataAdapter. + return MetaType.MetaMaxVarChar; + } + return _metaType; } if (_value != null && DBNull.Value != _value) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index bfeccec88c..d2aac0c786 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -102,7 +102,7 @@ public void ReadsFloat16ColumnAsWidenedSingles() { // Requesting single precision from a float16 column widens the elements, which is // exact. This is the only strongly typed read available where System.Half is not. - Insert(_float16Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + Seed(_float16Table, "[1.5,2.5,3.5]"); using SqlDataReader reader = Select(_float16Table); Assert.True(reader.Read()); @@ -133,7 +133,7 @@ public void RendersValuesExactlyRatherThanShortestRoundTrip() [ConditionalFact(nameof(IsSupported))] public void ReportsUnsupportedElementTypes() { - Insert(_float16Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + Seed(_float16Table, "[1.5,2.5,3.5]"); using SqlDataReader reader = Select(_float16Table); Assert.True(reader.Read()); @@ -147,7 +147,7 @@ public void ReportsProviderSpecificValueAsASqlType() { // Every provider specific value is a type from System.Data.SqlTypes, including the // JSON rendering a float16 column falls back to where System.Half is unavailable. - Insert(_float16Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + Seed(_float16Table, "[1.5,2.5,3.5]"); using SqlDataReader reader = Select(_float16Table); Assert.True(reader.Read()); @@ -453,7 +453,7 @@ public void BulkCopiesFloat16ToFloat32ThroughTheTextualRepresentation() // On .NET Framework a float16 column reads as a JSON string, so a copy into a // float32 column takes the textual path and the value is converted rather than // rejected. This is the counterpart of the .NET case above. - Insert(_float16Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); + Seed(_float16Table, "[1.5,2.5,3.5]"); using SqlConnection sourceConnection = new(_connectionString); sourceConnection.Open(); @@ -862,6 +862,50 @@ public void UpdatesAFloat16ColumnThroughADataAdapter() } #endif + /// + /// Verifies that a JSON string parameter is sent as text rather than as a float32 + /// vector, by writing more elements than a float32 payload could carry at all. A vector + /// payload is capped at the size of a TDS packet, which is 1998 float32 elements, so a + /// 3000 element float16 column can only be written this way if the value travels as + /// text for the server to parse. + /// + /// + /// This is what keeps the JSON form working against servers which block conversion + /// between base types, and it is the only form available to a .NET Framework caller + /// writing a float16 column. + /// + [ConditionalFact(nameof(IsSupported))] + public void SendsAJsonStringParameterAsTextRatherThanAFloat32Vector() + { + const int Dimensions = 3000; + + using Table wide = new(_managementConnection, "VectorF16WideParamTable", + $"(Id INT PRIMARY KEY IDENTITY, {ColumnName} vector({Dimensions}, float16) NULL)"); + + float[] values = new float[Dimensions]; + for (int i = 0; i < values.Length; i++) + { + // Eighths are exactly representable in binary16 at this magnitude. + values[i] = (i % 8) * 0.125f; + } + + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using (SqlCommand insert = + new($"INSERT INTO {wide.Name} ({ColumnName}) VALUES ({ParameterName})", connection)) + { + insert.Parameters.Add(new SqlParameter(ParameterName, SqlDbTypeExtensions.Vector) + { + Value = JsonSerializer.Serialize(values) + }); + + Assert.Equal(1, insert.ExecuteNonQuery()); + } + + Assert.Equal(values, ReadLast(connection, wide)); + } + [ConditionalFact(nameof(IsSupported))] public void BulkCopyRejectsValuesOutsideTheFloat16Range() { @@ -891,6 +935,16 @@ public void BulkCopyRejectsValuesOutsideTheFloat16Range() #region Helpers + /// + /// Inserts a vector through a parameter of the given value's own type. + /// + /// The destination table. + /// The value to insert. + /// + /// Writing a value whose base type differs from the column's needs the server to convert + /// between base types, which not every build does. Use instead where + /// the write is only setting up for what the test actually checks. + /// private void Insert(Table table, object value) { using SqlConnection connection = new(_connectionString); @@ -903,6 +957,27 @@ private void Insert(Table table, object value) Assert.Equal(1, command.ExecuteNonQuery()); } + /// + /// Inserts a vector from its textual form, which every server accepts whatever its + /// column's base type and without converting between base types. + /// + /// The destination table. + /// A JSON array holding the vector's elements. + /// + /// Used where a row is only being set up for what the test actually checks, so that the + /// setup does not depend on a server behaviour the test is not about. + /// + private void Seed(Table table, string literal) + { + using SqlConnection connection = new(_connectionString); + connection.Open(); + + using SqlCommand command = + new($"INSERT INTO {table.Name} ({ColumnName}) VALUES ('{literal}')", connection); + + Assert.Equal(1, command.ExecuteNonQuery()); + } + private SqlDataReader Select(Table table) { SqlConnection connection = new(_connectionString); From de32fbfaf936581c249d70011363837fe813dc72 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Tue, 6 Oct 2026 20:27:51 +0530 Subject: [PATCH 24/24] Document the remaining vector test methods Inserting a test between a summary and its method left two summary blocks together, detaching the one for DrivesReadPathForACallerWhichDoesNotKnowTheSchema. Put each back above the method it describes. The testing instructions ask for a behaviour-focused summary on every test method, so promote the leading comments in VectorFloat16BehaviourTests to summaries as well; those were written before that requirement was raised and were the only ones in the vector suites still relying on inline comments. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e17ed782-c576-4cb7-9b4b-7ad286d7a7d0 --- .../VectorTest/VectorColumnMetadataTests.cs | 14 +- .../VectorTest/VectorFloat16BehaviourTests.cs | 125 +++++++++++++----- 2 files changed, 100 insertions(+), 39 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs index 2d135444dc..bfd8aa7eea 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorColumnMetadataTests.cs @@ -159,13 +159,6 @@ public void SchemaCollectionOmitsVectorTypeWhenItIsNotNegotiated() return null; } - /// - /// Verifies the use case the properties exist for: choosing a read path without knowing - /// the schema in advance. GetFieldType is not enough on its own, because it - /// reports string for a float16 column on .NET Framework just as it does for a varchar - /// one, and cannot distinguish the two base types at all for a caller which wants to - /// read both through a single representation. - /// /// /// Verifies that the vector properties read as null for a float16 column on a connection /// which did not negotiate that base type, because the server describes such a column as @@ -197,6 +190,13 @@ public void ReportsNullForAFloat16ColumnWhichWasNotNegotiated() Assert.Equal(typeof(string), column.DataType); } + /// + /// Verifies the use case the properties exist for: choosing a read path without knowing + /// the schema in advance. GetFieldType is not enough on its own, because it + /// reports string for a float16 column on .NET Framework just as it does for a varchar + /// one, and cannot distinguish the two base types at all for a caller which wants to + /// read both through a single representation. + /// [ConditionalFact(nameof(IsFloat16Supported))] public void DrivesReadPathForACallerWhichDoesNotKnowTheSchema() { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs index d2aac0c786..92ad892cdf 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorFloat16BehaviourTests.cs @@ -75,12 +75,14 @@ public VectorFloat16BehaviourTests() #region Column metadata + /// + /// Verifies that a float16 column reports its base type by name and its dimension count + /// correctly, the latter needing the smaller element size to be accounted for. Metadata + /// for vector columns in general is covered by VectorColumnMetadataTests. + /// [ConditionalFact(nameof(IsSupported))] public void ColumnSchemaReportsFloat16BaseType() { - // Metadata for vector columns in general is covered by VectorColumnMetadataTests; - // this checks only that the float16 base type is reported by its own name, and that - // its dimension count accounts for the smaller element size. using SqlConnection connection = new(_connectionString); connection.Open(); @@ -97,11 +99,14 @@ public void ColumnSchemaReportsFloat16BaseType() #region Reading + /// + /// Verifies that requesting single precision from a float16 column widens the elements, + /// which is exact. This is the only strongly typed read available where + /// System.Half is not. + /// [ConditionalFact(nameof(IsSupported))] public void ReadsFloat16ColumnAsWidenedSingles() { - // Requesting single precision from a float16 column widens the elements, which is - // exact. This is the only strongly typed read available where System.Half is not. Seed(_float16Table, "[1.5,2.5,3.5]"); using SqlDataReader reader = Select(_float16Table); @@ -113,12 +118,15 @@ public void ReadsFloat16ColumnAsWidenedSingles() Assert.Equal([1.5f, 2.5f, 3.5f], vector.Memory.ToArray()); } + /// + /// Verifies that the string read paths render a float16 element's true value. The + /// largest finite binary16 value renders as 65500 if the elements are formatted as + /// System.Half, because that is the shortest string which round trips to the same + /// Half; widening to single precision first renders the value itself. + /// [ConditionalFact(nameof(IsSupported))] public void RendersValuesExactlyRatherThanShortestRoundTrip() { - // The largest finite binary16 value renders as 65500 if the elements are formatted - // as System.Half, because that is the shortest string which round trips to the same - // Half. Widening to single precision first renders the value itself. using SqlConnection connection = new(_connectionString); connection.Open(); @@ -130,6 +138,10 @@ public void RendersValuesExactlyRatherThanShortestRoundTrip() Assert.Equal("[65504,1,2]", reader.GetFieldValue(0)); } + /// + /// Verifies that reading a vector as an element type which is not a supported base type + /// is rejected, rather than reinterpreting the payload. + /// [ConditionalFact(nameof(IsSupported))] public void ReportsUnsupportedElementTypes() { @@ -142,11 +154,14 @@ public void ReportsUnsupportedElementTypes() Assert.Throws(() => reader.GetSqlVector(0)); } + /// + /// Verifies that every provider specific value is a type from + /// System.Data.SqlTypes, including the JSON rendering a float16 column falls back + /// to where System.Half is unavailable. + /// [ConditionalFact(nameof(IsSupported))] public void ReportsProviderSpecificValueAsASqlType() { - // Every provider specific value is a type from System.Data.SqlTypes, including the - // JSON rendering a float16 column falls back to where System.Half is unavailable. Seed(_float16Table, "[1.5,2.5,3.5]"); using SqlDataReader reader = Select(_float16Table); @@ -166,12 +181,15 @@ public void ReportsProviderSpecificValueAsASqlType() #endif } + /// + /// Verifies that a narrowing read is rejected the same way for a null row as for a + /// populated one. Whether a payload can be read as a given element type is a property of + /// the column's base type, so a null row must not appear to succeed where a populated + /// row in the same column fails — which was a real defect found in review. + /// [ConditionalFact(nameof(IsSupported))] public void ReportsNarrowingReadsConsistentlyForNullAndNonNullRows() { - // The base type pairing is a property of the column, so a null row has to be - // rejected the same way a populated one is. Reading a float32 column as a vector of - // a narrower element type is not supported in either case. Insert(_float32Table, DBNull.Value); Insert(_float32Table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); @@ -193,6 +211,11 @@ public void ReportsNarrowingReadsConsistentlyForNullAndNonNullRows() AssertNarrowingReadIsRejected(reader); } + /// + /// The asynchronous counterpart of + /// , since the two + /// read paths resolve the element type separately. + /// [ConditionalFact(nameof(IsSupported))] public async Task ReportsNarrowingReadsConsistentlyForNullAndNonNullRowsAsync() { @@ -381,13 +404,16 @@ private static SqlConnection OpenAt(SqlVectorTypeSupport? setting) #region Bulk copy across base types + /// + /// Verifies that a payload read from a vector column is transferred to a column of the + /// same base type as-is, with no conversion and no intermediate representation. + /// + /// The base type of both the source and destination columns. [ConditionalTheory(nameof(IsSupported))] [InlineData("float16")] [InlineData("float32")] public void BulkCopiesBetweenColumnsOfTheSameBaseType(string baseType) { - // A payload read from a vector column is transferred to a column of the same base - // type as-is, with no conversion and no intermediate representation. Table table = baseType == "float16" ? _float16Table : _float32Table; Insert(table, new SqlVector(new float[] { 1.5f, 2.5f, 3.5f })); @@ -414,6 +440,14 @@ public void BulkCopiesBetweenColumnsOfTheSameBaseType(string baseType) Assert.Equal([1.5f, 2.5f, 3.5f], verifyReader.GetSqlVector(0).Memory.ToArray()); } + /// + /// Verifies that a reader copy between columns of different base types is reported by + /// the server rather than silently rewritten. A payload read from a vector column keeps + /// its own base type, and the INSERT BULK declaration states the destination's, + /// so the two disagree. A caller which wants the conversion reads the source as text. + /// + /// The source column's base type. + /// The destination column's base type. [ConditionalTheory(nameof(IsSupported))] [InlineData("float32", "float16")] #if NET @@ -423,10 +457,6 @@ public void BulkCopiesBetweenColumnsOfTheSameBaseType(string baseType) #endif public void BulkCopyRejectsColumnsOfDifferentBaseTypes(string sourceBaseType, string destinationBaseType) { - // A payload read from a vector column keeps its own base type, and the INSERT BULK - // declaration states the destination's, so the server reports the mismatch. The - // driver does not silently rewrite the payload: a caller which wants the conversion - // reads the source column as text, which the server converts. Table source = sourceBaseType == "float16" ? _float16Table : _float32Table; Table destination = destinationBaseType == "float16" ? _float16Table : _float32Table; @@ -447,12 +477,16 @@ public void BulkCopyRejectsColumnsOfDifferentBaseTypes(string sourceBaseType, st } #if !NET + /// + /// Verifies the .NET Framework counterpart of + /// : a float16 column reads as a + /// JSON string there, so a copy into a float32 column takes the textual path and the + /// value is converted rather than rejected. That divergence is inherent to the base type + /// having no native representation on .NET Framework. + /// [ConditionalFact(nameof(IsSupported))] public void BulkCopiesFloat16ToFloat32ThroughTheTextualRepresentation() { - // On .NET Framework a float16 column reads as a JSON string, so a copy into a - // float32 column takes the textual path and the value is converted rather than - // rejected. This is the counterpart of the .NET case above. Seed(_float16Table, "[1.5,2.5,3.5]"); using SqlConnection sourceConnection = new(_connectionString); @@ -478,6 +512,11 @@ public void BulkCopiesFloat16ToFloat32ThroughTheTextualRepresentation() } #endif + /// + /// Verifies that nulls survive a reader copy between columns of the same base type, and + /// are not confused with an empty or zeroed vector. + /// + /// The base type of both the source and destination columns. [ConditionalTheory(nameof(IsSupported))] [InlineData("float16")] [InlineData("float32")] @@ -528,11 +567,14 @@ public void BulkCopyPreservesNullsBetweenColumnsOfTheSameBaseType(string baseTyp } } + /// + /// Verifies that a JSON string source loads into a float16 column. This is the ordinary + /// table to table path where System.Half is unavailable, since a float16 column + /// reads back as a JSON string there. + /// [ConditionalFact(nameof(IsSupported))] public void BulkCopiesJsonStringSourceIntoFloat16Column() { - // A float16 column reads back as a JSON string where System.Half is unavailable, so - // this is the ordinary table to table path on those frameworks. DataTable table = new(); table.Columns.Add(ColumnName, typeof(string)); table.Rows.Add("[1.5,2.5,3.5]"); @@ -551,13 +593,16 @@ public void BulkCopiesJsonStringSourceIntoFloat16Column() Assert.Equal([1.5f, 2.5f, 3.5f], reader.GetSqlVector(0).Memory.ToArray()); } + /// + /// Verifies that a reader from another provider is handled, guarding a real defect: the + /// source column type has to be read through rather than the + /// SqlDataReader field, which is null unless the reader is a SqlDataReader. + /// Such a reader still reports a string column, so the value is parsed into the + /// destination's base type as for any other textual source. + /// [ConditionalFact(nameof(IsSupported))] public void BulkCopiesJsonStringSourceFromANonSqlClientReader() { - // The source column type has to be read through IDataReader rather than through the - // SqlDataReader field, which is null unless the reader is a SqlDataReader. A reader - // from another provider still reports a string column, so the value is parsed into - // the destination's base type as it is for any other textual source. DataTable table = new(); table.Columns.Add(ColumnName, typeof(string)); table.Rows.Add(DBNull.Value); @@ -579,11 +624,14 @@ public void BulkCopiesJsonStringSourceFromANonSqlClientReader() Assert.Equal([1.5f, 2.5f, 3.5f], verify.GetSqlVector(0).Memory.ToArray()); } + /// + /// Control for the float16 cases: at v1 a float32 column is still presented as a vector, + /// so the declaration says vector(N) and the string is coerced to a float32 + /// payload by the client rather than travelling as text. + /// [ConditionalFact(nameof(IsSupported))] public void BulkCopiesJsonStringSourceIntoFloat32ColumnAtV1() { - // Control: at v1 a float32 column IS presented as a vector, so the declaration says - // vector(N) and the string is coerced to a float32 payload by the client. string v1 = new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString) { VectorTypeSupport = SqlVectorTypeSupport.V1 @@ -607,11 +655,14 @@ public void BulkCopiesJsonStringSourceIntoFloat32ColumnAtV1() Assert.Contains("1.5", (string)command.ExecuteScalar()); } + /// + /// Verifies that a float16 column can be loaded at v1, where the server presents it as a + /// varchar(max): the ordinary text path applies and the server performs the + /// conversion. This is what makes the default level usable for loading float16 data. + /// [ConditionalFact(nameof(IsSupported))] public void BulkCopiesJsonStringSourceIntoFloat16ColumnAtV1() { - // At v1 the server presents a float16 column as varchar(max), so the ordinary text - // path applies and the server performs the conversion. string v1 = new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString) { VectorTypeSupport = SqlVectorTypeSupport.V1 @@ -906,6 +957,13 @@ public void SendsAJsonStringParameterAsTextRatherThanAFloat32Vector() Assert.Equal(values, ReadLast(connection, wide)); } + /// + /// Verifies that a value outside the float16 range is reported against the value rather + /// than reaching the server. A textual source is parsed into the destination's base type + /// by the client, so the client catches the overflow itself; letting the saturated + /// infinity through would have the server reject it as a malformed vector instead, which + /// says nothing about which value was at fault. + /// [ConditionalFact(nameof(IsSupported))] public void BulkCopyRejectsValuesOutsideTheFloat16Range() { @@ -989,6 +1047,9 @@ private SqlDataReader Select(Table table) return command.ExecuteReader(CommandBehavior.CloseConnection); } + /// + /// Drops the tables this fixture created and closes its management connection. + /// public void Dispose() { if (_disposed)