Name and publisher normalization for cross reference

.github/actions/spelling/excludes.txt

@@ -18,3 +18,6 @@
 ^src/Valijson/
 ^src/YamlCppLib/
 ^\.github/
+^src/AppInstallerCLITests/TestData/InputNames.txt$
+^src/AppInstallerCLITests/TestData/InputPublishers.txt$
+^src/AppInstallerCLITests/TestData/NormalizationInitialIds.txt$

.github/actions/spelling/expect.txt

@@ -27,11 +27,14 @@ bitmask
 bkup
 blargle
 blogs
+bomgar
 BOMs
 brk
 Buf
+casemap
 casemappings
 cch
+CDEF
 centralus
 certmgr
 certs
@@ -54,6 +57,7 @@ cstdint
 ctc
 Ctx
 curated
+CYRL
 deigh
 deleteifnotneeded
 dirs
@@ -71,13 +75,18 @@ EXEHASH
 experimentalfeatures
 fd
 fintimes
+Fixfor
 flargle
 foldc
 foldcase
 FOLDERID
+ftp
 FULLWIDTH
 fuzzer
 gcpi
+GES
+GESMBH
+GHS
 gity
 Globals
 google
@@ -93,6 +102,7 @@ IHost
 IID
 IInstalled
 incosistencies
+IName
 INET
 inor
 IPackage
@@ -113,6 +123,7 @@ KF
 KNOWNFOLDERID
 ktf
 Langs
+LATN
 ldcase
 lhs
 libyaml
@@ -123,6 +134,8 @@ localhost
 LPBYTE
 LPWSTR
 LSTATUS
+LTDA
+MBH
 megamorf
 memcpy
 MMmmbbbb
@@ -134,6 +147,7 @@ MSIHASH
 MSIXHASH
 msstore
 multimap
+mx
 mycustom
 myinstalldir
 mylog
@@ -142,6 +156,7 @@ mysilentwithprogress
 mytool
 Newtonsoft
 NOEXPAND
+normer
 nuffing
 nullopt
 objbase
@@ -166,14 +181,17 @@ pz
 qb
 qword
 readonly
+regexes
 REGSAM
 REINSTALLMODE
 rhs
+roblox
 rosoft
 rowids
 RRF
 rrr
 rzkzqaqjwj
+SARL
 schematab
 sddl
 seof
@@ -187,13 +205,16 @@ sortof
 spamming
 SPAPI
 Srinivasan
+SRL
 srs
 Standalone
 startswith
 strtoull
 subdir
 subkey
+superstring
 suppy
+swervy
 sysrefcomp
 Tagit
 temppath
@@ -204,6 +225,8 @@ tombstoned
 transitioning
 UCase
 ucasemap
+UChars
+uec
 uild
 uintptr
 Uninitialize
@@ -213,27 +236,36 @@ uninstallprevious
 uninstalls
 unparsable
 UNSCOPED
+UParse
 UPSERT
+URIs
 URLZONE
 userfilesetting
 usersettingstest
 USHORT
 Utils
 UWP
 vamus
+VERSI
+VERSIE
 vns
+vy
 wcslen
 webpages
 Webserver
 website
+WERSJA
 wesome
 windir
 windowsdeveloper
 winerror
 wingetdev
 winreg
 withstarts
+wn
+wsv
 Workflows
 wto
 Wunused
 WZDNCRFJ
+zy

.github/actions/spelling/patterns.txt

@@ -18,3 +18,5 @@ data:[a-zA-Z=;,/0-9+-]+
 El proyecto .* diferentes
 # Package family names
 \b[-.A-Za-z0-9]+_[a-z0-9]{13}\b
+# Locales for name normalization
+\b\p{Lu}{2,3}(?:-(?:CANS|CYRL|LATN|MONG))?-\p{Lu}{2}(?![A-Z])(?:-VALENCIA)?\b

src/AppInstallerCLITests/AppInstallerCLITests.vcxproj

@@ -185,8 +185,10 @@
     <ClCompile Include="ExperimentalFeature.cpp" />
     <ClCompile Include="HashCommand.cpp" />
     <ClCompile Include="MsixInfo.cpp" />
+    <ClCompile Include="NameNormalization.cpp" />
     <ClCompile Include="PredefinedInstalledSource.cpp" />
     <ClCompile Include="PreIndexedPackageSource.cpp" />
+    <ClCompile Include="Regex.cpp" />
     <ClCompile Include="Registry.cpp" />
     <ClCompile Include="SQLiteIndexSource.cpp" />
     <ClCompile Include="Strings.cpp" />
@@ -432,6 +434,15 @@
     <CopyFileToFolders Include="TestData\UpdateFlowTest_Msix.yaml">
       <DeploymentContent>true</DeploymentContent>
     </CopyFileToFolders>
+    <CopyFileToFolders Include="TestData\InputNames.txt">
+      <DeploymentContent>true</DeploymentContent>
+    </CopyFileToFolders>
+    <CopyFileToFolders Include="TestData\InputPublishers.txt">
+      <DeploymentContent>true</DeploymentContent>
+    </CopyFileToFolders>
+    <CopyFileToFolders Include="TestData\NormalizationInitialIds.txt">
+      <DeploymentContent>true</DeploymentContent>
+    </CopyFileToFolders>
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\AppInstallerCLICore\AppInstallerCLICore.vcxproj">

src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters

@@ -110,6 +110,12 @@
     <ClCompile Include="Registry.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="NameNormalization.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="Regex.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="PropertySheet.props" />
@@ -333,5 +339,14 @@
     <CopyFileToFolders Include="TestData\UpdateFlowTest_Msix.yaml">
       <Filter>TestData</Filter>
     </CopyFileToFolders>
+    <CopyFileToFolders Include="TestData\InputNames.txt">
+      <Filter>TestData</Filter>
+    </CopyFileToFolders>
+    <CopyFileToFolders Include="TestData\InputPublishers.txt">
+      <Filter>TestData</Filter>
+    </CopyFileToFolders>
+    <CopyFileToFolders Include="TestData\NormalizationInitialIds.txt">
+      <Filter>TestData</Filter>
+    </CopyFileToFolders>
   </ItemGroup>
 </Project>

src/AppInstallerCLITests/NameNormalization.cpp

@@ -0,0 +1,133 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+#include "pch.h"
+#include "TestCommon.h"
+#include <winget/NameNormalization.h>
+
+using namespace std::string_view_literals;
+using namespace AppInstaller::Utility;
+
+
+// This skipped test case can be used to update the test file.
+// It writes back to the output content location, so you must manually
+// copy the file(s) back to the git managed location to update.
+TEST_CASE("NameNorm_Update_Database_Initial", "[.]")
+{
+    std::ifstream namesStream(TestCommon::TestDataFile("InputNames.txt"));
+    REQUIRE(namesStream);
+    std::ifstream publishersStream(TestCommon::TestDataFile("InputPublishers.txt"));
+    REQUIRE(publishersStream);
+    std::ofstream resultsStream(TestCommon::TestDataFile("NormalizationInitialIdsUpdate.txt"), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
+    REQUIRE(resultsStream);
+
+    // Far larger than any one value; hopefully
+    char name[4096]{};
+    char publisher[4096]{};
+
+    NameNormalizer normer(NormalizationVersion::Initial);
+
+    for (;;)
+    {
+        namesStream.getline(name, ARRAYSIZE(name));
+        publishersStream.getline(publisher, ARRAYSIZE(publisher));
+
+        if (namesStream || publishersStream)
+        {
+            REQUIRE(namesStream);
+            REQUIRE(publishersStream);
+
+            INFO("Name[" << name << "], Publisher[" << publisher << "]");
+
+            auto normalized = normer.Normalize(name, publisher);
+
+            std::string normalizedId = normalized.Publisher();
+            normalizedId += '.';
+            normalizedId += normalized.Name();
+
+            resultsStream << normalizedId << std::endl;
+            REQUIRE(resultsStream);
+        }
+        else
+        {
+            break;
+        }
+    }
+}
+
+// If this test is failing, either changes to winget code or the ICU binaries have caused it.
+// This will impact the functionality of the PreIndexedPackageSource, as it is the primary
+// mechanism used to cross reference packages installed outside of winget with those in the
+// source.
+TEST_CASE("NameNorm_Database_Initial", "[name_norm]")
+{
+    std::ifstream namesStream(TestCommon::TestDataFile("InputNames.txt"));
+    REQUIRE(namesStream);
+    std::ifstream publishersStream(TestCommon::TestDataFile("InputPublishers.txt"));
+    REQUIRE(publishersStream);
+    std::ifstream resultsStream(TestCommon::TestDataFile("NormalizationInitialIds.txt"));
+    REQUIRE(resultsStream);
+
+    // Far larger than any one value; hopefully
+    char name[4096]{};
+    char publisher[4096]{};
+    char expectedId[4096]{};
+
+    NameNormalizer normer(NormalizationVersion::Initial);
+
+    for (;;)
+    {
+        namesStream.getline(name, ARRAYSIZE(name));
+        publishersStream.getline(publisher, ARRAYSIZE(publisher));
+        resultsStream.getline(expectedId, ARRAYSIZE(expectedId));
+
+        if (namesStream || publishersStream || resultsStream)
+        {
+            REQUIRE(namesStream);
+            REQUIRE(publishersStream);
+            REQUIRE(resultsStream);
+
+            INFO("Name[" << name << "], Publisher[" << publisher << "]");
+
+            auto normalized = normer.Normalize(name, publisher);
+
+            std::string normalizedId = normalized.Publisher();
+            normalizedId += '.';
+            normalizedId += normalized.Name();
+
+            REQUIRE(expectedId == normalizedId);
+        }
+        else
+        {
+            break;
+        }
+    }
+}
+
+TEST_CASE("NameNorm_Architecture", "[name_norm]")
+{
+    NameNormalizer normer(NormalizationVersion::Initial);
+
+    REQUIRE(normer.Normalize("Name", {}).Architecture() == Architecture::Unknown);
+    REQUIRE(normer.Normalize("Name x86", {}).Architecture() == Architecture::X86);
+    REQUIRE(normer.Normalize("Name x86_64", {}).Architecture() == Architecture::X64);
+    REQUIRE(normer.Normalize("Name (64 bit)", {}).Architecture() == Architecture::X64);
+    REQUIRE(normer.Normalize("Name 32/64 bit", {}).Architecture() == Architecture::Unknown);
+    REQUIRE(normer.Normalize("Fox86", {}).Architecture() == Architecture::Unknown);
+}
+
+TEST_CASE("NameNorm_Locale", "[name_norm]")
+{
+    NameNormalizer normer(NormalizationVersion::Initial);
+
+    REQUIRE(normer.Normalize("Name", {}).Locale() == "");
+    REQUIRE(normer.Normalize("Name en-US", {}).Locale() == "en-us");
+    REQUIRE(normer.Normalize("Name (es-mx)", {}).Locale() == "es-mx");
+    REQUIRE(normer.Normalize("Names-mx", {}).Locale() == "");
+}
+
+TEST_CASE("NameNorm_KBNumbers", "[name_norm]")
+{
+    NameNormalizer normer(NormalizationVersion::Initial);
+
+    REQUIRE(normer.Normalize("Fix for (KB42)", {}).Name() == "FixforKB42");
+}