diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index a8655c4401..96750dc58c 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -130,6 +130,7 @@ ENDSESSION EPester epth EQU +errcode errmsg ERRORONEXIT ESource diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 38908034b7..a20708ba6b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,6 +7,7 @@ pr: branches: include: - master + - release-v1.6 paths: include: - azure-pipelines.yml @@ -123,9 +124,11 @@ jobs: /p:UapAppxPackageBuildMode=SideloadOnly' - task: CopyFiles@2 - displayName: 'Copy WindowsPackageManager.dll Symbols to artifacts folder' + displayName: 'Copy Symbols to artifacts folder' inputs: - Contents: '$(buildOutDir)\WindowsPackageManager\WindowsPackageManager.pdb' + Contents: | + $(buildOutDir)\WindowsPackageManager\WindowsPackageManager.pdb + $(buildOutDir)\Microsoft.Management.Configuration\Microsoft.Management.Configuration.pdb TargetFolder: '$(artifactsDir)' condition: succeededOrFailed() @@ -305,7 +308,8 @@ jobs: codeCoverageEnabled: true platform: '$(buildPlatform)' configuration: '$(BuildConfiguration)' - condition: succeededOrFailed() + diagnosticsEnabled: true + condition: and(succeededOrFailed(), eq(variables.buildPlatform, 'x86')) - task: PowerShell@2 displayName: Prepare for Microsoft.Management.Configuration.UnitTests (OutOfProc) diff --git a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp index 782344d97e..88bfa120e5 100644 --- a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp +++ b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp @@ -15,6 +15,7 @@ #include "WorkflowBase.h" #include "DependenciesFlow.h" #include "PromptFlow.h" +#include "SourceFlow.h" #include #include #include @@ -531,6 +532,7 @@ namespace AppInstaller::CLI::Workflow Workflow::ReportExecutionStage(ExecutionStage::PostExecution) << Workflow::ReportARPChanges << Workflow::RecordInstall << + Workflow::ForceInstalledCacheUpdate << Workflow::RemoveInstaller << Workflow::DisplayInstallationNotes; } diff --git a/src/AppInstallerCLICore/Workflows/SourceFlow.cpp b/src/AppInstallerCLICore/Workflows/SourceFlow.cpp index c0a9a6fb6c..6dbc951f87 100644 --- a/src/AppInstallerCLICore/Workflows/SourceFlow.cpp +++ b/src/AppInstallerCLICore/Workflows/SourceFlow.cpp @@ -301,4 +301,10 @@ namespace AppInstaller::CLI::Workflow } } } + + void ForceInstalledCacheUpdate(Execution::Context&) + { + // Creating this object is currently sufficient to mark the cache as needing an update for the next time it is opened. + Repository::Source ignore{ Repository::PredefinedSource::InstalledForceCacheUpdate }; + } } diff --git a/src/AppInstallerCLICore/Workflows/SourceFlow.h b/src/AppInstallerCLICore/Workflows/SourceFlow.h index fb1fee5e0d..f3d651148c 100644 --- a/src/AppInstallerCLICore/Workflows/SourceFlow.h +++ b/src/AppInstallerCLICore/Workflows/SourceFlow.h @@ -77,4 +77,10 @@ namespace AppInstaller::CLI::Workflow // Inputs: SourceList // Outputs: None void ExportSourceList(Execution::Context& context); + + // Forces an update to the cache of installed packages. + // Required Args: None + // Inputs: None + // Outputs: None + void ForceInstalledCacheUpdate(Execution::Context& context); } diff --git a/src/AppInstallerCLIE2ETests/Constants.cs b/src/AppInstallerCLIE2ETests/Constants.cs index aaec17b071..97c49816e8 100644 --- a/src/AppInstallerCLIE2ETests/Constants.cs +++ b/src/AppInstallerCLIE2ETests/Constants.cs @@ -80,6 +80,7 @@ public class Constants public const string PortableExePackageId = "AppInstallerTest.TestPortableExe"; public const string PortableExeWithCommandPackageId = "AppInstallerTest.TestPortableExeWithCommand"; + public const string ExeInstalledDefaultProductCode = "{A499DD5E-8DC5-4AD2-911A-BCD0263295E9}"; public const string MsiInstallerProductCode = "{A5D36CF1-1993-4F63-BFB4-3ACD910D36A1}"; public const string MsixInstallerName = "6c6338fe-41b7-46ca-8ba6-b5ad5312bb0e"; public const string MsixInstallerPackageFamilyName = "6c6338fe-41b7-46ca-8ba6-b5ad5312bb0e_8wekyb3d8bbwe"; diff --git a/src/AppInstallerCLIE2ETests/Interop/InstallInterop.cs b/src/AppInstallerCLIE2ETests/Interop/InstallInterop.cs index 4e3e284e82..72eea928ef 100644 --- a/src/AppInstallerCLIE2ETests/Interop/InstallInterop.cs +++ b/src/AppInstallerCLIE2ETests/Interop/InstallInterop.cs @@ -657,5 +657,72 @@ public void GetApplicableInstaller() Assert.AreEqual(PackageInstallerScope.User, packageInstallerInfo.Scope); Assert.AreEqual("en-US", packageInstallerInfo.Locale); } + + /// + /// Install exe and verify that we can find it as installed after. + /// + /// A representing the asynchronous unit test. + [Test] + public async Task InstallExe_VerifyInstalledCatalog() + { + var installedCatalogReference = this.packageManager.GetLocalPackageCatalog(LocalPackageCatalog.InstalledPackages); + + // Ensure package is not installed + var installedResult = this.FindAllPackages(installedCatalogReference, PackageMatchField.ProductCode, PackageFieldMatchOption.Equals, Constants.ExeInstalledDefaultProductCode); + Assert.IsNotNull(installedResult); + Assert.AreEqual(0, installedResult.Count); + + // Find package + var searchResult = this.FindOnePackage(this.testSource, PackageMatchField.Id, PackageFieldMatchOption.Equals, "AppInstallerTest.TestExeInstaller"); + + // Configure installation + var installOptions = this.TestFactory.CreateInstallOptions(); + installOptions.PackageInstallMode = PackageInstallMode.Silent; + installOptions.PreferredInstallLocation = this.installDir; + installOptions.AcceptPackageAgreements = true; + + // Install + var installResult = await this.packageManager.InstallPackageAsync(searchResult.CatalogPackage, installOptions); + + // Assert + Assert.AreEqual(InstallResultStatus.Ok, installResult.Status); + + // Check installed catalog after + this.FindOnePackage(installedCatalogReference, PackageMatchField.ProductCode, PackageFieldMatchOption.Equals, Constants.ExeInstalledDefaultProductCode); + + Assert.True(TestCommon.VerifyTestExeInstalledAndCleanup(this.installDir)); + } + + /// + /// Install msix and verify that we can find it as installed after. + /// + /// A representing the asynchronous unit test. + [Test] + public async Task InstallMSIX_VerifyInstalledCatalog() + { + var installedCatalogReference = this.packageManager.GetLocalPackageCatalog(LocalPackageCatalog.InstalledPackages); + + // Ensure package is not installed + var installedResult = this.FindAllPackages(installedCatalogReference, PackageMatchField.PackageFamilyName, PackageFieldMatchOption.Equals, Constants.MsixInstallerPackageFamilyName); + Assert.IsNotNull(installedResult); + Assert.AreEqual(0, installedResult.Count); + + // Find package + var searchResult = this.FindOnePackage(this.testSource, PackageMatchField.Name, PackageFieldMatchOption.Equals, "TestMsixInstaller"); + + // Configure installation + var installOptions = this.TestFactory.CreateInstallOptions(); + + // Install + var installResult = await this.packageManager.InstallPackageAsync(searchResult.CatalogPackage, installOptions); + + // Assert + Assert.AreEqual(InstallResultStatus.Ok, installResult.Status); + + // Check installed catalog after + this.FindOnePackage(installedCatalogReference, PackageMatchField.PackageFamilyName, PackageFieldMatchOption.Equals, Constants.MsixInstallerPackageFamilyName); + + Assert.True(TestCommon.VerifyTestMsixInstalledAndCleanup()); + } } } \ No newline at end of file diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj index 0e3db21fbb..c91740b515 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -197,6 +197,7 @@ + diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters index ff2ca53711..d6dc985837 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -317,6 +317,9 @@ Source Files\Common + + Source Files\Repository + diff --git a/src/AppInstallerCLITests/ArpHelper.cpp b/src/AppInstallerCLITests/ArpHelper.cpp new file mode 100644 index 0000000000..fdbfd2cb4c --- /dev/null +++ b/src/AppInstallerCLITests/ArpHelper.cpp @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestCommon.h" +#include "TestHooks.h" +#include "Microsoft/ARPHelper.h" +#include "AppInstallerStrings.h" + +using namespace AppInstaller::Manifest; +using namespace AppInstaller::Repository::Microsoft; +using namespace AppInstaller::Utility; +using namespace AppInstaller::Registry; + + +TEST_CASE("ARPHelper_Watcher", "[ARPHelper]") +{ + ARPHelper helper; + + wil::unique_event callbackEvent; + callbackEvent.create(); + + ScopeEnum scopeCallback = ScopeEnum::Unknown; + Architecture architectureCallback = Architecture::Unknown; + + ScopeEnum scopeTarget = ScopeEnum::Machine; + Architecture architectureTarget = Architecture::X86; + + auto fakeRoot = TestCommon::RegCreateVolatileTestRoot(); + TestHook::SetGetARPKey_Override arpOverride([&](ScopeEnum scope, Architecture arch) + { + if (scope == scopeTarget && arch == architectureTarget) + { + return Key(fakeRoot.get(), L""); + } + else + { + return Key{}; + } + }); + + auto watchers = helper.CreateRegistryWatchers(scopeTarget, [&](ScopeEnum scope, Architecture arch, wil::RegistryChangeKind) + { + scopeCallback = scope; + architectureCallback = arch; + callbackEvent.SetEvent(); + }); + + auto arpKey = helper.GetARPKey(scopeTarget, architectureTarget); + REQUIRE(!!arpKey); + + GUID guid; + std::ignore = CoCreateGuid(&guid); + std::ostringstream stream; + stream << guid; + + auto testKey = TestCommon::RegCreateVolatileSubKey(arpKey, ConvertToUTF16(stream.str())); + + REQUIRE(callbackEvent.wait(1000)); + REQUIRE(scopeTarget == scopeCallback); + REQUIRE(architectureTarget == architectureCallback); + + // Reset for changing a value + scopeCallback = ScopeEnum::Unknown; + architectureCallback = Architecture::Unknown; + + TestCommon::SetRegistryValue(testKey.get(), L"testValue", L"valueValue"); + + REQUIRE(callbackEvent.wait(1000)); + REQUIRE(scopeTarget == scopeCallback); + REQUIRE(architectureTarget == architectureCallback); +} diff --git a/src/AppInstallerCLITests/PredefinedInstalledSource.cpp b/src/AppInstallerCLITests/PredefinedInstalledSource.cpp index cfdd731edb..23efc240bf 100644 --- a/src/AppInstallerCLITests/PredefinedInstalledSource.cpp +++ b/src/AppInstallerCLITests/PredefinedInstalledSource.cpp @@ -7,6 +7,7 @@ #include #include #include +#include using namespace std::string_literals; using namespace std::string_view_literals; @@ -18,6 +19,7 @@ using namespace AppInstaller::Runtime; using namespace AppInstaller::Utility; using SQLiteIndex = AppInstaller::Repository::Microsoft::SQLiteIndex; +using SQLiteIndexSource = AppInstaller::Repository::Microsoft::SQLiteIndexSource; using Factory = AppInstaller::Repository::Microsoft::PredefinedInstalledSourceFactory; using ARPHelper = AppInstaller::Repository::Microsoft::ARPHelper; @@ -400,3 +402,82 @@ TEST_CASE("PredefinedInstalledSource_Search", "[installed][list]") REQUIRE_FALSE(results.Matches.empty()); } + +std::string GetDatabaseIdentifier(const std::shared_ptr& source) +{ + return reinterpret_cast(source->CastTo(ISourceType::SQLiteIndexSource))->GetIndex().GetDatabaseIdentifier(); +} + +void RequirePackagesHaveSameNames(std::shared_ptr& source1, std::shared_ptr& source2) +{ + auto result1 = source1->Search({}); + REQUIRE(!result1.Matches.empty()); + + // Ensure that all packages have the same name values + for (const auto& match : result1.Matches) + { + std::string packageId = match.Package->GetProperty(PackageProperty::Id).get(); + INFO(packageId); + + SearchRequest id2; + id2.Inclusions.emplace_back(PackageMatchFilter{ PackageMatchField::Id, MatchType::CaseInsensitive, packageId }); + auto result2 = source2->Search(id2); + REQUIRE(result2.Matches.size() == 1); + REQUIRE(match.Package->GetProperty(PackageProperty::Name) == result2.Matches[0].Package->GetProperty(PackageProperty::Name)); + } +} + +TEST_CASE("PredefinedInstalledSource_Create_Cached", "[installed][list][installed-cache]") +{ + auto source1 = CreatePredefinedInstalledSource(); + auto source2 = CreatePredefinedInstalledSource(); + + // Ensure the same identifier (which should mean the cache was not updated) + REQUIRE( + GetDatabaseIdentifier(source1) + == + GetDatabaseIdentifier(source2) + ); + + RequirePackagesHaveSameNames(source1, source2); + RequirePackagesHaveSameNames(source2, source1); +} + +TEST_CASE("PredefinedInstalledSource_Create_ForceCacheUpdate", "[installed][list][installed-cache]") +{ + auto source1 = CreatePredefinedInstalledSource(); + auto source2 = CreatePredefinedInstalledSource(Factory::Filter::NoneWithForcedCacheUpdate); + + // Ensure different identifier (which should mean the cache was updated) + REQUIRE( + GetDatabaseIdentifier(source1) + != + GetDatabaseIdentifier(source2) + ); + + RequirePackagesHaveSameNames(source1, source2); + RequirePackagesHaveSameNames(source2, source1); +} + +TEST_CASE("PredefinedInstalledSource_Create_ForceCacheUpdate_StillCached", "[installed][list][installed-cache]") +{ + auto source1 = CreatePredefinedInstalledSource(); + auto source2 = CreatePredefinedInstalledSource(Factory::Filter::NoneWithForcedCacheUpdate); + auto source3 = CreatePredefinedInstalledSource(); + + CAPTURE(GetDatabaseIdentifier(source1), GetDatabaseIdentifier(source2), GetDatabaseIdentifier(source3)); + + // Ensure different identifier (which should mean the cache was updated) + REQUIRE( + GetDatabaseIdentifier(source1) + != + GetDatabaseIdentifier(source2) + ); + + // Ensure the same identifier (which should mean the cache was not updated) + REQUIRE( + GetDatabaseIdentifier(source2) + == + GetDatabaseIdentifier(source3) + ); +} diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp index 9884fbe33f..b8041c3abc 100644 --- a/src/AppInstallerCLITests/SQLiteIndex.cpp +++ b/src/AppInstallerCLITests/SQLiteIndex.cpp @@ -321,7 +321,14 @@ bool AreArpVersionsSupported(const SQLiteIndex& index, const Schema::Version& te return (index.GetVersion() >= Schema::Version{ 1, 5 } && testVersion >= Schema::Version{ 1, 5 }); } -std::string GetPropertyStringByKey(const SQLiteIndex& index, SQLite::rowid_t id, PackageVersionProperty property, std::string_view version, std::string_view channel) +std::string GetPropertyStringByKey(const SQLiteIndex& index, rowid_t manifestId, PackageVersionProperty property) +{ + auto result = index.GetPropertyByManifestId(manifestId, property); + REQUIRE(result); + return result.value(); +} + +std::string GetPropertyStringByKey(const SQLiteIndex& index, rowid_t id, PackageVersionProperty property, std::string_view version, std::string_view channel) { auto manifestId = index.GetManifestIdByKey(id, version, channel); REQUIRE(manifestId); @@ -334,7 +341,7 @@ std::string GetPropertyStringById(const SQLiteIndex& index, SQLite::rowid_t id, { auto versions = index.GetVersionKeysById(id); REQUIRE(!versions.empty()); - return GetPropertyStringByKey(index, id, property, versions[0].GetVersion().ToString(), versions[0].GetChannel().ToString()); + return GetPropertyStringByKey(index, versions[0].ManifestId, property); } std::string GetIdStringById(const SQLiteIndex& index, SQLite::rowid_t id) @@ -1821,8 +1828,8 @@ TEST_CASE("SQLiteIndex_Versions", "[sqliteindex]") auto result = index.GetVersionKeysById(results.Matches[0].first); REQUIRE(result.size() == 1); - REQUIRE(result[0].GetVersion().ToString() == manifest.Version); - REQUIRE(result[0].GetChannel().ToString() == manifest.Channel); + REQUIRE(result[0].VersionAndChannel.GetVersion().ToString() == manifest.Version); + REQUIRE(result[0].VersionAndChannel.GetChannel().ToString() == manifest.Channel); } TEST_CASE("SQLiteIndex_Search_VersionSorting", "[sqliteindex]") @@ -1867,7 +1874,7 @@ TEST_CASE("SQLiteIndex_Search_VersionSorting", "[sqliteindex]") for (size_t i = 0; i < result.size(); ++i) { const VersionAndChannel& sortedVAC = sortedList[i]; - const VersionAndChannel& resultVAC = result[i]; + const VersionAndChannel& resultVAC = result[i].VersionAndChannel; INFO(i); REQUIRE(sortedVAC.GetVersion().ToString() == resultVAC.GetVersion().ToString()); @@ -1955,7 +1962,7 @@ TEST_CASE("SQLiteIndex_PathString_CaseInsensitive", "[sqliteindex]") REQUIRE(result.has_value()); result = index.GetManifestIdByKey(results.Matches[0].first, "13.2.0-BugFix", "BETA"); - REQUIRE(!result.has_value()); + REQUIRE(result.has_value()); } TEST_CASE("SQLiteIndex_SearchResultsTableSearches", "[sqliteindex][V1_0]") diff --git a/src/AppInstallerCLITests/TestHooks.h b/src/AppInstallerCLITests/TestHooks.h index b66aa7a507..5b003274cf 100644 --- a/src/AppInstallerCLITests/TestHooks.h +++ b/src/AppInstallerCLITests/TestHooks.h @@ -40,6 +40,9 @@ namespace AppInstaller namespace Repository::Microsoft { void TestHook_SetPinningIndex_Override(std::optional&& indexPath); + + using GetARPKeyFunc = std::function; + void SetGetARPKeyOverride(GetARPKeyFunc value); } namespace Logging @@ -228,4 +231,19 @@ namespace TestHook private: std::vector m_extractedIcons; }; + + struct SetGetARPKey_Override + { + SetGetARPKey_Override(std::function function) + { + AppInstaller::Repository::Microsoft::SetGetARPKeyOverride(function); + } + + ~SetGetARPKey_Override() + { + AppInstaller::Repository::Microsoft::SetGetARPKeyOverride({}); + } + + private: + }; } \ No newline at end of file diff --git a/src/AppInstallerCommonCore/Regex.cpp b/src/AppInstallerCommonCore/Regex.cpp index 74d4a0d12b..934be59b39 100644 --- a/src/AppInstallerCommonCore/Regex.cpp +++ b/src/AppInstallerCommonCore/Regex.cpp @@ -4,6 +4,7 @@ #include "Public/winget/Regex.h" #include "Public/AppInstallerErrors.h" #include "Public/AppInstallerLogging.h" +#include "Public/AppInstallerLanguageUtilities.h" #define WINGET_THROW_REGEX_ERROR_IF_FAILED(_err_,_func_) \ if (U_FAILURE(_err_)) \ @@ -20,6 +21,69 @@ namespace AppInstaller::Regex using uregex_ptr = wil::unique_any; using utext_ptr = wil::unique_any; + // Create caches the original ICU regex objects in a static map and hands out copies of them + // when requested. Since we have a limited set, this is a very simple cache-all-forever pattern. + static std::unique_ptr Create(std::string_view pattern, Options options) + { + struct key + { + std::string pattern; + Options options = Options::None; + + bool operator<(const key& other) const + { + if (pattern < other.pattern) + { + return true; + } + else if (pattern == other.pattern) + { + return ToIntegral(options) < ToIntegral(other.options); + } + else + { + return false; + } + } + }; + + struct statics + { + std::map map; + wil::srwlock lock; + }; + + static statics s_regex_cache; + + key requested; + requested.pattern = pattern; + requested.options = options; + + { + // Attempt to find in the cache + auto sharedLock = s_regex_cache.lock.lock_shared(); + + auto itr = s_regex_cache.map.find(requested); + if (itr != s_regex_cache.map.end()) + { + return std::make_unique(itr->second); + } + } + + auto exclusiveLock = s_regex_cache.lock.lock_exclusive(); + + // Check if another thread created it while we waited for the lock. + auto itr = s_regex_cache.map.find(requested); + if (itr != s_regex_cache.map.end()) + { + return std::make_unique(itr->second); + } + else + { + return std::make_unique(s_regex_cache.map.emplace(std::move(requested), impl{ pattern, options }).first->second); + } + } + impl(std::string_view pattern, Options options) { UErrorCode uec = U_ZERO_ERROR; @@ -167,7 +231,7 @@ namespace AppInstaller::Regex Expression::Expression() = default; - Expression::Expression(std::string_view pattern, Options options) : pImpl(std::make_unique(pattern, options)) {} + Expression::Expression(std::string_view pattern, Options options) : pImpl(impl::Create(pattern, options)) {} Expression::Expression(const Expression& other) { diff --git a/src/AppInstallerRepositoryCore/ArpVersionValidation.cpp b/src/AppInstallerRepositoryCore/ArpVersionValidation.cpp index 8f567750de..cbff934dd7 100644 --- a/src/AppInstallerRepositoryCore/ArpVersionValidation.cpp +++ b/src/AppInstallerRepositoryCore/ArpVersionValidation.cpp @@ -1,49 +1,45 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#pragma once -#include "pch.h" -#include "ArpVersionValidation.h" -#include - -namespace AppInstaller::Repository -{ - namespace - { - std::vector GetArpVersionRangesByPackageRowId(const Microsoft::SQLiteIndex* index, Microsoft::SQLiteIndex::IdType packageRowId, const Utility::VersionAndChannel& excludeVersionAndChannel = {}) - { - std::vector result; - - auto versionKeys = index->GetVersionKeysById(packageRowId); - for (auto const& versionKey : versionKeys) - { - // For manifest update, the manifest to be updated does not need to be checked. - // In unlikely cases if both version 1.0.0 and 1.0 of the same package exist, we compare raw values here as what sqlite index does. - if (versionKey.GetVersion().ToString() == excludeVersionAndChannel.GetVersion().ToString() && - versionKey.GetChannel().ToString() == excludeVersionAndChannel.GetChannel().ToString()) - { - continue; - } - - std::optional manifestRowId = index->GetManifestIdByKey(packageRowId, versionKey.GetVersion().ToString(), versionKey.GetChannel().ToString()); - if (manifestRowId) +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "pch.h" +#include "ArpVersionValidation.h" +#include + +namespace AppInstaller::Repository +{ + namespace + { + std::vector GetArpVersionRangesByPackageRowId(const Microsoft::SQLiteIndex* index, Microsoft::SQLiteIndex::IdType packageRowId, const Utility::VersionAndChannel& excludeVersionAndChannel = {}) + { + std::vector result; + + auto versionKeys = index->GetVersionKeysById(packageRowId); + for (auto const& versionKey : versionKeys) + { + // For manifest update, the manifest to be updated does not need to be checked. + // In unlikely cases if both version 1.0.0 and 1.0 of the same package exist, we compare raw values here as what sqlite index does. + if (versionKey.VersionAndChannel.GetVersion().ToString() == excludeVersionAndChannel.GetVersion().ToString() && + versionKey.VersionAndChannel.GetChannel().ToString() == excludeVersionAndChannel.GetChannel().ToString()) { - auto arpMinVersion = index->GetPropertyByManifestId(manifestRowId.value(), PackageVersionProperty::ArpMinVersion).value_or(""); - auto arpMaxVersion = index->GetPropertyByManifestId(manifestRowId.value(), PackageVersionProperty::ArpMaxVersion).value_or(""); + continue; + } + + auto arpMinVersion = index->GetPropertyByManifestId(versionKey.ManifestId, PackageVersionProperty::ArpMinVersion).value_or(""); + auto arpMaxVersion = index->GetPropertyByManifestId(versionKey.ManifestId, PackageVersionProperty::ArpMaxVersion).value_or(""); - // Either both empty or both not empty - THROW_HR_IF(E_UNEXPECTED, arpMinVersion.empty() != arpMaxVersion.empty()); + // Either both empty or both not empty + THROW_HR_IF(E_UNEXPECTED, arpMinVersion.empty() != arpMaxVersion.empty()); + + if (!arpMinVersion.empty() && !arpMaxVersion.empty()) + { + result.emplace_back(Utility::VersionRange{ Utility::Version{ std::move(arpMinVersion) }, Utility::Version{ std::move(arpMaxVersion) } }); + } + } + + return result; + } + } - if (!arpMinVersion.empty() && !arpMaxVersion.empty()) - { - result.emplace_back(Utility::VersionRange{ Utility::Version{ std::move(arpMinVersion) }, Utility::Version{ std::move(arpMaxVersion) } }); - } - } - } - - return result; - } - } - void ValidateManifestArpVersion(const Microsoft::SQLiteIndex* index, const Manifest::Manifest& manifest) { try @@ -54,14 +50,14 @@ namespace AppInstaller::Repository return; } - SearchRequest request; - request.Filters.emplace_back(PackageMatchField::Id, MatchType::CaseInsensitive, manifest.Id); - auto searchResult = index->Search(request); - if (searchResult.Matches.empty()) - { - return; - } - + SearchRequest request; + request.Filters.emplace_back(PackageMatchField::Id, MatchType::CaseInsensitive, manifest.Id); + auto searchResult = index->Search(request); + if (searchResult.Matches.empty()) + { + return; + } + auto arpVersionRangesInIndex = GetArpVersionRangesByPackageRowId(index, searchResult.Matches[0].first, { Utility::Version{ manifest.Version }, Utility::Channel{ manifest.Channel } }); for (auto const& arpInIndex : arpVersionRangesInIndex) { @@ -89,5 +85,5 @@ namespace AppInstaller::Repository { Manifest::ValidationError(Manifest::ManifestError::ArpVersionValidationInternalError) }, APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED)); } - } + } } \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.cpp b/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.cpp index 26a60f56a1..5bb9ef32ed 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.cpp @@ -108,15 +108,32 @@ namespace AppInstaller::Repository::Microsoft CATCH_LOG_MSG("Failed to read upgrade code: %hs", keyName.c_str()); } } - } + } CATCH_LOG_MSG("Failed to read upgrade codes."); return upgradeCodes; } } +#ifndef AICLI_DISABLE_TEST_HOOKS + using GetARPKeyFunc = std::function; + static GetARPKeyFunc s_GetARPKey_Override; + + void SetGetARPKeyOverride(GetARPKeyFunc value) + { + s_GetARPKey_Override = value; + } +#endif + Registry::Key ARPHelper::GetARPKey(Manifest::ScopeEnum scope, Utility::Architecture architecture) const { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_GetARPKey_Override) + { + return s_GetARPKey_Override(scope, architecture); + } +#endif + HKEY rootKey = NULL; switch (scope) @@ -368,7 +385,7 @@ namespace AppInstaller::Repository::Microsoft void ARPHelper::PopulateIndexFromKey(SQLiteIndex& index, const Registry::Key& key, std::string_view scope, std::string_view architecture, const std::map& upgradeCodes) const { - AICLI_LOG(Repo, Info, << "Examining ARP entries for " << scope << " | " << architecture); + AICLI_LOG(Repo, Verbose, << "Examining ARP entries for " << scope << " | " << architecture); for (const auto& arpEntry : key) { @@ -532,4 +549,34 @@ namespace AppInstaller::Repository::Microsoft } } } + + std::vector ARPHelper::CreateRegistryWatchers(Manifest::ScopeEnum scope, std::function callback) + { + std::vector result; + + auto addToResult = [&](Manifest::ScopeEnum scopeToUse) + { + for (auto architecture : Utility::GetApplicableArchitectures()) + { + Registry::Key arpRootKey = GetARPKey(scopeToUse, architecture); + + if (arpRootKey) + { + result.emplace_back(wil::make_registry_watcher(arpRootKey, L"", true, [scopeToUse, architecture, callback](wil::RegistryChangeKind change) { callback(scopeToUse, architecture, change); })); + } + } + }; + + if (scope == Manifest::ScopeEnum::Unknown) + { + addToResult(Manifest::ScopeEnum::User); + addToResult(Manifest::ScopeEnum::Machine); + } + else + { + addToResult(scope); + } + + return result; + } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.h b/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.h index 3c0b4ac77b..e0afa19924 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.h +++ b/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.h @@ -5,9 +5,11 @@ #include #include #include +#include #include #include +#include namespace AppInstaller::Repository::Microsoft { @@ -86,5 +88,8 @@ namespace AppInstaller::Repository::Microsoft // This entry point is primarily to allow unit tests to operate of arbitrary keys; // product code should use PopulateIndexFromARP. void PopulateIndexFromKey(SQLiteIndex& index, const Registry::Key& key, std::string_view scope, std::string_view architecture, const std::map& upgradeCodes = {}) const; + + // Creates registry watchers for the given scope + std::vector CreateRegistryWatchers(Manifest::ScopeEnum scope, std::function callback); }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.cpp index 878d60aa6e..dc6ac46c95 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.cpp @@ -17,13 +17,54 @@ namespace AppInstaller::Repository::Microsoft { namespace { + std::optional GetCachedMSIXName(const Utility::NormalizedString& id, const Utility::Version& version, SQLiteIndex& cacheData) + { + SearchRequest searchRequest; + searchRequest.Inclusions.emplace_back(PackageMatchField::Id, MatchType::Exact, id); + + SQLiteIndex::SearchResult searchResult = cacheData.Search(searchRequest); + + if (searchResult.Matches.empty()) + { + return std::nullopt; + } + + if (searchResult.Matches.size() != 1) + { + // This is very unexpected, but just log it and carry on + AICLI_LOG(Repo, Warning, << "Found multiple (" << searchResult.Matches.size() << ") cache entries for: " << id); + return std::nullopt; + } + + auto versionKeys = cacheData.GetVersionKeysById(searchResult.Matches[0].first); + const SQLiteIndex::VersionKey* versionKey = nullptr; + + for (const auto& key : versionKeys) + { + if (key.VersionAndChannel.GetVersion() == version) + { + versionKey = &key; + break; + } + } + + if (!versionKey) + { + return std::nullopt; + } + + return cacheData.GetPropertyByManifestId(versionKey->ManifestId, PackageVersionProperty::Name); + } + // Populates the index with the entries from MSIX. - void PopulateIndexFromMSIX(SQLiteIndex& index, Manifest::ScopeEnum scope) + void PopulateIndexFromMSIX(SQLiteIndex& index, Manifest::ScopeEnum scope, SQLiteIndex* cacheData = nullptr) { using namespace winrt::Windows::ApplicationModel; using namespace winrt::Windows::Management::Deployment; using namespace winrt::Windows::Foundation::Collections; + AICLI_LOG(Repo, Verbose, << "Examining MSIX entries for " << ScopeToString(scope)); + IIterable packages; PackageManager packageManager; if (scope == Manifest::ScopeEnum::Machine) @@ -63,10 +104,31 @@ namespace AppInstaller::Repository::Microsoft manifest.Id = familyName; + // Get version + std::ostringstream strstr; + auto packageVersion = packageId.Version(); + strstr << packageVersion.Major << '.' << packageVersion.Minor << '.' << packageVersion.Build << '.' << packageVersion.Revision; + + manifest.Version = strstr.str(); + + // Determine package name bool isPackageNameSet = false; + + // Look for the name in the cache data first + if (cacheData) + { + std::optional cachedName = GetCachedMSIXName(manifest.Id, manifest.Version, *cacheData); + + if (cachedName) + { + manifest.DefaultLocalization.Add(cachedName.value()); + isPackageNameSet = true; + } + } + // Attempt to get the DisplayName. Since this will retrieve the localized value, it has a chance to fail. // Rather than completely skip this package in that case, we will simply fall back to using the package name below. - if (!Runtime::IsRunningAsSystem()) + if (!isPackageNameSet && !Runtime::IsRunningAsSystem()) { try { @@ -79,12 +141,12 @@ namespace AppInstaller::Repository::Microsoft } catch (const winrt::hresult_error& hre) { - AICLI_LOG(Repo, Info, << "winrt::hresult_error[0x" << Logging::SetHRFormat << hre.code() << ": " << + AICLI_LOG(Repo, Warning, << "winrt::hresult_error[0x" << Logging::SetHRFormat << hre.code() << ": " << Utility::ConvertToUTF8(hre.message()) << "] exception thrown when getting DisplayName for " << familyName); } catch (...) { - AICLI_LOG(Repo, Info, << "Unknown exception thrown when getting DisplayName for " << familyName); + AICLI_LOG(Repo, Warning, << "Unknown exception thrown when getting DisplayName for " << familyName); } } @@ -93,12 +155,6 @@ namespace AppInstaller::Repository::Microsoft manifest.DefaultLocalization.Add(Utility::ConvertToUTF8(packageId.Name())); } - std::ostringstream strstr; - auto packageVersion = packageId.Version(); - strstr << packageVersion.Major << '.' << packageVersion.Minor << '.' << packageVersion.Build << '.' << packageVersion.Revision; - - manifest.Version = strstr.str(); - manifest.Installers[0].PackageFamilyName = familyName; // Use the full name as a unique key for the path @@ -109,11 +165,126 @@ namespace AppInstaller::Repository::Microsoft } } + SQLiteIndex CreateAndPopulateIndex(PredefinedInstalledSourceFactory::Filter filter) + { + AICLI_LOG(Repo, Verbose, << "Creating PredefinedInstalledSource with filter [" << PredefinedInstalledSourceFactory::FilterToString(filter) << ']'); + + // Create an in memory index + SQLiteIndex index = SQLiteIndex::CreateNew(SQLITE_MEMORY_DB_CONNECTION_TARGET, Schema::Version::Latest()); + + // Put installed packages into the index + if (filter == PredefinedInstalledSourceFactory::Filter::None || filter == PredefinedInstalledSourceFactory::Filter::ARP || + filter == PredefinedInstalledSourceFactory::Filter::User || filter == PredefinedInstalledSourceFactory::Filter::Machine) + { + ARPHelper arpHelper; + if (filter != PredefinedInstalledSourceFactory::Filter::User) + { + arpHelper.PopulateIndexFromARP(index, Manifest::ScopeEnum::Machine); + } + if (filter != PredefinedInstalledSourceFactory::Filter::Machine) + { + arpHelper.PopulateIndexFromARP(index, Manifest::ScopeEnum::User); + } + } + + if (filter == PredefinedInstalledSourceFactory::Filter::None || + filter == PredefinedInstalledSourceFactory::Filter::MSIX || + filter == PredefinedInstalledSourceFactory::Filter::User) + { + PopulateIndexFromMSIX(index, Manifest::ScopeEnum::User); + } + else if (filter == PredefinedInstalledSourceFactory::Filter::Machine) + { + PopulateIndexFromMSIX(index, Manifest::ScopeEnum::Machine); + } + + AICLI_LOG(Repo, Verbose, << " ... finished creating PredefinedInstalledSource"); + + return index; + } + + struct CachedInstalledIndex + { + CachedInstalledIndex() + { + ARPHelper arpHelper; + m_registryWatchers = arpHelper.CreateRegistryWatchers(Manifest::ScopeEnum::Unknown, + [this](Manifest::ScopeEnum, Utility::Architecture, wil::RegistryChangeKind) { ForceNextUpdate(); }); + + m_catalog = winrt::Windows::ApplicationModel::PackageCatalog::OpenForCurrentUser(); + m_eventRevoker = m_catalog.PackageStatusChanged(winrt::auto_revoke, [this](auto...) { ForceNextUpdate(); }); + } + + void UpdateIndexIfNeeded() + { + auto sharedLock = m_lock.lock_shared(); + if (CheckForUpdate()) + { + // Upgrade to exclusive + sharedLock.reset(); + auto exclusiveLock = m_lock.lock_exclusive(); + + if (CheckForUpdate()) + { + // TODO: To support servicing, the initial implementation of update will simply leverage + // some data from the existing index to speed up the MSIX populate function. + // In a larger update, we may want to make it possible to actually update the cache directly. + // We may even persist the cache to disk to speed things up further. + + // Set the update indicator to false before we start reading so that an external change can + // reindicate a need to update in the middle. But in the event that we error here, set it back to true + // to prevent an error from blocking further attempts. + m_forceNextUpdate = false; + auto scopeExit = wil::scope_exit([&]() { m_forceNextUpdate = true; }); + + // Populate from ARP using standard mechanism. + SQLiteIndex update = CreateAndPopulateIndex(PredefinedInstalledSourceFactory::Filter::ARP); + + // Populate from MSIX, using localization data from the existing index if applicable. + PopulateIndexFromMSIX(update, Manifest::ScopeEnum::User, m_index.get()); + + m_index = std::make_unique(std::move(update)); + scopeExit.release(); + } + } + } + + SQLiteIndex GetCopy() + { + auto lock = m_lock.lock_shared(); + THROW_HR_IF(E_POINTER, !m_index); + return SQLiteIndex::CopyFrom(SQLITE_MEMORY_DB_CONNECTION_TARGET, *m_index); + } + + void ForceNextUpdate() + { + m_forceNextUpdate = true; + } + + private: + bool CheckForUpdate() + { + return (!m_index || m_forceNextUpdate.load()); + } + + wil::srwlock m_lock; + std::atomic_bool m_forceNextUpdate{ false }; + std::unique_ptr m_index; + std::vector m_registryWatchers; + winrt::Windows::ApplicationModel::PackageCatalog m_catalog = nullptr; + decltype(winrt::Windows::ApplicationModel::PackageCatalog{ nullptr }.PackageStatusChanged(winrt::auto_revoke, nullptr)) m_eventRevoker; + }; + struct PredefinedInstalledSourceReference : public ISourceReference { PredefinedInstalledSourceReference(const SourceDetails& details) : m_details(details) { m_details.Identifier = "*PredefinedInstalledSource"; + + if (PredefinedInstalledSourceFactory::StringToFilter(m_details.Arg) == PredefinedInstalledSourceFactory::Filter::NoneWithForcedCacheUpdate) + { + GetCachedInstalledIndex().ForceNextUpdate(); + } } std::string GetIdentifier() override { return m_details.Identifier; } @@ -127,41 +298,27 @@ namespace AppInstaller::Repository::Microsoft // Determine the filter PredefinedInstalledSourceFactory::Filter filter = PredefinedInstalledSourceFactory::StringToFilter(m_details.Arg); - AICLI_LOG(Repo, Info, << "Creating PredefinedInstalledSource with filter [" << PredefinedInstalledSourceFactory::FilterToString(filter) << ']'); - - // Create an in memory index - SQLiteIndex index = SQLiteIndex::CreateNew(SQLITE_MEMORY_DB_CONNECTION_TARGET, Schema::Version::Latest()); - // Put installed packages into the index - if (filter == PredefinedInstalledSourceFactory::Filter::None || filter == PredefinedInstalledSourceFactory::Filter::ARP || - filter == PredefinedInstalledSourceFactory::Filter::User || filter == PredefinedInstalledSourceFactory::Filter::Machine) + // Only cache for the unfiltered install data + if (filter == PredefinedInstalledSourceFactory::Filter::None || filter == PredefinedInstalledSourceFactory::Filter::NoneWithForcedCacheUpdate) { - ARPHelper arpHelper; - if (filter != PredefinedInstalledSourceFactory::Filter::User) - { - arpHelper.PopulateIndexFromARP(index, Manifest::ScopeEnum::Machine); - } - if (filter != PredefinedInstalledSourceFactory::Filter::Machine) - { - arpHelper.PopulateIndexFromARP(index, Manifest::ScopeEnum::User); - } - } - - if (filter == PredefinedInstalledSourceFactory::Filter::None || - filter == PredefinedInstalledSourceFactory::Filter::MSIX || - filter == PredefinedInstalledSourceFactory::Filter::User) - { - PopulateIndexFromMSIX(index, Manifest::ScopeEnum::User); + CachedInstalledIndex& cachedIndex = GetCachedInstalledIndex(); + cachedIndex.UpdateIndexIfNeeded(); + return std::make_shared(m_details, cachedIndex.GetCopy(), true); } - else if (filter == PredefinedInstalledSourceFactory::Filter::Machine) + else { - PopulateIndexFromMSIX(index, Manifest::ScopeEnum::Machine); + return std::make_shared(m_details, CreateAndPopulateIndex(filter), true); } - - return std::make_shared(m_details, std::move(index), true); } private: + CachedInstalledIndex& GetCachedInstalledIndex() + { + static CachedInstalledIndex s_installedIndex; + return s_installedIndex; + } + SourceDetails m_details; }; @@ -214,6 +371,8 @@ namespace AppInstaller::Repository::Microsoft return "User"sv; case AppInstaller::Repository::Microsoft::PredefinedInstalledSourceFactory::Filter::Machine: return "Machine"sv; + case AppInstaller::Repository::Microsoft::PredefinedInstalledSourceFactory::Filter::NoneWithForcedCacheUpdate: + return "NoneWithForcedCacheUpdate"sv; default: return "Unknown"sv; } @@ -237,6 +396,10 @@ namespace AppInstaller::Repository::Microsoft { return Filter::Machine; } + else if (filter == FilterToString(Filter::NoneWithForcedCacheUpdate)) + { + return Filter::NoneWithForcedCacheUpdate; + } else { return Filter::None; diff --git a/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.h b/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.h index 3a6bcc216f..8c10648be2 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.h @@ -34,6 +34,8 @@ namespace AppInstaller::Repository::Microsoft User, // Contains machine ARP and machine MSIX Machine, + // Same as None but creating the source reference causes the next Open to always update the cache + NoneWithForcedCacheUpdate, }; // Converts a filter to its string. diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp index 1a43480f4d..6836076460 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -35,6 +35,16 @@ namespace AppInstaller::Repository::Microsoft return result; } + SQLiteIndex SQLiteIndex::Open(const std::string& filePath, OpenDisposition disposition, Utility::ManagedFile&& indexFile) + { + return { filePath, disposition, std::move(indexFile) }; + } + + SQLiteIndex SQLiteIndex::CopyFrom(const std::string& filePath, SQLiteIndex& source) + { + return { filePath, source }; + } + std::unique_ptr SQLiteIndex::CreateISQLiteIndex() const { using namespace Schema; @@ -90,6 +100,13 @@ namespace AppInstaller::Repository::Microsoft THROW_HR_IF(APPINSTALLER_CLI_ERROR_CANNOT_WRITE_TO_UPLEVEL_INDEX, disposition == SQLiteStorageBase::OpenDisposition::ReadWrite && m_version != m_interface->GetVersion()); } + SQLiteIndex::SQLiteIndex(const std::string& target, SQLiteIndex& source) : + SQLiteStorageBase(target, source) + { + m_dbconn.EnableICU(); + m_interface = CreateISQLiteIndex(); + } + #ifndef AICLI_DISABLE_TEST_HOOKS void SQLiteIndex::ForceVersion(const Schema::Version& version) { @@ -257,7 +274,7 @@ namespace AppInstaller::Repository::Microsoft return m_interface->GetManifestIdByManifest(m_dbconn, manifest); } - std::vector SQLiteIndex::GetVersionKeysById(IdType id) const + std::vector SQLiteIndex::GetVersionKeysById(IdType id) const { std::lock_guard lockInterface{ *m_interfaceLock }; return m_interface->GetVersionKeysById(m_dbconn, id); diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h index f8201adc7e..8e64c5474a 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -37,6 +37,9 @@ namespace AppInstaller::Repository::Microsoft // Options for creating a new index. using CreateOptions = Schema::ISQLiteIndex::CreateOptions; + // The type of version keys. + using VersionKey = Schema::ISQLiteIndex::VersionKey; + SQLiteIndex(const SQLiteIndex&) = delete; SQLiteIndex& operator=(const SQLiteIndex&) = delete; @@ -47,10 +50,10 @@ namespace AppInstaller::Repository::Microsoft static SQLiteIndex CreateNew(const std::string& filePath, Schema::Version version = Schema::Version::Latest(), CreateOptions options = CreateOptions::None); // Opens an existing SQLiteIndex database. - static SQLiteIndex Open(const std::string& filePath, OpenDisposition disposition, Utility::ManagedFile&& indexFile = {}) - { - return { filePath, disposition, std::move(indexFile) }; - } + static SQLiteIndex Open(const std::string& filePath, OpenDisposition disposition, Utility::ManagedFile&& indexFile = {}); + + // Creates a copy of the given index. + static SQLiteIndex CopyFrom(const std::string& filePath, SQLiteIndex& source); #ifndef AICLI_DISABLE_TEST_HOOKS // Changes the version of the interface being used to operate on the database. @@ -121,7 +124,7 @@ namespace AppInstaller::Repository::Microsoft std::optional GetManifestIdByManifest(const Manifest::Manifest& manifest) const; // Gets all versions and channels for the given id. - std::vector GetVersionKeysById(IdType id) const; + std::vector GetVersionKeysById(IdType id) const; // Gets the string for the given metadata and manifest id, if present. MetadataResult GetMetadataByManifestId(SQLite::rowid_t manifestId) const; @@ -136,6 +139,7 @@ namespace AppInstaller::Repository::Microsoft // Get all the dependencies for a specific manifest. std::set> GetDependenciesByManifestRowId(SQLite::rowid_t manifestRowId) const; std::vector> GetDependentsById(AppInstaller::Manifest::string_t packageId) const; + private: // Constructor used to create a new index. SQLiteIndex(const std::string& target, Schema::Version version); @@ -143,6 +147,9 @@ namespace AppInstaller::Repository::Microsoft // Constructor used to open an existing index. SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile); + // Constructor used to copy the given index. + SQLiteIndex(const std::string& target, SQLiteIndex& source); + // Internal functions to normalize on the relativePath being present. IdType AddManifestInternal(const Manifest::Manifest& manifest, const std::optional& relativePath); bool UpdateManifestInternal(const Manifest::Manifest& manifest, const std::optional& relativePath); diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp index b8c6b4dcbe..013e331637 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp @@ -288,14 +288,34 @@ namespace AppInstaller::Repository::Microsoft std::vector GetAvailableVersionKeys(PinBehavior) const override { std::shared_ptr source = GetReferenceSource(); - std::vector versions = source->GetIndex().GetVersionKeysById(m_idId); - std::vector result; - for (const auto& vac : versions) { - result.emplace_back(source->GetIdentifier(), vac.GetVersion().ToString(), vac.GetChannel().ToString()); + auto sharedLock = m_availableVersionKeysLock.lock_shared(); + + if (!m_availableVersionKeys.empty()) + { + return m_availableVersionKeys; + } } - return result; + + auto exclusiveLock = m_availableVersionKeysLock.lock_exclusive(); + + if (!m_availableVersionKeys.empty()) + { + return m_availableVersionKeys; + } + + std::vector versions = source->GetIndex().GetVersionKeysById(m_idId); + + for (const auto& vk : versions) + { + std::string version = vk.VersionAndChannel.GetVersion().ToString(); + std::string channel = vk.VersionAndChannel.GetChannel().ToString(); + m_availableVersionKeys.emplace_back(source->GetIdentifier(), version, channel); + m_availableVersionKeysMap.emplace(MapKey{ std::move(version), std::move(channel) }, vk.ManifestId); + } + + return m_availableVersionKeys; } std::shared_ptr GetLatestAvailableVersion(PinBehavior) const override @@ -313,7 +333,23 @@ namespace AppInstaller::Repository::Microsoft return {}; } - std::optional manifestId = source->GetIndex().GetManifestIdByKey(m_idId, versionKey.Version, versionKey.Channel); + std::optional manifestId; + + { + MapKey requested{ versionKey.Version, versionKey.Channel }; + auto sharedLock = m_availableVersionKeysLock.lock_shared(); + + auto itr = m_availableVersionKeysMap.find(requested); + if (itr != m_availableVersionKeysMap.end()) + { + manifestId = itr->second; + } + } + + if (!manifestId) + { + manifestId = source->GetIndex().GetManifestIdByKey(m_idId, versionKey.Version, versionKey.Channel); + } if (manifestId) { @@ -349,6 +385,35 @@ namespace AppInstaller::Repository::Microsoft return nullptr; } + + private: + // Contains the information needed to map a version key to it's rows. + struct MapKey + { + Utility::NormalizedString Version; + Utility::NormalizedString Channel; + + bool operator<(const MapKey& other) const + { + if (Version < other.Version) + { + return true; + } + else if (Version == other.Version) + { + return Channel < other.Channel; + } + else + { + return false; + } + } + }; + + // To avoid removing const from the interface + mutable wil::srwlock m_availableVersionKeysLock; + mutable std::vector m_availableVersionKeys; + mutable std::map m_availableVersionKeysMap; }; // The IPackage impl for SQLiteIndexSource of Installed packages. diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp index e2e3fc7804..692753aac7 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp @@ -39,8 +39,13 @@ namespace AppInstaller::Repository::Microsoft return Utility::ConvertUnixEpochToSystemClock(lastWriteTime); } - SQLiteStorageBase::SQLiteStorageBase(const std::string& filePath, OpenDisposition disposition, Utility::ManagedFile&& indexFile) : - m_indexFile(std::move(indexFile)) + std::string SQLiteStorageBase::GetDatabaseIdentifier() + { + return Schema::MetadataTable::TryGetNamedValue(m_dbconn, Schema::s_MetadataValueName_DatabaseIdentifier).value_or(std::string{}); + } + + SQLiteStorageBase::SQLiteStorageBase(const std::string& filePath, OpenDisposition disposition, Utility::ManagedFile&& file) : + m_indexFile(std::move(file)) { AICLI_LOG(Repo, Info, << "Opening SQLite Index for " << GetOpenDispositionString(disposition) << " at '" << filePath << "'"); switch (disposition) @@ -110,5 +115,21 @@ namespace AppInstaller::Repository::Microsoft { m_version = version; Schema::MetadataTable::Create(m_dbconn); + + // Write a new identifier for this database + GUID databaseIdentifier; + THROW_IF_FAILED(CoCreateGuid(&databaseIdentifier)); + std::ostringstream stream; + stream << databaseIdentifier; + Schema::MetadataTable::SetNamedValue(m_dbconn, Schema::s_MetadataValueName_DatabaseIdentifier, stream.str()); + } + + SQLiteStorageBase::SQLiteStorageBase(const std::string& target, SQLiteStorageBase& source) : + m_dbconn(SQLite::Connection::Create(target, SQLite::Connection::OpenDisposition::Create)), + m_version(source.m_version) + { + std::string mainDatabase = "main"; + SQLite::Backup backup = SQLite::Backup::Create(m_dbconn, mainDatabase, source.m_dbconn, mainDatabase); + backup.Step(); } -} \ No newline at end of file +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h index 26d86c5f40..8c7bd46351 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h @@ -26,7 +26,10 @@ namespace AppInstaller::Repository::Microsoft // Gets the last write time for the index. std::chrono::system_clock::time_point GetLastWriteTime(); - // Gets the schema version of the index. + // Gets the identifier written to the database when it was created. + std::string GetDatabaseIdentifier(); + + // Gets the schema version of the database. Schema::Version GetVersion() const { return m_version; } protected: @@ -34,7 +37,9 @@ namespace AppInstaller::Repository::Microsoft SQLiteStorageBase(const std::string& filePath, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile); - // Sets the last write time metadata value in the index. + SQLiteStorageBase(const std::string& target, SQLiteStorageBase& source); + + // Sets the last write time metadata value in the database. void SetLastWriteTime(); // Gets the corresponding OpenFlags based on the disposition. diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h index 39e57e8e12..b33d26981c 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h @@ -27,7 +27,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 std::vector GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; std::optional GetManifestIdByKey(const SQLite::Connection& connection, SQLite::rowid_t id, std::string_view version, std::string_view channel) const override; std::optional GetManifestIdByManifest(const SQLite::Connection& connection, const Manifest::Manifest& manifest) const override; - std::vector GetVersionKeysById(const SQLite::Connection& connection, SQLite::rowid_t id) const override; + std::vector GetVersionKeysById(const SQLite::Connection& connection, SQLite::rowid_t id) const override; // Version 1.1 MetadataResult GetMetadataByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId) const override; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp index 45f1e941a8..22da18edad 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp @@ -63,53 +63,60 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 std::optional channelIdOpt = ChannelTable::SelectIdByValue(connection, channel, true); if (!channelIdOpt && !channel.empty()) { - // If a non-empty channel was given but none was found, we will just not filter on channel. AICLI_LOG(Repo, Info, << "Did not find a Channel { " << channel << " }"); return {}; } std::optional versionIdOpt; + std::vector> versionStrings; - if (version.empty()) + if (channelIdOpt) { - std::vector versionStrings; - - if (channelIdOpt) - { - versionStrings = ManifestTable::GetAllValuesByIds(connection, { id, channelIdOpt.value() }); - } - else - { - versionStrings = ManifestTable::GetAllValuesByIds(connection, { id }); - } + versionStrings = ManifestTable::GetAllValuesByIds(connection, { id, channelIdOpt.value() }); + } + else + { + versionStrings = ManifestTable::GetAllValuesByIds(connection, { id }); + } - if (versionStrings.empty()) - { - AICLI_LOG(Repo, Info, << "Did not find any Versions { " << id << ", " << channel << " }"); - return {}; - } + if (versionStrings.empty()) + { + AICLI_LOG(Repo, Info, << "Did not find any Versions { " << id << ", " << channel << " }"); + return {}; + } - // Convert the strings to Versions and sort them - std::vector versions; - for (std::string& v : versionStrings) - { - versions.emplace_back(std::move(v)); - } + // Convert the strings to Versions and sort them + struct VersionAndRow + { + SQLite::rowid_t Row = 0; + Utility::Version Version; - std::sort(versions.begin(), versions.end()); + bool operator<(const VersionAndRow& other) const { return Version < other.Version; } + }; + + std::vector versions; + for (auto& v : versionStrings) + { + versions.emplace_back(VersionAndRow{ v.first, std::move(v.second) }); + } - // Get the last version in the list (the highest version) and its rowid - const std::string& latestVersion = versions.back().ToString(); - versionIdOpt = VersionTable::SelectIdByValue(connection, latestVersion); + std::sort(versions.begin(), versions.end()); - if (!versionIdOpt) - { - AICLI_LOG(Repo, Warning, << "Did not find a Version row for the latest version { " << latestVersion << " }"); - } + if (version.empty()) + { + // Get the last version in the list (the highest version) + versionIdOpt = versions.back().Row; } else { - versionIdOpt = VersionTable::SelectIdByValue(connection, version, true); + VersionAndRow requested; + requested.Version = Utility::Version{ std::string(version) }; + + auto itr = std::lower_bound(versions.begin(), versions.end(), requested); + if (itr != versions.end() && itr->Version == requested.Version) + { + versionIdOpt = itr->Row; + } } if (!versionIdOpt) @@ -510,12 +517,6 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 std::optional Interface::GetPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const { - if (!ManifestTable::ExistsById(connection, manifestId)) - { - AICLI_LOG(Repo, Info, << "Did not find manifest by id: " << manifestId); - return {}; - } - return GetPropertyByManifestIdInternal(connection, manifestId, property); } @@ -544,15 +545,15 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return {}; } - std::vector Interface::GetVersionKeysById(const SQLite::Connection& connection, SQLite::rowid_t id) const + std::vector Interface::GetVersionKeysById(const SQLite::Connection& connection, SQLite::rowid_t id) const { auto versionsAndChannels = ManifestTable::GetAllValuesById(connection, id); - std::vector result; + std::vector result; result.reserve(versionsAndChannels.size()); for (auto&& vac : versionsAndChannels) { - result.emplace_back(Utility::Version{ std::move(std::get<0>(vac)) }, Utility::Channel{ std::move(std::get<1>(vac)) }); + result.emplace_back(ISQLiteIndex::VersionKey{ Utility::VersionAndChannel{ Utility::Version{ std::move(std::get<1>(vac)) }, Utility::Channel{ std::move(std::get<2>(vac)) } }, std::get<0>(vac) }); } std::sort(result.begin(), result.end()); @@ -627,13 +628,13 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 switch (property) { case AppInstaller::Repository::PackageVersionProperty::Id: - return std::get<0>(ManifestTable::GetValuesById(connection, manifestId)); + return ManifestTable::GetValueById(connection, manifestId); case AppInstaller::Repository::PackageVersionProperty::Name: - return std::get<0>(ManifestTable::GetValuesById(connection, manifestId)); + return ManifestTable::GetValueById(connection, manifestId); case AppInstaller::Repository::PackageVersionProperty::Version: - return std::get<0>(ManifestTable::GetValuesById(connection, manifestId)); + return ManifestTable::GetValueById(connection, manifestId); case AppInstaller::Repository::PackageVersionProperty::Channel: - return std::get<0>(ManifestTable::GetValuesById(connection, manifestId)); + return ManifestTable::GetValueById(connection, manifestId); case AppInstaller::Repository::PackageVersionProperty::RelativePath: return PathPartTable::GetPathById(connection, std::get<0>(ManifestTable::GetIdsById(connection, manifestId))); default: diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp index cabae1e99a..4c5537c0f2 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp @@ -63,14 +63,18 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 SQLite::Statement ManifestTableGetIdsById_Statement( const SQLite::Connection& connection, SQLite::rowid_t id, - std::initializer_list values) + std::initializer_list values, + bool stepAndVerify) { SQLite::Builder::StatementBuilder builder; builder.Select(values).From(s_ManifestTable_Table_Name).Where(SQLite::RowIDName).Equals(id); SQLite::Statement result = builder.Prepare(connection); - THROW_HR_IF(E_NOT_SET, !result.Step()); + if (stepAndVerify) + { + THROW_HR_IF(E_NOT_SET, !result.Step()); + } return result; } @@ -84,7 +88,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 const SQLite::Connection& connection, SQLite::rowid_t id, std::initializer_list columns, - std::initializer_list manifestColumnNames) + std::initializer_list manifestColumnNames, + bool stepAndVerify) { THROW_HR_IF(E_UNEXPECTED, manifestColumnNames.size() != columns.size()); @@ -108,7 +113,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 SQLite::Statement result = builder.Prepare(connection); - THROW_HR_IF(E_NOT_SET, !result.Step()); + if (stepAndVerify) + { + THROW_HR_IF(E_NOT_SET, !result.Step()); + } return result; } @@ -116,6 +124,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 SQLite::Statement ManifestTableGetAllValuesByIds_Statement( const SQLite::Connection& connection, std::initializer_list valueColumns, + std::initializer_list joinColumns, std::initializer_list idColumns, std::initializer_list ids) { @@ -126,9 +135,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 SQLite::Builder::StatementBuilder builder; builder.Select(valueColumns).From(s_ManifestTable_Table_Name); - for (const auto& valueColumn : valueColumns) + for (const auto& joinColumn : joinColumns) { - builder.Join(valueColumn.Table).On(QCol{ s_ManifestTable_Table_Name, valueColumn.Column }, QCol{ valueColumn.Table, SQLite::RowIDName }); + builder.Join(joinColumn.Table).On(QCol{ s_ManifestTable_Table_Name, joinColumn.Column }, QCol{ joinColumn.Table, SQLite::RowIDName }); } bool isFirst = true; @@ -157,18 +166,19 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return select; } - std::vector ManifestTableGetAllValuesByIds( + std::vector> ManifestTableGetAllValuesByIds( const SQLite::Connection& connection, std::initializer_list valueColumns, + std::initializer_list joinColumns, std::initializer_list idColumns, std::initializer_list ids) { - auto select = ManifestTableGetAllValuesByIds_Statement(connection, valueColumns, idColumns, ids); + auto select = ManifestTableGetAllValuesByIds_Statement(connection, valueColumns, joinColumns, idColumns, ids); - std::vector result; + std::vector> result; while (select.Step()) { - result.emplace_back(select.GetColumn(0)); + result.emplace_back(select.GetColumn(0), select.GetColumn(1)); } return result; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h index 6a4cc06d93..ea9d6016e4 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -37,26 +38,30 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 SQLite::Statement ManifestTableGetIdsById_Statement( const SQLite::Connection& connection, SQLite::rowid_t id, - std::initializer_list values); + std::initializer_list values, + bool stepAndVerify = true); // Gets the requested values for the manifest with the given rowid. SQLite::Statement ManifestTableGetValuesById_Statement( const SQLite::Connection& connection, SQLite::rowid_t id, std::initializer_list columns, - std::initializer_list manifestColumnNames); + std::initializer_list manifestColumnNames, + bool stepAndVerify = true); // Gets all values for rows that match the given ids. SQLite::Statement ManifestTableGetAllValuesByIds_Statement( const SQLite::Connection& connection, std::initializer_list valueColumns, + std::initializer_list joinColumns, std::initializer_list idColumns, std::initializer_list ids); // Gets all values for rows that match the given ids. - std::vector ManifestTableGetAllValuesByIds( + std::vector> ManifestTableGetAllValuesByIds( const SQLite::Connection& connection, std::initializer_list valueColumns, + std::initializer_list joinColumns, std::initializer_list idColumns, std::initializer_list ids); @@ -137,6 +142,15 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return details::ManifestTableGetIdsById_Statement(connection, id, { details::GetManifestTableColumnName()...}).GetRow(); } + // Gets the id requested for the manifest with the given rowid, if it exists. + template + static std::optional GetIdById(const SQLite::Connection& connection, SQLite::rowid_t id) + { + auto statement = details::ManifestTableGetIdsById_Statement(connection, id, { details::GetManifestTableColumnName() }, false); + if (statement.Step()) { return statement.GetColumn(0); } + else { return std::nullopt; } + } + // Gets the values requested for the manifest with the given rowid. template static auto GetValuesById(const SQLite::Connection& connection, SQLite::rowid_t id) @@ -144,22 +158,37 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return details::ManifestTableGetValuesById_Statement(connection, id, { SQLite::Builder::QualifiedColumn{ Tables::TableName(), Tables::ValueName() }... }, { details::GetManifestTableColumnName()... }).GetRow(); } - // Gets the values for rows that match the given ids. + // Gets the value requested for the manifest with the given rowid, if it exists. + template + static std::optional GetValueById(const SQLite::Connection& connection, SQLite::rowid_t id) + { + auto statement = details::ManifestTableGetValuesById_Statement(connection, id, { SQLite::Builder::QualifiedColumn{ Table::TableName(), Table::ValueName() } }, { details::GetManifestTableColumnName
() }, false); + if (statement.Step()) { return statement.GetColumn(0); } + else { return std::nullopt; } + } + + // Gets the row ids and values for rows that match the given ids. template - static std::vector GetAllValuesByIds(const SQLite::Connection& connection, std::initializer_list ids) + static std::vector> GetAllValuesByIds(const SQLite::Connection& connection, std::initializer_list ids) { - return details::ManifestTableGetAllValuesByIds(connection, { SQLite::Builder::QualifiedColumn{ ValueTable::TableName(), ValueTable::ValueName() } }, { IdTables::ValueName()... }, ids); + return details::ManifestTableGetAllValuesByIds(connection, + { SQLite::Builder::QualifiedColumn{ ValueTable::TableName(), SQLite::RowIDName }, SQLite::Builder::QualifiedColumn{ ValueTable::TableName(), ValueTable::ValueName() } }, + { SQLite::Builder::QualifiedColumn{ ValueTable::TableName(), ValueTable::ValueName() } }, + { IdTables::ValueName()... }, ids); } // Gets all values for rows that match the given id. template - static std::vector> GetAllValuesById(const SQLite::Connection& connection, SQLite::rowid_t id) + static std::vector> GetAllValuesById(const SQLite::Connection& connection, SQLite::rowid_t id) { - auto stmt = details::ManifestTableGetAllValuesByIds_Statement(connection, { SQLite::Builder::QualifiedColumn{ ValueTables::TableName(), ValueTables::ValueName() }... }, { IdTable::ValueName() }, { id }); - std::vector> result; + auto stmt = details::ManifestTableGetAllValuesByIds_Statement(connection, + { SQLite::Builder::QualifiedColumn{ TableName(), SQLite::RowIDName }, SQLite::Builder::QualifiedColumn{ ValueTables::TableName(), ValueTables::ValueName() }... }, + { SQLite::Builder::QualifiedColumn{ ValueTables::TableName(), ValueTables::ValueName() }... }, + { IdTable::ValueName() }, { id }); + std::vector> result; while (stmt.Step()) { - result.emplace_back(stmt.GetRow()); + result.emplace_back(stmt.GetRow()); } return result; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface_1_3.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface_1_3.cpp index 2bcc69fbd9..44a225059b 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface_1_3.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface_1_3.cpp @@ -80,8 +80,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_3 { case AppInstaller::Repository::PackageVersionProperty::ManifestSHA256Hash: { - SQLite::blob_t hash = std::get<0>(V1_0::ManifestTable::GetIdsById(connection, manifestId)); - return hash.empty() ? std::optional{} : Utility::SHA256::ConvertToString(hash); + std::optional hash = V1_0::ManifestTable::GetIdById(connection, manifestId); + return (!hash || hash->empty()) ? std::optional{} : Utility::SHA256::ConvertToString(hash.value()); } default: return V1_2::Interface::GetPropertyByManifestIdInternal(connection, manifestId, property); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface_1_4.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface_1_4.cpp index a741b0fc2a..10f826212f 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface_1_4.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface_1_4.cpp @@ -150,7 +150,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_4 { auto versionKeys = GetVersionKeysById(connection, dependency.first); THROW_HR_IF(E_UNEXPECTED, versionKeys.empty()); - checkedVersions.emplace(dependency.first, versionKeys[0].GetVersion()); + checkedVersions.emplace(dependency.first, versionKeys[0].VersionAndChannel.GetVersion()); } // If the latest version is less than min version required, fail the validation. diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_5/Interface_1_5.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_5/Interface_1_5.cpp index a3d6703265..296d64ae51 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_5/Interface_1_5.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_5/Interface_1_5.cpp @@ -173,9 +173,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_5 switch (property) { case AppInstaller::Repository::PackageVersionProperty::ArpMinVersion: - return std::get<0>(V1_0::ManifestTable::GetValuesById(connection, manifestId)); + return V1_0::ManifestTable::GetValueById(connection, manifestId); case AppInstaller::Repository::PackageVersionProperty::ArpMaxVersion: - return std::get<0>(V1_0::ManifestTable::GetValuesById(connection, manifestId)); + return V1_0::ManifestTable::GetValueById(connection, manifestId); default: return V1_4::Interface::GetPropertyByManifestIdInternal(connection, manifestId, property); } @@ -197,19 +197,15 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_5 auto versionKeys = GetVersionKeysById(connection, match.first); for (auto const& versionKey : versionKeys) { - auto manifestRowId = GetManifestIdByKey(connection, match.first, versionKey.GetVersion().ToString(), versionKey.GetChannel().ToString()); - if (manifestRowId) - { - auto arpMinVersion = GetPropertyByManifestId(connection, manifestRowId.value(), PackageVersionProperty::ArpMinVersion).value_or(""); - auto arpMaxVersion = GetPropertyByManifestId(connection, manifestRowId.value(), PackageVersionProperty::ArpMaxVersion).value_or(""); + auto arpMinVersion = GetPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionProperty::ArpMinVersion).value_or(""); + auto arpMaxVersion = GetPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionProperty::ArpMaxVersion).value_or(""); - // Either both empty or both not empty - THROW_HR_IF(E_UNEXPECTED, arpMinVersion.empty() != arpMaxVersion.empty()); + // Either both empty or both not empty + THROW_HR_IF(E_UNEXPECTED, arpMinVersion.empty() != arpMaxVersion.empty()); - if (!arpMinVersion.empty() && !arpMaxVersion.empty()) - { - ranges.emplace_back(Utility::VersionRange{ Utility::Version{ std::move(arpMinVersion) }, Utility::Version{ std::move(arpMaxVersion) } }); - } + if (!arpMinVersion.empty() && !arpMaxVersion.empty()) + { + ranges.emplace_back(Utility::VersionRange{ Utility::Version{ std::move(arpMinVersion) }, Utility::Version{ std::move(arpMaxVersion) } }); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h index 1b8b048da5..0ceac963e2 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h @@ -49,6 +49,15 @@ namespace AppInstaller::Repository::Microsoft::Schema DisableDependenciesSupport = 0x2, }; + // Contains both the object representation of the version key and the rows. + struct VersionKey + { + Utility::VersionAndChannel VersionAndChannel; + SQLite::rowid_t ManifestId; + + bool operator<(const VersionKey& other) const { return VersionAndChannel < other.VersionAndChannel; } + }; + // Creates all of the version dependent tables within the database. virtual void CreateTables(SQLite::Connection& connection, CreateOptions options) = 0; @@ -92,7 +101,7 @@ namespace AppInstaller::Repository::Microsoft::Schema virtual std::optional GetManifestIdByManifest(const SQLite::Connection& connection, const Manifest::Manifest& manifest) const = 0; // Gets all versions and channels for the given id. - virtual std::vector GetVersionKeysById(const SQLite::Connection& connection, SQLite::rowid_t id) const = 0; + virtual std::vector GetVersionKeysById(const SQLite::Connection& connection, SQLite::rowid_t id) const = 0; // Version 1.1 diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp index 6244086154..9653adf61d 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp @@ -36,6 +36,23 @@ CREATE TABLE [metadata]( return result; } + std::optional MetadataTable::TryGetNamedValueStatement(SQLite::Connection& connection, std::string_view name) + { + THROW_HR_IF(E_INVALIDARG, name.empty()); + + SQLite::Statement result = SQLite::Statement::Create(connection, s_MetadataTableStmt_GetNamedValue); + result.Bind(1, name); + + if (result.Step()) + { + return result; + } + else + { + return std::nullopt; + } + } + SQLite::Statement MetadataTable::SetNamedValueStatement(SQLite::Connection& connection, std::string_view name) { THROW_HR_IF(E_INVALIDARG, name.empty()); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h index 5205a270d2..c4ad1f88fd 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h @@ -12,6 +12,7 @@ namespace AppInstaller::Repository::Microsoft::Schema // Named metadata values are defined here to reduce the risk of duplicate names. // Version 1.0 + static constexpr std::string_view s_MetadataValueName_DatabaseIdentifier = "databaseIdentifier"sv; static constexpr std::string_view s_MetadataValueName_MajorVersion = "majorVersion"sv; static constexpr std::string_view s_MetadataValueName_MinorVersion = "minorVersion"sv; static constexpr std::string_view s_MetadataValueName_LastWriteTime = "lastwritetime"sv; @@ -30,6 +31,22 @@ namespace AppInstaller::Repository::Microsoft::Schema return statement.GetColumn(0); } + // Gets the named value from the metadata table, interpreting it as the given type. + // Returns nullopt if the value is not present. + template + static std::optional TryGetNamedValue(SQLite::Connection& connection, std::string_view name) + { + std::optional statement = TryGetNamedValueStatement(connection, name); + if (statement) + { + return statement->GetColumn(0); + } + else + { + return std::nullopt; + } + } + // Sets the named value into the metadata table. template static void SetNamedValue(SQLite::Connection& connection, std::string_view name, Value&& v) @@ -43,6 +60,9 @@ namespace AppInstaller::Repository::Microsoft::Schema // Internal function that gets the named value. static SQLite::Statement GetNamedValueStatement(SQLite::Connection& connection, std::string_view name); + // Internal function that gets the named value, or nullopt if it is not present. + static std::optional TryGetNamedValueStatement(SQLite::Connection& connection, std::string_view name); + // Internal function that sets the named value. static SQLite::Statement SetNamedValueStatement(SQLite::Connection& connection, std::string_view name); }; diff --git a/src/AppInstallerRepositoryCore/PackageDependenciesValidation.cpp b/src/AppInstallerRepositoryCore/PackageDependenciesValidation.cpp index dca5f8261b..d6a5945b15 100644 --- a/src/AppInstallerRepositoryCore/PackageDependenciesValidation.cpp +++ b/src/AppInstallerRepositoryCore/PackageDependenciesValidation.cpp @@ -33,7 +33,7 @@ namespace AppInstaller::Repository return depList; } - std::optional> GetPackageLatestVersion( + std::optional GetPackageLatestVersion( SQLiteIndex* index, Manifest::string_t packageId, std::set exclusions = {}) { SearchRequest request; @@ -54,31 +54,28 @@ namespace AppInstaller::Repository return {}; } - Utility::VersionAndChannel maxVersion(Utility::Version::CreateUnknown(), Utility::Channel("")); + SQLiteIndex::VersionKey maxVersion{ Utility::VersionAndChannel{ Utility::Version::CreateUnknown(), Utility::Channel("") } }; for (auto& v : vac) { - auto currentVersion = v.GetVersion(); + auto currentVersion = v.VersionAndChannel.GetVersion(); if (exclusions.find(currentVersion) != exclusions.end()) { continue; } - if (currentVersion > maxVersion.GetVersion()) + if (currentVersion > maxVersion.VersionAndChannel.GetVersion()) { maxVersion = v; } } - if (maxVersion.GetVersion().IsUnknown()) + if (maxVersion.VersionAndChannel.GetVersion().IsUnknown()) { return {}; } - auto manifestRowId = index->GetManifestIdByKey( - packageRowId, maxVersion.GetVersion().ToString(), maxVersion.GetChannel().ToString()); - - return std::make_pair(manifestRowId.value(), maxVersion.GetVersion()); + return maxVersion; } void ThrowOnManifestValidationFailed( @@ -130,14 +127,14 @@ namespace AppInstaller::Repository return depList; } - if (node.MinVersion > packageLatest.value().second) + if (node.MinVersion > packageLatest->VersionAndChannel.GetVersion()) { dependenciesError.emplace_back(ManifestError::NoSuitableMinVersionDependency, "PackageIdentifier", node.Id()); foundErrors = true; return depList; } - auto packageLatestDependencies = index->GetDependenciesByManifestRowId(packageLatest.value().first); + auto packageLatestDependencies = index->GetDependenciesByManifestRowId(packageLatest->ManifestId); std::for_each( packageLatestDependencies.begin(), packageLatestDependencies.end(), @@ -201,13 +198,13 @@ namespace AppInstaller::Repository THROW_HR(APPINSTALLER_CLI_ERROR_MISSING_PACKAGE); } - if (Utility::Version(manifest.Version) < packageLatest.value().second) + if (Utility::Version(manifest.Version) < packageLatest->VersionAndChannel.GetVersion()) { // all good, since it's min version the criteria is still satisfied. return true; } - auto nextLatestAfterDelete = GetPackageLatestVersion(index, manifest.Id, { packageLatest.value().second }); + auto nextLatestAfterDelete = GetPackageLatestVersion(index, manifest.Id, { packageLatest->VersionAndChannel.GetVersion() }); if (!nextLatestAfterDelete.has_value()) { @@ -224,7 +221,7 @@ namespace AppInstaller::Repository std::back_inserter(breakingManifests), [&](std::pair current) { - return current.second > nextLatestAfterDelete.value().second; + return current.second > nextLatestAfterDelete->VersionAndChannel.GetVersion(); } ); diff --git a/src/AppInstallerRepositoryCore/PackageTrackingCatalog.cpp b/src/AppInstallerRepositoryCore/PackageTrackingCatalog.cpp index 57b17f3e83..5bc5f7ec11 100644 --- a/src/AppInstallerRepositoryCore/PackageTrackingCatalog.cpp +++ b/src/AppInstallerRepositoryCore/PackageTrackingCatalog.cpp @@ -273,12 +273,7 @@ namespace AppInstaller::Repository for (const auto& version : versions) { - auto manifestId = index.GetManifestIdByKey(match.first, version.GetVersion().ToString(), version.GetChannel().ToString()); - - if (manifestId) - { - index.RemoveManifestById(manifestId.value()); - } + index.RemoveManifestById(version.ManifestId); } } } diff --git a/src/AppInstallerRepositoryCore/Public/winget/RepositorySource.h b/src/AppInstallerRepositoryCore/Public/winget/RepositorySource.h index ff68e41bab..fc0b9d5737 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/RepositorySource.h +++ b/src/AppInstallerRepositoryCore/Public/winget/RepositorySource.h @@ -68,6 +68,9 @@ namespace AppInstaller::Repository ARP, MSIX, Installing, + // Same as `Installed`, but creating the source reference for this is sufficient to cause the cache to be updated + // on next Open of any `Installed` or `InstalledForceCacheUpdate`. + InstalledForceCacheUpdate, }; // A well known source. diff --git a/src/AppInstallerRepositoryCore/RepositorySource.cpp b/src/AppInstallerRepositoryCore/RepositorySource.cpp index 0c514fe074..b9fc498f42 100644 --- a/src/AppInstallerRepositoryCore/RepositorySource.cpp +++ b/src/AppInstallerRepositoryCore/RepositorySource.cpp @@ -162,6 +162,10 @@ namespace AppInstaller::Repository details.Type = Microsoft::PredefinedInstalledSourceFactory::Type(); details.Arg = Microsoft::PredefinedInstalledSourceFactory::FilterToString(Microsoft::PredefinedInstalledSourceFactory::Filter::None); return details; + case PredefinedSource::InstalledForceCacheUpdate: + details.Type = Microsoft::PredefinedInstalledSourceFactory::Type(); + details.Arg = Microsoft::PredefinedInstalledSourceFactory::FilterToString(Microsoft::PredefinedInstalledSourceFactory::Filter::NoneWithForcedCacheUpdate); + return details; case PredefinedSource::InstalledUser: details.Type = Microsoft::PredefinedInstalledSourceFactory::Type(); details.Arg = Microsoft::PredefinedInstalledSourceFactory::FilterToString(Microsoft::PredefinedInstalledSourceFactory::Filter::User); diff --git a/src/AppInstallerRepositoryCore/SQLiteWrapper.cpp b/src/AppInstallerRepositoryCore/SQLiteWrapper.cpp index 38b88d6982..1d1c258d31 100644 --- a/src/AppInstallerRepositoryCore/SQLiteWrapper.cpp +++ b/src/AppInstallerRepositoryCore/SQLiteWrapper.cpp @@ -17,14 +17,14 @@ using namespace std::string_view_literals; #define THROW_SQLITE(_error_,_connection_) \ do { \ - int _ts_sqliteReturnValue = _error_; \ - sqlite3* _ts_sqliteConnection = _connection_; \ + int _ts_sqliteReturnValue = (_error_); \ + sqlite3* _ts_sqliteConnection = (_connection_); \ THROW_EXCEPTION_MSG(SQLiteException(_ts_sqliteReturnValue), _ts_sqliteConnection ? sqlite3_errmsg(_ts_sqliteConnection) : sqlite3_errstr(_ts_sqliteReturnValue)); \ } while (0,0) #define THROW_IF_SQLITE_FAILED(_statement_,_connection_) \ do { \ - int _tisf_sqliteReturnValue = _statement_; \ + int _tisf_sqliteReturnValue = (_statement_); \ if (_tisf_sqliteReturnValue != SQLITE_OK) \ { \ THROW_SQLITE(_tisf_sqliteReturnValue,_connection_); \ @@ -349,6 +349,46 @@ namespace AppInstaller::Repository::SQLite } } + Backup::Backup(Connection& destination, const std::string& destinationName, Connection& source, const std::string& sourceName) + { + m_backup.reset(sqlite3_backup_init(destination, destinationName.c_str(), source, sourceName.c_str())); + + if (!m_backup) + { + THROW_SQLITE(sqlite3_errcode(destination), destination); + } + } + + Backup Backup::Create(Connection& destination, const std::string& destinationName, Connection& source, const std::string& sourceName) + { + return { destination, destinationName, source, sourceName }; + } + + bool Backup::Step(int pages) + { + int stepResult = sqlite3_backup_step(m_backup.get(), pages); + + if (stepResult == SQLITE_OK) + { + // A negative number of pages should finish the operation + if (pages < 0) + { + THROW_HR(E_UNEXPECTED); + } + + // Success but not done + return false; + } + else if (stepResult == SQLITE_DONE) + { + return true; + } + else + { + THROW_SQLITE(stepResult, nullptr); + } + } + std::string_view EscapeCharForLike = "'"sv; std::string EscapeStringForLike(std::string_view value) diff --git a/src/AppInstallerRepositoryCore/SQLiteWrapper.h b/src/AppInstallerRepositoryCore/SQLiteWrapper.h index 3a5913ce95..cbc93414ba 100644 --- a/src/AppInstallerRepositoryCore/SQLiteWrapper.h +++ b/src/AppInstallerRepositoryCore/SQLiteWrapper.h @@ -317,6 +317,28 @@ namespace AppInstaller::Repository::SQLite Statement m_release; }; + // A SQLite backup operation. + struct Backup + { + // Creates a backup. + static Backup Create(Connection& destination, const std::string& destinationName, Connection& source, const std::string& sourceName); + + Backup(const Backup&) = delete; + Backup& operator=(const Backup&) = delete; + + Backup(Backup&&) = default; + Backup& operator=(Backup&&) = default; + + // Performs some or all of the backup. + // Returns true if the backup is completed, false if not. + bool Step(int pages = -1); + + private: + Backup(Connection& destination, const std::string& destinationName, Connection& source, const std::string& sourceName); + + wil::unique_any m_backup; + }; + // The escape character used in the EscapeStringForLike function. extern std::string_view EscapeCharForLike; diff --git a/src/AppInstallerRepositoryCore/pch.h b/src/AppInstallerRepositoryCore/pch.h index df1d971c19..cf1f381611 100644 --- a/src/AppInstallerRepositoryCore/pch.h +++ b/src/AppInstallerRepositoryCore/pch.h @@ -14,6 +14,7 @@ #pragma warning( push ) #pragma warning ( disable : 6001 6340 6387 6388 26495 28196 ) #include +#include #include #include #include diff --git a/src/AppInstallerSharedLib/Public/AppInstallerVersions.h b/src/AppInstallerSharedLib/Public/AppInstallerVersions.h index aa98c64c3f..9c2ee43849 100644 --- a/src/AppInstallerSharedLib/Public/AppInstallerVersions.h +++ b/src/AppInstallerSharedLib/Public/AppInstallerVersions.h @@ -93,6 +93,9 @@ namespace AppInstaller::Utility uint64_t Integer = 0; std::string Other; + + private: + std::string m_foldedOther; }; // Gets the part breakdown for a given version. diff --git a/src/AppInstallerSharedLib/Public/winget/Registry.h b/src/AppInstallerSharedLib/Public/winget/Registry.h index fcbae166f2..97da282e0c 100644 --- a/src/AppInstallerSharedLib/Public/winget/Registry.h +++ b/src/AppInstallerSharedLib/Public/winget/Registry.h @@ -277,6 +277,7 @@ namespace AppInstaller::Registry ValueList Values() const; operator bool() const { return m_key.operator bool(); } + operator HKEY() const { return m_key.get(); } // Open a Key; will return an empty Key if the subkey does not exist. static Key OpenIfExists(HKEY key, std::string_view subKey = {}, DWORD options = 0, REGSAM access = KEY_READ); diff --git a/src/AppInstallerSharedLib/Versions.cpp b/src/AppInstallerSharedLib/Versions.cpp index abb097b724..2b84e33a52 100644 --- a/src/AppInstallerSharedLib/Versions.cpp +++ b/src/AppInstallerSharedLib/Versions.cpp @@ -295,10 +295,15 @@ namespace AppInstaller::Utility { Other = end; } + + m_foldedOther = Utility::FoldCase(static_cast(Other)); } Version::Part::Part(uint64_t integer, std::string other) : - Integer(integer), Other(std::move(other)) {} + Integer(integer), Other(std::move(other)) + { + m_foldedOther = Utility::FoldCase(static_cast(Other)); + } bool Version::Part::operator<(const Part& other) const { @@ -320,8 +325,9 @@ namespace AppInstaller::Utility // If the other Other is empty and this is not, this is less. return true; } - else if (Other < other.Other) + else if (m_foldedOther < other.m_foldedOther) { + // Compare the folded versions return true; } @@ -331,7 +337,7 @@ namespace AppInstaller::Utility bool Version::Part::operator==(const Part& other) const { - return Integer == other.Integer && Other == other.Other; + return Integer == other.Integer && m_foldedOther == other.m_foldedOther; } bool Version::Part::operator!=(const Part& other) const