Pack reqs (#679)

libs/rtemodel/include/RteCprjProject.h

@@ -66,6 +66,13 @@ class RteCprjProject : public RteProject
   */
    void Initialize() override;
 
+   /**
+    * @brief get all packs required in the specified target
+    * @param packs collection of package IDs mapped to RtePackage pointers to fill
+    * @param targetName target name
+   */
+   void GetRequiredPacks(RtePackageMap& packs, const std::string& targetName) const override;
+
 public:
   /**
    * @brief set toolchain for the project

libs/rtemodel/include/RteKernel.h

@@ -173,6 +173,29 @@ class RteKernel
   */
   static RteKernel* GetNullKernel() { return &NULL_RTE_KERNEL; }
 
+  /**
+   * @brief get installed packs
+   * @param pdscFiles list of installed packs
+   * @param bool latest get only latest versions (default true)
+  */
+  bool GetInstalledPacks(std::list<std::string>& pdscFiles, bool latest = true);
+
+  /**
+   * @brief load and insert pack into global model
+   * @param packs list of loaded packages
+   * @param pdscFiles list of packs to be loaded
+   * @return true if executed successfully
+  */
+  bool LoadAndInsertPacks(std::list<RtePackage*>& packs, std::list<std::string>& pdscFiles);
+
+  /**
+   * @brief get list of installed pdsc files
+   * @param files collection to fill with absolute pdsc filenames;
+   * @param bool latest get only latest versions (default true)
+   * @param rtePath pack path
+  */
+  static void GetInstalledPdscFiles(std::list<std::string>& files, const std::string& rtePath, bool latest = true);
+
   /**
    * @brief getter for pdsc file determined by pack ID, pack path and pack attributes
    * @param attributes pack attributes

libs/rtemodel/include/RtePackage.h

@@ -20,11 +20,16 @@
 
 class RteTarget;
 class RtePackage;
+class RtePackageComparator;
 class RteExample;
 class RteGeneratorContainer;
 class RteGeneratorProject;
 class RteBoard;
 
+
+typedef std::map<std::string, RtePackage*, RtePackageComparator > RtePackageMap;
+typedef std::map<std::string, RteItem*, RtePackageComparator > RteItemPackageMap;
+
 /**
  * @brief class representing <releases> element in *.pdsc files
 */
@@ -150,6 +155,16 @@ class RtePackage : public RteRootItem
   */
   static int ComparePackageIDs(const std::string& id1, const std::string& id2);
 
+  /**
+   * @brief helper static method to compare pack filenames.
+   * Alpha-numeric comparison for vendor and name.
+   * Semantic version comparison for pack version.
+   * @param pdsc1 first pdsc file path
+   * @param pdsc2 second pdsc file path
+   * @return 0 if both files are equal, less than 0 if pdsc1 < pdsc2, greater than 0 if pdsc1 > pdsc2
+  */
+  static int ComparePdscFileNames(const std::string& pdsc1, const std::string& pdsc2);
+
   /**
    * @brief get pack display name
    * @return display string
@@ -417,12 +432,28 @@ class RtePackage : public RteRootItem
 
 public:
 
+  /**
+   * @brief get collection of packs required by this one
+   * @param map of id to RtePackage pointers to fill
+   * @param model pointer to RteModel to resolve requirement
+  */
+  void GetRequiredPacks(RtePackageMap& packs, RteModel* model) const;
+
   /**
    * @brief get list of packs required by this one
    * @return list of pointers to RteItem objects with pack requirement information
   */
   virtual const Collection<RteItem*>& GetPackRequirements() const;
 
+ /**
+  * @brief resolve packs for specified requirements
+  * @param originatingItem RteItem* supplying requirements, can be NULL
+  * @param requirements collection of RteItem pointers representing requirements
+  * @param RtePackageMap to fill
+  * @param model pointer to RteModel to resolve requirement
+ */
+  static void ResolveRequiredPacks(const RteItem* originatingItem, const Collection<RteItem*>& requirements, RtePackageMap& packs, RteModel* model);
+
   /**
    * @brief get list of language requirements imposed by this pack
    * @return list of pointers to RteItem objects with language requirement information
@@ -651,9 +682,22 @@ class RtePackageComparator
   bool operator()(const std::string& a, const std::string& b) const { return RtePackage::ComparePackageIDs(a, b) < 0; }
 };
 
+/**
+ * @brief helper compare class for sorted pack maps
+ * compares common ID alpha-numerically in ascending order and version semantically in descending order
+*/
+class RtePdscComparator
+{
+public:
+  /**
+   * @brief compare two pdsc file names by their pack IDs
+   * @param a first absolute pdsc file name
+   * @param b second absolute pdsc file name
+   * @return true if first pack is 'less' than the second one
+  */
+  bool operator()(const std::string& a, const std::string& b) const { return RtePackage::ComparePdscFileNames(a, b) < 0; }
+};
 
-typedef std::map<std::string, RtePackage*, RtePackageComparator > RtePackageMap;
-typedef std::map<std::string, RteItem*, RtePackageComparator > RteItemPackageMap;
 
 /**
  * @brief class that replicates frequently used information of a pack object or a pack release.

libs/rtemodel/include/RteProject.h

@@ -410,6 +410,13 @@ class RteProject : public RteRootItem
   */
   void GetUsedPacks(RtePackageMap& packs, const std::string& targetName) const;
 
+  /**
+   * @brief get all packs required in the specified target
+   * @param packs collection of package IDs mapped to RtePackage pointers to fill
+   * @param targetName target name
+  */
+  virtual void GetRequiredPacks(RtePackageMap& packs, const std::string& targetName) const;
+
   /**
    * @brief get collection of RteGpdscInfo objects
    * @return collection of gpdsc files mapped to associated RteGpdscInfo pointers

libs/rtemodel/src/RteCprjProject.cpp

@@ -134,6 +134,17 @@ void RteCprjProject::Initialize()
   }
 }
 
+void RteCprjProject::GetRequiredPacks(RtePackageMap& packs, const std::string& targetName) const
+{
+  RteTarget* t = GetTarget(targetName);
+  if (!t)
+    return;
+  CprjFile* cprjFile = GetCprjFile();
+
+  RtePackage::ResolveRequiredPacks(nullptr, cprjFile->GetPackRequirements(), packs, t->GetFilteredModel());
+  RteProject::GetRequiredPacks(packs, targetName);
+}
+
 
 void RteCprjProject::FillToolchainAttributes(XmlItem &attributes) const{
   if (m_toolchain == "AC5" || m_toolchain == "AC6") {

libs/rtemodel/src/RteKernel.cpp

@@ -305,12 +305,6 @@ bool RteKernel::LoadRequiredPdscFiles(CprjFile* cprjFile)
 
       msg += packRequirement->GetPackageID(true);
 
-      const string& name = packRequirement->GetAttribute("name");
-      const string& vendor = packRequirement->GetAttribute("vendor");
-      const string& version = packRequirement->GetAttribute("version");
-      msg += vendor + "." + name;
-      if (!version.empty())
-        msg += "." + version;
       GetRteCallback()->Err("R821", msg, cprjFile->GetPackageFileName());
       // TODO: install missing pack (not needed for web version)
       return false;
@@ -332,7 +326,85 @@ bool RteKernel::LoadRequiredPdscFiles(CprjFile* cprjFile)
   return true;
 }
 
+bool RteKernel::GetInstalledPacks(std::list<std::string>& pdscFiles, bool latest)
+{
+   auto& cmsisPackRoot = GetCmsisPackRoot();
+   if (cmsisPackRoot.empty()) {
+     return false;
+   }
+   RteKernel::GetInstalledPdscFiles(pdscFiles, GetCmsisPackRoot(), latest);
+
+  // Find pdsc files from local repository
+  list<string> localPdscUrls;
+  if (!GetLocalPacksUrls(GetCmsisPackRoot(), localPdscUrls)) {
+    return false;
+  }
+  for (const auto& localPdscUrl : localPdscUrls) {
+    list<string> localPdscFiles;
+    RteFsUtils::GetPackageDescriptionFiles(localPdscFiles, localPdscUrl, 1);
+    // Insert local pdsc files first
+    pdscFiles.insert(pdscFiles.begin(), localPdscFiles.begin(), localPdscFiles.end());
+  }
+  return true;
+}
+
+bool RteKernel::LoadAndInsertPacks(std::list<RtePackage*>& packs, std::list<std::string>& pdscFiles) {
+  RteGlobalModel* globalModel = GetGlobalModel();
+  if (!globalModel) {
+    return false;
+  }
+  std::list<RtePackage*> newPacks;
+  pdscFiles.unique();
+  for (const auto& pdscFile : pdscFiles) {
+    RtePackage* pack = LoadPack(pdscFile);
+    if (!pack) {
+      return false;
+    }
+    bool loaded = false;
+    for (const auto& loadedPack : packs) {
+      if (pack->GetPackageID() == loadedPack->GetPackageID()) {
+        loaded = true;
+        break;
+      }
+    }
+    if (!loaded) {
+      newPacks.push_back(pack);
+    } else {
+      delete pack;
+    }
+  }
+
+  globalModel->InsertPacks(newPacks);
+
+  // Track only packs that were actually inserted into the model
+  packs.clear();
+  for (const auto& [_, pack] : globalModel->GetPackages()) {
+    packs.push_back(pack);
+  }
+  return true;
+}
 
+
+void RteKernel::GetInstalledPdscFiles(list<string>& files, const std::string& rtePath, bool latest)
+{
+  if (!latest) {
+    RteFsUtils::GetPackageDescriptionFiles(files, rtePath, 3);
+    files.sort(RtePdscComparator());
+  } else {
+    list<string> allFiles;
+    RteFsUtils::GetPackageDescriptionFiles(allFiles, rtePath, 3);
+    allFiles.sort(RtePdscComparator());
+    string commonId;
+    for (auto& f : allFiles) {
+      string id = RtePackage::CommonIdFromId(RtePackage::PackIdFromPath(f));
+      if (id == commonId) {
+        continue; // skip packs with the same common ID
+      }
+      commonId = id;
+      files.push_back(f);
+    }
+  }
+}
 string RteKernel::GetInstalledPdscFile(const XmlItem& attributes, const string& rtePath, string& packId)
 {
   const string& name = attributes.GetAttribute("name");
@@ -347,7 +419,6 @@ string RteKernel::GetInstalledPdscFile(const XmlItem& attributes, const string&
     packId = RtePackage::ComposePackageID(vendor, name, installedVersion);
     return path + '/' + installedVersion + '/' + vendor + '.' + name + ".pdsc";
   }
-
   return RteUtils::EMPTY_STRING;
 }
 

libs/rtemodel/src/RtePackage.cpp

@@ -282,6 +282,11 @@ int RtePackage::ComparePackageIDs(const string& a, const string& b)
   return VersionCmp::Compare(verB, verA); // reverse comparison!
 }
 
+int RtePackage::ComparePdscFileNames(const std::string& pdsc1, const std::string& pdsc2)
+{
+  return ComparePackageIDs(PackIdFromPath(pdsc1), PackIdFromPath(pdsc2));
+}
+
 RteItem* RtePackage::GetRelease(const string& version) const
 {
   if (m_releases == nullptr) {
@@ -417,6 +422,28 @@ XMLTreeElement* RtePackage::CreatePackXmlTreeElement(XMLTreeElement* parent) con
   return package;
 }
 
+void RtePackage::GetRequiredPacks(RtePackageMap& packs, RteModel* model) const
+{
+  RtePackage::ResolveRequiredPacks(this, GetPackRequirements(), packs, model);
+}
+
+void RtePackage::ResolveRequiredPacks(const RteItem* originatingItem,
+        const Collection<RteItem*>& requirements, RtePackageMap& packs, RteModel* model)
+{
+  for (auto item : requirements) {
+    string id = RtePackage::GetPackageIDfromAttributes(*item);
+    if (packs.find(id) != packs.end()) {
+      continue; // already in the map
+    }
+    RtePackage* pack = model ? model->GetPackage(*item) : nullptr;
+    packs[id] = pack;
+    if (pack && pack != originatingItem) {
+      pack->GetRequiredPacks(packs, model);
+    }
+  }
+}
+
+
 const Collection<RteItem*>& RtePackage::GetPackRequirements() const
 {
   return GetItemGrandChildren(m_requirements, "packages");

libs/rtemodel/src/RteProject.cpp

@@ -2493,19 +2493,33 @@ void RteProject::GetUsedPacks(RtePackageMap& packs, const string& targetName) co
     }
   }
 
-
-  for (auto it = m_components.begin(); it != m_components.end(); it++) {
-    RteComponentInstance* ci = it->second;
+  for (auto [_, ci] : m_components) {
     if (!ci->IsUsedByTarget(targetName))
       continue;
     RtePackage* pack = ci->GetEffectivePackage(targetName);
     if (pack) {
       const string& id = pack->GetID();
-      packs[id] = pack;
+      if (packs.find(id) == packs.end()) {
+        packs[id] = pack;
+      }
     }
   }
 }
 
+void RteProject::GetRequiredPacks(RtePackageMap& packs, const std::string& targetName) const
+{
+  RteTarget* t = GetTarget(targetName);
+  if (!t)
+    return;
+  RtePackageMap usedPacks;
+  GetUsedPacks(usedPacks, targetName); // get all used packs
+  // add required packs
+  RteModel* model = t->GetFilteredModel();
+  for (auto [_, pack] : usedPacks) {
+    pack->GetRequiredPacks(packs, model);
+  }
+}
+
 bool RteProject::HasProjectGroup(const string& group) const
 {
   map<string, RteTarget*>::const_iterator it;

libs/rtemodel/test/src/RteChkTest.cpp

@@ -15,7 +15,7 @@ using namespace std;
 
 TEST(RteChkTest, Summary) {
 
-const string summary = "Collecting pdsc files 5 files found\n\
+const string summary = "Collecting pdsc files 7 files found\n\
 Parsing XML passed\n\
 \n\
 Constructing Model passed\n\
@@ -25,8 +25,8 @@ Cleaning XML data\n\
 Validating Model passed\n\
 \n\
 Summary:\n\
-Packs: 5\n\
-Generic: 1\n\
+Packs: 7\n\
+Generic: 3\n\
 DFP: 3\n\
 BSP: 1\n\
 \n\
@@ -50,7 +50,7 @@ completed\n";
   rteChk.AddFileDir(RteModelTestConfig::CMSIS_PACK_ROOT);
   int res = rteChk.RunCheckRte();
   EXPECT_EQ(res, 0);
-  EXPECT_EQ(rteChk.GetPackCount(), 5);
+  EXPECT_EQ(rteChk.GetPackCount(), 7);
   EXPECT_EQ(rteChk.GetComponentCount(), 52);
   EXPECT_EQ(rteChk.GetDeviceCount(), 10);
   EXPECT_EQ(rteChk.GetBoardCount(), 13);