[projmgr] Global Generator: update context selection, defer partial processing failures

Author: brondani

tools/projmgr/src/ProjMgrWorker.cpp

@@ -526,8 +526,10 @@ bool ProjMgrWorker::LoadAllRelevantPacks() {
   m_loadedPacks.clear(); // the list will be updated, it should not contain dangling pointers
   // Get required pdsc files
   std::list<std::string> pdscFiles;
+  bool noPackRequirements = true;
   for (const auto& context : m_selectedContexts) {
     auto& contextItem = m_contexts.at(context);
+    noPackRequirements &= contextItem.packRequirements.empty();
     for (const auto& [pdscFile, pathVer] : contextItem.pdscFiles) {
       const string& path = pathVer.first;
       if (!path.empty()) {
@@ -543,12 +545,12 @@ bool ProjMgrWorker::LoadAllRelevantPacks() {
     }
   }
   // Check load packs policy
-  if (pdscFiles.empty() && (m_loadPacksPolicy == LoadPacksPolicy::REQUIRED)) {
+  if (noPackRequirements && (m_loadPacksPolicy == LoadPacksPolicy::REQUIRED)) {
     ProjMgrLogger::Get().Error("required packs must be specified");
     return false;
   }
   // Get installed packs
-  if (pdscFiles.empty() || (m_loadPacksPolicy == LoadPacksPolicy::ALL) || (m_loadPacksPolicy == LoadPacksPolicy::LATEST)) {
+  if (noPackRequirements || (m_loadPacksPolicy == LoadPacksPolicy::ALL) || (m_loadPacksPolicy == LoadPacksPolicy::LATEST)) {
     const bool latest = (m_loadPacksPolicy == LoadPacksPolicy::LATEST) || (m_loadPacksPolicy == LoadPacksPolicy::DEFAULT);
     if (!m_kernel->GetEffectivePdscFiles(pdscFiles, latest)) {
       ProjMgrLogger::Get().Error("parsing installed packs failed");
@@ -585,16 +587,15 @@ bool ProjMgrWorker::LoadPacks(ContextItem& context) {
   if (!InitializeTarget(context)) {
     return false;
   }
+  bool deferredFail = false;
   if (!CollectAllRequiredPdscFiles()) {
     PrintContextErrors(context.name);
-    if(!m_rpcMode) {
-      return false;
-    }
+    // defer return due to pdsc collection failure
+    deferredFail = true;
   }
   if ((m_loadedPacks.empty() || m_rpcMode) && !LoadAllRelevantPacks()) {
-    if(!m_rpcMode) {
-      return false;
-    }
+    // defer return due to pack loading failure
+    deferredFail = true;
   }
   // Filter context specific packs
   set<string> selectedPacks;
@@ -624,6 +625,10 @@ bool ProjMgrWorker::LoadPacks(ContextItem& context) {
   filter.SetSelectedPackages(selectedPacks);
   context.rteActiveTarget->SetPackageFilter(filter);
   context.rteActiveTarget->UpdateFilterModel();
+  // report deferred failure; in rpc mode continue processing despite it
+  if (!m_rpcMode && deferredFail) {
+    return false;
+  }
   return CheckRteErrors();
 }
 
@@ -4807,12 +4812,12 @@ bool ProjMgrWorker::ListGenerators(vector<string>& generators) {
   set<string> generatorsSet;
   map <string, map<string, map<string, StrVec>>> sortedGeneratorsMap;
   StrMap generatorsDescription;
+  bool deferredFail = false;
   // classic generators
   for (const auto& selectedContext : m_selectedContexts) {
     ContextItem& context = m_contexts[selectedContext];
-    if (!ProcessContext(context, false, true, false)) {
-      return false;
-    }
+    // defer failure report
+    deferredFail |= !ProcessContext(context, false, true, false);
     for (const auto& [id, generator] : context.generators) {
       for (const auto& [gpdsc, item] : context.gpdscs) {
         if (item.generator == id) {
@@ -4827,6 +4832,11 @@ bool ProjMgrWorker::ListGenerators(vector<string>& generators) {
       }
     }
   }
+  if (deferredFail) {
+    // report deferred failure
+    return false;
+  }
+
   // global generators
   for (const auto& [options, contexts] : m_extGenerator->GetUsedGenerators()) {
     sortedGeneratorsMap.insert({ options.id, {} });
@@ -5678,22 +5688,31 @@ std::string ProjMgrWorker::GetSelectedToochain(void) {
 bool ProjMgrWorker::ProcessGlobalGenerators(ContextItem* selectedContext, const string& generatorId,
   string& projectType, StrVec& siblings) {
 
-  // iterate over contexts with same build and target types
-  m_selectedContexts.clear();
-  for (auto& [_, context] : m_contexts) {
-    if ((context.type.build != selectedContext->type.build) ||
-      (context.type.target != selectedContext->type.target)) {
-      continue;
-    }
-    if (!ParseContextLayers(context)) {
-      return false;
+  if (m_activeTargetType.empty()) {
+    // if active target type is not given, select all contexts with same build and target types
+    m_selectedContexts.clear();
+    for (auto& [_, context] : m_contexts) {
+      if ((context.type.build != selectedContext->type.build) ||
+        (context.type.target != selectedContext->type.target)) {
+        continue;
+      }
+      m_selectedContexts.push_back(context.name);
     }
-    m_selectedContexts.push_back(context.name);
   }
-  for (auto& context : m_selectedContexts) {
-    if (!ProcessContext(m_contexts.at(context), false, true, false)) {
-      return false;
+  // parse layers and process selected contexts
+  bool deferredFail = false;
+  for (auto& contextName : m_selectedContexts) {
+    if (!ParseContextLayers(m_contexts.at(contextName))) {
+      // defer failure report due to layers parsing
+      deferredFail = true;
+      continue;
     }
+    // defer failure report due to context processing
+    deferredFail |= !ProcessContext(m_contexts.at(contextName), false, true, false);
+  }
+  // report deferred failure
+  if (deferredFail) {
+    return false;
   }
   StrVec contextVec;
   const string& genDir = selectedContext->extGen[generatorId].path;
@@ -6147,7 +6166,8 @@ void ProjMgrWorker::CollectUnusedPacks() {
       continue;
     }
     context.unusedPacks.clear();
-    for (const auto& [packId, _] : context.rteFilteredModel->GetPackages()) {
+    for (const auto& [_, packItem] : context.rteFilteredModel->GetPackages()) {
+      const auto packId = packItem->GetPackageID();
       if (context.packages.find(packId) == context.packages.end()) {
         CollectionUtils::PushBackUniquely(context.unusedPacks, packId);
       }

tools/projmgr/test/data/TestGenerator/RTE/Device/RteTestGen_ARMCM0/ac6_linker_script.sct.src

@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2023 Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------------
+  Stack seal size definition
+ *----------------------------------------------------------------------------*/
+#if defined (__ARM_FEATURE_CMSE) && (__ARM_FEATURE_CMSE == 3U)
+#define __STACKSEAL_SIZE 8
+#else
+#define __STACKSEAL_SIZE 0
+#endif
+
+/*----------------------------------------------------------------------------
+  Scatter File Definitions definition
+ *----------------------------------------------------------------------------*/
+
+LR_ROM0 __ROM0_BASE __ROM0_SIZE  {
+
+  ER_ROM0 __ROM0_BASE __ROM0_SIZE {
+    *.o (RESET, +First)
+    *(InRoot$$Sections)
+    *(+RO +XO)
+  }
+
+#if defined (__ARM_FEATURE_CMSE) && (__ARM_FEATURE_CMSE == 3U)
+  ER_CMSE_VENEER AlignExpr(+0, 32) (__ROM0_SIZE - AlignExpr(ImageLength(ER_ROM0), 32)) {
+    *(Veneer$$CMSE)
+  }
+#endif
+
+  RW_NOINIT __RAM0_BASE UNINIT (__RAM0_SIZE - __HEAP_SIZE - __STACK_SIZE - __STACKSEAL_SIZE) {
+    *.o(.bss.noinit)
+    *.o(.bss.noinit.*)
+  }
+
+  RW_RAM0 AlignExpr(+0, 8) (__RAM0_SIZE - __HEAP_SIZE - __STACK_SIZE - __STACKSEAL_SIZE - AlignExpr(ImageLength(RW_NOINIT), 8)) {
+    *(+RW +ZI)
+  }
+
+#if __HEAP_SIZE > 0
+  ARM_LIB_HEAP  (AlignExpr(+0, 8)) EMPTY __HEAP_SIZE  {   ; Reserve empty region for heap
+  }
+#endif
+
+  ARM_LIB_STACK (__RAM0_BASE + __RAM0_SIZE - __STACKSEAL_SIZE) EMPTY -__STACK_SIZE {   ; Reserve empty region for stack
+  }
+
+#if __STACKSEAL_SIZE > 0
+  STACKSEAL +0 EMPTY __STACKSEAL_SIZE {             ; Reserve empty region for stack seal immediately after stack
+  }
+#endif
+
+#if __RAM1_SIZE > 0
+  RW_RAM1 __RAM1_BASE __RAM1_SIZE  {
+   .ANY (+RW +ZI)
+  }
+#endif
+
+#if __RAM2_SIZE > 0
+  RW_RAM2 __RAM2_BASE __RAM2_SIZE  {
+   .ANY (+RW +ZI)
+  }
+#endif
+
+#if __RAM3_SIZE > 0
+  RW_RAM3 __RAM3_BASE __RAM3_SIZE  {
+   .ANY (+RW +ZI)
+  }
+#endif
+}
+
+#if __ROM1_SIZE > 0
+LR_ROM1 __ROM1_BASE __ROM1_SIZE  {
+  ER_ROM1 +0 __ROM1_SIZE {
+   .ANY (+RO +XO)
+  }
+}
+#endif
+
+#if __ROM2_SIZE > 0
+LR_ROM2 __ROM2_BASE __ROM2_SIZE  {
+  ER_ROM2 +0 __ROM2_SIZE {
+   .ANY (+RO +XO)
+  }
+}
+#endif
+
+#if __ROM3_SIZE > 0
+LR_ROM3 __ROM3_BASE __ROM3_SIZE  {
+  ER_ROM3 +0 __ROM3_SIZE {
+   .ANY (+RO +XO)
+  }
+}
+#endif

tools/projmgr/test/data/TestGenerator/RTE/Device/RteTestGen_ARMCM0/regions_RteTestGen_ARMCM0.h

@@ -0,0 +1,101 @@
+#ifndef REGIONS_RTETESTGEN_ARMCM0_H
+#define REGIONS_RTETESTGEN_ARMCM0_H
+
+
+//-------- <<< Use Configuration Wizard in Context Menu >>> --------------------
+//------ With VS Code: Open Preview for Configuration Wizard -------------------
+
+// <n> Auto-generated using information from packs
+// <i> Device Family Pack (DFP):   ARM::RteTestGenerator@0.1.0
+
+// <h> ROM Configuration
+// =======================
+// <h> __ROM0 (unused)
+//   <o> Base address <0x0-0xFFFFFFFF:8>
+//   <i> Defines base address of memory region.
+//   <i> Contains Startup and Vector Table
+#define __ROM0_BASE 0x0
+//   <o> Region size [bytes] <0x0-0xFFFFFFFF:8>
+//   <i> Defines size of memory region.
+#define __ROM0_SIZE 0x0
+// </h>
+
+// <h> __ROM1 (unused)
+//   <o> Base address <0x0-0xFFFFFFFF:8>
+//   <i> Defines base address of memory region.
+#define __ROM1_BASE 0x0
+//   <o> Region size [bytes] <0x0-0xFFFFFFFF:8>
+//   <i> Defines size of memory region.
+#define __ROM1_SIZE 0x0
+// </h>
+
+// <h> __ROM2 (unused)
+//   <o> Base address <0x0-0xFFFFFFFF:8>
+//   <i> Defines base address of memory region.
+#define __ROM2_BASE 0x0
+//   <o> Region size [bytes] <0x0-0xFFFFFFFF:8>
+//   <i> Defines size of memory region.
+#define __ROM2_SIZE 0x0
+// </h>
+
+// <h> __ROM3 (unused)
+//   <o> Base address <0x0-0xFFFFFFFF:8>
+//   <i> Defines base address of memory region.
+#define __ROM3_BASE 0x0
+//   <o> Region size [bytes] <0x0-0xFFFFFFFF:8>
+//   <i> Defines size of memory region.
+#define __ROM3_SIZE 0x0
+// </h>
+
+// </h>
+
+// <h> RAM Configuration
+// =======================
+// <h> __RAM0 (unused)
+//   <o> Base address <0x0-0xFFFFFFFF:8>
+//   <i> Defines base address of memory region.
+//   <i> Contains uninitialized RAM, Stack, and Heap
+#define __RAM0_BASE 0x0
+//   <o> Region size [bytes] <0x0-0xFFFFFFFF:8>
+//   <i> Defines size of memory region.
+#define __RAM0_SIZE 0x0
+// </h>
+
+// <h> __RAM1 (unused)
+//   <o> Base address <0x0-0xFFFFFFFF:8>
+//   <i> Defines base address of memory region.
+#define __RAM1_BASE 0x0
+//   <o> Region size [bytes] <0x0-0xFFFFFFFF:8>
+//   <i> Defines size of memory region.
+#define __RAM1_SIZE 0x0
+// </h>
+
+// <h> __RAM2 (unused)
+//   <o> Base address <0x0-0xFFFFFFFF:8>
+//   <i> Defines base address of memory region.
+#define __RAM2_BASE 0x0
+//   <o> Region size [bytes] <0x0-0xFFFFFFFF:8>
+//   <i> Defines size of memory region.
+#define __RAM2_SIZE 0x0
+// </h>
+
+// <h> __RAM3 (unused)
+//   <o> Base address <0x0-0xFFFFFFFF:8>
+//   <i> Defines base address of memory region.
+#define __RAM3_BASE 0x0
+//   <o> Region size [bytes] <0x0-0xFFFFFFFF:8>
+//   <i> Defines size of memory region.
+#define __RAM3_SIZE 0x0
+// </h>
+
+// </h>
+
+// <h> Stack / Heap Configuration
+//   <o0> Stack Size (in Bytes) <0x0-0xFFFFFFFF:8>
+//   <o1> Heap Size (in Bytes) <0x0-0xFFFFFFFF:8>
+#define __STACK_SIZE 0x00000200
+#define __HEAP_SIZE 0x00000000
+// </h>
+
+
+#endif /* REGIONS_RTETESTGEN_ARMCM0_H */

tools/projmgr/test/data/TestSolution/PackMissing/ref/missing_pack.cbuild-idx.yml

@@ -1,5 +1,5 @@
 build-idx:
-  generated-by: csolution version 0.0.0+g79b66b5f
+  generated-by: csolution version 0.0.0
   csolution: ../data/TestSolution/PackMissing/missing_pack.csolution.yml
   tmpdir: tmp
   cprojects:
@@ -12,7 +12,6 @@ build-idx:
       messages:
         errors:
           - "required pack: ARM::Missing_DFP@0.0.9 not installed"
-          - "specified device 'RteTest_ARMCM0' not found in the installed packs. Use:\n  cpackget add Vendor::PackName"
           - processing context 'project+CM0' failed
         info:
           - missing_pack.cbuild-pack.yml - file generated successfully
@@ -26,7 +25,6 @@ build-idx:
       messages:
         errors:
           - "required pack: ARM::Missing_PACK@0.0.1 not installed"
-          - "specified device 'RteTest_ARMCM3' not found in the installed packs. Use:\n  cpackget add Vendor::PackName"
           - processing context 'project+Gen' failed
         info:
           - missing_pack.cbuild-pack.yml - file generated successfully
@@ -42,7 +40,6 @@ build-idx:
           - "required pack: ARM::Missing_DFP@>=0.0.5 not installed"
           - "required pack: ARM::Missing_PACK not installed"
           - "required pack: ARM::RteTest@3.0.1 not installed"
-          - "specified device 'RteTest_ARMCM0' not found in the installed packs. Use:\n  cpackget add Vendor::PackName"
           - processing context 'project+Miss' failed
         info:
           - missing_pack.cbuild-pack.yml - file generated successfully