Merge pull request #628 from astrorama/feature/max_files_option

Added option to specify max nb of simultaneous files

Author: marcschefer

SEFramework/SEFramework/FITS/FitsImageSource.h

@@ -46,6 +46,7 @@ using Euclid::FilePool::FileHandler;
 class FitsImageSource : public ImageSource, public std::enable_shared_from_this<ImageSource> {
 public:
 
+  static std::shared_ptr<FileManager> getFileManager(unsigned int max_open_files = 500);
 
   /**
    * Constructor
@@ -57,14 +58,14 @@ class FitsImageSource : public ImageSource, public std::enable_shared_from_this<
    */
   explicit FitsImageSource(const std::string& filename, int hdu_number = 0,
                   ImageTile::ImageType image_type = ImageTile::AutoType,
-                  std::shared_ptr<FileManager> manager = FileManager::getDefault());
+                  std::shared_ptr<FileManager> manager = getFileManager());
 
   FitsImageSource(const std::string& filename, int width, int height,
                   ImageTile::ImageType image_type,
                   const std::shared_ptr<CoordinateSystem> coord_system = nullptr,
                   bool append = false,
                   bool empty_primary = false,
-                  std::shared_ptr<FileManager> manager = FileManager::getDefault());
+                  std::shared_ptr<FileManager> manager = getFileManager());
 
   virtual ~FitsImageSource() = default;
 
@@ -137,6 +138,8 @@ class FitsImageSource : public ImageSource, public std::enable_shared_from_this<
   ImageTile::ImageType m_image_type;
 
   int m_current_layer;
+
+  static std::shared_ptr<FileManager> s_file_manager;
 };
 
 }

SEFramework/src/lib/FITS/FitsImageSource.cpp

@@ -23,10 +23,13 @@
  */
 
 #include "SEFramework/FITS/FitsImageSource.h"
+
 #include "SEFramework/FITS/FitsFile.h"
 #include "SEUtils/VariantCast.h"
 #include <AlexandriaKernel/memory_tools.h>
+#include <FilePool/LRUFileManager.h>
 #include <ElementsKernel/Exception.h>
+#include "ElementsKernel/Logging.h"
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/algorithm/string/trim.hpp>
 #include <boost/filesystem/operations.hpp>
@@ -39,6 +42,9 @@
 
 namespace SourceXtractor {
 
+static Elements::Logging logger = Elements::Logging::getLogger("FitsFile");
+
+
 using Euclid::make_unique;
 
 namespace {
@@ -386,7 +392,18 @@ int FitsImageSource::getImageType() const {
       return LONGLONG_IMG;
   }
 }
-
 //FIXME add missing types
 
+
+std::shared_ptr<FileManager> FitsImageSource::getFileManager(unsigned int max_open_files) {
+  if (!s_file_manager) {
+    logger.info() << "Creating shared LRUFileManager with limit of " << max_open_files << " open files.";
+    s_file_manager = std::make_shared<Euclid::FilePool::LRUFileManager>(max_open_files);
+  }
+  return s_file_manager;
 }
+
+std::shared_ptr<FileManager> FitsImageSource::s_file_manager = nullptr;
+
+}  // namespace SourceXtractor
+

SEImplementation/SEImplementation/Configuration/FileManagerConfig.h

@@ -0,0 +1,44 @@
+/** Copyright © 2019-2025 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université
+ *
+ * This library is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 3.0 of the License, or (at your option)
+ * any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this library; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef SEIMPLEMENTATION_CONFIGURATION_FILEMANAGERCONFIG_H
+#define SEIMPLEMENTATION_CONFIGURATION_FILEMANAGERCONFIG_H
+
+#include "Configuration/Configuration.h"
+
+namespace SourceXtractor {
+
+class FileManagerConfig : public Euclid::Configuration::Configuration {
+public:
+    explicit FileManagerConfig(long manager_id);
+    virtual ~FileManagerConfig() = default;
+
+    std::map<std::string, Configuration::OptionDescriptionList> getProgramOptions() override;
+    void preInitialize(const UserValues& args) override;
+    void initialize(const UserValues& args) override;
+
+    int getMaxSimultaneousFiles() const {
+        return m_max_simultaneous_files;
+    }
+
+private:
+    int m_max_simultaneous_files;
+};
+
+} // namespace SourceXtractor
+
+#endif // SEIMPLEMENTATION_CONFIGURATION_FILEMANAGERCONFIG_H

SEImplementation/src/lib/Configuration/FileManagerConfig.cpp

@@ -0,0 +1,58 @@
+/** Copyright © 2019-2025 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université
+ *
+ * This library is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 3.0 of the License, or (at your option)
+ * any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this library; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "SEImplementation/Configuration/FileManagerConfig.h"
+
+#include <boost/algorithm/string.hpp>
+
+#include "ElementsKernel/Exception.h"
+#include "Configuration/ConfigManager.h"
+
+#include "SEFramework/FITS/FitsImageSource.h"
+
+#include <iostream>
+
+using namespace Euclid::Configuration;
+namespace po = boost::program_options;
+
+namespace SourceXtractor {
+
+static const std::string MAX_SIMULTANEOUS_FILES {"max-simultaneous-files" };
+
+
+ // Constructor
+FileManagerConfig::FileManagerConfig(long manager_id)
+ : Configuration(manager_id), m_max_simultaneous_files(500) {
+}
+
+std::map<std::string, Configuration::OptionDescriptionList> FileManagerConfig::getProgramOptions() {
+    return { {"File Manager", {
+        {MAX_SIMULTANEOUS_FILES.c_str(), po::value<int>()->default_value(500),
+            "Maximum number of simultaneous open files."}
+    }} };
+}
+
+void FileManagerConfig::preInitialize(const UserValues& args) {
+    // We have to configure the FileManager before any other configuration tries to use it
+    m_max_simultaneous_files = args.find(MAX_SIMULTANEOUS_FILES)->second.as<int>();
+    FitsImageSource::getFileManager(m_max_simultaneous_files);
+}
+
+void FileManagerConfig::initialize(const UserValues& args) {
+}
+
+} // namespace SourceXtractor

SEMain/src/program/SourceXtractor.cpp

@@ -74,6 +74,7 @@
 #include "SEImplementation/Configuration/MemoryConfig.h"
 #include "SEImplementation/Configuration/OutputConfig.h"
 #include "SEImplementation/Configuration/SamplingConfig.h"
+#include "SEImplementation/Configuration/FileManagerConfig.h"
 #include "SEImplementation/CheckImages/CheckImages.h"
 #include "SEImplementation/Prefetcher/Prefetcher.h"
 
@@ -191,6 +192,7 @@ class SEMain : public Elements::Program {
       config_manager.registerConfiguration<BackgroundAnalyzerFactory>();
       config_manager.registerConfiguration<SamplingConfig>();
       config_manager.registerConfiguration<DetectionFrameConfig>();
+      config_manager.registerConfiguration<FileManagerConfig>();
 
       CheckImages::getInstance().reportConfigDependencies(config_manager);
 
@@ -319,15 +321,15 @@ class SEMain : public Elements::Program {
         throw Elements::Exception() << "The configuration file '" << cfg_file << "' does not exist";
       }
     }
-
+    
     // Create the progress listener and printer ASAP
     progress_printer_factory.configure(args);
     auto progress_mediator = progress_printer_factory.createProgressMediator();
-
+    
     // Initialize the rest of the components
     auto& config_manager = ConfigManager::getInstance(config_manager_id);
     config_manager.initialize(args);
-
+    
     // Configure TileManager
     auto memory_config = config_manager.getConfiguration<MemoryConfig>();
     TileManager::getInstance()->setOptions(memory_config.getTileSize(),