added parquet support for control networks (#5905)

* added parquet support

* checkpoint, more additions

* checkpoint

* adding deps to meta.yml

* added VSI support for image lists

* minor changes

* added changelog

* comments

* fixed log

* rm frag

Author: Kelvinrr

changes/+20260630130235.add.md

@@ -0,0 +1 @@
+Added Parquet support for control networks

environment.yml

@@ -35,6 +35,7 @@ dependencies:
   - krb5
   - libboost >=1.88.0,<1.89
   - libgdal >=3.12.2,<4
+  - libgdal-arrow-parquet >=3.12,<4
   - libopencv >=4.12,<5
   - libpng >=1.6.34,<1.7
   - libprotobuf >=6,<7

environment_arm.yml

@@ -33,8 +33,9 @@ dependencies:
   - jama
   - kalasiris
   - krb5
-  - libboost >=1.88.0,<1.89
   - libgdal >=3.12.2,<4
+  - libgdal-arrow-parquet >=3.12,<4
+  - libboost >=1.88.0,<1.89
   - libopencv >=4.12,<5
   - libpng >=1.6.34,<1.7
   - libprotobuf >=6,<7

isis/src/base/objs/FileList/FileList.cpp

@@ -11,6 +11,9 @@ find files of those names at the top level of this repository. **/
 #include "IString.h"
 #include <iostream>
 #include <fstream>
+#include <sstream>
+
+#include "cpl_vsi.h"
 
 using namespace std;
 namespace Isis {
@@ -50,6 +53,36 @@ namespace Isis {
    * @throws Isis::iException::Io - Cannot open file
    */
   void FileList::read(FileName listFile) {
+    QString path = listFile.toString();
+
+    if (path.startsWith("/vsi")) {
+      VSILFILE *fp = VSIFOpenL(path.toUtf8().constData(), "rb");
+      if (!fp) {
+        QString message = Isis::Message::FileOpen(path);
+        throw IException(IException::Io, message, _FILEINFO_);
+      }
+
+      // Read the entire file in chunks (a cube list can be arbitrarily large,
+      // so do not assume a fixed buffer size).
+      std::string contents;
+      char buffer[65536];
+      size_t nRead;
+      while ((nRead = VSIFReadL(buffer, 1, sizeof(buffer), fp)) > 0) {
+        contents.append(buffer, nRead);
+      }
+      VSIFCloseL(fp);
+
+      try {
+        std::istringstream iss(contents);
+        read(iss);
+      }
+      catch (IException &e) {
+        QString msg = "File [" + path + "] contains no data";
+        throw IException(IException::User, msg, _FILEINFO_);
+      }
+      return;
+    }
+
     // Open the file
     ifstream istm;
     istm.open(listFile.toString().toLatin1().data(), std::ios::in);
@@ -156,6 +189,23 @@ namespace Isis {
    * @throws Isis::iException::Io File could not be created.
    */
   void FileList::write(FileName outputFileList) {
+    QString path = outputFileList.toString();
+
+    if (path.startsWith("/vsi")) {
+      std::ostringstream oss;
+      write(oss);
+      std::string data = oss.str();
+
+      VSILFILE *fp = VSIFOpenL(path.toUtf8().constData(), "wb");
+      if (!fp) {
+        QString message = Message::FileOpen(path);
+        throw IException(IException::Io, message, _FILEINFO_);
+      }
+      VSIFWriteL(data.data(), 1, data.size(), fp);
+      VSIFCloseL(fp);
+      return;
+    }
+
     // Open the file
     ofstream ostm;
     ostm.open(outputFileList.toString().toLatin1().data(), std::ios::out);

isis/src/control/objs/ControlNet/ControlNet.cpp

@@ -310,6 +310,26 @@ namespace Isis {
   void ControlNet::Write(const QString &ptfile, bool pvl) {
     ControlNetVersioner versionedWriter(this);
 
+    // Check if writing to Parquet format. Strip any VSI query string (e.g.
+    // "/vsicurl/https://host/net.parquet?list=no") before testing the suffix.
+    bool isParquet = ptfile.split("?").first().endsWith(".parquet", Qt::CaseInsensitive);
+
+    if (isParquet) {
+      if (pvl) {
+        QString msg = "Cannot write control network as Pvl to a Parquet file [" + ptfile
+                      + "]. Parquet (binary) and Pvl output are mutually exclusive.";
+        throw IException(IException::User, msg, _FILEINFO_);
+      }
+      try {
+        versionedWriter.write(FileName(ptfile));
+      }
+      catch (IException &e) {
+        QString msg = "Failed writing control network to Parquet file [" + ptfile + "]";
+        throw IException(e, IException::Io, msg, _FILEINFO_);
+      }
+      return;
+    }
+
     if (pvl) {
       Pvl network;
       try {