Core: Integrated unified recursive statistics for directory file count, size, and uploads

Author: jolavillette

src/file_sharing/dir_hierarchy.cc

@@ -22,6 +22,7 @@
  ******************************************************************************/
 #include <sstream>
 #include <algorithm>
+#include <functional>
 
 #include "util/rstime.h"
 #include "util/rsdir.h"
@@ -669,26 +670,36 @@ bool InternalFileHierarchyStorage::setTS(const DirectoryStorage::EntryIndex& ind
 
 // Do a complete recursive sweep of directory hierarchy and update cumulative size of directories
 
-uint64_t InternalFileHierarchyStorage::recursUpdateCumulatedSize(const DirectoryStorage::EntryIndex& dir_index)
+uint64_t InternalFileHierarchyStorage::recursUpdateCumulatedSize(const DirectoryStorage::EntryIndex& dir_index, std::function<uint64_t(const RsFileHash&)> get_uploads)
 {
     DirEntry& d(*static_cast<DirEntry*>(mNodes[dir_index])) ;
 
     uint64_t local_cumulative_size = 0;
     uint32_t local_cumulative_files = 0;
+    uint64_t local_cumulative_uploads = 0;
 
     for(uint32_t i=0;i<d.subfiles.size();++i)
         if(mNodes[d.subfiles[i]]) {		// normally not needed, but an extra-security
-            local_cumulative_size += static_cast<FileEntry*>(mNodes[d.subfiles[i]])->file_size;
+            FileEntry *f = static_cast<FileEntry*>(mNodes[d.subfiles[i]]);
+            local_cumulative_size += f->file_size;
             local_cumulative_files++;
+
+            if(get_uploads)
+                local_cumulative_uploads += get_uploads(f->file_hash);
         }
 
     for(uint32_t i=0;i<d.subdirs.size();++i) {
-        local_cumulative_size += recursUpdateCumulatedSize(d.subdirs[i]);
+        local_cumulative_size += recursUpdateCumulatedSize(d.subdirs[i], get_uploads);
         local_cumulative_files += static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_cumulated_files;
+        local_cumulative_uploads += static_cast<DirEntry*>(mNodes[d.subdirs[i]])->dir_cumulated_uploads;
     }
 
     d.dir_cumulated_size = local_cumulative_size;
     d.dir_cumulated_files = local_cumulative_files;
+    d.dir_cumulated_uploads = local_cumulative_uploads;
+
+
+
     return local_cumulative_size;
 }
 // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method.

src/file_sharing/dir_hierarchy.h

@@ -63,7 +63,7 @@ class InternalFileHierarchyStorage
     class DirEntry: public FileStorageNode
     {
     public:
-        explicit DirEntry(const std::string& name) : dir_name(name), dir_cumulated_size(0), dir_cumulated_files(0), dir_modtime(0),dir_most_recent_time(0),dir_update_time(0) {}
+        explicit DirEntry(const std::string& name) : dir_name(name), dir_cumulated_size(0), dir_cumulated_files(0), dir_cumulated_uploads(0), dir_modtime(0),dir_most_recent_time(0),dir_update_time(0) {}
         virtual ~DirEntry() {}
 
         virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
@@ -74,6 +74,7 @@ class InternalFileHierarchyStorage
         RsFileHash  dir_hash ;
         uint64_t    dir_cumulated_size;
         uint32_t    dir_cumulated_files;
+        uint64_t    dir_cumulated_uploads;
 
         std::vector<DirectoryStorage::EntryIndex> subdirs ;
         std::vector<DirectoryStorage::EntryIndex> subfiles ;
@@ -112,7 +113,7 @@ class InternalFileHierarchyStorage
 
     // Do a complete recursive sweep over sub-directories and files, and update the cumulative size.
 
-    uint64_t recursUpdateCumulatedSize(const DirectoryStorage::EntryIndex& dir_index);
+    uint64_t recursUpdateCumulatedSize(const DirectoryStorage::EntryIndex& dir_index, std::function<uint64_t(const RsFileHash&)> get_uploads = nullptr);
 
     // hash stuff
 

src/file_sharing/directory_storage.cc

@@ -249,6 +249,7 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
         d.hash.clear() ;
         d.size   = dir_entry->dir_cumulated_size;//dir_entry->subdirs.size() + dir_entry->subfiles.size();
         d.count  = dir_entry->dir_cumulated_files;
+        d.uploads = dir_entry->dir_cumulated_uploads;
         d.max_mtime = dir_entry->dir_most_recent_time ;
         d.mtime     = dir_entry->dir_modtime ;
         d.name    = dir_entry->dir_name;
@@ -260,6 +261,8 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
             d.type = DIR_TYPE_PERSON ;
             d.name = mPeerId.toStdString();
         }
+
+
     }
     else if(type == InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)
     {
@@ -299,13 +302,13 @@ bool DirectoryStorage::getIndexFromDirHash(const RsFileHash& hash,EntryIndex& in
     return mFileHierarchy->getIndexFromDirHash(hash,index) ;
 }
 
-void DirectoryStorage::checkSave()
+void DirectoryStorage::checkSave(std::function<uint64_t(const RsFileHash&)> get_uploads)
 {
     rstime_t now = time(NULL);
 
     if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now)
 	{
-        mFileHierarchy->recursUpdateCumulatedSize(mFileHierarchy->mRoot);
+        mFileHierarchy->recursUpdateCumulatedSize(mFileHierarchy->mRoot, get_uploads);
 
 	   {
 		  RS_STACK_MUTEX(mDirStorageMtx) ;

src/file_sharing/directory_storage.h

@@ -24,6 +24,7 @@
 #include <string>
 #include <stdint.h>
 #include <list>
+#include <functional>
 
 #include "retroshare/rsids.h"
 #include "retroshare/rsfiles.h"
@@ -150,7 +151,7 @@ class DirectoryStorage
 		 * \brief checkSave
 		 * 			Checks the time of last saving, last modification time, and saves if needed.
 		 */
-		void checkSave() ;
+		void checkSave(std::function<uint64_t(const RsFileHash&)> get_uploads = nullptr) ;
 
 		const std::string& filename() const { return mFileName ; }
 

src/file_sharing/p3filelists.cc

@@ -271,11 +271,11 @@ int p3FileDatabase::tick()
                   mRemoteDirectories[i]->lastSweepTime() = now ;
                }
 
-               mRemoteDirectories[i]->checkSave() ;
+               mRemoteDirectories[i]->checkSave([this](const RsFileHash& h){ return this->locked_getCumulativeUpload(h); }) ;
             }
 
         mLastRemoteDirSweepTS = now;
-		mLocalSharedDirs->checkSave() ;
+		mLocalSharedDirs->checkSave([this](const RsFileHash& h){ return this->locked_getCumulativeUpload(h); }) ;
 
         // This is a hack to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up,
         // so the first notify is ignored, and no other notify will happen next. We only do it if no data was received in the last 5 secs, in order to
@@ -1093,6 +1093,11 @@ int p3FileDatabase::getSharedDirStatistics(const RsPeerId& pid,SharedDirStats& s
 uint64_t p3FileDatabase::getCumulativeUpload(const RsFileHash& hash) const
 {
 	RS_STACK_MUTEX(mFLSMtx);
+	return locked_getCumulativeUpload(hash);
+}
+
+uint64_t p3FileDatabase::locked_getCumulativeUpload(const RsFileHash& hash) const
+{
 	auto it = mCumulativeUploaded.find(hash);
 	if (it != mCumulativeUploaded.end())
 		return it->second.total_bytes;
@@ -1439,6 +1444,13 @@ int p3FileDatabase::RequestDirDetails(
     }
 
     d.id = storage->peerId();
+ 
+    if (d.type == DIR_TYPE_FILE)
+    {
+        d.uploads = locked_getCumulativeUpload(d.hash);
+    }
+ 
+
 
 #ifdef DEBUG_FILE_HIERARCHY
     P3FILELISTS_DEBUG() << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl;

src/file_sharing/p3filelists.h

@@ -203,6 +203,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
 		bool hashingProcessPaused();
 
     protected:
+        uint64_t locked_getCumulativeUpload(const RsFileHash& hash) const;
         void getExtraFilesDirDetails_locked(void *ref,DirectoryStorage::EntryIndex e,DirDetails& d) const;
 
         int filterResults(const std::list<void*>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const;

src/retroshare/rstypes.h

@@ -280,7 +280,7 @@ struct DirStub : RsSerializable
 struct DirDetails : RsSerializable
 {
 	DirDetails() : parent(nullptr), prow(0), ref(nullptr),
-        type(DIR_TYPE_UNKNOWN), size(0), mtime(0), max_mtime(0), count(0) {}
+        type(DIR_TYPE_UNKNOWN), size(0), mtime(0), max_mtime(0), count(0), uploads(0) {}
 
 
 	/* G10h4ck do we still need to keep this as void* instead of uint64_t for
@@ -303,6 +303,7 @@ struct DirDetails : RsSerializable
 	FileStorageFlags flags;
 	uint32_t max_mtime ;	// maximum modification time of the whole hierarchy below.
 	uint32_t count;			// cumulative number of files in the directory hierarchy.
+	uint64_t uploads;		// cumulative number of bytes uploaded for this directory/file.
 
     std::vector<DirStub> children;
     std::list<RsNodeGroupId> parent_groups;	// parent groups for the shared directory
@@ -333,6 +334,7 @@ struct DirDetails : RsSerializable
 		RS_SERIAL_PROCESS(flags);
 		RS_SERIAL_PROCESS(max_mtime);
 		RS_SERIAL_PROCESS(count);
+		RS_SERIAL_PROCESS(uploads);
 		RS_SERIAL_PROCESS(children);
 		RS_SERIAL_PROCESS(parent_groups);
 	}