add unique CID file export function to monitor (#51)

ihlec · web-flow · commit e4a5fd4d6340 · 2026-03-13T12:17:52.000+01:00
* re-run bootstrap if all nodes are lost

* fixed inconsitent docu

* removed compose v2 syntax

* dlockss monitor to allow export of unique CID list

* ci fix
diff --git a/README.md b/README.md
@@ -99,7 +99,7 @@ services:
       # DLOCKSS_NODE_NAME: my-node               # human-readable name shown in the monitor;
       #                                          # if empty the peer ID is displayed instead
     volumes:
-      - dlockss-data:/data                       # persistent D-LOCKSS data (identity, cluster state, ingested files)
+      - ./dlockss-files:/data                    # persistent D-LOCKSS data (identity, cluster state, ingested files)
       - ipfs-data:/ipfs-repo:ro                  # read-only access to Kubo config for identity
     depends_on:
       - ipfs
@@ -138,7 +138,6 @@ services:
 volumes:
   ipfs-staging:  # IPFS staging area on /export
   ipfs-data:     # IPFS repo on /data/ipfs (shared read-only with D-LOCKSS for identity)
-  dlockss-data:  # persistent D-LOCKSS data (identity key, cluster state, ingested files)
 ```
 
 See [docs/DLOCKSS_PROTOCOL.md](docs/DLOCKSS_PROTOCOL.md) for protocol details.
diff --git a/cmd/dlockss-monitor/main.go b/cmd/dlockss-monitor/main.go
@@ -44,6 +44,10 @@ func main() {
 		cfg.PubsubTopicPrefix = v
 		slog.Info("pubsub topic prefix from env", "prefix", cfg.PubsubTopicPrefix)
 	}
+	if v := os.Getenv("DLOCKSS_TOPIC_NAME"); v != "" {
+		cfg.TopicName = v
+		slog.Info("topic name from env", "topic", cfg.TopicName)
+	}
 
 	geoDBPath := *geoipDB
 	if geoDBPath == "" {
diff --git a/internal/keywords/keywords.go b/internal/keywords/keywords.go
@@ -827,6 +827,20 @@ func (s *Store) GetRecentSearches() []RecentSearch {
 	return result
 }
 
+// PayloadCIDs returns all known payload CIDs from the keyword index.
+func (s *Store) PayloadCIDs() []string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	out := make([]string, 0, len(s.cidKeywords))
+	for _, entry := range s.cidKeywords {
+		if entry.PayloadCID != "" {
+			out = append(out, entry.PayloadCID)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
 // GetStats returns keyword extraction progress statistics.
 func (s *Store) GetStats(totalUniqueCIDs int) Stats {
 	s.mu.RLock()
diff --git a/internal/monitor/monitor_models.go b/internal/monitor/monitor_models.go
@@ -39,15 +39,19 @@ type MonitorConfig struct {
 	NodeCleanupTimeout  time.Duration
 	BootstrapShardDepth int
 	PubsubTopicPrefix   string
+	TopicName           string
 }
 
 const DefaultPubsubTopicPrefix = "dlockss-v0.0.3"
 
+const DefaultTopicName = "creative-commons"
+
 func DefaultMonitorConfig() MonitorConfig {
 	return MonitorConfig{
 		NodeCleanupTimeout:  DefaultNodeCleanupTimeout,
 		BootstrapShardDepth: DefaultBootstrapShardDepth,
 		PubsubTopicPrefix:   DefaultPubsubTopicPrefix,
+		TopicName:           DefaultTopicName,
 	}
 }
 
@@ -62,7 +66,7 @@ type NodeState struct {
 	PeerID         string              `json:"peer_id"`
 	NodeName       string              `json:"node_name,omitempty"`
 	CurrentShard   string              `json:"current_shard"`
-	Role           string              `json:"role,omitempty"` // ACTIVE, PASSIVE, or PROBE (empty = ACTIVE)
+	Role           string              `json:"role,omitempty"` // ACTIVE, PASSIVE, REPLICATOR, or PROBE (empty = ACTIVE)
 	PinnedFiles    int                 `json:"pinned_files"`
 	KnownFiles     int                 `json:"known_files"`
 	LastSeen       time.Time           `json:"last_seen"`
diff --git a/internal/monitor/monitor_routes.go b/internal/monitor/monitor_routes.go
@@ -29,6 +29,7 @@ func (m *Monitor) RegisterRoutes(mux *http.ServeMux) {
 	mux.HandleFunc("/api/root-topic", m.handleRootTopic)
 	mux.HandleFunc("/api/node-files", m.handleNodeFiles)
 	mux.HandleFunc("/api/unique-cids", m.handleUniqueCIDs)
+	mux.HandleFunc("/api/payload-cids-export", m.handlePayloadCIDsExport)
 	mux.HandleFunc("/api/replication", m.handleReplication)
 	mux.HandleFunc("/api/replication-cids", m.handleReplicationCIDs)
 	mux.HandleFunc("/api/manifest-payload", m.handleManifestPayload)
@@ -172,11 +173,11 @@ func (m *Monitor) handleRootTopic(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		m.SwitchTopicPrefix(r.Context(), body.TopicPrefix)
-		rootTopic := fmt.Sprintf("%s-creative-commons-shard-", m.getTopicPrefix())
+		rootTopic := fmt.Sprintf("%s-%s-shard-", m.getTopicPrefix(), m.cfg.TopicName)
 		json.NewEncoder(w).Encode(map[string]string{"root_topic": rootTopic, "topic_prefix": m.getTopicPrefix()})
 		return
 	}
-	rootTopic := fmt.Sprintf("%s-creative-commons-shard-", m.getTopicPrefix())
+	rootTopic := fmt.Sprintf("%s-%s-shard-", m.getTopicPrefix(), m.cfg.TopicName)
 	json.NewEncoder(w).Encode(map[string]string{"root_topic": rootTopic, "topic_prefix": m.getTopicPrefix()})
 }
 
@@ -206,6 +207,16 @@ func (m *Monitor) handleUniqueCIDs(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(map[string]interface{}{"cids": entries, "count": len(entries)})
 }
 
+func (m *Monitor) handlePayloadCIDsExport(w http.ResponseWriter, r *http.Request) {
+	cids := m.keywords.PayloadCIDs()
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("Content-Disposition", "attachment; filename=\"payload_cids.txt\"")
+	for _, c := range cids {
+		w.Write([]byte(c))
+		w.Write([]byte("\n"))
+	}
+}
+
 func (m *Monitor) handleReplication(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodGet {
 		writeJSONError(w, "method not allowed", http.StatusMethodNotAllowed)
diff --git a/internal/monitor/monitor_subscription.go b/internal/monitor/monitor_subscription.go
@@ -73,7 +73,7 @@ func (m *Monitor) ensureShardSubscriptionUnlocked(ctx context.Context, shardID s
 	if _, exists := m.shardTopics[shardID]; exists {
 		return
 	}
-	topicName := fmt.Sprintf("%s-creative-commons-shard-%s", m.getTopicPrefixUnlocked(), shardID)
+	topicName := fmt.Sprintf("%s-%s-shard-%s", m.getTopicPrefixUnlocked(), m.cfg.TopicName, shardID)
 	topic, err := m.ps.Join(topicName)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || strings.Contains(err.Error(), "context canceled") {
@@ -134,6 +134,8 @@ func (m *Monitor) handleShardMessages(ctx context.Context, sub *pubsub.Subscript
 						role = "PASSIVE"
 					case "PROBE":
 						role = "PROBE"
+					case "REPLICATOR":
+						role = "REPLICATOR"
 					}
 				}
 				nodeName := ""
@@ -189,8 +191,13 @@ func (m *Monitor) handleShardMessages(ctx context.Context, sub *pubsub.Subscript
 				parts := strings.SplitN(string(msg.Data[5:]), ":", 3)
 				peerIDStr := strings.TrimSpace(parts[0])
 				role := "ACTIVE"
-				if len(parts) >= 2 && strings.ToUpper(parts[1]) == "PASSIVE" {
-					role = "PASSIVE"
+				if len(parts) >= 2 {
+					switch strings.ToUpper(parts[1]) {
+					case "PASSIVE":
+						role = "PASSIVE"
+					case "REPLICATOR":
+						role = "REPLICATOR"
+					}
 				}
 				nodeName := ""
 				if len(parts) >= 3 {
diff --git a/internal/monitor/static/dashboard.html b/internal/monitor/static/dashboard.html
@@ -50,9 +50,11 @@
         .modal { background: white; border: 2px solid #333; max-width: 90vw; max-height: 85vh; overflow: hidden; display: flex; flex-direction: column; }
         .modal-header { padding: 15px 20px; border-bottom: 1px solid #333; display: flex; justify-content: space-between; align-items: center; }
         .modal-body { padding: 20px; overflow-y: auto; flex: 1; }
-        .cid-row { display: flex; align-items: center; gap: 12px; padding: 8px 0; border-bottom: 1px solid #eee; font-size: 0.85em; flex-wrap: wrap; }
-        .cid-row .cid-value { font-family: monospace; word-break: break-all; flex: 1; min-width: 200px; }
-        .cid-row .cid-meta { color: #666; font-size: 0.9em; }
+        .cid-row { display: flex; align-items: center; gap: 12px; padding: 8px 0; border-bottom: 1px solid #eee; font-size: 0.85em; flex-wrap: nowrap; }
+        .cid-row .cid-value { font-family: monospace; white-space: nowrap; flex: 1; min-width: 200px; cursor: pointer; }
+        .cid-row .cid-value:hover { color: #06A77D; }
+        .cid-row .cid-meta { color: #666; font-size: 0.9em; white-space: nowrap; }
+        .copy-toast { position: fixed; bottom: 30px; left: 50%; transform: translateX(-50%); background: #333; color: #fff; padding: 8px 20px; font-size: 0.85em; z-index: 2000; opacity: 0; transition: opacity 0.3s; pointer-events: none; }
         .cid-row .cid-actions { display: flex; gap: 6px; flex-shrink: 0; }
         .cid-payload { margin-top: 4px; font-size: 0.8em; color: #06A77D; }
         .load-spinner { display: inline-block; width: 14px; height: 14px; border: 2px solid #ccc; border-top-color: #06A77D; border-radius: 50%; animation: spin 0.8s linear infinite; vertical-align: middle; }
@@ -147,9 +149,10 @@ <h3 style="margin:0; text-transform:uppercase; font-size:1em;">Network Nodes</h3
             <table id="nodeTable"><thead><tr><th style="width: 80px;">Action</th><th>Peer ID</th><th>Shard</th><th>Peers</th><th>Pinned</th><th>Known</th><th>Uptime</th><th>Last Seen</th></tr></thead><tbody id="nodeTableBody"></tbody></table>
         </div>
     </div>
+    <div id="copy-toast" class="copy-toast">Copied to clipboard</div>
     <div id="cids-modal" class="modal-overlay" style="display: none;">
-        <div class="modal" style="width: 700px;">
-            <div class="modal-header"><h3 style="margin:0; text-transform: uppercase; font-size: 1em;">Unique CIDs</h3><input type="text" id="cids-modal-search" placeholder="Filter CIDs..." style="padding: 6px; font-family: inherit; font-size: 0.85em; width: 180px; border: 1px solid #333;"><button class="btn-text" id="cids-modal-close">CLOSE</button></div>
+        <div class="modal" style="width: 1100px;">
+            <div class="modal-header"><h3 style="margin:0; text-transform: uppercase; font-size: 1em;">Unique CIDs</h3><div style="display:flex; gap:8px; align-items:center;"><input type="text" id="cids-modal-search" placeholder="Filter CIDs..." style="padding: 6px; font-family: inherit; font-size: 0.85em; width: 180px; border: 1px solid #333;"><button class="btn-text" id="cids-export-btn" title="Export payload CIDs as text file">EXPORT PAYLOAD CIDS</button><button class="btn-text" id="cids-modal-close">CLOSE</button></div></div>
             <div class="modal-body"><div id="cids-modal-list"></div></div>
         </div>
     </div>
@@ -273,18 +276,29 @@ <h3 style="margin:0; text-transform:uppercase; font-size:1em;">Network Nodes</h3
         document.getElementById('root-topic-cancel-btn').onclick = hideTopicEdit;
         document.getElementById('root-topic-input').onkeydown = function(e) { if (e.key === 'Enter') saveTopicPrefix(); else if (e.key === 'Escape') hideTopicEdit(); };
         const GATEWAY = 'https://ipfs.io';
+        function showCopyToast() {
+            const toast = document.getElementById('copy-toast');
+            toast.style.opacity = '1';
+            setTimeout(function() { toast.style.opacity = '0'; }, 1200);
+        }
+        function copyCID(text) {
+            navigator.clipboard.writeText(text).then(showCopyToast).catch(function() {});
+        }
         function showCidsModal() {
             document.getElementById('cids-modal').style.display = 'flex';
             fetch('/api/unique-cids?t=' + Date.now()).then(r=>r.json()).then(data => {
                 const list = document.getElementById('cids-modal-list');
                 const cids = data.cids || [];
                 list.innerHTML = cids.length === 0 ? '<p style="color:#666;">No CIDs yet. CIDs appear when nodes announce pinned files.</p>' : cids.map((entry, i) => {
                     const cid = entry.cid || entry.CID || entry; const shard = (entry.shard || entry.Shard || '') === '' ? 'root' : (entry.shard || entry.Shard); const replicas = entry.replicas ?? entry.Replicas ?? 0;
-                    return '<div class="cid-row" data-cid="' + escapeHtml(cid) + '"><div class="cid-value">' + escapeHtml(cid) + '</div><div class="cid-meta">Shard: ' + escapeHtml(shard) + ' | Replicas: ' + replicas + '</div><div class="cid-actions"><button class="btn-text" onclick="loadManifest(\'' + escapeJs(cid) + '\', this.parentElement)">LOAD MANIFEST</button></div></div>';
+                    return '<div class="cid-row" data-cid="' + escapeHtml(cid) + '"><div class="cid-value" onclick="copyCID(\'' + escapeJs(cid) + '\')" title="Click to copy">' + escapeHtml(cid) + '</div><div class="cid-meta">Shard: ' + escapeHtml(shard) + ' | Replicas: ' + replicas + '</div><div class="cid-actions"><button class="btn-text" onclick="loadManifest(\'' + escapeJs(cid) + '\', this.parentElement)">LOAD MANIFEST</button></div></div>';
                 }).join('');
             }).catch(() => { document.getElementById('cids-modal-list').innerHTML = '<p style="color:#999;">Failed to load CIDs.</p>'; });
         }
         function hideCidsModal() { document.getElementById('cids-modal').style.display = 'none'; }
+        function exportPayloadCIDs() {
+            window.open('/api/payload-cids-export', '_blank');
+        }
         function showNodeFilesModal(peerId) {
             const aliases = loadAliases();
             const label = aliases[peerId] || (window._nodeNames && window._nodeNames[peerId]) || peerId.slice(-6);
@@ -296,7 +310,7 @@ <h3 style="margin:0; text-transform:uppercase; font-size:1em;">Network Nodes</h3
                 const cids = data.cids || [];
                 list.innerHTML = cids.length === 0 ? '<p style="color:#666;">No pinned files for this node.</p>' : cids.map((entry, i) => {
                     const cid = entry.cid || entry.CID || entry; const shard = (entry.shard || entry.Shard || '') === '' ? 'root' : (entry.shard || entry.Shard); const replicas = entry.replicas ?? entry.Replicas ?? 0;
-                    return '<div class="cid-row" data-cid="' + escapeHtml(cid) + '"><div class="cid-value">' + escapeHtml(cid) + '</div><div class="cid-meta">Shard: ' + escapeHtml(shard) + ' | Replicas: ' + replicas + '</div><div class="cid-actions"><button class="btn-text" onclick="loadManifest(\'' + escapeJs(cid) + '\', this.parentElement)">LOAD MANIFEST</button></div></div>';
+                    return '<div class="cid-row" data-cid="' + escapeHtml(cid) + '"><div class="cid-value" onclick="copyCID(\'' + escapeJs(cid) + '\')" title="Click to copy">' + escapeHtml(cid) + '</div><div class="cid-meta">Shard: ' + escapeHtml(shard) + ' | Replicas: ' + replicas + '</div><div class="cid-actions"><button class="btn-text" onclick="loadManifest(\'' + escapeJs(cid) + '\', this.parentElement)">LOAD MANIFEST</button></div></div>';
                 }).join('');
             }).catch(() => { document.getElementById('node-files-modal-list').innerHTML = '<p style="color:#999;">Failed to load node files.</p>'; });
         }
@@ -311,7 +325,7 @@ <h3 style="margin:0; text-transform:uppercase; font-size:1em;">Network Nodes</h3
                 const cids = data.cids || [];
                 list.innerHTML = cids.length === 0 ? '<p style="color:#666;">No files at this replication level.</p>' : cids.map((entry, i) => {
                     const cid = entry.cid || entry.CID || entry; const shard = (entry.shard || entry.Shard || '') === '' ? 'root' : (entry.shard || entry.Shard); const replicas = entry.replicas ?? entry.Replicas ?? 0;
-                    return '<div class="cid-row" data-cid="' + escapeHtml(cid) + '"><div class="cid-value">' + escapeHtml(cid) + '</div><div class="cid-meta">Shard: ' + escapeHtml(shard) + ' | Replicas: ' + replicas + '</div><div class="cid-actions"><button class="btn-text" onclick="loadManifest(\'' + escapeJs(cid) + '\', this.parentElement)">LOAD MANIFEST</button></div></div>';
+                    return '<div class="cid-row" data-cid="' + escapeHtml(cid) + '"><div class="cid-value" onclick="copyCID(\'' + escapeJs(cid) + '\')" title="Click to copy">' + escapeHtml(cid) + '</div><div class="cid-meta">Shard: ' + escapeHtml(shard) + ' | Replicas: ' + replicas + '</div><div class="cid-actions"><button class="btn-text" onclick="loadManifest(\'' + escapeJs(cid) + '\', this.parentElement)">LOAD MANIFEST</button></div></div>';
                 }).join('');
             }).catch(() => { document.getElementById('replication-modal-list').innerHTML = '<p style="color:#999;">Failed to load CIDs.</p>'; });
         }
@@ -398,6 +412,7 @@ <h3 style="margin:0; text-transform:uppercase; font-size:1em;">Network Nodes</h3
                 alert('Failed to load manifest');
             });
         }
+        document.getElementById('cids-export-btn').onclick = exportPayloadCIDs;
         document.getElementById('unique-cids-card').onclick = showCidsModal;
         document.getElementById('total-nodes-card').onclick = function() { document.getElementById('network-nodes-section').scrollIntoView({ behavior: 'smooth' }); };
         document.getElementById('total-shards-card').onclick = function() { document.getElementById('shard-topology-section').scrollIntoView({ behavior: 'smooth' }); };
