Add pcap subfolder support

frontend/.dockerignore

@@ -1 +1,3 @@
-node_modules
+node_modules
+dist
+Dockerfile-frontend

frontend/Dockerfile-frontend

@@ -1,4 +1,4 @@
-FROM node:17
+FROM node:17 AS build
 
 COPY package.json /app/
 COPY yarn.lock /app/
@@ -11,6 +11,9 @@ COPY . /app/
 
 RUN yarn run build
 
-EXPOSE 3000
+FROM nginx
+
+COPY --from=build /app/dist /usr/share/nginx/html
+COPY nginx.conf /etc/nginx/conf.d/default.conf
 
-CMD ["yarn", "run", "preview", "--host", "--port", "3000"]
+EXPOSE 3000

frontend/nginx.conf

@@ -0,0 +1,45 @@
+server {
+	listen 3000;
+
+	keepalive_timeout 1d;
+	send_timeout 1d;
+	client_body_timeout 1d;
+	client_header_timeout 1d;
+	proxy_connect_timeout 1d;
+	proxy_read_timeout 1d;
+	proxy_send_timeout 1d;
+	fastcgi_connect_timeout 1d;
+	fastcgi_read_timeout 1d;
+	fastcgi_send_timeout 1d;
+	memcached_connect_timeout 1d;
+	memcached_read_timeout 1d;
+	memcached_send_timeout 1d;
+
+	gzip on;
+	gzip_types text/html application/javascript application/json text/css;
+
+	# where the root here
+	root /usr/share/nginx/html;
+	# what file to server as index
+	index index.html;
+
+	location /api/ {
+		proxy_pass http://api:5000/;
+	}
+
+	location / {
+		# First attempt to serve request as file, then
+		# as directory, then fall back to redirecting to index.html
+		try_files $uri $uri/ $uri.html /index.html;
+	}
+
+	location ~* \.(?:css|js|jpg|svg)$ {
+		expires 30d;
+		add_header Cache-Control "public";
+	}
+
+	location ~* \.(?:json)$ {
+		expires 1d;
+		add_header Cache-Control "public";
+	}
+}

frontend/src/components/FlowList.tsx

@@ -188,8 +188,7 @@ export function FlowList() {
         }
       }
     }
-  }
-  );
+  });
   useHotkeys('k', () => setFlowIndex(fi => Math.max(0, fi - 1)));
   useHotkeys('i', () => {
     setShowFilters(true)

frontend/src/components/Header.tsx

@@ -250,8 +250,8 @@ export function Header() {
   let [searchParams] = useSearchParams();
 
   let navigate = useNavigate();
-
-  useHotkeys('g', () => navigate(`/corrie?${searchParams}`, { replace: true }))
+ 
+  useHotkeys('g', () => navigate(`/corrie?${searchParams}`, { replace: true }));
   useHotkeys('a', () => setToLastnTicks(5));
   useHotkeys('c', () => {
     (document.getElementById("startdateselection") as HTMLInputElement).value = "";

services/api/wsgi.py

@@ -0,0 +1,4 @@
+from webservice import application
+
+if __name__ == "__main__":
+    application.run(host='0.0.0.0', threaded=True)

services/go-importer/cmd/assembler/main.go

@@ -467,29 +467,57 @@ func connectToPCAPOverIP(service *AssemblerService, pcapIP string) {
 	}
 }
 
-func (service *AssemblerService) WatchDir(watch_dir string) {
-	stat, err := os.Stat(watch_dir)
+func (service AssemblerService) recursiveHandleExistingFiles(directory string) {
+	log.Println("Handling already existing files in ", directory)
+	files, err := ioutil.ReadDir(directory)
 	if err != nil {
-		log.Fatal("Failed to open the watch_dir with error: ", err)
+		log.Fatal(err)
 	}
 
-	if !stat.IsDir() {
-		log.Fatal("watch_dir is not a directory")
+	for _, file := range files {
+		if file.IsDir() {
+			service.recursiveHandleExistingFiles(filepath.Join(directory, file.Name()))
+			continue
+		}
+		// accepts files with prefixes that start with .pcap (.pcapng .pcap1 etc)
+		if strings.HasPrefix(filepath.Ext(file.Name()), ".pcap") {
+			service.HandlePcapUri(filepath.Join(directory, file.Name())) //FIXME; this is a little clunky
+		}
 	}
+}
 
-	log.Println("Monitoring dir: ", watch_dir)
+func (service AssemblerService) recursiveWatchDir(watcher *fsnotify.Watcher, directory string) {
+	log.Println("Watching inode events for ", directory)
+	err := watcher.Add(directory)
+	if err != nil {
+		log.Fatal(err)
+	}
 
-	files, err := ioutil.ReadDir(watch_dir)
+	files, err := ioutil.ReadDir(directory)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	for _, file := range files {
-		// accepts files with prefixes that start with .pcap (.pcapng .pcap1 etc)
-		if strings.HasPrefix(filepath.Ext(file.Name()), ".pcap") {
-			service.HandlePcapUri(filepath.Join(watch_dir, file.Name())) //FIXME; this is a little clunky
+		if file.IsDir() {
+			service.recursiveWatchDir(watcher, filepath.Join(directory, file.Name()))
 		}
 	}
+}
+
+func (service AssemblerService) WatchDir(watch_dir string) {
+	stat, err := os.Stat(watch_dir)
+	if err != nil {
+		log.Fatal("Failed to open the watch_dir with error: ", err)
+	}
+
+	if !stat.IsDir() {
+		log.Fatal("watch_dir is not a directory")
+	}
+
+	log.Println("Monitoring dir: ", watch_dir)
+
+	service.recursiveHandleExistingFiles(watch_dir)
 
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
@@ -514,6 +542,21 @@ func (service *AssemblerService) WatchDir(watch_dir string) {
 						log.Println("Found new file", event.Name, event.Op.String())
 						time.Sleep(2 * time.Second) // FIXME; bit of race here between file creation and writes.
 						service.HandlePcapUri(event.Name)
+					} else {
+						// test if the file is a directory
+						stat, err := os.Stat(event.Name)
+						if err != nil {
+							log.Fatal("Failed to open the watch_dir with error: ", err)
+						} else if stat.IsDir() {
+							if event.Op&fsnotify.Rename != 0 {
+								watcher.Remove(event.Name)
+								log.Println("Removed watch for ", event.Name)
+							} else if event.Op&fsnotify.Create != 0 {
+								watcher.Add(event.Name)
+								log.Println("Added watch for ", event.Name)
+								service.recursiveHandleExistingFiles(event.Name)
+							}
+						}
 					}
 				}
 			case err, ok := <-watcher.Errors:
@@ -525,10 +568,7 @@ func (service *AssemblerService) WatchDir(watch_dir string) {
 		}
 	}()
 
-	err = watcher.Add(watch_dir)
-	if err != nil {
-		log.Fatal(err)
-	}
+	service.recursiveWatchDir(watcher, watch_dir)
 	<-signalChan
 	log.Println("Watcher stopped")
 

services/timescale/Dockerfile

@@ -1,6 +1,6 @@
 FROM timescale/timescaledb:latest-pg15
 
-RUN apk add build-base make clang15 llvm15 git
+RUN apk add build-base make clang19 llvm19 git
 COPY tulip /tulip
 RUN cd /tulip && make USE_PGXS=1 install && cd / && \
 	git clone https://github.com/ossc-db/pg_hint_plan --branch PG15 && \