consolidate into single db

Author: TheTechromancer

bbot_server/applets/base.py

@@ -223,7 +223,9 @@ async def _native_setup(self):
                     self.collection = self.parent.collection
                     self.strict_collection = self.parent.strict_collection
                 else:
-                    self.collection = self.db[self.table_name]
+                    store = getattr(self, f"{self.store_type}_store")
+                    prefix = getattr(store, "collection_prefix", "")
+                    self.collection = self.db[f"{prefix}{self.table_name}"]
                     # WriteConcern options:
                     #  w=1: Acknowledges the write operation only after it has been written to the primary. (the default)
                     #  j=True: Ensures the write operation is committed to the journal. (default is False)

bbot_server/config.py

@@ -42,6 +42,7 @@
 
 class StoreConfig(BaseModel):
     uri: str
+    collection_prefix: str = ""
 
 
 class MessageQueueConfig(BaseModel):

bbot_server/defaults.yml

@@ -10,16 +10,19 @@ api_key: null
 
 # event store is just a big bucket of BBOT scan events that act as a read-only time machine
 event_store:
-  uri: mongodb://localhost:27017/bbot_eventstore
+  uri: mongodb://localhost:27017/bbot
+  collection_prefix: "eventstore__"
 
 # assets are derived from the event store, and can be recreated at any time
 asset_store:
-  uri: mongodb://localhost:27017/bbot_assetstore
+  uri: mongodb://localhost:27017/bbot
+  collection_prefix: "assetstore__"
 
 # user_store holds any user-specific data, overrides, etc. which are not derived from events
 # examples include targets, scans, etc.
 user_store:
-  uri: mongodb://localhost:27017/bbot_userstore
+  uri: mongodb://localhost:27017/bbot
+  collection_prefix: "userstore__"
 
 message_queue:
   uri: redis://localhost:6379/0

bbot_server/defaults_docker.yml

@@ -3,16 +3,19 @@ url: http://server:8807/v1/
 
 # event store is just a big bucket of BBOT scan events
 event_store:
-  uri: mongodb://mongodb:27017/bbot_eventstore
+  uri: mongodb://mongodb:27017/bbot
+  collection_prefix: "eventstore__"
 
 # assets are derived in real time from the event store, and can be recreated at any time
 asset_store:
-  uri: mongodb://mongodb:27017/bbot_assetstore
+  uri: mongodb://mongodb:27017/bbot
+  collection_prefix: "assetstore__"
 
 # user_store holds any user-specific data, overrides, etc. which are not derived from events
 # examples include targets, scans, etc.
 user_store:
-  uri: mongodb://mongodb:27017/bbot_userstore
+  uri: mongodb://mongodb:27017/bbot
+  collection_prefix: "userstore__"
 
 message_queue:
   uri: redis://redis:6379/0

bbot_server/modules/server/server_cli.py

@@ -128,49 +128,34 @@ def logs(
     @subcommand(help="Clear the database (drop Mongodb collections).")
     def cleardb(
         self,
-        event_store: Annotated[bool, Option("--event-store", "-e", help="Clear the event store database")] = False,
-        asset_store: Annotated[bool, Option("--asset-store", "-a", help="Clear the asset store database")] = False,
-        user_store: Annotated[bool, Option("--user-store", "-u", help="Clear the user store database")] = False,
+        event_store: Annotated[bool, Option("--event-store", "-e", help="Clear the event store collections")] = False,
+        asset_store: Annotated[bool, Option("--asset-store", "-a", help="Clear the asset store collections")] = False,
+        user_store: Annotated[bool, Option("--user-store", "-u", help="Clear the user store collections")] = False,
     ):
         if not event_store and not asset_store and not user_store:
-            raise self.BBOTServerError(f"Must specify at least one database to clear")
+            raise self.BBOTServerError(f"Must specify at least one store to clear")
 
+        stores_to_clear = []
         if event_store:
-            event_store_db = self.config.event_store.uri.split("/")[-1]
-            if not event_store_db:
-                raise self.BBOTServerError("Event store database not found in config")
-            response = input(
-                f"Are you sure you want to clear the event store database: {event_store_db}? This will permanently delete all BBOT scan events! (y/N) "
-            )
-            if response.lower() != "y":
-                raise self.BBOTServerError("Aborting")
-
-            self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", "db.dropDatabase()", event_store_db])
-            self.log.info(f"Successfully cleared event store database: {event_store_db}")
-
+            stores_to_clear.append(("event store", self.config.event_store, "all BBOT scan events"))
         if asset_store:
-            asset_store_db = self.config.asset_store.uri.split("/")[-1]
-            if not asset_store_db:
-                raise self.BBOTServerError("Asset store database not found in config")
-            response = input(
-                f"Are you sure you want to clear the asset store database: {asset_store_db}? This will permanently delete all BBOT asset data! (y/N) "
-            )
-            if response.lower() != "y":
-                raise self.BBOTServerError("Aborting")
-            self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", "db.dropDatabase()", asset_store_db])
-            self.log.info(f"Successfully cleared asset store database: {asset_store_db}")
-
+            stores_to_clear.append(("asset store", self.config.asset_store, "all BBOT asset data"))
         if user_store:
-            user_store_db = self.config.user_store.uri.split("/")[-1]
-            if not user_store_db:
-                raise self.BBOTServerError("User store database not found in config")
+            stores_to_clear.append(("user store", self.config.user_store, "all BBOT user data, including presets and targets"))
+
+        for store_name, store_config, data_desc in stores_to_clear:
+            db_name = store_config.uri.split("/")[-1]
+            prefix = store_config.collection_prefix
+            if not db_name:
+                raise self.BBOTServerError(f"{store_name.title()} database not found in config")
             response = input(
-                f"Are you sure you want to clear the user store database: {user_store_db}? This will permanently delete all BBOT user data, including presets and targets! (y/N) "
+                f"Are you sure you want to clear the {store_name} (prefix: {prefix})? This will permanently delete {data_desc}! (y/N) "
             )
             if response.lower() != "y":
                 raise self.BBOTServerError("Aborting")
-            self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", "db.dropDatabase()", user_store_db])
-            self.log.info(f"Successfully cleared user store database: {user_store_db}")
+            drop_eval = f"db.getCollectionNames().filter(c => c.startsWith('{prefix}')).forEach(c => db[c].drop())"
+            self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", drop_eval, db_name])
+            self.log.info(f"Successfully cleared {store_name} collections (prefix: {prefix})")
 
     def _run_docker_compose(self, args, **kwargs):
         kwargs["cwd"] = self.docker_compose_dir

bbot_server/store.py

@@ -8,7 +8,9 @@ class BaseMongoStore(BaseDB):
     async def setup(self):
         self.client = AsyncMongoClient(self.uri)
         self.db = self.client.get_database(self.db_name)
-        self.fs = AsyncGridFSBucket(self.db)
+        self.collection_prefix = getattr(self.db_config, "collection_prefix", "")
+        bucket_name = f"{self.collection_prefix}fs" if self.collection_prefix else "fs"
+        self.fs = AsyncGridFSBucket(self.db, bucket_name=bucket_name)
 
     async def cleanup(self):
         await self.client.close()

helm/templates/server-deployment.yaml

@@ -59,11 +59,17 @@ spec:
               key: api-key
         # MongoDB URIs (credentials injected via $(VAR) interpolation)
         - name: BBOT_SERVER_EVENT_STORE__URI
-          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_eventstore?authSource=admin"
+          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
+        - name: BBOT_SERVER_EVENT_STORE__COLLECTION_PREFIX
+          value: "eventstore__"
         - name: BBOT_SERVER_ASSET_STORE__URI
-          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_assetstore?authSource=admin"
+          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
+        - name: BBOT_SERVER_ASSET_STORE__COLLECTION_PREFIX
+          value: "assetstore__"
         - name: BBOT_SERVER_USER_STORE__URI
-          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_userstore?authSource=admin"
+          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
+        - name: BBOT_SERVER_USER_STORE__COLLECTION_PREFIX
+          value: "userstore__"
         # Redis URI
         - name: BBOT_SERVER_MESSAGE_QUEUE__URI
           value: "redis://:$(REDIS_PASSWORD)@{{ .Release.Name }}-redis-master:6379/0"

helm/templates/watchdog-deployment.yaml

@@ -72,11 +72,17 @@ spec:
               key: api-key
         # MongoDB URIs (credentials injected via $(VAR) interpolation)
         - name: BBOT_SERVER_EVENT_STORE__URI
-          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_eventstore?authSource=admin"
+          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
+        - name: BBOT_SERVER_EVENT_STORE__COLLECTION_PREFIX
+          value: "eventstore__"
         - name: BBOT_SERVER_ASSET_STORE__URI
-          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_assetstore?authSource=admin"
+          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
+        - name: BBOT_SERVER_ASSET_STORE__COLLECTION_PREFIX
+          value: "assetstore__"
         - name: BBOT_SERVER_USER_STORE__URI
-          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_userstore?authSource=admin"
+          value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
+        - name: BBOT_SERVER_USER_STORE__COLLECTION_PREFIX
+          value: "userstore__"
         # Redis URI
         - name: BBOT_SERVER_MESSAGE_QUEUE__URI
           value: "redis://:$(REDIS_PASSWORD)@{{ .Release.Name }}-redis-master:6379/0"

tests/conftest.py

@@ -41,7 +41,7 @@
 
 bbcfg.refresh(config_path=str(TEST_CONFIG_PATH))
 
-assert bbcfg.asset_store.uri == "mongodb://localhost:27017/test_bbot_server_assets"
+assert bbcfg.asset_store.uri == "mongodb://localhost:27017/test_bbot"
 
 if not bbcfg.get_api_keys():
     # create a new api key if we don't have one yet
@@ -263,9 +263,7 @@ async def mongo_cleanup():
     client = AsyncMongoClient(bbcfg.event_store.uri)
 
     async def clear_everything():
-        await client.drop_database("test_bbot_server_events")
-        await client.drop_database("test_bbot_server_assets")
-        await client.drop_database("test_bbot_server_userdata")
+        await client.drop_database("test_bbot")
 
     try:
         # Clear before test

tests/test_cli/test_cli_basic.py

@@ -32,7 +32,7 @@ def test_cli_config():
     result = subprocess.run(BBCTL_COMMAND + ["--current-config"], capture_output=True, text=True)
     assert result.returncode == 0
     config = yaml.safe_load(result.stdout)
-    assert config["event_store"]["uri"] == "mongodb://localhost:27017/test_bbot_server_events"
+    assert config["event_store"]["uri"] == "mongodb://localhost:27017/test_bbot"
 
     yaml_config_str = """
 event_store: