Skip to content

Commit 4ce6960

Browse files
TheTechromancerTheTechromancer
authored
Merge pull request #132 from blacklanternsecurity/consolidate-dbs
Consolidate Into Single DB
2 parents f529297 + fdc0a6e commit 4ce6960

17 files changed

Lines changed: 79 additions & 64 deletions

‎.github/workflows/helm-tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ jobs:
3636
uses: medyagh/setup-minikube@latest
3737

3838
- name: Run helm deployment tests
39-
run: python test_helm_deployment.py
39+
run: python helm_deployment_test.py
4040

4141
- name: Cleanup minikube
4242
if: always()

‎bbot_server/applets/base.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,9 @@ async def _native_setup(self):
223223
self.collection = self.parent.collection
224224
self.strict_collection = self.parent.strict_collection
225225
else:
226-
self.collection = self.db[self.table_name]
226+
store = getattr(self, f"{self.store_type}_store")
227+
prefix = getattr(store, "collection_prefix", "")
228+
self.collection = self.db[f"{prefix}{self.table_name}"]
227229
# WriteConcern options:
228230
# w=1: Acknowledges the write operation only after it has been written to the primary. (the default)
229231
# j=True: Ensures the write operation is committed to the journal. (default is False)

‎bbot_server/config.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242

4343
class StoreConfig(BaseModel):
4444
uri: str
45+
collection_prefix: str = ""
4546

4647

4748
class MessageQueueConfig(BaseModel):

‎bbot_server/defaults.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,19 @@ api_key: null
1010

1111
# event store is just a big bucket of BBOT scan events that act as a read-only time machine
1212
event_store:
13-
uri: mongodb://localhost:27017/bbot_eventstore
13+
uri: mongodb://localhost:27017/bbot
14+
collection_prefix: "eventstore__"
1415

1516
# assets are derived from the event store, and can be recreated at any time
1617
asset_store:
17-
uri: mongodb://localhost:27017/bbot_assetstore
18+
uri: mongodb://localhost:27017/bbot
19+
collection_prefix: "assetstore__"
1820

1921
# user_store holds any user-specific data, overrides, etc. which are not derived from events
2022
# examples include targets, scans, etc.
2123
user_store:
22-
uri: mongodb://localhost:27017/bbot_userstore
24+
uri: mongodb://localhost:27017/bbot
25+
collection_prefix: "userstore__"
2326

2427
message_queue:
2528
uri: redis://localhost:6379/0

‎bbot_server/defaults_docker.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,19 @@ url: http://server:8807/v1/
33

44
# event store is just a big bucket of BBOT scan events
55
event_store:
6-
uri: mongodb://mongodb:27017/bbot_eventstore
6+
uri: mongodb://mongodb:27017/bbot
7+
collection_prefix: "eventstore__"
78

89
# assets are derived in real time from the event store, and can be recreated at any time
910
asset_store:
10-
uri: mongodb://mongodb:27017/bbot_assetstore
11+
uri: mongodb://mongodb:27017/bbot
12+
collection_prefix: "assetstore__"
1113

1214
# user_store holds any user-specific data, overrides, etc. which are not derived from events
1315
# examples include targets, scans, etc.
1416
user_store:
15-
uri: mongodb://mongodb:27017/bbot_userstore
17+
uri: mongodb://mongodb:27017/bbot
18+
collection_prefix: "userstore__"
1619

1720
message_queue:
1821
uri: redis://redis:6379/0

‎bbot_server/modules/server/server_cli.py‎

Lines changed: 19 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -128,49 +128,36 @@ def logs(
128128
@subcommand(help="Clear the database (drop Mongodb collections).")
129129
def cleardb(
130130
self,
131-
event_store: Annotated[bool, Option("--event-store", "-e", help="Clear the event store database")] = False,
132-
asset_store: Annotated[bool, Option("--asset-store", "-a", help="Clear the asset store database")] = False,
133-
user_store: Annotated[bool, Option("--user-store", "-u", help="Clear the user store database")] = False,
131+
event_store: Annotated[bool, Option("--event-store", "-e", help="Clear the event store collections")] = False,
132+
asset_store: Annotated[bool, Option("--asset-store", "-a", help="Clear the asset store collections")] = False,
133+
user_store: Annotated[bool, Option("--user-store", "-u", help="Clear the user store collections")] = False,
134134
):
135135
if not event_store and not asset_store and not user_store:
136-
raise self.BBOTServerError(f"Must specify at least one database to clear")
136+
raise self.BBOTServerError(f"Must specify at least one store to clear")
137137

138+
stores_to_clear = []
138139
if event_store:
139-
event_store_db = self.config.event_store.uri.split("/")[-1]
140-
if not event_store_db:
141-
raise self.BBOTServerError("Event store database not found in config")
142-
response = input(
143-
f"Are you sure you want to clear the event store database: {event_store_db}? This will permanently delete all BBOT scan events! (y/N) "
144-
)
145-
if response.lower() != "y":
146-
raise self.BBOTServerError("Aborting")
147-
148-
self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", "db.dropDatabase()", event_store_db])
149-
self.log.info(f"Successfully cleared event store database: {event_store_db}")
150-
140+
stores_to_clear.append(("event store", self.config.event_store, "all BBOT scan events"))
151141
if asset_store:
152-
asset_store_db = self.config.asset_store.uri.split("/")[-1]
153-
if not asset_store_db:
154-
raise self.BBOTServerError("Asset store database not found in config")
155-
response = input(
156-
f"Are you sure you want to clear the asset store database: {asset_store_db}? This will permanently delete all BBOT asset data! (y/N) "
142+
stores_to_clear.append(("asset store", self.config.asset_store, "all BBOT asset data"))
143+
if user_store:
144+
stores_to_clear.append(
145+
("user store", self.config.user_store, "all BBOT user data, including presets and targets")
157146
)
158-
if response.lower() != "y":
159-
raise self.BBOTServerError("Aborting")
160-
self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", "db.dropDatabase()", asset_store_db])
161-
self.log.info(f"Successfully cleared asset store database: {asset_store_db}")
162147

163-
if user_store:
164-
user_store_db = self.config.user_store.uri.split("/")[-1]
165-
if not user_store_db:
166-
raise self.BBOTServerError("User store database not found in config")
148+
for store_name, store_config, data_desc in stores_to_clear:
149+
db_name = store_config.uri.split("/")[-1]
150+
prefix = store_config.collection_prefix
151+
if not db_name:
152+
raise self.BBOTServerError(f"{store_name.title()} database not found in config")
167153
response = input(
168-
f"Are you sure you want to clear the user store database: {user_store_db}? This will permanently delete all BBOT user data, including presets and targets! (y/N) "
154+
f"Are you sure you want to clear the {store_name} (prefix: {prefix})? This will permanently delete {data_desc}! (y/N) "
169155
)
170156
if response.lower() != "y":
171157
raise self.BBOTServerError("Aborting")
172-
self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", "db.dropDatabase()", user_store_db])
173-
self.log.info(f"Successfully cleared user store database: {user_store_db}")
158+
drop_eval = f"db.getCollectionNames().filter(c => c.startsWith('{prefix}')).forEach(c => db[c].drop())"
159+
self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", drop_eval, db_name])
160+
self.log.info(f"Successfully cleared {store_name} collections (prefix: {prefix})")
174161

175162
def _run_docker_compose(self, args, **kwargs):
176163
kwargs["cwd"] = self.docker_compose_dir

‎bbot_server/store.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ class BaseMongoStore(BaseDB):
88
async def setup(self):
99
self.client = AsyncMongoClient(self.uri)
1010
self.db = self.client.get_database(self.db_name)
11-
self.fs = AsyncGridFSBucket(self.db)
11+
self.collection_prefix = getattr(self.db_config, "collection_prefix", "")
12+
bucket_name = f"{self.collection_prefix}fs" if self.collection_prefix else "fs"
13+
self.fs = AsyncGridFSBucket(self.db, bucket_name=bucket_name)
1214

1315
async def cleanup(self):
1416
await self.client.close()

‎helm/templates/server-deployment.yaml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,17 @@ spec:
5959
key: api-key
6060
# MongoDB URIs (credentials injected via $(VAR) interpolation)
6161
- name: BBOT_SERVER_EVENT_STORE__URI
62-
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_eventstore?authSource=admin"
62+
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
63+
- name: BBOT_SERVER_EVENT_STORE__COLLECTION_PREFIX
64+
value: "eventstore__"
6365
- name: BBOT_SERVER_ASSET_STORE__URI
64-
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_assetstore?authSource=admin"
66+
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
67+
- name: BBOT_SERVER_ASSET_STORE__COLLECTION_PREFIX
68+
value: "assetstore__"
6569
- name: BBOT_SERVER_USER_STORE__URI
66-
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_userstore?authSource=admin"
70+
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
71+
- name: BBOT_SERVER_USER_STORE__COLLECTION_PREFIX
72+
value: "userstore__"
6773
# Redis URI
6874
- name: BBOT_SERVER_MESSAGE_QUEUE__URI
6975
value: "redis://:$(REDIS_PASSWORD)@{{ .Release.Name }}-redis-master:6379/0"

‎helm/templates/watchdog-deployment.yaml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,17 @@ spec:
7272
key: api-key
7373
# MongoDB URIs (credentials injected via $(VAR) interpolation)
7474
- name: BBOT_SERVER_EVENT_STORE__URI
75-
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_eventstore?authSource=admin"
75+
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
76+
- name: BBOT_SERVER_EVENT_STORE__COLLECTION_PREFIX
77+
value: "eventstore__"
7678
- name: BBOT_SERVER_ASSET_STORE__URI
77-
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_assetstore?authSource=admin"
79+
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
80+
- name: BBOT_SERVER_ASSET_STORE__COLLECTION_PREFIX
81+
value: "assetstore__"
7882
- name: BBOT_SERVER_USER_STORE__URI
79-
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot_userstore?authSource=admin"
83+
value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin"
84+
- name: BBOT_SERVER_USER_STORE__COLLECTION_PREFIX
85+
value: "userstore__"
8086
# Redis URI
8187
- name: BBOT_SERVER_MESSAGE_QUEUE__URI
8288
value: "redis://:$(REDIS_PASSWORD)@{{ .Release.Name }}-redis-master:6379/0"

‎helm/values.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ mongodb:
6969
# -- Existing secret name (created by this chart if empty)
7070
existingSecret: ""
7171
persistence:
72+
size: 50Gi
7273
resourcePolicy: keep
7374

7475
# =============================================================================

0 commit comments

Comments
 (0)