Skip to content

Commit 1df8f3a

Browse files
dav1dodav1do
committed
wip: create new scenario for nightly benchmark and some clean up
ceramic-new-streams-benchmark: creates a new 1kb document repeatedly
1 parent 33108ac commit 1df8f3a

12 files changed

Lines changed: 159 additions & 58 deletions

File tree

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM public.ecr.aws/r5b3e0r5/3box/rust-builder:latest as builder
1+
FROM rust-builder:latest as builder
22

33
RUN mkdir -p /home/builder/keramik/
44
WORKDIR /home/builder/keramik/

‎k8s/operator/kustomization.yaml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,15 @@ resources:
1414
#
1515
# Additionally ensure that ./k8s/operator/manifests/operator.yaml uses Always as the imagePullPolicy
1616
#
17-
images:
18-
- name: keramik/operator
19-
newName: public.ecr.aws/r5b3e0r5/3box/keramik-operator
20-
newTag: latest
17+
# images:
18+
# - name: keramik/operator
19+
# newName: public.ecr.aws/r5b3e0r5/3box/keramik-operator
20+
# newTag: latest
2121

2222
# Uncomment for development
2323
#
2424
# Additionally ensure that ./k8s/operator/manifests/operator.yaml uses IfNotPresent as the imagePullPolicy
2525
#
26-
#images:
27-
# - name: keramik/operator
28-
# newTag: dev
26+
images:
27+
- name: keramik/operator
28+
newTag: dev

‎k8s/operator/manifests/operator.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ spec:
106106
containers:
107107
- name: keramik-operator
108108
image: "keramik/operator"
109-
imagePullPolicy: Always # Should be IfNotPresent when using imageTag: dev, but Always if using imageTag: latest
109+
imagePullPolicy: IfNotPresent # Should be IfNotPresent when using imageTag: dev, but Always if using imageTag: latest
110110
command:
111111
- "/usr/bin/keramik-operator"
112112
- "daemon"

‎local-scripts/build-local.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
set -e
3+
4+
docker buildx build --load -t keramik/operator:dev --target operator .
5+
docker buildx build --load -t keramik/runner:dev --target runner .
6+
7+
kind load docker-image keramik/runner:dev
8+
kind load docker-image keramik/operator:dev

‎local-scripts/small.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ metadata:
44
name: small
55
spec:
66
replicas: 2
7+
monitoring:
8+
namespaced: true
79
devMode: true
8-
# bootstrap:
9-
# image: keramik/runner:dev
10-
# imagePullPolicy: IfNotPresent
10+
bootstrap:
11+
image: keramik/runner:dev
12+
imagePullPolicy: IfNotPresent
1113
cas:
1214
ipfs:
1315
go: {}

‎runner/src/scenario/ceramic/mod.rs‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use ceramic_http_client::ceramic_event::{DidDocument, JwkSigner};
1010
use ceramic_http_client::CeramicHttpClient;
1111

1212
use models::RandomModelInstance;
13+
use serde::{Deserialize, Serialize};
1314

1415
use crate::simulate::Scenario;
1516

@@ -36,7 +37,8 @@ impl Credentials {
3637
}
3738
}
3839

39-
#[derive(Debug, Clone, PartialEq, Eq)]
40+
#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
41+
#[serde(rename_all = "camelCase")]
4042
pub enum DidType {
4143
/// One DID for all users
4244
Shared,
@@ -46,7 +48,8 @@ pub enum DidType {
4648
//UserCacao,
4749
}
4850

49-
#[derive(Debug, Clone, PartialEq, Eq)]
51+
#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
52+
#[serde(rename_all = "camelCase")]
5053
pub enum ReuseType {
5154
/// Create a new model or model instance document for each user
5255
PerUser,
@@ -56,7 +59,8 @@ pub enum ReuseType {
5659
Shared,
5760
}
5861

59-
#[derive(Debug, Clone)]
62+
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
63+
#[serde(rename_all = "camelCase")]
6064
pub struct CeramicScenarioParameters {
6165
pub did_type: DidType,
6266
/// Whether models should be shared or independent
@@ -68,43 +72,45 @@ pub struct CeramicScenarioParameters {
6872

6973
impl From<Scenario> for CeramicScenarioParameters {
7074
fn from(value: Scenario) -> Self {
75+
// did_type: DidType::UserDidKey and model_instance_reuse: ReuseType::Shared is an invalid combination
76+
// as we'll try to update documents owned by another controller and just log lots of errors
7177
match value {
7278
Scenario::CeramicSimple => Self {
7379
did_type: DidType::Shared,
7480
model_reuse: ReuseType::PerUser,
7581
model_instance_reuse: ReuseType::PerUser,
7682
number_of_documents: 1,
7783
},
78-
Scenario::CeramicUserSimple => Self {
84+
Scenario::CeramicModelReuse => Self {
7985
did_type: DidType::UserDidKey,
80-
model_reuse: ReuseType::PerUser,
86+
model_reuse: ReuseType::Shared,
8187
model_instance_reuse: ReuseType::PerUser,
8288
number_of_documents: 1,
8389
},
8490
Scenario::CeramicWriteOnly => Self {
8591
did_type: DidType::UserDidKey,
86-
model_reuse: ReuseType::PerUser,
92+
model_reuse: ReuseType::Shared,
8793
model_instance_reuse: ReuseType::PerUser,
8894
number_of_documents: 1,
8995
},
9096
Scenario::CeramicNewStreams => Self {
9197
did_type: DidType::UserDidKey,
9298
model_reuse: ReuseType::PerUser,
9399
model_instance_reuse: ReuseType::PerUser,
94-
number_of_documents: 1,
100+
number_of_documents: 0,
101+
},
102+
Scenario::CeramicNewStreamsBenchmark => Self {
103+
did_type: DidType::UserDidKey,
104+
model_reuse: ReuseType::Shared,
105+
model_instance_reuse: ReuseType::PerUser,
106+
number_of_documents: 0,
95107
},
96108
Scenario::CeramicQuery => Self {
97109
did_type: DidType::Shared,
98110
model_reuse: ReuseType::PerUser,
99111
model_instance_reuse: ReuseType::PerUser,
100112
number_of_documents: 3,
101113
},
102-
Scenario::CeramicModelReuse => Self {
103-
did_type: DidType::UserDidKey,
104-
model_reuse: ReuseType::Shared,
105-
model_instance_reuse: ReuseType::Shared,
106-
number_of_documents: 1,
107-
},
108114
Scenario::IpfsRpc | Scenario::ReconEventSync | Scenario::ReconEventKeySync => {
109115
panic!("Not supported for non ceramic scenarios")
110116
}

‎runner/src/scenario/ceramic/model_instance.rs‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use ceramic_http_client::{
77
};
88
use goose::{metrics::GooseRequestMetric, prelude::*};
99
use redis::AsyncCommands;
10-
use tracing::warn;
10+
use tracing::{debug, info, warn};
1111

1212
use crate::{
1313
goose_try,
@@ -111,31 +111,34 @@ impl CeramicModelInstanceTestUser {
111111
user: &mut GooseUser,
112112
config: EnvBasedConfig,
113113
) -> anyhow::Result<()> {
114+
debug!(params=?config.params, "setting up scenario");
114115
let (small_model_id, large_model_id) = match config.params.model_reuse {
115116
super::ReuseType::PerUser => {
116117
Self::generate_list_models(user, &config.admin_cli).await?
117118
}
118119
super::ReuseType::Shared => {
119120
let mut conn = config.redis_cli.get_async_connection().await.unwrap();
120121
if is_goose_leader() {
122+
info!("generating shared model IDs...");
121123
let (small, large) =
122124
Self::generate_list_models(user, &config.admin_cli).await?;
123-
ModelInstanceRequests::index_model(user, &config.admin_cli, &small, "small")
124-
.await?;
125-
ModelInstanceRequests::index_model(user, &config.admin_cli, &large, "large")
126-
.await?;
125+
127126
let _ = set_key_to_stream_id(&mut conn, SMALL_MODEL_ID_KEY, &small).await;
128127
let _ = set_key_to_stream_id(&mut conn, LARGE_MODEL_ID_KEY, &large).await;
129128

130129
(small, large)
131130
} else {
131+
info!("waiting for shared model IDs to be set in redis...");
132132
let small = loop_until_key_value_set(&mut conn, SMALL_MODEL_ID_KEY).await;
133133
let large = loop_until_key_value_set(&mut conn, LARGE_MODEL_ID_KEY).await;
134134
(small, large)
135135
}
136136
}
137137
};
138138

139+
// ModelInstanceRequests::index_model(user, &config.admin_cli, &small, "small").await?;
140+
// ModelInstanceRequests::index_model(user, &config.admin_cli, &large, "large").await?;
141+
139142
let (small_model_instance_ids, large_model_instance_ids) = match config
140143
.params
141144
.model_instance_reuse
@@ -156,6 +159,8 @@ impl CeramicModelInstanceTestUser {
156159
}
157160
let mut conn = config.redis_cli.get_async_connection().await.unwrap();
158161
if is_goose_leader() {
162+
info!("generating shared model instance IDs...");
163+
159164
let (small, large) = Self::generate_mids(
160165
user,
161166
&config.user_cli,
@@ -171,6 +176,8 @@ impl CeramicModelInstanceTestUser {
171176

172177
(small, large)
173178
} else {
179+
info!("waiting for shared model instance IDs...");
180+
174181
let small = loop_until_key_value_set(&mut conn, SMALL_MID_ID_KEY).await;
175182
let large = loop_until_key_value_set(&mut conn, LARGE_MID_ID_KEY).await;
176183
(vec![small], vec![large])
@@ -187,6 +194,7 @@ impl CeramicModelInstanceTestUser {
187194
};
188195

189196
user.set_session_data(Arc::new(resp));
197+
info!("scenario setup complete");
190198
Ok(())
191199
}
192200

@@ -236,14 +244,13 @@ impl CeramicModelInstanceTestUser {
236244
large_model_id: &StreamId,
237245
number_of_documents: usize,
238246
) -> Result<(Vec<StreamId>, Vec<StreamId>), TransactionError> {
239-
let doc_cnt = if number_of_documents == 0 {
240-
1
241-
} else {
242-
number_of_documents
243-
};
247+
if number_of_documents == 0 {
248+
// not all scenarios care to prep documents in advance
249+
return Ok((vec![], vec![]));
250+
}
244251
let mut small_model_instance_ids = vec![];
245252
let mut large_model_instance_ids = vec![];
246-
for i in 0..doc_cnt {
253+
for i in 0..number_of_documents {
247254
let small_model_instance_id = ModelInstanceRequests::create_model_instance(
248255
user,
249256
cli,
@@ -432,6 +439,7 @@ impl ModelInstanceRequests {
432439
Ok(resp.stream_id)
433440
}
434441

442+
#[allow(dead_code)]
435443
pub async fn index_model(
436444
user: &mut GooseUser,
437445
cli: &CeramicClient,

‎runner/src/scenario/ceramic/models.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,17 @@ impl LargeModel {
5252
tpe,
5353
}
5454
}
55+
56+
pub fn random_1kb() -> Self {
57+
let mut rng = thread_rng();
58+
let name: String = (1..40).map(|_| rng.gen::<char>()).collect();
59+
Self {
60+
creator: "keramik".to_string(),
61+
name: format!("keramik-large-model-{}", name),
62+
description: (1..200).map(|_| rng.gen::<char>()).collect(),
63+
tpe: rng.gen_range(0..100),
64+
}
65+
}
5566
}
5667

5768
impl RandomModelInstance for LargeModel {

‎runner/src/scenario/ceramic/new_streams.rs‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ use super::{
99
CeramicScenarioParameters,
1010
};
1111

12-
pub async fn scenario(params: CeramicScenarioParameters) -> Result<Scenario, GooseError> {
12+
pub async fn small_large_scenario(
13+
params: CeramicScenarioParameters,
14+
) -> Result<Scenario, GooseError> {
1315
let config = CeramicModelInstanceTestUser::prep_scenario(params)
1416
.await
1517
.unwrap();
@@ -29,6 +31,23 @@ pub async fn scenario(params: CeramicScenarioParameters) -> Result<Scenario, Goo
2931
.register_transaction(instantiate_large_model))
3032
}
3133

34+
pub async fn benchmark_scenario(params: CeramicScenarioParameters) -> Result<Scenario, GooseError> {
35+
let config = CeramicModelInstanceTestUser::prep_scenario(params)
36+
.await
37+
.unwrap();
38+
39+
let test_start = Transaction::new(Arc::new(move |user| Box::pin(setup(user, config.clone()))))
40+
.set_name("setup")
41+
.set_on_start();
42+
43+
let instantiate_large_model =
44+
transaction!(instantiate_large_model_1kb).set_name("instantiate_1k_mid");
45+
46+
Ok(scenario!("CeramicNewStreamsBenchmark")
47+
.register_transaction(test_start)
48+
.register_transaction(instantiate_large_model))
49+
}
50+
3251
async fn instantiate_small_model(user: &mut GooseUser) -> TransactionResult {
3352
let user_data = CeramicModelInstanceTestUser::user_data(user).to_owned();
3453
ModelInstanceRequests::create_model_instance(
@@ -54,3 +73,16 @@ async fn instantiate_large_model(user: &mut GooseUser) -> TransactionResult {
5473
.await?;
5574
Ok(())
5675
}
76+
77+
async fn instantiate_large_model_1kb(user: &mut GooseUser) -> TransactionResult {
78+
let user_data = CeramicModelInstanceTestUser::user_data(user).to_owned();
79+
ModelInstanceRequests::create_model_instance(
80+
user,
81+
user_data.user_cli(),
82+
&user_data.large_model_id,
83+
"instantiate_1kb_mid",
84+
&models::LargeModel::random_1kb(),
85+
)
86+
.await?;
87+
Ok(())
88+
}

‎runner/src/scenario/ceramic/simple.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use std::sync::Arc;
22

33
use goose::prelude::*;
4-
use tracing::instrument;
4+
use tracing::{error, instrument};
55

66
use crate::scenario::ceramic::{
77
model_instance::CeramicModelInstanceTestUser,
@@ -17,6 +17,7 @@ use super::{
1717
pub(crate) async fn setup(user: &mut GooseUser, config: EnvBasedConfig) -> TransactionResult {
1818
CeramicModelInstanceTestUser::setup_scenario(user, config)
1919
.await
20+
.map_err(|e| error!("Error setting up scenario: {}", e))
2021
.unwrap();
2122
Ok(())
2223
}

0 commit comments

Comments
 (0)