Skip to content

Commit b1d523d

Browse files
densumeshskeptrunedev
densumesh
authored and
skeptrunedev
committed
feature: added a/b testing infra to backend
1 parent 4432c70 commit b1d523d

11 files changed

Lines changed: 1227 additions & 5 deletions

File tree

‎clients/ts-sdk/openapi.json‎

Lines changed: 442 additions & 0 deletions
Large diffs are not rendered by default.

‎clients/ts-sdk/src/types.gen.ts‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22

33
export type APIVersion = 'V1' | 'V2';
44

5+
export type AbTestReqBody = {
6+
experiment_id: string;
7+
user_id: string;
8+
};
9+
510
export type AddChunkToGroupReqPayload = {
611
/**
712
* Id of the chunk to make a member of the group.
@@ -970,6 +975,11 @@ export type CreateDatasetReqPayload = {
970975
tracking_id?: (string) | null;
971976
};
972977

978+
export type CreateExperimentReqBody = {
979+
experiment_config: ExperimentConfig;
980+
name: string;
981+
};
982+
973983
/**
974984
* Will use [chunkr.ai](https://chunkr.ai) to process the file when this object is defined. See [docs.chunkr.ai/api-references/task/create-task](https://docs.chunkr.ai/api-references/task/create-task) for detailed information about what each field on this request payload does.
975985
*/
@@ -1908,6 +1918,25 @@ export type EventsForTopicResponse = {
19081918
events: Array<EventData>;
19091919
};
19101920

1921+
export type Experiment = {
1922+
control_name: string;
1923+
control_split: number;
1924+
created_at: string;
1925+
dataset_id: string;
1926+
id: string;
1927+
name: string;
1928+
t1_name: string;
1929+
t1_split: number;
1930+
updated_at: string;
1931+
};
1932+
1933+
export type ExperimentConfig = {
1934+
control_name: string;
1935+
control_split: number;
1936+
t1_name: string;
1937+
t1_split: number;
1938+
};
1939+
19111940
export type ExtendedOrganizationUsageCount = {
19121941
bytes_ingested: number;
19131942
chunk_count: number;
@@ -4643,6 +4672,12 @@ export type UpdateDatasetReqPayload = {
46434672
tracking_id?: (string) | null;
46444673
};
46454674

4675+
export type UpdateExperimentReqBody = {
4676+
experiment_config?: ((ExperimentConfig) | null);
4677+
id: string;
4678+
name?: (string) | null;
4679+
};
4680+
46464681
export type UpdateGroupByTrackingIDReqPayload = {
46474682
/**
46484683
* Description to assign to the chunk_group. Convenience field for you to avoid having to remember what the group is for. If not provided, the description will not be updated.
@@ -4796,6 +4831,12 @@ export type UserOrganization = {
47964831
user_id: string;
47974832
};
47984833

4834+
export type UserTreatmentResponse = {
4835+
experiment_id: string;
4836+
treatment_name: string;
4837+
user_id: string;
4838+
};
4839+
47994840
export type V1RecommendChunksResponseBody = Array<ChunkMetadataWithScore>;
48004841

48014842
export type WorkerEvent = {
@@ -5857,6 +5898,64 @@ export type CreateEtlJobData = {
58575898

58585899
export type CreateEtlJobResponse = (void);
58595900

5901+
export type GetExperimentsData = {
5902+
/**
5903+
* The dataset id to use for the request
5904+
*/
5905+
trDataset: string;
5906+
};
5907+
5908+
export type GetExperimentsResponse = (Array<Experiment>);
5909+
5910+
export type CreateExperimentData = {
5911+
/**
5912+
* JSON request payload to create a new experiment
5913+
*/
5914+
requestBody: CreateExperimentReqBody;
5915+
/**
5916+
* The dataset id to use for the request
5917+
*/
5918+
trDataset: string;
5919+
};
5920+
5921+
export type CreateExperimentResponse = (Experiment);
5922+
5923+
export type UpdateExperimentData = {
5924+
/**
5925+
* JSON request payload to update an experiment
5926+
*/
5927+
requestBody: UpdateExperimentReqBody;
5928+
/**
5929+
* The dataset id to use for the request
5930+
*/
5931+
trDataset: string;
5932+
};
5933+
5934+
export type UpdateExperimentResponse = (Experiment);
5935+
5936+
export type AbTestData = {
5937+
/**
5938+
* JSON request payload to get a user's treatment
5939+
*/
5940+
requestBody: AbTestReqBody;
5941+
/**
5942+
* The dataset id to use for the request
5943+
*/
5944+
trDataset: string;
5945+
};
5946+
5947+
export type AbTestResponse = (UserTreatmentResponse);
5948+
5949+
export type DeleteExperimentData = {
5950+
experimentId: string;
5951+
/**
5952+
* The dataset id to use for the request
5953+
*/
5954+
trDataset: string;
5955+
};
5956+
5957+
export type DeleteExperimentResponse = (void);
5958+
58605959
export type UploadFileHandlerData = {
58615960
/**
58625961
* JSON request payload to upload a file
@@ -7572,6 +7671,77 @@ export type $OpenApiTs = {
75727671
};
75737672
};
75747673
};
7674+
'/api/experiment': {
7675+
get: {
7676+
req: GetExperimentsData;
7677+
res: {
7678+
/**
7679+
* Experiments retrieved successfully
7680+
*/
7681+
200: Array<Experiment>;
7682+
/**
7683+
* Service error relating to getting the experiments
7684+
*/
7685+
400: ErrorResponseBody;
7686+
};
7687+
};
7688+
post: {
7689+
req: CreateExperimentData;
7690+
res: {
7691+
/**
7692+
* Experiment created successfully
7693+
*/
7694+
200: Experiment;
7695+
/**
7696+
* Service error relating to creating the experiment
7697+
*/
7698+
400: ErrorResponseBody;
7699+
};
7700+
};
7701+
put: {
7702+
req: UpdateExperimentData;
7703+
res: {
7704+
/**
7705+
* Experiment updated successfully
7706+
*/
7707+
200: Experiment;
7708+
/**
7709+
* Service error relating to updating the experiment
7710+
*/
7711+
400: ErrorResponseBody;
7712+
};
7713+
};
7714+
};
7715+
'/api/experiment/ab-test': {
7716+
post: {
7717+
req: AbTestData;
7718+
res: {
7719+
/**
7720+
* User treatment response
7721+
*/
7722+
200: UserTreatmentResponse;
7723+
/**
7724+
* Service error relating to getting the user's treatment
7725+
*/
7726+
400: ErrorResponseBody;
7727+
};
7728+
};
7729+
};
7730+
'/api/experiment/{experiment_id}': {
7731+
delete: {
7732+
req: DeleteExperimentData;
7733+
res: {
7734+
/**
7735+
* Experiment deleted successfully
7736+
*/
7737+
204: void;
7738+
/**
7739+
* Service error relating to deleting the experiment
7740+
*/
7741+
400: ErrorResponseBody;
7742+
};
7743+
};
7744+
};
75757745
'/api/file': {
75767746
post: {
75777747
req: UploadFileHandlerData;

‎frontends/dashboard/src/pages/dataset/PublicPageSettings.tsx‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ const PublicPageControls = () => {
303303
{loadingDefaultConfig() ? "Loading..." : "Auto Configure"}
304304
</button>
305305
</div>
306-
<div class="mb-6 py-2 flex content-center items-center gap-1.5 gap-x-2.5">
306+
<div class="mb-6 flex content-center items-center gap-1.5 gap-x-2.5 py-2">
307307
<span class="font-medium">Published Url:</span>{" "}
308308
<a class="text-magenta-400" href={publicUrl()} target="_blank">
309309
{publicUrl()}
@@ -359,11 +359,11 @@ const PublicPageControls = () => {
359359
options={["light", "dark"]}
360360
/>
361361
</div>
362-
<div class="grow max-w-[50%]">
362+
<div class="max-w-[50%] grow">
363363
<For each={docColors()}>
364364
{(color) => (
365365
<button
366-
class="w-6 h-6 rounded-lg"
366+
class="h-6 w-6 rounded-lg"
367367
style={{ "background-color": color }}
368368
onClick={() => {
369369
setExtraParams("brandColor", color);
@@ -1756,7 +1756,7 @@ export const SingleProductOptions = () => {
17561756
});
17571757
}}
17581758
disabled={loadingAutoFill()}
1759-
class="inline-flex justify-center rounded-md bg-magenta-500 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-magenta-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-magenta-900 disabled:animate-pulse min-w-[130px]"
1759+
class="inline-flex min-w-[130px] justify-center rounded-md bg-magenta-500 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-magenta-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-magenta-900 disabled:animate-pulse"
17601760
>
17611761
{loadingAutoFill() ? "Loading..." : "Auto Fill"}
17621762
</button>

‎server/ch_migrations/1747440134_add_experiments_table/down.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
DROP TABLE IF EXISTS experiments;
2+
DROP TABLE IF EXISTS experiment_user_assignments;

‎server/ch_migrations/1747440134_add_experiments_table/up.sql‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
CREATE TABLE IF NOT EXISTS experiments (
2+
id UUID,
3+
name String,
4+
t1_name String,
5+
t1_split Float32,
6+
control_name String,
7+
control_split Float32,
8+
dataset_id UUID,
9+
created_at DateTime DEFAULT now(),
10+
updated_at DateTime DEFAULT now(),
11+
)
12+
ORDER BY (created_at, id)
13+
PARTITION BY
14+
(dataset_id);
15+
16+
CREATE TABLE IF NOT EXISTS experiment_user_assignments (
17+
id UUID,
18+
experiment_id UUID,
19+
user_id String,
20+
dataset_id UUID,
21+
treatment_name String,
22+
created_at DateTime DEFAULT now(),
23+
updated_at DateTime DEFAULT now(),
24+
)
25+
ORDER BY (created_at, id)
26+
PARTITION BY
27+
(experiment_id);
28+

‎server/src/data/models.rs‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8901,6 +8901,92 @@ impl Default for ContextOptions {
89018901
}
89028902
}
89038903

8904+
#[derive(Debug, Serialize, Deserialize, ToSchema, Clone, Row)]
8905+
pub struct ExperimentClickhouse {
8906+
#[serde(with = "clickhouse::serde::uuid")]
8907+
pub id: uuid::Uuid,
8908+
pub name: String,
8909+
pub t1_name: String,
8910+
pub t1_split: f32,
8911+
pub control_name: String,
8912+
pub control_split: f32,
8913+
#[serde(with = "clickhouse::serde::uuid")]
8914+
pub dataset_id: uuid::Uuid,
8915+
#[serde(with = "clickhouse::serde::time::datetime")]
8916+
pub created_at: OffsetDateTime,
8917+
#[serde(with = "clickhouse::serde::time::datetime")]
8918+
pub updated_at: OffsetDateTime,
8919+
}
8920+
8921+
#[derive(Debug, Serialize, Deserialize, ToSchema, Clone)]
8922+
pub struct Experiment {
8923+
pub id: uuid::Uuid,
8924+
pub name: String,
8925+
pub t1_name: String,
8926+
pub t1_split: f32,
8927+
pub control_name: String,
8928+
pub control_split: f32,
8929+
pub dataset_id: uuid::Uuid,
8930+
pub created_at: chrono::NaiveDateTime,
8931+
pub updated_at: chrono::NaiveDateTime,
8932+
}
8933+
8934+
impl From<Experiment> for ExperimentClickhouse {
8935+
fn from(experiment: Experiment) -> Self {
8936+
ExperimentClickhouse {
8937+
id: experiment.id,
8938+
name: experiment.name,
8939+
t1_name: experiment.t1_name,
8940+
t1_split: experiment.t1_split,
8941+
control_name: experiment.control_name,
8942+
control_split: experiment.control_split,
8943+
dataset_id: experiment.dataset_id,
8944+
created_at: OffsetDateTime::from_unix_timestamp(experiment.created_at.timestamp())
8945+
.unwrap(),
8946+
updated_at: OffsetDateTime::from_unix_timestamp(experiment.updated_at.timestamp())
8947+
.unwrap(),
8948+
}
8949+
}
8950+
}
8951+
8952+
impl From<ExperimentClickhouse> for Experiment {
8953+
fn from(experiment: ExperimentClickhouse) -> Self {
8954+
Experiment {
8955+
id: experiment.id,
8956+
name: experiment.name,
8957+
t1_name: experiment.t1_name,
8958+
t1_split: experiment.t1_split,
8959+
control_name: experiment.control_name,
8960+
control_split: experiment.control_split,
8961+
dataset_id: experiment.dataset_id,
8962+
created_at: chrono::NaiveDateTime::from_timestamp(
8963+
experiment.created_at.unix_timestamp(),
8964+
0,
8965+
),
8966+
updated_at: chrono::NaiveDateTime::from_timestamp(
8967+
experiment.updated_at.unix_timestamp(),
8968+
0,
8969+
),
8970+
}
8971+
}
8972+
}
8973+
8974+
#[derive(Debug, Clone, Serialize, Deserialize, Row, ToSchema)]
8975+
pub struct ExperimentUserAssignment {
8976+
#[serde(with = "clickhouse::serde::uuid")]
8977+
pub id: uuid::Uuid,
8978+
#[serde(with = "clickhouse::serde::uuid")]
8979+
pub experiment_id: uuid::Uuid,
8980+
pub user_id: String,
8981+
#[serde(with = "clickhouse::serde::uuid")]
8982+
pub dataset_id: uuid::Uuid,
8983+
pub treatment_name: String,
8984+
#[serde(with = "clickhouse::serde::time::datetime")]
8985+
pub created_at: OffsetDateTime,
8986+
#[serde(with = "clickhouse::serde::time::datetime")]
8987+
pub updated_at: OffsetDateTime,
8988+
}
8989+
89048990
#[derive(Debug, Serialize, Deserialize, ToSchema, Clone, Default)]
89058991
/// LLM options to use for the completion. If not specified, this defaults to the dataset's LLM options.
89068992
pub struct LLMOptions {

0 commit comments

Comments
 (0)