Skip to content

Commit 9ccc075

Browse files
ian-siftian-sift
committed
Merge remote-tracking branch 'origin/main' into FD-74
2 parents 3c8e26a + a345fcb commit 9ccc075

59 files changed

Lines changed: 1933 additions & 283 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/python_build_docs.yml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
name: Build and Deploy Python Docs (Dev)
2+
3+
on:
4+
pull_request:
5+
types: [opened, synchronize, closed]
6+
paths:
7+
- 'python/docs/**'
8+
- 'python/lib/**'
9+
- 'python/mkdocs.yml'
10+
- 'python/pyproject.toml'
11+
workflow_dispatch:
12+
inputs:
13+
deploy_to_dev:
14+
description: 'Deploy to dev alias using commit hash as version'
15+
required: false
16+
default: true
17+
type: boolean
18+
19+
jobs:
20+
build_docs:
21+
if: github.event.action != 'closed'
22+
runs-on: ubuntu-latest
23+
permissions:
24+
contents: write
25+
steps:
26+
- name: Checkout code
27+
uses: actions/checkout@v4
28+
with:
29+
fetch-depth: 1
30+
31+
- name: Set up Python
32+
uses: actions/setup-python@v4
33+
with:
34+
python-version: '3.11'
35+
36+
- name: Install dependencies
37+
run: |
38+
cd python
39+
pip install -e .[docs,development]
40+
41+
- name: Extract version
42+
id: version
43+
run: |
44+
# Use commit hash as version for dev deployments
45+
VERSION=$(git rev-parse --short HEAD)
46+
47+
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
48+
# Use PR number as alias for PR deployments
49+
ALIAS="pr-${{ github.event.number }}"
50+
else
51+
# Use 'dev' for manual workflow dispatch
52+
ALIAS="dev"
53+
fi
54+
55+
echo "version=$VERSION" >> $GITHUB_OUTPUT
56+
echo "alias=$ALIAS" >> $GITHUB_OUTPUT
57+
echo "Dev deployment - Version: $VERSION, Alias: $ALIAS"
58+
59+
- name: Fetch gh-pages branch
60+
run: git fetch origin gh-pages --depth=1
61+
62+
- name: Configure Git
63+
run: |
64+
git config --global user.name "github-actions[bot]"
65+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
66+
67+
- name: Deploy docs with mike
68+
run: |
69+
cd python
70+
# Deploy dev/PR docs with hidden property to hide from version dropdown
71+
mike deploy ${{ steps.version.outputs.alias }} --push --update-aliases --prop-set hidden=true
72+
env:
73+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
74+
75+
cleanup_docs:
76+
if: github.event.action == 'closed'
77+
runs-on: ubuntu-latest
78+
permissions:
79+
contents: write
80+
steps:
81+
- name: Checkout code
82+
uses: actions/checkout@v4
83+
with:
84+
fetch-depth: 1
85+
86+
- name: Set up Python
87+
uses: actions/setup-python@v4
88+
with:
89+
python-version: '3.11'
90+
91+
- name: Install dependencies
92+
run: |
93+
cd python
94+
pip install -e .[docs,development]
95+
96+
- name: Fetch gh-pages branch
97+
run: git fetch origin gh-pages --depth=1
98+
99+
- name: Configure Git
100+
run: |
101+
git config --global user.name "github-actions[bot]"
102+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
103+
104+
- name: Delete PR docs
105+
run: |
106+
cd python
107+
PR_ALIAS="pr-${{ github.event.number }}"
108+
echo "Deleting docs for: $PR_ALIAS"
109+
110+
# Check if the PR docs exist before trying to delete
111+
if mike list | grep -q "$PR_ALIAS"; then
112+
mike delete "$PR_ALIAS" --push
113+
echo "Successfully deleted docs for $PR_ALIAS"
114+
else
115+
echo "No docs found for $PR_ALIAS, nothing to delete"
116+
fi
117+
env:
118+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/python_release.yaml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,4 +87,89 @@ jobs:
8787
echo "Uploading archive: $archive"
8888
gh release upload "$TAG_NAME" "$archive" --clobber
8989
done
90+
91+
deploy-docs:
92+
name: Deploy Documentation
93+
needs: publish-to-pypi
94+
runs-on: ubuntu-latest
95+
permissions:
96+
contents: write
97+
steps:
98+
- name: Checkout code
99+
uses: actions/checkout@v4
100+
with:
101+
fetch-depth: 1
102+
103+
- name: Set up Python
104+
uses: actions/setup-python@v4
105+
with:
106+
python-version: '3.11'
107+
108+
- name: Install dependencies
109+
run: |
110+
cd python
111+
pip install -e .[docs,development]
112+
113+
- name: Extract version and check if stable
114+
id: version
115+
run: |
116+
# Extract version from tag (everything after 'python/')
117+
FULL_VERSION=${GITHUB_REF#refs/tags/python/}
118+
echo "Full version from tag: $FULL_VERSION"
119+
120+
# Extract major.minor from version (drop patch)
121+
if [[ "$FULL_VERSION" =~ ^v?([0-9]+)\.([0-9]+)\.[0-9]+(.*)?$ ]]; then
122+
MAJOR=${BASH_REMATCH[1]}
123+
MINOR=${BASH_REMATCH[2]}
124+
SUFFIX=${BASH_REMATCH[3]}
125+
VERSION="v${MAJOR}.${MINOR}"
126+
127+
# Check if this is a stable release (no suffix like -alpha, -beta, -rc)
128+
if [[ -z "$SUFFIX" ]]; then
129+
# Stable release - use 'latest' alias and make visible
130+
ALIAS="latest"
131+
HIDDEN="false"
132+
echo "Stable release detected: $FULL_VERSION -> $VERSION"
133+
else
134+
# Pre-release (alpha, beta, rc) - no 'latest' alias and hide from dropdown
135+
VERSION="${VERSION}${SUFFIX}"
136+
ALIAS=""
137+
HIDDEN="true"
138+
echo "Pre-release detected: $FULL_VERSION -> $VERSION"
139+
fi
140+
else
141+
echo "Error: Could not parse version format: $FULL_VERSION"
142+
exit 1
143+
fi
144+
145+
echo "version=$VERSION" >> $GITHUB_OUTPUT
146+
echo "alias=$ALIAS" >> $GITHUB_OUTPUT
147+
echo "hidden=$HIDDEN" >> $GITHUB_OUTPUT
148+
echo "Final - Version: $VERSION, Alias: $ALIAS, Hidden: $HIDDEN"
149+
150+
- name: Fetch gh-pages branch
151+
run: git fetch origin gh-pages --depth=1
152+
153+
- name: Configure Git
154+
run: |
155+
git config --global user.name "github-actions[bot]"
156+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
157+
158+
- name: Deploy docs with mike
159+
run: |
160+
cd python
161+
VERSION="${{ steps.version.outputs.version }}"
162+
ALIAS="${{ steps.version.outputs.alias }}"
163+
HIDDEN="${{ steps.version.outputs.hidden }}"
90164
165+
# Always deploy the full version first, but hide it
166+
echo "Deploying full version $FULL_VERSION (hidden)"
167+
mike deploy "$FULL_VERSION" --push --prop-set hidden=true
168+
169+
if [[ "$HIDDEN" == "false" ]]; then
170+
# Stable release: deploy abbreviated version with latest alias, visible in dropdown
171+
echo "Deploying stable release $VERSION with $ALIAS alias"
172+
mike deploy "$VERSION" "$ALIAS" --push --update-aliases
173+
fi
174+
env:
175+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎protos/sift/protobuf_descriptors/v2/protobuf_descriptors.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ message IncompatibleProtobufField {
9595
string field_number = 5;
9696
string reason = 6;
9797
string details = 7;
98+
string field_kind = 8;
9899
}
99100

100101
message CheckProtobufDescriptorCompatibilityResponse {

‎protos/sift/rule_evaluation/v1/rule_evaluation.proto‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ message EvaluateRulesRequest {
5858
EvaluateRulesFromCurrentRuleVersions rules = 3;
5959
EvaluateRulesFromRuleVersions rule_versions = 4;
6060
EvaluateRulesFromReportTemplate report_template = 5;
61+
bool all_applicable_rules = 10;
6162
}
6263
EvaluateRulesAnnotationOptions annotation_options = 6;
6364

@@ -129,8 +130,18 @@ message EvaluateRulesFromRuleConfigs {
129130
repeated sift.rules.v1.UpdateRuleRequest configs = 1 [(google.api.field_behavior) = REQUIRED];
130131
}
131132

133+
message RulePreviewOutput {
134+
string rule_name = 1 [(google.api.field_behavior) = REQUIRED];
135+
string rule_id = 2;
136+
string rule_version_id = 3;
137+
string asset_id = 4 [(google.api.field_behavior) = REQUIRED];
138+
int32 exit_code = 5 [(google.api.field_behavior) = REQUIRED];
139+
string stdout = 6;
140+
string stderr = 7;
141+
}
132142

133143
message EvaluateRulesPreviewResponse {
134144
int32 created_annotation_count = 1 [(google.api.field_behavior) = REQUIRED];
135145
repeated sift.rules.v1.DryRunAnnotation dry_run_annotations = 2;
146+
repeated RulePreviewOutput rule_outputs = 3;
136147
}

‎protos/sift/rules/v1/rules.proto‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -656,6 +656,8 @@ message RuleConditionExpression {
656656
oneof expression {
657657
SingleChannelComparisonExpression single_channel_comparison = 1 [deprecated = true];
658658
CalculatedChannelConfig calculated_channel = 2;
659+
// Experimental - not currently enabled
660+
PythonCode python_code = 3;
659661
}
660662
}
661663

@@ -693,6 +695,15 @@ message ChannelReference {
693695
// Deprecated - use name instead. If provided, name will be joined with the component as `component.name`
694696
string component = 2 [(google.api.field_behavior) = REQUIRED];
695697
}
698+
message PythonChannelReference {
699+
string reference = 1 [(google.api.field_behavior) = REQUIRED];
700+
string name = 2 [(google.api.field_behavior) = REQUIRED];
701+
}
702+
message PythonCode {
703+
repeated PythonChannelReference channel_references = 1 [(google.api.field_behavior) = REQUIRED];
704+
string code = 2 [(google.api.field_behavior) = REQUIRED];
705+
string dependencies = 3 [(google.api.field_behavior) = OPTIONAL];
706+
}
696707

697708
message RuleActionConfiguration {
698709
oneof configuration {

‎protos/sift/runs/v2/runs.proto‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,18 @@ service RunService {
5353
};
5454
}
5555

56+
// Create adhoc run, this is used to create a run after the data has been ingested
57+
rpc CreateAdhocRun(CreateAdhocRunRequest) returns (CreateAdhocRunResponse) {
58+
option (google.api.http) = {
59+
post: "/api/v2/runs:adhoc"
60+
body: "*"
61+
};
62+
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
63+
summary: "CreateAdhocRun"
64+
description: "Create an adhoc run."
65+
};
66+
}
67+
5668
// Updates an existing run using using the list of fields specified in `update_mask`.
5769
rpc UpdateRun(UpdateRunRequest) returns (UpdateRunResponse) {
5870
option (google.api.http) = {
@@ -118,6 +130,7 @@ message Run {
118130
repeated sift.metadata.v1.MetadataValue metadata = 15 [(google.api.field_behavior) = REQUIRED];
119131
repeated string asset_ids = 16 [(google.api.field_behavior) = REQUIRED];
120132
optional google.protobuf.Timestamp archived_date = 17 [(google.api.field_behavior) = OPTIONAL];
133+
bool is_adhoc = 18 [(google.api.field_behavior) = REQUIRED];
121134
}
122135

123136
// The request for a call to `RunService_GetRun` to retrieve run.
@@ -148,8 +161,8 @@ message ListRunsRequest {
148161
string page_token = 2 [(google.api.field_behavior) = OPTIONAL];
149162

150163
// A [Common Expression Language (CEL)](https://github.com/google/cel-spec) filter string.
151-
// Available fields to filter by are `run_id`, `organization_id`, `name`, `description`, `created_by_user_id`, `modified_by_user_id`,
152-
// `created_date`, `modified_date`, `start_time`, `stop_time`, `client_key`, `is_pinned`, `asset_id`, `asset_name`, `archived_date`,
164+
// Available fields to filter by are `run_id` `organization_id`, `asset_id`, `asset_name`, `client_key`, `name`, `description`, `created_by_user_id`, `modified_by_user_id`,
165+
// `created_date`, `modified_date`, `start_time`, `stop_time`, `tag_id`, `asset_tag_id`, `duration`, `annotation_comments_count`, `annotation_state`, `archived_date`,
153166
// and `metadata`. Metadata can be used in filters by using `metadata.{metadata_key_name}` as the field name.
154167
// For further information about how to use CELs, please refer to [this guide](https://github.com/google/cel-spec/blob/master/doc/langdef.md#standard-definitions).
155168
// For more information about the fields used for filtering, please refer to [this definition](/docs/api/grpc/protocol-buffers/runs#run). Optional.
@@ -203,11 +216,39 @@ message CreateRunRequest {
203216
repeated sift.metadata.v1.MetadataValue metadata = 9 [(google.api.field_behavior) = OPTIONAL];
204217
}
205218

206-
// The response of a call to `RunService_CreateRuns` containing the newly created run.
207219
message CreateRunResponse {
208220
Run run = 1 [(google.api.field_behavior) = REQUIRED];
209221
}
210222

223+
// The request for a call to `RunService_CreateAdhocRun` to create an adhoc run.
224+
message CreateAdhocRunRequest {
225+
// The name that will be assigned to the new run.
226+
string name = 1 [(google.api.field_behavior) = REQUIRED];
227+
// A description about the new run.
228+
string description = 2 [(google.api.field_behavior) = REQUIRED];
229+
// The time at which data ingestion began for this new run. It must be before the `stop_time`
230+
google.protobuf.Timestamp start_time = 3 [(google.api.field_behavior) = REQUIRED];
231+
// The time at which data ingestion concluded for this new run.
232+
google.protobuf.Timestamp stop_time = 4 [(google.api.field_behavior) = REQUIRED];
233+
// A list of asset IDs to associate with the new run.
234+
repeated string asset_ids = 5 [(google.api.field_behavior) = REQUIRED];
235+
// Tags to associate with the new run.
236+
repeated string tags = 6 [(google.api.field_behavior) = OPTIONAL];
237+
// The metadata values associated with this run.
238+
repeated sift.metadata.v1.MetadataValue metadata = 7 [(google.api.field_behavior) = OPTIONAL];
239+
// An arbitrary user-chosen key that uniquely identifies this run. Optional, though it is recommended to provide.
240+
optional string client_key = 8 [
241+
(google.api.field_behavior) = OPTIONAL
242+
];
243+
}
244+
245+
246+
// The response of a call to `RunService_CreateAdhocRun` containing the newly created adhoc run.
247+
message CreateAdhocRunResponse {
248+
Run run = 1 [(google.api.field_behavior) = REQUIRED];
249+
}
250+
251+
211252
// The request for a call to `RunService_UpdateRun` to update an existing run.
212253
message UpdateRunRequest {
213254
// The run to update. The run's `run_id` field is used to identify the run to update

‎python/CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file.
33

44
This project adheres to [Semantic Versioning](http://semver.org/).
55

6+
## [v0.8.4] - August 18, 2025
7+
- [Add experimental protos for development](https://github.com/sift-stack/sift/pull/291)
8+
9+
## [v0.8.3] - August 11, 2025
10+
- [Fix windows utf-8 encoding bug with Hdf5UploadService](https://github.com/sift-stack/sift/pull/289)
11+
612
## [v0.8.2] - August 1, 2025
713
- [Use name only in ChannelReference creation](https://github.com/sift-stack/sift/pull/284)
814

0 commit comments

Comments
 (0)