Skip to content

Commit 506c9b5

Browse files
Merge pull request #27 from microsoft/dev
chore: Merge Dev to Main
2 parents 187de94 + c391bb7 commit 506c9b5

15 files changed

Lines changed: 743 additions & 211 deletions

.flake8

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
[flake8]
2+
max-line-length = 88
3+
extend-ignore =
4+
# Docstring warnings (D-series) - these are style preferences
5+
D100, D101, D102, D103, D107, D400, D401
6+
# Type annotation warnings (ANN-series) - gradually add these
7+
ANN001, ANN101, ANN201, ANN202, ANN204, ANN205
8+
# Import order warnings (I-series) - handled by isort
9+
I100, I101, I201, I202
10+
# Whitespace in blank lines
11+
W293
12+
# F-string without placeholders (often intentional for consistency)
13+
F541
14+
exclude =
15+
.git,
16+
__pycache__,
17+
.venv,
18+
venv,
19+
*.egg-info,
20+
.tox,
21+
.eggs
22+
per-file-ignores =
23+
__init__.py:F401

.github/CODEOWNERS

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Lines starting with '#' are comments.
2+
3+
# Each line is a file pattern followed by one or more owners.
4+
5+
# These owners will be the default owners for everything in the repo.
6+
7+
* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft @Vinay-Microsoft @aniaroramsft @toherman-msft @nchandhi @dgp10801

.github/dependabot.yml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
version: 2
2+
3+
updates:
4+
# GitHub Actions - grouped
5+
- package-ecosystem: "github-actions"
6+
directory: "/"
7+
schedule:
8+
interval: "monthly"
9+
target-branch: "dependabotchanges"
10+
commit-message:
11+
prefix: "build"
12+
open-pull-requests-limit: 10
13+
groups:
14+
github-actions:
15+
patterns:
16+
- "*"
17+
18+
# Python dependencies - grouped
19+
- package-ecosystem: "pip"
20+
directory: "/"
21+
schedule:
22+
interval: "monthly"
23+
target-branch: "dependabotchanges"
24+
commit-message:
25+
prefix: "build"
26+
open-pull-requests-limit: 10
27+
groups:
28+
python-deps:
29+
patterns:
30+
- "*"
31+
# url: "https://registry.npmjs.org/"
32+
# token: ${{ secrets.TOKEN }}
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Broken Link Checker
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- '**/*.md'
7+
workflow_dispatch:
8+
9+
permissions:
10+
contents: read
11+
12+
jobs:
13+
markdown-link-check:
14+
name: Check Markdown Broken Links
15+
runs-on: ubuntu-latest
16+
17+
steps:
18+
- name: Checkout Repo
19+
uses: actions/checkout@v4
20+
with:
21+
fetch-depth: 0
22+
23+
# For PR : Get only changed markdown files
24+
- name: Get changed markdown files (PR only)
25+
id: changed-markdown-files
26+
if: github.event_name == 'pull_request'
27+
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46
28+
with:
29+
files: |
30+
**/*.md
31+
32+
33+
# For PR: Check broken links only in changed files
34+
- name: Check Broken Links in Changed Markdown Files
35+
id: lychee-check-pr
36+
if: github.event_name == 'pull_request' && steps.changed-markdown-files.outputs.any_changed == 'true'
37+
uses: lycheeverse/lychee-action@v2.4.1
38+
with:
39+
args: >
40+
--verbose --exclude-mail --no-progress --exclude ^https?://
41+
${{ steps.changed-markdown-files.outputs.all_changed_files }}
42+
failIfEmpty: false
43+
env:
44+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
45+
46+
# For manual trigger: Check all markdown files in repo
47+
- name: Check Broken Links in All Markdown Files in Entire Repo (Manual Trigger)
48+
id: lychee-check-manual
49+
if: github.event_name == 'workflow_dispatch'
50+
uses: lycheeverse/lychee-action@v2.4.1
51+
with:
52+
args: >
53+
--verbose --exclude-mail --no-progress --exclude ^https?://
54+
'**/*.md'
55+
failIfEmpty: false
56+
env:
57+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: "Create Release"
2+
3+
on:
4+
push:
5+
branches: ["main"]
6+
7+
workflow_dispatch:
8+
9+
permissions:
10+
contents: write
11+
pull-requests: write
12+
13+
jobs:
14+
create-release:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- name: Checkout repository
18+
uses: actions/checkout@v4
19+
with:
20+
ref: ${{ github.sha }}
21+
22+
- uses: codfish/semantic-release-action@v3
23+
id: semantic
24+
with:
25+
tag-format: 'v${version}'
26+
additional-packages: |
27+
['conventional-changelog-conventionalcommits@7']
28+
plugins: |
29+
[
30+
[
31+
"@semantic-release/commit-analyzer",
32+
{
33+
"preset": "conventionalcommits"
34+
}
35+
],
36+
[
37+
"@semantic-release/release-notes-generator",
38+
{
39+
"preset": "conventionalcommits",
40+
"presetConfig": {
41+
"types": [
42+
{ type: 'feat', section: 'Features', hidden: false },
43+
{ type: 'fix', section: 'Bug Fixes', hidden: false },
44+
{ type: 'perf', section: 'Performance Improvements', hidden: false },
45+
{ type: 'revert', section: 'Reverts', hidden: false },
46+
{ type: 'docs', section: 'Other Updates', hidden: false },
47+
{ type: 'style', section: 'Other Updates', hidden: false },
48+
{ type: 'chore', section: 'Other Updates', hidden: false },
49+
{ type: 'refactor', section: 'Other Updates', hidden: false },
50+
{ type: 'test', section: 'Other Updates', hidden: false },
51+
{ type: 'build', section: 'Other Updates', hidden: false },
52+
{ type: 'ci', section: 'Other Updates', hidden: false }
53+
]
54+
}
55+
}
56+
],
57+
'@semantic-release/github'
58+
]
59+
env:
60+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
61+
62+
- run: echo ${{ steps.semantic.outputs.release-version }}
63+
64+
- run: echo "$OUTPUTS"
65+
env:
66+
OUTPUTS: ${{ toJson(steps.semantic.outputs) }}

.github/workflows/pylint.yml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: PyLint
2+
3+
on: [push]
4+
5+
jobs:
6+
build:
7+
runs-on: ubuntu-latest
8+
strategy:
9+
matrix:
10+
python-version: ["3.11"]
11+
steps:
12+
- uses: actions/checkout@v4
13+
14+
- name: Set up Python ${{ matrix.python-version }}
15+
uses: actions/setup-python@v5
16+
with:
17+
python-version: ${{ matrix.python-version }}
18+
19+
- name: Install dependencies
20+
run: |
21+
python -m pip install --upgrade pip
22+
pip install -r requirements.txt
23+
pip install flake8 # Ensure flake8 is installed explicitly
24+
25+
- name: Run flake8
26+
run: |
27+
flake8 src
28+
29+
30+

.github/workflows/stale-bot.yml

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: "Manage Stale Issues, PRs & Unmerged Branches"
2+
on:
3+
schedule:
4+
- cron: '30 1 * * *' # Runs daily at 1:30 AM UTC
5+
workflow_dispatch: # Allows manual triggering
6+
permissions:
7+
contents: write
8+
issues: write
9+
pull-requests: write
10+
jobs:
11+
stale:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Mark Stale Issues and PRs
15+
uses: actions/stale@v9
16+
with:
17+
stale-issue-message: "This issue is stale because it has been open 180 days with no activity. Remove stale label or comment, or it will be closed in 30 days."
18+
stale-pr-message: "This PR is stale because it has been open 180 days with no activity. Please update or it will be closed in 30 days."
19+
days-before-stale: 180
20+
days-before-close: 30
21+
exempt-issue-labels: "keep"
22+
exempt-pr-labels: "keep"
23+
cleanup-branches:
24+
runs-on: ubuntu-latest
25+
steps:
26+
- name: Checkout Repository
27+
uses: actions/checkout@v4
28+
with:
29+
fetch-depth: 0 # Fetch full history for accurate branch checks
30+
- name: Fetch All Branches
31+
run: git fetch --all --prune
32+
- name: List Merged Branches With No Activity in Last 3 Months
33+
run: |
34+
35+
echo "Branch Name,Last Commit Date,Committer,Committed In Branch,Action" > merged_branches_report.csv
36+
37+
for branch in $(git for-each-ref --format '%(refname:short) %(committerdate:unix)' refs/remotes/origin | awk -v date=$(date -d '3 months ago' +%s) '$2 < date {print $1}'); do
38+
if [[ "$branch" != "origin/main" && "$branch" != "origin/dev" ]]; then
39+
branch_name=${branch#origin/}
40+
# Ensure the branch exists locally before getting last commit date
41+
git fetch origin "$branch_name" || echo "Could not fetch branch: $branch_name"
42+
last_commit_date=$(git log -1 --format=%ci "origin/$branch_name" || echo "Unknown")
43+
committer_name=$(git log -1 --format=%cn "origin/$branch_name" || echo "Unknown")
44+
committed_in_branch=$(git branch -r --contains "origin/$branch_name" | tr -d ' ' | paste -sd "," -)
45+
echo "$branch_name,$last_commit_date,$committer_name,$committed_in_branch,Delete" >> merged_branches_report.csv
46+
fi
47+
done
48+
- name: List PR Approved and Merged Branches Older Than 30 Days
49+
run: |
50+
51+
for branch in $(gh api repos/${{ github.repository }}/pulls --jq '.[] | select(.merged_at != null and (.base.ref == "main" or .base.ref == "dev")) | select(.merged_at | fromdateiso8601 < (now - 2592000)) | .head.ref'); do
52+
# Ensure the branch exists locally before getting last commit date
53+
git fetch origin "$branch" || echo "Could not fetch branch: $branch"
54+
last_commit_date=$(git log -1 --format=%ci origin/$branch || echo "Unknown")
55+
committer_name=$(git log -1 --format=%cn origin/$branch || echo "Unknown")
56+
committed_in_branch=$(git branch -r --contains "origin/$branch" | tr -d ' ' | paste -sd "," -)
57+
echo "$branch,$last_commit_date,$committer_name,$committed_in_branch,Delete" >> merged_branches_report.csv
58+
done
59+
env:
60+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
61+
- name: List Open PR Branches With No Activity in Last 3 Months
62+
run: |
63+
64+
for branch in $(gh api repos/${{ github.repository }}/pulls --state open --jq '.[] | select(.base.ref == "main" or .base.ref == "dev") | .head.ref'); do
65+
# Ensure the branch exists locally before getting last commit date
66+
git fetch origin "$branch" || echo "Could not fetch branch: $branch"
67+
last_commit_date=$(git log -1 --format=%ci origin/$branch || echo "Unknown")
68+
committer_name=$(git log -1 --format=%cn origin/$branch || echo "Unknown")
69+
if [[ $(date -d "$last_commit_date" +%s) -lt $(date -d '3 months ago' +%s) ]]; then
70+
# If no commit in the last 3 months, mark for deletion
71+
committed_in_branch=$(git branch -r --contains "origin/$branch" | tr -d ' ' | paste -sd "," -)
72+
echo "$branch,$last_commit_date,$committer_name,$committed_in_branch,Delete" >> merged_branches_report.csv
73+
fi
74+
done
75+
env:
76+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
77+
- name: Upload CSV Report of Inactive Branches
78+
uses: actions/upload-artifact@v4
79+
with:
80+
name: merged-branches-report
81+
path: merged_branches_report.csv
82+
retention-days: 30

infra/scripts/fabric/deploy_fabric_rti.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,10 @@ def main():
365365

366366
print_step(11, 11, "Updating Eventstream Definition", workspace_id=workspace_id, eventstream_id=eventstream_id, eventhouse_database_name=eventhouse_database_name)
367367
try:
368+
# Create destination-friendly names (without underscores) for eventstream
369+
eventhouse_destination_name = eventhouse_name.replace('_', '-')
370+
activator_destination_name = activator_name.replace('_', '-')
371+
368372
eventstream_definition_result = update_eventstream_definition(
369373
workspace_client=workspace_client,
370374
workspace_id=workspace_id,
@@ -375,9 +379,9 @@ def main():
375379
eventhouse_table_name="events",
376380
eventhub_connection_id=eventhub_connection_id,
377381
source_name=event_hub_name,
378-
eventhouse_name=eventhouse_name,
382+
eventhouse_name=eventhouse_destination_name,
379383
stream_name=eventstream_name,
380-
activator_name=activator_name,
384+
activator_name=activator_destination_name,
381385
activator_id=activator_id
382386
)
383387
if eventstream_definition_result is None:

requirements.txt

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,16 @@
33

44
# === FABRIC DEPLOYMENT SCRIPTS (infra/scripts/fabric/) ===
55
# Used by: deploy_fabric_rti.py, fabric_*.py files for Microsoft Fabric configuration deployment
6-
azure-identity>=1.25.1 # Authentication for Azure services (fabric_api.py, fabric_database.py, fabric_data_ingester.py, fabric_event_hub.py)
7-
azure-core>=1.29.0 # Core Azure SDK functionality (fabric_api.py, graph_api.py)
8-
azure-storage-file-datalake>=12.14.0 # OneLake operations and Data Lake Storage (fabric_api.py)
9-
azure-mgmt-eventhub>=11.2.0 # Event Hub management operations (fabric_event_hub.py)
10-
azure-kusto-data>=6.0.0 # Kusto/KQL database connections and queries (fabric_database.py, fabric_data_ingester.py)
11-
azure-kusto-ingest>=6.0.0 # Data ingestion to Kusto databases (fabric_data_ingester.py)
12-
requests>=2.32.5 # HTTP API calls to Microsoft Fabric REST APIs (fabric_api.py, graph_api.py)
13-
python-dateutil>=2.8.2 # Date/time utilities (fabric_api.py, graph_api.py)
6+
azure-identity==1.25.1 # Authentication for Azure services (fabric_api.py, fabric_database.py, fabric_data_ingester.py, fabric_event_hub.py)
7+
azure-core==1.35.0 # Core Azure SDK functionality (fabric_api.py, graph_api.py)
8+
azure-storage-file-datalake==12.14.0 # OneLake operations and Data Lake Storage (fabric_api.py)
9+
azure-mgmt-eventhub==11.2.0 # Event Hub management operations (fabric_event_hub.py)
10+
azure-kusto-data==6.0.0 # Kusto/KQL database connections and queries (fabric_database.py, fabric_data_ingester.py)
11+
azure-kusto-ingest==6.0.0 # Data ingestion to Kusto databases (fabric_data_ingester.py)
12+
requests==2.32.5 # HTTP API calls to Microsoft Fabric REST APIs (fabric_api.py, graph_api.py)
13+
python-dateutil==2.9.0 # Date/time utilities (fabric_api.py, graph_api.py)
1414

1515
# === EVENT SIMULATION SCRIPTS (infra/scripts/) ===
1616
# Used by: event_simulator.py, sample_data.py, event_hub_service.py for event generation and simulation
17-
azure-eventhub>=5.15.1 # Sending events to Event Hub (event_hub_service.py, event_simulator.py)
18-
pandas>=2.3.3 # Data manipulation and CSV operations (sample_data.py)
17+
azure-eventhub==5.15.1 # Sending events to Event Hub (event_hub_service.py, event_simulator.py)
18+
pandas==2.3.3 # Data manipulation and CSV operations (sample_data.py)

0 commit comments

Comments
 (0)