forked from ostris/ai-toolkit
-
Notifications
You must be signed in to change notification settings - Fork 3
Expand file tree
/
Copy pathstart_toolkit.ps1
More file actions
376 lines (324 loc) · 11.7 KB
/
start_toolkit.ps1
File metadata and controls
376 lines (324 loc) · 11.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
# Startup script for AI Toolkit on Windows
# This script sets up the environment and launches the toolkit
$ErrorActionPreference = "Stop"
# Colors for output
function Write-Info {
Write-Host "[INFO] $args" -ForegroundColor Blue
}
function Write-Success {
Write-Host "[SUCCESS] $args" -ForegroundColor Green
}
function Write-Warning {
Write-Host "[WARNING] $args" -ForegroundColor Yellow
}
function Write-Error {
Write-Host "[ERROR] $args" -ForegroundColor Red
}
# Default values
$MODE = "help"
$CONFIG_FILE = ""
$RECOVER = $false
$JOB_NAME = ""
$LOG_FILE = ""
$UI_PORT = 8675
$UI_DEV_MODE = $false
# Function to show usage
function Show-Usage {
Write-Host "AI Toolkit Startup Script"
Write-Host "=========================="
Write-Host ""
Write-Host "Usage: .\start_toolkit.ps1 [MODE] [OPTIONS]"
Write-Host ""
Write-Host "Modes:"
Write-Host " setup - Setup/validate the toolkit environment"
Write-Host " train <config_file> [config_file2 ...] - Run training job(s) with config file(s)"
Write-Host " gradio - Launch Gradio UI for FLUX training"
Write-Host " ui - Launch web UI (Next.js, production mode)"
Write-Host " ui --dev - Launch web UI (development mode with hot reload)"
Write-Host " help - Show this help message"
Write-Host ""
Write-Host "Training Options:"
Write-Host " -r, --recover - Continue running additional jobs if one fails"
Write-Host " -n, --name NAME - Name to replace [name] tag in config"
Write-Host " -l, --log FILE - Log file to write output to"
Write-Host ""
Write-Host "UI Options:"
Write-Host " -p, --port PORT - Port for web UI (default: 8675)"
Write-Host ""
Write-Host "Examples:"
Write-Host " .\start_toolkit.ps1 train config\examples\train_lora_wan22_14b_24gb.yaml"
Write-Host " .\start_toolkit.ps1 train config\my_config.yaml -r -n my_training_run"
Write-Host " .\start_toolkit.ps1 gradio"
Write-Host " .\start_toolkit.ps1 ui"
Write-Host ""
}
# Function to get Python executable path
function Get-PythonCmd {
# If we're in a virtual environment, use its Python
if ($env:VIRTUAL_ENV) {
return "$env:VIRTUAL_ENV\Scripts\python.exe"
}
# Check for uv venv
if ((Test-Path ".venv") -and (Test-Path ".venv\Scripts\python.exe")) {
return ".venv\Scripts\python.exe"
}
# Check for standard venv
if ((Test-Path "venv") -and (Test-Path "venv\Scripts\python.exe")) {
return "venv\Scripts\python.exe"
}
# Fall back to system python
return "python"
}
# Function to check and activate virtual environment
function Setup-Venv {
# Check if we're already in a virtual environment
if ($env:VIRTUAL_ENV) {
Write-Info "Virtual environment already active: $env:VIRTUAL_ENV"
return
}
# Check for uv venv
if (Test-Path ".venv") {
Write-Info "Activating uv virtual environment..."
& .venv\Scripts\Activate.ps1
Write-Success "Virtual environment activated"
return
}
# Check for standard venv
if (Test-Path "venv") {
Write-Info "Activating virtual environment..."
& venv\Scripts\Activate.ps1
Write-Success "Virtual environment activated"
return
}
Write-Warning "No virtual environment found. Using system Python."
Write-Info "Consider creating one with: python -m venv venv"
}
# Function to detect backend and set ROCm environment variables
function Detect-Backend {
Write-Info "Detecting GPU backend..."
$pythonCmd = Get-PythonCmd
try {
$backendOutput = & $pythonCmd -c "import torch; print('ROCm' if hasattr(torch.version, 'hip') and torch.version.hip else 'CUDA')" 2>&1
if ($LASTEXITCODE -eq 0) {
$script:BACKEND = $backendOutput
Write-Success "Backend detected: $BACKEND"
if ($BACKEND -eq "ROCm") {
# Set ROCm environment variables (Windows ROCm support is limited)
Write-Info "ROCm detected. Note: ROCm on Windows has limited support."
Write-Info "Consider using CUDA on Windows for better compatibility."
}
# Verify GPU is available
$gpuCheck = & $pythonCmd -c "import torch; exit(0 if torch.cuda.is_available() else 1)" 2>&1
if ($LASTEXITCODE -eq 0) {
$deviceName = & $pythonCmd -c "import torch; print(torch.cuda.get_device_name(0))" 2>&1
Write-Success "GPU detected: $deviceName"
} else {
Write-Warning "No GPU detected. Training will run on CPU (very slow)."
}
} else {
Write-Warning "PyTorch not available. Make sure it's installed."
$script:BACKEND = "UNKNOWN"
}
} catch {
Write-Warning "PyTorch not available. Make sure it's installed."
$script:BACKEND = "UNKNOWN"
}
}
# Function to verify dependencies
function Verify-Dependencies {
Write-Info "Verifying dependencies..."
$pythonCmd = Get-PythonCmd
try {
& $pythonCmd -c "import torch" 2>&1 | Out-Null
if ($LASTEXITCODE -ne 0) {
Write-Error "PyTorch is not installed!"
Write-Info "Run: .\setup.ps1 to install dependencies"
Write-Info "Or manually: pip install torch torchvision torchaudio"
return $false
}
} catch {
Write-Error "PyTorch is not installed!"
Write-Info "Run: .\setup.ps1 to install dependencies"
return $false
}
# Check for other critical dependencies
$missingDeps = @()
$deps = @("accelerate", "diffusers", "transformers")
foreach ($dep in $deps) {
try {
& $pythonCmd -c "import $dep" 2>&1 | Out-Null
if ($LASTEXITCODE -ne 0) {
$missingDeps += $dep
}
} catch {
$missingDeps += $dep
}
}
if ($missingDeps.Count -gt 0) {
Write-Warning "Missing dependencies: $($missingDeps -join ', ')"
Write-Info "Install with: pip install -r requirements.txt"
return $false
} else {
Write-Success "Core dependencies verified"
return $true
}
}
# Parse arguments
function Parse-Args {
if ($args.Count -eq 0) {
Show-Usage
exit 0
}
$script:MODE = $args[0]
$remainingArgs = $args[1..($args.Count - 1)]
$i = 0
while ($i -lt $remainingArgs.Count) {
$arg = $remainingArgs[$i]
switch ($arg) {
{ $_ -in "-r", "--recover" } {
$script:RECOVER = $true
}
{ $_ -in "-n", "--name" } {
$script:JOB_NAME = $remainingArgs[$i + 1]
$i++
}
{ $_ -in "-l", "--log" } {
$script:LOG_FILE = $remainingArgs[$i + 1]
$i++
}
{ $_ -in "-p", "--port" } {
$script:UI_PORT = $remainingArgs[$i + 1]
$i++
}
"--dev" {
if ($MODE -eq "ui") {
$script:UI_DEV_MODE = $true
} else {
Write-Warning "--dev flag is only valid for 'ui' mode"
}
}
default {
if ($MODE -eq "train") {
if ([string]::IsNullOrEmpty($CONFIG_FILE)) {
$script:CONFIG_FILE = $arg
} else {
$script:CONFIG_FILE = "$CONFIG_FILE $arg"
}
}
}
}
$i++
}
# Validate config file for train mode
if ($MODE -eq "train" -and [string]::IsNullOrEmpty($CONFIG_FILE)) {
Write-Error "No config file specified for training mode"
Show-Usage
exit 1
}
}
# Main execution
function Main {
Write-Info "AI Toolkit Startup Script"
Write-Info "=========================="
Write-Host ""
# Parse arguments
Parse-Args $args
# Setup environment
Setup-Venv
# For setup mode, skip dependency verification initially
if ($MODE -ne "setup") {
Detect-Backend
if (-not (Verify-Dependencies)) {
Write-Error "Dependencies not satisfied. Run '.\start_toolkit.ps1 setup' to install them."
exit 1
}
}
Write-Host ""
Write-Info "Starting in mode: $MODE"
Write-Host ""
# Execute based on mode
switch ($MODE) {
"setup" {
Write-Info "Setting up/validating toolkit environment..."
Write-Info "Please run .\setup.ps1 for full setup"
& .\setup.ps1
}
"train" {
Write-Info "Running training job(s)..."
$pythonCmd = Get-PythonCmd
$cmdArgs = @("run.py")
# Add config files
foreach ($config in $CONFIG_FILE.Split(' ')) {
if (-not [string]::IsNullOrEmpty($config)) {
$cmdArgs += $config
}
}
# Add options
if ($RECOVER) {
$cmdArgs += "--recover"
}
if (-not [string]::IsNullOrEmpty($JOB_NAME)) {
$cmdArgs += "--name"
$cmdArgs += $JOB_NAME
}
if (-not [string]::IsNullOrEmpty($LOG_FILE)) {
$cmdArgs += "--log"
$cmdArgs += $LOG_FILE
}
Write-Info "Command: $pythonCmd $($cmdArgs -join ' ')"
& $pythonCmd $cmdArgs
}
"gradio" {
Write-Info "Launching Gradio UI..."
$pythonCmd = Get-PythonCmd
try {
& $pythonCmd -c "import gradio" 2>&1 | Out-Null
if ($LASTEXITCODE -ne 0) {
Write-Error "Gradio is not installed!"
Write-Info "Install with: pip install gradio"
exit 1
}
} catch {
Write-Error "Gradio is not installed!"
Write-Info "Install with: pip install gradio"
exit 1
}
& $pythonCmd flux_train_ui.py
}
"ui" {
Write-Info "Launching web UI on port $UI_PORT..."
if (-not (Test-Path "ui")) {
Write-Error "UI directory not found!"
exit 1
}
Push-Location ui
if (-not (Test-Path "node_modules")) {
Write-Info "Installing UI dependencies..."
npm install
}
# Check if --dev flag is set for development mode with hot reload
if ($UI_DEV_MODE) {
Write-Info "Starting UI in DEVELOPMENT mode (hot reload enabled)..."
Write-Info "UI will be available at http://localhost:3000 (or next available port)"
$env:PORT = $UI_PORT
npm run dev
} else {
Write-Info "Starting UI in PRODUCTION mode..."
Write-Info "To use dev mode with hot reload, run: .\start_toolkit.ps1 ui --dev"
$env:PORT = $UI_PORT
npm run build_and_start
}
Pop-Location
}
"help" {
Show-Usage
}
default {
Write-Error "Unknown mode: $MODE"
Show-Usage
exit 1
}
}
}
# Run main function
Main $args