Skip to content

Commit 8ba6510

Browse files
mohnjilesmohnjiles
authored
Merge pull request LykosAI#1212 from ionite34/things-n-stuff-2
Add Unicode support for PNG metadata, configure portable Git, and improve environment variable handling
2 parents ba1879f + e188a36 commit 8ba6510

14 files changed

Lines changed: 764 additions & 132 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2
99
### Fixed
1010
- Fixed an issue where `Align Your Steps` scheduler and Unet Loader workflows ignored Regional Prompting (and other addon) conditioning modifiers.
1111

12+
## v2.16.0-pre.1
13+
### Added
14+
- Added enable/disable toggle for environment variables in Settings, allowing variables to be temporarily disabled without deleting them
15+
### Changed
16+
- Improved safetensor checkpoint classification to correctly detect UNet-only models for Wan Video, HiDream, Z-Image, Hunyuan3D, and diffusers-format Flux architectures, ensuring they are routed to the DiffusionModels folder
17+
- GGUF checkpoint downloads now go directly to the DiffusionModels folder instead of StableDiffusion
18+
- Configured portable Git to suppress detached HEAD advice messages
19+
### Fixed
20+
- Fixed downloaded checkpoint going to StableDiffusion folder when a saved download preference existed, even for GGUF files that should always go to DiffusionModels
21+
- Fixed potential crash when adding metadata to malformed or non-PNG image data in Inference
22+
- Fixed non-Latin-1 characters (e.g. Japanese, Chinese, Korean, emoji) in image generation parameters being stored in PNG tEXt chunks, violating the PNG specification and causing character corruption (mojibake) in standard-compliant parsers. Non-Latin-1 content now uses spec-compliant iTXt chunks with proper UTF-8 encoding ([#1535](https://github.com/LykosAI/StabilityMatrix/issues/1535))
23+
1224
## v2.16.0-dev.2
1325
### Added
1426
- Added Regional Prompting addon to Inference - paint detailed masks to apply different prompts, strengths, and settings to specific regions of your image

‎StabilityMatrix.Avalonia/Helpers/PngDataHelper.cs‎

Lines changed: 103 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,30 @@
33
using System.IO;
44
using System.Linq;
55
using System.Text;
6+
using System.Text.Encodings.Web;
67
using System.Text.Json;
8+
using System.Text.Unicode;
79
using Force.Crc32;
10+
using NLog;
811
using StabilityMatrix.Avalonia.Models;
912
using StabilityMatrix.Core.Models;
1013

1114
namespace StabilityMatrix.Avalonia.Helpers;
1215

1316
public static class PngDataHelper
1417
{
18+
private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
19+
20+
private static readonly byte[] PngHeader = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A];
1521
private static readonly byte[] Idat = { 0x49, 0x44, 0x41, 0x54 };
1622
private static readonly byte[] Text = { 0x74, 0x45, 0x58, 0x74 };
1723
private static readonly byte[] Iend = { 0x49, 0x45, 0x4E, 0x44 };
24+
private static readonly byte[] InternationalText = { 0x69, 0x54, 0x58, 0x74 };
25+
26+
private static readonly JsonSerializerOptions UnicodeJsonOptions = new()
27+
{
28+
Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
29+
};
1830

1931
public static byte[] AddMetadata(
2032
Stream inputStream,
@@ -33,18 +45,47 @@ public static byte[] AddMetadata(
3345
InferenceProjectDocument projectDocument
3446
)
3547
{
48+
// Validate PNG header
49+
if (inputImage.Length < 8 || !inputImage[..8].AsSpan().SequenceEqual(PngHeader))
50+
{
51+
Logger.Warn(
52+
"AddMetadata: Image data ({Size} bytes) does not have a valid PNG header, "
53+
+ "the file may not actually be a PNG. Returning image as-is",
54+
inputImage.Length
55+
);
56+
return inputImage;
57+
}
58+
3659
using var memoryStream = new MemoryStream();
3760
var position = 8; // Skip the PNG signature
3861
memoryStream.Write(inputImage, 0, position);
3962

4063
var metadataInserted = false;
4164

42-
while (position < inputImage.Length)
65+
while (position + 12 <= inputImage.Length)
4366
{
4467
var chunkLength = BitConverter.ToInt32(
4568
inputImage[position..(position + 4)].AsEnumerable().Reverse().ToArray(),
4669
0
4770
);
71+
72+
var totalChunkSize = chunkLength + 12; // 4 (length) + 4 (type) + data + 4 (CRC)
73+
74+
// Validate chunk bounds
75+
if (chunkLength < 0 || position + totalChunkSize > inputImage.Length)
76+
{
77+
// Malformed chunk — write remaining bytes as-is and stop parsing
78+
Logger.Warn(
79+
"Malformed PNG chunk at position {Position}: declared length {ChunkLength} "
80+
+ "exceeds image size {ImageSize}. Image may be truncated or corrupted",
81+
position,
82+
chunkLength,
83+
inputImage.Length
84+
);
85+
memoryStream.Write(inputImage, position, inputImage.Length - position);
86+
break;
87+
}
88+
4889
var chunkType = Encoding.ASCII.GetString(inputImage[(position + 4)..(position + 8)]);
4990

5091
switch (chunkType)
@@ -64,7 +105,7 @@ InferenceProjectDocument projectDocument
64105
}
65106
case "IDAT" when !metadataInserted:
66107
{
67-
var smprojJson = JsonSerializer.Serialize(projectDocument);
108+
var smprojJson = JsonSerializer.Serialize(projectDocument, UnicodeJsonOptions);
68109
var smprojChunk = BuildTextChunk("smproj", smprojJson);
69110

70111
var paramsData =
@@ -75,7 +116,7 @@ InferenceProjectDocument projectDocument
75116
+ $"Model hash: {generationParameters.ModelHash}, Model: {generationParameters.ModelName}";
76117
var paramsChunk = BuildTextChunk("parameters", paramsData);
77118

78-
var paramsJson = JsonSerializer.Serialize(generationParameters);
119+
var paramsJson = JsonSerializer.Serialize(generationParameters, UnicodeJsonOptions);
79120
var paramsJsonChunk = BuildTextChunk("parameters-json", paramsJson);
80121

81122
memoryStream.Write(paramsChunk, 0, paramsChunk.Length);
@@ -88,42 +129,78 @@ InferenceProjectDocument projectDocument
88129
}
89130

90131
// Write the current chunk to the output stream
91-
memoryStream.Write(inputImage, position, chunkLength + 12); // Write the length, type, data, and CRC
92-
position += chunkLength + 12;
132+
memoryStream.Write(inputImage, position, totalChunkSize); // Write the length, type, data, and CRC
133+
position += totalChunkSize;
93134
}
94135

95136
return memoryStream.ToArray();
96137
}
97138

98139
public static byte[] RemoveMetadata(byte[] inputImage)
99140
{
141+
// Validate PNG header
142+
if (inputImage.Length < 8 || !inputImage[..8].AsSpan().SequenceEqual(PngHeader))
143+
{
144+
Logger.Warn(
145+
"RemoveMetadata: Image data ({Size} bytes) does not have a valid PNG header, "
146+
+ "the file may not actually be a PNG. Returning image as-is",
147+
inputImage.Length
148+
);
149+
return inputImage;
150+
}
151+
100152
using var memoryStream = new MemoryStream();
101153
var position = 8; // Skip the PNG signature
102154
memoryStream.Write(inputImage, 0, position);
103155

104-
while (position < inputImage.Length)
156+
while (position + 12 <= inputImage.Length)
105157
{
106158
var chunkLength = BitConverter.ToInt32(
107159
inputImage[position..(position + 4)].AsEnumerable().Reverse().ToArray(),
108160
0
109161
);
162+
163+
var totalChunkSize = chunkLength + 12; // 4 (length) + 4 (type) + data + 4 (CRC)
164+
165+
// Validate chunk bounds
166+
if (chunkLength < 0 || position + totalChunkSize > inputImage.Length)
167+
{
168+
// Malformed chunk — write remaining bytes as-is and stop parsing
169+
Logger.Warn(
170+
"Malformed PNG chunk at position {Position}: declared length {ChunkLength} "
171+
+ "exceeds image size {ImageSize}. Image may be truncated or corrupted",
172+
position,
173+
chunkLength,
174+
inputImage.Length
175+
);
176+
memoryStream.Write(inputImage, position, inputImage.Length - position);
177+
break;
178+
}
179+
110180
var chunkType = Encoding.ASCII.GetString(inputImage[(position + 4)..(position + 8)]);
111181

112182
// If the chunk is not a text chunk, write it to the output
113183
if (chunkType != "tEXt" && chunkType != "zTXt" && chunkType != "iTXt")
114184
{
115-
memoryStream.Write(inputImage, position, chunkLength + 12); // Write the length, type, data, and CRC
185+
memoryStream.Write(inputImage, position, totalChunkSize); // Write the length, type, data, and CRC
116186
}
117187

118188
// Move to the next chunk
119-
position += chunkLength + 12;
189+
position += totalChunkSize;
120190
}
121191

122192
return memoryStream.ToArray();
123193
}
124194

125195
private static byte[] BuildTextChunk(string key, string value)
126196
{
197+
// Use iTXt chunk for non-Latin-1 characters (per PNG specification,
198+
// tEXt chunks only support Latin-1 / ISO 8859-1 encoding)
199+
if (value.Any(c => c > 0xFF))
200+
{
201+
return BuildInternationalTextChunk(key, value);
202+
}
203+
127204
var textData = $"{key}\0{value}";
128205
var dataBytes = Encoding.UTF8.GetBytes(textData);
129206
var textDataLength = BitConverter.GetBytes(dataBytes.Length).AsEnumerable().Reverse().ToArray();
@@ -136,4 +213,22 @@ private static byte[] BuildTextChunk(string key, string value)
136213

137214
return textDataLength.Concat(textDataBytes).Concat(crc).ToArray();
138215
}
216+
217+
private static byte[] BuildInternationalTextChunk(string key, string value)
218+
{
219+
// iTXt chunk format (uncompressed):
220+
// Keyword(Latin-1) \0 CompressionFlag(0) CompressionMethod(0) LanguageTag \0 TranslatedKeyword \0 Text(UTF-8)
221+
var keyBytes = Encoding.Latin1.GetBytes(key);
222+
var valueBytes = Encoding.UTF8.GetBytes(value);
223+
byte[] dataBytes = [.. keyBytes, 0, 0, 0, 0, 0, .. valueBytes];
224+
var dataLength = BitConverter.GetBytes(dataBytes.Length).AsEnumerable().Reverse().ToArray();
225+
var chunkTypeAndData = InternationalText.Concat(dataBytes).ToArray();
226+
var crc = BitConverter
227+
.GetBytes(Crc32Algorithm.Compute(chunkTypeAndData))
228+
.AsEnumerable()
229+
.Reverse()
230+
.ToArray();
231+
232+
return dataLength.Concat(chunkTypeAndData).Concat(crc).ToArray();
233+
}
139234
}

‎StabilityMatrix.Avalonia/Helpers/WindowsPrerequisiteHelper.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -660,6 +660,7 @@ await downloadService.DownloadToFileAsync(
660660
await UnzipGit(progress);
661661

662662
await FixGitLongPaths();
663+
await ConfigurePortableGit();
663664
}
664665

665666
[SupportedOSPlatform("windows")]
@@ -681,6 +682,18 @@ public async Task<bool> FixGitLongPaths()
681682
return false;
682683
}
683684

685+
private async Task ConfigurePortableGit()
686+
{
687+
try
688+
{
689+
await RunGit(["config", "--system", "advice.detachedHead", "false"]);
690+
}
691+
catch (Exception e)
692+
{
693+
Logger.Warn(e, "Failed to configure portable git");
694+
}
695+
}
696+
684697
[SupportedOSPlatform("windows")]
685698
public async Task InstallVcRedistIfNecessary(IProgress<ProgressReport>? progress = null)
686699
{

‎StabilityMatrix.Avalonia/Services/InferenceClientManager.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -652,12 +652,18 @@ .. modelIndexService.FindByModelType(SharedFolderType.Sams).Select(HybridModelFi
652652
);
653653
downloadableClipVisionModelsSource.EditDiff(downloadableClipVisionModels, HybridModelFile.Comparer);
654654

655-
samplersSource.EditDiff(ComfySampler.Defaults, ComfySampler.Comparer);
655+
// When connected, skip resetting samplers/schedulers to defaults only.
656+
// EditDiff would remove server-only items (e.g. custom samplers from extensions),
657+
// causing the ComboBox to clear its selection before LoadSharedPropertiesAsync
658+
// re-fetches the full list from the server.
659+
if (!IsConnected)
660+
{
661+
samplersSource.EditDiff(ComfySampler.Defaults, ComfySampler.Comparer);
662+
schedulersSource.EditDiff(ComfyScheduler.Defaults, ComfyScheduler.Comparer);
663+
}
656664

657665
latentUpscalersSource.EditDiff(ComfyUpscaler.Defaults, ComfyUpscaler.Comparer);
658666

659-
schedulersSource.EditDiff(ComfyScheduler.Defaults, ComfyScheduler.Comparer);
660-
661667
// Load Upscalers
662668
modelUpscalersSource.EditDiff(
663669
modelIndexService

0 commit comments

Comments
 (0)