Skip to content

Commit 0f792b0

Browse files
jaybarden1claude
jaybarden1
and
claude
authored
fix(wallpaper-scrapper,onedrive-sync,app-db): stop duplicate categories and re-classification of already-classified files (#712)
* test(wallpaper-scrapper,onedrive-sync): specify classification dedup and ancestry behaviour (#709 #710 #711) Failing tests for: on-disk filename persisted to FileDetail (#709), tag classification reusing existing hierarchy categories instead of creating parentless Level-3 duplicates (#710), and FileClassifier emitting full ancestry for Level-2/3 keyword matches (#711). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(wallpaper-scrapper,onedrive-sync,app-db): stop duplicate categories and re-classification of already-classified files (#709 #710 #711) - Scrapper now persists the same lowercased, prefixed filename it writes to disk (new ScrapedFileNameFactory), so FileDetailResolver matches the uploaded file and the skip-once-classified rule holds (#709). - Scrapper tag classification reuses an existing category by case-insensitive name across all levels (deepest wins) instead of creating parentless Level-3 duplicates (#710). - FileClassifier emits the full Level1/2/3 ancestry for Level-2/3 keyword matches, so CategoryResolutionService reuses existing hierarchy nodes instead of dropping the leaf or duplicating under Unclassified (#711). - RepairClassificationData migration merges the existing damage: re-points junction rows from duplicate categories to their hierarchy twins, merges prefixed/unprefixed twin FileDetail rows (keeping the on-disk name and a union of classifications), and removes bare Unclassified rows where real classifications exist. Verified against a copy of the live db. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * test(wallpaper-scrapper): specify new tag categories created as Level 2 under the Unclassified root (#710) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(wallpaper-scrapper,app-db): create new tag categories as Level 2 under the Unclassified root (#710) New tag categories were still created as parentless Level 3, which the category validation skips and the hierarchy UI cannot place. They are now created as Level 2 under the shared Unclassified Level-1 root (created on demand, reused when present). The RepairClassificationData migration also re-homes the existing parentless Level-3 rows: self-duplicates and existing children of the root are merged first, then the remainder moves to Level 2 under the root. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
1 parent 937b550 commit 0f792b0

11 files changed

Lines changed: 1744 additions & 39 deletions

File tree

‎apps/desktop/AStar.Dev.OneDrive.Sync.Client.Tests.Unit/Domain/GivenAFileClassifier.cs‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,12 @@ public sealed class GivenAFileClassifier
88
private static FileClassificationCategory Category(string name, int level, bool isFamous = false, bool isInternet = false)
99
=> ((Result<FileClassificationCategory, string>.Ok)FileClassificationCategoryFactory.Create(new(1), name, level, isFamous, isInternet, Option.None<FileClassificationCategoryId>())).Value;
1010

11+
private static FileClassificationCategory ChildCategory(int id, string name, int level, int parentId, bool isFamous = false, bool isInternet = false)
12+
=> ((Result<FileClassificationCategory, string>.Ok)FileClassificationCategoryFactory.Create(new(id), name, level, isFamous, isInternet, Option.Some(new FileClassificationCategoryId(parentId)))).Value;
13+
14+
private static FileClassificationCategory RootCategory(int id, string name)
15+
=> ((Result<FileClassificationCategory, string>.Ok)FileClassificationCategoryFactory.Create(new(id), name, 1, false, false, Option.None<FileClassificationCategoryId>())).Value;
16+
1117
[Fact]
1218
public void when_classifying_segment_based_path_then_path_segments_produce_tokens()
1319
{
@@ -177,6 +183,54 @@ public void when_keyword_has_spaces_and_all_words_appear_as_tokens_then_mapping_
177183
result[0].Level1.ShouldBe("red car");
178184
}
179185

186+
[Fact]
187+
public void when_a_level_two_mapping_matches_then_its_level_one_parent_is_emitted()
188+
{
189+
IReadOnlyList<FileClassificationCategory> mappings = [RootCategory(1, "Color"), ChildCategory(2, "Red", 2, 1)];
190+
191+
var result = FileClassifier.Classify("/photos/red.jpg", mappings);
192+
193+
result.ShouldHaveSingleItem();
194+
result[0].Level1.ShouldBe("Color");
195+
result[0].Level2.ShouldBe(Option.Some("Red"));
196+
result[0].Level3.ShouldBe(Option.None<string>());
197+
}
198+
199+
[Fact]
200+
public void when_a_level_three_mapping_matches_then_its_full_ancestry_is_emitted()
201+
{
202+
IReadOnlyList<FileClassificationCategory> mappings = [RootCategory(1, "Color"), ChildCategory(2, "Red", 2, 1), ChildCategory(3, "Red Car", 3, 2)];
203+
204+
var result = FileClassifier.Classify("/photos/redcar.jpg", mappings);
205+
206+
result.ShouldHaveSingleItem();
207+
result[0].Level1.ShouldBe("Color");
208+
result[0].Level2.ShouldBe(Option.Some("Red"));
209+
result[0].Level3.ShouldBe(Option.Some("Red Car"));
210+
}
211+
212+
[Fact]
213+
public void when_a_level_three_mapping_matches_alongside_its_level_two_parent_then_both_classifications_share_the_ancestry()
214+
{
215+
IReadOnlyList<FileClassificationCategory> mappings = [RootCategory(1, "Color"), ChildCategory(2, "Red", 2, 1), ChildCategory(3, "Red Car", 3, 2)];
216+
217+
var result = FileClassifier.Classify("/photos/red-car.jpg", mappings);
218+
219+
result.Count.ShouldBe(2);
220+
result.ShouldContain(c => c.Level1 == "Color" && c.Level2 == Option.Some("Red") && c.Level3 == Option.None<string>());
221+
result.ShouldContain(c => c.Level1 == "Color" && c.Level2 == Option.Some("Red") && c.Level3 == Option.Some("Red Car"));
222+
}
223+
224+
[Fact]
225+
public void when_a_matched_mapping_has_an_unresolvable_parent_then_that_mapping_is_skipped()
226+
{
227+
IReadOnlyList<FileClassificationCategory> mappings = [ChildCategory(3, "Red Car", 3, 99)];
228+
229+
var result = FileClassifier.Classify("/photos/redcar.jpg", mappings);
230+
231+
result.ShouldBeEmpty();
232+
}
233+
180234
[Fact]
181235
public void when_classifying_path_with_plus_separator_then_plus_delimited_parts_produce_tokens()
182236
{

‎apps/desktop/AStar.Dev.OneDrive.Sync.Client/appsettings.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"AuthorityForMicrosoftAccountsOnly": "https://login.microsoftonline.com/consumers"
88
},
99
"AStarDevOneDriveClient": {
10-
"ApplicationVersion": "0.32.0",
10+
"ApplicationVersion": "0.32.1",
1111
"ApplicationName": "AStar Dev OneDrive Sync Client",
1212
"CacheTag": 1,
1313
"UserPreferencesPath": "astar-dev/astar-dev-onedrive-client",

‎apps/desktop/Scrapper/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAFileClassificationService.cs‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,187 @@ public async Task when_classifying_a_file_that_already_has_classifications_then_
268268
count.ShouldBe(1);
269269
}
270270

271+
[Fact]
272+
public async Task when_classifying_with_a_tag_matching_an_existing_hierarchy_category_then_the_existing_category_is_reused()
273+
{
274+
await using var seedCtx = new AppDbContext(options);
275+
var parent = new FileClassificationCategoryEntity { Name = "Bum", Level = 1 };
276+
seedCtx.FileClassificationCategories.Add(parent);
277+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
278+
var existing = new FileClassificationCategoryEntity { Name = "Ass", Level = 2, ParentId = parent.Id };
279+
seedCtx.FileClassificationCategories.Add(existing);
280+
var fileDetail = new FileDetailEntity
281+
{
282+
FileName = new FileName("test.jpg"),
283+
DirectoryName = new DirectoryName("/tmp"),
284+
FileHandle = FileHandleFactory.Create("test-handle-hierarchy-reuse")
285+
};
286+
seedCtx.Files.Add(fileDetail);
287+
seedCtx.ScrapedTags.Add(new ScrapedTagEntity { Value = "ass", IncludeInSearch = true });
288+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
289+
290+
var pageData = await sut.LoadPageClassificationDataAsync("any-category", TestContext.Current.CancellationToken);
291+
await sut.ClassifyAsync(fileDetail, pageData, ["ass"], TestContext.Current.CancellationToken);
292+
293+
await using var verifyCtx = new AppDbContext(options);
294+
int categoryCount = await verifyCtx.FileClassificationCategories.CountAsync(c => c.Name == "Ass", TestContext.Current.CancellationToken);
295+
var junction = await verifyCtx.FileClassifications.SingleAsync(TestContext.Current.CancellationToken);
296+
categoryCount.ShouldBe(1);
297+
junction.CategoryId.ShouldBe(existing.Id);
298+
}
299+
300+
[Fact]
301+
public async Task when_classifying_with_a_tag_matching_multiple_existing_categories_then_the_deepest_level_category_is_reused()
302+
{
303+
await using var seedCtx = new AppDbContext(options);
304+
var levelOne = new FileClassificationCategoryEntity { Name = "Boobs", Level = 1 };
305+
seedCtx.FileClassificationCategories.Add(levelOne);
306+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
307+
var levelTwo = new FileClassificationCategoryEntity { Name = "Boobs", Level = 2, ParentId = levelOne.Id };
308+
seedCtx.FileClassificationCategories.Add(levelTwo);
309+
var fileDetail = new FileDetailEntity
310+
{
311+
FileName = new FileName("test.jpg"),
312+
DirectoryName = new DirectoryName("/tmp"),
313+
FileHandle = FileHandleFactory.Create("test-handle-deepest-reuse")
314+
};
315+
seedCtx.Files.Add(fileDetail);
316+
seedCtx.ScrapedTags.Add(new ScrapedTagEntity { Value = "boobs", IncludeInSearch = true });
317+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
318+
319+
var pageData = await sut.LoadPageClassificationDataAsync("any-category", TestContext.Current.CancellationToken);
320+
await sut.ClassifyAsync(fileDetail, pageData, ["boobs"], TestContext.Current.CancellationToken);
321+
322+
await using var verifyCtx = new AppDbContext(options);
323+
var junction = await verifyCtx.FileClassifications.SingleAsync(TestContext.Current.CancellationToken);
324+
junction.CategoryId.ShouldBe(levelTwo.Id);
325+
}
326+
327+
[Fact]
328+
public async Task when_classifying_with_a_tag_matching_an_existing_category_with_different_casing_then_no_duplicate_category_is_created()
329+
{
330+
await using var seedCtx = new AppDbContext(options);
331+
var parent = new FileClassificationCategoryEntity { Name = "Clothes", Level = 1 };
332+
seedCtx.FileClassificationCategories.Add(parent);
333+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
334+
var existing = new FileClassificationCategoryEntity { Name = "BATHROBES", Level = 2, ParentId = parent.Id };
335+
seedCtx.FileClassificationCategories.Add(existing);
336+
var fileDetail = new FileDetailEntity
337+
{
338+
FileName = new FileName("test.jpg"),
339+
DirectoryName = new DirectoryName("/tmp"),
340+
FileHandle = FileHandleFactory.Create("test-handle-casing-reuse")
341+
};
342+
seedCtx.Files.Add(fileDetail);
343+
seedCtx.ScrapedTags.Add(new ScrapedTagEntity { Value = "bathrobes", IncludeInSearch = true });
344+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
345+
346+
var pageData = await sut.LoadPageClassificationDataAsync("any-category", TestContext.Current.CancellationToken);
347+
await sut.ClassifyAsync(fileDetail, pageData, ["bathrobes"], TestContext.Current.CancellationToken);
348+
349+
await using var verifyCtx = new AppDbContext(options);
350+
int categoryCount = await verifyCtx.FileClassificationCategories.CountAsync(c => EF.Functions.Collate(c.Name, "NOCASE") == "Bathrobes", TestContext.Current.CancellationToken);
351+
var junction = await verifyCtx.FileClassifications.SingleAsync(TestContext.Current.CancellationToken);
352+
categoryCount.ShouldBe(1);
353+
junction.CategoryId.ShouldBe(existing.Id);
354+
}
355+
356+
[Fact]
357+
public async Task when_classifying_with_a_tag_matching_no_existing_category_then_a_level_two_category_under_the_unclassified_root_is_created()
358+
{
359+
var fileDetail = new FileDetailEntity
360+
{
361+
FileName = new FileName("test.jpg"),
362+
DirectoryName = new DirectoryName("/tmp"),
363+
FileHandle = FileHandleFactory.Create("test-handle-new-tag")
364+
};
365+
await using var seedCtx = new AppDbContext(options);
366+
seedCtx.Files.Add(fileDetail);
367+
seedCtx.ScrapedTags.Add(new ScrapedTagEntity { Value = "brand new tag", IncludeInSearch = true });
368+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
369+
370+
var pageData = await sut.LoadPageClassificationDataAsync("any-category", TestContext.Current.CancellationToken);
371+
await sut.ClassifyAsync(fileDetail, pageData, ["brand new tag"], TestContext.Current.CancellationToken);
372+
373+
await using var verifyCtx = new AppDbContext(options);
374+
var created = await verifyCtx.FileClassificationCategories.SingleAsync(c => c.Name == "Brand New Tag", TestContext.Current.CancellationToken);
375+
var root = await verifyCtx.FileClassificationCategories.SingleAsync(c => c.Name == "Unclassified", TestContext.Current.CancellationToken);
376+
created.Level.ShouldBe(2);
377+
created.ParentId.ShouldBe(root.Id);
378+
root.Level.ShouldBe(1);
379+
root.ParentId.ShouldBeNull();
380+
}
381+
382+
[Fact]
383+
public async Task when_classifying_with_a_new_tag_and_the_unclassified_root_already_exists_then_it_is_reused()
384+
{
385+
var fileDetail = new FileDetailEntity
386+
{
387+
FileName = new FileName("test.jpg"),
388+
DirectoryName = new DirectoryName("/tmp"),
389+
FileHandle = FileHandleFactory.Create("test-handle-existing-root")
390+
};
391+
await using var seedCtx = new AppDbContext(options);
392+
var root = new FileClassificationCategoryEntity { Name = "Unclassified", Level = 1 };
393+
seedCtx.FileClassificationCategories.Add(root);
394+
seedCtx.Files.Add(fileDetail);
395+
seedCtx.ScrapedTags.Add(new ScrapedTagEntity { Value = "another new tag", IncludeInSearch = true });
396+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
397+
398+
var pageData = await sut.LoadPageClassificationDataAsync("any-category", TestContext.Current.CancellationToken);
399+
await sut.ClassifyAsync(fileDetail, pageData, ["another new tag"], TestContext.Current.CancellationToken);
400+
401+
await using var verifyCtx = new AppDbContext(options);
402+
var created = await verifyCtx.FileClassificationCategories.SingleAsync(c => c.Name == "Another New Tag", TestContext.Current.CancellationToken);
403+
int rootCount = await verifyCtx.FileClassificationCategories.CountAsync(c => c.Name == "Unclassified", TestContext.Current.CancellationToken);
404+
created.Level.ShouldBe(2);
405+
created.ParentId.ShouldBe(root.Id);
406+
rootCount.ShouldBe(1);
407+
}
408+
409+
[Fact]
410+
public async Task when_classifying_with_two_new_tags_then_both_share_a_single_unclassified_root()
411+
{
412+
var fileDetail = new FileDetailEntity
413+
{
414+
FileName = new FileName("test.jpg"),
415+
DirectoryName = new DirectoryName("/tmp"),
416+
FileHandle = FileHandleFactory.Create("test-handle-two-new-tags")
417+
};
418+
await using var seedCtx = new AppDbContext(options);
419+
seedCtx.Files.Add(fileDetail);
420+
seedCtx.ScrapedTags.Add(new ScrapedTagEntity { Value = "first new tag", IncludeInSearch = true });
421+
seedCtx.ScrapedTags.Add(new ScrapedTagEntity { Value = "second new tag", IncludeInSearch = true });
422+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
423+
424+
var pageData = await sut.LoadPageClassificationDataAsync("any-category", TestContext.Current.CancellationToken);
425+
await sut.ClassifyAsync(fileDetail, pageData, ["first new tag", "second new tag"], TestContext.Current.CancellationToken);
426+
427+
await using var verifyCtx = new AppDbContext(options);
428+
int rootCount = await verifyCtx.FileClassificationCategories.CountAsync(c => c.Name == "Unclassified", TestContext.Current.CancellationToken);
429+
int childCount = await verifyCtx.FileClassificationCategories.CountAsync(c => c.Level == 2 && c.Name != "Unclassified", TestContext.Current.CancellationToken);
430+
rootCount.ShouldBe(1);
431+
childCount.ShouldBe(2);
432+
}
433+
434+
[Fact]
435+
public async Task when_resolving_the_category_classification_matching_an_existing_hierarchy_category_then_the_existing_category_is_reused()
436+
{
437+
await using var seedCtx = new AppDbContext(options);
438+
var parent = new FileClassificationCategoryEntity { Name = "Person", Level = 1 };
439+
seedCtx.FileClassificationCategories.Add(parent);
440+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
441+
var existing = new FileClassificationCategoryEntity { Name = "Animals", Level = 2, ParentId = parent.Id };
442+
seedCtx.FileClassificationCategories.Add(existing);
443+
seedCtx.ScrapeConfiguration.Add(CreateScrapeConfigEntityWithCategory("cat1", "animals", true));
444+
await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken);
445+
446+
var result = await sut.LoadPageClassificationDataAsync("cat1", TestContext.Current.CancellationToken);
447+
448+
result.CategoryClassification.ShouldNotBeNull();
449+
result.CategoryClassification.Id.ShouldBe(existing.Id);
450+
}
451+
271452
[Fact]
272453
public async Task when_exporting_classifications_then_categories_and_keywords_are_returned()
273454
{

‎apps/desktop/Scrapper/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenAScrapedFileNameFactory.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
using AStar.Dev.Wallpaper.Scrapper.Support;
2+
3+
namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support;
4+
5+
public sealed class GivenAScrapedFileNameFactory
6+
{
7+
[Fact]
8+
public void when_a_prefix_is_supplied_then_the_name_is_the_lowercased_prefix_and_filename()
9+
{
10+
string result = ScrapedFileNameFactory.Create("Azami", "https://w.wallhaven.cc/full/p9/Wallhaven-P9ldle.JPG");
11+
12+
result.ShouldBe("azami wallhaven-p9ldle.jpg");
13+
}
14+
15+
[Fact]
16+
public void when_no_prefix_is_supplied_then_the_name_is_the_lowercased_filename()
17+
{
18+
string result = ScrapedFileNameFactory.Create(string.Empty, "https://w.wallhaven.cc/full/p9/Wallhaven-P9ldle.JPG");
19+
20+
result.ShouldBe("wallhaven-p9ldle.jpg");
21+
}
22+
23+
[Fact]
24+
public void when_the_prefix_is_null_then_the_name_is_the_lowercased_filename()
25+
{
26+
string result = ScrapedFileNameFactory.Create(null, "https://w.wallhaven.cc/full/p9/wallhaven-p9ldle.jpg");
27+
28+
result.ShouldBe("wallhaven-p9ldle.jpg");
29+
}
30+
}

‎apps/desktop/Scrapper/AStar.Dev.Wallpaper.Scrapper/Services/FileClassificationService.cs‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -162,15 +162,42 @@ private static async Task<FileClassificationCategoryEntity> FindOrCreateClassifi
162162
string normalizedName = name.ToTitleCase();
163163

164164
var tracked = context.ChangeTracker.Entries<FileClassificationCategoryEntity>()
165-
.FirstOrDefault(e => e.Entity.Level == 3 && e.Entity.ParentId == null && e.Entity.Name.Equals(normalizedName, StringComparison.OrdinalIgnoreCase))?.Entity;
165+
.Select(e => e.Entity)
166+
.Where(e => e.Name.Equals(normalizedName, StringComparison.OrdinalIgnoreCase))
167+
.OrderByDescending(e => e.Level)
168+
.FirstOrDefault();
166169
if (tracked is not null) return tracked;
167170

168171
var existing = await context.FileClassificationCategories
169-
.FirstOrDefaultAsync(c => c.Level == 3 && c.ParentId == null && c.Name == normalizedName, token)
172+
.Where(c => EF.Functions.Collate(c.Name, "NOCASE") == normalizedName)
173+
.OrderByDescending(c => c.Level)
174+
.ThenBy(c => c.Id)
175+
.FirstOrDefaultAsync(token)
176+
.ConfigureAwait(false);
177+
if (existing is not null) return existing;
178+
179+
var root = await FindOrCreateUnclassifiedRootAsync(context, token).ConfigureAwait(false);
180+
var created = new FileClassificationCategoryEntity { Name = normalizedName, Level = 2, Parent = root };
181+
context.FileClassificationCategories.Add(created);
182+
183+
return created;
184+
}
185+
186+
private static async Task<FileClassificationCategoryEntity> FindOrCreateUnclassifiedRootAsync(AppDbContext context, CancellationToken token)
187+
{
188+
const string rootName = "Unclassified";
189+
190+
var tracked = context.ChangeTracker.Entries<FileClassificationCategoryEntity>()
191+
.Select(e => e.Entity)
192+
.FirstOrDefault(e => e.Level == 1 && e.Name.Equals(rootName, StringComparison.OrdinalIgnoreCase));
193+
if (tracked is not null) return tracked;
194+
195+
var existing = await context.FileClassificationCategories
196+
.FirstOrDefaultAsync(c => c.Level == 1 && EF.Functions.Collate(c.Name, "NOCASE") == rootName, token)
170197
.ConfigureAwait(false);
171198
if (existing is not null) return existing;
172199

173-
var created = new FileClassificationCategoryEntity { Name = normalizedName, Level = 3 };
200+
var created = new FileClassificationCategoryEntity { Name = rootName, Level = 1 };
174201
context.FileClassificationCategories.Add(created);
175202

176203
return created;

0 commit comments

Comments
 (0)