Transliterate Liquid Filter (#19009)

urbanit · Piedone · web-flow · commit 671fc8bdef0a · 2026-04-13T22:28:44.000+02:00
Co-authored-by: Zoltán Lehóczky &lt;zoltan.lehoczky@lombiq.com&gt;
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -4,6 +4,7 @@
     <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
   <ItemGroup>
+    <PackageVersion Include="AnyAscii" Version="0.3.2" />
     <PackageVersion Include="AngleSharp" Version="1.4.0" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.19.4" />
     <PackageVersion Include="AWSSDK.Extensions.NETCore.Setup" Version="4.0.3.28" />
diff --git a/src/OrchardCore.Cms.Web/appsettings.json b/src/OrchardCore.Cms.Web/appsettings.json
@@ -62,6 +62,10 @@
     //  "MaxUploadChunkSize": 104857600,
     //  "TemporaryFileLifetime": "01:00:00"
     //},
+    // See https://docs.orchardcore.net/en/latest/reference/modules/Media/#configuration to configure Media Slugify.
+    //"OrchardCore_Media_Slugify": {
+    //  "Transliterate": true
+    //},
     // See https://docs.orchardcore.net/en/latest/reference/modules/Media.AmazonS3/#configuration to configure media storage in Amazon S3 Storage.
     //"OrchardCore_Media_AmazonS3": {
     //  "Region": "eu-central-1",
diff --git a/src/OrchardCore.Modules/OrchardCore.Liquid/Filters/SlugifyFilter.cs b/src/OrchardCore.Modules/OrchardCore.Liquid/Filters/SlugifyFilter.cs
@@ -14,8 +14,9 @@ public SlugifyFilter(ISlugService slugService)
     }
     public ValueTask<FluidValue> ProcessAsync(FluidValue input, FilterArguments arguments, LiquidTemplateContext ctx)
     {
-        var text = input.ToStringValue();
+        var transliterateArg = arguments["transliterate"];
+        var transliterate = transliterateArg.IsNil() || transliterateArg.ToBooleanValue();
 
-        return new StringValue(_slugService.Slugify(text));
+        return new StringValue(_slugService.Slugify(input.ToStringValue(), transliterate));
     }
 }
diff --git a/src/OrchardCore.Modules/OrchardCore.Liquid/Filters/TransliterateFilter.cs b/src/OrchardCore.Modules/OrchardCore.Liquid/Filters/TransliterateFilter.cs
@@ -0,0 +1,20 @@
+using AnyAscii;
+using Fluid;
+using Fluid.Values;
+
+namespace OrchardCore.Liquid.Filters;
+
+public class TransliterateFilter : ILiquidFilter
+{
+    public ValueTask<FluidValue> ProcessAsync(FluidValue input, FilterArguments arguments, LiquidTemplateContext ctx)
+    {
+        var text = input.ToStringValue();
+
+        if (string.IsNullOrEmpty(text))
+        {
+            return ValueTask.FromResult(input);
+        }
+
+        return ValueTask.FromResult<FluidValue>(StringValue.Create(text.Transliterate()));
+    }
+}
diff --git a/src/OrchardCore.Modules/OrchardCore.Liquid/OrchardCore.Liquid.csproj b/src/OrchardCore.Modules/OrchardCore.Liquid/OrchardCore.Liquid.csproj
@@ -27,6 +27,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="AnyAscii" />
     <PackageReference Include="System.IO.Hashing" />
   </ItemGroup>
 
diff --git a/src/OrchardCore.Modules/OrchardCore.Liquid/Startup.cs b/src/OrchardCore.Modules/OrchardCore.Liquid/Startup.cs
@@ -72,6 +72,7 @@ public override void ConfigureServices(IServiceCollection services)
         .AddLiquidFilter<LocalTimeZoneFilter>("local")
         .AddLiquidFilter<UtcTimeZoneFilter>("utc")
         .AddLiquidFilter<SlugifyFilter>("slugify")
+        .AddLiquidFilter<TransliterateFilter>("transliterate")
         .AddLiquidFilter<LiquidFilter>("liquid")
         .AddLiquidFilter<ContentUrlFilter>("href")
         .AddLiquidFilter<AbsoluteUrlFilter>("absolute_url")
diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaSlugifyOptionsConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaSlugifyOptionsConfiguration.cs
@@ -0,0 +1,22 @@
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
+using OrchardCore.Environment.Shell.Configuration;
+
+namespace OrchardCore.Media.Services;
+
+public sealed class MediaSlugifyOptionsConfiguration : IConfigureOptions<MediaSlugifyOptions>
+{
+    private readonly IShellConfiguration _shellConfiguration;
+
+    public MediaSlugifyOptionsConfiguration(IShellConfiguration shellConfiguration)
+    {
+        _shellConfiguration = shellConfiguration;
+    }
+
+    public void Configure(MediaSlugifyOptions options)
+    {
+        var section = _shellConfiguration.GetSection("OrchardCore_Media_Slugify");
+
+        options.Transliterate = section.GetValue(nameof(options.Transliterate), true);
+    }
+}
diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/SlugifyMediaNameNormalizerService.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/SlugifyMediaNameNormalizerService.cs
@@ -1,23 +1,28 @@
+using Microsoft.Extensions.Options;
 using OrchardCore.Modules.Services;
 
 namespace OrchardCore.Media.Services;
 
 public class SlugifyMediaNameNormalizerService : IMediaNameNormalizerService
 {
     private readonly ISlugService _slugService;
+    private readonly MediaSlugifyOptions _options;
 
-    public SlugifyMediaNameNormalizerService(ISlugService slugService)
+    public SlugifyMediaNameNormalizerService(
+        ISlugService slugService,
+        IOptions<MediaSlugifyOptions> options)
     {
         _slugService = slugService;
+        _options = options.Value;
     }
 
     public string NormalizeFolderName(string folderName)
     {
-        return _slugService.Slugify(folderName);
+        return _slugService.Slugify(folderName, _options.Transliterate);
     }
 
     public string NormalizeFileName(string fileName)
     {
-        return _slugService.Slugify(Path.GetFileNameWithoutExtension(fileName)) + Path.GetExtension(fileName);
+        return _slugService.Slugify(Path.GetFileNameWithoutExtension(fileName), _options.Transliterate) + Path.GetExtension(fileName);
     }
 }
diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs b/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs
@@ -246,6 +246,8 @@ public sealed class MediaSlugifyStartup : StartupBase
 {
     public override void ConfigureServices(IServiceCollection services)
     {
+        services.AddTransient<IConfigureOptions<MediaSlugifyOptions>, MediaSlugifyOptionsConfiguration>();
+
         // Media Name Normalizer
         services.AddScoped<IMediaNameNormalizerService, SlugifyMediaNameNormalizerService>();
     }
diff --git a/src/OrchardCore/OrchardCore.Abstractions/Modules/Services/ISlugService.cs b/src/OrchardCore/OrchardCore.Abstractions/Modules/Services/ISlugService.cs
@@ -9,6 +9,15 @@ public interface ISlugService
     /// <returns>The slug created from the input text.</returns>
     string Slugify(string text);
 
+    /// <summary>
+    /// Transforms specified text to the form suitable for URL slugs,
+    /// optionally transliterating non-Latin characters to their ASCII equivalents first.
+    /// </summary>
+    /// <param name="text">The text to transform.</param>
+    /// <param name="transliterate">Whether to transliterate non-Latin characters before slugifying.</param>
+    /// <returns>The slug created from the input text.</returns>
+    string Slugify(string text, bool transliterate);
+
     /// <summary>
     /// Transforms specified text to a custom form generally not suitable for URL slugs.
     /// Allows you to use a specified separator char.
diff --git a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaSlugifyOptions.cs b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaSlugifyOptions.cs
@@ -0,0 +1,6 @@
+namespace OrchardCore.Media;
+
+public class MediaSlugifyOptions
+{
+    public bool Transliterate { get; set; } = true;
+}
diff --git a/src/OrchardCore/OrchardCore/Modules/Services/SlugService.cs b/src/OrchardCore/OrchardCore/Modules/Services/SlugService.cs
@@ -1,5 +1,6 @@
 using System.Globalization;
 using System.Text;
+using AnyAscii;
 using Cysharp.Text;
 
 namespace OrchardCore.Modules.Services;
@@ -9,6 +10,16 @@ public class SlugService : ISlugService
     private const char Hyphen = '-';
     private const int MaxLength = 1000;
 
+    public string Slugify(string text, bool transliterate)
+    {
+        if (transliterate && !string.IsNullOrEmpty(text))
+        {
+            text = text.Transliterate();
+        }
+
+        return Slugify(text);
+    }
+
     public string Slugify(string text, char separator)
     {
         throw new NotImplementedException();
diff --git a/src/OrchardCore/OrchardCore/OrchardCore.csproj b/src/OrchardCore/OrchardCore/OrchardCore.csproj
@@ -14,6 +14,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="AnyAscii" />
     <PackageReference Include="ncrontab" />
     <PackageReference Include="NodaTime" />
   </ItemGroup>
diff --git a/src/docs/reference/modules/Liquid/README.md b/src/docs/reference/modules/Liquid/README.md
@@ -70,6 +70,22 @@ Blog
 
 ## String Filters
 
+### `transliterate`
+Convert Unicode text (e.g. Greek, Cyrillic, Arabic, Chinese, etc.) into its closest ASCII/Latin representation using the <https://github.com/anyascii/anyascii> library.
+
+Input
+
+```liquid
+{{ "Ελληνικά" | transliterate }}
+```
+
+Output
+
+```text
+Ellinika
+```
+
+
 ### `slugify`
 
 Convert a text into a string that can be used in a URL.
@@ -86,6 +102,23 @@ Output
 this-is-some-text
 ```
 
+There is an option to transliterate (by default `true`) which first transliterates and slugifies afterwards:
+
+```liquid
+{{ "Ελληνικά" | slugify }}
+```
+or 
+```liquid
+{{ "Ελληνικά" | slugify: transliterate: true}}
+```
+
+Output
+
+```text
+ellinika
+```
+
+
 ### `local`
 
 Converts a UTC date and time to the local date and time based on the site settings.
diff --git a/src/docs/reference/modules/Media.Slugify/README.md b/src/docs/reference/modules/Media.Slugify/README.md
@@ -12,5 +12,18 @@ Folders and files can be _slugified_ automatically by enabling the Media Slugify
 
 Different files can have the same slug, making it impossible to upload both without renaming one (e.g. `The team (2020).jpg` and `The Team 2020.jpg`).
 
+By default, transliteration happens when the feature is enabled. The following configuration values are used by default and can be customized:
+
+```json
+{
+  "OrchardCore": {
+    "OrchardCore_Media_Slugify": {
+      // Enable/Disable Transliteration.
+      "Transilterate": true
+    }
+  }
+}
+```
+
 !!! note
     Enabling the Media Slugify feature will not rename existing folders and files, only new folders and files will be _slugified_.
diff --git a/src/docs/releases/3.0.0.md b/src/docs/releases/3.0.0.md
@@ -31,6 +31,14 @@ To re-enable the `liquid` tag, you can configure it in your `appsettings.json`:
 }
 ```
 
+#### Liquid Slugify Filter
+By default, Slugify string filter transliterates the string provided. The same applies to MediaSlugify module (if enabled) for Media files. 
+#### Liquid Slugify Filter
+
+By default, the `slugify` string filter now transliterates the string provided, i.e. it converts Unicode text (e.g. Greek, Cyrillic, Arabic, Chinese, etc.) into its closest ASCII/Latin representation. The same applies to the Media Slugify module (if enabled) for Media files.
+
+There's also a new `transliterate` filter that just does transliteration.
+
 ### Localization Module
 
 Previously, the `ILocalizationService` interface had a `static` default method called `GetAllCulturesAsync()`, which returned a list of all cultures available in the system with their aliases.
diff --git a/test/OrchardCore.Tests/Modules/OrchardCore.Media/SlugifyMediaNameNormalizerServiceTests.cs b/test/OrchardCore.Tests/Modules/OrchardCore.Media/SlugifyMediaNameNormalizerServiceTests.cs
@@ -0,0 +1,79 @@
+using System.Collections.Generic;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
+using OrchardCore.Environment.Shell.Configuration;
+using OrchardCore.Media;
+using OrchardCore.Media.Services;
+using OrchardCore.Modules.Services;
+
+namespace OrchardCore.Tests.Modules.OrchardCore.Media;
+
+public class SlugifyMediaNameNormalizerServiceTests
+{
+    private readonly SlugService _slugService = new();
+
+    [Fact]
+    public void ShouldNormalizeFolderNameWithTransliterationEnabledByDefault()
+    {
+        var options = new MediaSlugifyOptions();
+        var service = new SlugifyMediaNameNormalizerService(_slugService, Options.Create(options));
+
+        var normalizedFolderName = service.NormalizeFolderName("Æther");
+
+        Assert.Equal("aether", normalizedFolderName);
+    }
+
+    [Fact]
+    public void ShouldNormalizeFileNameWithoutTransliterationWhenDisabled()
+    {
+        var options = new MediaSlugifyOptions
+        {
+            Transliterate = false,
+        };
+        var service = new SlugifyMediaNameNormalizerService(_slugService, Options.Create(options));
+
+        var normalizedFileName = service.NormalizeFileName("Æther.png");
+
+        Assert.Equal("æther.png", normalizedFileName);
+    }
+
+    [Fact]
+    public void ShouldDefaultTransliterationToTrueWhenConfigurationIsMissing()
+    {
+        var options = new MediaSlugifyOptions
+        {
+            Transliterate = false,
+        };
+
+        var shellConfiguration = CreateShellConfiguration([]);
+        var configuration = new MediaSlugifyOptionsConfiguration(shellConfiguration);
+
+        configuration.Configure(options);
+
+        Assert.True(options.Transliterate);
+    }
+
+    [Fact]
+    public void ShouldReadTransliterationFromConfiguration()
+    {
+        var shellConfiguration = CreateShellConfiguration(
+        [
+            new KeyValuePair<string, string>("OrchardCore:OrchardCore_Media_Slugify:Transliterate", "false"),
+        ]);
+        var options = new MediaSlugifyOptions();
+        var configuration = new MediaSlugifyOptionsConfiguration(shellConfiguration);
+
+        configuration.Configure(options);
+
+        Assert.False(options.Transliterate);
+    }
+
+    private static ShellConfiguration CreateShellConfiguration(IEnumerable<KeyValuePair<string, string>> values)
+    {
+        var configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(values)
+            .Build();
+
+        return new ShellConfiguration(configuration.GetSection("OrchardCore"));
+    }
+}
diff --git a/test/OrchardCore.Tests/Tokens.Content/SlugServiceTests.cs b/test/OrchardCore.Tests/Tokens.Content/SlugServiceTests.cs
@@ -77,4 +77,18 @@ public void ShouldPreserveNonLatinCharacters(string input, string expected)
         var slug = _slugService.Slugify(input);
         Assert.Equal(expected, slug);
     }
+
+    [Fact]
+    public void ShouldTransliterateWhenRequested()
+    {
+        var slug = _slugService.Slugify("Æneid", transliterate: true);
+        Assert.Equal("aeneid", slug);
+    }
+
+    [Fact]
+    public void ShouldNotTransliterateWhenDisabled()
+    {
+        var slug = _slugService.Slugify("Æneid", transliterate: false);
+        Assert.Equal("æneid", slug);
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -14,8 +14,9 @@ public SlugifyFilter(ISlugService slugService)`
`14`	`14`	`}`
`15`	`15`	`public ValueTask<FluidValue> ProcessAsync(FluidValue input, FilterArguments arguments, LiquidTemplateContext ctx)`
`16`	`16`	`{`
`17`		`- var text = input.ToStringValue();`
	`17`	`+ var transliterateArg = arguments["transliterate"];`
	`18`	`+ var transliterate = transliterateArg.IsNil() \|\| transliterateArg.ToBooleanValue();`
`18`	`19`
`19`		`- return new StringValue(_slugService.Slugify(text));`
	`20`	`+ return new StringValue(_slugService.Slugify(input.ToStringValue(), transliterate));`
`20`	`21`	`}`
`21`	`22`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,23 +1,28 @@`
	`1`	`+using Microsoft.Extensions.Options;`
`1`	`2`	`using OrchardCore.Modules.Services;`
`2`	`3`
`3`	`4`	`namespace OrchardCore.Media.Services;`
`4`	`5`
`5`	`6`	`public class SlugifyMediaNameNormalizerService : IMediaNameNormalizerService`
`6`	`7`	`{`
`7`	`8`	`private readonly ISlugService _slugService;`
	`9`	`+ private readonly MediaSlugifyOptions _options;`
`8`	`10`
`9`		`- public SlugifyMediaNameNormalizerService(ISlugService slugService)`
	`11`	`+ public SlugifyMediaNameNormalizerService(`
	`12`	`+ ISlugService slugService,`
	`13`	`+ IOptions<MediaSlugifyOptions> options)`
`10`	`14`	`{`
`11`	`15`	`_slugService = slugService;`
	`16`	`+ _options = options.Value;`
`12`	`17`	`}`
`13`	`18`
`14`	`19`	`public string NormalizeFolderName(string folderName)`
`15`	`20`	`{`
`16`		`- return _slugService.Slugify(folderName);`
	`21`	`+ return _slugService.Slugify(folderName, _options.Transliterate);`
`17`	`22`	`}`
`18`	`23`
`19`	`24`	`public string NormalizeFileName(string fileName)`
`20`	`25`	`{`
`21`		`- return _slugService.Slugify(Path.GetFileNameWithoutExtension(fileName)) + Path.GetExtension(fileName);`
	`26`	`+ return _slugService.Slugify(Path.GetFileNameWithoutExtension(fileName), _options.Transliterate) + Path.GetExtension(fileName);`
`22`	`27`	`}`
`23`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -246,6 +246,8 @@ public sealed class MediaSlugifyStartup : StartupBase`
`246`	`246`	`{`
`247`	`247`	`public override void ConfigureServices(IServiceCollection services)`
`248`	`248`	`{`
	`249`	`+ services.AddTransient<IConfigureOptions<MediaSlugifyOptions>, MediaSlugifyOptionsConfiguration>();`
	`250`	`+`
`249`	`251`	`// Media Name Normalizer`
`250`	`252`	`services.AddScoped<IMediaNameNormalizerService, SlugifyMediaNameNormalizerService>();`
`251`	`253`	`}`