fix: Add encoding option to Digest->new() for charset normalization

.github/workflows/testsuite.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - run: perl -V
       - name: install dependencies
         uses: perl-actions/install-with-cpm@v1
@@ -46,26 +46,21 @@ jobs:
       matrix:
         perl-version:
           [
+            "5.40",
+            "5.38",
+            "5.36",
+            "5.34",
             "5.32",
             "5.30",
             "5.28",
             "5.26",
-            "5.24",
-            "5.22",
-            "5.20",
-            "5.18",
-            "5.16",
-            "5.14",
-            "5.12",
-            "5.10",
-            "5.8",
           ]
 
     container:
       image: perl:${{ matrix.perl-version }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - run: perl -V
       - name: install dependencies
         uses: perl-actions/install-with-cpm@v1
@@ -77,6 +72,37 @@ jobs:
       - name: make test
         run: make test
 
+  windows:
+    needs: [ubuntu]
+    env:
+      PERL_USE_UNSAFE_INC: 0
+      AUTHOR_TESTING: 1
+      AUTOMATED_TESTING: 1
+      RELEASE_TESTING: 1
+
+    runs-on: windows-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        perl-version: [latest]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Perl
+        uses: shogo82148/actions-setup-perl@v1
+        with:
+          perl-version: ${{ matrix.perl-version }}
+      - run: perl -V
+      - name: install dependencies
+        uses: perl-actions/install-with-cpm@v1
+        with:
+          cpanfile: "cpanfile"
+      - name: Makefile.PL
+        run: perl Makefile.PL
+      - name: make test
+        run: gmake test
+
   macOS:
     needs: [ubuntu]
     env:
@@ -93,7 +119,7 @@ jobs:
         perl-version: [latest]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - run: perl -V
       - name: install dependencies
         uses: perl-actions/install-with-cpm@v1

MANIFEST

@@ -2,12 +2,14 @@ Changes
 digest-bench
 lib/Digest.pm
 lib/Digest/base.pm
+lib/Digest/Encoder.pm
 lib/Digest/file.pm
 Makefile.PL
 MANIFEST			This list of files
 README
 t/base.t
 t/digest.t
+t/encoding.t
 t/file.t
 t/lib/Digest/Dummy.pm
 t/security.t

lib/Digest.pm

@@ -26,6 +26,17 @@ our %MMAP = (
 sub new {
     shift;    # class ignored
     my $algorithm = shift;
+
+    # Extract encoding option if present
+    my $encoding;
+    for (my $i = 0; $i < @_; $i++) {
+        if (defined $_[$i] && $_[$i] eq 'encoding') {
+            $encoding = $_[$i + 1];
+            splice(@_, $i, 2);
+            last;
+        }
+    }
+
     my $impl      = $MMAP{$algorithm} || do {
         $algorithm =~ s/\W+//g;
         "Digest::$algorithm";
@@ -51,7 +62,16 @@ sub new {
                 next;
             }
         }
-        return $class->new( @args, @_ );
+        my $ctx = $class->new( @args, @_ );
+        if ($encoding) {
+            require Encode;
+            require Carp;
+            Encode::find_encoding($encoding)
+                or Carp::croak("Unknown encoding '$encoding'");
+            require Digest::Encoder;
+            $ctx = Digest::Encoder->_wrap($ctx, $encoding);
+        }
+        return $ctx;
     }
     die $err;
 }
@@ -152,6 +172,8 @@ The following methods are available for all C<Digest::> modules:
 
 =item $ctx = Digest->new(XXX => $arg,...)
 
+=item $ctx = Digest->new(XXX => $arg,..., encoding => $enc)
+
 =item $ctx = Digest::XXX->new($arg,...)
 
 The constructor returns some object that encapsulate the state of the
@@ -165,6 +187,22 @@ algorithm names which contains letters which are not legal perl
 identifiers, e.g. "SHA-1".  If no implementation for the given algorithm
 can be found, then an exception is raised.
 
+If the C<encoding> option is provided, all strings passed to C<add()>
+will be encoded using L<Encode> before being digested.  This ensures
+that the same characters always produce the same digest regardless of
+Perl's internal string representation:
+
+  # These will always produce the same digest:
+  my $ctx = Digest->new("SHA-256", encoding => "UTF-8");
+  $ctx->add($unicode_string);
+
+Without an explicit encoding, digest algorithms operate on the raw bytes
+of the string's internal representation, which may differ between Latin-1
+and UTF-8 encoded strings even when they contain the same characters.
+
+The C<encoding> name can be any encoding supported by the L<Encode> module.
+An exception is raised if the encoding is not recognized.
+
 To know what arguments (if any) the constructor takes (the C<$args,...> above)
 consult the docs for the specific digest implementation.
 

lib/Digest/Encoder.pm

@@ -0,0 +1,90 @@
+package Digest::Encoder;
+
+use strict;
+use warnings;
+
+our $VERSION = "1.20";
+
+use Encode ();
+
+sub _wrap {
+    my ($class, $ctx, $encoding) = @_;
+    bless {
+        _ctx      => $ctx,
+        _encoding => $encoding,
+    }, $class;
+}
+
+sub add {
+    my $self = shift;
+    $self->{_ctx}->add(map { Encode::encode($self->{_encoding}, $_) } @_);
+    return $self;
+}
+
+# Methods that return $self for chaining need to return the wrapper
+sub addfile {
+    my $self = shift;
+    $self->{_ctx}->addfile(@_);
+    return $self;
+}
+
+sub add_bits {
+    my $self = shift;
+    $self->{_ctx}->add_bits(@_);
+    return $self;
+}
+
+sub reset {
+    my $self = shift;
+    $self->{_ctx}->reset(@_);
+    return $self;
+}
+
+# Delegate all other methods to the underlying context
+our $AUTOLOAD;
+
+sub AUTOLOAD {
+    my $self = shift;
+    my $method = substr($AUTOLOAD, rindex($AUTOLOAD, '::') + 2);
+    return if $method eq 'DESTROY';
+    $self->{_ctx}->$method(@_);
+}
+
+sub can {
+    my ($self, $method) = @_;
+    my $code = $self->SUPER::can($method);
+    return $code if $code;
+    return $self->{_ctx}->can($method);
+}
+
+sub isa {
+    my ($self, $class) = @_;
+    return 1 if $self->SUPER::isa($class);
+    return $self->{_ctx}->isa($class);
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+Digest::Encoder - Encoding wrapper for Digest objects
+
+=head1 DESCRIPTION
+
+This is an internal wrapper class used by L<Digest> when the C<encoding>
+option is passed to C<< Digest->new() >>. It intercepts C<add()> calls
+and encodes strings using L<Encode> before passing them to the underlying
+digest implementation.
+
+You should not need to use this class directly. Instead, use:
+
+  my $ctx = Digest->new("SHA-256", encoding => "UTF-8");
+  $ctx->add($unicode_string);  # automatically encoded
+
+=head1 SEE ALSO
+
+L<Digest>, L<Encode>
+
+=cut