Remove Google.Protobuf dependency from Microsoft.ML.Tokenizers (#7587)

Co-authored-by: Eric StJohn <ericstj@microsoft.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: 3 people

NuGet.config

@@ -5,7 +5,6 @@
   </solution>
   <packageSources>
     <clear />
-    <add key="darc-pub-dotnet-maintenance-packages-ab95a1f1" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/darc-pub-dotnet-maintenance-packages-ab95a1f1/nuget/v3/index.json" />
     <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
     <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
     <add key="dotnet-libraries" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries/nuget/v3/index.json" />

src/Microsoft.ML.Tokenizers/Microsoft.ML.Tokenizers.csproj

@@ -6,6 +6,7 @@
     <IsPackable>true</IsPackable>
     <PackageDescription>Microsoft.ML.Tokenizers contains the implmentation of the tokenization used in the NLP transforms.</PackageDescription>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <NoWarn>$(NoWarn);MSML_NoInstanceInitializers</NoWarn>
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
@@ -16,10 +17,6 @@
     <Compile Remove="Utils/Helpers.netstandard.cs" />
   </ItemGroup>
 
-  <ItemGroup>
-    <PackageReference Include="Google.Protobuf" />
-  </ItemGroup>
-
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
     <PackageReference Include="Microsoft.Bcl.HashCode" />
     <PackageReference Include="Microsoft.Bcl.Memory" />

src/Microsoft.ML.Tokenizers/Model/BertOptions.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
@@ -9,7 +9,6 @@ namespace Microsoft.ML.Tokenizers
     /// </summary>
     public sealed class BertOptions : WordPieceOptions
     {
-#pragma warning disable MSML_NoInstanceInitializers
         /// <summary>
         /// Gets or sets a value indicating whether to lower case the input before tokenization.
         /// </summary>
@@ -66,7 +65,5 @@ public sealed class BertOptions : WordPieceOptions
         /// Gets or sets a value indicating whether to remove non-spacing marks.
         /// </summary>
         public bool RemoveNonSpacingMarks { get; set; }
-
-#pragma warning restore MSML_NoInstanceInitializers
     }
-}
+}

src/Microsoft.ML.Tokenizers/Model/WordPieceOptions.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
@@ -11,7 +11,6 @@ namespace Microsoft.ML.Tokenizers
     /// </summary>
     public class WordPieceOptions
     {
-#pragma warning disable MSML_NoInstanceInitializers
         internal const int DefaultMaxInputCharsPerWord = 100;
         internal const string DefaultContinuingSubwordPrefix = "##";
 
@@ -44,6 +43,5 @@ public class WordPieceOptions
         /// Gets or set the maximum number of characters to consider for a single word.
         /// </summary>
         public int MaxInputCharsPerWord { get; set; } = DefaultMaxInputCharsPerWord;
-#pragma warning restore MSML_NoInstanceInitializers
     }
-}
+}

src/Microsoft.ML.Tokenizers/SentencepieceModel.cs

@@ -913,5 +913,11 @@ public void TestPhi3TokenizerIdEncoding(string text, string decodedWithNoSpecial
             Assert.Equal(textWithSpecialTokens.Length, charactersWritten);
             Assert.Equal(textWithSpecialTokens, destinationBuffer.AsSpan(0, charactersWritten).ToString());
         }
+
+        [Fact]
+        public void CreateWithNullStreamThrows()
+        {
+            Assert.ThrowsAny<ArgumentException>(() => LlamaTokenizer.Create(null!));
+        }
     }
 }