-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Expand file tree
/
Copy pathF1Evaluator.xml
More file actions
174 lines (174 loc) · 13 KB
/
Copy pathF1Evaluator.xml
File metadata and controls
174 lines (174 loc) · 13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
<Type Name="F1Evaluator" FullName="Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator">
<TypeSignature Language="C#" Value="public sealed class F1Evaluator : Microsoft.Extensions.AI.Evaluation.IEvaluator" />
<TypeSignature Language="ILAsm" Value=".class public auto ansi sealed beforefieldinit F1Evaluator extends System.Object implements class Microsoft.Extensions.AI.Evaluation.IEvaluator" />
<TypeSignature Language="DocId" Value="T:Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator" />
<TypeSignature Language="VB.NET" Value="Public NotInheritable Class F1Evaluator
Implements IEvaluator" />
<TypeSignature Language="F#" Value="type F1Evaluator = class
 interface IEvaluator" />
<TypeSignature Language="C++ CLI" Value="public ref class F1Evaluator sealed : Microsoft::Extensions::AI::Evaluation::IEvaluator" />
<AssemblyInfo>
<AssemblyName>Microsoft.Extensions.AI.Evaluation.NLP</AssemblyName>
<AssemblyVersion>9.7.0.0</AssemblyVersion>
<AssemblyVersion>9.9.0.0</AssemblyVersion>
<AssemblyVersion>9.10.0.0</AssemblyVersion>
<AssemblyVersion>10.0.0.0</AssemblyVersion>
<AssemblyVersion>10.1.0.0</AssemblyVersion>
<AssemblyVersion>10.2.0.0</AssemblyVersion>
<AssemblyVersion>10.3.0.0</AssemblyVersion>
<AssemblyVersion>10.4.0.0</AssemblyVersion>
<AssemblyVersion>10.6.0.0</AssemblyVersion>
</AssemblyInfo>
<Base>
<BaseTypeName>System.Object</BaseTypeName>
</Base>
<Interfaces>
<Interface>
<InterfaceName>Microsoft.Extensions.AI.Evaluation.IEvaluator</InterfaceName>
</Interface>
</Interfaces>
<Attributes>
<Attribute FrameworkAlternate="net-10.0-pp;net-11.0-pp;net-8.0-pp;net-9.0-pp">
<AttributeName Language="C#">[System.Runtime.CompilerServices.Nullable(0)]</AttributeName>
<AttributeName Language="F#">[<System.Runtime.CompilerServices.Nullable(0)>]</AttributeName>
</Attribute>
</Attributes>
<Docs>
<summary>
An <see cref="T:Microsoft.Extensions.AI.Evaluation.IEvaluator" /> that evaluates the quality of a response produced by an AI model by comparing
it to a reference response using the F1 scoring algorithm. F1 score is the ratio of the number of shared
words between the generated response and the reference response.
</summary>
<remarks>
<para>
The <see cref="T:Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator" /> computes the F1 score of a response ("hypothesis") in relation to a ground-truth reference
supplied by <see cref="P:Microsoft.Extensions.AI.Evaluation.NLP.F1EvaluatorContext.GroundTruth" />. The score is returned in a <see cref="T:Microsoft.Extensions.AI.Evaluation.NumericMetric" />
with a value between 0.0 and 1.0 where 0.0 represents no match at all and 1.0 indicates a perfect match.
By default, the score is interpreted with a pass/fail cutoff of 0.5. So a score of 0.5 or higher is
passing and a score below 0.5 is failing.
</para>
</remarks>
</Docs>
<Members>
<Member MemberName=".ctor">
<MemberSignature Language="C#" Value="public F1Evaluator ();" />
<MemberSignature Language="ILAsm" Value=".method public hidebysig specialname rtspecialname instance void .ctor() cil managed" />
<MemberSignature Language="DocId" Value="M:Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator.#ctor" />
<MemberSignature Language="VB.NET" Value="Public Sub New ()" />
<MemberSignature Language="C++ CLI" Value="public:
 F1Evaluator();" />
<MemberType>Constructor</MemberType>
<AssemblyInfo>
<AssemblyName>Microsoft.Extensions.AI.Evaluation.NLP</AssemblyName>
<AssemblyVersion>10.6.0.0</AssemblyVersion>
</AssemblyInfo>
<Parameters />
<Docs>
<summary>To be added.</summary>
<remarks>To be added.</remarks>
</Docs>
</Member>
<Member MemberName="EvaluateAsync">
<MemberSignature Language="C#" Value="public System.Threading.Tasks.ValueTask<Microsoft.Extensions.AI.Evaluation.EvaluationResult> EvaluateAsync (System.Collections.Generic.IEnumerable<Microsoft.Extensions.AI.ChatMessage> messages, Microsoft.Extensions.AI.ChatResponse modelResponse, Microsoft.Extensions.AI.Evaluation.ChatConfiguration? chatConfiguration = default, System.Collections.Generic.IEnumerable<Microsoft.Extensions.AI.Evaluation.EvaluationContext>? additionalContext = default, System.Threading.CancellationToken cancellationToken = default);" />
<MemberSignature Language="ILAsm" Value=".method public hidebysig newslot virtual instance valuetype System.Threading.Tasks.ValueTask`1<class Microsoft.Extensions.AI.Evaluation.EvaluationResult> EvaluateAsync(class System.Collections.Generic.IEnumerable`1<class Microsoft.Extensions.AI.ChatMessage> messages, class Microsoft.Extensions.AI.ChatResponse modelResponse, class Microsoft.Extensions.AI.Evaluation.ChatConfiguration chatConfiguration, class System.Collections.Generic.IEnumerable`1<class Microsoft.Extensions.AI.Evaluation.EvaluationContext> additionalContext, valuetype System.Threading.CancellationToken cancellationToken) cil managed" />
<MemberSignature Language="DocId" Value="M:Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator.EvaluateAsync(System.Collections.Generic.IEnumerable{Microsoft.Extensions.AI.ChatMessage},Microsoft.Extensions.AI.ChatResponse,Microsoft.Extensions.AI.Evaluation.ChatConfiguration,System.Collections.Generic.IEnumerable{Microsoft.Extensions.AI.Evaluation.EvaluationContext},System.Threading.CancellationToken)" />
<MemberSignature Language="VB.NET" Value="Public Function EvaluateAsync (messages As IEnumerable(Of ChatMessage), modelResponse As ChatResponse, Optional chatConfiguration As ChatConfiguration = Nothing, Optional additionalContext As IEnumerable(Of EvaluationContext) = Nothing, Optional cancellationToken As CancellationToken = Nothing) As ValueTask(Of EvaluationResult)" />
<MemberSignature Language="F#" Value="abstract member EvaluateAsync : seq<Microsoft.Extensions.AI.ChatMessage> * Microsoft.Extensions.AI.ChatResponse * Microsoft.Extensions.AI.Evaluation.ChatConfiguration * seq<Microsoft.Extensions.AI.Evaluation.EvaluationContext> * System.Threading.CancellationToken -> System.Threading.Tasks.ValueTask<Microsoft.Extensions.AI.Evaluation.EvaluationResult>
override this.EvaluateAsync : seq<Microsoft.Extensions.AI.ChatMessage> * Microsoft.Extensions.AI.ChatResponse * Microsoft.Extensions.AI.Evaluation.ChatConfiguration * seq<Microsoft.Extensions.AI.Evaluation.EvaluationContext> * System.Threading.CancellationToken -> System.Threading.Tasks.ValueTask<Microsoft.Extensions.AI.Evaluation.EvaluationResult>" Usage="f1Evaluator.EvaluateAsync (messages, modelResponse, chatConfiguration, additionalContext, cancellationToken)" />
<MemberType>Method</MemberType>
<Implements>
<InterfaceMember>M:Microsoft.Extensions.AI.Evaluation.IEvaluator.EvaluateAsync(System.Collections.Generic.IEnumerable{Microsoft.Extensions.AI.ChatMessage},Microsoft.Extensions.AI.ChatResponse,Microsoft.Extensions.AI.Evaluation.ChatConfiguration,System.Collections.Generic.IEnumerable{Microsoft.Extensions.AI.Evaluation.EvaluationContext},System.Threading.CancellationToken)</InterfaceMember>
</Implements>
<AssemblyInfo>
<AssemblyName>Microsoft.Extensions.AI.Evaluation.NLP</AssemblyName>
<AssemblyVersion>10.6.0.0</AssemblyVersion>
</AssemblyInfo>
<ReturnValue>
<ReturnType>System.Threading.Tasks.ValueTask<Microsoft.Extensions.AI.Evaluation.EvaluationResult></ReturnType>
<Attributes>
<Attribute FrameworkAlternate="net-10.0-pp">
<AttributeName Language="C#">[System.Runtime.CompilerServices.Nullable(new System.Byte[] { 0, 1 })]</AttributeName>
<AttributeName Language="F#">[<System.Runtime.CompilerServices.Nullable(new System.Byte[] { 0, 1 })>]</AttributeName>
</Attribute>
</Attributes>
</ReturnValue>
<Parameters>
<Parameter Name="messages" Type="System.Collections.Generic.IEnumerable<Microsoft.Extensions.AI.ChatMessage>" />
<Parameter Name="modelResponse" Type="Microsoft.Extensions.AI.ChatResponse" />
<Parameter Name="chatConfiguration" Type="Microsoft.Extensions.AI.Evaluation.ChatConfiguration">
<Attributes>
<Attribute FrameworkAlternate="net-10.0-pp;net-11.0-pp;net-8.0-pp;net-9.0-pp">
<AttributeName Language="C#">[System.Runtime.CompilerServices.Nullable(2)]</AttributeName>
<AttributeName Language="F#">[<System.Runtime.CompilerServices.Nullable(2)>]</AttributeName>
</Attribute>
</Attributes>
</Parameter>
<Parameter Name="additionalContext" Type="System.Collections.Generic.IEnumerable<Microsoft.Extensions.AI.Evaluation.EvaluationContext>">
<Attributes>
<Attribute FrameworkAlternate="net-10.0-pp;net-11.0-pp;net-8.0-pp;net-9.0-pp">
<AttributeName Language="C#">[System.Runtime.CompilerServices.Nullable(new System.Byte[] { 2, 1 })]</AttributeName>
<AttributeName Language="F#">[<System.Runtime.CompilerServices.Nullable(new System.Byte[] { 2, 1 })>]</AttributeName>
</Attribute>
</Attributes>
</Parameter>
<Parameter Name="cancellationToken" Type="System.Threading.CancellationToken" />
</Parameters>
<Docs>
<param name="messages">To be added.</param>
<param name="modelResponse">To be added.</param>
<param name="chatConfiguration">To be added.</param>
<param name="additionalContext">To be added.</param>
<param name="cancellationToken">To be added.</param>
<summary>To be added.</summary>
<returns>To be added.</returns>
<remarks>To be added.</remarks>
<inheritdoc />
</Docs>
</Member>
<Member MemberName="EvaluationMetricNames">
<MemberSignature Language="C#" Value="public System.Collections.Generic.IReadOnlyCollection<string> EvaluationMetricNames { get; }" />
<MemberSignature Language="ILAsm" Value=".property instance class System.Collections.Generic.IReadOnlyCollection`1<string> EvaluationMetricNames" />
<MemberSignature Language="DocId" Value="P:Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator.EvaluationMetricNames" />
<MemberSignature Language="VB.NET" Value="Public ReadOnly Property EvaluationMetricNames As IReadOnlyCollection(Of String)" />
<MemberSignature Language="F#" Value="member this.EvaluationMetricNames : System.Collections.Generic.IReadOnlyCollection<string>" Usage="Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator.EvaluationMetricNames" />
<MemberSignature Language="C++ CLI" Value="public:
 property System::Collections::Generic::IReadOnlyCollection<System::String ^> ^ EvaluationMetricNames { System::Collections::Generic::IReadOnlyCollection<System::String ^> ^ get(); };" />
<MemberType>Property</MemberType>
<Implements>
<InterfaceMember>P:Microsoft.Extensions.AI.Evaluation.IEvaluator.EvaluationMetricNames</InterfaceMember>
</Implements>
<AssemblyInfo>
<AssemblyName>Microsoft.Extensions.AI.Evaluation.NLP</AssemblyName>
<AssemblyVersion>10.6.0.0</AssemblyVersion>
</AssemblyInfo>
<ReturnValue>
<ReturnType>System.Collections.Generic.IReadOnlyCollection<System.String></ReturnType>
</ReturnValue>
<Docs>
<summary>To be added.</summary>
<value>To be added.</value>
<remarks>To be added.</remarks>
<inheritdoc />
</Docs>
</Member>
<Member MemberName="F1MetricName">
<MemberSignature Language="C#" Value="public static string F1MetricName { get; }" />
<MemberSignature Language="ILAsm" Value=".property string F1MetricName" />
<MemberSignature Language="DocId" Value="P:Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator.F1MetricName" />
<MemberSignature Language="VB.NET" Value="Public Shared ReadOnly Property F1MetricName As String" />
<MemberSignature Language="F#" Value="static member F1MetricName : string" Usage="Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator.F1MetricName" />
<MemberSignature Language="C++ CLI" Value="public:
 static property System::String ^ F1MetricName { System::String ^ get(); };" />
<MemberType>Property</MemberType>
<AssemblyInfo>
<AssemblyName>Microsoft.Extensions.AI.Evaluation.NLP</AssemblyName>
<AssemblyVersion>10.6.0.0</AssemblyVersion>
</AssemblyInfo>
<ReturnValue>
<ReturnType>System.String</ReturnType>
</ReturnValue>
<Docs>
<summary>
Gets the <see cref="P:Microsoft.Extensions.AI.Evaluation.EvaluationMetric.Name" /> of the <see cref="T:Microsoft.Extensions.AI.Evaluation.NumericMetric" /> returned by
<see cref="T:Microsoft.Extensions.AI.Evaluation.NLP.F1Evaluator" />.
</summary>
<value>To be added.</value>
<remarks>To be added.</remarks>
</Docs>
</Member>
</Members>
</Type>