[JA DateTimeV2] Merged refinements (#2950)

* Merged refinements

* Fixed specs formatting

* Minor renamed variables according to review

Co-authored-by: aitelint <Fabrizio.Sorba@telusinternational.com>

Author: aitelint

.NET/Microsoft.Recognizers.Definitions.Common/Japanese/DateTimeDefinitions.cs

@@ -56,6 +56,8 @@ public ChineseDurationExtractorConfiguration(IDateTimeOptionsConfiguration confi
 
         public Dictionary<string, long> UnitValueMap { get; }
 
+        public Dictionary<Regex, Regex> AmbiguityDurationFiltersDict => null;
+
         Regex ICJKDurationExtractorConfiguration.DurationUnitRegex => DurationUnitRegex;
 
         Regex ICJKDurationExtractorConfiguration.DurationConnectorRegex => DurationConnectorRegex;

.NET/Microsoft.Recognizers.Text.DateTime/Chinese/Extractors/ChineseDurationExtractorConfiguration.cs

@@ -53,6 +53,7 @@ public static class Constants
 
         // AmPm time representation for time parser
         public const string Comment_AmPm = "ampm";
+        public const string Comment_Am = "am";
 
         // Prefix early/late for time parser
         public const string Comment_Early = "early";
@@ -267,6 +268,7 @@ public static class Constants
         public const string TimexFuzzyDay = "XX";
         public const string DateTimexConnector = "-";
         public const string TimeTimexConnector = ":";
+        public const string TimexSeparator = ",";
         public const string GeneralPeriodPrefix = "P";
         public const string TimeTimexPrefix = "T";
 

.NET/Microsoft.Recognizers.Text.DateTime/Constants.cs

@@ -2,7 +2,8 @@
 // Licensed under the MIT License.
 
 using System.Collections.Generic;
-
+using System.Linq;
+using System.Text.RegularExpressions;
 using Microsoft.Recognizers.Text.Utilities;
 using DateObject = System.DateTime;
 
@@ -54,9 +55,32 @@ public List<ExtractResult> Extract(string source, DateObject referenceTime)
                 res = MergeMultipleDuration(source, res);
             }
 
+            res = FilterAmbiguity(res, source);
+
             return res;
         }
 
+        private List<ExtractResult> FilterAmbiguity(List<ExtractResult> extractResults, string text)
+        {
+            if (this.config.AmbiguityDurationFiltersDict != null)
+            {
+                foreach (var regex in this.config.AmbiguityDurationFiltersDict)
+                {
+                    foreach (var extractResult in extractResults)
+                    {
+                        if (regex.Key.IsMatch(text))
+                        {
+                            var matches = regex.Value.Matches(text).Cast<Match>();
+                            extractResults = extractResults.Where(er => !matches.Any(m => m.Index < er.Start + er.Length && m.Index + m.Length > er.Start))
+                                .ToList();
+                        }
+                    }
+                }
+            }
+
+            return extractResults;
+        }
+
         private List<ExtractResult> MergeMultipleDuration(string text, List<ExtractResult> extractorResults)
         {
             if (extractorResults.Count <= 1)

.NET/Microsoft.Recognizers.Text.DateTime/Extractors/CJK/BaseCJKDurationExtractor.cs

@@ -129,6 +129,17 @@ private void AddMod(List<ExtractResult> ers, string text)
                     er.Metadata = AssignModMetadata(er.Metadata);
                 }
 
+                match = this.config.UntilRegex.MatchBegin(afterStr, trim: true);
+
+                if (match.Success)
+                {
+                    var modLength = match.Index + match.Length;
+                    er.Length += modLength;
+                    er.Text = text.Substring(er.Start ?? 0, er.Length ?? 0);
+
+                    er.Metadata = AssignModMetadata(er.Metadata);
+                }
+
                 match = this.config.SincePrefixRegex.MatchEnd(beforeStr, trim: true);
 
                 if (match.Success && AmbiguousRangeChecker(beforeStr, text, er))

.NET/Microsoft.Recognizers.Text.DateTime/Extractors/CJK/BaseCJKMergedDateTimeExtractor.cs

@@ -32,5 +32,7 @@ public interface ICJKDurationExtractorConfiguration : IDateTimeOptionsConfigurat
 
         Dictionary<string, long> UnitValueMap { get; }
 
+        Dictionary<Regex, Regex> AmbiguityDurationFiltersDict { get; }
+
     }
 }

.NET/Microsoft.Recognizers.Text.DateTime/Extractors/CJK/ICJKDurationExtractorConfiguration.cs

@@ -6,8 +6,8 @@
 using System.Globalization;
 using System.Linq;
 using System.Text.RegularExpressions;
-
 using Microsoft.Recognizers.Definitions.Japanese;
+using Microsoft.Recognizers.Definitions.Utilities;
 using Microsoft.Recognizers.Text.NumberWithUnit;
 using Microsoft.Recognizers.Text.NumberWithUnit.Japanese;
 
@@ -48,6 +48,8 @@ public JapaneseDurationExtractorConfiguration(IDateTimeOptionsConfiguration conf
 
             UnitMap = DateTimeDefinitions.ParserConfigurationUnitMap.ToDictionary(k => k.Key, k => k.Value);
             UnitValueMap = DateTimeDefinitions.DurationUnitValueMap;
+            AmbiguityDurationFiltersDict = DefinitionLoader.LoadAmbiguityFilters(DateTimeDefinitions.AmbiguityDurationFiltersDict);
+
         }
 
         public IExtractor InternalExtractor { get; }
@@ -56,6 +58,8 @@ public JapaneseDurationExtractorConfiguration(IDateTimeOptionsConfiguration conf
 
         public Dictionary<string, long> UnitValueMap { get; }
 
+        public Dictionary<Regex, Regex> AmbiguityDurationFiltersDict { get; }
+
         Regex ICJKDurationExtractorConfiguration.DurationUnitRegex => DurationUnitRegex;
 
         Regex ICJKDurationExtractorConfiguration.DurationConnectorRegex => DurationConnectorRegex;

.NET/Microsoft.Recognizers.Text.DateTime/Japanese/Extractors/JapaneseDurationExtractorConfiguration.cs

@@ -56,6 +56,8 @@ public KoreanDurationExtractorConfiguration(IDateTimeOptionsConfiguration config
 
         public Dictionary<string, long> UnitValueMap { get; }
 
+        public Dictionary<Regex, Regex> AmbiguityDurationFiltersDict => null;
+
         Regex ICJKDurationExtractorConfiguration.DurationUnitRegex => DurationUnitRegex;
 
         Regex ICJKDurationExtractorConfiguration.DurationConnectorRegex => DurationConnectorRegex;

.NET/Microsoft.Recognizers.Text.DateTime/Korean/Extractors/KoreanDurationExtractorConfiguration.cs

@@ -318,8 +318,9 @@ protected DateTimeResolutionResult ParseImplicitDate(string text, DateObject ref
             // handle "明日から3週間" (3 weeks from tomorrow)
             var durationResult = this.config.DurationExtractor.Extract(text, referenceDate);
             var unitMatch = this.config.DurationRelativeDurationUnitRegex.Match(text);
+            var isWithin = this.config.DurationRelativeDurationUnitRegex.MatchEnd(text, trim: true).Groups[Constants.WithinGroupName].Success;
 
-            if (exactMatch.Success && unitMatch.Success && (durationResult.Count > 0) &&
+            if ((exactMatch.Success || isWithin) && unitMatch.Success && (durationResult.Count > 0) &&
                 string.IsNullOrEmpty(unitMatch.Groups["few"].Value))
             {
                 var pr = this.config.DurationParser.Parse(durationResult[0], referenceDate);

.NET/Microsoft.Recognizers.Text.DateTime/Parsers/CJK/BaseCJKDateParser.cs

@@ -158,6 +158,14 @@ private DateTimeResolutionResult MergeDateAndTime(string text, DateObject refere
             var pastDate = (DateObject)((DateTimeResolutionResult)pr1.Value).PastValue;
             var time = (DateObject)((DateTimeResolutionResult)pr2.Value).FutureValue;
 
+            // handle cases with time like 25時 which resolve to the next day
+            var timexHour = TimexUtility.ParseHourFromTimeTimex(pr2.TimexStr);
+            if (timexHour > Constants.DayHourCount)
+            {
+                futureDate = futureDate.AddDays(1);
+                pastDate = pastDate.AddDays(1);
+            }
+
             var hour = time.Hour;
             var min = time.Minute;
             var sec = time.Second;
@@ -178,7 +186,6 @@ private DateTimeResolutionResult MergeDateAndTime(string text, DateObject refere
                 timeStr = timeStr.Substring(0, timeStr.Length - 4);
             }
 
-            timeStr = "T" + hour.ToString("D2", CultureInfo.InvariantCulture) + timeStr.Substring(3);
             ret.Timex = pr1.TimexStr + timeStr;
 
             var val = (DateTimeResolutionResult)pr2.Value;