Merge pull request #140 from diffix/andrei/text-columns-pattern

More cleanup for TextColumnExplorer

Author: Andrei Bozantan

src/explorer/Common/ValueWithCountList.cs

@@ -0,0 +1,69 @@
+namespace Explorer.Common
+{
+    using System;
+    using System.Collections.Generic;
+
+    internal class ValueWithCountList<T> : List<(T Value, long Count)>
+    {
+        public long TotalCount => Count == 0 ? 0 : this[^1].Count;
+
+        public static ValueWithCountList<T> FromValueWithCountEnum(IEnumerable<ValueWithCount<T>> valueCounts)
+        {
+            var ret = new ValueWithCountList<T>();
+            foreach (var vc in valueCounts)
+            {
+                ret.AddValueCount(vc.Value, vc.Count);
+            }
+            return ret;
+        }
+
+        public void AddValueCount(T value, long count)
+        {
+            Add((value, TotalCount + count));
+        }
+
+        public T GetRandomValue(Random rand, T @default)
+        {
+            if (Count == 0)
+            {
+                return @default;
+            }
+            var rcount = rand.NextLong(TotalCount);
+            return FindSubstring(rcount);
+        }
+
+        private T FindSubstring(long count)
+        {
+            var left = 0;
+            var right = Count - 1;
+            while (true)
+            {
+                var middle = (left + right) / 2;
+                if (middle == 0 || middle == Count - 1)
+                {
+                    return this[middle].Value;
+                }
+                if (count < this[middle].Count)
+                {
+                    if (count >= this[middle - 1].Count)
+                    {
+                        return this[middle - 1].Value;
+                    }
+                    right = middle;
+                }
+                else if (count > this[middle].Count)
+                {
+                    if (count <= this[middle + 1].Count)
+                    {
+                        return this[middle].Value;
+                    }
+                    left = middle;
+                }
+                else
+                {
+                    return this[middle].Value;
+                }
+            }
+        }
+    }
+}

src/explorer/Explorers/TextColumnExplorer.cs

@@ -10,6 +10,8 @@ namespace Explorer.Explorers
     using Explorer.Common;
     using Explorer.Queries;
 
+    using SubstringWithCountList = Explorer.Common.ValueWithCountList<string>;
+
     internal class TextColumnExplorer : ExplorerBase
     {
         public const string EmailAddressChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_.";
@@ -64,11 +66,7 @@ private static async Task<IEnumerable<string>> GenerateStrings(DConnection conn,
             var substrings = await ExploreSubstrings(conn, ctx, substringLengths: new int[] { 3, 4 });
             var rand = new Random(Environment.TickCount);
             return Enumerable.Range(0, GeneratedValuesCount).Select(_
-                => substrings.GenerateString(
-                        minLength: 3,
-                        minSubstringLength: 3,
-                        maxSubstringLength: 4,
-                        rand));
+                => GenerateString(substrings, minLength: 3, rand));
         }
 
         private static async Task<IEnumerable<string>> GenerateEmails(DConnection conn, ExplorerContext ctx)
@@ -80,12 +78,7 @@ private static async Task<IEnumerable<string>> GenerateEmails(DConnection conn,
             var emails = new List<string>(GeneratedValuesCount);
             for (var i = 0; emails.Count < GeneratedValuesCount && i < GeneratedValuesCount * 100; i++)
             {
-                var s = substrings.GenerateString(
-                    minLength: 3,
-                    minSubstringLength: 3,
-                    maxSubstringLength: 4,
-                    rand);
-                var email = GenerateEmail(s, domains, tlds, rand);
+                var email = GenerateEmail(substrings, domains, tlds, rand);
                 if (!string.IsNullOrEmpty(email))
                 {
                     emails.Add(email);
@@ -94,9 +87,27 @@ private static async Task<IEnumerable<string>> GenerateEmails(DConnection conn,
             return emails;
         }
 
-        private static string GenerateEmail(string str, SubstringWithCountList domains, SubstringWithCountList tlds, Random rand)
+        private static string GenerateString(SubstringsData substrings, int minLength, Random rand)
+        {
+            var sb = new StringBuilder();
+            var len = rand.Next(minLength, substrings.Count);
+            for (var pos = 0; pos < substrings.Count && sb.Length < len; pos++)
+            {
+                var str = substrings.GetRandomSubstring(pos, rand);
+                sb.Append(str);
+                pos += str.Length;
+            }
+            return sb.ToString();
+        }
+
+        private static string GenerateEmail(
+            SubstringsData substrings,
+            SubstringWithCountList domains,
+            SubstringWithCountList tlds,
+            Random rand)
         {
             // create local-part section
+            var str = GenerateString(substrings, minLength: 6, rand);
             var allParts = str.Split('@', StringSplitOptions.RemoveEmptyEntries);
             var sb = new StringBuilder();
             var partIndex = 0;
@@ -107,12 +118,6 @@ private static string GenerateEmail(string str, SubstringWithCountList domains,
                 pnext /= 2;
                 partIndex++;
             }
-            for (var replaced = 1; replaced != 0;)
-            {
-                var oldlen = sb.Length;
-                sb.Replace("..", ".");
-                replaced = oldlen - sb.Length;
-            }
             var localParts = sb.ToString()
                 .Split('.', StringSplitOptions.RemoveEmptyEntries)
                 .Where(s => s.Length == 1 || s.Length > 3)
@@ -125,7 +130,7 @@ private static string GenerateEmail(string str, SubstringWithCountList domains,
             if (domains.Count >= EmailDomainsCountThreshold)
             {
                 // if the number of distinct domains is big enough we select one from the extracted list
-                return localPart + domains.GetSubstring(rand);
+                return localPart + domains.GetRandomValue(rand, @default: string.Empty);
             }
 
             // create domain section
@@ -145,17 +150,17 @@ private static string GenerateEmail(string str, SubstringWithCountList domains,
             {
                 return string.Empty;
             }
-            return localPart + "@" + domain + tlds.GetSubstring(rand);
+            return localPart + "@" + domain + tlds.GetRandomValue(rand, @default: string.Empty);
         }
 
         /// <summary>
         /// Finds common substrings for each position in the texts of the specified column.
         /// It uses a batch approach to query for several positions (specified using SubstringQueryColumnCount)
         /// using a single query.
         /// </summary>
-        private static async Task<SubstringDataCollection> ExploreSubstrings(DConnection conn, ExplorerContext ctx, params int[] substringLengths)
+        private static async Task<SubstringsData> ExploreSubstrings(DConnection conn, ExplorerContext ctx, params int[] substringLengths)
         {
-            var substrings = new SubstringDataCollection(maxSubstringLength: substringLengths.Max());
+            var substrings = new SubstringsData();
             foreach (var length in substringLengths)
             {
                 var hasRows = true;
@@ -182,166 +187,54 @@ private static async Task<bool> CheckIsEmail(DConnection conn, ExplorerContext c
             var emailCheck = await conn.Exec(
                 new TextColumnTrim(ctx.Table, ctx.Column, TextColumnTrimType.Both, EmailAddressChars));
 
-            var counts = ValueCounts.Compute(emailCheck.Rows);
-
-            return counts.TotalCount == emailCheck.Rows
-                .Where(r => r.IsNull || r.Value == "@")
-                .Sum(r => r.Count);
+            return emailCheck.Rows.All(r => r.IsNull || r.Value == "@");
         }
 
         private static async Task<SubstringWithCountList> ExploreEmailDomains(DConnection conn, ExplorerContext ctx)
         {
             var domains = await conn.Exec(new TextColumnTrim(ctx.Table, ctx.Column, TextColumnTrimType.Leading, EmailAddressChars));
-            var totalCount = 0L;
-            var domain = new SubstringWithCountList();
-            foreach (var row in domains.Rows)
-            {
-                if (row.HasValue && row.Value.StartsWith("@", StringComparison.InvariantCulture))
-                {
-                    totalCount += row.Count;
-                    domain.Add((row.Value, totalCount));
-                }
-            }
-            return domain;
+
+            return SubstringWithCountList.FromValueWithCountEnum(
+                domains.Rows
+                    .Where(r => r.HasValue && r.Value.StartsWith("@", StringComparison.InvariantCulture)));
         }
 
         private static async Task<SubstringWithCountList> ExploreEmailTopLevelDomains(DConnection conn, ExplorerContext ctx)
         {
             var suffixes = await conn.Exec(new TextColumnSuffix(ctx.Table, ctx.Column, 3, 7));
-            var totalCount = 0L;
-            var tlds = new SubstringWithCountList();
-            foreach (var row in suffixes.Rows)
-            {
-                if (row.HasValue && row.Value.StartsWith(".", StringComparison.InvariantCulture))
-                {
-                    totalCount += row.Count;
-                    tlds.Add((row.Value, totalCount));
-                }
-            }
-            return tlds;
-        }
-    }
-
-    internal class SubstringWithCountList : List<(string Value, long Count)>
-    {
-        public long TotalCount => Count == 0 ? 0 : this[^1].Count;
-
-        public string GetSubstring(Random rand)
-        {
-            if (Count == 0)
-            {
-                return string.Empty;
-            }
-            var rcount = rand.NextLong(TotalCount);
-            return FindSubstring(rcount);
-        }
-
-        private string FindSubstring(long count)
-        {
-            var left = 0;
-            var right = Count - 1;
-            while (true)
-            {
-                var middle = (left + right) / 2;
-                if (middle == 0 || middle == Count - 1)
-                {
-                    return this[middle].Value;
-                }
-                if (count < this[middle].Count)
-                {
-                    if (count >= this[middle - 1].Count)
-                    {
-                        return this[middle - 1].Value;
-                    }
-                    right = middle;
-                }
-                else if (count > this[middle].Count)
-                {
-                    if (count <= this[middle + 1].Count)
-                    {
-                        return this[middle].Value;
-                    }
-                    left = middle;
-                }
-                else
-                {
-                    return this[middle].Value;
-                }
-            }
-        }
-    }
-
-    internal class SubstringDataCollection
-    {
-        public SubstringDataCollection(int maxSubstringLength)
-        {
-            MaxSubstringLength = maxSubstringLength;
-            Substrings = new List<Item>();
-        }
-
-        private List<Item> Substrings { get; }
-
-        private int MaxSubstringLength { get; }
-
-        public void Add(int pos, string s, long count)
-        {
-            while (Substrings.Count <= pos)
-            {
-                Substrings.Add(new Item(MaxSubstringLength));
-            }
-            Substrings[pos].Add(s, count);
-        }
 
-        public string GenerateString(int minLength, int minSubstringLength, int maxSubstringLength, Random rand)
-        {
-            var sb = new StringBuilder();
-            var len = rand.Next(minLength, Substrings.Count);
-            for (var pos = 0; pos < Substrings.Count && sb.Length < len; pos++)
-            {
-                var str = Substrings[pos].GetSubstring(minSubstringLength, maxSubstringLength, rand);
-                sb.Append(str);
-                pos += str.Length;
-            }
-            return sb.ToString();
+            return SubstringWithCountList.FromValueWithCountEnum(
+                suffixes.Rows
+                    .Where(r => r.HasValue && r.Value.StartsWith(".", StringComparison.InvariantCulture)));
         }
 
         /// <summary>
-        /// Stores the substrings from a certain position in a column,
+        /// Stores the substrings at each position in a column,
         /// together with the number of occurences (counts) for each substring.
-        /// The substrings are grouped separately by length.
         /// </summary>
-        internal class Item
+        internal class SubstringsData
         {
-            public Item(int maxSubstringLength)
-            {
-                Data = new List<SubstringWithCountList>(maxSubstringLength)
+            public SubstringsData()
             {
-                new SubstringWithCountList() { (string.Empty, 0) },
-            };
-                for (var i = 1; i <= maxSubstringLength; i++)
-                {
-                    Data.Add(new SubstringWithCountList());
-                }
+                Substrings = new List<SubstringWithCountList>();
             }
 
-            private List<SubstringWithCountList> Data { get; }
+            public int Count => Substrings.Count;
 
-            public void Add(string s, long count)
-            {
-                var substrings = Data[s.Length];
-                substrings.Add((s, substrings.TotalCount + count));
-            }
+            private List<SubstringWithCountList> Substrings { get; }
 
-            public string GetSubstring(int minLength, int maxLength, Random rand)
+            public void Add(int pos, string s, long count)
             {
-                if (maxLength >= Data.Count)
+                while (Substrings.Count <= pos)
                 {
-                    throw new ArgumentException($"{nameof(maxLength)} should be smaller than {Data.Count}.", nameof(maxLength));
+                    Substrings.Add(new SubstringWithCountList());
                 }
-                // TODO: distribute value over all alternatives according to counts (not with the same probability)
-                var sslen = rand.Next(minLength, maxLength + 1);
-                var substrings = Data[sslen];
-                return substrings.GetSubstring(rand);
+                Substrings[pos].AddValueCount(s, count);
+            }
+
+            public string GetRandomSubstring(int pos, Random rand)
+            {
+                return Substrings[pos].GetRandomValue(rand, string.Empty);
             }
         }
     }

tests/explorer.api.tests/TextColumnExplorerTests.cs

@@ -28,8 +28,9 @@ public async void TestClientsEmail()
             var metric_svalues = metrics.First(m => m.Name == "synthetic_values");
             var values = metric_svalues.Metric as IEnumerable<string>;
             Assert.True(values.All(v => v.Length >= 3));
-            Assert.True(values.All(v => v.Contains('@', StringComparison.InvariantCulture)));
+            Assert.True(values.All(v => v.Count(c => c == '@') == 1));
             Assert.True(values.All(v => v.Contains('.', StringComparison.InvariantCulture)));
+            Assert.True(values.All(v => !v.Contains("..", StringComparison.InvariantCulture)));
         }
 
         [Fact]
@@ -41,8 +42,9 @@ public async void TestCardsEmail()
             var metric_svalues = metrics.First(m => m.Name == "synthetic_values");
             var values = metric_svalues.Metric as IEnumerable<string>;
             Assert.True(values.All(v => v.Length >= 3));
-            Assert.True(values.All(v => v.Contains('@', StringComparison.InvariantCulture)));
+            Assert.True(values.All(v => v.Count(c => c == '@') == 1));
             Assert.True(values.All(v => v.Contains('.', StringComparison.InvariantCulture)));
+            Assert.True(values.All(v => !v.Contains("..", StringComparison.InvariantCulture)));
         }
 
         [Fact]