Fixed bootstrapper and data loading

AndMu · AndMu · commit e23186eba7a6 · 2019-02-10T21:35:46.000Z
diff --git a/src/Sentiment/Wikiled.Sentiment.AcceptanceTests/Wikiled.Sentiment.AcceptanceTests.csproj b/src/Sentiment/Wikiled.Sentiment.AcceptanceTests/Wikiled.Sentiment.AcceptanceTests.csproj
@@ -11,7 +11,7 @@
     <PackageReference Include="Newtonsoft.Json" version="12.0.1" />
     <PackageReference Include="NLog" version="4.5.11" />
     <PackageReference Include="NUnit" version="3.11.0" />
-    <PackageReference Include="Polly" version="6.1.2" />
+    <PackageReference Include="Polly" version="7.0.1" />
     <PackageReference Include="protobuf-net" version="2.4.0" />
     <PackageReference Include="Snappy.NET" version="1.1.1.8" />
     <PackageReference Include="StackExchange.Redis" Version="2.0.519" />
diff --git a/src/Sentiment/Wikiled.Sentiment.Analysis.Tests/Wikiled.Sentiment.Analysis.Tests.csproj b/src/Sentiment/Wikiled.Sentiment.Analysis.Tests/Wikiled.Sentiment.Analysis.Tests.csproj
@@ -12,7 +12,7 @@
     <PackageReference Include="Newtonsoft.Json" version="12.0.1" />
     <PackageReference Include="NLog" version="4.5.11" />
     <PackageReference Include="NUnit" version="3.11.0" />
-    <PackageReference Include="Polly" version="6.1.2" />
+    <PackageReference Include="Polly" version="7.0.1" />
     <PackageReference Include="protobuf-net" version="2.4.0" />
     <PackageReference Include="StackExchange.Redis" Version="2.0.519" />
     <PackageReference Include="System.Reactive" version="4.1.2" />
diff --git a/src/Sentiment/Wikiled.Sentiment.Analysis/Processing/Persistency/DataLoader.cs b/src/Sentiment/Wikiled.Sentiment.Analysis/Processing/Persistency/DataLoader.cs
@@ -1,6 +1,7 @@
 ﻿using Microsoft.Extensions.Logging;
 using System;
 using System.Collections.Generic;
+using System.IO;
 
 namespace Wikiled.Sentiment.Analysis.Processing.Persistency
 {
@@ -18,22 +19,28 @@ public DataLoader(ILoggerFactory loggerFactory)
 
         public IDataSource Load(IDataSourceConfig source)
         {
-            if (source == null)
+            if (source == null ||
+                (source.All == null && source.Negative == null && source.Positive == null))
             {
                 throw new ArgumentNullException(nameof(source));
             }
 
             if (source.All != null)
             {
-                logger.LogInformation("Loading {0}", source.All);
-                if (source.All.EndsWith(".xml", StringComparison.OrdinalIgnoreCase))
+                if (File.Exists(source.All))
                 {
-                    return new XmlDataLoader(loggerFactory.CreateLogger<XmlDataLoader>()).LoadOldXml(source.All);
+                    logger.LogInformation("Loading {0}", source.All);
+                    if (source.All.EndsWith(".xml", StringComparison.OrdinalIgnoreCase))
+                    {
+                        return new XmlDataLoader(loggerFactory.CreateLogger<XmlDataLoader>()).LoadOldXml(source.All);
+                    }
+
+                    logger.LogInformation("Loading {0} as JSON", source.All);
+                    var data = new JsonDataSource(loggerFactory.CreateLogger<JsonDataSource>(), source.All);
+                    return data;
                 }
 
-                logger.LogInformation("Loading {0} as JSON", source.All);
-                var data = new JsonDataSource(loggerFactory.CreateLogger< JsonDataSource>(), source.All);
-                return data;
+                return new SimpleDataSource(loggerFactory.CreateLogger<SimpleDataSource>(), source.All, null);
             }
 
             var loaders = new List<IDataSource>();
diff --git a/src/Sentiment/Wikiled.Sentiment.Analysis/Wikiled.Sentiment.Analysis.csproj b/src/Sentiment/Wikiled.Sentiment.Analysis/Wikiled.Sentiment.Analysis.csproj
@@ -18,13 +18,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Autofac" Version="4.8.1" />
+    <PackageReference Include="Autofac" Version="4.9.0" />
     <PackageReference Include="Autofac.Extras.AggregateService" Version="4.1.0" />
     <PackageReference Include="Knuppe.SharpNL" Version="1.3.0" />
     <PackageReference Include="morelinq" Version="3.1.0" />
-    <PackageReference Include="CsvHelper" version="12.1.1" />
+    <PackageReference Include="CsvHelper" version="12.1.2" />
     <PackageReference Include="Newtonsoft.Json" version="12.0.1" />
-    <PackageReference Include="Polly" version="6.1.2" />
+    <PackageReference Include="Polly" version="7.0.1" />
     <PackageReference Include="protobuf-net" version="2.4.0" />
     <PackageReference Include="StackExchange.Redis" Version="2.0.519" />
     <PackageReference Include="System.Reactive" version="4.1.2" />
diff --git a/src/Sentiment/Wikiled.Sentiment.Integration.Tests/Wikiled.Sentiment.Integration.Tests.csproj b/src/Sentiment/Wikiled.Sentiment.Integration.Tests/Wikiled.Sentiment.Integration.Tests.csproj
@@ -11,7 +11,7 @@
     <PackageReference Include="Newtonsoft.Json" version="12.0.1" />
     <PackageReference Include="NLog" version="4.5.11" />
     <PackageReference Include="NUnit" version="3.11.0" />
-    <PackageReference Include="Polly" version="6.1.2" />
+    <PackageReference Include="Polly" version="7.0.1" />
     <PackageReference Include="protobuf-net" version="2.4.0" />
     <PackageReference Include="StackExchange.Redis" Version="2.0.519" />
     <PackageReference Include="System.Reactive" version="4.1.2" />
diff --git a/src/Sentiment/Wikiled.Sentiment.TestLogic.Shared/Wikiled.Sentiment.TestLogic.Shared.csproj b/src/Sentiment/Wikiled.Sentiment.TestLogic.Shared/Wikiled.Sentiment.TestLogic.Shared.csproj
@@ -13,7 +13,7 @@
     <PackageReference Include="Newtonsoft.Json" version="12.0.1" />
     <PackageReference Include="NLog" version="4.5.11" />
     <PackageReference Include="NUnit" version="3.11.0" />
-    <PackageReference Include="Polly" version="6.1.2" />
+    <PackageReference Include="Polly" version="7.0.1" />
     <PackageReference Include="protobuf-net" version="2.4.0" />
     <PackageReference Include="StackExchange.Redis" Version="2.0.519" />
     <PackageReference Include="System.Reactive" version="4.1.2" />
diff --git a/src/Sentiment/Wikiled.Sentiment.Text.Tests/Wikiled.Sentiment.Text.Tests.csproj b/src/Sentiment/Wikiled.Sentiment.Text.Tests/Wikiled.Sentiment.Text.Tests.csproj
@@ -15,7 +15,7 @@
     <PackageReference Include="Newtonsoft.Json" version="12.0.1" />
     <PackageReference Include="NLog" version="4.5.11" />
     <PackageReference Include="NUnit" version="3.11.0" />
-    <PackageReference Include="Polly" version="6.1.2" />
+    <PackageReference Include="Polly" version="7.0.1" />
     <PackageReference Include="protobuf-net" version="2.4.0" />
     <PackageReference Include="StackExchange.Redis" Version="2.0.519" />
     <PackageReference Include="System.Reactive" version="4.1.2" />
diff --git a/src/Sentiment/Wikiled.Sentiment.Text/Wikiled.Sentiment.Text.csproj b/src/Sentiment/Wikiled.Sentiment.Text/Wikiled.Sentiment.Text.csproj
@@ -14,11 +14,11 @@
 
   <ItemGroup>
     <PackageReference Include="Accord.MachineLearning" Version="3.8.0" />
-    <PackageReference Include="Autofac" Version="4.8.1" />
+    <PackageReference Include="Autofac" Version="4.9.0" />
     <PackageReference Include="morelinq" Version="3.1.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.7.0" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
-    <PackageReference Include="Polly" Version="6.1.2" />
+    <PackageReference Include="Polly" Version="7.0.1" />
     <PackageReference Include="protobuf-net" Version="2.4.0" />
     <PackageReference Include="SharpZipLib" Version="1.1.0" />
     <PackageReference Include="StackExchange.Redis" Version="2.0.519" />
diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Analysis/BaseRawCommand.cs b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Analysis/BaseRawCommand.cs
@@ -62,7 +62,6 @@ private async Task<IParsedDocumentHolder> SynchronizedReviews(IParsedDocumentHol
 
         private IObservable<IParsedDocumentHolder> GetAllDocuments()
         {
-            Logger.LogInformation("Loading {0}", Config);
             var dataSource = loader.Load(Config);
             return container.GetTextSplitter().GetParsedReviewHolders(dataSource);
         }
diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Analysis/BoostrapCommand.cs b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Analysis/BoostrapCommand.cs
@@ -3,6 +3,7 @@
 using System.Linq;
 using System.Reactive.Concurrency;
 using System.Reactive.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Wikiled.Arff.Logic;
@@ -23,24 +24,10 @@ public class BoostrapCommand : BaseRawCommand<BootsrapConfig>
 
         private string positiveResult;
 
-        public BoostrapCommand(ILogger log, BootsrapConfig config, IDataLoader loader, ISessionContainer container)
+        public BoostrapCommand(ILogger<BoostrapCommand> log, BootsrapConfig config, IDataLoader loader, ISessionContainer container)
             : base(log, config, loader, container)
         {
-        }
-
-        private bool ProcessReview(ProcessingContext context)
-        {
-            var bootSentimentValue = context.Review.CalculateRawRating();
-            var bootAllSentiments = context.Review.GetAllSentiments()
-                .Where(item => !item.Owner.IsInvertor && item.Owner.IsSentiment)
-                .ToArray();
-
-            if (bootSentimentValue.StarsRating.HasValue)
-            {
-                return true;
-            }
-
-            return false;
+            Semaphore = new SemaphoreSlim(3000);
         }
 
         protected override async Task Process(IObservable<IParsedDocumentHolder> reviews,
@@ -65,8 +52,6 @@ protected override async Task Process(IObservable<IParsedDocumentHolder> reviews
                     .ToArray();
                 SaveDocuments(result);
             }
-
-            Logger.LogInformation($"Testing performance {client.GetPerformanceDescription()}");
         }
 
         private EvalData Resolve(ProcessingContext context)
@@ -88,6 +73,7 @@ private EvalData Resolve(ProcessingContext context)
 
         private void SaveDocuments(EvalData[] context)
         {
+            Logger.LogInformation("SaveDocuments");
             InitOutput();
             var types = context.Where(item => item.Stars.HasValue && item.TotalSentiments >= Config.Minimum).ToArray();
             var positive = types.Count(item => item.CalculatedPositivity == PositivityType.Positive);
diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Analysis/Config/BaseRawConfig.cs b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Analysis/Config/BaseRawConfig.cs
@@ -28,7 +28,7 @@ public abstract class BaseRawConfig : ICommandConfig, IDataSourceConfig
 
         public POSTaggerType Tagger { get; set; } = POSTaggerType.SharpNLP;
 
-        public void Build(ContainerBuilder builder)
+        public virtual void Build(ContainerBuilder builder)
         { 
             MainContainerFactory.Setup(builder)
                 .Config()
diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Analysis/Config/BootsrapConfig.cs b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Analysis/Config/BootsrapConfig.cs
@@ -14,11 +14,5 @@ public class BootsrapConfig : BaseRawConfig
         public double? BalancedTop { get; set; }
 
         public string Model { get; set; }
-
-        [Required]
-        public string Path { get; set; }
-
-        [Required]
-        public string Words { get; set; }
     }
 }
diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Properties/launchSettings.json b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Properties/launchSettings.json
@@ -2,7 +2,7 @@
   "profiles": {
     "Wikiled.Sentiment.ConsoleApp": {
       "commandName": "Project",
-      "commandLineArgs": "test -Model=.\\svm -Positive=e:\\DataSets\\aclImdb\\All\\Train\\pos -Negative=e:\\DataSets\\aclImdb\\All\\Train\\neg -out=IMDB_SMALL"
+      "commandLineArgs": "boot -Destination=c:\\imdbdOut -Weights=e:\\Source\\Research.Unsupervised.Sentiment\\src\\Latest\\words_imdb.csv -All=c:\\2"
     }
   }
 }
diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Wikiled.Sentiment.ConsoleApp.csproj b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Wikiled.Sentiment.ConsoleApp.csproj
@@ -17,11 +17,11 @@
 
   <ItemGroup>
     <PackageReference Include="Accord.MachineLearning" Version="3.8.0" />
-    <PackageReference Include="CsvHelper" version="12.1.1" />
+    <PackageReference Include="CsvHelper" version="12.1.2" />
     <PackageReference Include="Newtonsoft.Json" version="12.0.1" />
     <PackageReference Include="NLog" version="4.5.11" />
     <PackageReference Include="NLog.Extensions.Logging" Version="1.4.0" />
-    <PackageReference Include="Polly" version="6.1.2" />
+    <PackageReference Include="Polly" version="7.0.1" />
     <PackageReference Include="protobuf-net" version="2.4.0" />
     <PackageReference Include="StackExchange.Redis" Version="2.0.519" />
     <PackageReference Include="System.Reactive" version="4.1.2" />

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`using Microsoft.Extensions.Logging;`
`2`	`2`	`using System;`
`3`	`3`	`using System.Collections.Generic;`
	`4`	`+using System.IO;`
`4`	`5`
`5`	`6`	`namespace Wikiled.Sentiment.Analysis.Processing.Persistency`
`6`	`7`	`{`
`@@ -18,22 +19,28 @@ public DataLoader(ILoggerFactory loggerFactory)`
`18`	`19`
`19`	`20`	`public IDataSource Load(IDataSourceConfig source)`
`20`	`21`	`{`
`21`		`- if (source == null)`
	`22`	`+ if (source == null \|\|`
	`23`	`+ (source.All == null && source.Negative == null && source.Positive == null))`
`22`	`24`	`{`
`23`	`25`	`throw new ArgumentNullException(nameof(source));`
`24`	`26`	`}`
`25`	`27`
`26`	`28`	`if (source.All != null)`
`27`	`29`	`{`
`28`		`- logger.LogInformation("Loading {0}", source.All);`
`29`		`- if (source.All.EndsWith(".xml", StringComparison.OrdinalIgnoreCase))`
	`30`	`+ if (File.Exists(source.All))`
`30`	`31`	`{`
`31`		`- return new XmlDataLoader(loggerFactory.CreateLogger<XmlDataLoader>()).LoadOldXml(source.All);`
	`32`	`+ logger.LogInformation("Loading {0}", source.All);`
	`33`	`+ if (source.All.EndsWith(".xml", StringComparison.OrdinalIgnoreCase))`
	`34`	`+ {`
	`35`	`+ return new XmlDataLoader(loggerFactory.CreateLogger<XmlDataLoader>()).LoadOldXml(source.All);`
	`36`	`+ }`
	`37`	`+`
	`38`	`+ logger.LogInformation("Loading {0} as JSON", source.All);`
	`39`	`+ var data = new JsonDataSource(loggerFactory.CreateLogger<JsonDataSource>(), source.All);`
	`40`	`+ return data;`
`32`	`41`	`}`
`33`	`42`
`34`		`- logger.LogInformation("Loading {0} as JSON", source.All);`
`35`		`- var data = new JsonDataSource(loggerFactory.CreateLogger< JsonDataSource>(), source.All);`
`36`		`- return data;`
	`43`	`+ return new SimpleDataSource(loggerFactory.CreateLogger<SimpleDataSource>(), source.All, null);`
`37`	`44`	`}`
`38`	`45`
`39`	`46`	`var loaders = new List<IDataSource>();`
Original file line number	Diff line number	Diff line change
`@@ -62,7 +62,6 @@ private async Task<IParsedDocumentHolder> SynchronizedReviews(IParsedDocumentHol`
`62`	`62`
`63`	`63`	`private IObservable<IParsedDocumentHolder> GetAllDocuments()`
`64`	`64`	`{`
`65`		`- Logger.LogInformation("Loading {0}", Config);`
`66`	`65`	`var dataSource = loader.Load(Config);`
`67`	`66`	`return container.GetTextSplitter().GetParsedReviewHolders(dataSource);`
`68`	`67`	`}`