Fix news search with new website layout (#295)

* More robustness when downloading news

* Adapt news search to new website

* Formatting

* Bump Lib_CSharp

* Fix small comments

Author: francescolf

Backend/Other/DB/DBPolifemo.sql

@@ -49,6 +49,16 @@ create table if not exists Tags
     blurhash varchar(80) null
     );
 
+create table if not exists ArticleContent
+(
+    id       int auto_increment
+    primary key,
+    title    varchar(100) not null,
+    subtitle varchar(200) null,
+    content  text         not null,
+    url      varchar(500) null
+    );
+
 create table if not exists Articles
 (
     article_id   int unsigned auto_increment
@@ -75,15 +85,6 @@ create table if not exists Articles
     foreign key (tag_id) references Tags (name)
     );
 
-create table if not exists ArticleContent
-(
-    id       int auto_increment
-    primary key,
-    title    varchar(100) not null,
-    subtitle varchar(200) null,
-    content  text         not null,
-    url      varchar(500) null
-    );
 
 create table if not exists Types
 (

Backend/Source/Objects/Articles/Articles.cs

@@ -2,7 +2,7 @@
 
 using HtmlAgilityPack;
 using PoliFemoBackend.Source.Utils.Article;
-using PoliNetwork.Core.Data;
+using PoliFemoBackend.Source.Utils.News.PoliMi;
 using ReverseMarkdown;
 
 #endregion
@@ -34,53 +34,35 @@ public static ArticleContent[] LoadContentFromURL(string url)
         var r = new ArticleContent[2];
         r[0] = new ArticleContent();
         r[1] = new ArticleContent();
-        if (url.Contains("landingpages"))
-            return r; // If this is a landing page, return an empty array as this is not a news article
-        var web = new HtmlWeb();
 
         for (var i = 0; i < 2; i++)
         {
-            var doc = web.Load(url);
-            var urls1 = doc.DocumentNode.SelectNodes("//div");
+            var doc = HtmlPageUtil.LoadUrl(url);
+            var article = doc.DocumentNode.SelectSingleNode("//div[@class='article']");
             try
             {
-                // Try to get the news-single-item class, if none are found, try to get the content id
-                var urls = urls1.FirstOrDefault(x =>
-                    x.GetAttributeValue("class", "") == "news-single-item"
-                );
+                // Get the html article text
+                var htmlContent = article.SelectSingleNode("//div[@itemprop='articleBody']");
 
-                // If empty, try to get the "content" id
-                if (urls == null)
-                {
-                    GlobalVariables.DefaultLogger.Info(
-                        "No news-single-item class found, trying to get the content element"
-                    );
-                    urls = urls1.First(x => x.GetAttributeValue("id", "") == "content");
-                }
+                r[i].title = article
+                    .SelectSingleNode("//h1[@itemprop='headline']")
+                    .InnerText.Trim();
+                r[i].subtitle = article
+                    .SelectSingleNode("//div[@itemprop='description']")
+                    .InnerText.Trim();
 
-                r[i].title = urls.SelectSingleNode("//h1").InnerHtml.Trim();
-                r[i].subtitle = urls.SelectSingleNode("//h2").InnerHtml.Trim();
+                var content = converter.Convert(htmlContent.InnerHtml);
 
-                var content = converter.Convert(
-                    ArticleContentUtil.CleanContentString(urls.InnerHtml)
-                );
                 content = content.Replace("](/", "](https://www.polimi.it/"); // Replace relative PoliMi links with absolute ones
-                content = content.Replace("\r\n\r\n* * *", ""); // Remove the final horizontal line
                 r[i].content = content.Trim();
                 r[i].url = url;
 
                 if (i == 0)
                 {
-                    var pathnode = doc
-                        .DocumentNode.Descendants("li")
-                        .FirstOrDefault(li => li.GetAttributeValue("id", "") == "lienglish")
-                        ?.Descendants("span")
-                        .First()
-                        .Descendants("a")
-                        .FirstOrDefault();
+                    //Get the second li element (the link to the english version, if available)
+                    var pathnode = doc.DocumentNode.SelectNodes("//li//a").Skip(1).FirstOrDefault();
                     if (pathnode == null)
                         break;
-                    //var b = a.SelectSingleNode("//span").InnerHtml;
                     url = "https://polimi.it" + pathnode.GetAttributeValue("href", "");
                 }
             }

Backend/Source/Objects/Articles/News/ArticleContent.cs

@@ -11,15 +11,12 @@ public class ArticleNews
     //internal News
     private static readonly Config config = new() { RemoveComments = true };
 
-    private static Converter converter = new(config);
-
     public ArticleContent[] content = new ArticleContent[2];
 
     public ArticleNews(string? tag, string? image)
     {
         this.tag = tag;
         this.image = image;
-        internalNews = true;
         author_id = 1;
         content = new ArticleContent[2];
     }
@@ -33,7 +30,6 @@ public ArticleNews(
         double? longitude,
         string? blurhash,
         int platforms,
-        bool? internalNews,
         string? tag,
         ArticleContent[] content
     )
@@ -46,7 +42,6 @@ ArticleContent[] content
         this.longitude = longitude;
         this.blurhash = blurhash;
         this.platforms = platforms;
-        this.internalNews = internalNews;
         this.tag = tag;
         this.content = content;
     }
@@ -61,7 +56,6 @@ public ArticleNews() { }
     public double? longitude { get; set; }
     public string? blurhash { get; set; }
     public int? platforms { get; set; }
-    public bool? internalNews { get; set; }
     public string? tag { get; set; }
 
     public void AddContent(ArticleContent c)