Merge pull request #5223 from kinga-altF4/searchcrawllog

Increase time out for Get-PnPSearchCrawlLog

Author: reshmee011

.vscode/launch.json

@@ -4,6 +4,23 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+      "name": "PnP.PowerShell (pwsh + import)",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "C:/Program Files/PowerShell/7/pwsh.exe",
+      "args": [
+        "-NoLogo",
+        "-NoProfile",
+        "-ExecutionPolicy", "Bypass",
+        "-Command",
+        "Import-Module \"$PWD/build/debug/PnP.PowerShell/PnP.PowerShell.psd1\" -Force; 'Ready: Import complete'; Read-Host 'Press Enter to exit'"
+      ],
+      "cwd": "${workspaceFolder}",
+      "console": "externalTerminal",
+      "justMyCode": true
+    }
+,
         {
             "name": "Debug with nugets in new PowerShell session",
             "type": "coreclr",

documentation/Get-PnPSearchCrawlLog.md

@@ -16,7 +16,7 @@ Returns entries from the SharePoint search crawl log. Make sure you are granted
 
 ```powershell
 Get-PnPSearchCrawlLog [-LogLevel <LogLevel>] [-RowLimit <Int32>] [-Filter <String>]
- [-ContentSource <ContentSource>] [-StartDate <DateTime>] [-EndDate <DateTime>] [-RawFormat]
+ [-ContentSource <ContentSource>] [-StartDate <DateTime>] [-EndDate <DateTime>] [-RawFormat] [-IncreaseRequestTimeout]
  [-Connection <PnPConnection>] 
 ```
 
@@ -75,6 +75,16 @@ Get-PnPSearchCrawlLog -RowLimit 3 -RawFormat
 
 Returns the last 3 crawl log entries showing the raw crawl log data.
 
+### EXAMPLE 8
+```powershell
+$ClientID= "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+$env:SharePointPnPHttpTimeout = -1 #👈
+Connect-PnPOnline -Url https://<tenant>-admin.sharepoint.com/ -Interactive -ClientId $ClientID -ErrorAction Stop # 👈
+
+Get-PnPSearchCrawlLog -Filter "https://contoso-my.sharepoint.com/sites/Intranet" -IncreaseRequestTimeout
+```
+Increases the request timeout allowing the call to last up to 3 minutes. The `ClientRuntimeContext` enforces a three-minute limit; when increasing the timeout to its maximum of three minutes, this threshold may still be exceeded.
+
 ## PARAMETERS
 
 ### -Connection
@@ -134,6 +144,20 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -IncreaseRequestTimeout
+Extend the request timeout to permit command execution for up to 3 minutes.
+
+```yaml
+Type: Switch
+Parameter Sets: (All)
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -LogLevel
 Filter what log entries to return (All, Success, Warning, Error). Defaults to All
 

src/Commands/Search/GetSearchCrawlLog.cs

@@ -3,7 +3,7 @@
 using System.Linq;
 using System.Management.Automation;
 using Microsoft.SharePoint.Client;
-using Microsoft.SharePoint.Client.Search.Administration;
+using Microsoft.SharePoint.Client.Search.Administration; 
 using PnP.PowerShell.Commands.Attributes;
 
 namespace PnP.PowerShell.Commands.Search
@@ -13,7 +13,9 @@ public enum LogLevel
         All = -1,
         Success = 0,
         Warning = 1,
-        Error = 2
+        Error = 2,
+        Deleted = 3,
+        TopLEvel =4
     }
 
     public enum ContentSource
@@ -26,14 +28,17 @@ public class CrawlEntry
     {
         public string Url { get; set; }
         public DateTime CrawlTime { get; set; }
+        public DateTime LastTouchedTime { get; set; }
         public DateTime ItemTime { get; set; }
         public LogLevel LogLevel { get; set; }
         public string Status { get; set; }
         public int ItemId { get; set; }
         public int ContentSourceId { get; set; }
+
+        public string DatabaseName { get; set; }
     }
 
-    [Cmdlet(VerbsCommon.Get, "PnPSearchCrawlLog", DefaultParameterSetName = "Xml")]
+    [Cmdlet(VerbsCommon.Get, "PnPSearchCrawlLog")]
     [ApiNotAvailableUnderApplicationPermissions]
     public class GetSearchCrawlLog : PnPWebCmdlet
     {
@@ -58,12 +63,34 @@ public class GetSearchCrawlLog : PnPWebCmdlet
         [Parameter(Mandatory = false)]
         public SwitchParameter RawFormat;
 
+        [Parameter(Mandatory = false)]
+        public SwitchParameter GetCountOnly;
+
+        [Parameter(Mandatory = false)]
+        public SwitchParameter IncreaseRequestTimeout;  
+
+
         private const int MaxRows = 100000;
 
         protected override void ExecuteCmdlet()
         {
             try
             {
+                if(IncreaseRequestTimeout)
+                {
+                    string timeoutValue = Environment.GetEnvironmentVariable("SharePointPnPHttpTimeout");
+                    if (string.IsNullOrEmpty(timeoutValue))
+                    {
+                        LogWarning("The timeout may be only increased if the SharePointPnPHttpTimeout environment variable is set to 180000 or -1.");
+                        LogWarning("Use  $env:SharePointPnPHttpTimeout = -1 command and then, establish new connection with Connect-PnPOnline.");
+                        return;
+                    }
+                    else
+                    {
+                        //Max 3 minutes, because Default CSOM timeout is 180,000 ms
+                        ClientContext.RequestTimeout=3*60*1000;
+                    }
+                }
                 var crawlLog = new DocumentCrawlLog(ClientContext, ClientContext.Site);
                 ClientContext.Load(crawlLog);
 
@@ -94,7 +121,9 @@ protected override void ExecuteCmdlet()
                     RowLimit = MaxRows;
                 }
 
-                var logEntries = crawlLog.GetCrawledUrls(false, RowLimit, Filter, true, contentSourceId, (int)LogLevel, -1, StartDate, EndDate);
+                bool countOnly= GetCountOnly ? true: false;
+
+                var logEntries = crawlLog.GetCrawledUrls(countOnly, RowLimit, Filter, true, contentSourceId, (int)LogLevel, -1, StartDate, EndDate);
                 ClientContext.ExecuteQueryRetry();
 
                 if (RawFormat)
@@ -139,7 +168,16 @@ protected override void ExecuteCmdlet()
             }
             catch (Exception e)
             {
-                LogError($"Error: {e.Message}. Make sure you are granted access to the crawl log via the SharePoint search admin center at https://<tenant>-admin.sharepoint.com/_layouts/15/searchadmin/crawllogreadpermission.aspx");
+                if(e.Message=="The operation has timed out." )
+                {
+                    
+                    LogError($"Error: {e.Message}.  Default CSOM timeout is 180,000 ms (≈3 minutes). If you are querying large crawl logs or broad ranges, the server may take longer than that. ");
+                    
+                }
+                else
+                {
+                    LogError($"Error: {e.Message}. Make sure you are granted access to the crawl log via the SharePoint search admin center at https://<tenant>-admin.sharepoint.com/_layouts/15/searchadmin/crawllogreadpermission.aspx");
+                }
             }
         }
 
@@ -175,7 +213,9 @@ private static CrawlEntry MapCrawlLogEntry(Dictionary<string, object> dictionary
                 ItemId = (int)dictionary["URLID"],
                 ContentSourceId = (int)dictionary["ContentSourceID"],
                 Url = dictionary["FullUrl"].ToString(),
-                CrawlTime = (DateTime)dictionary["TimeStampUtc"]
+                CrawlTime = (DateTime)dictionary["TimeStampUtc"],
+                LastTouchedTime= (DateTime)dictionary["LastTouchedTime"],
+                DatabaseName= (string)dictionary["DatabaseName"]
             };
             long.TryParse(dictionary["LastRepositoryModifiedTime"] + "", out long ticks);
             if (ticks != 0)

src/Tests/Search/GetPnPSearchCrawlLogTests.cs

@@ -77,14 +77,17 @@ public void GetPnPSearchCrawlLogTest()
 				// From Cmdlet Help: Show raw crawl log data
 				var rawFormat = "";
 
+                var increaseRequestTimeout= "";
+
                 var results = scope.ExecuteCommand("Get-PnPSearchCrawlLog",
 					new CommandParameter("LogLevel", logLevel),
 					new CommandParameter("RowLimit", rowLimit),
 					new CommandParameter("Filter", filter),
 					new CommandParameter("ContentSource", contentSource),
 					new CommandParameter("StartDate", startDate),
 					new CommandParameter("EndDate", endDate),
-					new CommandParameter("RawFormat", rawFormat));
+					new CommandParameter("RawFormat", rawFormat),
+					new CommandParameter("IncreaseRequestTimeout", increaseRequestTimeout));
 
                 Assert.IsNotNull(results);
             }