Merge branch '6.0/progressive-search-result-rows' into 6.0-trunk

Author: cbrandtbuffalo

etc/RT_Config.pm.in

@@ -3092,6 +3092,26 @@ Include "0" to provide the "Unlimited" option.
 
 Set(@SearchResultsPerPage, qw(0 10 25 50 100));
 
+=item C<$ProgressiveSearchResultRows>
+
+For unpaginated searches (including TSV, iCal, and RSS exports), RT
+progressively generates results by paging through the collection
+C<$ProgressiveSearchResultRows> rows at a time. This does not change the
+results, but can significantly reduce memory usage on large result sets.
+The system default is 100.
+
+Set this to 0 to disable progressive generation and load the entire
+result set into memory at once.
+
+For large downloads that use complicated queries, progressive generation
+can make the download slower. If you frequently download large exports,
+you can increase this number or set it to 0 to disable it for better
+performance, at the cost of higher memory use on the server.
+
+=cut
+
+Set( $ProgressiveSearchResultRows, 100 );
+
 =item C<$UserTicketDataResultFormat>
 
 This is the format of ticket search result for "Download User Tickets" links. It

lib/RT/Config.pm

@@ -2034,6 +2034,9 @@ our %META;
     TicketsItemMapSize => {
         Widget => '/Widgets/Form/Integer',
     },
+    ProgressiveSearchResultRows => {
+        Widget => '/Widgets/Form/Integer',
+    },
 
     AssetDefaultSearchResultOrderBy => {
         Widget => '/Widgets/Form/String',

share/html/Elements/CollectionList

@@ -76,9 +76,10 @@ $Collection->RowsPerPage( $Rows ) if $Rows;
 $Page = 1 unless $Page && $Page =~ /\A\d+\z/ && $Page > 1;    # workaround problems with $Page = '' or undef
 $Collection->GotoPage( $Page - 1 ); # SB uses page 0 as the first page
 
-# $Collection might not have CombineSearchAndCount method, like dashboards and saved searches.
-# Don't combine search and count for unpaginated searches, as we will progressively generate the result.
-$Collection->CombineSearchAndCount(1) if $Rows && $Collection->can('CombineSearchAndCount');
+# For unpaginated searches we progressively generate the result in chunks of
+# $ProgressiveSearchResultRows rows, and must not combine search and count then.
+my $ProgressiveRows = $Rows ? 0 : RT->Config->Get('ProgressiveSearchResultRows');
+$Collection->CombineSearchAndCount(1) unless $ProgressiveRows;
 $TotalFound = $Collection->CountAll() unless defined $TotalFound;
 return '' if !$TotalFound && !$ShowEmpty;
 
@@ -159,11 +160,9 @@ if ( $ShowHeader ) {
 
 my ($i, $column_map) = (0, {});
 $m->out('<tbody>');
-if ( !$Rows ) {
-    $Collection->RowsPerPage(100);
-}
+$Collection->RowsPerPage($ProgressiveRows) if $ProgressiveRows;
 
-while ( my $record = $Rows ? $Collection->Next : ( $Collection->Next || ( $Collection->NextPage && $Collection->Next ) ) ) {
+while ( my $record = $ProgressiveRows ? ( $Collection->Next || ( $Collection->NextPage && $Collection->Next ) ) : $Collection->Next ) {
     # Every ten rows, flush the buffer and put something on the page.
     $m->flush_buffer unless ++$i % 10;
 

share/html/Elements/TSVExport

@@ -51,7 +51,7 @@ $Collection
 $Format
 $PreserveNewLines => 0
 $Filename  => undef
-$RowsPerPage => 100
+$RowsPerPage => RT->Config->Get('ProgressiveSearchResultRows')
 </%ARGS>
 <%ONCE>
 my $no_html = HTML::Scrubber->new( deny => '*' );
@@ -137,10 +137,9 @@ for (@columns) {
 my $i = 0;
 my $ii = 0;
 
-$RowsPerPage = 1000 if !$RowsPerPage || $RowsPerPage > 1000;
-$Collection->RowsPerPage($RowsPerPage);
+$Collection->RowsPerPage($RowsPerPage) if $RowsPerPage;
 
-while ( my $row = ( $Collection->Next || ( $Collection->NextPage && $Collection->Next ) ) ) {
+while ( my $row = $RowsPerPage ? ( $Collection->Next || ( $Collection->NextPage && $Collection->Next ) ) : $Collection->Next ) {
     for my $col (@columns) {
         $m->out(join("\t", map {
             my $val = ProcessColumnMapValue($_->{map}, Arguments => [$row, $ii++], Escape => 0);

share/html/NoAuth/iCal/dhandler

@@ -98,10 +98,9 @@ $feed->add_properties('calscale' => ['gregorian']);
 $feed->add_properties('method'   => ['publish']);
 $feed->add_properties('prodid'   => ["-//" . RT->Config->Get('rtname') ."//"]);
 
-$RowsPerPage = 1000 if !$RowsPerPage || $RowsPerPage > 1000;
-$tickets->RowsPerPage($RowsPerPage);
+$tickets->RowsPerPage($RowsPerPage) if $RowsPerPage;
 
-while ( my $t = ( $tickets->Next || ( $tickets->NextPage && $tickets->Next ) ) ) {
+while ( my $t = $RowsPerPage ? ( $tickets->Next || ( $tickets->NextPage && $tickets->Next ) ) : $tickets->Next ) {
     next unless $t->DueObj->IsSet;
 
     my $starttime = $t->StartsObj->IsSet ? $t->StartsObj : $t->CreatedObj;
@@ -168,5 +167,5 @@ $m->abort;
 </%init>
 
 <%args>
-$RowsPerPage => 100
+$RowsPerPage => RT->Config->Get('ProgressiveSearchResultRows')
 </%args>

share/html/Search/Elements/ResultsRSSView

@@ -88,9 +88,8 @@ $rss->channel(
     },
 );
 
-$RowsPerPage = 1000 if !$RowsPerPage || $RowsPerPage > 1000;
-$Tickets->RowsPerPage($RowsPerPage);
-while ( my $Ticket = ( $Tickets->Next || ( $Tickets->NextPage && $Tickets->Next ) ) ) {
+$Tickets->RowsPerPage($RowsPerPage) if $RowsPerPage;
+while ( my $Ticket = $RowsPerPage ? ( $Tickets->Next || ( $Tickets->NextPage && $Tickets->Next ) ) : $Tickets->Next ) {
     my $creator_str = $Ticket->CreatorObj->Format;
     $creator_str =~ s/[\r\n]//g;
 
@@ -118,6 +117,6 @@ $m->abort();
 <%ARGS>
 $OrderBy => 'Created'
 $Order => 'ASC'
-$RowsPerPage => 100
+$RowsPerPage => RT->Config->Get('ProgressiveSearchResultRows')
 </%ARGS>
 

share/html/Search/Results.html

@@ -173,8 +173,8 @@
 }
 $session{$session_name}->RowsPerPage( $Rows ) if $Rows;
 $session{$session_name}->GotoPage( $Page - 1 );
-# Don't combine search and count for unpaginated searches, as we will progressively generate the result.
-$session{$session_name}->CombineSearchAndCount(1) if $Rows;
+# Don't combine search and count for unpaginated searches, as we progressively generate the result.
+$session{$session_name}->CombineSearchAndCount(1) if $Rows || !RT->Config->Get('ProgressiveSearchResultRows');
 
 # Save the session again because we made changes
 # Otherwise the set below will restore it back to