Skip to content

Commit 3abdf1b

Browse files
gnat42gnat42
committed
Improve Result interface
Allow the Offset filter to stop the process function loop. The impetus is the lack of detail the result object contains - particularly when using the Offset filter. When processing a file in chunks the StepAggregator reports the number of successfully imported results and errors but not the number of skipped. This allows a user to seek the reader, and then process a batch and then stop. Receiving an accurate set of processed,skipped and errors.
1 parent 37fd273 commit 3abdf1b

6 files changed

Lines changed: 72 additions & 16 deletions

File tree

‎src/Exception/StopException.php‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?php
2+
3+
namespace Ddeboer\DataImport\Exception;
4+
5+
use Ddeboer\DataImport\Exception;
6+
7+
class StopException extends \RuntimeException implements Exception
8+
{
9+
10+
}

‎src/Filter/OffsetFilter.php‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<?php
22

33
namespace Ddeboer\DataImport\Filter;
4+
use Ddeboer\DataImport\Exception\StopException;
45

56
/**
67
* This filter can be used to filter out some items from the beginning and/or
@@ -35,14 +36,20 @@ class OffsetFilter
3536
*/
3637
protected $maxLimitHit = false;
3738

39+
/**
40+
* @var boolean
41+
*/
42+
protected $stopOnMaxLimit = false;
43+
3844
/**
3945
* @param integer $offset 0-based index of the item to start read from
4046
* @param integer|null $limit Maximum count of items to read. null = no limit
4147
*/
42-
public function __construct($offset = 0, $limit = null)
48+
public function __construct($offset = 0, $limit = null, $stopOnLimit = false)
4349
{
4450
$this->offset = $offset;
4551
$this->limit = $limit;
52+
$this->stopOnMaxLimit = ($limit>0 && $stopOnLimit);
4653
}
4754

4855
/**
@@ -52,6 +59,10 @@ public function __invoke(array $item)
5259
{
5360
// In case we've already filtered up to limited
5461
if ($this->maxLimitHit) {
62+
if($this->stopOnMaxLimit) {
63+
throw new StopException();
64+
}
65+
5566
return false;
5667
}
5768

‎src/Result.php‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,18 +57,25 @@ class Result
5757
* @param string $name
5858
* @param \DateTime $startTime
5959
* @param \DateTime $endTime
60-
* @param integer $totalCount
60+
* @param integer $processed
61+
* @param integer $imported
62+
* @param integer $skipped
63+
* @param integer $errors
6164
* @param \SplObjectStorage $exceptions
6265
*/
63-
public function __construct($name, \DateTime $startTime, \DateTime $endTime, $totalCount, \SplObjectStorage $exceptions)
66+
public function __construct($name, \DateTime $startTime, \DateTime $endTime, $processed, $imported, $skipped, $errors, \SplObjectStorage $exceptions)
6467
{
6568
$this->name = $name;
6669
$this->startTime = $startTime;
6770
$this->endTime = $endTime;
6871
$this->elapsed = $startTime->diff($endTime);
69-
$this->totalProcessedCount = $totalCount;
70-
$this->errorCount = count($exceptions);
71-
$this->successCount = $totalCount - $this->errorCount;
72+
73+
//Should expect $processed = $errors+$imported+$skipped
74+
$this->totalProcessedCount = $processed;
75+
$this->errorCount = $errors;
76+
$this->successCount = $imported;
77+
$this->skippedCount = $skipped;
78+
7279
$this->exceptions = $exceptions;
7380
}
7481

‎src/Workflow/StepAggregator.php‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,11 @@ public function addWriter(Writer $writer)
107107
*/
108108
public function process()
109109
{
110-
$count = 0;
110+
$processed = 0;
111+
$skipped = 0;
112+
$errors = 0;
113+
$imported = 0;
114+
111115
$exceptions = new \SplObjectStorage();
112116
$startTime = new \DateTime;
113117

@@ -131,9 +135,12 @@ public function process()
131135
break;
132136
}
133137

138+
$processed++;
139+
134140
try {
135141
foreach (clone $this->steps as $step) {
136142
if (false === $step->process($item)) {
143+
$skipped++;
137144
continue 2;
138145
}
139146
}
@@ -145,23 +152,26 @@ public function process()
145152
foreach ($this->writers as $writer) {
146153
$writer->writeItem($item);
147154
}
155+
} catch(Exception\StopException $e) {
156+
break;
148157
} catch(Exception $e) {
149158
if (!$this->skipItemOnFailure) {
150159
throw $e;
151160
}
152161

162+
$errors++;
153163
$exceptions->attach($e, $index);
154164
$this->logger->error($e->getMessage());
155165
}
156166

157-
$count++;
167+
$imported++;
158168
}
159169

160170
foreach ($this->writers as $writer) {
161171
$writer->finish();
162172
}
163173

164-
return new Result($this->name, $startTime, new \DateTime, $count, $exceptions);
174+
return new Result($this->name, $startTime, new \DateTime, $processed, $imported, $skipped, $errors, $exceptions);
165175
}
166176

167177
/**

‎tests/Filter/OffsetFilterTest.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,22 @@ public function testOffsetWithMaxCount()
4848
$resultItems = $this->applyFilter(new OffsetFilter(1, 1), $items);
4949
$this->assertEquals($resultItems, array('second'));
5050
}
51+
52+
/**
53+
* @expectedException \Ddeboer\DataImport\Exception\StopException
54+
*/
55+
public function testMaxCountStop()
56+
{
57+
$items = array('first','second','third','fourth');
58+
$this->applyFilter(new OffsetFilter(0, 2, true), $items);
59+
}
60+
61+
/**
62+
* @expectedException \Ddeboer\DataImport\Exception\StopException
63+
*/
64+
public function testOffsetWithMaxCountStop()
65+
{
66+
$items = array('first','second','third','fourth');
67+
$this->applyFilter(new OffsetFilter(1, 1, true), $items);
68+
}
5169
}

‎tests/ResultTest.php‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,21 @@ class ResultTest extends \PHPUnit_Framework_TestCase
1313
{
1414
public function testResultName()
1515
{
16-
$result = new Result('export', new \DateTime, new \DateTime, 10, new \SplObjectStorage());
16+
$result = new Result('export', new \DateTime, new \DateTime, 10, 10, 0, 0, new \SplObjectStorage());
1717
$this->assertSame('export', $result->getName());
1818
}
1919

2020
public function testResultCounts()
2121
{
22-
$result = new Result('export', new \DateTime, new \DateTime, 10, new \SplObjectStorage());
22+
$result = new Result('export', new \DateTime, new \DateTime, 10, 10, 0, 0, new \SplObjectStorage());
2323
$this->assertSame(10, $result->getTotalProcessedCount());
2424
$this->assertSame(10, $result->getSuccessCount());
2525
$this->assertSame(0, $result->getErrorCount());
2626

2727
$exceptions = new \SplObjectStorage();
2828
$exceptions->attach(new \Exception());
2929
$exceptions->attach(new \Exception());
30-
$result = new Result('export', new \DateTime, new \DateTime, 10, $exceptions);
30+
$result = new Result('export', new \DateTime, new \DateTime, 10, 8, 0, 2, $exceptions);
3131
$this->assertSame(10, $result->getTotalProcessedCount());
3232
$this->assertSame(8, $result->getSuccessCount());
3333
$this->assertSame(2, $result->getErrorCount());
@@ -39,7 +39,7 @@ public function testDates()
3939
$startDate = new \DateTime("22-07-2014 22:00");
4040
$endDate = new \DateTime("22-07-2014 23:30");
4141

42-
$result = new Result('export', $startDate, $endDate, 10, new \SplObjectStorage());
42+
$result = new Result('export', $startDate, $endDate, 10, 10, 0, 0, new \SplObjectStorage());
4343

4444
$this->assertSame($startDate, $result->getStartTime());
4545
$this->assertSame($endDate, $result->getEndTime());
@@ -52,13 +52,13 @@ public function testHasErrorsReturnsTrueIfAnyExceptions()
5252
$exceptions->attach(new \Exception());
5353
$exceptions->attach(new \Exception());
5454

55-
$result = new Result('export', new \DateTime, new \DateTime, 10, $exceptions);
55+
$result = new Result('export', new \DateTime, new \DateTime, 10, 10, 0, 2, $exceptions);
5656
$this->assertTrue($result->hasErrors());
5757
}
5858

5959
public function testHasErrorsReturnsFalseIfNoExceptions()
6060
{
61-
$result = new Result('export', new \DateTime, new \DateTime, 10, new \SplObjectStorage());
61+
$result = new Result('export', new \DateTime, new \DateTime, 10, 10, 0, 0, new \SplObjectStorage());
6262
$this->assertFalse($result->hasErrors());
6363
}
6464

@@ -68,7 +68,7 @@ public function testGetExceptions()
6868
$exceptions->attach(new \Exception());
6969
$exceptions->attach(new \Exception());
7070

71-
$result = new Result('export', new \DateTime, new \DateTime, 10, $exceptions);
71+
$result = new Result('export', new \DateTime, new \DateTime, 10, 10, 0, 2, $exceptions);
7272
$this->assertSame($exceptions, $result->getExceptions());
7373
}
7474
}

0 commit comments

Comments
 (0)