Skip to content

Commit 0bd8238

Browse files
committed
Added DataReader for parallel training with one DB session
- Makes sure each solver accesses a different subset of the data - Sequential reading of DB for performance - Prefetches a configurable amount of data to host memory - Distributes data to solvers in round-robin way for determinism
1 parent 01cbda5 commit 0bd8238

8 files changed

Lines changed: 249 additions & 35 deletions

File tree

include/caffe/data_layers.hpp

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@
55
#include <utility>
66
#include <vector>
77

8-
#include "boost/scoped_ptr.hpp"
98
#include "hdf5.h"
109

1110
#include "caffe/blob.hpp"
1211
#include "caffe/common.hpp"
12+
#include "caffe/data_reader.hpp"
1313
#include "caffe/data_transformer.hpp"
1414
#include "caffe/filler.hpp"
1515
#include "caffe/internal_thread.hpp"
@@ -90,8 +90,7 @@ class BasePrefetchingDataLayer :
9090
template <typename Dtype>
9191
class DataLayer : public BasePrefetchingDataLayer<Dtype> {
9292
public:
93-
explicit DataLayer(const LayerParameter& param)
94-
: BasePrefetchingDataLayer<Dtype>(param) {}
93+
explicit DataLayer(const LayerParameter& param);
9594
virtual ~DataLayer();
9695
virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
9796
const vector<Blob<Dtype>*>& top);
@@ -104,8 +103,7 @@ class DataLayer : public BasePrefetchingDataLayer<Dtype> {
104103
protected:
105104
virtual void load_batch(Batch<Dtype>* batch);
106105

107-
shared_ptr<db::DB> db_;
108-
shared_ptr<db::Cursor> cursor_;
106+
DataReader reader_;
109107
};
110108

111109
/**

include/caffe/data_reader.hpp

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#ifndef CAFFE_DATA_READER_HPP_
2+
#define CAFFE_DATA_READER_HPP_
3+
4+
#include <map>
5+
#include <string>
6+
#include <vector>
7+
8+
#include "caffe/common.hpp"
9+
#include "caffe/internal_thread.hpp"
10+
#include "caffe/util/blocking_queue.hpp"
11+
#include "caffe/util/db.hpp"
12+
13+
namespace caffe {
14+
15+
/**
16+
* @brief Reads data from a source to queues available to data layers.
17+
* A single reading thread is created per source, even if multiple solvers
18+
* are running in parallel, e.g. for multi-GPU training. This makes sure
19+
* databases are read sequentially, and that each solver accesses a different
20+
* subset of the database. Data is distributed to solvers in a round-robin
21+
* way to keep parallel training deterministic.
22+
*/
23+
class DataReader {
24+
public:
25+
explicit DataReader(const LayerParameter& param);
26+
~DataReader();
27+
28+
inline BlockingQueue<Datum*>& free() const {
29+
return queue_pair_->free_;
30+
}
31+
inline BlockingQueue<Datum*>& full() const {
32+
return queue_pair_->full_;
33+
}
34+
35+
protected:
36+
// Queue pairs are shared between a body and its readers
37+
class QueuePair {
38+
public:
39+
explicit QueuePair(int size);
40+
~QueuePair();
41+
42+
BlockingQueue<Datum*> free_;
43+
BlockingQueue<Datum*> full_;
44+
45+
DISABLE_COPY_AND_ASSIGN(QueuePair);
46+
};
47+
48+
// A single body is created per source
49+
class Body : public InternalThread {
50+
public:
51+
explicit Body(const LayerParameter& param);
52+
virtual ~Body();
53+
54+
protected:
55+
void InternalThreadEntry();
56+
void read_one(db::Cursor* cursor, QueuePair* qp);
57+
58+
const LayerParameter param_;
59+
BlockingQueue<shared_ptr<QueuePair> > new_queue_pairs_;
60+
61+
friend class DataReader;
62+
63+
DISABLE_COPY_AND_ASSIGN(Body);
64+
};
65+
66+
// A source is uniquely identified by its layer name + path, in case
67+
// the same database is read from two different locations in the net.
68+
static inline string source_key(const LayerParameter& param) {
69+
return param.name() + ":" + param.data_param().source();
70+
}
71+
72+
const shared_ptr<QueuePair> queue_pair_;
73+
shared_ptr<Body> body_;
74+
75+
static map<const string, boost::weak_ptr<DataReader::Body> > bodies_;
76+
77+
DISABLE_COPY_AND_ASSIGN(DataReader);
78+
};
79+
80+
} // namespace caffe
81+
82+
#endif // CAFFE_DATA_READER_HPP_

src/caffe/data_reader.cpp

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#include <boost/thread.hpp>
2+
#include <map>
3+
#include <string>
4+
#include <vector>
5+
6+
#include "caffe/common.hpp"
7+
#include "caffe/data_layers.hpp"
8+
#include "caffe/data_reader.hpp"
9+
#include "caffe/proto/caffe.pb.h"
10+
11+
namespace caffe {
12+
13+
using boost::weak_ptr;
14+
15+
map<const string, weak_ptr<DataReader::Body> > DataReader::bodies_;
16+
static boost::mutex bodies_mutex_;
17+
18+
DataReader::DataReader(const LayerParameter& param)
19+
: queue_pair_(new QueuePair( //
20+
param.data_param().prefetch() * param.data_param().batch_size())) {
21+
// Get or create a body
22+
boost::mutex::scoped_lock lock(bodies_mutex_);
23+
string key = source_key(param);
24+
weak_ptr<Body>& weak = bodies_[key];
25+
body_ = weak.lock();
26+
if (!body_) {
27+
body_.reset(new Body(param));
28+
bodies_[key] = weak_ptr<Body>(body_);
29+
}
30+
body_->new_queue_pairs_.push(queue_pair_);
31+
}
32+
33+
DataReader::~DataReader() {
34+
string key = source_key(body_->param_);
35+
body_.reset();
36+
boost::mutex::scoped_lock lock(bodies_mutex_);
37+
if (bodies_[key].expired()) {
38+
bodies_.erase(key);
39+
}
40+
}
41+
42+
//
43+
44+
DataReader::QueuePair::QueuePair(int size) {
45+
// Initialize the free queue with requested number of datums
46+
for (int i = 0; i < size; ++i) {
47+
free_.push(new Datum());
48+
}
49+
}
50+
51+
DataReader::QueuePair::~QueuePair() {
52+
Datum* datum;
53+
while (free_.try_pop(&datum)) {
54+
delete datum;
55+
}
56+
while (full_.try_pop(&datum)) {
57+
delete datum;
58+
}
59+
}
60+
61+
//
62+
63+
DataReader::Body::Body(const LayerParameter& param)
64+
: param_(param),
65+
new_queue_pairs_() {
66+
StartInternalThread();
67+
}
68+
69+
DataReader::Body::~Body() {
70+
StopInternalThread();
71+
}
72+
73+
void DataReader::Body::InternalThreadEntry() {
74+
shared_ptr<db::DB> db(db::GetDB(param_.data_param().backend()));
75+
db->Open(param_.data_param().source(), db::READ);
76+
shared_ptr<db::Cursor> cursor(db->NewCursor());
77+
vector<shared_ptr<QueuePair> > qps;
78+
try {
79+
// int solver_count = param_.phase() == TRAIN ? Caffe::solver_count() : 1;
80+
// TODO single solver until multi-gpu merge
81+
int solver_count = 1;
82+
83+
// To ensure deterministic runs, only start running once all solvers
84+
// are ready. But solvers need to peek on one item during initialization,
85+
// so read one item, then wait for the next solver.
86+
for (int i = 0; i < solver_count; ++i) {
87+
shared_ptr<QueuePair> qp(new_queue_pairs_.pop());
88+
read_one(cursor.get(), qp.get());
89+
qps.push_back(qp);
90+
}
91+
// Main loop
92+
while (!must_stop()) {
93+
for (int i = 0; i < solver_count; ++i) {
94+
read_one(cursor.get(), qps[i].get());
95+
}
96+
// Check no additional readers have been created. This can happen if
97+
// more than one net is trained at a time per process, whether single
98+
// or multi solver. It might also happen if two data layers have same
99+
// name and same source.
100+
CHECK_EQ(new_queue_pairs_.size(), 0);
101+
}
102+
} catch (boost::thread_interrupted&) {
103+
// Interrupted exception is expected on shutdown
104+
}
105+
}
106+
107+
void DataReader::Body::read_one(db::Cursor* cursor, QueuePair* qp) {
108+
Datum* datum = qp->free_.pop();
109+
// TODO deserialize in-place instead of copy?
110+
datum->ParseFromString(cursor->value());
111+
qp->full_.push(datum);
112+
113+
// go to the next iter
114+
cursor->Next();
115+
if (!cursor->valid()) {
116+
DLOG(INFO) << "Restarting data prefetching from start.";
117+
cursor->SeekToFirst();
118+
}
119+
}
120+
121+
} // namespace caffe

src/caffe/layers/data_layer.cpp

Lines changed: 11 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,15 @@
1111
#include "caffe/proto/caffe.pb.h"
1212
#include "caffe/util/benchmark.hpp"
1313
#include "caffe/util/io.hpp"
14-
#include "caffe/util/math_functions.hpp"
15-
#include "caffe/util/rng.hpp"
1614

1715
namespace caffe {
1816

17+
template <typename Dtype>
18+
DataLayer<Dtype>::DataLayer(const LayerParameter& param)
19+
: BasePrefetchingDataLayer<Dtype>(param),
20+
reader_(param) {
21+
}
22+
1923
template <typename Dtype>
2024
DataLayer<Dtype>::~DataLayer() {
2125
this->StopInternalThread();
@@ -24,23 +28,8 @@ DataLayer<Dtype>::~DataLayer() {
2428
template <typename Dtype>
2529
void DataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
2630
const vector<Blob<Dtype>*>& top) {
27-
// Initialize DB
28-
db_.reset(db::GetDB(this->layer_param_.data_param().backend()));
29-
db_->Open(this->layer_param_.data_param().source(), db::READ);
30-
cursor_.reset(db_->NewCursor());
31-
32-
// Check if we should randomly skip a few data points
33-
if (this->layer_param_.data_param().rand_skip()) {
34-
unsigned int skip = caffe_rng_rand() %
35-
this->layer_param_.data_param().rand_skip();
36-
LOG(INFO) << "Skipping first " << skip << " data points.";
37-
while (skip-- > 0) {
38-
cursor_->Next();
39-
}
40-
}
4131
// Read a data point, and use it to initialize the top blob.
42-
Datum datum;
43-
datum.ParseFromString(cursor_->value());
32+
Datum& datum = *(reader_.full().peek());
4433

4534
bool force_color = this->layer_param_.data_param().force_encoded_color();
4635
if ((force_color && DecodeDatum(&datum, true)) ||
@@ -97,8 +86,7 @@ void DataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
9786
const int crop_size = this->layer_param_.transform_param().crop_size();
9887
bool force_color = this->layer_param_.data_param().force_encoded_color();
9988
if (batch_size == 1 && crop_size == 0) {
100-
Datum datum;
101-
datum.ParseFromString(cursor_->value());
89+
Datum& datum = *(reader_.full().peek());
10290
if (datum.encoded()) {
10391
if (force_color) {
10492
DecodeDatum(&datum, true);
@@ -121,9 +109,7 @@ void DataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
121109
for (int item_id = 0; item_id < batch_size; ++item_id) {
122110
timer.Start();
123111
// get a blob
124-
Datum datum;
125-
datum.ParseFromString(cursor_->value());
126-
112+
Datum& datum = *(reader_.full().pop("Waiting for data"));
127113
cv::Mat cv_img;
128114
if (datum.encoded()) {
129115
if (force_color) {
@@ -153,12 +139,8 @@ void DataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
153139
top_label[item_id] = datum.label();
154140
}
155141
trans_time += timer.MicroSeconds();
156-
// go to the next iter
157-
cursor_->Next();
158-
if (!cursor_->valid()) {
159-
DLOG(INFO) << "Restarting data prefetching from start.";
160-
cursor_->SeekToFirst();
161-
}
142+
143+
reader_.free().push(const_cast<Datum*>(&datum));
162144
}
163145
batch_timer.Stop();
164146
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";

src/caffe/proto/caffe.proto

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,7 @@ message DataParameter {
455455
// to avoid all asynchronous sgd clients to start at the same point. The skip
456456
// point would be set as rand_skip * rand(0,1). Note that rand_skip should not
457457
// be larger than the number of keys in the database.
458+
// DEPRECATED. Each solver accesses a different subset of the database.
458459
optional uint32 rand_skip = 7 [default = 0];
459460
optional DB backend = 8 [default = LEVELDB];
460461
// DEPRECATED. See TransformationParameter. For data pre-processing, we can do
@@ -470,6 +471,9 @@ message DataParameter {
470471
optional bool mirror = 6 [default = false];
471472
// Force the encoded image to have 3 color channels
472473
optional bool force_encoded_color = 9 [default = false];
474+
// Prefetch queue (Number of batches to prefetch to host memory, increase if
475+
// data access bandwidth varies).
476+
optional uint32 prefetch = 10 [default = 4];
473477
}
474478

475479
message DropoutParameter {

src/caffe/test/test_layer_factory.cpp

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
#include <map>
22
#include <string>
33

4+
#include "boost/scoped_ptr.hpp"
45
#include "gtest/gtest.h"
56

67
#include "caffe/common.hpp"
78
#include "caffe/layer.hpp"
89
#include "caffe/layer_factory.hpp"
10+
#include "caffe/util/db.hpp"
11+
#include "caffe/util/io.hpp"
912

1013
#include "caffe/test/test_caffe_main.hpp"
1114

@@ -21,11 +24,20 @@ TYPED_TEST(LayerFactoryTest, TestCreateLayer) {
2124
typename LayerRegistry<Dtype>::CreatorRegistry& registry =
2225
LayerRegistry<Dtype>::Registry();
2326
shared_ptr<Layer<Dtype> > layer;
24-
LayerParameter layer_param;
2527
for (typename LayerRegistry<Dtype>::CreatorRegistry::iterator iter =
2628
registry.begin(); iter != registry.end(); ++iter) {
2729
// Special case: PythonLayer is checked by pytest
2830
if (iter->first == "Python") { continue; }
31+
LayerParameter layer_param;
32+
// Data layers expect a DB
33+
if (iter->first == "Data") {
34+
string tmp;
35+
MakeTempDir(&tmp);
36+
boost::scoped_ptr<db::DB> db(db::GetDB(DataParameter_DB_LEVELDB));
37+
db->Open(tmp, db::NEW);
38+
db->Close();
39+
layer_param.mutable_data_param()->set_source(tmp);
40+
}
2941
layer_param.set_type(iter->first);
3042
layer = LayerRegistry<Dtype>::CreateLayer(layer_param);
3143
EXPECT_EQ(iter->first, layer->type());

0 commit comments

Comments
 (0)