Skip to content

Commit 7d5f7ed

Browse files
Yangqing Jiafacebook-github-bot
Yangqing Jia
authored and
facebook-github-bot
committed
Using c10 namespace across caffe2. (pytorch#12714)
Summary: Pull Request resolved: pytorch#12714 This is a short change to enable c10 namespace in caffe2. We did not enable it before due to gflags global variable confusion, but it should have been mostly cleaned now. Right now, the plan on record is that namespace caffe2 and namespace aten will fully be supersets of namespace c10. Most of the diff is codemod, and only two places of non-codemod is in caffe2/core/common.h, where ``` using namespace c10; ``` is added, and in Flags.h, where instead of creating aliasing variables in c10 namespace, we directly put it in the global namespace to match gflags (and same behavior if gflags is not being built with). Reviewed By: dzhulgakov Differential Revision: D10390486 fbshipit-source-id: 5e2df730e28e29a052f513bddc558d9f78a23b9b
1 parent 348867c commit 7d5f7ed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+414
-433
lines changed

‎aten/src/ATen/core/TensorImpl.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -594,11 +594,10 @@ struct CAFFE2_API TensorImpl : public c10::intrusive_ptr_target {
594594
} else {
595595
reset_tensor = storage_.capacity() <
596596
(storage_offset_ + numel_) * storage_.itemsize() ||
597-
!c10::FLAGS_caffe2_keep_on_shrink ||
597+
!FLAGS_caffe2_keep_on_shrink ||
598598
storage_.capacity() -
599599
(storage_offset_ + numel_) * storage_.itemsize() >
600-
static_cast<size_t>(
601-
c10::FLAGS_caffe2_max_keep_on_shrink_memory);
600+
static_cast<size_t>(FLAGS_caffe2_max_keep_on_shrink_memory);
602601
}
603602

604603
if (reset_tensor && !is_init) {

‎binaries/caffe2_benchmark.cc

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -78,19 +78,19 @@ int main(int argc, char** argv) {
7878
benchmark(
7979
argc,
8080
argv,
81-
c10::FLAGS_backend,
82-
c10::FLAGS_init_net,
83-
c10::FLAGS_input,
84-
c10::FLAGS_input_dims,
85-
c10::FLAGS_input_file,
86-
c10::FLAGS_input_type,
87-
c10::FLAGS_iter,
88-
c10::FLAGS_net,
89-
c10::FLAGS_output,
90-
c10::FLAGS_output_folder,
91-
c10::FLAGS_run_individual,
92-
c10::FLAGS_sleep_before_run,
93-
c10::FLAGS_text_output,
94-
c10::FLAGS_warmup,
95-
c10::FLAGS_wipe_cache);
81+
FLAGS_backend,
82+
FLAGS_init_net,
83+
FLAGS_input,
84+
FLAGS_input_dims,
85+
FLAGS_input_file,
86+
FLAGS_input_type,
87+
FLAGS_iter,
88+
FLAGS_net,
89+
FLAGS_output,
90+
FLAGS_output_folder,
91+
FLAGS_run_individual,
92+
FLAGS_sleep_before_run,
93+
FLAGS_text_output,
94+
FLAGS_warmup,
95+
FLAGS_wipe_cache);
9696
}

‎binaries/convert_caffe_image_db.cc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ int main(int argc, char** argv) {
3737
caffe2::GlobalInit(&argc, &argv);
3838

3939
std::unique_ptr<DB> in_db(caffe2::db::CreateDB(
40-
c10::FLAGS_input_db_type, c10::FLAGS_input_db, caffe2::db::READ));
40+
FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ));
4141
std::unique_ptr<DB> out_db(caffe2::db::CreateDB(
42-
c10::FLAGS_output_db_type, c10::FLAGS_output_db, caffe2::db::NEW));
42+
FLAGS_output_db_type, FLAGS_output_db, caffe2::db::NEW));
4343
std::unique_ptr<Cursor> cursor(in_db->NewCursor());
4444
std::unique_ptr<Transaction> transaction(out_db->NewTransaction());
4545
int count = 0;
@@ -80,7 +80,7 @@ int main(int argc, char** argv) {
8080
data->set_byte_data(buffer, datum.data().size());
8181
}
8282
transaction->Put(cursor->key(), protos.SerializeAsString());
83-
if (++count % c10::FLAGS_batch_size == 0) {
83+
if (++count % FLAGS_batch_size == 0) {
8484
transaction->Commit();
8585
LOG(INFO) << "Converted " << count << " items so far.";
8686
}

‎binaries/convert_db.cc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,15 @@ int main(int argc, char** argv) {
3333
caffe2::GlobalInit(&argc, &argv);
3434

3535
std::unique_ptr<DB> in_db(caffe2::db::CreateDB(
36-
c10::FLAGS_input_db_type, c10::FLAGS_input_db, caffe2::db::READ));
36+
FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ));
3737
std::unique_ptr<DB> out_db(caffe2::db::CreateDB(
38-
c10::FLAGS_output_db_type, c10::FLAGS_output_db, caffe2::db::NEW));
38+
FLAGS_output_db_type, FLAGS_output_db, caffe2::db::NEW));
3939
std::unique_ptr<Cursor> cursor(in_db->NewCursor());
4040
std::unique_ptr<Transaction> transaction(out_db->NewTransaction());
4141
int count = 0;
4242
for (; cursor->Valid(); cursor->Next()) {
4343
transaction->Put(cursor->key(), cursor->value());
44-
if (++count % c10::FLAGS_batch_size == 0) {
44+
if (++count % FLAGS_batch_size == 0) {
4545
transaction->Commit();
4646
LOG(INFO) << "Converted " << count << " items so far.";
4747
}

‎binaries/convert_encoded_to_raw_leveldb.cc

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616

1717
// This script converts an image dataset to leveldb.
1818
//
19-
// c10::FLAGS_input_folder is the root folder that holds all the images, and
20-
// c10::FLAGS_list_file should be a list of files as well as their labels, in
19+
// FLAGS_input_folder is the root folder that holds all the images, and
20+
// FLAGS_list_file should be a list of files as well as their labels, in
2121
// the format as
2222
// subfolder1/file1.JPEG 7
2323
// ....
@@ -41,7 +41,7 @@ C10_DEFINE_bool(color, true, "If set, load images in color.");
4141
C10_DEFINE_int(
4242
scale,
4343
256,
44-
"If c10::FLAGS_raw is set, scale all the images' shorter edge to the given "
44+
"If FLAGS_raw is set, scale all the images' shorter edge to the given "
4545
"value.");
4646
C10_DEFINE_bool(warp, false, "If warp is set, warp the images to square.");
4747

@@ -93,7 +93,7 @@ void ConvertToRawDataset(
9393
data->set_data_type(TensorProto::BYTE);
9494
data->add_dims(0);
9595
data->add_dims(0);
96-
if (c10::FLAGS_color) {
96+
if (FLAGS_color) {
9797
data->add_dims(3);
9898
}
9999
string value;
@@ -108,21 +108,20 @@ void ConvertToRawDataset(
108108
const string& encoded_image = input_protos.protos(0).string_data(0);
109109
int encoded_size = encoded_image.size();
110110
cv::Mat img = cv::imdecode(
111-
cv::Mat(1, &encoded_size, CV_8UC1,
112-
const_cast<char*>(encoded_image.data())),
113-
c10::FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
111+
cv::Mat(
112+
1, &encoded_size, CV_8UC1, const_cast<char*>(encoded_image.data())),
113+
FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
114114
cv::Mat resized_img;
115115
int scaled_width, scaled_height;
116-
if (c10::FLAGS_warp) {
117-
scaled_width = c10::FLAGS_scale;
118-
scaled_height = c10::FLAGS_scale;
116+
if (FLAGS_warp) {
117+
scaled_width = FLAGS_scale;
118+
scaled_height = FLAGS_scale;
119119
} else if (img.rows > img.cols) {
120-
scaled_width = c10::FLAGS_scale;
121-
scaled_height =
122-
static_cast<float>(img.rows) * c10::FLAGS_scale / img.cols;
120+
scaled_width = FLAGS_scale;
121+
scaled_height = static_cast<float>(img.rows) * FLAGS_scale / img.cols;
123122
} else {
124-
scaled_height = c10::FLAGS_scale;
125-
scaled_width = static_cast<float>(img.cols) * c10::FLAGS_scale / img.rows;
123+
scaled_height = FLAGS_scale;
124+
scaled_width = static_cast<float>(img.cols) * FLAGS_scale / img.rows;
126125
}
127126
cv::resize(img, resized_img, cv::Size(scaled_width, scaled_height), 0, 0,
128127
cv::INTER_LINEAR);
@@ -131,7 +130,7 @@ void ConvertToRawDataset(
131130
DCHECK(resized_img.isContinuous());
132131
data->set_byte_data(
133132
resized_img.ptr(),
134-
scaled_height * scaled_width * (c10::FLAGS_color ? 3 : 1));
133+
scaled_height * scaled_width * (FLAGS_color ? 3 : 1));
135134
output_protos.SerializeToString(&value);
136135
// Put in db
137136
batch->Put(iter->key(), value);
@@ -153,7 +152,6 @@ void ConvertToRawDataset(
153152

154153
int main(int argc, char** argv) {
155154
caffe2::GlobalInit(&argc, &argv);
156-
caffe2::ConvertToRawDataset(
157-
c10::FLAGS_input_db_name, c10::FLAGS_output_db_name);
155+
caffe2::ConvertToRawDataset(FLAGS_input_db_name, FLAGS_output_db_name);
158156
return 0;
159157
}

‎binaries/convert_image_to_tensor.cc

Lines changed: 27 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,15 @@ namespace caffe2 {
4444
cv::Mat resizeImage(cv::Mat& img) {
4545
cv::Mat resized_img;
4646
int scaled_width, scaled_height;
47-
if (c10::FLAGS_warp) {
48-
scaled_width = c10::FLAGS_scale;
49-
scaled_height = c10::FLAGS_scale;
47+
if (FLAGS_warp) {
48+
scaled_width = FLAGS_scale;
49+
scaled_height = FLAGS_scale;
5050
} else if (img.rows > img.cols) {
51-
scaled_width = c10::FLAGS_scale;
52-
scaled_height = static_cast<float>(img.rows) * c10::FLAGS_scale / img.cols;
51+
scaled_width = FLAGS_scale;
52+
scaled_height = static_cast<float>(img.rows) * FLAGS_scale / img.cols;
5353
} else {
54-
scaled_height = c10::FLAGS_scale;
55-
scaled_width = static_cast<float>(img.cols) * c10::FLAGS_scale / img.rows;
54+
scaled_height = FLAGS_scale;
55+
scaled_width = static_cast<float>(img.cols) * FLAGS_scale / img.rows;
5656
}
5757
cv::resize(
5858
img,
@@ -87,9 +87,9 @@ std::vector<float> convertToVector(cv::Mat& img) {
8787
std::vector<float> mean(3, 0);
8888
std::vector<float> std(3, 1);
8989
bool bgrtorgb = false;
90-
assert(img.cols == c10::FLAGS_scale);
91-
assert(img.rows == c10::FLAGS_scale);
92-
vector<string> steps = caffe2::split(',', c10::FLAGS_preprocess);
90+
assert(img.cols == FLAGS_scale);
91+
assert(img.rows == FLAGS_scale);
92+
vector<string> steps = caffe2::split(',', FLAGS_preprocess);
9393
for (int i = 0; i < steps.size(); i++) {
9494
auto step = steps[i];
9595
if (step == "subtract128") {
@@ -112,8 +112,8 @@ std::vector<float> convertToVector(cv::Mat& img) {
112112
}
113113
}
114114

115-
int C = c10::FLAGS_color ? 3 : 1;
116-
int total_size = C * c10::FLAGS_scale * c10::FLAGS_scale;
115+
int C = FLAGS_color ? 3 : 1;
116+
int total_size = C * FLAGS_scale * FLAGS_scale;
117117
std::vector<float> values(total_size);
118118
if (C == 1) {
119119
cv::MatIterator_<uchar> it, end;
@@ -130,9 +130,9 @@ std::vector<float> convertToVector(cv::Mat& img) {
130130
for (it = img.begin<cv::Vec3b>(), end = img.end<cv::Vec3b>(); it != end;
131131
++it, i++) {
132132
values[i] = (((*it)[b] / normalize[0] - mean[0]) / std[0]);
133-
int offset = c10::FLAGS_scale * c10::FLAGS_scale + i;
133+
int offset = FLAGS_scale * FLAGS_scale + i;
134134
values[offset] = (((*it)[g] / normalize[1] - mean[1]) / std[1]);
135-
offset = c10::FLAGS_scale * c10::FLAGS_scale + offset;
135+
offset = FLAGS_scale * FLAGS_scale + offset;
136136
values[offset] = (((*it)[r] / normalize[2] - mean[2]) / std[2]);
137137
}
138138
}
@@ -145,8 +145,7 @@ std::vector<float> convertOneImage(std::string& filename) {
145145
std::cout << "Converting " << filename << std::endl;
146146
// Load image
147147
cv::Mat img = cv::imread(
148-
filename,
149-
c10::FLAGS_color ? CV_LOAD_IMAGE_COLOR : CV_LOAD_IMAGE_GRAYSCALE);
148+
filename, FLAGS_color ? CV_LOAD_IMAGE_COLOR : CV_LOAD_IMAGE_GRAYSCALE);
150149

151150
cv::Mat crop = cropToSquare(img);
152151

@@ -155,17 +154,17 @@ std::vector<float> convertOneImage(std::string& filename) {
155154
// Assert we don't have to deal with alignment
156155
DCHECK(resized_img.isContinuous());
157156
assert(resized_img.rows == resized_img.cols);
158-
assert(resized_img.rows == c10::FLAGS_scale);
157+
assert(resized_img.rows == FLAGS_scale);
159158
std::vector<float> one_image_values = convertToVector(resized_img);
160159
return one_image_values;
161160
}
162161

163162
void convertImages() {
164163
vector<string> file_names;
165-
if (c10::FLAGS_input_images != "") {
166-
file_names = caffe2::split(',', c10::FLAGS_input_images);
167-
} else if (c10::FLAGS_input_image_file != "") {
168-
std::ifstream infile(c10::FLAGS_input_image_file);
164+
if (FLAGS_input_images != "") {
165+
file_names = caffe2::split(',', FLAGS_input_images);
166+
} else if (FLAGS_input_image_file != "") {
167+
std::ifstream infile(FLAGS_input_image_file);
169168
std::string line;
170169
while (std::getline(infile, line)) {
171170
vector<string> file_name = caffe2::split(',', line);
@@ -181,7 +180,7 @@ void convertImages() {
181180
assert(false);
182181
}
183182
std::vector<std::vector<float>> values;
184-
int C = c10::FLAGS_color ? 3 : 1;
183+
int C = FLAGS_color ? 3 : 1;
185184
for (int i = 0; i < file_names.size(); i++) {
186185
std::vector<float> one_image_values = convertOneImage(file_names[i]);
187186
values.push_back(one_image_values);
@@ -193,19 +192,19 @@ void convertImages() {
193192
data->set_data_type(TensorProto::FLOAT);
194193
data->add_dims(values.size());
195194
data->add_dims(C);
196-
data->add_dims(c10::FLAGS_scale);
197-
data->add_dims(c10::FLAGS_scale);
195+
data->add_dims(FLAGS_scale);
196+
data->add_dims(FLAGS_scale);
198197

199198
for (int i = 0; i < values.size(); i++) {
200-
assert(values[i].size() == C * c10::FLAGS_scale * c10::FLAGS_scale);
199+
assert(values[i].size() == C * FLAGS_scale * FLAGS_scale);
201200
for (int j = 0; j < values[i].size(); j++) {
202201
data->add_float_data(values[i][j]);
203202
}
204203
}
205-
if (c10::FLAGS_text_output) {
206-
caffe2::WriteProtoToTextFile(protos, c10::FLAGS_output_tensor);
204+
if (FLAGS_text_output) {
205+
caffe2::WriteProtoToTextFile(protos, FLAGS_output_tensor);
207206
} else {
208-
caffe2::WriteProtoToBinaryFile(protos, c10::FLAGS_output_tensor);
207+
caffe2::WriteProtoToBinaryFile(protos, FLAGS_output_tensor);
209208
}
210209
}
211210

‎binaries/db_throughput.cc

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,11 @@ using caffe2::string;
4040

4141
void TestThroughputWithDB() {
4242
std::unique_ptr<DB> in_db(caffe2::db::CreateDB(
43-
c10::FLAGS_input_db_type, c10::FLAGS_input_db, caffe2::db::READ));
43+
FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ));
4444
std::unique_ptr<Cursor> cursor(in_db->NewCursor());
45-
for (int iter_id = 0; iter_id < c10::FLAGS_repeat; ++iter_id) {
45+
for (int iter_id = 0; iter_id < FLAGS_repeat; ++iter_id) {
4646
caffe2::Timer timer;
47-
for (int i = 0; i < c10::FLAGS_report_interval; ++i) {
47+
for (int i = 0; i < FLAGS_report_interval; ++i) {
4848
string key = cursor->key();
4949
string value = cursor->value();
5050
//VLOG(1) << "Key " << key;
@@ -58,15 +58,15 @@ void TestThroughputWithDB() {
5858
"Iteration %03d, took %4.5f seconds, throughput %f items/sec.\n",
5959
iter_id,
6060
elapsed_seconds,
61-
c10::FLAGS_report_interval / elapsed_seconds);
61+
FLAGS_report_interval / elapsed_seconds);
6262
}
6363
}
6464

6565
void TestThroughputWithReaderWorker(const DBReader* reader, int thread_id) {
6666
string key, value;
67-
for (int iter_id = 0; iter_id < c10::FLAGS_repeat; ++iter_id) {
67+
for (int iter_id = 0; iter_id < FLAGS_repeat; ++iter_id) {
6868
caffe2::Timer timer;
69-
for (int i = 0; i < c10::FLAGS_report_interval; ++i) {
69+
for (int i = 0; i < FLAGS_report_interval; ++i) {
7070
reader->Read(&key, &value);
7171
}
7272
double elapsed_seconds = timer.Seconds();
@@ -76,14 +76,14 @@ void TestThroughputWithReaderWorker(const DBReader* reader, int thread_id) {
7676
thread_id,
7777
iter_id,
7878
elapsed_seconds,
79-
c10::FLAGS_report_interval / elapsed_seconds);
79+
FLAGS_report_interval / elapsed_seconds);
8080
}
8181
}
8282

8383
void TestThroughputWithReader() {
84-
caffe2::db::DBReader reader(c10::FLAGS_input_db_type, c10::FLAGS_input_db);
84+
caffe2::db::DBReader reader(FLAGS_input_db_type, FLAGS_input_db);
8585
std::vector<std::unique_ptr<std::thread>> reading_threads(
86-
c10::FLAGS_num_read_threads);
86+
FLAGS_num_read_threads);
8787
for (int i = 0; i < reading_threads.size(); ++i) {
8888
reading_threads[i].reset(new std::thread(
8989
TestThroughputWithReaderWorker, &reader, i));
@@ -95,7 +95,7 @@ void TestThroughputWithReader() {
9595

9696
int main(int argc, char** argv) {
9797
caffe2::GlobalInit(&argc, &argv);
98-
if (c10::FLAGS_use_reader) {
98+
if (FLAGS_use_reader) {
9999
TestThroughputWithReader();
100100
} else {
101101
TestThroughputWithDB();

0 commit comments

Comments
 (0)