From a73044eef857988b9bfa668d00108303e5205a51 Mon Sep 17 00:00:00 2001 From: Owen Hoffend Date: Wed, 14 Apr 2021 02:04:59 -0700 Subject: [PATCH 1/2] Fixed whitespace error causing failures in DFSL_parser Previous version of DFSL_parser did not consider mapping files containing extra whitespace characters \n and \r, added by some IDEs/OSes. This caused a failing error where the parser would fail to match tokens to anything in the list of string constants. Fixed by adding \n and \r to the list of separation characters. Also added some more descriptive debug printouts to the ParseError calls. --- .../DFSL_parser.hpp | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/cost-model/include/dataflow-specification-language/DFSL_parser.hpp b/cost-model/include/dataflow-specification-language/DFSL_parser.hpp index 9ee1541..16dfcaa 100644 --- a/cost-model/include/dataflow-specification-language/DFSL_parser.hpp +++ b/cost-model/include/dataflow-specification-language/DFSL_parser.hpp @@ -77,8 +77,9 @@ namespace maestro { std::string file_name_; std::ifstream in_file_; - void ParseError(int line_num) { + void ParseError(int line_num, std::string err = "Not reported") { std::cout << "[MAESTRO Parser] Parse error at line number " + std::to_string(line_num) + " in target file " + file_name_ << std::endl; + std::cout << "Error was: " + err << std::endl; exit(-1); } }; // End of class InputParser @@ -90,7 +91,7 @@ namespace maestro { void ParseDFSL(std::shared_ptr network) { std::string line; - boost::char_separator sep(" ,->():\t;"); + boost::char_separator sep(" ,->():\t\n\r;"); int line_number = 1; // network = std::make_shared(); @@ -167,7 +168,7 @@ namespace maestro { network->SetName(tkn); } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state Network_Identifier"); } break; @@ -181,7 +182,7 @@ namespace maestro { state_ = ParserState::Layer_Identifier; } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state Network_Body: " + tkn); } break; } @@ -202,7 +203,7 @@ namespace maestro { case ParserState::Layer_Body: { if(tkn == DFSL::brace_close_) { if(curr_layer == nullptr) { - ParseError(line_number); + ParseError(line_number, "curr_layer is nullptr"); } curr_layer->SetDimensions(dim_vector); @@ -249,7 +250,7 @@ namespace maestro { state_ = ParserState::Dataflow_Decl; } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state Layer_Body"); } break; @@ -355,7 +356,7 @@ namespace maestro { //TODO } else { - ParseError(line_number); + ParseError(line_number, "Unsupported layer type"); } state_ = ParserState::Layer_Body; @@ -367,7 +368,7 @@ namespace maestro { state_ = ParserState::Dimension_Body; } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state Dimension_Dec1"); } break; } @@ -381,7 +382,7 @@ namespace maestro { tmp_name = tkn; } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state Dimension_Body"); } state_ = ParserState::Dimension_Size; } @@ -400,7 +401,7 @@ namespace maestro { if(size == 0) { - ParseError(line_number); + ParseError(line_number, "Dimension size was 0"); } else { @@ -420,7 +421,7 @@ namespace maestro { state_ = ParserState::Dataflow_Body; } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state Dataflow_Dec1"); } break; } @@ -442,7 +443,7 @@ namespace maestro { state_ = ParserState::Dataflow_ClusterSize; } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state Dataflow_Body"); } break; } @@ -478,7 +479,7 @@ namespace maestro { } if(map_size <= 0) { - ParseError(line_number); + ParseError(line_number, "map_size was <= 0"); } get_dim_size = false; @@ -516,7 +517,7 @@ namespace maestro { } if(map_offset <= 0) { - ParseError(line_number); + ParseError(line_number, "map_offset was <= 0"); } get_dim_size = false; @@ -526,7 +527,7 @@ namespace maestro { case ParserState::Dataflow_MapVar: { if(tkn.empty()) { - ParseError(line_number); + ParseError(line_number, "Token was empty in Dataflow_MapVar state"); } switch(curr_directive_class) { @@ -539,7 +540,7 @@ namespace maestro { break; } default: { - ParseError(line_number); + ParseError(line_number, "Bad curr_directive_class"); } } @@ -579,7 +580,7 @@ namespace maestro { } if(cluster_size <= 0) { - ParseError(line_number); + ParseError(line_number, "cluster_size was <= 0"); } get_dim_size = false; @@ -607,7 +608,7 @@ namespace maestro { state_ = ParserState::Acclerator_Body; } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state Accelerator_Identifier"); } break; } @@ -623,7 +624,7 @@ namespace maestro { state_ = ParserState::NoC_Identifier; } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state Accelerator_Body"); } break; } @@ -634,7 +635,7 @@ namespace maestro { } else { //TODO: Add an error message - ParseError(line_number); + ParseError(line_number, "Bad token in state PE_Identifier"); } break; } @@ -645,7 +646,7 @@ namespace maestro { } else { //TODO: Add an error message - ParseError(line_number); + ParseError(line_number, "Bad token in state Buffer_Identifier"); } break; } @@ -676,7 +677,7 @@ namespace maestro { break; } else { - ParseError(line_number); + ParseError(line_number, "Bad token in state PE_Body"); } break; @@ -826,7 +827,7 @@ namespace maestro { } else { //TODO: Add an error message - ParseError(line_number); + ParseError(line_number, "Bad token in state NoC_Identifier"); } break; } @@ -848,7 +849,7 @@ namespace maestro { } else { //TODO: Add an error message - ParseError(line_number); + ParseError(line_number, "Bad token in state Noc_Name_Identifier"); } break; } @@ -865,7 +866,7 @@ namespace maestro { } else { //TODO: Add an error message - ParseError(line_number); + ParseError(line_number, "Bad token in state SubNoC_Body"); } break; } @@ -873,7 +874,7 @@ namespace maestro { case ParserState::NoC_BW: { int noc_bw = std::atoi(tkn.c_str()); if(noc_bw < 1) { - ParseError(line_number); + ParseError(line_number, "noc_bw was < 1"); } else { noc_bandwidth_.push_back(noc_bw); @@ -885,7 +886,7 @@ namespace maestro { case ParserState::NoC_Latency: { int noc_latency = std::atoi(tkn.c_str()); if(noc_latency < 1) { - ParseError(line_number); + ParseError(line_number, "noc_latency was < 1"); } else { noc_latency_.push_back(noc_latency); @@ -895,7 +896,7 @@ namespace maestro { } default: { - ParseError(line_number); + ParseError(line_number, "Bad parser state"); break; } } // End of switch(state_) From 3ffc16b73e0af29c3121b2db69e27a7ed71bf2c8 Mon Sep 17 00:00:00 2001 From: OwenHoffend Date: Wed, 28 Apr 2021 22:04:14 -0700 Subject: [PATCH 2/2] Add ability to import custom PyTorch models --- tools/frontend/frameworks_to_modelfile_maestro.py | 11 +++++++++-- tools/frontend/modelfile_to_mapping.py | 6 +++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/frontend/frameworks_to_modelfile_maestro.py b/tools/frontend/frameworks_to_modelfile_maestro.py index 7539418..ed0214e 100755 --- a/tools/frontend/frameworks_to_modelfile_maestro.py +++ b/tools/frontend/frameworks_to_modelfile_maestro.py @@ -85,9 +85,16 @@ import torch import torchvision.models as models from helpers.torch_maestro_summary import summary - + import sys, os device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') - model = getattr(models, opt.model)() + model = None + if opt.model == 'custom': + sys.path.insert(0,os.path.dirname(opt.custom)) + fn = os.path.basename(opt.custom).split('.')[0] + new_module = __import__(fn, fromlist=[fn]) + model = getattr(new_module, fn)() + else: + model = getattr(models, opt.model)() model = model.to(device) mae_summary = summary(model, INPUT_SIZE) with open("../../data/model/"+opt.outfile, "w") as fo: diff --git a/tools/frontend/modelfile_to_mapping.py b/tools/frontend/modelfile_to_mapping.py index c920bdc..2dc3df5 100644 --- a/tools/frontend/modelfile_to_mapping.py +++ b/tools/frontend/modelfile_to_mapping.py @@ -13,10 +13,10 @@ dsconv = 0 base_path = '../../data/' if os.path.exists(base_path + 'model/' + opt.model_file): - with open('./dataflow/' + opt.dataflow + ".m" ,"r") as fd: - with open('./dataflow/'+ 'dpt.m' , "r") as fdpt: + with open('./dataflow/' + opt.dataflow + ".m" ,"r", encoding="utf8") as fd: + with open('./dataflow/'+ 'dpt.m' , "r", encoding="utf8") as fdpt: with open(base_path + 'mapping/' + opt.outfile, "w") as fo: - with open(base_path + 'model/' + opt.model_file, "r") as fm: + with open(base_path + 'model/' + opt.model_file, "r", encoding="utf8") as fm: for line in fm: if(re.search("DSCONV",line)): dsconv = 1