DPsim
CSVReader.cpp
1 /* Copyright 2017-2021 Institute for Automation of Complex Power Systems,
2  * EONERC, RWTH Aachen University
3  *
4  * This Source Code Form is subject to the terms of the Mozilla Public
5  * License, v. 2.0. If a copy of the MPL was not distributed with this
6  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
7  *********************************************************************************/
8 
9 #include <dpsim-models/CSVReader.h>
10 
11 using namespace CPS;
12 
13 MatrixRow CSVReader::csv2Eigen(const String &path) {
14  std::ifstream inputFile;
15  inputFile.open(path);
16  String line;
17  std::vector<double> values;
18  UInt rows = 0;
19  while (std::getline(inputFile, line)) {
20  std::stringstream lineStream(line);
21  String cell;
22  while (std::getline(lineStream, cell, ',')) {
23  values.push_back(std::stod(cell));
24  }
25  ++rows;
26  }
27  UInt columns = values.size() / rows;
28  return Eigen::Map<const MatrixRow>(values.data(), rows, columns);
29 }
30 
31 void CSVRow::readNextRow(std::istream &str) {
32  std::string line;
33  std::getline(str, line);
34 
35  std::stringstream lineStream(line);
36  std::string cell;
37 
38  m_data.clear();
39  while (std::getline(lineStream >> std::ws, cell, ',')) {
40  m_data.push_back(cell);
41  }
42  // This checks for a trailing comma with no data after it.
43  if (!lineStream && cell.empty()) {
44  // If there was a trailing comma then add an empty element.
45  m_data.push_back("");
46  }
47 }
48 
49 int CSVRow::size() const { return m_data.size(); }
50 
51 CSVReaderIterator &CSVReaderIterator::next() {
52  if (m_str) {
53  m_row.readNextRow(*m_str);
54  if (!(*m_str)) {
55  m_str = NULL;
56  }
57  }
58  return *this;
59 }
60 
61 CSVReaderIterator CSVReaderIterator::next(int) {
62  CSVReaderIterator tmp(*this);
63  this->next();
64  return tmp;
65 }
66 
67 CSVReaderIterator &CSVReaderIterator::step(int time_step) {
68  while (time_step != 0) {
69  if (m_str) {
70  m_row.readNextRow(*m_str);
71  if (!(*m_str)) {
72  m_str = NULL;
73  }
74  }
75  time_step -= 1;
76  }
77  return *this;
78 }
79 
80 CSVReaderIterator::CSVReaderIterator(std::istream &str)
81  : m_str(str.good() ? &str : NULL) {
82  this->next();
83 }
84 
85 CSVReaderIterator::CSVReaderIterator() : m_str(NULL) {}
86 
87 CSVReader::CSVReader(CPS::String name, std::list<fs::path> paths,
88  CPS::Logger::Level logLevel) {
89  mSLog = Logger::get(name + "_csvReader", logLevel);
90  //mFileList = paths;
91  for (auto file : paths) {
92  if (file.string().find(".csv") != std::string::npos) {
93  mFileList.push_back(file);
94  std::cout << "add " << file << std::endl;
95  }
96  }
97 }
98 
99 CSVReader::CSVReader(CPS::String name, CPS::String path,
100  CPS::Logger::Level logLevel) {
101  mSLog = Logger::get(name + "_csvReader", logLevel);
102 
103  mPath = path;
104  for (const auto &entry : fs::directory_iterator(path)) {
105  mFileList.push_back(entry.path());
106  }
107 }
108 
109 CSVReader::CSVReader(CPS::String name, CPS::String path,
110  std::map<String, String> &assignList,
111  CPS::Logger::Level logLevel)
112  : CSVReader(name, path, logLevel) {
113 
114  mAssignPattern = assignList;
115 }
116 
117 CSVReader::CSVReader(CPS::String name, std::list<fs::path> paths,
118  std::map<String, String> &assignList,
119  CPS::Logger::Level logLevel)
120  : CSVReader(name, paths, logLevel) {
121  mAssignPattern = assignList;
122 }
123 
124 CPS::Real CSVReader::time_format_convert(const CPS::String &time) {
125  int hh, mm, ss = 0;
126  CPS::Real secs = 0;
127  if (sscanf(time.c_str(), "%d:%d:%d", &hh, &mm, &ss) >= 2) {
128  secs = hh * 3600 + mm * 60 + ss;
129  }
130  return secs;
131 }
132 
133 std::vector<PQData> CSVReader::readLoadProfileDP(fs::path file, Real start_time,
134  Real time_step, Real end_time,
135  Real scale_factor,
136  CSVReader::DataFormat format) {
137 
138  std::vector<PQData> load_profileDP;
139  std::ifstream csvfile(file);
140  CSVReaderIterator row_(csvfile);
141 
142  // ignore the first row if it is a title
143  if (mSkipFirstRow && !std::isdigit((*row_).get(0)[0])) {
144  row_.next();
145  }
146 
147  /*
148  loop over rows of the csv file to find the entry point to read in.
149  if start_time and end_time are negative (as default), it reads in all rows.
150  */
151  Real presentTime = 0;
152  for (; row_ != CSVReaderIterator(); row_.next()) {
153  if ((start_time < 0) | (Int(presentTime) + 1 > Int(start_time)))
154  break;
155  presentTime++;
156  }
157  /*
158  reading data after entry point until end_time is reached
159  */
160  for (; row_ != CSVReaderIterator(); row_.next()) {
161  // IMPORTANT: take care of units. assume kW
162  PQData pq;
163  // multiplied by 1000 due to unit conversion (kw to w)
164  pq.p = std::stod((*row_).get(1)) * 1000 * scale_factor;
165  pq.q = std::stod((*row_).get(2)) * 1000 * scale_factor;
166  load_profileDP.push_back(pq);
167  if (end_time > 0 && presentTime > end_time)
168  break;
169  /*while (presentTime + time_step < Int(presentTime) + 1)
170  {
171  presentTime += time_step;
172  p_data.push_back(p_data.back());
173  }*/
174  presentTime = Int(presentTime) + 1;
175  }
176  std::cout << "CSV loaded." << std::endl;
177  return load_profileDP;
178 }
179 
180 // void CSVReader::assignLoadProfileSP(std::vector<std::shared_ptr<CPS::SP::Ph1::AvVoltageSourceInverterDQ>>& loads,
181 // Real start_time, Real time_step, Real end_time, Real scale_factor,
182 // CSVReader::Mode mode, CSVReader::DataFormat format) {
183 
184 // switch (mode) {
185 // case CSVReader::Mode::AUTO: {
186 // for (auto load : loads) {
187 // if(!load->isLoad())
188 // continue;
189 // String load_name = load->name();
190 // for (auto file : mFileList) {
191 // String file_name = file.filename().string();
192 // /// changing file name and load name to upper case for later matching
193 // for (auto & c : load_name) c = toupper(c);
194 // for (auto & c : file_name) c = toupper(c);
195 // /// strip off all non-alphanumeric characters
196 // load_name.erase(remove_if(load_name.begin(), load_name.end(), [](char c) { return !isalnum(c); }), load_name.end());
197 // file_name.erase(remove_if(file_name.begin(), file_name.end(), [](char c) { return !isalnum(c); }), file_name.end());
198 // if (std::stoi(file_name) == std::stoi(load_name)) {
199 // load->mLoadProfile = readLoadProfileDP(file, start_time, time_step, end_time, scale_factor, format);
200 // SPDLOG_LOGGER_INFO(mSLog, "Assigned {} to {}", file.filename().string(), load->name());
201 // }
202 // }
203 // }
204 
205 // break;
206 // }
207 // case CSVReader::Mode::MANUAL: {
208 // Int LP_assigned_counter = 0;
209 // Int LP_not_assigned_counter = 0;
210 // SPDLOG_LOGGER_INFO(mSLog, "Assigning load profiles with user defined pattern ...");
211 // for (auto load : loads) {
212 // std::map<String, String>::iterator file = mAssignPattern.find(load->name());
213 // if (file == mAssignPattern.end()) {
214 
215 // SPDLOG_LOGGER_INFO(mSLog, "{} has no profile given.", load->name());
216 // LP_not_assigned_counter++;
217 // continue;
218 // }
219 // for(auto path: mFileList){
220 // if(path.string().find(file->second)!= std::string::npos){
221 // load->mLoadProfile = readLoadProfileDP(path, start_time, time_step, end_time, scale_factor);
222 // SPDLOG_LOGGER_INFO(mSLog, "Assigned {}.csv to {}", file->second, load->name());
223 // LP_assigned_counter++;
224 // }
225 
226 // }
227 // }
228 // SPDLOG_LOGGER_INFO(mSLog, "Assigned profiles for {} loads, {} not assigned.", LP_assigned_counter, LP_not_assigned_counter);
229 // break;
230 // }
231 // default: {
232 // throw std::invalid_argument(
233 // "Load profile assign mode error");
234 // break;
235 // }
236 // }
237 // }
238 
239 // void CSVReader::assignLoadProfilePF(std::vector<std::shared_ptr<CPS::SP::Ph1::AvVoltageSourceInverterDQ>>& loads,
240 // Real start_time, Real time_step, Real end_time, Real scale_factor,
241 // CSVReader::Mode mode, CSVReader::DataFormat format) {
242 
243 // switch (mode) {
244 // case CSVReader::Mode::AUTO: {
245 // for (auto load : loads) {
246 // if(!load->isLoad())
247 // continue;
248 // String load_name = load->name();
249 // for (auto file : mFileList) {
250 // String file_name = file.filename().string();
251 // /// changing file name and load name to upper case for later matching
252 // for (auto & c : load_name) c = toupper(c);
253 // for (auto & c : file_name) c = toupper(c);
254 // /// strip off all non-alphanumeric characters
255 // load_name.erase(remove_if(load_name.begin(), load_name.end(), [](char c) { return !isalnum(c); }), load_name.end());
256 // file_name.erase(remove_if(file_name.begin(), file_name.end(), [](char c) { return !isalnum(c); }), file_name.end());
257 // if (std::stoi(file_name) == std::stoi(load_name)) {
258 // load->mPFAvVoltageSourceInverter->mLoadProfile = readLoadProfile(fs::path(mPath + file->second + ".csv"), start_time, time_step, end_time);
259 // load->mPFAvVoltageSourceInverter->use_profile = true;
260 // SPDLOG_LOGGER_INFO(mSLog, "Assigned {} to {}", file.filename().string(), load->name());
261 // }
262 // }
263 // }
264 
265 // break;
266 // }
267 // case CSVReader::Mode::MANUAL: {
268 // Int LP_assigned_counter = 0;
269 // Int LP_not_assigned_counter = 0;
270 // SPDLOG_LOGGER_INFO(mSLog, "Assigning load profiles with user defined pattern ...");
271 // for (auto load : loads) {
272 // std::map<String, String>::iterator file = mAssignPattern.find(load->name());
273 // if (file == mAssignPattern.end()) {
274 
275 // SPDLOG_LOGGER_INFO(mSLog, "{} has no profile given.", load->name());
276 // LP_not_assigned_counter++;
277 // continue;
278 // }
279 // for(auto path: mFileList){
280 // if(path.string().find(file->second)!= std::string::npos){
281 // load->mPFAvVoltageSourceInverter->mLoadProfile = readLoadProfile(fs::path(mPath + file->second + ".csv"), start_time, time_step, end_time);
282 // load->mPFAvVoltageSourceInverter->use_profile = true;
283 // SPDLOG_LOGGER_INFO(mSLog, "Assigned {}.csv to {}", file->second, load->name());
284 // LP_assigned_counter++;
285 // }
286 
287 // }
288 // }
289 // SPDLOG_LOGGER_INFO(mSLog, "Assigned profiles for {} loads, {} not assigned.", LP_assigned_counter, LP_not_assigned_counter);
290 // break;
291 // }
292 // default: {
293 // throw std::invalid_argument(
294 // "Load profile assign mode error");
295 // break;
296 // }
297 // }
298 // }
299 
300 // TODO: profile handling currently not supported by average inverter in DP1ph
301 // void CSVReader::assignLoadProfileDP(std::vector<std::shared_ptr<CPS::DP::Ph1::AvVoltageSourceInverterDQ>>& loads,
302 // Real start_time, Real time_step, Real end_time, Real scale_factor,
303 // CSVReader::Mode mode, CSVReader::DataFormat format) {
304 
305 // switch (mode) {
306 // case CSVReader::Mode::AUTO: {
307 // for (auto load : loads) {
308 // if(!load->isLoad())
309 // continue;
310 // String load_name = load->name();
311 // for (auto file : mFileList) {
312 // String file_name = file.filename().string();
313 // /// changing file name and load name to upper case for later matching
314 // for (auto & c : load_name) c = toupper(c);
315 // for (auto & c : file_name) c = toupper(c);
316 // /// strip off all non-alphanumeric characters
317 // load_name.erase(remove_if(load_name.begin(), load_name.end(), [](char c) { return !isalnum(c); }), load_name.end());
318 // file_name.erase(remove_if(file_name.begin(), file_name.end(), [](char c) { return !isalnum(c); }), file_name.end());
319 // if (std::stoi(file_name) == std::stoi(load_name)) {
320 // load->mLoadProfile = readLoadProfileDP(file, start_time, time_step, end_time, scale_factor, format);
321 // SPDLOG_LOGGER_INFO(mSLog, "Assigned {} to {}", file.filename().string(), load->name());
322 // }
323 // }
324 // }
325 
326 // break;
327 // }
328 // case CSVReader::Mode::MANUAL: {
329 // Int LP_assigned_counter = 0;
330 // Int LP_not_assigned_counter = 0;
331 // SPDLOG_LOGGER_INFO(mSLog, "Assigning load profiles with user defined pattern ...");
332 // for (auto load : loads) {
333 // std::map<String, String>::iterator file = mAssignPattern.find(load->name());
334 // if (file == mAssignPattern.end()) {
335 
336 // SPDLOG_LOGGER_INFO(mSLog, "{} has no profile given.", load->name());
337 // LP_not_assigned_counter++;
338 // continue;
339 // }
340 // for(auto path: mFileList){
341 // if(path.string().find(file->second)!= std::string::npos){
342 // load->mLoadProfile = readLoadProfileDP(path, start_time, time_step, end_time, scale_factor);
343 // SPDLOG_LOGGER_INFO(mSLog, "Assigned {}.csv to {}", file->second, load->name());
344 // LP_assigned_counter++;
345 // }
346 
347 // }
348 // }
349 // SPDLOG_LOGGER_INFO(mSLog, "Assigned profiles for {} loads, {} not assigned.", LP_assigned_counter, LP_not_assigned_counter);
350 // break;
351 // }
352 // default: {
353 // throw std::invalid_argument(
354 // "Load profile assign mode error");
355 // break;
356 // }
357 // }
358 // }
359 
360 PowerProfile CSVReader::readLoadProfile(fs::path file, Real start_time,
361  Real time_step, Real end_time,
362  CSVReader::DataFormat format) {
363 
364  PowerProfile load_profile;
365  std::ifstream csvfile(file);
366  bool need_that_conversion = (format == DataFormat::HHMMSS) ? true : false;
367  bool data_with_weighting_factor = false;
368 
369  CSVReaderIterator loop(csvfile);
370 
371  // ignore the first row if it is a title
372  if (mSkipFirstRow && !std::isdigit((*loop).get(0)[0])) {
373  loop.next();
374  }
375 
376  /*
377  loop over rows of the csv file to find the entry point to read in.
378  and determine data type prior to read in. (assuming only time,p,q or time,weighting factor)
379  if start_time and end_time are negative (as default), it reads in all rows.
380  */
381  for (; loop != CSVReaderIterator(); loop.next()) {
382  CSVReaderIterator nextRow = loop;
383  nextRow.next();
384  CPS::Real nextTime = (need_that_conversion)
385  ? time_format_convert((*nextRow).get(0))
386  : std::stod((*nextRow).get(0));
387  if ((*nextRow).size() == 2) {
388  data_with_weighting_factor = true;
389  }
390  if ((start_time < 0) | (nextTime >= Int(start_time))) {
391  break;
392  }
393  }
394  /*
395  reading data after entry point until end_time is reached
396  */
397  for (; loop != CSVReaderIterator(); loop.next()) {
398  CPS::Real currentTime = (need_that_conversion)
399  ? time_format_convert((*loop).get(0))
400  : std::stod((*loop).get(0));
401  if (data_with_weighting_factor) {
402  Real wf = std::stod((*loop).get(1));
403  load_profile.weightingFactors.insert(
404  std::pair<Real, Real>(currentTime, wf));
405  } else {
406  PQData pq;
407  // multiplied by 1000 due to unit conversion (kw to w)
408  pq.p = std::stod((*loop).get(1)) * 1000;
409  pq.q = std::stod((*loop).get(2)) * 1000;
410  load_profile.pqData.insert(std::pair<Real, PQData>(currentTime, pq));
411  }
412 
413  if (end_time > 0 && currentTime > end_time)
414  break;
415  }
416  std::vector<CPS::Real> times;
417  for (CPS::Real x_ = start_time; x_ <= end_time; x_ += time_step) {
418  times.push_back(x_);
419  }
420 
421  for (auto x : times) {
422  if (load_profile.pqData.find(x) == load_profile.pqData.end()) {
423  if (data_with_weighting_factor) {
424  Real y = interpol_linear(load_profile.weightingFactors, x);
425  load_profile.weightingFactors.insert(std::pair<Real, Real>(x, y));
426  } else {
427  PQData y = interpol_linear(load_profile.pqData, x);
428  load_profile.pqData.insert(std::pair<Real, PQData>(x, y));
429  }
430  }
431  }
432 
433  return load_profile;
434 }
435 
436 // can only read one file for now
437 std::vector<Real> CSVReader::readPQData(fs::path file, Real start_time,
438  Real time_step, Real end_time,
439  CSVReader::DataFormat format) {
440 
441  std::vector<Real> p_data;
442  std::ifstream csvfile(file);
443  CSVReaderIterator row_(csvfile);
444 
445  // ignore the first row if it is a title
446  if (mSkipFirstRow && !std::isdigit((*row_).get(0)[0])) {
447  row_.next();
448  }
449 
450  /*
451  loop over rows of the csv file to find the entry point to read in.
452  if start_time and end_time are negative (as default), it reads in all rows.
453  */
454  Real presentTime = 0;
455  for (; row_ != CSVReaderIterator(); row_.next()) {
456  if ((start_time < 0) | (Int(presentTime) + 1 > Int(start_time)))
457  break;
458  presentTime++;
459  }
460  /*
461  reading data after entry point until end_time is reached
462  */
463  for (; row_ != CSVReaderIterator(); row_.next()) {
464  // IMPORTANT: take care of units. assume kW
465  p_data.push_back(std::stod((*row_).get(0)) * 1000);
466  if (end_time > 0 && presentTime > end_time)
467  break;
468  /*while (presentTime + time_step < Int(presentTime) + 1)
469  {
470  presentTime += time_step;
471  p_data.push_back(p_data.back());
472  }*/
473  presentTime = Int(presentTime) + 1;
474  }
475  std::cout << "CSV loaded." << std::endl;
476  return p_data;
477 }
478 
480  Real time_step, Real end_time,
481  CSVReader::Mode mode,
482  CSVReader::DataFormat format) {
483 
484  switch (mode) {
485  case CSVReader::Mode::AUTO: {
486  for (auto obj : sys.mComponents) {
487  if (std::shared_ptr<CPS::SP::Ph1::Load> load =
488  std::dynamic_pointer_cast<CPS::SP::Ph1::Load>(obj)) {
489  SPDLOG_LOGGER_INFO(mSLog,
490  "Comparing csv file names with load mRIDs ...");
491  String load_name = load->name();
492  for (auto file : mFileList) {
493  String file_name = file.filename().string();
495  for (auto &c : load_name)
496  c = toupper(c);
497  for (auto &c : file_name)
498  c = toupper(c);
500  load_name.erase(remove_if(load_name.begin(), load_name.end(),
501  [](char c) { return !isalnum(c); }),
502  load_name.end());
503  file_name.erase(remove_if(file_name.begin(), file_name.end(),
504  [](char c) { return !isalnum(c); }),
505  file_name.end());
506  if (std::string(file_name.begin(), file_name.end() - 3)
507  .compare(load_name) == 0) {
508  load->mLoadProfile =
509  readLoadProfile(file, start_time, time_step, end_time, format);
510  load->use_profile = true;
511  SPDLOG_LOGGER_INFO(mSLog, "Assigned {} to {}",
512  file.filename().string(), load->name());
513  }
514  }
515  }
516  }
517  break;
518  }
519  case CSVReader::Mode::MANUAL: {
520  Int LP_assigned_counter = 0;
521  Int LP_not_assigned_counter = 0;
522  SPDLOG_LOGGER_INFO(mSLog,
523  "Assigning load profiles with user defined pattern ...");
524  for (auto obj : sys.mComponents) {
525  if (std::shared_ptr<CPS::SP::Ph1::Load> load =
526  std::dynamic_pointer_cast<CPS::SP::Ph1::Load>(obj)) {
527  std::map<String, String>::iterator file =
528  mAssignPattern.find(load->name());
529  if (file == mAssignPattern.end()) {
530  std::cout << load->name() << " has no profile given." << std::endl;
531  SPDLOG_LOGGER_INFO(mSLog, "{} has no profile given.", load->name());
532  LP_not_assigned_counter++;
533  continue;
534  }
535  load->mLoadProfile =
536  readLoadProfile(fs::path(mPath + file->second + ".csv"), start_time,
537  time_step, end_time);
538  load->use_profile = true;
539  std::cout << " Assigned " << file->second << " to " << load->name()
540  << std::endl;
541  SPDLOG_LOGGER_INFO(mSLog, "Assigned {}.csv to {}", file->second,
542  load->name());
543  LP_assigned_counter++;
544  }
545  }
546  SPDLOG_LOGGER_INFO(mSLog,
547  "Assigned profiles for {} loads, {} not assigned.",
548  LP_assigned_counter, LP_not_assigned_counter);
549  break;
550  }
551  default: {
552  throw std::invalid_argument("Load profile assign mode error");
553  break;
554  }
555  }
556 }
557 
558 CPS::PQData CSVReader::interpol_linear(std::map<CPS::Real, CPS::PQData> &pqData,
559  CPS::Real x) {
560  std::map<Real, PQData>::const_iterator entry = pqData.upper_bound(x);
561  PQData y;
562 
563  if (entry == pqData.end()) {
564  return (--entry)->second;
565  }
566  if (entry == pqData.begin()) {
567  return entry->second;
568  }
569  std::map<Real, PQData>::const_iterator prev = entry;
570  --prev;
571 
572  const CPS::Real delta = (x - prev->first) / (entry->first - prev->first);
573 
574  y.p = delta * entry->second.p + (1 - delta) * prev->second.p;
575  y.q = delta * entry->second.q + (1 - delta) * prev->second.q;
576  return y;
577 }
578 
579 CPS::Real
580 CSVReader::interpol_linear(std::map<CPS::Real, CPS::Real> &weightingFactors,
581  CPS::Real x) {
582  std::map<Real, Real>::const_iterator entry = weightingFactors.upper_bound(x);
583  CPS::Real y;
584 
585  if (entry == weightingFactors.end()) {
586  return (--entry)->second;
587  }
588  if (entry == weightingFactors.begin()) {
589  return entry->second;
590  }
591  std::map<Real, Real>::const_iterator prev = entry;
592  --prev;
593 
594  const CPS::Real delta = (x - prev->first) / (entry->first - prev->first);
595 
596  y = delta * entry->second + (1 - delta) * prev->second;
597  return y;
598 }
reads load profiles (csv files only) and assign them to the corresponding load object
Definition: CSVReader.h:28
PowerProfile readLoadProfile(fs::path file, Real start_time=-1, Real time_step=1, Real end_time=-1, CSVReader::DataFormat format=CSVReader::DataFormat::SECONDS)
Definition: CSVReader.cpp:360
PQData interpol_linear(std::map< Real, PQData > &data_PQ, Real x)
interpolation for PQ data points
void assignLoadProfile(SystemTopology &sys, Real start_time=-1, Real time_step=1, Real end_time=-1, CSVReader::Mode mode=CSVReader::Mode::AUTO, CSVReader::DataFormat format=CSVReader::DataFormat::SECONDS)
assign load profile to corresponding load object
Definition: CSVReader.cpp:479
Real time_format_convert(const String &time)
Definition: CSVReader.cpp:124
IdentifiedObject::List mComponents
List of network components.