1 // ANNA - Anna is Not Nothingness Anymore //
3 // (c) Copyright 2005-2015 Eduardo Ramos Testillano & Francisco Ruiz Rayo //
5 // See project site at http://redmine.teslayout.com/projects/anna-suite //
6 // See accompanying file LICENSE or copy at http://www.teslayout.com/projects/public/anna.LICENSE //
9 #include <anna/core/util/Tokenizer.hpp>
10 #include <anna/core/functions.hpp>
11 #include <anna/config/defines.hpp>
20 const int Tokenizer::MaxItem = 64;
23 Tokenizer::Tokenizer() :
25 a_activateStrip(false) {
26 a_items = new char* [MaxItem];
27 anna_memset(a_items, 0, sizeof(char*) * MaxItem);
31 Tokenizer::Tokenizer(const char* str, const char* separator) :
33 a_activateStrip(false) {
34 a_items = new char* [MaxItem];
35 anna_memset(a_items, 0, sizeof(char*) * MaxItem);
39 apply(str, separator);
44 Tokenizer::Tokenizer(const string& str, const char* separator) :
46 a_activateStrip(false) {
47 a_items = new char* [MaxItem];
48 anna_memset(a_items, 0, sizeof(char*) * MaxItem);
51 apply(str.c_str(), separator);
56 Tokenizer::~Tokenizer() {
60 int Tokenizer::_apply(const char* str, const char* separator)
68 DataBlock mb(str, anna_strlen(str) + 1, false);
70 const char* buffer = a_dataBlock.getData();
71 char *token = const_cast <char*>(buffer);
74 while((token = strtok_r(token, separator, &last)) != NULL) {
75 if(a_activateStrip == true)
78 a_items [a_maxItem ++] = token;
81 if(a_maxItem == MaxItem) {
82 string msg("Tokenizer::apply | String: ");
84 msg += " | Separator: ";
86 msg += " | Tokenizer has not enough space";
87 throw RuntimeException(msg, ANNA_FILE_LOCATION);
94 int Tokenizer::apply(const char *str, const char* separator, const char *tokenizeContiguous) noexcept(false) {
100 throw RuntimeException("Cannot tokenize with a NULL separator", ANNA_FILE_LOCATION);
102 if (!tokenizeContiguous) return _apply(str, separator);
104 std::string _str = str;
105 std::string _sep = separator;
106 std::string _tok = tokenizeContiguous;
108 throw RuntimeException("Using the separator as artifial token is a nonsense (original behaviour)", ANNA_FILE_LOCATION);
110 throw RuntimeException("Use another artifial token. Empty is a nonsense (original behaviour)", ANNA_FILE_LOCATION);
112 std::string seps = _sep + _sep;
113 std::size_t pos, sepsL = seps.size();
114 std::string artificialToken = _sep + _tok + _sep;
116 while ((pos = _str.find(seps)) != std::string::npos) _str.replace(pos, sepsL, artificialToken);
117 return _apply(_str.c_str(), separator);
120 const char* Tokenizer::at(const int i)
123 indexException(i, ANNA_FILE_LOCATION);
125 return data(begin() + i);
128 const char* Tokenizer::at(const int i) const
131 indexException(i, ANNA_FILE_LOCATION);
133 return data(begin() + i);
136 const char* Tokenizer::last() const
139 throw RuntimeException("There is any token to select", ANNA_FILE_LOCATION);
141 return data(begin() + a_maxItem - 1);
144 char* Tokenizer::strip(char* str)
149 while(*result != 0 && *result == ' ')
153 char* final = result + anna_strlen(result) - 1;
155 while(final > result && *final == ' ') {
165 void Tokenizer::indexException(const int index, const char* fromFile, const int fromLine) const
167 string msg(functions::asString("Index %d out of range [0,%d] | Items: ", index, a_maxItem));
169 for(const_iterator ii = begin(), maxii = end(); ii != maxii; ii ++) {
174 throw RuntimeException(msg, fromFile, fromLine);