X-Git-Url: https://git.teslayout.com/public/public/public/?a=blobdiff_plain;ds=sidebyside;f=source%2Fcore%2Futil%2FTokenizer.cpp;h=c51cf00ac87bd137a477e2899e45f7a21d394bd2;hb=5a6cba5fde2b2f538a7515f8293cc0a8d9589dfa;hp=b4f6d121a1aed251988179591bf0031e4577025e;hpb=93366a0bda79e6fd6e7dad6316bfcf8cc82f5731;p=anna.git

diff --git a/source/core/util/Tokenizer.cpp b/source/core/util/Tokenizer.cpp
index b4f6d12..c51cf00 100644
--- a/source/core/util/Tokenizer.cpp
+++ b/source/core/util/Tokenizer.cpp
@@ -10,12 +10,16 @@
 #include <anna/core/functions.hpp>
 #include <anna/config/defines.hpp>
 
+// temporary
+#include <iostream>
+
 using namespace std;
 using namespace anna;
 
 //static
 const int Tokenizer::MaxItem = 64;
 
+
 Tokenizer::Tokenizer() :
   a_dataBlock(true),
   a_activateStrip(false) {
@@ -53,12 +57,13 @@ Tokenizer::~Tokenizer() {
   delete [] a_items;
 }
 
-int Tokenizer::apply(const char* str, const char* separator)
-throw(RuntimeException) {
+int Tokenizer::_apply(const char* str, const char* separator)
+noexcept(false) {
+
   a_maxItem = 0;
 
-  if(str == NULL)
-    return 0;
+  //if(str == NULL)
+  //  return 0;
 
   DataBlock mb(str, anna_strlen(str) + 1, false);
   a_dataBlock = mb;
@@ -86,8 +91,34 @@ throw(RuntimeException) {
   return a_maxItem;
 }
 
+int Tokenizer::apply(const char *str, const char* separator, const char *tokenizeContiguous) noexcept(false) {
+
+  if(str == NULL)
+    return 0;
+
+  if (!separator)
+    throw RuntimeException("Cannot tokenize with a NULL separator", ANNA_FILE_LOCATION);
+
+  if (!tokenizeContiguous) return _apply(str, separator);
+
+  std::string _str = str;
+  std::string _sep = separator;
+  std::string _tok = tokenizeContiguous;
+  if (_sep == _tok)
+    throw RuntimeException("Using the separator as artifial token is a nonsense (original behaviour)", ANNA_FILE_LOCATION);
+  if (_tok == "")
+    throw RuntimeException("Use another artifial token. Empty is a nonsense (original behaviour)", ANNA_FILE_LOCATION);
+
+  std::string seps = _sep + _sep;
+  std::size_t pos, sepsL = seps.size();
+  std::string artificialToken = _sep + _tok + _sep;
+
+  while ((pos = _str.find(seps)) != std::string::npos) _str.replace(pos, sepsL, artificialToken); 
+  return _apply(_str.c_str(), separator);
+}
+
 const char* Tokenizer::at(const int i)
-throw(RuntimeException) {
+noexcept(false) {
   if(i >= a_maxItem)
     indexException(i, ANNA_FILE_LOCATION);
 
@@ -95,7 +126,7 @@ throw(RuntimeException) {
 }
 
 const char* Tokenizer::at(const int i) const
-throw(RuntimeException) {
+noexcept(false) {
   if(i >= a_maxItem)
     indexException(i, ANNA_FILE_LOCATION);
 
@@ -103,7 +134,7 @@ throw(RuntimeException) {
 }
 
 const char* Tokenizer::last() const
-throw(RuntimeException) {
+noexcept(false) {
   if(a_maxItem == 0)
     throw RuntimeException("There is any token to select", ANNA_FILE_LOCATION);
 
@@ -111,7 +142,7 @@ throw(RuntimeException) {
 }
 
 char* Tokenizer::strip(char* str)
-throw() {
+{
   char* result(str);
 
   if(str != NULL) {
@@ -132,7 +163,7 @@ throw() {
 }
 
 void Tokenizer::indexException(const int index, const char* fromFile, const int fromLine) const
-throw(RuntimeException) {
+noexcept(false) {
   string msg(functions::asString("Index %d out of range [0,%d] | Items: ", index, a_maxItem));
 
   for(const_iterator ii = begin(), maxii = end(); ii != maxii; ii ++) {