X-Git-Url: https://git.teslayout.com/public/public/public/?a=blobdiff_plain;f=source%2Fcore%2Futil%2FTokenizer.cpp;h=d49f6edd2069a5798e6ecefcc387d1e1299113b7;hb=af9c86ffb0e28d35ad94d99c5f77e41578c972b4;hp=663c60fa4a26b6283a31ebdfa5e033ccd589f603;hpb=39033fd99e58e994a5e98c1060dcc79e0d81f9c9;p=anna.git

diff --git a/source/core/util/Tokenizer.cpp b/source/core/util/Tokenizer.cpp
index 663c60f..d49f6ed 100644
--- a/source/core/util/Tokenizer.cpp
+++ b/source/core/util/Tokenizer.cpp
@@ -1,49 +1,25 @@
-// ANNA - Anna is Not Nothingness Anymore
-//
-// (c) Copyright 2005-2014 Eduardo Ramos Testillano & Francisco Ruiz Rayo
-//
-// http://redmine.teslayout.com/projects/anna-suite
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions
-// are met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     *  Neither the name of the copyright holder nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Authors: eduardo.ramos.testillano@gmail.com
-//          cisco.tierra@gmail.com
+// ANNA - Anna is Not Nothingness Anymore                                                         //
+//                                                                                                //
+// (c) Copyright 2005-2015 Eduardo Ramos Testillano & Francisco Ruiz Rayo                         //
+//                                                                                                //
+// See project site at http://redmine.teslayout.com/projects/anna-suite                           //
+// See accompanying file LICENSE or copy at http://www.teslayout.com/projects/public/anna.LICENSE //
 
 
 #include <anna/core/util/Tokenizer.hpp>
 #include <anna/core/functions.hpp>
 #include <anna/config/defines.hpp>
 
+// temporary
+#include <iostream>
+
 using namespace std;
 using namespace anna;
 
 //static
 const int Tokenizer::MaxItem = 64;
 
+
 Tokenizer::Tokenizer() :
   a_dataBlock(true),
   a_activateStrip(false) {
@@ -81,12 +57,13 @@ Tokenizer::~Tokenizer() {
   delete [] a_items;
 }
 
-int Tokenizer::apply(const char* str, const char* separator)
+int Tokenizer::_apply(const char* str, const char* separator)
 throw(RuntimeException) {
+
   a_maxItem = 0;
 
-  if(str == NULL)
-    return 0;
+  //if(str == NULL)
+  //  return 0;
 
   DataBlock mb(str, anna_strlen(str) + 1, false);
   a_dataBlock = mb;
@@ -114,6 +91,32 @@ throw(RuntimeException) {
   return a_maxItem;
 }
 
+int Tokenizer::apply(const char *str, const char* separator, const char *tokenizeContiguous) throw(RuntimeException) {
+
+  if(str == NULL)
+    return 0;
+
+  if (!separator)
+    throw RuntimeException("Cannot tokenize with a NULL separator", ANNA_FILE_LOCATION);
+
+  if (!tokenizeContiguous) return _apply(str, separator);
+
+  std::string _str = str;
+  std::string _sep = separator;
+  std::string _tok = tokenizeContiguous;
+  if (_sep == _tok)
+    throw RuntimeException("Using the separator as artifial token is a nonsense (original behaviour)", ANNA_FILE_LOCATION);
+  if (_tok == "")
+    throw RuntimeException("Use another artifial token. Empty is a nonsense (original behaviour)", ANNA_FILE_LOCATION);
+
+  std::string seps = _sep + _sep;
+  std::size_t pos, sepsL = seps.size();
+  std::string artificialToken = _sep + _tok + _sep;
+
+  while ((pos = _str.find(seps)) != std::string::npos) _str.replace(pos, sepsL, artificialToken); 
+  return _apply(_str.c_str(), separator);
+}
+
 const char* Tokenizer::at(const int i)
 throw(RuntimeException) {
   if(i >= a_maxItem)