X-Git-Url: https://git.teslayout.com/public/public/public/?a=blobdiff_plain;f=include%2Fanna%2Fcore%2Futil%2FTokenizer.hpp;h=829077577342ad817d4842f264fe3b4152fd3816;hb=refs%2Fheads%2Fdynamic_procedure_library;hp=b0fbada9f08127607913c55aef9001c9a24cbf9b;hpb=4e12ac57e93c052f716a6305ad8fc099c45899d1;p=anna.git

diff --git a/include/anna/core/util/Tokenizer.hpp b/include/anna/core/util/Tokenizer.hpp
index b0fbada..8290775 100644
--- a/include/anna/core/util/Tokenizer.hpp
+++ b/include/anna/core/util/Tokenizer.hpp
@@ -1,37 +1,9 @@
-// ANNA - Anna is Not 'N' Anymore
-//
-// (c) Copyright 2005-2014 Eduardo Ramos Testillano & Francisco Ruiz Rayo
-//
-// https://bitbucket.org/testillano/anna
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions
-// are met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Authors: eduardo.ramos.testillano@gmail.com
-//          cisco.tierra@gmail.com
+// ANNA - Anna is Not Nothingness Anymore                                                         //
+//                                                                                                //
+// (c) Copyright 2005-2015 Eduardo Ramos Testillano & Francisco Ruiz Rayo                         //
+//                                                                                                //
+// See project site at http://redmine.teslayout.com/projects/anna-suite                           //
+// See accompanying file LICENSE or copy at http://www.teslayout.com/projects/public/anna.LICENSE //
 
 
 #ifndef anna_core_util_Tokenizer_hpp
@@ -46,9 +18,12 @@ namespace anna {
 class RuntimeException;
 
 /**
-   Separa la cadena recibida en distintos elementos.
+   Tokenize the input string into several elements
 */
 class Tokenizer {
+
+  int _apply(const char* str, const char* separator) throw(RuntimeException);
+
 public:
   typedef char* const* const_iterator;
 
@@ -121,30 +96,28 @@ public:
   const char* operator [](const int i) const throw(RuntimeException) { return at(i); }
 
   /**
-     Aplica la separacion sobre la cadena str con el separador recibido como parametro.
+     Process the separation over the string str with the separator provided.
 
-     @param str Cadena sobre la que aplicar la separacion.
-     @param separator Caracteres que van a actuar como separador de las subcadenas contenidas en el
-     primer parametro.
+     Internally used strtok_r has these imitations: sequence of two or more contiguous delimiter
+     bytes in the parsed string is considered to be a single delimiter. Delimiter bytes at the start
+     or end of the string are ignored. Put another way: the tokens returned by strtok() are always
+     nonempty strings. To override these limitations, the string provided can be internally modified
+     inserting a artificial token in order to cheat on strtok_r. For this feature, you may provide
+     '<null>' or whatever string (non-empty) you prefer.
 
-     @return Numero de elementos obtenidos al aplicar la separacion.
+     @param str String to apply the separation.
+     @param separator Characters used as separator within the string tokenized
+     @param tokenizeContiguous If provided, it will be the artificial token used internally. The
+            resulting tokens shall store this string in case of contiguous separators. NULL by
+            default (original strtok_r behaviour).
 
+     @return Number of tokens
   */
-  int apply(const std::string& str, const char* separator) throw(RuntimeException) {
-    return apply(str.c_str(), separator);
+  int apply(const char* str, const char* separator, const char *tokenizeContiguous = NULL) throw(RuntimeException);
+  int apply(const std::string& str, const char* separator, const char *tokenizeContiguous = NULL) throw(RuntimeException) {
+    return apply(str.c_str(), separator, tokenizeContiguous);
   }
 
-  /**
-     Aplica la separacion sobre la cadena str con el separador recibido como parametro.
-
-     @param str Cadena sobre la que aplicar la separacion.
-     @param separator Caracteres que van a actuar como separador de las subcadenas contenidas en el
-     primer parametro.
-
-     @return Numero de elementos obtenidos al aplicar la separacion.
-
-  */
-  int apply(const char* str, const char* separator) throw(RuntimeException);
 
   // Metodos
   /**