From 415985b3f67878c2e3cee785a0b1cb36f4eff901 Mon Sep 17 00:00:00 2001 From: "Eduardo Ramos Testillano (eramedu)" Date: Sun, 3 May 2020 02:15:48 +0200 Subject: [PATCH] Ensures normalization on waitfe/fc-xml operations Normalize the xml representation for loaded messages, and also the regular expression is adapted as usual, regarding hop-by-hop, end-to-end and Origin-State-Id. isLike is fixed to do there the full normalization, including the removing of newlines. isLike docygen documentation is also improved. Also, move getContentFromFile to core::functions. --- example/diameter/launcher/Launcher.cpp | 58 +++++---------- example/diameter/launcher/Launcher.hpp | 1 - include/anna/core/functions.hpp | 11 +++ include/anna/diameter/codec/Avp.hpp | 5 +- include/anna/diameter/codec/Message.hpp | 99 +++++++++++-------------- source/core/functions.cpp | 17 +++++ source/diameter/codec/Avp.cpp | 14 +++- source/diameter/codec/Message.cpp | 14 +++- source/testing/TestCondition.cpp | 12 +-- 9 files changed, 115 insertions(+), 116 deletions(-) diff --git a/example/diameter/launcher/Launcher.cpp b/example/diameter/launcher/Launcher.cpp index bb28699..60a9cb0 100644 --- a/example/diameter/launcher/Launcher.cpp +++ b/example/diameter/launcher/Launcher.cpp @@ -833,21 +833,6 @@ bool Launcher::getDataBlockFromHexFile(const std::string &pathfile, anna::DataBl return false; } -bool Launcher::getContentFromFile(const std::string &pathfile, std::string &content) const throw(anna::RuntimeException) { - - std::ifstream inFile(pathfile.c_str(), std::ifstream::in); - if(!inFile.good()) { - throw RuntimeException(anna::functions::asString("Unable to open file '%s'", pathfile.c_str()), ANNA_FILE_LOCATION); - } - - std::stringstream strStream; - strStream << inFile.rdbuf(); //read the file - content = strStream.str(); // holds the content of the file - inFile.close(); - - return true; -} - void Launcher::resetStatistics() throw() { if (a_workingNode) { a_workingNode->getCommEngine()->resetStatistics(); @@ -1777,49 +1762,42 @@ bool Launcher::eventOperation(const std::string &operation, std::string &respons throw anna::RuntimeException("Wrong body content format on HTTP Request. Check 'HELP.md' for more information.", ANNA_FILE_LOCATION); if(param3 == "") throw anna::RuntimeException(anna::functions::asString("Missing xml file for '%s' command in test id operation", param2.c_str()), ANNA_FILE_LOCATION); - // Get xml content from file: - std::string regexp; - if(!getContentFromFile(param3, regexp)) - throw anna::RuntimeException("Error reading xml content from file provided", ANNA_FILE_LOCATION); + codecMsg.loadXMLFile(param3); + std::string regexp = codecMsg.asXMLString(true /* normalization */); + + // Now we must insert regular expressions in hop-by-hop, end-to-end and Origin-State-Id: // optional 'full': if(param4 != "strict") { - - // TODO: use this from gcc4.9.0: http://stackoverflow.com/questions/8060025/is-this-c11-regex-error-me-or-the-compiler -/* - std::string s_from = "hop-by-hop-id=\"[0-9]+\" end-to-end-id=\"[0-9]+\""; - std::string s_to = s_from; - std::string s_from2 = "avp name=\"Origin-State-Id\" data=\"[0-9]+\""; - std::string s_to2 = s_from2; - - try { - regexp = std::regex_replace (regexp, std::regex(s_from), s_to); - regexp = std::regex_replace (regexp, std::regex(s_from2), s_to2); - } - catch (const std::regex_error& e) { - throw anna::RuntimeException(e.what(), ANNA_FILE_LOCATION); - } - -*/ std::string::size_type pos, pos_1, pos_2; - pos = regexp.find("hop-by-hop-id=", 0u); + pos = regexp.find("end-to-end-id=", 0u); pos = regexp.find("\"", pos); pos_1 = pos; pos = regexp.find("\"", pos+1); pos_2 = pos; regexp.replace(pos_1 + 1, pos_2 - pos_1 - 1, "[0-9]+"); - pos = regexp.find("end-to-end-id=", 0u); + pos = regexp.find("hop-by-hop-id=", 0u); pos = regexp.find("\"", pos); pos_1 = pos; pos = regexp.find("\"", pos+1); pos_2 = pos; regexp.replace(pos_1 + 1, pos_2 - pos_1 - 1, "[0-9]+"); + // For this representation: + //pos = regexp.find("Origin-State-Id", 0u); + //pos = regexp.find("\"", pos); + //pos = regexp.find("\"", pos+1); + //pos_1 = pos; + //pos = regexp.find("\"", pos+1); + //pos_2 = pos; + //regexp.replace(pos_1 + 1, pos_2 - pos_1 - 1, "[0-9]+"); + // But we have this one: pos = regexp.find("Origin-State-Id", 0u); - pos = regexp.find("\"", pos); - pos = regexp.find("\"", pos+1); + pos = regexp.rfind("\"", pos); + pos = regexp.rfind("\"", pos-1); + pos = regexp.rfind("\"", pos-1); pos_1 = pos; pos = regexp.find("\"", pos+1); pos_2 = pos; diff --git a/example/diameter/launcher/Launcher.hpp b/example/diameter/launcher/Launcher.hpp index 8a68888..399a0f8 100644 --- a/example/diameter/launcher/Launcher.hpp +++ b/example/diameter/launcher/Launcher.hpp @@ -119,7 +119,6 @@ public: // helpers bool getDataBlockFromHexFile(const std::string &pathfile, anna::DataBlock &db) const throw(anna::RuntimeException); - bool getContentFromFile(const std::string &pathfile, std::string &content) const throw(anna::RuntimeException); friend class TestManager; }; diff --git a/include/anna/core/functions.hpp b/include/anna/core/functions.hpp index 58b8cd8..78fe13b 100644 --- a/include/anna/core/functions.hpp +++ b/include/anna/core/functions.hpp @@ -1119,6 +1119,17 @@ struct functions { * @return Returns decoded representation */ static std::string decodeBase64(const std::string & encodedString); + + + /* + * Reads a file into passed string + * + * @param pathfile Path file to read + * @param content String where file content is dump + * + * @return success for read operation + */ + static bool getContentFromFile(const std::string &pathfile, std::string &content) throw(anna::RuntimeException); }; } diff --git a/include/anna/diameter/codec/Avp.hpp b/include/anna/diameter/codec/Avp.hpp index 5bffde7..2454d4d 100644 --- a/include/anna/diameter/codec/Avp.hpp +++ b/include/anna/diameter/codec/Avp.hpp @@ -712,11 +712,12 @@ public: /** Class xml string representation - @param sortAttributes Optional normalization used to match xml representation with regexps + @param normalize Optional normalization which sorts attribute names and removes + newlines in the xml representation in order to ease regexp matching. \return XML string representation with relevant information for this instance. */ - std::string asXMLString(bool sortAttributes = false) const throw(); + std::string asXMLString(bool normalize = false) const throw(); /** Comparison operator by mean serialization diff --git a/include/anna/diameter/codec/Message.hpp b/include/anna/diameter/codec/Message.hpp index 3a583ce..2ec338b 100644 --- a/include/anna/diameter/codec/Message.hpp +++ b/include/anna/diameter/codec/Message.hpp @@ -715,11 +715,12 @@ public: /** Class xml string representation - @param sortAttributes Optional normalization used to match xml representation with regexps + @param normalize Optional normalization which sorts attribute names and removes + newlines in the xml representation in order to ease regexp matching. \return XML string representation with relevant information for this instance. */ - std::string asXMLString(bool sortAttributes = false) const throw(); + std::string asXMLString(bool normalize = false) const throw(); /** Comparison operator by mean serialization @@ -732,69 +733,59 @@ public: friend bool operator == (const Message & m1, const Message & m2) throw() { return (m1.asXMLString() == m2.asXMLString()); } /** - Match a regular expression (string pattern) regarding xml string serialization for this message. - Using a complex pattern (many avps, grouped ones) it could be necessary to fix the message before - using the method in order to perform a more controlled comparison. In the same way, flags could be - ignored to simplify message xml presentation. - This powerful tool could be used to program traffic analysis and decide future behaviour (routing, - traslation, etc.). + Matchs a regular expression (string pattern) regarding xml string serialization for this message. + The message xml representation is internally normalized (attribute names are sort and newlines + are removed) in order to ease regexp matching. - 
-     Examples:
+     You could use simple regular expressions.
+     For example, the pattern '' detects
+     PS charging contexts because of data suffix specification '32251@3gpp.org' for that AVP.
+     The pattern '' detects PS charging contexts
-     because of data suffix specification '32251@3gpp.org' for that AVP.
+     It would seems strange or 'creative' to use regular expressions within an hex string representation,
+     but anyway you could also do such kind of things to check non-printable data parts within the message:
+     for example, the pattern ''
+     matchs IP addresses for '10.x.10.10' where x = [0..255].
 
-     The pattern '
-        
-        
+     
+     ...
+     
+        
+        
      '
+     ...
+     

 
-     detects MSISDN (not IMSI) equal to 606000106
+     You could also extract AVP xml normalized representation in this way:
 
-     It would seems strange or 'creative' to use regular expressions within an hex string representation,
-     but anyway you could also do such kind of things to check non-printable data parts within the message:
-     for example, the pattern ''
-     matchs IP addresses for '10.x.10.10' where x = [0..255].
+     
+     anna::diameter::codec::Message myMessage;
+     myMessage.loadXMLFile("message.xml");
+     std::string subscriptionId = myMessage.getAvp("Subscription-Id")->getAvp("Subscription-Id-Type")->asXMLString(true);
+     // Former is ''
+     

+
+     And then use to match incoming messages:
 
-     Note that string pattern could also be generated via #loadXMLFile/#loadXMLString and then #asXML, thus, you
-     could get patterns through xml files which act as conditional triggers over message. In that case,
-     it is not possible to specify regular expressions within xml 'hex-data' fields because parser will fail
-     during hexadecimal read. Normally only printable 'data' fields are used for matching issues.
-
-     For example, imagine a 'pattern.xml' file like:
-     
-        
-           
-           
-        
-     
-
-     Then you could do:
-
-     anna::diameter::codec::Message patternMessage;
-     patternMessage.loadXMLFile("pattern.xml");
-     std::string pattern = patternMessage.getAvp("Subscription-Id")->getAvp("Subscription-Id-Type")->asXMLString();
-     // Former is ''
-     bool match = incomingMessage.isLike(pattern);
-
-     Then, messages having MSISDN numbers starting with '616' will match the pattern.
-     Note, that any other message codes (and not only Credit-Control-Request ones), could pass the test...
-     You could also build that string manually:
-
-     Example 1:
-     std::string pattern = "\n";
-     pattern += std::string(ANNA_XML_INDENTATION_SPACES, ' '); pattern += "\n"
-     pattern += std::string(ANNA_XML_INDENTATION_SPACES, ' '); pattern += ""
-
-     Example 2:
-     std::string pattern = "name=\"Subscription-Id\"(.)*name=\"Subscription-Id-Type\" data=\"0\"(.)*name=\"Subscription-Id-Data\" data=\"616[0-9]{6,6}\"";
+     
+     bool match = incomingMessage.isLike(subscriptionId);
      

 
+     Using a complex pattern (many avps, grouped ones) is possible, indeed testing ADML engine supports 'waitfe/fc-xml'
+     operations which load entire diameter messages to be used as a whole regular expression (hop-by-hop, end-to-end and
+     Origin-State-Id avp is automatically replaced by '[0-9]+' to make possible the comparison).
+
+     Those operations makes all the work, but if you use the API, you may take into account:
+
+     - Respect indentation for inner Message xml representation (normally 3 spaces).
+     - Sort alphabetically the attribute names in every xml node.
+     - Remove all the newlines in the xml representation as normalization stage.
+     - Ignore flags and set the fix mode for the message.
+
      \return Returns the match result
   */
   bool isLike(const std::string &pattern) const throw();
diff --git a/source/core/functions.cpp b/source/core/functions.cpp
index 8a77994..217fd0a 100644
--- a/source/core/functions.cpp
+++ b/source/core/functions.cpp
@@ -30,6 +30,8 @@
 #include 
 
 #include 
+#include 
+#include 
 
 
 using namespace anna;
@@ -1612,3 +1614,18 @@ std::string functions::decodeBase64(const std::string & encodedString) {
   return ret;
 }
 
+bool functions::getContentFromFile(const std::string &pathfile, std::string &content) throw(anna::RuntimeException) {
+
+  std::ifstream inFile(pathfile.c_str(), std::ifstream::in);
+  if(!inFile.good()) {
+    throw RuntimeException(anna::functions::asString("Unable to open file '%s'", pathfile.c_str()), ANNA_FILE_LOCATION);
+  }
+
+  std::stringstream strStream;
+  strStream << inFile.rdbuf(); //read the file
+  content = strStream.str(); // holds the content of the file
+  inFile.close();
+
+  return true;
+}
+
diff --git a/source/diameter/codec/Avp.cpp b/source/diameter/codec/Avp.cpp
index f0477e2..2338111 100644
--- a/source/diameter/codec/Avp.cpp
+++ b/source/diameter/codec/Avp.cpp
@@ -1648,11 +1648,16 @@ anna::xml::Node* Avp::asXML(anna::xml::Node* parent) const throw() {
 //------------------------------------------------------------------------------
 //----------------------------------------------------------- Avp::asXMLString()
 //------------------------------------------------------------------------------
-std::string Avp::asXMLString(bool sortAttributes) const throw() {
+std::string Avp::asXMLString(bool normalize) const throw() {
   anna::xml::Node root("root");
 
-  anna::xml::Compiler::Mode::_v mode = sortAttributes ? anna::xml::Compiler::Mode::Sort : anna::xml::Compiler::Mode::Visual;
-  return anna::xml::Compiler().apply(asXML(&root), mode);
+  anna::xml::Compiler::Mode::_v mode = normalize ? anna::xml::Compiler::Mode::Sort : anna::xml::Compiler::Mode::Visual;
+  std::string result = anna::xml::Compiler().apply(asXML(&root), mode);
+
+  if (normalize)
+    result.erase(std::remove(result.begin(), result.end(), '\n'), result.end());
+
+  return result;
 }
 
 
@@ -1661,5 +1666,6 @@ std::string Avp::asXMLString(bool sortAttributes) const throw() {
 //------------------------------------------------------------------------------
 bool Avp::isLike(const std::string &pattern) const throw() {
   anna::RegularExpression re(pattern);
-  return re.isLike(asXMLString());
+  return re.isLike(asXMLString(true /* normalize by mean sorting attribute names and removing new lines */));
 }
+
diff --git a/source/diameter/codec/Message.cpp b/source/diameter/codec/Message.cpp
index 0cf70cd..a686c71 100644
--- a/source/diameter/codec/Message.cpp
+++ b/source/diameter/codec/Message.cpp
@@ -998,11 +998,16 @@ anna::xml::Node* Message::asXML(anna::xml::Node* parent) const throw() {
 //------------------------------------------------------------------------------
 //------------------------------------------------------- Message::asXMLString()
 //------------------------------------------------------------------------------
-std::string Message::asXMLString(bool sortAttributes) const throw() {
+std::string Message::asXMLString(bool normalize) const throw() {
   anna::xml::Node root("root");
 
-  anna::xml::Compiler::Mode::_v mode = sortAttributes ? anna::xml::Compiler::Mode::Sort : anna::xml::Compiler::Mode::Visual;
-  return anna::xml::Compiler().apply(asXML(&root), mode);
+  anna::xml::Compiler::Mode::_v mode = normalize ? anna::xml::Compiler::Mode::Sort : anna::xml::Compiler::Mode::Visual;
+  std::string result = anna::xml::Compiler().apply(asXML(&root), mode);
+
+  if (normalize)
+    result.erase(std::remove(result.begin(), result.end(), '\n'), result.end());
+
+  return result;
 }
 
 
@@ -1011,5 +1016,6 @@ std::string Message::asXMLString(bool sortAttributes) const throw() {
 //------------------------------------------------------------------------------
 bool Message::isLike(const std::string &pattern) const throw() {
   anna::RegularExpression re(pattern);
-  return re.isLike(asXMLString());
+  return re.isLike(asXMLString(true /* normalize by mean sorting attribute names and removing new lines */));
 }
+
diff --git a/source/testing/TestCondition.cpp b/source/testing/TestCondition.cpp
index 6f0bba0..416b893 100644
--- a/source/testing/TestCondition.cpp
+++ b/source/testing/TestCondition.cpp
@@ -45,17 +45,7 @@ bool TestDiameterCondition::comply(const anna::DataBlock &message) const throw()
   if (a_type == Type::RegexpXml) {
     anna::diameter::codec::Message codecMsg;
     try { codecMsg.decode(message); } catch (anna::RuntimeException &ex) { ex.trace(); }
-
-    //return codecMsg.isLike(getRegexp());
-    // We will remove LF from both sides to ease regexp management:
-    std::string regexp = getRegexp();
-    regexp.erase(std::remove(regexp.begin(), regexp.end(), '\n'), regexp.end());
-    anna::RegularExpression re(regexp);
-
-    std::string msgString = codecMsg.asXMLString();
-    msgString.erase(std::remove(msgString.begin(), msgString.end(), '\n'), msgString.end());
-
-    return re.isLike(msgString);
+    return codecMsg.isLike(getRegexp());
   }
   else if (a_type == Type::RegexpHex) {
     anna::RegularExpression re(getRegexp());
-- 
2.20.1