example/diameter/launcher/deployments/aots-setup/loader.py

   1 #!/usr/bin/env python
   2 # Anna Agents-Oriented Testing Setup Loader
   3
   4 # Requires PyYAML: pip install pyyaml
   5
   6 import os, sys, datetime
   7 from argparse import ArgumentParser
   8
   9 # PyYAML
  10 from yaml import load, dump
  11 try:
  12     from yaml import CLoader as Loader, CDumper as Dumper
  13 except ImportError:
  14     from yaml import Loader, Dumper
  15
  16
  17 def _exit(message = None, rc=0):
  18   if (message): printMsg(message)
  19   printMsg("Exiting [rc={}]".format(rc))
  20   sys.exit(rc)
  21
  22
  23 def timeFormat():
  24   return '%b.%d.%Y-%H.%M.%S'
  25
  26
  27 def printMsg(message):
  28   print ("[{}] {}".format(datetime.datetime.now().strftime(timeFormat()), message))
  29
  30
  31 class YamlConfigParser():
  32   """
  33   Yaml parser
  34   """
  35   def __init__(self, yaml_config_file):
  36     """
  37     Convert the yaml file into a Python object
  38     """
  39     self.data = None
  40     try:
  41       with open(yaml_config_file, 'r') as ss:
  42         self.data = load(ss, Loader=Loader)
  43         ss.close()
  44
  45     except Exception:
  46      raise
  47
  48   def getData(self):
  49     return self.data
  50
  51   def show(self, options):
  52     output = dump(self.data, Dumper=Dumper)
  53     print(output)
  54
  55
  56 def writeFileContent(filename, content):
  57   _file = open(filename, "w")
  58   _file.write(content)
  59   _file.close()
  60
  61
  62 def getNodeValue(dictionary, key):
  63
  64   value = dictionary[key]
  65   printMsg("  {}: {}".format(key, value))
  66   return value
  67
  68
  69 def configure_agent_adml(node, description, application_id, dictionary, _type, origin_host, origin_realm, address):
  70
  71   # Agent ADML configuration is basically generation of services:
  72
  73   if (_type == "client"):
  74     stacks='''  <stack id="{application_id}" dictionary="{dictionary}" fixMode="Always" ignoreFlagsOnValidation="yes"/>'''.format(application_id=application_id, dictionary=dictionary)
  75
  76     nodes='''  <node originHost="{origin_host}" applicationId="{application_id}" entity="{address}" cer="cer.{application_id}.xml" answersTimeout="300000" dumpLog="yes"/>'''.format(application_id=application_id, dictionary=dictionary, origin_host=origin_host, address=address)
  77
  78     ce_path = agents_ADML_dir + "/cer.{}.xml".format(application_id)
  79     ce_content='''<message version="1" name="CER" application-id="0">
  80    <avp name="Origin-Host" data="{origin_host}"/>
  81    <avp name="Origin-Realm" data="{origin_realm}"/>
  82    <avp name="Host-IP-Address" data="1|127.0.0.1"/>
  83    <avp name="Vendor-Id" data="10415"/>
  84    <avp name="Product-Name" data="{description}"/>
  85    <avp name="Auth-Application-Id" data="{application_id}"/>
  86 </message>'''.format(application_id=application_id, origin_host=origin_host, origin_realm=origin_realm, description=description)
  87
  88   elif (_type == "server"):
  89     stacks='''  <stack id="{application_id}" dictionary="{dictionary}" fixMode="Never"/>'''.format(application_id=application_id, dictionary=dictionary)
  90
  91     nodes='''  <node originHost="{origin_host}" applicationId="{application_id}" diameterServer="{address}" diameterServerSessions="10" cea="cea.{application_id}.xml" answersTimeout="300000" dumpLog="yes"/>'''.format(application_id=application_id, dictionary=dictionary, origin_host=origin_host, address=address)
  92
  93     ce_path = agents_ADML_dir + "/cea.{}.xml".format(application_id)
  94     ce_content='''<message version="1" name="CEA" application-id="0">
  95    <avp name="Result-Code" data="2001" alias="DIAMETER_SUCCESS"/>
  96    <avp name="Origin-Host" data="{origin_host}"/>
  97    <avp name="Origin-Realm" data="{origin_realm}"/>
  98    <avp name="Host-IP-Address" data="1|127.0.0.1"/>
  99    <avp name="Vendor-Id" data="10415"/>
 100    <avp name="Product-Name" data="{description}"/>
 101 </message>'''.format(origin_host=origin_host, origin_realm=origin_realm, description=description)
 102
 103   else:
 104     raise Exception("Invalid ADML type '{}' (allowed: client|server)".format(_type))
 105
 106   # Create cer/cea files:
 107   writeFileContent(ce_path, ce_content)
 108
 109
 110   return stacks, nodes
 111
 112
 113 def configure_agent_kafka(node, description, topic):
 114
 115   # Producer script:
 116   content = '''#!/bin/bash
 117 # Positional arguments: json, delay, debug indicator
 118 [ -z "$3" ] && echo "Usage: $0 <json file> <delay in milliseconds: 0 is no delay> <debug indicator: yes|no>" && exit 1
 119
 120 json=$1
 121 delay=""
 122 [ "$2" != "0" ] && delay="-w $2"
 123 debug=""
 124 [ "$3" = "yes" ] && debug="-d"
 125
 126 echo "[{description}] Kafka Producer Step"
 127 cd $(dirname $0)
 128 python3 Producer.py -n {node} -t {topic} -f $json $delay $debug 2>&1 | tee -a {node}-producer.sh.output
 129 exit ${{PIPESTATUS[0]}}
 130 '''.format(description=description, node=node, topic=topic)
 131
 132   path = agents_KAFKA_dir + "/{}-producer.sh".format(node)
 133   writeFileContent(path, content)
 134   os.chmod(path, 0o755)
 135   #printMsg("Created procedure '{}'".format(path))
 136
 137   # Consumer script:
 138   content = '''#!/bin/bash
 139 # Positional arguments: json, offset, timeout, debug indicator
 140 [ -z "$4" ] && echo "Usage: $0 <json file ('any' to skip matching)> <auto offset reset> <timeout in seconds: 0 is no timeout> <debug indicator: yes|no>" && exit 1
 141
 142 file=""
 143 [ "$1" != "any" ] && file="-f $1"
 144 offset=$2
 145 timeout=""
 146 [ "$3" != "0" ] && timeout="-w $3"
 147 debug=""
 148 [ "$4" = "yes" ] && debug="-d"
 149
 150 echo "[{description}] Kafka Consumer Step"
 151 cd $(dirname $0)
 152 python3 Consumer.py -n {node} -t {topic} $file -o "$offset" $timeout $debug 2>&1 | tee -a {node}-consumer.sh.output
 153 exit ${{PIPESTATUS[0]}}
 154 '''.format(description=description, node=node, topic=topic)
 155
 156   # We don't use -m (match as regular expression) because some problems have been detected with big json files.
 157
 158   path = agents_KAFKA_dir + "/{}-consumer.sh".format(node)
 159   writeFileContent(path, content)
 160   os.chmod(path, 0o755)
 161   #printMsg("Created procedure '{}'".format(path))
 162
 163   # Admin script:
 164   content = '''#!/bin/bash
 165 # Positional arguments: operation, debug indicator
 166 [ -z "$2" ] && echo "Usage: $0 <operation: clean_topic (at the moment)> <debug indicator: yes|no>" && exit 1
 167 operation=""
 168 case $1 in
 169   clean_topic)
 170     operation="-r"
 171     ;;
 172 esac
 173 debug=""
 174 [ "$2" = "yes" ] && debug="-d"
 175
 176 echo "[{description}] Kafka Administration Step"
 177 cd $(dirname $0)
 178 python3 Admin.py -n {node} -t {topic} $operation $debug 2>&1 | tee -a {node}-admin.sh.output
 179 exit ${{PIPESTATUS[0]}}
 180 '''.format(description=description, node=node, topic=topic)
 181
 182   path = agents_KAFKA_dir + "/{}-admin.sh".format(node)
 183   writeFileContent(path, content)
 184   os.chmod(path, 0o755)
 185   #printMsg("Created procedure '{}'".format(path))
 186
 187
 188 def configure_agent_kafkacpp(node, description, topic):
 189
 190   # Consumer script:
 191   content = '''#!/bin/bash
 192 # Positional arguments: json, offset, timeout, debug indicator
 193 [ -z "$4" ] && echo "Usage: $0 <json file ('any' to skip matching)> <auto offset reset> <timeout in seconds: 0 is no timeout> <debug indicator: yes|no>" && exit 1
 194
 195 file=""
 196 [ "$1" != "any" ] && file="-f $1"
 197 offset=$2
 198 timeout=""
 199 [ "$3" != "0" ] && timeout="-w $3"
 200 debug=""
 201 [ "$4" = "yes" ] && debug="-d"
 202
 203 echo "[{description}] Kafkacpp Consumer Step"
 204 cd $(dirname $0)
 205 sh Consumer -n {node} -t {topic} $file -o "$offset" $timeout $debug 2>&1 | tee -a {node}-consumer.sh.output
 206 exit ${{PIPESTATUS[0]}}
 207 '''.format(description=description, node=node, topic=topic)
 208
 209   # We don't use -m (match as regular expression) because some problems have been detected with big json files.
 210
 211   path = agents_KAFKACPP_dir + "/{}-consumer.sh".format(node)
 212   writeFileContent(path, content)
 213   os.chmod(path, 0o755)
 214   #printMsg("Created procedure '{}'".format(path))
 215
 216
 217 def configure_agent_httpmock(node, description, address):
 218
 219   # Provisioning script:
 220   content = '''#!/bin/bash
 221 # Positional arguments: json, method, uri
 222 [ -z "$3" ] && echo "Usage: $0 <test id> <step id> <test path> <json file> <method: POST|PUT|GET|DELETE> <uri>" && exit 1
 223
 224 echo "[{description}] Http Mock Provision Step"
 225 cd $(dirname $0)
 226 sh Provision.sh $1 $2 $3 "{address}" "$4" "$5" "$6" 2>&1 | tee -a {node}-provision.sh.output
 227 exit ${{PIPESTATUS[0]}}
 228 '''.format(description=description, node=node, address=address)
 229
 230   path = agents_HTTPMOCK_dir + "/{}-provision.sh".format(node)
 231   writeFileContent(path, content)
 232   os.chmod(path, 0o755)
 233   #printMsg("Created procedure '{}'".format(path))
 234
 235
 236 def configure_agents(agents):
 237
 238   stacks = ""
 239   nodes = ""
 240
 241   for node in agents:
 242
 243     if (node == "ADML"): raise Exception('Invalid node id: "ADML" is reserved')
 244     if (node == "KAFKA"): raise Exception('Invalid node id: "KAFKA" is reserved')
 245     if (node == "KAFKACPP"): raise Exception('Invalid node id: "KAFKACPP" is reserved')
 246     if (node == "HTTPMOCK"): raise Exception('Invalid node id: "HTTPMOCK" is reserved')
 247
 248     printMsg("Node: {}".format(node))
 249
 250     description = getNodeValue(agents[node], "description")
 251     template = getNodeValue(agents[node], "template")
 252
 253     if (template == "ADML"):
 254       # Node file:
 255       path = agents_ADML_dir + "/{}.node".format(node)
 256       writeFileContent(path, description)
 257
 258       application_id = getNodeValue(agents[node], "application_id")
 259       dictionary = getNodeValue(agents[node], "dictionary")
 260       _type = getNodeValue(agents[node], "type")
 261       origin_host = getNodeValue(agents[node], "origin_host")
 262       origin_realm = getNodeValue(agents[node], "origin_realm")
 263       address = getNodeValue(agents[node], "address")
 264
 265       # Here stacks and nodes are accumulated. Consolidation of 'services.xml' is done at the end of iteration (*)
 266       s,n = configure_agent_adml(node, description, application_id, dictionary, _type, origin_host, origin_realm, address)
 267       stacks += s
 268       stacks += "\n"
 269       nodes += n
 270       nodes += "\n"
 271
 272     if (template == "KAFKA"):
 273       # Node file:
 274       path = agents_KAFKA_dir + "/{}.node".format(node)
 275       writeFileContent(path, description)
 276
 277       topic = getNodeValue(agents[node], "topic")
 278
 279       configure_agent_kafka(node, description, topic)
 280
 281
 282     if (template == "KAFKACPP"):
 283       # Node file:
 284       path = agents_KAFKACPP_dir + "/{}.node".format(node)
 285       writeFileContent(path, description)
 286
 287       topic = getNodeValue(agents[node], "topic")
 288
 289       configure_agent_kafkacpp(node, description, topic)
 290
 291
 292     if (template == "HTTPMOCK"):
 293       # Node file:
 294       path = agents_HTTPMOCK_dir + "/{}.node".format(node)
 295       writeFileContent(path, description)
 296
 297       address = getNodeValue(agents[node], "address")
 298
 299       configure_agent_httpmock(node, description, address)
 300
 301
 302   #(*) Consolidate services.xml
 303   path = agents_ADML_dir + "/services.xml"
 304
 305   content='''<services>
 306 {}
 307 {}
 308 </services>'''.format(stacks, nodes)
 309
 310   writeFileContent(path, content)
 311
 312
 313
 314 def parse_arguments():
 315
 316   parser = ArgumentParser(description='Anna Agents-Oriented Testing Setup Loader')
 317   parser.add_argument('-f', '--file', help='Agents yaml configuration file', required=True)
 318
 319
 320   arguments = parser.parse_args()
 321
 322   return arguments
 323
 324
 325 #####################
 326 #      M A I N      #
 327 #####################
 328
 329 if __name__ == "__main__":
 330
 331   # Agents:
 332   abspath = os.path.abspath(__file__)
 333   dname = os.path.dirname(abspath)
 334   agents_ADML_dir = dname + "/agents/ADML"
 335   agents_KAFKA_dir = dname + "/agents/KAFKA"
 336   agents_KAFKACPP_dir = dname + "/agents/KAFKACPP"
 337   agents_HTTPMOCK_dir = dname + "/agents/HTTPMOCK"
 338
 339   arguments = parse_arguments()
 340
 341   afile = arguments.file
 342
 343   try:
 344     agents = YamlConfigParser(arguments.file)
 345
 346     configure_agents(agents.getData())
 347     #start_agents(agents)
 348
 349     _exit()
 350
 351   except Exception as e:
 352     _exit(e, 1)