Add deployment for ADML agent with http interface
[anna.git] / example / diameter / launcher / deployments / aots-setup / loader.py
1 #!/usr/bin/env python
2 # Anna Agents-Oriented Testing Setup Loader
3
4 # Requires PyYAML: pip install pyyaml
5
6 import os, sys, datetime
7 from argparse import ArgumentParser
8
9 # PyYAML
10 from yaml import load, dump
11 try:
12     from yaml import CLoader as Loader, CDumper as Dumper
13 except ImportError:
14     from yaml import Loader, Dumper
15
16
17 def _exit(message = None, rc=0):
18   if (message): printMsg(message)
19   printMsg("Exiting [rc={}]".format(rc))
20   sys.exit(rc)
21
22
23 def timeFormat():
24   return '%b.%d.%Y-%H.%M.%S'
25
26
27 def printMsg(message):
28   print ("[{}] {}".format(datetime.datetime.now().strftime(timeFormat()), message))
29
30
31 class YamlConfigParser():
32   """
33   Yaml parser
34   """
35   def __init__(self, yaml_config_file):
36     """
37     Convert the yaml file into a Python object
38     """
39     self.data = None
40     try:
41       with open(yaml_config_file, 'r') as ss:
42         self.data = load(ss, Loader=Loader)
43         ss.close()
44
45     except Exception:
46      raise
47
48   def getData(self):
49     return self.data
50
51   def show(self, options):
52     output = dump(self.data, Dumper=Dumper)
53     print(output)
54
55
56 def writeFileContent(filename, content):
57   _file = open(filename, "w")
58   _file.write(content)
59   _file.close()
60
61
62 def getNodeValue(dictionary, key):
63
64   value = dictionary[key]
65   printMsg("  {}: {}".format(key, value))
66   return value
67
68
69 def configure_agent_adml(node, description, application_id, dictionary, _type, origin_host, origin_realm, address):
70
71   # Agent ADML configuration is basically generation of services:
72
73   if (_type == "client"):
74     stacks='''  <stack id="{application_id}" dictionary="{dictionary}" fixMode="Always" ignoreFlagsOnValidation="yes"/>'''.format(application_id=application_id, dictionary=dictionary)
75
76     nodes='''  <node originHost="{origin_host}" applicationId="{application_id}" entity="{address}" cer="cer.{application_id}.xml" answersTimeout="300000" dumpLog="yes"/>'''.format(application_id=application_id, dictionary=dictionary, origin_host=origin_host, address=address)
77
78     ce_path = agents_ADML_dir + "/cer.{}.xml".format(application_id)
79     ce_content='''<message version="1" name="CER" application-id="0">
80    <avp name="Origin-Host" data="{origin_host}"/>
81    <avp name="Origin-Realm" data="{origin_realm}"/>
82    <avp name="Host-IP-Address" data="1|127.0.0.1"/>
83    <avp name="Vendor-Id" data="10415"/>
84    <avp name="Product-Name" data="{description}"/>
85    <avp name="Auth-Application-Id" data="{application_id}"/>
86 </message>'''.format(application_id=application_id, origin_host=origin_host, origin_realm=origin_realm, description=description)
87
88   elif (_type == "server"):
89     stacks='''  <stack id="{application_id}" dictionary="{dictionary}" fixMode="Never"/>'''.format(application_id=application_id, dictionary=dictionary)
90
91     nodes='''  <node originHost="{origin_host}" applicationId="{application_id}" diameterServer="{address}" diameterServerSessions="10" cea="cea.{application_id}.xml" answersTimeout="300000" dumpLog="yes"/>'''.format(application_id=application_id, dictionary=dictionary, origin_host=origin_host, address=address)
92
93     ce_path = agents_ADML_dir + "/cea.{}.xml".format(application_id)
94     ce_content='''<message version="1" name="CEA" application-id="0">
95    <avp name="Result-Code" data="2001" alias="DIAMETER_SUCCESS"/>
96    <avp name="Origin-Host" data="{origin_host}"/>
97    <avp name="Origin-Realm" data="{origin_realm}"/>
98    <avp name="Host-IP-Address" data="1|127.0.0.1"/>
99    <avp name="Vendor-Id" data="10415"/>
100    <avp name="Product-Name" data="{description}"/>
101 </message>'''.format(origin_host=origin_host, origin_realm=origin_realm, description=description)
102
103   else:
104     raise Exception("Invalid ADML type '{}' (allowed: client|server)".format(_type))
105
106   # Create cer/cea files:
107   writeFileContent(ce_path, ce_content)
108
109
110   return stacks, nodes
111
112
113 def configure_agent_kafka(node, description, topic):
114
115   # Producer script:
116   content = '''#!/bin/bash
117 # Positional arguments: json, delay, debug indicator
118 [ -z "$3" ] && echo "Usage: $0 <json file> <delay in milliseconds: 0 is no delay> <debug indicator: yes|no>" && exit 1
119
120 json=$1
121 delay=""
122 [ "$2" != "0" ] && delay="-w $2"
123 debug=""
124 [ "$3" = "yes" ] && debug="-d"
125
126 echo "[{description}] Kafka Producer Step"
127 cd $(dirname $0)
128 python3 Producer.py -n {node} -t {topic} -f $json $delay $debug 2>&1 | tee -a {node}-producer.sh.output
129 exit ${{PIPESTATUS[0]}}
130 '''.format(description=description, node=node, topic=topic)
131
132   path = agents_KAFKA_dir + "/{}-producer.sh".format(node)
133   writeFileContent(path, content)
134   os.chmod(path, 0o755)
135   #printMsg("Created procedure '{}'".format(path))
136
137   # Consumer script:
138   content = '''#!/bin/bash
139 # Positional arguments: json, offset, timeout, debug indicator
140 [ -z "$4" ] && echo "Usage: $0 <json file ('any' to skip matching)> <auto offset reset> <timeout in seconds: 0 is no timeout> <debug indicator: yes|no>" && exit 1
141
142 file=""
143 [ "$1" != "any" ] && file="-f $1"
144 offset=$2
145 timeout=""
146 [ "$3" != "0" ] && timeout="-w $3"
147 debug=""
148 [ "$4" = "yes" ] && debug="-d"
149
150 echo "[{description}] Kafka Consumer Step"
151 cd $(dirname $0)
152 python3 Consumer.py -n {node} -t {topic} $file -o "$offset" $timeout $debug 2>&1 | tee -a {node}-consumer.sh.output
153 exit ${{PIPESTATUS[0]}}
154 '''.format(description=description, node=node, topic=topic)
155
156   # We don't use -m (match as regular expression) because some problems have been detected with big json files.
157
158   path = agents_KAFKA_dir + "/{}-consumer.sh".format(node)
159   writeFileContent(path, content)
160   os.chmod(path, 0o755)
161   #printMsg("Created procedure '{}'".format(path))
162
163   # Admin script:
164   content = '''#!/bin/bash
165 # Positional arguments: operation, debug indicator
166 [ -z "$2" ] && echo "Usage: $0 <operation: clean_topic (at the moment)> <debug indicator: yes|no>" && exit 1
167 operation=""
168 case $1 in
169   clean_topic)
170     operation="-r"
171     ;;
172 esac
173 debug=""
174 [ "$2" = "yes" ] && debug="-d"
175
176 echo "[{description}] Kafka Administration Step"
177 cd $(dirname $0)
178 python3 Admin.py -n {node} -t {topic} $operation $debug 2>&1 | tee -a {node}-admin.sh.output
179 exit ${{PIPESTATUS[0]}}
180 '''.format(description=description, node=node, topic=topic)
181
182   path = agents_KAFKA_dir + "/{}-admin.sh".format(node)
183   writeFileContent(path, content)
184   os.chmod(path, 0o755)
185   #printMsg("Created procedure '{}'".format(path))
186
187
188 def configure_agent_kafkacpp(node, description, topic):
189
190   # Consumer script:
191   content = '''#!/bin/bash
192 # Positional arguments: json, offset, timeout, debug indicator
193 [ -z "$4" ] && echo "Usage: $0 <json file ('any' to skip matching)> <auto offset reset> <timeout in seconds: 0 is no timeout> <debug indicator: yes|no>" && exit 1
194
195 file=""
196 [ "$1" != "any" ] && file="-f $1"
197 offset=$2
198 timeout=""
199 [ "$3" != "0" ] && timeout="-w $3"
200 debug=""
201 [ "$4" = "yes" ] && debug="-d"
202
203 echo "[{description}] Kafkacpp Consumer Step"
204 cd $(dirname $0)
205 sh Consumer -n {node} -t {topic} $file -o "$offset" $timeout $debug 2>&1 | tee -a {node}-consumer.sh.output
206 exit ${{PIPESTATUS[0]}}
207 '''.format(description=description, node=node, topic=topic)
208
209   # We don't use -m (match as regular expression) because some problems have been detected with big json files.
210
211   path = agents_KAFKACPP_dir + "/{}-consumer.sh".format(node)
212   writeFileContent(path, content)
213   os.chmod(path, 0o755)
214   #printMsg("Created procedure '{}'".format(path))
215
216
217 def configure_agent_httpmock(node, description, address):
218
219   # Provisioning script:
220   content = '''#!/bin/bash
221 # Positional arguments: json, method, uri
222 [ -z "$3" ] && echo "Usage: $0 <test id> <step id> <test path> <json file> <method: POST|PUT|GET|DELETE> <uri>" && exit 1
223
224 echo "[{description}] Http Mock Provision Step"
225 cd $(dirname $0)
226 sh Provision.sh $1 $2 $3 "{address}" "$4" "$5" "$6" 2>&1 | tee -a {node}-provision.sh.output
227 exit ${{PIPESTATUS[0]}}
228 '''.format(description=description, node=node, address=address)
229
230   path = agents_HTTPMOCK_dir + "/{}-provision.sh".format(node)
231   writeFileContent(path, content)
232   os.chmod(path, 0o755)
233   #printMsg("Created procedure '{}'".format(path))
234
235
236 def configure_agents(agents):
237
238   stacks = ""
239   nodes = ""
240
241   for node in agents:
242
243     if (node == "ADML"): raise Exception('Invalid node id: "ADML" is reserved')
244     if (node == "KAFKA"): raise Exception('Invalid node id: "KAFKA" is reserved')
245     if (node == "KAFKACPP"): raise Exception('Invalid node id: "KAFKACPP" is reserved')
246     if (node == "HTTPMOCK"): raise Exception('Invalid node id: "HTTPMOCK" is reserved')
247
248     printMsg("Node: {}".format(node))
249
250     description = getNodeValue(agents[node], "description")
251     template = getNodeValue(agents[node], "template")
252
253     if (template == "ADML"):
254       # Node file:
255       path = agents_ADML_dir + "/{}.node".format(node)
256       writeFileContent(path, description)
257
258       application_id = getNodeValue(agents[node], "application_id")
259       dictionary = getNodeValue(agents[node], "dictionary")
260       _type = getNodeValue(agents[node], "type")
261       origin_host = getNodeValue(agents[node], "origin_host")
262       origin_realm = getNodeValue(agents[node], "origin_realm")
263       address = getNodeValue(agents[node], "address")
264
265       # Here stacks and nodes are accumulated. Consolidation of 'services.xml' is done at the end of iteration (*)
266       s,n = configure_agent_adml(node, description, application_id, dictionary, _type, origin_host, origin_realm, address)
267       stacks += s
268       stacks += "\n"
269       nodes += n
270       nodes += "\n"
271
272     if (template == "KAFKA"):
273       # Node file:
274       path = agents_KAFKA_dir + "/{}.node".format(node)
275       writeFileContent(path, description)
276
277       topic = getNodeValue(agents[node], "topic")
278
279       configure_agent_kafka(node, description, topic)
280
281
282     if (template == "KAFKACPP"):
283       # Node file:
284       path = agents_KAFKACPP_dir + "/{}.node".format(node)
285       writeFileContent(path, description)
286
287       topic = getNodeValue(agents[node], "topic")
288
289       configure_agent_kafkacpp(node, description, topic)
290
291
292     if (template == "HTTPMOCK"):
293       # Node file:
294       path = agents_HTTPMOCK_dir + "/{}.node".format(node)
295       writeFileContent(path, description)
296
297       address = getNodeValue(agents[node], "address")
298
299       configure_agent_httpmock(node, description, address)
300
301
302   #(*) Consolidate services.xml
303   path = agents_ADML_dir + "/services.xml"
304
305   content='''<services>
306 {}
307 {}
308 </services>'''.format(stacks, nodes)
309
310   writeFileContent(path, content)
311
312
313
314 def parse_arguments():
315
316   parser = ArgumentParser(description='Anna Agents-Oriented Testing Setup Loader')
317   parser.add_argument('-f', '--file', help='Agents yaml configuration file', required=True)
318
319
320   arguments = parser.parse_args()
321
322   return arguments
323
324
325 #####################
326 #      M A I N      #
327 #####################
328
329 if __name__ == "__main__":
330
331   # Agents:
332   abspath = os.path.abspath(__file__)
333   dname = os.path.dirname(abspath)
334   agents_ADML_dir = dname + "/agents/ADML"
335   agents_KAFKA_dir = dname + "/agents/KAFKA"
336   agents_KAFKACPP_dir = dname + "/agents/KAFKACPP"
337   agents_HTTPMOCK_dir = dname + "/agents/HTTPMOCK"
338
339   arguments = parse_arguments()
340
341   afile = arguments.file
342
343   try:
344     agents = YamlConfigParser(arguments.file)
345
346     configure_agents(agents.getData())
347     #start_agents(agents)
348
349     _exit()
350
351   except Exception as e:
352     _exit(e, 1)