Merge test folder of test branch
[iotivity.git] / test / test_manager / ite / exec / tc_executor.py
1 import re
2 import os
3 import io
4 import sys
5 import time
6 from time import strftime
7 from datetime import datetime
8 import subprocess
9 import traceback
10 import threading
11
12 from ite.constants import *
13 from ite.util import *
14 from ite.multi_logger import print_runner_output, show_exeception, print_to_file
15 from ite.exec.run_options import TestRunOption
16 from ite.exec.tc_utility_action import TcUtilityAction
17
18 class TcExecutor:
19
20     def __init__(self, timeout_seconds, test_result_dir):
21         self.timeout_seconds = timeout_seconds
22         self.test_result_dir = test_result_dir
23
24     def print_tc_list(self, file_path, dynamic_runner):
25         fp = open(file_path, "w")
26         list_of_testcase = dynamic_runner.get_tc_list()
27         for testcase in list_of_testcase:
28             command = 'binary_name:{}, suite_name:{}, tc_name:{}, package_name:{}'.format(testcase.binary_name, testcase.suite_name, testcase.tc_name, testcase.package_name)
29             fp.write (command + '\n')
30         fp.close()
31         print_runner_output('TC List Generated!!')
32
33     def run_selected_testcase(self, binary_name, suite_name, tc_name, package_name, dynamic_runner, tc_utility_action):
34
35         fw_output_file = 'temporary_test_output'
36         platform_type = dynamic_runner.platform_type
37         target = dynamic_runner.target
38         build_type = dynamic_runner.build_type
39         transport = dynamic_runner.transport
40         network = dynamic_runner.network
41         module = dynamic_runner.module
42         search_text, expected, search_file_type = dynamic_runner.get_pass_criteria()
43         crash_texts = dynamic_runner.get_crash_text_list()
44         command = dynamic_runner.get_tc_run_command(binary_name, suite_name, tc_name, package_name, fw_output_file)
45         print_runner_output(command)
46
47         if os.path.exists(fw_output_file):
48             os.remove(fw_output_file)
49
50         dynamic_runner.remove_fw_output_file(fw_output_file)
51
52         failure_msg_list = []
53
54         for attempt in range(0, 2):
55
56             log = ''
57             result = 0
58             delta = 0
59             status = GT_ATT_STATUS[GT_LOG.OK]
60             is_execution_finshed = False
61             failure_msg_list = []
62
63             dynamic_runner.clear_device_log()
64             print_runner_output('current_path: {}'.format(os.getcwd()))
65             dynamic_runner.change_current_dir_to_app_path()
66
67             #function_name = '{}_{}_tc_pre_action'.format(module, platform_type)
68             #if hasattr(TcUtilityAction, function_name) and callable(getattr(TcUtilityAction, function_name)):
69             #    getattr(tc_utility_action, function_name)()
70             #else:
71             #    print_runner_output('no {} function found'.format(function_name))
72
73             start_time = datetime.now()
74
75             try:
76
77                 def start_tc_execution():
78                     nonlocal log
79                     nonlocal is_execution_finshed
80
81                     rc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, bufsize=1)
82                     #encoding utf-8 wasn't working, ISO-8859-1 was worked for a testcase
83                     for line in io.TextIOWrapper(rc.stdout, encoding="ISO-8859-1"):
84                         print_to_file(line.strip())
85                         log += line
86                         for crash_text in crash_texts:
87                             if crash_text in log:
88                                 break
89                     is_execution_finshed = True
90
91                 t = threading.Thread(target=start_tc_execution)
92                 t.start()
93                 cnt = 0
94                 timeout = self.timeout_seconds / 10
95                 while t.is_alive() and cnt < timeout:
96                     print_runner_output('Observe Session No: {}'.format(cnt))
97                     t.join(10)
98                     cnt += 1
99                     if is_execution_finshed:
100                         break
101
102                 for crash_text in crash_texts:
103                     if crash_text in log:
104                         failure_msg_list.append("crashed")
105                         status = GT_ATT_STATUS[GT_LOG.CRASHED]
106                         result = 3
107                         break
108
109                 if result == 0 and not is_execution_finshed:
110                     failure_msg_list.append("hang")
111                     status = GT_ATT_STATUS[GT_LOG.HANG]
112                     result = 2
113
114                 if result == 0:
115                     text_found = False
116                     if search_file_type == TestRunOption.XML_PASS_CRITERIA:
117                         dynamic_runner.download_fw_output_file(fw_output_file)
118                         if os.path.exists(fw_output_file):
119                             contents = open(fw_output_file, 'r').read()
120                             text_found = True
121                         else:
122                             print_runner_output('No output file found')
123                     else:
124                         contents = log
125                         text_found = True
126
127                     if text_found and ((expected and contents.find(search_text) == -1) or (not expected and contents.find(search_text) >= 0)):
128                         failure_msg_list.append(dynamic_runner.get_failure_msg(log))
129                         result = 1
130                     if not text_found:
131                         failure_msg_list.append("unexpected")
132                         result = 4
133
134             except subprocess.CalledProcessError:
135                     show_exeception()
136                     failure_msg_list.append("crashed")
137                     result = 3
138                     status = GT_ATT_STATUS[GT_LOG.CRASHED]
139
140             end_time = datetime.now()
141
142             delta = end_time - start_time
143
144             #function_name = '{}_{}_tc_post_action'.format(module, platform_type)
145             #if hasattr(TcUtilityAction, function_name) and callable(getattr(TcUtilityAction, function_name)):
146             #    getattr(tc_utility_action, function_name)(log_file)
147             #else:
148             #    print_runner_output('no {} function found'.format(function_name))
149
150             print_runner_output('executing {} complete'.format(command))
151             print_runner_output('current_path {}'.format(os.getcwd()))
152             dynamic_runner.change_back_to_runner_path()
153
154             if result == 0:
155                 break
156
157             if dynamic_runner.check_connectivity():
158                 break
159             else:
160                 failure_msg_list.append("low network quality")
161
162         xml_output = '<testsuites>\n'
163         xml_output += '\t<testsuite name="{}">\n'.format(suite_name)
164         xml_output += '\t\t<testcase name="{}" status="{}" time="{}">\n'.format(tc_name, status, int(delta.total_seconds() * 1000))
165
166         for failure_msg in failure_msg_list:
167             xml_output += ('\t\t\t<failure message="{}"> </failure>\n'.format(failure_msg))
168
169         xml_output += '\t\t</testcase>\n'
170         xml_output += '\t</testsuite>\n'
171         xml_output += '</testsuites>\n'
172         
173         timestring = datetime.now().strftime("%Y%m%d_%H%M%S.%f")
174
175         process_id = binary_name
176         if not process_id:
177             process_id = package_name
178
179         file_name = "{}_{}_{}_{}_{}_{}_{}" .format(platform_type, target, build_type, transport, network, timestring, process_id.replace('.', '_'))
180         log_file_path = os.path.join(self.test_result_dir, file_name + '.log')
181         xml_file_path = os.path.join(self.test_result_dir, file_name + '.xml')
182
183         if os.path.isfile(log_file_path) or os.path.isfile(xml_file_path):
184             print_runner_output('', Colors.FAIL, 'File name exist !!!\nRunner Internal Error', Colors.ENDC)
185             return command, 4
186
187         dynamic_runner.append_device_log(log_file_path)
188
189         if not os.path.exists(self.test_result_dir):
190             print_runner_output('{} path not found'.format(self.test_result_dir))
191             result_path_parent = self.test_result_dir
192
193             result_path_count = 0
194             while result_path_parent:
195                 pos = result_path_parent.rfind(os.sep)
196                 if pos == -1:
197                     break
198                 result_path_parent = result_path_parent[:pos]
199                 if os.path.exists(result_path_parent):
200                     print_runner_output('{} path found'.format(result_path_parent))
201                     file_list = os.listdir(result_path_parent)
202                     print_runner_output('printing file list ...')
203                     for f in file_list:
204                         print_runner_output(f)
205                     print_runner_output('file list printed')
206                     break
207                 else:
208                     print_runner_output('{} path not found'.format(result_path_parent))
209                 result_path_count += 1
210                 if result_path_count > 20:
211                     break
212
213             return command, 4
214
215         if not os.path.isdir(self.test_result_dir):
216             print_runner_output('{} directory not found'.format(self.test_result_dir))
217             return command, 4
218
219         if not os.path.isfile(log_file_path):
220             print_runner_output('No log file {} found'.format(log_file_path))
221
222         log_file = open(log_file_path, 'a')
223         log_file.write('\n' + log)
224
225         xml_file = open(xml_file_path, 'w')
226         xml_file.write(xml_output)
227
228         return command, result
229
230     def remove_pass_tc(self, list_of_testcase, verdict_file_path):
231
232         if not os.path.exists(verdict_file_path):
233             return list_of_testcase
234
235         f = open(verdict_file_path,'r')
236         contents = f.readlines()
237         f.close()
238
239         mydi={}
240         for row in contents:
241             d = row.split('|')
242             d = [x.strip() for x in d]
243
244             key1 = d[1]
245             key2 = d[4]
246             key3 = d[2]
247             key4 = d[3]
248             value = d[5]
249
250             if key1 not in mydi.keys():
251                 mydi[key1] = {}
252             if key2 not in mydi[key1].keys():
253                 mydi[key1][key2] = {}
254             if key3 not in mydi[key1][key2].keys():
255                 mydi[key1][key2][key3] = {}
256             if key4 not in mydi[key1][key2][key3].keys():
257                 mydi[key1][key2][key3][key4] = value
258             else:
259                 if value == 'pass':
260                     mydi[key1][key2][key3][key4] = value
261
262         new_tc_list = []
263
264         for testcase in list_of_testcase:
265             if testcase.binary_name in mydi.keys():
266                 if testcase.package_name in mydi[testcase.binary_name].keys():
267                     if testcase.suite_name in mydi[testcase.binary_name][testcase.package_name].keys():
268                         if testcase.tc_name in mydi[testcase.binary_name][testcase.package_name][testcase.suite_name].keys():
269                             if 'pass' == mydi[testcase.binary_name][testcase.package_name][testcase.suite_name][testcase.tc_name]:
270                                 continue
271             new_tc_list.append(testcase)
272         return new_tc_list
273
274     def run_selected_testcases(self, dynamic_runner, verdict_file_path, save_verdict):
275
276         verdict_types = ['pass', 'fail', 'timeout', 'crash', 'unexpected']
277         color_types = [Colors.OKGREEN, Colors.FAIL, Colors.FAIL, Colors.FAIL, Colors.FAIL]
278
279         list_of_testcase = dynamic_runner.get_tc_list()
280
281         if not list_of_testcase:
282             print_runner_output(Colors.FAIL + 'No testcase Found !!!\nPlease, Check command parameter(s)' + Colors.ENDC)
283             return
284
285         if save_verdict:
286             list_of_testcase = self.remove_pass_tc(list_of_testcase, verdict_file_path)
287
288         fp = None
289         if save_verdict:
290             fp = open(verdict_file_path, 'a')
291
292         for testcase in list_of_testcase:
293             if fp:
294                 text = 'verdict | '
295                 text += testcase.binary_name + ' |'
296                 text += testcase.suite_name + ' |'
297                 text += testcase.tc_name + ' |'
298                 text += testcase.package_name + ' |'
299                 text += 'in_queue'
300
301                 fp.write(text + '\n')
302                 fp.flush()
303
304         tc_utility_action = TcUtilityAction(dynamic_runner)
305
306         while list_of_testcase:
307
308             print_runner_output('{} testcase(s) needed to be run'.format(len(list_of_testcase)))
309
310             for testcase in list_of_testcase:
311                 result = 1
312                 print_runner_output('Executing ' + testcase.binary_name + ' ' + testcase.package_name + '.' + testcase.suite_name + '.' + testcase.tc_name)
313
314                 try:
315                     command, result = self.run_selected_testcase (testcase.binary_name, testcase.suite_name, testcase.tc_name, testcase.package_name, dynamic_runner, tc_utility_action)
316                     print_runner_output('result: {}'.format(result))
317
318                     if result == 0:
319                         testcase.increase_pass_count()
320                     elif result == 2:
321                         testcase.increase_timeout_count()
322                         testcase.increase_fail_count()
323                     else:
324                         testcase.increase_fail_count()
325
326                     testcase.increase_total_count()
327
328                     print_runner_output('increase tc verdict count')
329                 except:
330                     show_exeception()
331                     break
332
333                 print_runner_output(color_types[result] + '[ ' + 'Result of ' + testcase.suite_name + '.' + testcase.tc_name + ': ' + verdict_types[result] + ' ]' + Colors.ENDC)
334
335                 if fp:
336                     text = 'verdict | '
337                     text += testcase.binary_name + ' |'
338                     text += testcase.suite_name + ' |'
339                     text += testcase.tc_name + ' |'
340                     text += testcase.package_name + ' |'
341                     text += verdict_types[result]
342
343                     fp.write(text + '\n')
344                     fp.flush()
345
346                 if result == 4:
347                     return
348
349             list_of_testcase[:] = [x for x in list_of_testcase if not x.is_execution_complete()]
350
351         if fp:
352             fp.close()
353
354         print_runner_output("### Test Is Done!!")
355