Merge test folder of test branch
[iotivity.git] / test / test_manager / ite / reporter / result_reporter.py
1 '''
2 /******************************************************************
3 *
4 * Copyright 2018 Samsung Electronics All Rights Reserved.
5 *
6 *
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 ******************************************************************/
21 '''
22
23 import sys
24 import operator
25 import os
26
27 from xlsxwriter.workbook import Workbook
28
29 from ite.config import *
30 from ite.constants import *
31 from ite.result.collector import TestResultCollector
32 from ite.tc.container import TestSpecContainer
33 from ite.reporter.excel_util import *
34 from ite.reporter.reporter_util import *
35 from ite.util import *
36
37 class TestResultReporter:
38
39
40     def __init__(self):
41         self.summary = dict()
42         self.failtc = dict()
43         self.passtc = dict()
44         self.testspec = None
45
46     def make_summary(self, data):
47         for platform in TEST_PLATFORM:
48             if not platform in data:
49                 continue
50
51             self.summary[platform] = dict()
52             self.failtc[platform] = dict()
53             self.passtc[platform] = dict()
54
55             for target in data[platform]:
56                 self.summary[platform][target] = dict()
57                 self.failtc[platform][target] = dict()
58                 self.passtc[platform][target] = dict()
59
60                 for build_type in data[platform][target]:
61                     self.summary[platform][target][build_type] = dict()
62                     self.failtc[platform][target][build_type] = dict()
63                     self.passtc[platform][target][build_type] = dict()
64
65                     for transport in data[platform][target][build_type]:
66                         self.summary[platform][target][build_type][transport] = dict()
67                         self.failtc[platform][target][build_type][transport] = dict()
68                         self.passtc[platform][target][build_type][transport] = dict()
69
70                         for network in data[platform][target][build_type][transport]:
71                             self.summary[platform][target][build_type][transport][network] = dict()
72                             self.failtc[platform][target][build_type][transport][network] = dict()
73                             self.passtc[platform][target][build_type][transport][network] = dict()
74
75                             for tctype in TESTCASE_TYPES:
76                                 self.summary[platform][target][build_type][transport][network][tctype] = dict()
77                                 self.failtc[platform][target][build_type][transport][network][tctype] = dict()
78                                 self.passtc[platform][target][build_type][transport][network][tctype] = dict()
79
80                                 for module in TESTSUITE_MODULES:
81                                     self.summary[platform][target][build_type][transport][network][tctype][module] = dict()
82                                     self.failtc[platform][target][build_type][transport][network][tctype][module] = dict()
83                                     self.passtc[platform][target][build_type][transport][network][tctype][module] = dict()
84
85                                     for result in TC_RESULT:
86                                         self.summary[platform][target][build_type][transport][network][tctype][module][result] = 0
87
88                                     if ((not tctype in list(data[platform][target][build_type][transport][network])) or (not module in list(data[platform][target][build_type][transport][network][tctype]))):
89                                         continue
90
91                                     for suite in list(data[platform][target][build_type][transport][network][tctype][module]):
92                                         self.failtc[platform][target][build_type][transport][network][tctype][module][suite] = list()
93                                         self.passtc[platform][target][build_type][transport][network][tctype][module][suite] = list()
94
95                                         for tc_name in list(data[platform][target][build_type][transport][network][tctype][module][suite]):
96                                             testcase = data[platform][target][build_type][transport][network][tctype][module][suite][tc_name]
97                                             if (testcase.success == 0):
98                                                 self.summary[platform][target][build_type][transport][network][tctype][module][TC_RESULT.FAIL] += 1
99                                                 self.failtc[platform][target][build_type][transport][network][tctype][module][suite].append(testcase)
100                                             else:
101                                                 self.summary[platform][target][build_type][transport][network][tctype][module][TC_RESULT.PASS] += 1
102                                                 self.passtc[platform][target][build_type][transport][network][tctype][module][suite].append(testcase)
103
104
105     def generate_testresult_report(self, path, spec_data):
106         collector = TestResultCollector()
107         collector.collect_results(path)
108
109         self.testspec = spec_data
110
111         self.make_summary(collector.result.data)
112
113     def report_fail_result_to_txt(self, dir_path):
114         self.report_test_result_to_txt(dir_path, self.failtc)
115
116     def report_pass_result_to_txt(self, dir_path):
117         self.report_test_result_to_txt(dir_path, self.passtc)
118
119     def report_test_result_to_txt(self, dir_path, result_suite):
120         for platform in TEST_PLATFORM:
121             if not platform in result_suite:
122                 continue
123
124             for target in result_suite[platform]:
125                 for build_type in  result_suite[platform][target]:
126                     for transport in result_suite[platform][target][build_type]:
127                         transport_name = transport
128                         if transport_name == NO_TRANSPORT:
129                             transport_name = ''
130                         for network in result_suite[platform][target][build_type][transport]:
131                             network_name = network
132                             if network_name == NO_NETWORK:
133                                 network_name = ''
134                             new_dir = os.path.join(dir_path, '%s_%s_%s_%s_%s' %(platform, target.strip(), build_type, transport_name, network_name))
135                             os.makedirs(new_dir)
136                             for tctype in result_suite[platform][target][build_type][transport][network]:
137                                 for module in result_suite[platform][target][build_type][transport][network][tctype]:
138                                     for suite in list(result_suite[platform][target][build_type][transport][network][tctype][module]):
139                                         for testcase in list(result_suite[platform][target][build_type][transport][network][tctype][module][suite]):
140                                             text_file_name = suite
141                                             if not ('_' + tctype.lower()) in suite.lower():
142                                                 text_file_name += '_' + tctype.lower()
143                                             text_file_name += '_' + testcase.name + '_' + build_type + '.txt'
144                                             file_path = os.path.join(new_dir, text_file_name)
145                                             txt = open_write_file(file_path)
146                                             if txt == False:
147                                                 continue
148                                             print("[Defect Info]", file=txt)
149                                             print(" Title: ", file=txt)
150                                             print(" Description: \n", file=txt)
151
152                                             if platform in self.testspec:
153                                                 if tctype in self.testspec[platform]:
154                                                     if module in self.testspec[platform][tctype]:
155                                                         if transport in self.testspec[platform][tctype][module]:
156                                                             if network in self.testspec[platform][tctype][module][transport]:
157                                                                 if suite in self.testspec[platform][tctype][module][transport][network]:
158                                                                     if testcase.name in self.testspec[platform][tctype][module][transport][network][suite]:
159                                                                         spec = self.testspec[platform][tctype][module][transport][network][suite][testcase.name]
160                                                                         print("[Test Case Info]", file=txt)
161                                                                         print(spec.to_string(), file=txt)
162
163                                             print("\n[Test Result]", file=txt)
164                                             print("Fail Rate: %d/%d\n" % (testcase.fail, testcase.fail+testcase.success), file=txt)
165
166                                             index = 1
167                                             for result in testcase.runresult:
168                                                 print("[%d Try]" % index, file=txt)
169                                                 print("--------------------------------------------------------------------", file=txt)
170                                                 print("Result: " + result.result, file=txt)
171                                                 print("Run Type: " + result.runtype, file=txt)
172                                                 print("Run Time: " + str(result.runtime), file=txt)
173                                                 print("\n<<<Fail Message>>> \n" + result.fail_msg, file=txt)
174                                                 print("\n<<<Test Log>>> \n" + result.test_log, file=txt)
175                                                 print("--------------------------------------------------------------------\n\n", file=txt)
176                                                 index += 1
177
178     def report_result(self, sheet, form, report_title, result_writer, result_suite):
179         row = 0
180         col = 0
181         for title, size in RESULT_CAT_TITLE:
182             if (title == 'Line'):
183                 continue
184             sheet.write(row, col, title, form.title)
185             sheet.set_column(col, col, size)
186             col += 1
187
188         for title, size in report_title:
189             sheet.write(row, col, title, form.title)
190             sheet.set_column(col, col, size)
191             col += 1
192
193         row += 1
194         col = 0
195
196         def write_platform(platforms):
197             for platform in platforms:
198                 nonlocal row, col
199                 platform_first_row = row
200
201                 if not platform in result_suite:
202                     continue
203
204                 col += 1
205
206                 yield result_suite[platform], platform
207
208                 col -= 1
209                 merge_cell(sheet, platform_first_row, col, row-1, col, platform, form.cell)
210
211         def write_target(platforms):
212             for platform in platforms:
213                 targets, platform_name = platform
214                 for target in list(targets):
215                     nonlocal row, col
216                     target_first_row = row
217
218                     col += 1
219
220                     yield targets[target], platform_name, target
221
222                     col -= 1
223                     merge_cell(sheet, target_first_row, col, row-1, col, target, form.cell)
224
225         def write_build_type(targets):
226             for target in targets:
227                 build_types, platform, target = target
228                 for build_type in list(build_types):
229                     nonlocal row, col
230                     build_type_first_row = row
231
232                     col += 1
233
234                     yield build_types[build_type], platform, target, build_type
235
236                     col -= 1
237                     merge_cell(sheet, build_type_first_row, col, row-1, col, build_type, form.cell)
238
239         def write_transport(build_types):
240             for build_type in build_types:
241                 transports, platform, target, build_type = build_type
242                 for transport in list(transports):
243                     nonlocal row, col
244                     transport_first_row = row
245
246                     col += 1
247
248                     yield transports[transport], platform, target, build_type, transport
249
250                     col -= 1
251                     merge_cell(sheet, transport_first_row, col, row-1, col, transport, form.cell)
252
253
254         def write_network(transports):
255             for transport in transports:
256                 networks, platform, target, build_type, transport = transport
257                 for network in list(networks):
258                     nonlocal row, col
259                     network_first_row = row
260
261                     col += 1
262
263                     yield networks[network], platform, target, build_type, transport, network
264
265                     col -= 1
266                     merge_cell(sheet, network_first_row, col, row-1, col, network, form.cell)
267
268
269         def write_tctype(networks):
270             for network in networks:
271                 types, platform, target, build_type, transport, network = network
272                 for tctype in list(types):
273                     nonlocal row, col
274                     type_first_row = row
275
276                     module_count = 0
277                     for module in TESTSUITE_MODULES:
278                         module_count += len(types[tctype][module])
279                     if module_count == 0:
280                         continue
281
282                     col += 1
283
284                     yield TESTSUITE_MODULES, types[tctype], platform, target, build_type, transport, network, tctype
285
286                     col -= 1
287                     merge_cell(sheet, type_first_row, col, row-1, col, tctype, form.cell)
288
289         def write_module(tctypes):
290             for tctype in tctypes:
291                 module_names, modules, platform, target, build_type, transport, network, tctype = tctype
292                 for module in module_names:
293                     nonlocal row, col
294                     module_first_row = row
295
296                     if (len(list(modules[module])) == 0):
297                         continue
298
299                     col += 1
300
301                     yield modules[module], platform, target, build_type, transport, network, tctype, module
302
303                     col -= 1
304                     merge_cell(sheet, module_first_row, col, row-1, col, module, form.cell)
305
306         def write_suite(modules):
307             for module in modules:
308                 suites, platform, target, build_type, transport, network, tctype, module = module
309                 for suite in list(suites):
310                     nonlocal row, col
311                     suite_first_row = row
312
313                     if (len(suites[suite]) == 0):
314                         continue
315
316                     if not suite in suites and not suite in self.testspec[platform][build_type][tctype][module][transport][network]:
317                         print ('suite not found: ', platform, tctype, module, transport, network, suite)
318                         continue
319
320                     col += 1
321
322                     yield suites[suite], platform, target, build_type, transport, network, tctype, module, suite
323
324                     col -=1
325                     merge_cell(sheet, suite_first_row, col, row-1, col, suite, form.cell)
326
327         def write_tc(suites):
328             for suite in suites:
329                 testcases, platform, target, build_type, transport, network, tctype, module, suite = suite
330                 testspec = self.testspec[platform][build_type][tctype][module][transport][network][suite]
331                 for testcase in testcases:
332                     nonlocal row, col
333                     row, col = result_writer(row, col, testcase, testspec, sheet, platform, target, transport, network)
334
335
336         walk_through_results(write_platform, write_target, write_build_type, write_transport, write_network, write_tctype, write_module, write_suite, write_tc)
337
338
339     def report_to_xlsx(self, path):
340         workbook = Workbook(path)
341         form = ExcelFormat(workbook)
342
343         summarysheet = workbook.add_worksheet('Summary')
344
345         row = 0
346         col = 0
347
348         summarysheet.merge_range(row, col, row+3, col, '', form.cell)
349
350         col += 1
351         for platform in TEST_PLATFORM:
352             if not platform in self.summary:
353                 continue
354
355             row += 1
356             platform_col = col
357
358             for target in self.summary[platform]:
359                 row += 1
360                 target_col = col
361
362                 for tc_type in TESTCASE_TYPES + ('Total',):
363                     row += 1
364                     tc_type_col = col
365
366                     for result in TC_RESULT + ('Total',):
367                         summarysheet.write(row, col, result, form.result_title[result])
368                         col += 1
369
370                     row -= 1
371                     summarysheet.merge_range(row, tc_type_col, row, col-1, tc_type, form.title)
372
373                 row -= 1
374                 summarysheet.merge_range(row, target_col, row, col-1, target, form.title)
375
376                 row -= 1
377                 summarysheet.merge_range(row, platform_col, row, col-1, platform, form.title)
378
379         total_col = col
380         for result in TC_RESULT + ('Total','Pass Rate',):
381             summarysheet.write(row+3, col, result, form.result_title[result])
382             col += 1
383         summarysheet.merge_range(row, total_col, row+2, col-1, "Total", form.total)
384
385         row += 4
386         col = 0
387
388         for module in TESTSUITE_MODULES :
389             col = 0
390             summarysheet.write(row, col, module, form.title)
391             col += 1
392
393             module_total_txt = dict()
394             for result in TC_RESULT + ('Total',):
395                 module_total_txt[result] = '=SUM('
396
397             for platform in TEST_PLATFORM:
398                 if not platform in self.summary:
399                     continue
400
401                 for target in self.summary[platform]:
402                     for tc_type in TESTCASE_TYPES:
403                         for result in TC_RESULT:
404                             result_sum = 0
405                             for build_type in self.summary[platform][target]:
406                                 for transport in self.summary[platform][target][build_type]:
407                                     for network in self.summary[platform][target][build_type][transport]:
408                                         result_sum += self.summary[platform][target][build_type][transport][network][tc_type][module][result]
409                             summarysheet.write(row, col, result_sum, form.cell)
410                             col += 1
411
412                         total_txt = '=SUM(%s:%s)' % (get_cell_name(col - len(TC_RESULT), row), get_cell_name(col - 1, row))
413                         summarysheet.write(row, col, total_txt, form.cell)
414                         col += 1
415
416                     for result in TC_RESULT + ('Total',):
417                                 total_txt = '=SUM(%s,%s)' % (get_cell_name(col - (len(TC_RESULT) + 1)*2, row), get_cell_name(col - (len(TC_RESULT) + 1), row))
418                                 summarysheet.write(row, col, total_txt, form.cell)
419                                 module_total_txt[result] += '%s,' % get_cell_name(col, row)
420                                 col += 1
421
422             for result in TC_RESULT + ('Total',):
423                 module_total_txt[result] += ')'
424                 summarysheet.write(row, col, module_total_txt[result], form.cell)
425                 col += 1
426
427             total_txt = '=IF(%s=0,"",ROUND(%s/%s*100, 2))' % (get_cell_name(col - 1, row), get_cell_name(col - 3, row), get_cell_name(col - 1, row))
428             summarysheet.write(row, col, total_txt, form.cell)
429
430             row += 1
431
432         col = 0
433         summarysheet.write(row, col, 'Total', form.total)
434         col += 1
435
436         for platform in TEST_PLATFORM:
437                 if not platform in self.summary:
438                     continue
439
440                 for target in self.summary[platform]:
441                     for tc_type in TESTCASE_TYPES + ('Total',):
442                         for result in TC_RESULT + ('Total',):
443                             total_txt = '=SUM(%s:%s)' % (get_cell_name(col, row - len(TESTSUITE_MODULES)), get_cell_name(col, row - 1))
444                             summarysheet.write(row, col, total_txt, form.total_no)
445                             col += 1
446
447         for result in TC_RESULT + ('Total',):
448             total_txt = '=SUM(%s:%s)' % (get_cell_name(col, row - len(TESTSUITE_MODULES)), get_cell_name(col, row - 1))
449             summarysheet.write(row, col, total_txt, form.total_no)
450             col += 1
451         total_txt = '=IF(%s=0,"",ROUND(%s/%s*100, 2))' % (get_cell_name(col - 1, row), get_cell_name(col - 3, row), get_cell_name(col - 1, row))
452         summarysheet.write(row, col, total_txt, form.total_no)
453
454         def write_pass_result(row, col, testcase, testspec, sheet, platform, target, transport, network):
455             tc_col = col
456             tc_row = row
457             col += 2
458
459             index = 0
460             while(index < len(testcase.runresult)):
461                 sheet.write(row, col, testcase.runresult[index].result, form.cell)
462                 sheet.write(row, col + 1, testcase.runresult[index].runtype, form.cell)
463                 sheet.write(row, col + 2, testcase.runresult[index].runtime, form.cell)
464                 sheet.write(row, col + 3, testcase.runresult[index].fail_msg, form.cell_wrap)
465                 temp_log = get_log_content_or_filename(testcase.runresult[index].test_log)
466                 sheet.write(row, col + 4, temp_log, form.cell_wrap)
467                 index += 1
468                 row +=1
469
470             col = tc_col
471             merge_cell(sheet, tc_row, col, row - 1, col, testcase.name, form.cell)
472             col += 1
473             merge_cell(sheet, tc_row, col, row - 1, col,
474                             "%d/%d" % (testcase.fail, testcase.fail + testcase.success), form.cell)
475             col += 6
476
477             if testcase.name in testspec:
478                 spec = testspec[testcase.name]
479                 for key, title in sorted(TAG_DIC.items(), key=operator.itemgetter(1)):
480                     if (title[0] < 5):
481                         continue;
482                     merge_cell(sheet, tc_row, col, row -1, col, spec.__dict__[key], form.cell_wrap)
483                     col += 1
484
485             col = tc_col
486
487             return row, col
488
489         def write_fail_result(row, col, testcase, testspec, sheet, platform, target, transport, network):
490             tc_col = col
491             tc_row = row
492             col += 2
493
494             index = 0
495             while(index < len(testcase.runresult)):
496                 sheet.write(row, col, testcase.runresult[index].result, form.cell)
497                 sheet.write(row, col + 1, testcase.runresult[index].runtype, form.cell)
498                 sheet.write(row, col + 2, testcase.runresult[index].runtime, form.cell)
499                 sheet.write(row, col + 3, testcase.runresult[index].fail_msg, form.cell_wrap)
500                 temp_log = get_log_content_or_filename(testcase.runresult[index].test_log)
501                 sheet.write(row, col + 4, temp_log, form.cell_wrap)
502                 index += 1
503                 row +=1
504
505             col = tc_col
506             merge_cell(sheet, tc_row, col, row - 1, col, testcase.name, form.cell)
507             col += 1
508             merge_cell(sheet, tc_row, col, row - 1, col,
509                             "%d/%d" % (testcase.fail, testcase.fail + testcase.success), form.cell)
510             col += 6
511
512             if testcase.name in testspec:
513                 spec = testspec[testcase.name]
514                 for key, title in sorted(TAG_DIC.items(), key=operator.itemgetter(1)):
515                     if (title[0] < 5):
516                         continue;
517                     merge_cell(sheet, tc_row, col, row -1, col, spec.__dict__[key], form.cell_wrap)
518                     col += 1
519
520             col = tc_col
521
522             return row, col
523
524         def write_defect_result(row, col, testcase, testspec, sheet, platform, target, transport, network):
525             tc_col = col
526             tc_row = row
527             col += 2
528
529             index = 0
530             while(index < len(testcase.runresult)):
531                 sheet.write(row, col, testcase.runresult[index].result, form.cell)
532                 sheet.write(row, col + 1, testcase.runresult[index].runtype, form.cell)
533                 sheet.write(row, col + 2, testcase.runresult[index].runtime, form.cell)
534                 index += 1
535                 row +=1
536
537             col = tc_col
538             merge_cell(sheet, tc_row, col, row - 1, col, testcase.name, form.cell)
539             col += 1
540             merge_cell(sheet, tc_row, col, row - 1, col,
541                             "%d/%d" % (testcase.fail, testcase.fail + testcase.success), form.cell)
542
543             col += 4
544             merge_cell(sheet, tc_row, col, row-1, col, '', form.cell_wrap)
545             col += 1
546             merge_cell(sheet, tc_row, col, row-1, col, '', form.cell)
547             col += 1
548
549             test_target = ''
550             if testcase.name in list(testspec):
551                 test_target = testspec[testcase.name].target
552
553             description = '[Device: %s %s]\n' %(platform, target)\
554                         + '[TC: %s]\n' % (testcase.name)\
555                         + '[Target: %s]\n\n' %(test_target) \
556                         + DEFECT_DESCRIPTION
557             merge_cell(sheet, tc_row, col, row-1, col, description, form.cell_wrap)
558
559             col = tc_col
560
561             return row, col
562
563         def get_log_content_or_filename(log):
564             if len(log) > 10000:
565                 begin_index = log.find('Log File Name:') + len('Log File Name:')
566                 end_index = log.find('Content:')
567                 log = log[begin_index:end_index].strip()
568             return log
569
570         passsheet = workbook.add_worksheet('PassTC')
571         self.report_result(passsheet, form, RESULT_TITLE, write_pass_result, self.passtc)
572
573         failsheet = workbook.add_worksheet('FailedTC')
574         self.report_result(failsheet, form, RESULT_TITLE, write_fail_result, self.failtc)
575
576         defectsheet = workbook.add_worksheet('DefectReport')
577         self.report_result(defectsheet, form, DEFECT_TITLE, write_defect_result, self.failtc)
578
579         workbook.close()