test/test_manager/ite/reporter/result_reporter.py

   1 '''
   2 /******************************************************************
   3 *
   4 * Copyright 2018 Samsung Electronics All Rights Reserved.
   5 *
   6 *
   7 *
   8 * Licensed under the Apache License, Version 2.0 (the "License");
   9 * you may not use this file except in compliance with the License.
  10 * You may obtain a copy of the License at
  11 *
  12 * http://www.apache.org/licenses/LICENSE-2.0
  13 *
  14 * Unless required by applicable law or agreed to in writing, software
  15 * distributed under the License is distributed on an "AS IS" BASIS,
  16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 * See the License for the specific language governing permissions and
  18 * limitations under the License.
  19 *
  20 ******************************************************************/
  21 '''
  22
  23 import sys
  24 import operator
  25 import os
  26
  27 from xlsxwriter.workbook import Workbook
  28
  29 from ite.config import *
  30 from ite.constants import *
  31 from ite.result.collector import TestResultCollector
  32 from ite.tc.container import TestSpecContainer
  33 from ite.reporter.excel_util import *
  34 from ite.reporter.reporter_util import *
  35 from ite.util import *
  36
  37 class TestResultReporter:
  38
  39
  40     def __init__(self):
  41         self.summary = dict()
  42         self.failtc = dict()
  43         self.passtc = dict()
  44         self.testspec = None
  45
  46     def make_summary(self, data):
  47         for platform in TEST_PLATFORM:
  48             if not platform in data:
  49                 continue
  50
  51             self.summary[platform] = dict()
  52             self.failtc[platform] = dict()
  53             self.passtc[platform] = dict()
  54
  55             for target in data[platform]:
  56                 self.summary[platform][target] = dict()
  57                 self.failtc[platform][target] = dict()
  58                 self.passtc[platform][target] = dict()
  59
  60                 for build_type in data[platform][target]:
  61                     self.summary[platform][target][build_type] = dict()
  62                     self.failtc[platform][target][build_type] = dict()
  63                     self.passtc[platform][target][build_type] = dict()
  64
  65                     for transport in data[platform][target][build_type]:
  66                         self.summary[platform][target][build_type][transport] = dict()
  67                         self.failtc[platform][target][build_type][transport] = dict()
  68                         self.passtc[platform][target][build_type][transport] = dict()
  69
  70                         for network in data[platform][target][build_type][transport]:
  71                             self.summary[platform][target][build_type][transport][network] = dict()
  72                             self.failtc[platform][target][build_type][transport][network] = dict()
  73                             self.passtc[platform][target][build_type][transport][network] = dict()
  74
  75                             for tctype in TESTCASE_TYPES:
  76                                 self.summary[platform][target][build_type][transport][network][tctype] = dict()
  77                                 self.failtc[platform][target][build_type][transport][network][tctype] = dict()
  78                                 self.passtc[platform][target][build_type][transport][network][tctype] = dict()
  79
  80                                 for module in TESTSUITE_MODULES:
  81                                     self.summary[platform][target][build_type][transport][network][tctype][module] = dict()
  82                                     self.failtc[platform][target][build_type][transport][network][tctype][module] = dict()
  83                                     self.passtc[platform][target][build_type][transport][network][tctype][module] = dict()
  84
  85                                     for result in TC_RESULT:
  86                                         self.summary[platform][target][build_type][transport][network][tctype][module][result] = 0
  87
  88                                     if ((not tctype in list(data[platform][target][build_type][transport][network])) or (not module in list(data[platform][target][build_type][transport][network][tctype]))):
  89                                         continue
  90
  91                                     for suite in list(data[platform][target][build_type][transport][network][tctype][module]):
  92                                         self.failtc[platform][target][build_type][transport][network][tctype][module][suite] = list()
  93                                         self.passtc[platform][target][build_type][transport][network][tctype][module][suite] = list()
  94
  95                                         for tc_name in list(data[platform][target][build_type][transport][network][tctype][module][suite]):
  96                                             testcase = data[platform][target][build_type][transport][network][tctype][module][suite][tc_name]
  97                                             if (testcase.success == 0):
  98                                                 self.summary[platform][target][build_type][transport][network][tctype][module][TC_RESULT.FAIL] += 1
  99                                                 self.failtc[platform][target][build_type][transport][network][tctype][module][suite].append(testcase)
 100                                             else:
 101                                                 self.summary[platform][target][build_type][transport][network][tctype][module][TC_RESULT.PASS] += 1
 102                                                 self.passtc[platform][target][build_type][transport][network][tctype][module][suite].append(testcase)
 103
 104
 105     def generate_testresult_report(self, path, spec_data):
 106         collector = TestResultCollector()
 107         collector.collect_results(path)
 108
 109         self.testspec = spec_data
 110
 111         self.make_summary(collector.result.data)
 112
 113     def report_fail_result_to_txt(self, dir_path):
 114         self.report_test_result_to_txt(dir_path, self.failtc)
 115
 116     def report_pass_result_to_txt(self, dir_path):
 117         self.report_test_result_to_txt(dir_path, self.passtc)
 118
 119     def report_test_result_to_txt(self, dir_path, result_suite):
 120         for platform in TEST_PLATFORM:
 121             if not platform in result_suite:
 122                 continue
 123
 124             for target in result_suite[platform]:
 125                 for build_type in  result_suite[platform][target]:
 126                     for transport in result_suite[platform][target][build_type]:
 127                         transport_name = transport
 128                         if transport_name == NO_TRANSPORT:
 129                             transport_name = ''
 130                         for network in result_suite[platform][target][build_type][transport]:
 131                             network_name = network
 132                             if network_name == NO_NETWORK:
 133                                 network_name = ''
 134                             new_dir = os.path.join(dir_path, '%s_%s_%s_%s_%s' %(platform, target.strip(), build_type, transport_name, network_name))
 135                             os.makedirs(new_dir)
 136                             for tctype in result_suite[platform][target][build_type][transport][network]:
 137                                 for module in result_suite[platform][target][build_type][transport][network][tctype]:
 138                                     for suite in list(result_suite[platform][target][build_type][transport][network][tctype][module]):
 139                                         for testcase in list(result_suite[platform][target][build_type][transport][network][tctype][module][suite]):
 140                                             text_file_name = suite
 141                                             if not ('_' + tctype.lower()) in suite.lower():
 142                                                 text_file_name += '_' + tctype.lower()
 143                                             text_file_name += '_' + testcase.name + '_' + build_type + '.txt'
 144                                             file_path = os.path.join(new_dir, text_file_name)
 145                                             txt = open_write_file(file_path)
 146                                             if txt == False:
 147                                                 continue
 148                                             print("[Defect Info]", file=txt)
 149                                             print(" Title: ", file=txt)
 150                                             print(" Description: \n", file=txt)
 151
 152                                             if platform in self.testspec:
 153                                                 if tctype in self.testspec[platform]:
 154                                                     if module in self.testspec[platform][tctype]:
 155                                                         if transport in self.testspec[platform][tctype][module]:
 156                                                             if network in self.testspec[platform][tctype][module][transport]:
 157                                                                 if suite in self.testspec[platform][tctype][module][transport][network]:
 158                                                                     if testcase.name in self.testspec[platform][tctype][module][transport][network][suite]:
 159                                                                         spec = self.testspec[platform][tctype][module][transport][network][suite][testcase.name]
 160                                                                         print("[Test Case Info]", file=txt)
 161                                                                         print(spec.to_string(), file=txt)
 162
 163                                             print("\n[Test Result]", file=txt)
 164                                             print("Fail Rate: %d/%d\n" % (testcase.fail, testcase.fail+testcase.success), file=txt)
 165
 166                                             index = 1
 167                                             for result in testcase.runresult:
 168                                                 print("[%d Try]" % index, file=txt)
 169                                                 print("--------------------------------------------------------------------", file=txt)
 170                                                 print("Result: " + result.result, file=txt)
 171                                                 print("Run Type: " + result.runtype, file=txt)
 172                                                 print("Run Time: " + str(result.runtime), file=txt)
 173                                                 print("\n<<<Fail Message>>> \n" + result.fail_msg, file=txt)
 174                                                 print("\n<<<Test Log>>> \n" + result.test_log, file=txt)
 175                                                 print("--------------------------------------------------------------------\n\n", file=txt)
 176                                                 index += 1
 177
 178     def report_result(self, sheet, form, report_title, result_writer, result_suite):
 179         row = 0
 180         col = 0
 181         for title, size in RESULT_CAT_TITLE:
 182             if (title == 'Line'):
 183                 continue
 184             sheet.write(row, col, title, form.title)
 185             sheet.set_column(col, col, size)
 186             col += 1
 187
 188         for title, size in report_title:
 189             sheet.write(row, col, title, form.title)
 190             sheet.set_column(col, col, size)
 191             col += 1
 192
 193         row += 1
 194         col = 0
 195
 196         def write_platform(platforms):
 197             for platform in platforms:
 198                 nonlocal row, col
 199                 platform_first_row = row
 200
 201                 if not platform in result_suite:
 202                     continue
 203
 204                 col += 1
 205
 206                 yield result_suite[platform], platform
 207
 208                 col -= 1
 209                 merge_cell(sheet, platform_first_row, col, row-1, col, platform, form.cell)
 210
 211         def write_target(platforms):
 212             for platform in platforms:
 213                 targets, platform_name = platform
 214                 for target in list(targets):
 215                     nonlocal row, col
 216                     target_first_row = row
 217
 218                     col += 1
 219
 220                     yield targets[target], platform_name, target
 221
 222                     col -= 1
 223                     merge_cell(sheet, target_first_row, col, row-1, col, target, form.cell)
 224
 225         def write_build_type(targets):
 226             for target in targets:
 227                 build_types, platform, target = target
 228                 for build_type in list(build_types):
 229                     nonlocal row, col
 230                     build_type_first_row = row
 231
 232                     col += 1
 233
 234                     yield build_types[build_type], platform, target, build_type
 235
 236                     col -= 1
 237                     merge_cell(sheet, build_type_first_row, col, row-1, col, build_type, form.cell)
 238
 239         def write_transport(build_types):
 240             for build_type in build_types:
 241                 transports, platform, target, build_type = build_type
 242                 for transport in list(transports):
 243                     nonlocal row, col
 244                     transport_first_row = row
 245
 246                     col += 1
 247
 248                     yield transports[transport], platform, target, build_type, transport
 249
 250                     col -= 1
 251                     merge_cell(sheet, transport_first_row, col, row-1, col, transport, form.cell)
 252
 253
 254         def write_network(transports):
 255             for transport in transports:
 256                 networks, platform, target, build_type, transport = transport
 257                 for network in list(networks):
 258                     nonlocal row, col
 259                     network_first_row = row
 260
 261                     col += 1
 262
 263                     yield networks[network], platform, target, build_type, transport, network
 264
 265                     col -= 1
 266                     merge_cell(sheet, network_first_row, col, row-1, col, network, form.cell)
 267
 268
 269         def write_tctype(networks):
 270             for network in networks:
 271                 types, platform, target, build_type, transport, network = network
 272                 for tctype in list(types):
 273                     nonlocal row, col
 274                     type_first_row = row
 275
 276                     module_count = 0
 277                     for module in TESTSUITE_MODULES:
 278                         module_count += len(types[tctype][module])
 279                     if module_count == 0:
 280                         continue
 281
 282                     col += 1
 283
 284                     yield TESTSUITE_MODULES, types[tctype], platform, target, build_type, transport, network, tctype
 285
 286                     col -= 1
 287                     merge_cell(sheet, type_first_row, col, row-1, col, tctype, form.cell)
 288
 289         def write_module(tctypes):
 290             for tctype in tctypes:
 291                 module_names, modules, platform, target, build_type, transport, network, tctype = tctype
 292                 for module in module_names:
 293                     nonlocal row, col
 294                     module_first_row = row
 295
 296                     if (len(list(modules[module])) == 0):
 297                         continue
 298
 299                     col += 1
 300
 301                     yield modules[module], platform, target, build_type, transport, network, tctype, module
 302
 303                     col -= 1
 304                     merge_cell(sheet, module_first_row, col, row-1, col, module, form.cell)
 305
 306         def write_suite(modules):
 307             for module in modules:
 308                 suites, platform, target, build_type, transport, network, tctype, module = module
 309                 for suite in list(suites):
 310                     nonlocal row, col
 311                     suite_first_row = row
 312
 313                     if (len(suites[suite]) == 0):
 314                         continue
 315
 316                     if not suite in suites and not suite in self.testspec[platform][build_type][tctype][module][transport][network]:
 317                         print ('suite not found: ', platform, tctype, module, transport, network, suite)
 318                         continue
 319
 320                     col += 1
 321
 322                     yield suites[suite], platform, target, build_type, transport, network, tctype, module, suite
 323
 324                     col -=1
 325                     merge_cell(sheet, suite_first_row, col, row-1, col, suite, form.cell)
 326
 327         def write_tc(suites):
 328             for suite in suites:
 329                 testcases, platform, target, build_type, transport, network, tctype, module, suite = suite
 330                 testspec = self.testspec[platform][build_type][tctype][module][transport][network][suite]
 331                 for testcase in testcases:
 332                     nonlocal row, col
 333                     row, col = result_writer(row, col, testcase, testspec, sheet, platform, target, transport, network)
 334
 335
 336         walk_through_results(write_platform, write_target, write_build_type, write_transport, write_network, write_tctype, write_module, write_suite, write_tc)
 337
 338
 339     def report_to_xlsx(self, path):
 340         workbook = Workbook(path)
 341         form = ExcelFormat(workbook)
 342
 343         summarysheet = workbook.add_worksheet('Summary')
 344
 345         row = 0
 346         col = 0
 347
 348         summarysheet.merge_range(row, col, row+3, col, '', form.cell)
 349
 350         col += 1
 351         for platform in TEST_PLATFORM:
 352             if not platform in self.summary:
 353                 continue
 354
 355             row += 1
 356             platform_col = col
 357
 358             for target in self.summary[platform]:
 359                 row += 1
 360                 target_col = col
 361
 362                 for tc_type in TESTCASE_TYPES + ('Total',):
 363                     row += 1
 364                     tc_type_col = col
 365
 366                     for result in TC_RESULT + ('Total',):
 367                         summarysheet.write(row, col, result, form.result_title[result])
 368                         col += 1
 369
 370                     row -= 1
 371                     summarysheet.merge_range(row, tc_type_col, row, col-1, tc_type, form.title)
 372
 373                 row -= 1
 374                 summarysheet.merge_range(row, target_col, row, col-1, target, form.title)
 375
 376                 row -= 1
 377                 summarysheet.merge_range(row, platform_col, row, col-1, platform, form.title)
 378
 379         total_col = col
 380         for result in TC_RESULT + ('Total','Pass Rate',):
 381             summarysheet.write(row+3, col, result, form.result_title[result])
 382             col += 1
 383         summarysheet.merge_range(row, total_col, row+2, col-1, "Total", form.total)
 384
 385         row += 4
 386         col = 0
 387
 388         for module in TESTSUITE_MODULES :
 389             col = 0
 390             summarysheet.write(row, col, module, form.title)
 391             col += 1
 392
 393             module_total_txt = dict()
 394             for result in TC_RESULT + ('Total',):
 395                 module_total_txt[result] = '=SUM('
 396
 397             for platform in TEST_PLATFORM:
 398                 if not platform in self.summary:
 399                     continue
 400
 401                 for target in self.summary[platform]:
 402                     for tc_type in TESTCASE_TYPES:
 403                         for result in TC_RESULT:
 404                             result_sum = 0
 405                             for build_type in self.summary[platform][target]:
 406                                 for transport in self.summary[platform][target][build_type]:
 407                                     for network in self.summary[platform][target][build_type][transport]:
 408                                         result_sum += self.summary[platform][target][build_type][transport][network][tc_type][module][result]
 409                             summarysheet.write(row, col, result_sum, form.cell)
 410                             col += 1
 411
 412                         total_txt = '=SUM(%s:%s)' % (get_cell_name(col - len(TC_RESULT), row), get_cell_name(col - 1, row))
 413                         summarysheet.write(row, col, total_txt, form.cell)
 414                         col += 1
 415
 416                     for result in TC_RESULT + ('Total',):
 417                                 total_txt = '=SUM(%s,%s)' % (get_cell_name(col - (len(TC_RESULT) + 1)*2, row), get_cell_name(col - (len(TC_RESULT) + 1), row))
 418                                 summarysheet.write(row, col, total_txt, form.cell)
 419                                 module_total_txt[result] += '%s,' % get_cell_name(col, row)
 420                                 col += 1
 421
 422             for result in TC_RESULT + ('Total',):
 423                 module_total_txt[result] += ')'
 424                 summarysheet.write(row, col, module_total_txt[result], form.cell)
 425                 col += 1
 426
 427             total_txt = '=IF(%s=0,"",ROUND(%s/%s*100, 2))' % (get_cell_name(col - 1, row), get_cell_name(col - 3, row), get_cell_name(col - 1, row))
 428             summarysheet.write(row, col, total_txt, form.cell)
 429
 430             row += 1
 431
 432         col = 0
 433         summarysheet.write(row, col, 'Total', form.total)
 434         col += 1
 435
 436         for platform in TEST_PLATFORM:
 437                 if not platform in self.summary:
 438                     continue
 439
 440                 for target in self.summary[platform]:
 441                     for tc_type in TESTCASE_TYPES + ('Total',):
 442                         for result in TC_RESULT + ('Total',):
 443                             total_txt = '=SUM(%s:%s)' % (get_cell_name(col, row - len(TESTSUITE_MODULES)), get_cell_name(col, row - 1))
 444                             summarysheet.write(row, col, total_txt, form.total_no)
 445                             col += 1
 446
 447         for result in TC_RESULT + ('Total',):
 448             total_txt = '=SUM(%s:%s)' % (get_cell_name(col, row - len(TESTSUITE_MODULES)), get_cell_name(col, row - 1))
 449             summarysheet.write(row, col, total_txt, form.total_no)
 450             col += 1
 451         total_txt = '=IF(%s=0,"",ROUND(%s/%s*100, 2))' % (get_cell_name(col - 1, row), get_cell_name(col - 3, row), get_cell_name(col - 1, row))
 452         summarysheet.write(row, col, total_txt, form.total_no)
 453
 454         def write_pass_result(row, col, testcase, testspec, sheet, platform, target, transport, network):
 455             tc_col = col
 456             tc_row = row
 457             col += 2
 458
 459             index = 0
 460             while(index < len(testcase.runresult)):
 461                 sheet.write(row, col, testcase.runresult[index].result, form.cell)
 462                 sheet.write(row, col + 1, testcase.runresult[index].runtype, form.cell)
 463                 sheet.write(row, col + 2, testcase.runresult[index].runtime, form.cell)
 464                 sheet.write(row, col + 3, testcase.runresult[index].fail_msg, form.cell_wrap)
 465                 temp_log = get_log_content_or_filename(testcase.runresult[index].test_log)
 466                 sheet.write(row, col + 4, temp_log, form.cell_wrap)
 467                 index += 1
 468                 row +=1
 469
 470             col = tc_col
 471             merge_cell(sheet, tc_row, col, row - 1, col, testcase.name, form.cell)
 472             col += 1
 473             merge_cell(sheet, tc_row, col, row - 1, col,
 474                             "%d/%d" % (testcase.fail, testcase.fail + testcase.success), form.cell)
 475             col += 6
 476
 477             if testcase.name in testspec:
 478                 spec = testspec[testcase.name]
 479                 for key, title in sorted(TAG_DIC.items(), key=operator.itemgetter(1)):
 480                     if (title[0] < 5):
 481                         continue;
 482                     merge_cell(sheet, tc_row, col, row -1, col, spec.__dict__[key], form.cell_wrap)
 483                     col += 1
 484
 485             col = tc_col
 486
 487             return row, col
 488
 489         def write_fail_result(row, col, testcase, testspec, sheet, platform, target, transport, network):
 490             tc_col = col
 491             tc_row = row
 492             col += 2
 493
 494             index = 0
 495             while(index < len(testcase.runresult)):
 496                 sheet.write(row, col, testcase.runresult[index].result, form.cell)
 497                 sheet.write(row, col + 1, testcase.runresult[index].runtype, form.cell)
 498                 sheet.write(row, col + 2, testcase.runresult[index].runtime, form.cell)
 499                 sheet.write(row, col + 3, testcase.runresult[index].fail_msg, form.cell_wrap)
 500                 temp_log = get_log_content_or_filename(testcase.runresult[index].test_log)
 501                 sheet.write(row, col + 4, temp_log, form.cell_wrap)
 502                 index += 1
 503                 row +=1
 504
 505             col = tc_col
 506             merge_cell(sheet, tc_row, col, row - 1, col, testcase.name, form.cell)
 507             col += 1
 508             merge_cell(sheet, tc_row, col, row - 1, col,
 509                             "%d/%d" % (testcase.fail, testcase.fail + testcase.success), form.cell)
 510             col += 6
 511
 512             if testcase.name in testspec:
 513                 spec = testspec[testcase.name]
 514                 for key, title in sorted(TAG_DIC.items(), key=operator.itemgetter(1)):
 515                     if (title[0] < 5):
 516                         continue;
 517                     merge_cell(sheet, tc_row, col, row -1, col, spec.__dict__[key], form.cell_wrap)
 518                     col += 1
 519
 520             col = tc_col
 521
 522             return row, col
 523
 524         def write_defect_result(row, col, testcase, testspec, sheet, platform, target, transport, network):
 525             tc_col = col
 526             tc_row = row
 527             col += 2
 528
 529             index = 0
 530             while(index < len(testcase.runresult)):
 531                 sheet.write(row, col, testcase.runresult[index].result, form.cell)
 532                 sheet.write(row, col + 1, testcase.runresult[index].runtype, form.cell)
 533                 sheet.write(row, col + 2, testcase.runresult[index].runtime, form.cell)
 534                 index += 1
 535                 row +=1
 536
 537             col = tc_col
 538             merge_cell(sheet, tc_row, col, row - 1, col, testcase.name, form.cell)
 539             col += 1
 540             merge_cell(sheet, tc_row, col, row - 1, col,
 541                             "%d/%d" % (testcase.fail, testcase.fail + testcase.success), form.cell)
 542
 543             col += 4
 544             merge_cell(sheet, tc_row, col, row-1, col, '', form.cell_wrap)
 545             col += 1
 546             merge_cell(sheet, tc_row, col, row-1, col, '', form.cell)
 547             col += 1
 548
 549             test_target = ''
 550             if testcase.name in list(testspec):
 551                 test_target = testspec[testcase.name].target
 552
 553             description = '[Device: %s %s]\n' %(platform, target)\
 554                         + '[TC: %s]\n' % (testcase.name)\
 555                         + '[Target: %s]\n\n' %(test_target) \
 556                         + DEFECT_DESCRIPTION
 557             merge_cell(sheet, tc_row, col, row-1, col, description, form.cell_wrap)
 558
 559             col = tc_col
 560
 561             return row, col
 562
 563         def get_log_content_or_filename(log):
 564             if len(log) > 10000:
 565                 begin_index = log.find('Log File Name:') + len('Log File Name:')
 566                 end_index = log.find('Content:')
 567                 log = log[begin_index:end_index].strip()
 568             return log
 569
 570         passsheet = workbook.add_worksheet('PassTC')
 571         self.report_result(passsheet, form, RESULT_TITLE, write_pass_result, self.passtc)
 572
 573         failsheet = workbook.add_worksheet('FailedTC')
 574         self.report_result(failsheet, form, RESULT_TITLE, write_fail_result, self.failtc)
 575
 576         defectsheet = workbook.add_worksheet('DefectReport')
 577         self.report_result(defectsheet, form, DEFECT_TITLE, write_defect_result, self.failtc)
 578
 579         workbook.close()