Merge test folder of test branch
[iotivity.git] / test / test_manager / test_runner.py
1 #!/usr/bin/python3
2 '''
3 /******************************************************************
4 *
5 * Copyright 2018 Samsung Electronics All Rights Reserved.
6 *
7 *
8 *
9 * Licensed under the Apache License, Version 2.0 (the "License");
10 * you may not use this file except in compliance with the License.
11 * You may obtain a copy of the License at
12 *
13 * http://www.apache.org/licenses/LICENSE-2.0
14 *
15 * Unless required by applicable law or agreed to in writing, software
16 * distributed under the License is distributed on an "AS IS" BASIS,
17 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 * See the License for the specific language governing permissions and
19 * limitations under the License.
20 *
21 ******************************************************************/
22 '''
23 import sys
24 import os
25 import optparse
26 import subprocess
27 import re
28 import platform
29 import configuration
30 from configuration import *
31 from ite.exec.runner import TestRunner, TestRunnerOption
32 from ite.reporter.tc_reporter import TestSpecReporter
33 from ite.webreporter.tc_list_reporter import TCListReporter
34 from ite.tc.container import TestSpecContainer
35
36 oparser = optparse.OptionParser()
37 oparser.add_option("-f", action="store", dest="file_filter")
38 oparser.add_option("--filter", action="store", dest="file_filter")
39 oparser.add_option("-m", action="store", dest="module")
40 oparser.add_option("--module", action="store", dest="module")
41 oparser.add_option("-p", action="store", dest="platform")
42 oparser.add_option("--platform", action="store", dest="platform")
43 oparser.add_option("-x", action="store", dest="transport", default="")
44 oparser.add_option("--transport", action="store", dest="transport", default="")
45 oparser.add_option("-n", action="store", dest="network", default="")
46 oparser.add_option("--network", action="store", dest="network", default="")
47 oparser.add_option("-t", action="store", dest="target")
48 oparser.add_option("--target", action="store", dest="target")
49 oparser.add_option("-l", action="store", dest="testlist")
50 oparser.add_option("--list", action="store", dest="testlist")
51 oparser.add_option("-s", action="store", dest="testprogress")
52 oparser.add_option("--progress", action="store", dest="testprogress")
53 oparser.add_option("-r", action="store", dest="testresult")
54 oparser.add_option("--result", action="store", dest="testresult")
55 oparser.add_option("-a", action="store_true", dest="standalone")
56 oparser.add_option("--standalone", action="store_true", dest="standalone")
57 oparser.add_option("-o", action="store_true", dest="runonce")
58 oparser.add_option("--runonce", action="store_true", dest="runonce")
59 oparser.add_option("-u", action="store", dest="testsuite")
60 oparser.add_option("--testsuite", action="store", dest="testsuite")
61 oparser.add_option("-c", action="store", dest="testcase")
62 oparser.add_option("--testcase", action="store", dest="testcase")
63
64
65 oparser.add_option("--memcheck", action = "store", dest = "memcheck", default= "0")
66 oparser.add_option("-v", action= "store", dest= "memcheck", default="0")
67 oparser.add_option("--memcheckLocation", action="store", dest ="memcheckLocation", default = "")
68 oparser.add_option("-g", action="store", dest ="memcheckLocation", default = "")
69
70 oparser.set_defaults(file_filter='', platform=platform.system().lower(), target='', testlist='', testprogress='', testresult='',
71                      standalone=TEST_STANDALONE, runonce=False)
72
73 opts, args = oparser.parse_args()
74 file_filter = opts.file_filter
75 module = opts.module
76 platform = opts.platform
77 transport = opts.transport
78 network = opts.network
79 target = opts.target
80 testlist = opts.testlist
81 testprogress = opts.testprogress
82 testresult = opts.testresult
83 standalone = opts.standalone
84 runonce = opts.runonce
85 given_testsuite = opts.testsuite
86 memcheckOn = opts.memcheck
87 memcheckLocation = opts.memcheckLocation
88
89 if testresult == '':
90     testresult = TEST_RESULT_RUN_DIR
91
92 if not os.path.exists(testresult):
93     os.makedirs(testresult)
94
95 if not testlist == '':
96     list_analyzer = TCListReporter()
97     testgroup = list_analyzer.analyze(testlist)
98
99 if memcheckOn == "1":
100     if memcheckLocation =='':
101         memcheckLocation = MEMCHECK_RESULT_RUN_DIR
102     if not os.path.exists(memcheckLocation):
103         os.makedirs(memcheckLocation)
104
105 testspec_path = os.path.join(testresult, TEST_SPEC_XML_FOR_RESULT)
106 if not os.path.exists(testspec_path) and os.path.exists(API_TC_SRC_DIR):
107     container = TestSpecContainer()
108     container.extract_api_testspec(API_TC_SRC_DIR, '')
109
110     reporter = TestSpecReporter()
111     reporter.generate_testspec_report(container.data)
112     reporter.report('XML', testspec_path)
113
114 configuration.EXECUTION_LOG_FP = open(testresult + os.sep + EXECUTION_LOG_FILE, "w")
115
116 file_filter = file_filter.replace(TC_BIN_PREFIX, '')
117 total_found_tc = 0
118 runner = TestRunner()
119 for fname in os.listdir(TC_BIN_DIR):
120     if "." in fname:
121         if fname.endswith('.exe'):
122             fname = fname.split('.')[0]
123         else:
124             continue
125
126     if not fname.startswith(TC_BIN_PREFIX):
127         continue
128
129     if not fname.endswith(TC_BIN_SUFFIX):
130         continue
131
132     if module and module != fname.split('_')[1]:
133         continue
134
135     if file_filter and file_filter not in fname:
136         continue
137
138     tname = fname.replace(TC_BIN_PREFIX, '')
139
140     option = TestRunnerOption()
141
142     current_filter = ''
143     if not testlist == '':
144         for tcfile_filter in list(testgroup):
145             if tcfile_filter in fname:
146                 current_filter = tcfile_filter
147                 break
148
149         if current_filter == '':
150             continue
151
152         option.testset = set(testgroup[current_filter].keys())
153         option.testkey = testgroup[current_filter]
154
155     elif (file_filter != '') and (not file_filter in tname):
156         continue
157
158     filepath = os.path.join(TC_BIN_DIR, fname)
159
160     print("### Start to run : " + filepath)
161
162     if given_testsuite:
163         given_testsuites = opts.testsuite.split(',')
164     else:
165         given_testsuites = []
166
167     if opts.testcase:
168         given_testcases = opts.testcase.split(',')
169     else:
170         given_testcases = []
171
172     if option.testset == None:
173         if platform == 'linux':
174             command = ["./%s" % os.path.basename(filepath), '--gtest_list_tests']
175             process = subprocess.Popen(command, cwd=os.path.dirname(filepath), stdout=subprocess.PIPE)
176         if platform == 'windows':
177             filepath = filepath+'.exe'
178             command = ["%s" % os.path.basename(filepath), '--gtest_list_tests']
179             process = subprocess.Popen(command, cwd=os.path.dirname(filepath), stdout=subprocess.PIPE, shell= True)
180         testset = set()
181         testsuite = ''
182
183         for line in process.stdout.readlines():
184             line = re.sub(r'(b\'|\')', '', str(line.strip()))
185             if line.endswith('.'):
186                 testsuite = line
187                 continue
188             if given_testsuites:
189                 for ts in given_testsuites:
190
191                     if testsuite != (ts+'.'):
192                         continue
193
194                     if (testsuite != '' and line != ''):
195                         if not given_testcases:
196                             testset.add(testsuite +line)
197                         else:
198                             for tc in given_testcases:
199                                 if ( line == tc ):
200                                     testset.add(testsuite +line)
201             elif given_testcases:
202                 for tc in given_testcases:
203                     if ( line == tc):
204                         testset.add(testsuite +line)
205             else:
206                 testset.add(testsuite +line)
207
208     if not testset:
209         continue
210
211     total_found_tc = total_found_tc + len(testset)
212
213     option.testset = testset
214     option.exe_path = filepath
215     option.platform = platform
216     option.target = target
217     option.transport = transport
218     option.network = network
219     option.result_dir = testresult
220     option.runtime = TEST_REPEAT
221     option.run_standalone = standalone
222     option.testprogress_path = testprogress
223     option.repeat_failed = TEST_REPEAT_FAILED
224     option.repeat_crashed = TEST_REPEAT_CRASHED
225     if runonce == True:
226         option.repeat_failed = False
227         option.repeat_crashed = False
228
229     runner.run_test_executable(option)
230
231 if total_found_tc == 0:
232     print ('No testcase Found')
233     exit(0)
234
235 print("### Test Is Done!!")