31 """Unit test for Google Test test filters.
33 A user can specify which test(s) in a Google Test program to run via either
34 the GTEST_FILTER environment variable or the --gtest_filter flag.
35 This script tests such functionality by invoking
36 gtest_filter_unittest_ (a program written with Google Test) with different
37 environments and command line flags.
39 Note that test sharding may also influence which tests are filtered. Therefore,
40 we test that here also.
43 __author__ =
'wan@google.com (Zhanyong Wan)'
48 from sets
import Set
as set
53 import gtest_test_utils
62 os.environ[
'EMPTY_VAR'] =
''
64 [sys.executable,
'-c',
'import os; print(\'EMPTY_VAR\' in os.environ)'])
65 CAN_PASS_EMPTY_ENV = eval(child.output)
74 os.environ[
'UNSET_VAR'] =
'X'
75 del os.environ[
'UNSET_VAR']
77 [sys.executable,
'-c',
'import os; print(\'UNSET_VAR\' not in os.environ)'])
78 CAN_UNSET_ENV = eval(child.output)
85 CAN_TEST_EMPTY_FILTER = (CAN_PASS_EMPTY_ENV
and CAN_UNSET_ENV)
89 FILTER_ENV_VAR =
'GTEST_FILTER'
92 TOTAL_SHARDS_ENV_VAR =
'GTEST_TOTAL_SHARDS'
93 SHARD_INDEX_ENV_VAR =
'GTEST_SHARD_INDEX'
94 SHARD_STATUS_FILE_ENV_VAR =
'GTEST_SHARD_STATUS_FILE'
97 FILTER_FLAG =
'gtest_filter'
100 ALSO_RUN_DISABED_TESTS_FLAG =
'gtest_also_run_disabled_tests'
106 PARAM_TEST_REGEX = re.compile(
r'/ParamTest')
109 TEST_CASE_REGEX = re.compile(
r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)')
112 TEST_REGEX = re.compile(
r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)')
116 LIST_TESTS_FLAG =
'--gtest_list_tests'
120 [COMMAND, LIST_TESTS_FLAG]).output
124 'SeqP/ParamTest.TestX/0',
125 'SeqP/ParamTest.TestX/1',
126 'SeqP/ParamTest.TestY/0',
127 'SeqP/ParamTest.TestY/1',
128 'SeqQ/ParamTest.TestX/0',
129 'SeqQ/ParamTest.TestX/1',
130 'SeqQ/ParamTest.TestY/0',
131 'SeqQ/ParamTest.TestY/1',
135 'BarTest.DISABLED_TestFour',
136 'BarTest.DISABLED_TestFive',
137 'BazTest.DISABLED_TestC',
138 'DISABLED_FoobarTest.Test1',
139 'DISABLED_FoobarTest.DISABLED_Test2',
140 'DISABLED_FoobarbazTest.TestA',
143 if SUPPORTS_DEATH_TESTS:
145 'HasDeathTest.Test1',
146 'HasDeathTest.Test2',
163 ] + DEATH_TESTS + PARAM_TESTS
165 param_tests_present =
None
169 environ = os.environ.copy()
173 """Sets the env variable to 'value'; unsets it when 'value' is None."""
175 if value
is not None:
176 environ[env_var] = value
177 elif env_var
in environ:
182 """Runs the test program and returns its output."""
189 """Runs the test program and returns its exit code and a list of tests run."""
195 for line
in p.output.split(
'\n'):
196 match = TEST_CASE_REGEX.match(line)
197 if match
is not None:
198 test_case = match.group(1)
200 match = TEST_REGEX.match(line)
201 if match
is not None:
202 test = match.group(1)
203 tests_run.append(test_case +
'.' + test)
204 return (tests_run, p.exit_code)
208 """Runs the given function and arguments in a modified environment."""
210 original_env = environ.copy()
211 environ.update(extra_env)
212 return function(*args, **kwargs)
215 environ.update(original_env)
219 """Runs a test program shard and returns exit code and a list of tests run."""
221 extra_env = {SHARD_INDEX_ENV_VAR:
str(shard_index),
222 TOTAL_SHARDS_ENV_VAR:
str(total_shards)}
229 """Tests the env variable or the command line flag to filter tests."""
234 """Asserts that two sets are equal."""
237 self.assert_(elem
in rhs,
'%s in %s' % (elem, rhs))
240 self.assert_(elem
in lhs,
'%s in %s' % (elem, lhs))
243 """Asserts that list_of_sets is a valid partition of set_var."""
246 for slice_var
in list_of_sets:
247 full_partition.extend(slice_var)
248 self.assertEqual(len(set_var), len(full_partition))
249 self.assertEqual(
set(set_var),
set(full_partition))
252 """Adjust tests_to_run in case value parameterized tests are disabled."""
254 global param_tests_present
255 if not param_tests_present:
256 return list(
set(tests_to_run) -
set(PARAM_TESTS))
261 """Checks that the binary runs correct set of tests for a given filter."""
273 if CAN_TEST_EMPTY_FILTER
or gtest_filter !=
'':
282 if gtest_filter
is None:
285 args = [
'--%s=%s' % (FILTER_FLAG, gtest_filter)]
291 args=
None, check_exit_0=
False):
292 """Checks that binary runs correct tests for the given filter and shard.
294 Runs all shards of gtest_filter_unittest_ with the given filter, and
295 verifies that the right set of tests were run. The union of tests run
296 on each shard should be identical to tests_to_run, without duplicates.
299 gtest_filter: A filter to apply to the tests.
300 total_shards: A total number of shards to split test run into.
301 tests_to_run: A set of tests expected to run.
302 args : Arguments to pass to the to the test binary.
303 check_exit_0: When set to a true value, make sure that all shards
315 if CAN_TEST_EMPTY_FILTER
or gtest_filter !=
'':
318 for i
in range(0, total_shards):
321 self.assertEqual(0, exit_code)
322 partition.append(tests_run)
329 """Checks that the binary runs correct set of tests for the given filter.
331 Runs gtest_filter_unittest_ with the given filter, and enables
332 disabled tests. Verifies that the right set of tests were run.
335 gtest_filter: A filter to apply to the tests.
336 tests_to_run: A set of tests expected to run.
342 args = [
'--%s' % ALSO_RUN_DISABED_TESTS_FLAG]
343 if gtest_filter
is not None:
344 args.append(
'--%s=%s' % (FILTER_FLAG, gtest_filter))
350 """Sets up test case.
352 Determines whether value-parameterized tests are enabled in the binary and
353 sets the flags accordingly.
356 global param_tests_present
357 if param_tests_present
is None:
358 param_tests_present = PARAM_TEST_REGEX.search(
362 """Tests the behavior of not specifying the filter."""
367 """Tests the behavior without the filter, with sharding enabled."""
376 """Tests an empty filter."""
383 """Tests a filter that matches nothing."""
389 """Tests filtering by full name."""
396 """Tests filters that match everything."""
405 """Tests filtering by test case name."""
407 self.
RunAndVerify(
'FooTest.*', [
'FooTest.Abc',
'FooTest.Xyz'])
409 BAZ_TESTS = [
'BazTest.TestOne',
'BazTest.TestA',
'BazTest.TestB']
412 BAZ_TESTS + [
'BazTest.DISABLED_TestC'])
415 """Tests filtering by test name."""
417 self.
RunAndVerify(
'*.TestOne', [
'BarTest.TestOne',
'BazTest.TestOne'])
420 """Select only the disabled tests to run."""
424 [
'DISABLED_FoobarTest.Test1'])
431 'BarTest.DISABLED_TestFour',
432 'BarTest.DISABLED_TestFive',
433 'BazTest.DISABLED_TestC',
434 'DISABLED_FoobarTest.DISABLED_Test2',
439 'DISABLED_FoobarTest.Test1',
440 'DISABLED_FoobarTest.DISABLED_Test2',
441 'DISABLED_FoobarbazTest.TestA',
445 """Tests using wildcard in the test case name."""
454 'BazTest.TestB', ] + DEATH_TESTS + PARAM_TESTS)
457 """Tests using wildcard in the test name."""
459 self.
RunAndVerify(
'*.*A*', [
'FooTest.Abc',
'BazTest.TestA'])
462 """Tests a filter that has no '.' in it."""
473 """Tests filters that consist of two patterns."""
483 self.
RunAndVerify(
':*A*', [
'FooTest.Abc',
'BazTest.TestA'])
486 """Tests filters that consist of three patterns."""
525 ] + DEATH_TESTS + PARAM_TESTS)
533 ] + DEATH_TESTS + PARAM_TESTS)
541 self.
RunAndVerify(
'-FooTest.Abc:FooTest.Xyz:BazTest.*', [
545 ] + DEATH_TESTS + PARAM_TESTS)
552 'SeqP/ParamTest.TestX/0',
553 'SeqP/ParamTest.TestX/1',
554 'SeqP/ParamTest.TestY/0',
555 'SeqP/ParamTest.TestY/1',
560 'SeqP/ParamTest.TestX/0',
561 'SeqP/ParamTest.TestY/0',
562 'SeqQ/ParamTest.TestX/0',
563 'SeqQ/ParamTest.TestY/0',
567 """Tests that the filter flag overrides the filtering env. variable."""
570 args = [
'--%s=%s' % (FILTER_FLAG,
'*One')]
574 self.
AssertSetEqual(tests_run, [
'BarTest.TestOne',
'BazTest.TestOne'])
577 """Tests that the shard file is created if specified in the environment."""
581 self.assert_(
not os.path.exists(shard_status_file))
583 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
587 self.assert_(os.path.exists(shard_status_file))
588 os.remove(shard_status_file)
591 """Tests that the shard file is created with the "list_tests" flag."""
594 'shard_status_file2')
595 self.assert_(
not os.path.exists(shard_status_file))
597 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
605 self.assert_(
'[==========]' not in output,
606 'Unexpected output during test enumeration.\n'
607 'Please ensure that LIST_TESTS_FLAG is assigned the\n'
608 'correct flag value for listing Google Test tests.')
610 self.assert_(os.path.exists(shard_status_file))
611 os.remove(shard_status_file)
613 if SUPPORTS_DEATH_TESTS:
615 """Tests integration with death tests and sharding."""
617 gtest_filter =
'HasDeathTest.*:SeqP/*'
619 'HasDeathTest.Test1',
620 'HasDeathTest.Test2',
622 'SeqP/ParamTest.TestX/0',
623 'SeqP/ParamTest.TestX/1',
624 'SeqP/ParamTest.TestY/0',
625 'SeqP/ParamTest.TestY/1',
628 for flag
in [
'--gtest_death_test_style=threadsafe',
629 '--gtest_death_test_style=fast']:
631 check_exit_0=
True, args=[flag])
633 check_exit_0=
True, args=[flag])
635 if __name__ ==
'__main__':