31 """Verifies that test shuffling works."""
33 __author__ =
'wan@google.com (Zhanyong Wan)'
36 import gtest_test_utils
42 TOTAL_SHARDS_ENV_VAR =
'GTEST_TOTAL_SHARDS'
43 SHARD_INDEX_ENV_VAR =
'GTEST_SHARD_INDEX'
45 TEST_FILTER =
'A*.A:A*.B:C*'
52 SHUFFLED_ALL_TESTS = []
53 SHUFFLED_ACTIVE_TESTS = []
54 SHUFFLED_FILTERED_TESTS = []
55 SHUFFLED_SHARDED_TESTS = []
59 return '--gtest_also_run_disabled_tests'
63 return '--gtest_filter=%s' % (test_filter,)
67 return '--gtest_repeat=%s' % (n,)
71 return '--gtest_shuffle'
75 return '--gtest_random_seed=%s' % (n,)
79 """Runs the test program and returns its output."""
81 environ_copy = os.environ.copy()
82 environ_copy.update(extra_env)
88 """Runs the test program and returns a list of test lists.
91 extra_env: a map from environment variables to their values
92 args: command line flags to pass to gtest_shuffle_test_
95 A list where the i-th element is the list of tests run in the i-th
101 if line.startswith(
'----'):
103 test_iterations.append(tests)
105 tests.append(line.strip())
107 return test_iterations
111 """Returns a list of test cases in the given full test names.
114 tests: a list of full test names
117 A list of test cases from 'tests', in their original order.
118 Consecutive duplicates are removed.
123 test_case = test.split(
'.')[0]
124 if not test_case
in test_cases:
125 test_cases.append(test_case)
131 """Calculates the list of tests run under different flags."""
140 if not FILTERED_TESTS:
141 FILTERED_TESTS.extend(
144 if not SHARDED_TESTS:
145 SHARDED_TESTS.extend(
147 SHARD_INDEX_ENV_VAR:
'1'},
150 if not SHUFFLED_ALL_TESTS:
154 if not SHUFFLED_ACTIVE_TESTS:
158 if not SHUFFLED_FILTERED_TESTS:
162 if not SHUFFLED_SHARDED_TESTS:
163 SHUFFLED_SHARDED_TESTS.extend(
165 SHARD_INDEX_ENV_VAR:
'1'},
170 """Tests test shuffling."""
176 self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
177 self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
178 self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
179 self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
182 self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
183 self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)
184 self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,
185 SHUFFLED_FILTERED_TESTS)
186 self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,
187 SHUFFLED_SHARDED_TESTS)
203 for test
in SHUFFLED_ALL_TESTS:
204 self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),
205 '%s appears more than once' % (test,))
206 for test
in SHUFFLED_ACTIVE_TESTS:
207 self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),
208 '%s appears more than once' % (test,))
209 for test
in SHUFFLED_FILTERED_TESTS:
210 self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),
211 '%s appears more than once' % (test,))
212 for test
in SHUFFLED_SHARDED_TESTS:
213 self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),
214 '%s appears more than once' % (test,))
217 for test
in SHUFFLED_ALL_TESTS:
218 self.assert_(test
in ALL_TESTS,
'%s is an invalid test' % (test,))
219 for test
in SHUFFLED_ACTIVE_TESTS:
220 self.assert_(test
in ACTIVE_TESTS,
'%s is an invalid test' % (test,))
221 for test
in SHUFFLED_FILTERED_TESTS:
222 self.assert_(test
in FILTERED_TESTS,
'%s is an invalid test' % (test,))
223 for test
in SHUFFLED_SHARDED_TESTS:
224 self.assert_(test
in SHARDED_TESTS,
'%s is an invalid test' % (test,))
227 for test
in ALL_TESTS:
228 self.assert_(test
in SHUFFLED_ALL_TESTS,
'%s is missing' % (test,))
229 for test
in ACTIVE_TESTS:
230 self.assert_(test
in SHUFFLED_ACTIVE_TESTS,
'%s is missing' % (test,))
231 for test
in FILTERED_TESTS:
232 self.assert_(test
in SHUFFLED_FILTERED_TESTS,
'%s is missing' % (test,))
233 for test
in SHARDED_TESTS:
234 self.assert_(test
in SHUFFLED_SHARDED_TESTS,
'%s is missing' % (test,))
237 non_death_test_found =
False
238 for test
in SHUFFLED_ACTIVE_TESTS:
239 if 'DeathTest.' in test:
240 self.assert_(
not non_death_test_found,
241 '%s appears after a non-death test' % (test,))
243 non_death_test_found =
True
248 [test_case, _] = test.split(
'.')
249 if test_cases
and test_cases[-1] != test_case:
250 test_cases.append(test_case)
251 self.assertEqual(1, test_cases.count(test_case),
252 'Test case %s is not grouped together in %s' %
267 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
275 self.assertEqual(tests_in_iteration1, tests_with_seed1)
283 self.assertEqual(tests_in_iteration2, tests_with_seed2)
291 self.assertEqual(tests_in_iteration3, tests_with_seed3)
294 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
298 self.assert_(tests_in_iteration1 != tests_in_iteration2,
300 self.assert_(tests_in_iteration1 != tests_in_iteration3,
302 self.assert_(tests_in_iteration2 != tests_in_iteration3,
309 SHARD_INDEX_ENV_VAR:
'0'},
312 SHARD_INDEX_ENV_VAR:
'1'},
315 SHARD_INDEX_ENV_VAR:
'2'},
317 sorted_sharded_tests = tests1 + tests2 + tests3
318 sorted_sharded_tests.sort()
319 sorted_active_tests = []
320 sorted_active_tests.extend(ACTIVE_TESTS)
321 sorted_active_tests.sort()
322 self.assertEqual(sorted_active_tests, sorted_sharded_tests)
324 if __name__ ==
'__main__':