SDL  2.0
SDL_test_harness.c
Go to the documentation of this file.
1 /*
2  Simple DirectMedia Layer
3  Copyright (C) 1997-2018 Sam Lantinga <slouken@libsdl.org>
4 
5  This software is provided 'as-is', without any express or implied
6  warranty. In no event will the authors be held liable for any damages
7  arising from the use of this software.
8 
9  Permission is granted to anyone to use this software for any purpose,
10  including commercial applications, and to alter it and redistribute it
11  freely, subject to the following restrictions:
12 
13  1. The origin of this software must not be misrepresented; you must not
14  claim that you wrote the original software. If you use this software
15  in a product, an acknowledgment in the product documentation would be
16  appreciated but is not required.
17  2. Altered source versions must be plainly marked as such, and must not be
18  misrepresented as being the original software.
19  3. This notice may not be removed or altered from any source distribution.
20 */
21 
22 #include "SDL_config.h"
23 
24 #include "SDL_test.h"
25 
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <time.h>
30 
31 /* Invalid test name/description message format */
32 #define SDLTEST_INVALID_NAME_FORMAT "(Invalid)"
33 
34 /* Log summary message format */
35 #define SDLTEST_LOG_SUMMARY_FORMAT "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
36 
37 /* Final result message format */
38 #define SDLTEST_FINAL_RESULT_FORMAT ">>> %s '%s': %s\n"
39 
40 /* ! \brief Timeout for single test case execution */
42 
43 /**
44 * Generates a random run seed string for the harness. The generated seed
45 * will contain alphanumeric characters (0-9A-Z).
46 *
47 * Note: The returned string needs to be deallocated by the caller.
48 *
49 * \param length The length of the seed string to generate
50 *
51 * \returns The generated seed string
52 */
53 char *
55 {
56  char *seed = NULL;
57  SDLTest_RandomContext randomContext;
58  int counter;
59 
60  /* Sanity check input */
61  if (length <= 0) {
62  SDLTest_LogError("The length of the harness seed must be >0.");
63  return NULL;
64  }
65 
66  /* Allocate output buffer */
67  seed = (char *)SDL_malloc((length + 1) * sizeof(char));
68  if (seed == NULL) {
69  SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
71  return NULL;
72  }
73 
74  /* Generate a random string of alphanumeric characters */
75  SDLTest_RandomInitTime(&randomContext);
76  for (counter = 0; counter < length; counter++) {
77  unsigned int number = SDLTest_Random(&randomContext);
78  char ch = (char) (number % (91 - 48)) + 48;
79  if (ch >= 58 && ch <= 64) {
80  ch = 65;
81  }
82  seed[counter] = ch;
83  }
84  seed[length] = '\0';
85 
86  return seed;
87 }
88 
89 /**
90 * Generates an execution key for the fuzzer.
91 *
92 * \param runSeed The run seed to use
93 * \param suiteName The name of the test suite
94 * \param testName The name of the test
95 * \param iteration The iteration count
96 *
97 * \returns The generated execution key to initialize the fuzzer with.
98 *
99 */
100 static Uint64
101 SDLTest_GenerateExecKey(const char *runSeed, char *suiteName, char *testName, int iteration)
102 {
103  SDLTest_Md5Context md5Context;
104  Uint64 *keys;
105  char iterationString[16];
106  size_t runSeedLength;
107  size_t suiteNameLength;
108  size_t testNameLength;
109  size_t iterationStringLength;
110  size_t entireStringLength;
111  char *buffer;
112 
113  if (runSeed == NULL || runSeed[0] == '\0') {
114  SDLTest_LogError("Invalid runSeed string.");
115  return -1;
116  }
117 
118  if (suiteName == NULL || suiteName[0] == '\0') {
119  SDLTest_LogError("Invalid suiteName string.");
120  return -1;
121  }
122 
123  if (testName == NULL || testName[0] == '\0') {
124  SDLTest_LogError("Invalid testName string.");
125  return -1;
126  }
127 
128  if (iteration <= 0) {
129  SDLTest_LogError("Invalid iteration count.");
130  return -1;
131  }
132 
133  /* Convert iteration number into a string */
134  SDL_memset(iterationString, 0, sizeof(iterationString));
135  SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
136 
137  /* Combine the parameters into single string */
138  runSeedLength = SDL_strlen(runSeed);
139  suiteNameLength = SDL_strlen(suiteName);
140  testNameLength = SDL_strlen(testName);
141  iterationStringLength = SDL_strlen(iterationString);
142  entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
143  buffer = (char *)SDL_malloc(entireStringLength);
144  if (buffer == NULL) {
145  SDLTest_LogError("Failed to allocate buffer for execKey generation.");
147  return 0;
148  }
149  SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
150 
151  /* Hash string and use half of the digest as 64bit exec key */
152  SDLTest_Md5Init(&md5Context);
153  SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, (unsigned int) entireStringLength);
154  SDLTest_Md5Final(&md5Context);
155  SDL_free(buffer);
156  keys = (Uint64 *)md5Context.digest;
157 
158  return keys[0];
159 }
160 
161 /**
162 * \brief Set timeout handler for test.
163 *
164 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.
165 *
166 * \param timeout Timeout interval in seconds.
167 * \param callback Function that will be called after timeout has elapsed.
168 *
169 * \return Timer id or -1 on failure.
170 */
171 static SDL_TimerID
173 {
174  Uint32 timeoutInMilliseconds;
175  SDL_TimerID timerID;
176 
177  if (callback == NULL) {
178  SDLTest_LogError("Timeout callback can't be NULL");
179  return -1;
180  }
181 
182  if (timeout < 0) {
183  SDLTest_LogError("Timeout value must be bigger than zero.");
184  return -1;
185  }
186 
187  /* Init SDL timer if not initialized before */
188  if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
190  SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
191  return -1;
192  }
193  }
194 
195  /* Set timer */
196  timeoutInMilliseconds = timeout * 1000;
197  timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
198  if (timerID == 0) {
199  SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
200  return -1;
201  }
202 
203  return timerID;
204 }
205 
206 /**
207 * \brief Timeout handler. Aborts test run and exits harness process.
208 */
209 static SDL_NORETURN void
211 {
212  SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
213  exit(TEST_ABORTED); /* bail out from the test */
214 }
215 
216 /**
217 * \brief Execute a test using the given execution key.
218 *
219 * \param testSuite Suite containing the test case.
220 * \param testCase Case to execute.
221 * \param execKey Execution key for the fuzzer.
222 * \param forceTestRun Force test to run even if test was disabled in suite.
223 *
224 * \returns Test case result.
225 */
226 static int
227 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
228 {
229  SDL_TimerID timer = 0;
230  int testCaseResult = 0;
231  int testResult = 0;
232  int fuzzerCount;
233 
234  if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
235  {
236  SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
238  }
239 
240  if (!testCase->enabled && forceTestRun == SDL_FALSE)
241  {
242  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
243  return TEST_RESULT_SKIPPED;
244  }
245 
246  /* Initialize fuzzer */
247  SDLTest_FuzzerInit(execKey);
248 
249  /* Reset assert tracker */
251 
252  /* Set timeout timer */
254 
255  /* Maybe run suite initalizer function */
256  if (testSuite->testSetUp) {
257  testSuite->testSetUp(0x0);
259  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
261  }
262  }
263 
264  /* Run test case function */
265  testCaseResult = testCase->testCase(0x0);
266 
267  /* Convert test execution result into harness result */
268  if (testCaseResult == TEST_SKIPPED) {
269  /* Test was programatically skipped */
270  testResult = TEST_RESULT_SKIPPED;
271  } else if (testCaseResult == TEST_STARTED) {
272  /* Test did not return a TEST_COMPLETED value; assume it failed */
273  testResult = TEST_RESULT_FAILED;
274  } else if (testCaseResult == TEST_ABORTED) {
275  /* Test was aborted early; assume it failed */
276  testResult = TEST_RESULT_FAILED;
277  } else {
278  /* Perform failure analysis based on asserts */
279  testResult = SDLTest_AssertSummaryToTestResult();
280  }
281 
282  /* Maybe run suite cleanup function (ignore failed asserts) */
283  if (testSuite->testTearDown) {
284  testSuite->testTearDown(0x0);
285  }
286 
287  /* Cancel timeout timer */
288  if (timer) {
289  SDL_RemoveTimer(timer);
290  }
291 
292  /* Report on asserts and fuzzer usage */
293  fuzzerCount = SDLTest_GetFuzzerInvocationCount();
294  if (fuzzerCount > 0) {
295  SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
296  }
297 
298  /* Final log based on test execution result */
299  if (testCaseResult == TEST_SKIPPED) {
300  /* Test was programatically skipped */
301  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
302  } else if (testCaseResult == TEST_STARTED) {
303  /* Test did not return a TEST_COMPLETED value; assume it failed */
304  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
305  } else if (testCaseResult == TEST_ABORTED) {
306  /* Test was aborted early; assume it failed */
307  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
308  } else {
310  }
311 
312  return testResult;
313 }
314 
315 /* Prints summary of all suites/tests contained in the given reference */
316 #if 0
317 static void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites)
318 {
319  int suiteCounter;
320  int testCounter;
321  SDLTest_TestSuiteReference *testSuite;
322  SDLTest_TestCaseReference *testCase;
323 
324  /* Loop over all suites */
325  suiteCounter = 0;
326  while(&testSuites[suiteCounter]) {
327  testSuite=&testSuites[suiteCounter];
328  suiteCounter++;
329  SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
330  (testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
331 
332  /* Loop over all test cases */
333  testCounter = 0;
334  while(testSuite->testCases[testCounter])
335  {
336  testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
337  testCounter++;
338  SDLTest_Log(" Test Case %i - %s: %s", testCounter,
339  (testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT,
340  (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
341  }
342  }
343 }
344 #endif
345 
346 /* Gets a timer value in seconds */
347 static float GetClock()
348 {
349  float currentClock = clock() / (float) CLOCKS_PER_SEC;
350  return currentClock;
351 }
352 
353 /**
354 * \brief Execute a test suite using the given run seed and execution key.
355 *
356 * The filter string is matched to the suite name (full comparison) to select a single suite,
357 * or if no suite matches, it is matched to the test names (full comparison) to select a single test.
358 *
359 * \param testSuites Suites containing the test case.
360 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one.
361 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
362 * \param filter Filter specification. NULL disables. Case sensitive.
363 * \param testIterations Number of iterations to run each test case.
364 *
365 * \returns Test run result; 0 when all tests passed, 1 if any tests failed.
366 */
367 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
368 {
369  int totalNumberOfTests = 0;
370  int failedNumberOfTests = 0;
371  int suiteCounter;
372  int testCounter;
373  int iterationCounter;
374  SDLTest_TestSuiteReference *testSuite;
375  const SDLTest_TestCaseReference *testCase;
376  const char *runSeed = NULL;
377  char *currentSuiteName;
378  char *currentTestName;
379  Uint64 execKey;
380  float runStartSeconds;
381  float suiteStartSeconds;
382  float testStartSeconds;
383  float runEndSeconds;
384  float suiteEndSeconds;
385  float testEndSeconds;
386  float runtime;
387  int suiteFilter = 0;
388  char *suiteFilterName = NULL;
389  int testFilter = 0;
390  char *testFilterName = NULL;
391  SDL_bool forceTestRun = SDL_FALSE;
392  int testResult = 0;
393  int runResult = 0;
394  Uint32 totalTestFailedCount = 0;
395  Uint32 totalTestPassedCount = 0;
396  Uint32 totalTestSkippedCount = 0;
397  Uint32 testFailedCount = 0;
398  Uint32 testPassedCount = 0;
399  Uint32 testSkippedCount = 0;
400  Uint32 countSum = 0;
401  const SDLTest_TestCaseReference **failedTests;
402 
403  /* Sanitize test iterations */
404  if (testIterations < 1) {
405  testIterations = 1;
406  }
407 
408  /* Generate run see if we don't have one already */
409  if (userRunSeed == NULL || userRunSeed[0] == '\0') {
410  runSeed = SDLTest_GenerateRunSeed(16);
411  if (runSeed == NULL) {
412  SDLTest_LogError("Generating a random seed failed");
413  return 2;
414  }
415  } else {
416  runSeed = userRunSeed;
417  }
418 
419 
420  /* Reset per-run counters */
421  totalTestFailedCount = 0;
422  totalTestPassedCount = 0;
423  totalTestSkippedCount = 0;
424 
425  /* Take time - run start */
426  runStartSeconds = GetClock();
427 
428  /* Log run with fuzzer parameters */
429  SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
430 
431  /* Count the total number of tests */
432  suiteCounter = 0;
433  while (testSuites[suiteCounter]) {
434  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
435  suiteCounter++;
436  testCounter = 0;
437  while (testSuite->testCases[testCounter])
438  {
439  testCounter++;
440  totalNumberOfTests++;
441  }
442  }
443 
444  /* Pre-allocate an array for tracking failed tests (potentially all test cases) */
445  failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
446  if (failedTests == NULL) {
447  SDLTest_LogError("Unable to allocate cache for failed tests");
449  return -1;
450  }
451 
452  /* Initialize filtering */
453  if (filter != NULL && filter[0] != '\0') {
454  /* Loop over all suites to check if we have a filter match */
455  suiteCounter = 0;
456  while (testSuites[suiteCounter] && suiteFilter == 0) {
457  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
458  suiteCounter++;
459  if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
460  /* Matched a suite name */
461  suiteFilter = 1;
462  suiteFilterName = testSuite->name;
463  SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
464  break;
465  }
466 
467  /* Within each suite, loop over all test cases to check if we have a filter match */
468  testCounter = 0;
469  while (testSuite->testCases[testCounter] && testFilter == 0)
470  {
471  testCase = testSuite->testCases[testCounter];
472  testCounter++;
473  if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
474  /* Matched a test name */
475  suiteFilter = 1;
476  suiteFilterName = testSuite->name;
477  testFilter = 1;
478  testFilterName = testCase->name;
479  SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
480  break;
481  }
482  }
483  }
484 
485  if (suiteFilter == 0 && testFilter == 0) {
486  SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
487  SDLTest_Log("Exit code: 2");
488  SDL_free((void *) failedTests);
489  return 2;
490  }
491  }
492 
493  /* Loop over all suites */
494  suiteCounter = 0;
495  while(testSuites[suiteCounter]) {
496  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
497  currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
498  suiteCounter++;
499 
500  /* Filter suite if flag set and we have a name */
501  if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
502  SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
503  /* Skip suite */
504  SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
505  suiteCounter,
506  currentSuiteName);
507  } else {
508 
509  /* Reset per-suite counters */
510  testFailedCount = 0;
511  testPassedCount = 0;
512  testSkippedCount = 0;
513 
514  /* Take time - suite start */
515  suiteStartSeconds = GetClock();
516 
517  /* Log suite started */
518  SDLTest_Log("===== Test Suite %i: '%s' started\n",
519  suiteCounter,
520  currentSuiteName);
521 
522  /* Loop over all test cases */
523  testCounter = 0;
524  while(testSuite->testCases[testCounter])
525  {
526  testCase = testSuite->testCases[testCounter];
527  currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
528  testCounter++;
529 
530  /* Filter tests if flag set and we have a name */
531  if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
532  SDL_strcmp(testFilterName, testCase->name) != 0) {
533  /* Skip test */
534  SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
535  suiteCounter,
536  testCounter,
537  currentTestName);
538  } else {
539  /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
540  if (testFilter == 1 && !testCase->enabled) {
541  SDLTest_Log("Force run of disabled test since test filter was set");
542  forceTestRun = SDL_TRUE;
543  }
544 
545  /* Take time - test start */
546  testStartSeconds = GetClock();
547 
548  /* Log test started */
549  SDLTest_Log("----- Test Case %i.%i: '%s' started",
550  suiteCounter,
551  testCounter,
552  currentTestName);
553  if (testCase->description != NULL && testCase->description[0] != '\0') {
554  SDLTest_Log("Test Description: '%s'",
555  (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
556  }
557 
558  /* Loop over all iterations */
559  iterationCounter = 0;
560  while(iterationCounter < testIterations)
561  {
562  iterationCounter++;
563 
564  if (userExecKey != 0) {
565  execKey = userExecKey;
566  } else {
567  execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
568  }
569 
570  SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
571  testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
572 
573  if (testResult == TEST_RESULT_PASSED) {
574  testPassedCount++;
575  totalTestPassedCount++;
576  } else if (testResult == TEST_RESULT_SKIPPED) {
577  testSkippedCount++;
578  totalTestSkippedCount++;
579  } else {
580  testFailedCount++;
581  totalTestFailedCount++;
582  }
583  }
584 
585  /* Take time - test end */
586  testEndSeconds = GetClock();
587  runtime = testEndSeconds - testStartSeconds;
588  if (runtime < 0.0f) runtime = 0.0f;
589 
590  if (testIterations > 1) {
591  /* Log test runtime */
592  SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
593  SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
594  } else {
595  /* Log test runtime */
596  SDLTest_Log("Total Test runtime: %.1f sec", runtime);
597  }
598 
599  /* Log final test result */
600  switch (testResult) {
601  case TEST_RESULT_PASSED:
602  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
603  break;
604  case TEST_RESULT_FAILED:
605  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
606  break;
608  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
609  break;
610  }
611 
612  /* Collect failed test case references for repro-step display */
613  if (testResult == TEST_RESULT_FAILED) {
614  failedTests[failedNumberOfTests] = testCase;
615  failedNumberOfTests++;
616  }
617  }
618  }
619 
620  /* Take time - suite end */
621  suiteEndSeconds = GetClock();
622  runtime = suiteEndSeconds - suiteStartSeconds;
623  if (runtime < 0.0f) runtime = 0.0f;
624 
625  /* Log suite runtime */
626  SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
627 
628  /* Log summary and final Suite result */
629  countSum = testPassedCount + testFailedCount + testSkippedCount;
630  if (testFailedCount == 0)
631  {
632  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
633  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
634  }
635  else
636  {
637  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
638  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
639  }
640 
641  }
642  }
643 
644  /* Take time - run end */
645  runEndSeconds = GetClock();
646  runtime = runEndSeconds - runStartSeconds;
647  if (runtime < 0.0f) runtime = 0.0f;
648 
649  /* Log total runtime */
650  SDLTest_Log("Total Run runtime: %.1f sec", runtime);
651 
652  /* Log summary and final run result */
653  countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
654  if (totalTestFailedCount == 0)
655  {
656  runResult = 0;
657  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
658  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
659  }
660  else
661  {
662  runResult = 1;
663  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
664  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
665  }
666 
667  /* Print repro steps for failed tests */
668  if (failedNumberOfTests > 0) {
669  SDLTest_Log("Harness input to repro failures:");
670  for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
671  SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
672  }
673  }
674  SDL_free((void *) failedTests);
675 
676  SDLTest_Log("Exit code: %d", runResult);
677  return runResult;
678 }
679 
680 /* vi: set ts=4 sw=4 expandtab: */
#define SDL_GetError
#define SDLTEST_INVALID_NAME_FORMAT
#define TEST_ABORTED
SDLTest_TestCaseTearDownFp testTearDown
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
Execute a test suite using the given run seed and execution key.
char * SDLTest_GenerateRunSeed(const int length)
Generates a random run seed string for the harness. The generated seed will contain alphanumeric char...
Uint32(* SDL_TimerCallback)(Uint32 interval, void *param)
Definition: SDL_timer.h:81
#define SDL_NORETURN
Definition: begin_code.h:154
#define SDLTEST_FINAL_RESULT_FORMAT
GLfloat f
#define SDL_InitSubSystem
static Uint32 SDLTest_TestCaseTimeout
uint32_t Uint32
Definition: SDL_stdinc.h:181
SDLTest_TestCaseSetUpFp testSetUp
#define SDL_Error
uint64_t Uint64
Definition: SDL_stdinc.h:194
const SDLTest_TestCaseReference ** testCases
GLuint const GLchar * name
#define SDL_AddTimer
static void iteration()
int SDLTest_GetFuzzerInvocationCount(void)
#define TEST_RESULT_FAILED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
void SDLTest_RandomInitTime(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
#define SDL_free
static SDL_TimerID SDLTest_SetTestTimeout(int timeout, void(*callback)())
Set timeout handler for test.
void SDLTest_ResetAssertSummary(void)
Resets the assert summary counters to zero.
void SDLTest_Md5Update(SDLTest_Md5Context *mdContext, unsigned char *inBuf, unsigned int inLen)
update digest from variable length data
Definition: SDL_test_md5.c:131
void SDLTest_Md5Init(SDLTest_Md5Context *mdContext)
initialize the context
Definition: SDL_test_md5.c:110
#define TEST_RESULT_SETUP_FAILURE
static Uint32 callback(Uint32 interval, void *param)
Definition: testtimer.c:34
void SDLTest_LogAssertSummary(void)
Logs summary of all assertions (total, pass, fail) since last reset as INFO or ERROR.
#define TEST_STARTED
static float GetClock()
#define TEST_RESULT_NO_ASSERT
unsigned int SDLTest_Random(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
int SDLTest_AssertSummaryToTestResult(void)
Converts the current assert summary state to a test result.
#define NULL
Definition: begin_code.h:164
SDL_bool
Definition: SDL_stdinc.h:139
GLuint buffer
#define SDL_INIT_TIMER
Definition: SDL.h:76
#define TEST_RESULT_PASSED
#define SDL_strlen
GLbitfield GLuint64 timeout
void SDLTest_Md5Final(SDLTest_Md5Context *mdContext)
complete digest computation
Definition: SDL_test_md5.c:180
#define SDLTEST_LOG_SUMMARY_FORMAT
#define SDL_PRIu64
Definition: SDL_stdinc.h:216
GLuint counter
GLuint GLfloat x0
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:85
#define SDL_snprintf
static Uint64 SDLTest_GenerateExecKey(const char *runSeed, char *suiteName, char *testName, int iteration)
#define SDL_malloc
static SDL_NORETURN void SDLTest_BailOut()
Timeout handler. Aborts test run and exits harness process.
void SDLTest_FuzzerInit(Uint64 execKey)
#define SDL_strcmp
#define SDL_WasInit
#define TEST_RESULT_SKIPPED
GLuint GLsizei GLsizei * length
SDLTest_TestCaseFp testCase
#define TEST_SKIPPED
#define SDL_RemoveTimer
#define SDL_memset
int SDL_TimerID
Definition: SDL_timer.h:86
static int SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
Execute a test using the given execution key.
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
unsigned char digest[16]
Definition: SDL_test_md5.h:75
SDLTest_TestSuiteReference * testSuites[]