SDL  2.0
SDL_test_harness.c File Reference
#include "SDL_config.h"
#include "SDL_test.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
+ Include dependency graph for SDL_test_harness.c:

Go to the source code of this file.

Macros

#define SDLTEST_INVALID_NAME_FORMAT   "(Invalid)"
 
#define SDLTEST_LOG_SUMMARY_FORMAT   "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
 
#define SDLTEST_FINAL_RESULT_FORMAT   ">>> %s '%s': %s\n"
 

Functions

char * SDLTest_GenerateRunSeed (const int length)
 Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z). More...
 
static Uint64 SDLTest_GenerateExecKey (const char *runSeed, char *suiteName, char *testName, int iteration)
 
static SDL_TimerID SDLTest_SetTestTimeout (int timeout, void(*callback)())
 Set timeout handler for test. More...
 
static SDL_NORETURN void SDLTest_BailOut ()
 Timeout handler. Aborts test run and exits harness process. More...
 
static int SDLTest_RunTest (SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
 Execute a test using the given execution key. More...
 
static float GetClock ()
 
int SDLTest_RunSuites (SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
 Execute a test suite using the given run seed and execution key. More...
 

Variables

static Uint32 SDLTest_TestCaseTimeout = 3600
 

Macro Definition Documentation

◆ SDLTEST_FINAL_RESULT_FORMAT

#define SDLTEST_FINAL_RESULT_FORMAT   ">>> %s '%s': %s\n"

Definition at line 38 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites(), and SDLTest_RunTest().

◆ SDLTEST_INVALID_NAME_FORMAT

#define SDLTEST_INVALID_NAME_FORMAT   "(Invalid)"

Definition at line 32 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites(), and SDLTest_RunTest().

◆ SDLTEST_LOG_SUMMARY_FORMAT

#define SDLTEST_LOG_SUMMARY_FORMAT   "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"

Definition at line 35 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites().

Function Documentation

◆ GetClock()

static float GetClock ( )
static

Definition at line 347 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites().

348 {
349  float currentClock = clock() / (float) CLOCKS_PER_SEC;
350  return currentClock;
351 }

◆ SDLTest_BailOut()

static SDL_NORETURN void SDLTest_BailOut ( )
static

Timeout handler. Aborts test run and exits harness process.

Definition at line 210 of file SDL_test_harness.c.

References SDLTest_LogError(), and TEST_ABORTED.

Referenced by SDLTest_RunTest().

211 {
212  SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
213  exit(TEST_ABORTED); /* bail out from the test */
214 }
#define TEST_ABORTED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103

◆ SDLTest_GenerateExecKey()

static Uint64 SDLTest_GenerateExecKey ( const char *  runSeed,
char *  suiteName,
char *  testName,
int  iteration 
)
static

Generates an execution key for the fuzzer.

Parameters
runSeedThe run seed to use
suiteNameThe name of the test suite
testNameThe name of the test
iterationThe iteration count
Returns
The generated execution key to initialize the fuzzer with.

Definition at line 101 of file SDL_test_harness.c.

References SDLTest_Md5Context::digest, NULL, SDL_ENOMEM, SDL_Error, SDL_free, SDL_malloc, SDL_memset, SDL_snprintf, SDL_strlen, SDLTest_LogError(), SDLTest_Md5Final(), SDLTest_Md5Init(), and SDLTest_Md5Update().

Referenced by SDLTest_RunSuites().

102 {
103  SDLTest_Md5Context md5Context;
104  Uint64 *keys;
105  char iterationString[16];
106  size_t runSeedLength;
107  size_t suiteNameLength;
108  size_t testNameLength;
109  size_t iterationStringLength;
110  size_t entireStringLength;
111  char *buffer;
112 
113  if (runSeed == NULL || runSeed[0] == '\0') {
114  SDLTest_LogError("Invalid runSeed string.");
115  return -1;
116  }
117 
118  if (suiteName == NULL || suiteName[0] == '\0') {
119  SDLTest_LogError("Invalid suiteName string.");
120  return -1;
121  }
122 
123  if (testName == NULL || testName[0] == '\0') {
124  SDLTest_LogError("Invalid testName string.");
125  return -1;
126  }
127 
128  if (iteration <= 0) {
129  SDLTest_LogError("Invalid iteration count.");
130  return -1;
131  }
132 
133  /* Convert iteration number into a string */
134  SDL_memset(iterationString, 0, sizeof(iterationString));
135  SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
136 
137  /* Combine the parameters into single string */
138  runSeedLength = SDL_strlen(runSeed);
139  suiteNameLength = SDL_strlen(suiteName);
140  testNameLength = SDL_strlen(testName);
141  iterationStringLength = SDL_strlen(iterationString);
142  entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
143  buffer = (char *)SDL_malloc(entireStringLength);
144  if (buffer == NULL) {
145  SDLTest_LogError("Failed to allocate buffer for execKey generation.");
147  return 0;
148  }
149  SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
150 
151  /* Hash string and use half of the digest as 64bit exec key */
152  SDLTest_Md5Init(&md5Context);
153  SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, (unsigned int) entireStringLength);
154  SDLTest_Md5Final(&md5Context);
155  SDL_free(buffer);
156  keys = (Uint64 *)md5Context.digest;
157 
158  return keys[0];
159 }
#define SDL_Error
uint64_t Uint64
Definition: SDL_stdinc.h:194
static void iteration()
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
#define SDL_free
void SDLTest_Md5Update(SDLTest_Md5Context *mdContext, unsigned char *inBuf, unsigned int inLen)
update digest from variable length data
Definition: SDL_test_md5.c:131
void SDLTest_Md5Init(SDLTest_Md5Context *mdContext)
initialize the context
Definition: SDL_test_md5.c:110
#define NULL
Definition: begin_code.h:164
GLuint buffer
#define SDL_strlen
void SDLTest_Md5Final(SDLTest_Md5Context *mdContext)
complete digest computation
Definition: SDL_test_md5.c:180
#define SDL_snprintf
#define SDL_malloc
#define SDL_memset
unsigned char digest[16]
Definition: SDL_test_md5.h:75

◆ SDLTest_GenerateRunSeed()

char* SDLTest_GenerateRunSeed ( const int  length)

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Note: The returned string needs to be deallocated by the caller.

Parameters
lengthThe length of the seed string to generate
Returns
The generated seed string

Definition at line 54 of file SDL_test_harness.c.

References NULL, SDL_ENOMEM, SDL_Error, SDL_malloc, SDLTest_LogError(), SDLTest_Random(), and SDLTest_RandomInitTime().

Referenced by sdltest_generateRunSeed(), and SDLTest_RunSuites().

55 {
56  char *seed = NULL;
57  SDLTest_RandomContext randomContext;
58  int counter;
59 
60  /* Sanity check input */
61  if (length <= 0) {
62  SDLTest_LogError("The length of the harness seed must be >0.");
63  return NULL;
64  }
65 
66  /* Allocate output buffer */
67  seed = (char *)SDL_malloc((length + 1) * sizeof(char));
68  if (seed == NULL) {
69  SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
71  return NULL;
72  }
73 
74  /* Generate a random string of alphanumeric characters */
75  SDLTest_RandomInitTime(&randomContext);
76  for (counter = 0; counter < length; counter++) {
77  unsigned int number = SDLTest_Random(&randomContext);
78  char ch = (char) (number % (91 - 48)) + 48;
79  if (ch >= 58 && ch <= 64) {
80  ch = 65;
81  }
82  seed[counter] = ch;
83  }
84  seed[length] = '\0';
85 
86  return seed;
87 }
#define SDL_Error
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
void SDLTest_RandomInitTime(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
unsigned int SDLTest_Random(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
#define NULL
Definition: begin_code.h:164
GLuint counter
#define SDL_malloc
GLuint GLsizei GLsizei * length

◆ SDLTest_RunSuites()

int SDLTest_RunSuites ( SDLTest_TestSuiteReference testSuites[],
const char *  userRunSeed,
Uint64  userExecKey,
const char *  filter,
int  testIterations 
)

Execute a test suite using the given run seed and execution key.

The filter string is matched to the suite name (full comparison) to select a single suite, or if no suite matches, it is matched to the test names (full comparison) to select a single test.

Parameters
testSuitesSuites containing the test case.
userRunSeedCustom run seed provided by user, or NULL to autogenerate one.
userExecKeyCustom execution key provided by user, or 0 to autogenerate one.
filterFilter specification. NULL disables. Case sensitive.
testIterationsNumber of iterations to run each test case.
Returns
Test run result; 0 when all tests passed, 1 if any tests failed.

Definition at line 367 of file SDL_test_harness.c.

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, GetClock(), SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_ENOMEM, SDL_Error, SDL_FALSE, SDL_free, SDL_malloc, SDL_PRIu64, SDL_strcmp, SDL_TRUE, SDLTEST_FINAL_RESULT_FORMAT, SDLTest_GenerateExecKey(), SDLTest_GenerateRunSeed(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTEST_LOG_SUMMARY_FORMAT, SDLTest_LogError(), SDLTest_RunTest(), TEST_RESULT_FAILED, TEST_RESULT_NO_ASSERT, TEST_RESULT_PASSED, TEST_RESULT_SKIPPED, and SDLTest_TestSuiteReference::testCases.

Referenced by main().

368 {
369  int totalNumberOfTests = 0;
370  int failedNumberOfTests = 0;
371  int suiteCounter;
372  int testCounter;
373  int iterationCounter;
374  SDLTest_TestSuiteReference *testSuite;
375  const SDLTest_TestCaseReference *testCase;
376  const char *runSeed = NULL;
377  char *currentSuiteName;
378  char *currentTestName;
379  Uint64 execKey;
380  float runStartSeconds;
381  float suiteStartSeconds;
382  float testStartSeconds;
383  float runEndSeconds;
384  float suiteEndSeconds;
385  float testEndSeconds;
386  float runtime;
387  int suiteFilter = 0;
388  char *suiteFilterName = NULL;
389  int testFilter = 0;
390  char *testFilterName = NULL;
391  SDL_bool forceTestRun = SDL_FALSE;
392  int testResult = 0;
393  int runResult = 0;
394  Uint32 totalTestFailedCount = 0;
395  Uint32 totalTestPassedCount = 0;
396  Uint32 totalTestSkippedCount = 0;
397  Uint32 testFailedCount = 0;
398  Uint32 testPassedCount = 0;
399  Uint32 testSkippedCount = 0;
400  Uint32 countSum = 0;
401  const SDLTest_TestCaseReference **failedTests;
402 
403  /* Sanitize test iterations */
404  if (testIterations < 1) {
405  testIterations = 1;
406  }
407 
408  /* Generate run see if we don't have one already */
409  if (userRunSeed == NULL || userRunSeed[0] == '\0') {
410  runSeed = SDLTest_GenerateRunSeed(16);
411  if (runSeed == NULL) {
412  SDLTest_LogError("Generating a random seed failed");
413  return 2;
414  }
415  } else {
416  runSeed = userRunSeed;
417  }
418 
419 
420  /* Reset per-run counters */
421  totalTestFailedCount = 0;
422  totalTestPassedCount = 0;
423  totalTestSkippedCount = 0;
424 
425  /* Take time - run start */
426  runStartSeconds = GetClock();
427 
428  /* Log run with fuzzer parameters */
429  SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
430 
431  /* Count the total number of tests */
432  suiteCounter = 0;
433  while (testSuites[suiteCounter]) {
434  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
435  suiteCounter++;
436  testCounter = 0;
437  while (testSuite->testCases[testCounter])
438  {
439  testCounter++;
440  totalNumberOfTests++;
441  }
442  }
443 
444  /* Pre-allocate an array for tracking failed tests (potentially all test cases) */
445  failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
446  if (failedTests == NULL) {
447  SDLTest_LogError("Unable to allocate cache for failed tests");
449  return -1;
450  }
451 
452  /* Initialize filtering */
453  if (filter != NULL && filter[0] != '\0') {
454  /* Loop over all suites to check if we have a filter match */
455  suiteCounter = 0;
456  while (testSuites[suiteCounter] && suiteFilter == 0) {
457  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
458  suiteCounter++;
459  if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
460  /* Matched a suite name */
461  suiteFilter = 1;
462  suiteFilterName = testSuite->name;
463  SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
464  break;
465  }
466 
467  /* Within each suite, loop over all test cases to check if we have a filter match */
468  testCounter = 0;
469  while (testSuite->testCases[testCounter] && testFilter == 0)
470  {
471  testCase = testSuite->testCases[testCounter];
472  testCounter++;
473  if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
474  /* Matched a test name */
475  suiteFilter = 1;
476  suiteFilterName = testSuite->name;
477  testFilter = 1;
478  testFilterName = testCase->name;
479  SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
480  break;
481  }
482  }
483  }
484 
485  if (suiteFilter == 0 && testFilter == 0) {
486  SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
487  SDLTest_Log("Exit code: 2");
488  SDL_free((void *) failedTests);
489  return 2;
490  }
491  }
492 
493  /* Loop over all suites */
494  suiteCounter = 0;
495  while(testSuites[suiteCounter]) {
496  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
497  currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
498  suiteCounter++;
499 
500  /* Filter suite if flag set and we have a name */
501  if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
502  SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
503  /* Skip suite */
504  SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
505  suiteCounter,
506  currentSuiteName);
507  } else {
508 
509  /* Reset per-suite counters */
510  testFailedCount = 0;
511  testPassedCount = 0;
512  testSkippedCount = 0;
513 
514  /* Take time - suite start */
515  suiteStartSeconds = GetClock();
516 
517  /* Log suite started */
518  SDLTest_Log("===== Test Suite %i: '%s' started\n",
519  suiteCounter,
520  currentSuiteName);
521 
522  /* Loop over all test cases */
523  testCounter = 0;
524  while(testSuite->testCases[testCounter])
525  {
526  testCase = testSuite->testCases[testCounter];
527  currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
528  testCounter++;
529 
530  /* Filter tests if flag set and we have a name */
531  if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
532  SDL_strcmp(testFilterName, testCase->name) != 0) {
533  /* Skip test */
534  SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
535  suiteCounter,
536  testCounter,
537  currentTestName);
538  } else {
539  /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
540  if (testFilter == 1 && !testCase->enabled) {
541  SDLTest_Log("Force run of disabled test since test filter was set");
542  forceTestRun = SDL_TRUE;
543  }
544 
545  /* Take time - test start */
546  testStartSeconds = GetClock();
547 
548  /* Log test started */
549  SDLTest_Log("----- Test Case %i.%i: '%s' started",
550  suiteCounter,
551  testCounter,
552  currentTestName);
553  if (testCase->description != NULL && testCase->description[0] != '\0') {
554  SDLTest_Log("Test Description: '%s'",
555  (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
556  }
557 
558  /* Loop over all iterations */
559  iterationCounter = 0;
560  while(iterationCounter < testIterations)
561  {
562  iterationCounter++;
563 
564  if (userExecKey != 0) {
565  execKey = userExecKey;
566  } else {
567  execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
568  }
569 
570  SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
571  testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
572 
573  if (testResult == TEST_RESULT_PASSED) {
574  testPassedCount++;
575  totalTestPassedCount++;
576  } else if (testResult == TEST_RESULT_SKIPPED) {
577  testSkippedCount++;
578  totalTestSkippedCount++;
579  } else {
580  testFailedCount++;
581  totalTestFailedCount++;
582  }
583  }
584 
585  /* Take time - test end */
586  testEndSeconds = GetClock();
587  runtime = testEndSeconds - testStartSeconds;
588  if (runtime < 0.0f) runtime = 0.0f;
589 
590  if (testIterations > 1) {
591  /* Log test runtime */
592  SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
593  SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
594  } else {
595  /* Log test runtime */
596  SDLTest_Log("Total Test runtime: %.1f sec", runtime);
597  }
598 
599  /* Log final test result */
600  switch (testResult) {
601  case TEST_RESULT_PASSED:
602  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
603  break;
604  case TEST_RESULT_FAILED:
605  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
606  break;
608  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
609  break;
610  }
611 
612  /* Collect failed test case references for repro-step display */
613  if (testResult == TEST_RESULT_FAILED) {
614  failedTests[failedNumberOfTests] = testCase;
615  failedNumberOfTests++;
616  }
617  }
618  }
619 
620  /* Take time - suite end */
621  suiteEndSeconds = GetClock();
622  runtime = suiteEndSeconds - suiteStartSeconds;
623  if (runtime < 0.0f) runtime = 0.0f;
624 
625  /* Log suite runtime */
626  SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
627 
628  /* Log summary and final Suite result */
629  countSum = testPassedCount + testFailedCount + testSkippedCount;
630  if (testFailedCount == 0)
631  {
632  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
633  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
634  }
635  else
636  {
637  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
638  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
639  }
640 
641  }
642  }
643 
644  /* Take time - run end */
645  runEndSeconds = GetClock();
646  runtime = runEndSeconds - runStartSeconds;
647  if (runtime < 0.0f) runtime = 0.0f;
648 
649  /* Log total runtime */
650  SDLTest_Log("Total Run runtime: %.1f sec", runtime);
651 
652  /* Log summary and final run result */
653  countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
654  if (totalTestFailedCount == 0)
655  {
656  runResult = 0;
657  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
658  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
659  }
660  else
661  {
662  runResult = 1;
663  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
664  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
665  }
666 
667  /* Print repro steps for failed tests */
668  if (failedNumberOfTests > 0) {
669  SDLTest_Log("Harness input to repro failures:");
670  for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
671  SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
672  }
673  }
674  SDL_free((void *) failedTests);
675 
676  SDLTest_Log("Exit code: %d", runResult);
677  return runResult;
678 }
#define SDLTEST_INVALID_NAME_FORMAT
char * SDLTest_GenerateRunSeed(const int length)
Generates a random run seed string for the harness. The generated seed will contain alphanumeric char...
#define SDLTEST_FINAL_RESULT_FORMAT
GLfloat f
uint32_t Uint32
Definition: SDL_stdinc.h:181
#define SDL_Error
uint64_t Uint64
Definition: SDL_stdinc.h:194
const SDLTest_TestCaseReference ** testCases
GLuint const GLchar * name
#define TEST_RESULT_FAILED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
#define SDL_free
static float GetClock()
#define TEST_RESULT_NO_ASSERT
#define NULL
Definition: begin_code.h:164
SDL_bool
Definition: SDL_stdinc.h:139
#define TEST_RESULT_PASSED
#define SDLTEST_LOG_SUMMARY_FORMAT
#define SDL_PRIu64
Definition: SDL_stdinc.h:216
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:85
static Uint64 SDLTest_GenerateExecKey(const char *runSeed, char *suiteName, char *testName, int iteration)
#define SDL_malloc
#define SDL_strcmp
#define TEST_RESULT_SKIPPED
static int SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
Execute a test using the given execution key.
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter

◆ SDLTest_RunTest()

static int SDLTest_RunTest ( SDLTest_TestSuiteReference testSuite,
const SDLTest_TestCaseReference testCase,
Uint64  execKey,
SDL_bool  forceTestRun 
)
static

Execute a test using the given execution key.

Parameters
testSuiteSuite containing the test case.
testCaseCase to execute.
execKeyExecution key for the fuzzer.
forceTestRunForce test to run even if test was disabled in suite.
Returns
Test case result.

Definition at line 227 of file SDL_test_harness.c.

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_FALSE, SDL_RemoveTimer, SDLTest_AssertSummaryToTestResult(), SDLTest_BailOut(), SDLTEST_FINAL_RESULT_FORMAT, SDLTest_FuzzerInit(), SDLTest_GetFuzzerInvocationCount(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTest_LogAssertSummary(), SDLTest_LogError(), SDLTest_ResetAssertSummary(), SDLTest_SetTestTimeout(), SDLTest_TestCaseTimeout, TEST_ABORTED, TEST_RESULT_FAILED, TEST_RESULT_SETUP_FAILURE, TEST_RESULT_SKIPPED, TEST_SKIPPED, TEST_STARTED, SDLTest_TestCaseReference::testCase, SDLTest_TestSuiteReference::testCases, SDLTest_TestSuiteReference::testSetUp, testSuites, and SDLTest_TestSuiteReference::testTearDown.

Referenced by SDLTest_RunSuites().

228 {
229  SDL_TimerID timer = 0;
230  int testCaseResult = 0;
231  int testResult = 0;
232  int fuzzerCount;
233 
234  if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
235  {
236  SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
238  }
239 
240  if (!testCase->enabled && forceTestRun == SDL_FALSE)
241  {
242  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
243  return TEST_RESULT_SKIPPED;
244  }
245 
246  /* Initialize fuzzer */
247  SDLTest_FuzzerInit(execKey);
248 
249  /* Reset assert tracker */
251 
252  /* Set timeout timer */
254 
255  /* Maybe run suite initalizer function */
256  if (testSuite->testSetUp) {
257  testSuite->testSetUp(0x0);
259  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
261  }
262  }
263 
264  /* Run test case function */
265  testCaseResult = testCase->testCase(0x0);
266 
267  /* Convert test execution result into harness result */
268  if (testCaseResult == TEST_SKIPPED) {
269  /* Test was programatically skipped */
270  testResult = TEST_RESULT_SKIPPED;
271  } else if (testCaseResult == TEST_STARTED) {
272  /* Test did not return a TEST_COMPLETED value; assume it failed */
273  testResult = TEST_RESULT_FAILED;
274  } else if (testCaseResult == TEST_ABORTED) {
275  /* Test was aborted early; assume it failed */
276  testResult = TEST_RESULT_FAILED;
277  } else {
278  /* Perform failure analysis based on asserts */
279  testResult = SDLTest_AssertSummaryToTestResult();
280  }
281 
282  /* Maybe run suite cleanup function (ignore failed asserts) */
283  if (testSuite->testTearDown) {
284  testSuite->testTearDown(0x0);
285  }
286 
287  /* Cancel timeout timer */
288  if (timer) {
289  SDL_RemoveTimer(timer);
290  }
291 
292  /* Report on asserts and fuzzer usage */
293  fuzzerCount = SDLTest_GetFuzzerInvocationCount();
294  if (fuzzerCount > 0) {
295  SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
296  }
297 
298  /* Final log based on test execution result */
299  if (testCaseResult == TEST_SKIPPED) {
300  /* Test was programatically skipped */
301  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
302  } else if (testCaseResult == TEST_STARTED) {
303  /* Test did not return a TEST_COMPLETED value; assume it failed */
304  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
305  } else if (testCaseResult == TEST_ABORTED) {
306  /* Test was aborted early; assume it failed */
307  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
308  } else {
310  }
311 
312  return testResult;
313 }
#define TEST_ABORTED
SDLTest_TestCaseTearDownFp testTearDown
#define SDLTEST_FINAL_RESULT_FORMAT
static Uint32 SDLTest_TestCaseTimeout
SDLTest_TestCaseSetUpFp testSetUp
int SDLTest_GetFuzzerInvocationCount(void)
#define TEST_RESULT_FAILED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
static SDL_TimerID SDLTest_SetTestTimeout(int timeout, void(*callback)())
Set timeout handler for test.
void SDLTest_ResetAssertSummary(void)
Resets the assert summary counters to zero.
#define TEST_RESULT_SETUP_FAILURE
void SDLTest_LogAssertSummary(void)
Logs summary of all assertions (total, pass, fail) since last reset as INFO or ERROR.
#define TEST_STARTED
int SDLTest_AssertSummaryToTestResult(void)
Converts the current assert summary state to a test result.
#define NULL
Definition: begin_code.h:164
GLuint GLfloat x0
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:85
static SDL_NORETURN void SDLTest_BailOut()
Timeout handler. Aborts test run and exits harness process.
void SDLTest_FuzzerInit(Uint64 execKey)
#define TEST_RESULT_SKIPPED
SDLTest_TestCaseFp testCase
#define TEST_SKIPPED
#define SDL_RemoveTimer
int SDL_TimerID
Definition: SDL_timer.h:86

◆ SDLTest_SetTestTimeout()

static SDL_TimerID SDLTest_SetTestTimeout ( int  timeout,
void(*)()  callback 
)
static

Set timeout handler for test.

Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.

Parameters
timeoutTimeout interval in seconds.
callbackFunction that will be called after timeout has elapsed.
Returns
Timer id or -1 on failure.

Definition at line 172 of file SDL_test_harness.c.

References callback(), NULL, SDL_AddTimer, SDL_GetError, SDL_INIT_TIMER, SDL_InitSubSystem, SDL_NORETURN, SDL_WasInit, and SDLTest_LogError().

Referenced by SDLTest_RunTest().

173 {
174  Uint32 timeoutInMilliseconds;
175  SDL_TimerID timerID;
176 
177  if (callback == NULL) {
178  SDLTest_LogError("Timeout callback can't be NULL");
179  return -1;
180  }
181 
182  if (timeout < 0) {
183  SDLTest_LogError("Timeout value must be bigger than zero.");
184  return -1;
185  }
186 
187  /* Init SDL timer if not initialized before */
188  if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
190  SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
191  return -1;
192  }
193  }
194 
195  /* Set timer */
196  timeoutInMilliseconds = timeout * 1000;
197  timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
198  if (timerID == 0) {
199  SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
200  return -1;
201  }
202 
203  return timerID;
204 }
#define SDL_GetError
Uint32(* SDL_TimerCallback)(Uint32 interval, void *param)
Definition: SDL_timer.h:81
#define SDL_InitSubSystem
uint32_t Uint32
Definition: SDL_stdinc.h:181
#define SDL_AddTimer
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
static Uint32 callback(Uint32 interval, void *param)
Definition: testtimer.c:34
#define NULL
Definition: begin_code.h:164
#define SDL_INIT_TIMER
Definition: SDL.h:76
GLbitfield GLuint64 timeout
GLuint GLfloat x0
#define SDL_WasInit
int SDL_TimerID
Definition: SDL_timer.h:86

Variable Documentation

◆ SDLTest_TestCaseTimeout

Uint32 SDLTest_TestCaseTimeout = 3600
static

Definition at line 41 of file SDL_test_harness.c.

Referenced by SDLTest_RunTest().