shadps4-emu/ext-SDL
1
0
Fork 0
mirror of https://github.com/shadps4-emu/ext-SDL.git synced 2024-11-27 03:50:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Run test suites and test cases in non-linear order (see libsdl-org#9303)

Browse Source
This commit is contained in:
Sylvain 2024-08-07 09:55:08 +02:00 committed by Sylvain Becker
parent 27bb5bcef6
commit 20a6193eaa
3 changed files with 107 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/SDL3/SDL_test_harness.h
View File

@ -117,10 +117,11 @@ char *SDLTest_GenerateRunSeed(const int length);
* \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
* \param filter Filter specification. NULL disables. Case sensitive.
* \param testIterations Number of iterations to run each test case.
* \param randomOrder allow to run suites and tests in random order when there is no filter
*
* \returns the test run result: 0 when all tests passed, 1 if any tests failed.
*/
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations);
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations, SDL_bool randomOrder);
/* Ends C function definitions when using C++ */

96
src/test/SDL_test_harness.c
View File

@ -365,10 +365,11 @@ static float GetClock(void)
* \param userExecKey Custom execution key provided by user, or 0 to autogenerate one.
* \param filter Filter specification. NULL disables. Case sensitive.
* \param testIterations Number of iterations to run each test case.
* \param randomOrder allow to run suites and tests in random order when there is no filter
*
* \returns Test run result; 0 when all tests passed, 1 if any tests failed.
*/
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations, SDL_bool randomOrder)
{
int totalNumberOfTests = 0;
int failedNumberOfTests = 0;
@ -404,6 +405,9 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
int countSum = 0;
const SDLTest_TestCaseReference **failedTests;
char generatedSeed[16 + 1];
int nbSuites = 0;
int i = 0;
int *arraySuites = NULL;
/* Sanitize test iterations */
if (testIterations < 1) {
@ -509,11 +513,59 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
SDL_free((void *)failedTests);
return 2;
}
randomOrder = SDL_FALSE;
}
/* Number of test suites */
while (testSuites[nbSuites]) {
nbSuites++;
}
arraySuites = SDL_malloc(nbSuites * sizeof(int));
if (!arraySuites) {
return SDL_OutOfMemory();
}
for (i = 0; i < nbSuites; i++) {
arraySuites[i] = i;
}
/* Mix the list of suites to run them in random order */
{
/* Exclude last test "subsystemsTestSuite" which is said to interfer with other tests */
nbSuites--;
if (userExecKey != 0) {
execKey = userExecKey;
} else {
/* dummy values to have random numbers working */
execKey = SDLTest_GenerateExecKey(runSeed, "random testSuites", "initialisation", 1);
}
/* Initialize fuzzer */
SDLTest_FuzzerInit(execKey);
i = 100;
while (i--) {
int a, b;
int tmp;
a = SDLTest_RandomIntegerInRange(0, nbSuites - 1);
b = SDLTest_RandomIntegerInRange(0, nbSuites - 1);
/* Swap */
if (randomOrder) {
tmp = arraySuites[b];
arraySuites[b] = arraySuites[a];
arraySuites[a] = tmp;
}
}
/* re-add last lest */
nbSuites++;
}
/* Loop over all suites */
suiteCounter = 0;
while (testSuites[suiteCounter]) {
for (i = 0; i < nbSuites; i++) {
suiteCounter = arraySuites[i];;
testSuite = testSuites[suiteCounter];
currentSuiteName = (testSuite->name ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
suiteCounter++;
@ -527,6 +579,36 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
currentSuiteName);
} else {
int nbTestCases = 0;
int *arrayTestCases;
int j;
while (testSuite->testCases[nbTestCases]) {
nbTestCases++;
}
arrayTestCases = SDL_malloc(nbTestCases * sizeof(int));
if (!arrayTestCases) {
return SDL_OutOfMemory();
}
for (j = 0; j < nbTestCases; j++) {
arrayTestCases[j] = j;
}
/* Mix the list of testCases to run them in random order */
j = 100;
while (j--) {
int a, b;
int tmp;
a = SDLTest_RandomIntegerInRange(0, nbTestCases - 1);
b = SDLTest_RandomIntegerInRange(0, nbTestCases - 1);
/* Swap */
if (randomOrder) {
tmp = arrayTestCases[b];
arrayTestCases[b] = arrayTestCases[a];
arrayTestCases[a] = tmp;
}
}
/* Reset per-suite counters */
testFailedCount = 0;
testPassedCount = 0;
@ -541,8 +623,8 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
currentSuiteName);
/* Loop over all test cases */
testCounter = 0;
while (testSuite->testCases[testCounter]) {
for (j = 0; j < nbTestCases; j++) {
testCounter = arrayTestCases[j];
testCase = testSuite->testCases[testCounter];
currentTestName = (testCase->name ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
testCounter++;
@ -657,9 +739,13 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user
SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, COLOR_RED "Failed" COLOR_END);
}
SDL_free(arrayTestCases);
}
}
SDL_free(arraySuites);
/* Take time - run end */
runEndSeconds = GetClock();
runtime = runEndSeconds - runStartSeconds;

16
test/testautomation.c
View File

@ -75,6 +75,7 @@ int main(int argc, char *argv[])
int i, done;
SDL_Event event;
int list = 0;
SDL_bool randomOrder = SDL_FALSE;
/* Initialize test framework */
state = SDLTest_CommonCreateState(argv, SDL_INIT_VIDEO | SDL_INIT_AUDIO);
@ -118,10 +119,21 @@ int main(int argc, char *argv[])
} else if (SDL_strcasecmp(argv[i], "--list") == 0) {
consumed = 1;
list = 1;
} else if (SDL_strcasecmp(argv[i], "--random-order") == 0) {
consumed = 1;
randomOrder = SDL_TRUE;
}
}
if (consumed < 0) {
static const char *options[] = { "[--iterations #]", "[--execKey #]", "[--seed string]", "[--filter suite_name|test_name]", "[--list]", NULL };
static const char *options[] = {
"[--iterations #]",
"[--execKey #]",
"[--seed string]",
"[--filter suite_name|test_name]",
"[--list]",
"[--random-order]",
NULL };
SDLTest_CommonLogUsage(state, argv[0], options);
quit(1);
}
@ -157,7 +169,7 @@ int main(int argc, char *argv[])
}
/* Call Harness */
result = SDLTest_RunSuites(testSuites, userRunSeed, userExecKey, filter, testIterations);
result = SDLTest_RunSuites(testSuites, userRunSeed, userExecKey, filter, testIterations, randomOrder);
/* Empty event queue */
done = 0;