Merge pull request #15110 from unknownbrackets/headless-compare

Headless: Generate a difference highlighting png
This commit is contained in:
Unknown W. Brackets 2021-11-07 11:36:28 -08:00 committed by GitHub
commit da697d861c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 187 additions and 68 deletions

View File

@ -19,9 +19,10 @@
#include <cmath>
#include <cstdarg>
#include <iostream>
#include <png.h>
#include <vector>
#include "headless/Compare.h"
#include "headless/Compare.h"
#include "Common/Data/Convert/ColorConv.h"
#include "Common/Data/Format/PNGLoad.h"
#include "Common/File/FileUtil.h"
@ -209,7 +210,7 @@ std::string GetTestName(const Path &bootFilename)
}
bool CompareOutput(const Path &bootFilename, const std::string &output, bool verbose) {
Path expect_filename = bootFilename.WithReplacedExtension(".prx", ".expected");
Path expect_filename = bootFilename.GetFileExtension() == ".prx" ? bootFilename.WithReplacedExtension(".prx", ".expected") : bootFilename.WithExtraExtension(".expected");
std::unique_ptr<FileLoader> expect_loader(ConstructFileLoader(expect_filename));
if (expect_loader->Exists()) {
@ -365,32 +366,35 @@ std::vector<u32> TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32
return data;
}
double CompareScreenshot(const std::vector<u32> &pixels, u32 stride, u32 w, u32 h, const Path& screenshotFilename, std::string &error)
{
if (pixels.size() < stride * h)
{
error = "Buffer format conversion error";
ScreenshotComparer::~ScreenshotComparer() {
if (reference_)
free(reference_);
}
double ScreenshotComparer::Compare(const Path &screenshotFilename) {
if (pixels_.size() < stride_ * h_) {
error_ = "Buffer format conversion error";
return -1.0f;
}
// We assume the bitmap is the specified size, not including whatever stride.
u32 *reference = nullptr;
bool asBitmap = false;
std::unique_ptr<FileLoader> loader(ConstructFileLoader(screenshotFilename));
if (loader->Exists()) {
uint8_t header[2];
if (loader->ReadAt(0, 2, header) != 2) {
error = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();
error_ = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();
return -1.0f;
}
if (header[0] == 'B' && header[1] == 'M') {
reference = (u32 *)calloc(stride * h, sizeof(u32));
asBitmap = true;
reference_ = (u32 *)calloc(stride_ * h_, sizeof(u32));
asBitmap_ = true;
// The bitmap header is 14 + 40 bytes. We could validate it but the test would fail either way.
if (reference && loader->ReadAt(14 + 40, sizeof(u32), stride * h, reference) != stride * h) {
error = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();
if (reference_ && loader->ReadAt(14 + 40, sizeof(u32), stride_ * h_, reference_) != stride_ * h_) {
error_ = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();
free(reference_);
reference_ = nullptr;
return -1.0f;
}
} else {
@ -398,45 +402,151 @@ double CompareScreenshot(const std::vector<u32> &pixels, u32 stride, u32 w, u32
std::vector<uint8_t> compressed;
compressed.resize(loader->FileSize());
if (loader->ReadAt(0, compressed.size(), &compressed[0]) != compressed.size()) {
error = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();
error_ = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();
return -1.0f;
}
int width, height;
if (!pngLoadPtr(&compressed[0], compressed.size(), &width, &height, (unsigned char **)&reference)) {
error = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();
if (!pngLoadPtr(&compressed[0], compressed.size(), &width, &height, (unsigned char **)&reference_)) {
error_ = "Unable to read screenshot data: " + screenshotFilename.ToVisualString();
if (reference_)
free(reference_);
reference_ = nullptr;
return -1.0f;
}
}
} else {
error = "Unable to read screenshot: " + screenshotFilename.ToVisualString();
error_ = "Unable to read screenshot: " + screenshotFilename.ToVisualString();
return -1.0f;
}
if (!reference) {
error = "Unable to allocate screenshot data: " + screenshotFilename.ToVisualString();
if (!reference_) {
error_ = "Unable to allocate screenshot data: " + screenshotFilename.ToVisualString();
return -1.0f;
}
u32 errors = 0;
if (asBitmap) {
if (asBitmap_) {
// The reference is flipped and BGRA by default for the common BMP compare case.
for (u32 y = 0; y < h; ++y) {
u32 yoff = y * stride;
for (u32 x = 0; x < w; ++x)
errors += ComparePixel(pixels[y * stride + x], reference[yoff + x]);
for (u32 y = 0; y < h_; ++y) {
u32 yoff = y * stride_;
for (u32 x = 0; x < w_; ++x)
errors += ComparePixel(pixels_[y * stride_ + x], reference_[yoff + x]);
}
} else {
// Just convert to BGRA for simplicity.
ConvertRGBA8888ToBGRA8888(reference, reference, h * stride);
for (u32 y = 0; y < h; ++y) {
u32 yoff = (h - y - 1) * stride;
for (u32 x = 0; x < w; ++x)
errors += ComparePixel(pixels[y * stride + x], reference[yoff + x]);
ConvertRGBA8888ToBGRA8888(reference_, reference_, h_ * stride_);
for (u32 y = 0; y < h_; ++y) {
u32 yoff = (h_ - y - 1) * stride_;
for (u32 x = 0; x < w_; ++x)
errors += ComparePixel(pixels_[y * stride_ + x], reference_[yoff + x]);
}
}
free(reference);
return (double) errors / (double) (w * h);
return (double) errors / (double) (w_ * h_);
}
bool ScreenshotComparer::SaveActualBitmap(const Path &resultFilename) {
static const u8 header[14 + 40] = {
0x42, 0x4D, 0x38, 0x80, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x28, 0x00,
0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x10, 0x01,
0x00, 0x00, 0x01, 0x00, 0x20, 0x00, 0x00, 0x00,
0x00, 0x00, 0x02, 0x80, 0x08, 0x00, 0x12, 0x0B,
0x00, 0x00, 0x12, 0x0B, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
};
FILE *saved = File::OpenCFile(resultFilename, "wb");
if (saved) {
fwrite(&header, sizeof(header), 1, saved);
fwrite(pixels_.data(), sizeof(u32), stride_ * h_, saved);
fclose(saved);
return true;
}
return false;
}
bool ScreenshotComparer::SaveVisualComparisonPNG(const Path &resultFilename) {
std::unique_ptr<u32[]> comparison(new u32[w_ * 2 * h_ * 2]);
if (asBitmap_) {
// The reference is flipped and BGRA by default for the common BMP compare case.
for (u32 y = 0; y < h_; ++y) {
u32 yoff = y * stride_;
u32 comparisonRow = (h_ - y - 1) * 2 * w_ * 2;
for (u32 x = 0; x < w_; ++x) {
PlotVisualComparison(comparison.get(), comparisonRow + x * 2, pixels_[y * stride_ + x], reference_[yoff + x]);
}
}
} else {
// Reference is already in BGRA either way.
for (u32 y = 0; y < h_; ++y) {
u32 yoff = (h_ - y - 1) * stride_;
u32 comparisonRow = (h_ - y - 1) * 2 * w_ * 2;
for (u32 x = 0; x < w_; ++x) {
PlotVisualComparison(comparison.get(), comparisonRow + x * 2, pixels_[y * stride_ + x], reference_[yoff + x]);
}
}
}
FILE *fp = File::OpenCFile(resultFilename, "wb");
if (!fp)
return false;
png_image png;
memset(&png, 0, sizeof(png));
png.version = PNG_IMAGE_VERSION;
png.format = PNG_FORMAT_BGRA;
png.width = w_ * 2;
png.height = h_ * 2;
bool success = png_image_write_to_stdio(&png, fp, 0, comparison.get(), w_ * 2 * 4, nullptr) != 0;
fclose(fp);
png_image_free(&png);
return success && png.warning_or_error < 2;
}
int ChannelDifference(u8 actual, u8 reference) {
int diff = actual > reference ? actual - reference : reference - actual;
if (diff == 0)
return 0;
if (diff < 4)
return 1;
if (diff < 8)
return 2;
if (diff < 16)
return 3;
if (diff < 32)
return 4;
return 5;
}
int PixelDifference(u32 actual, u32 reference) {
int b = ChannelDifference((actual >> 0) & 0xFF, (reference >> 0) & 0xFF);
int g = ChannelDifference((actual >> 8) & 0xFF, (reference >> 8) & 0xFF);
int r = ChannelDifference((actual >> 16) & 0xFF, (reference >> 16) & 0xFF);
return std::max(b, std::max(g, r));
}
void ScreenshotComparer::PlotVisualComparison(u32 *dst, u32 offset, u32 actual, u32 reference) {
int diff = PixelDifference(actual, reference);
dst[offset + 0] = actual | 0xFF000000;
dst[offset + 1] = actual | 0xFF000000;
dst[offset + w_ * 2 + 0] = reference | 0xFF000000;
int alpha = 0x00000000;
switch (diff) {
case 0: alpha = 0xFF000000; break;
case 1: alpha = 0xEF000000; break;
case 2: alpha = 0xCF000000; break;
case 3: alpha = 0xAF000000; break;
case 4: alpha = 0x7F000000; break;
default: break;
}
dst[offset + w_ * 2 + 1] = (reference & 0x00FFFFFF) | alpha;
}

View File

@ -34,4 +34,31 @@ std::string GetTestName(const Path &bootFilename);
bool CompareOutput(const Path &bootFilename, const std::string &output, bool verbose);
std::vector<u32> TranslateDebugBufferToCompare(const GPUDebugBuffer *buffer, u32 stride, u32 h);
double CompareScreenshot(const std::vector<u32> &pixels, u32 stride, u32 w, u32 h, const Path &screenshotFilename, std::string &error);
class ScreenshotComparer {
public:
ScreenshotComparer(const std::vector<u32> &pixels, u32 stride, u32 w, u32 h)
: pixels_(pixels), stride_(stride), w_(w), h_(h) {
}
~ScreenshotComparer();
double Compare(const Path &screenshotFilename);
std::string GetError() {
return error_;
}
bool SaveActualBitmap(const Path &filename);
bool SaveVisualComparisonPNG(const Path &filename);
protected:
void PlotVisualComparison(u32 *dst, u32 offset, u32 actual, u32 reference);
const std::vector<u32> &pixels_;
u32 *reference_ = nullptr;
bool asBitmap_ = false;
std::string error_;
u32 stride_;
u32 w_;
u32 h_;
};

View File

@ -176,7 +176,7 @@
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<PreprocessorDefinitions>_CRTDBG_MAP_ALLOC;USING_WIN_UI;GLEW_STATIC;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_ARCH_32=1;_CONSOLE;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>../ffmpeg/Windows/x86/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>../ffmpeg/Windows/x86/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew;../ext/libpng17</AdditionalIncludeDirectories>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<FloatingPointModel>Precise</FloatingPointModel>
@ -209,7 +209,7 @@
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<PreprocessorDefinitions>_CRTDBG_MAP_ALLOC;USING_WIN_UI;GLEW_STATIC;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_ARCH_64=1;_CONSOLE;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>../ffmpeg/Windows/x86_64/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>../ffmpeg/Windows/x86_64/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew;../ext/libpng17</AdditionalIncludeDirectories>
<EnableEnhancedInstructionSet>NotSet</EnableEnhancedInstructionSet>
<FloatingPointModel>Precise</FloatingPointModel>
<OmitFramePointers>false</OmitFramePointers>
@ -242,7 +242,7 @@
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<PreprocessorDefinitions>_CRTDBG_MAP_ALLOC;USING_WIN_UI;GLEW_STATIC;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_ARCH_64=1;_CONSOLE;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>../ffmpeg/Windows/aarch64/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>../ffmpeg/Windows/aarch64/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew;../ext/libpng17</AdditionalIncludeDirectories>
<FloatingPointModel>Precise</FloatingPointModel>
<OmitFramePointers>false</OmitFramePointers>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
@ -271,7 +271,7 @@
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<PreprocessorDefinitions>_CRTDBG_MAP_ALLOC;USING_WIN_UI;GLEW_STATIC;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_ARCH_32=1;_CONSOLE;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>../ffmpeg/Windows/arm/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>../ffmpeg/Windows/arm/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew;../ext/libpng17</AdditionalIncludeDirectories>
<FloatingPointModel>Precise</FloatingPointModel>
<OmitFramePointers>false</OmitFramePointers>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
@ -303,7 +303,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>USING_WIN_UI;GLEW_STATIC;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_ARCH_32=1;_CONSOLE;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>../ffmpeg/Windows/x86/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>../ffmpeg/Windows/x86/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew;../ext/libpng17</AdditionalIncludeDirectories>
<BufferSecurityCheck>false</BufferSecurityCheck>
<EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
<FloatingPointModel>Precise</FloatingPointModel>
@ -339,7 +339,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>USING_WIN_UI;GLEW_STATIC;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_ARCH_64=1;_CONSOLE;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>../ffmpeg/Windows/x86_64/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>../ffmpeg/Windows/x86_64/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew;../ext/libpng17</AdditionalIncludeDirectories>
<BufferSecurityCheck>false</BufferSecurityCheck>
<EnableEnhancedInstructionSet>NotSet</EnableEnhancedInstructionSet>
<FloatingPointModel>Precise</FloatingPointModel>
@ -377,7 +377,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>USING_WIN_UI;GLEW_STATIC;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_ARCH_64=1;_CONSOLE;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>../ffmpeg/Windows/aarch64/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>../ffmpeg/Windows/aarch64/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew;../ext/libpng17</AdditionalIncludeDirectories>
<BufferSecurityCheck>false</BufferSecurityCheck>
<FloatingPointModel>Precise</FloatingPointModel>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
@ -411,7 +411,7 @@
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>USING_WIN_UI;GLEW_STATIC;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_ARCH_32=1;_CONSOLE;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>../ffmpeg/Windows/arm/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>../ffmpeg/Windows/arm/include;../dx9sdk/Include/DX11;../Common;..;../Core;../ext/glew;../ext/libpng17</AdditionalIncludeDirectories>
<BufferSecurityCheck>false</BufferSecurityCheck>
<FloatingPointModel>Precise</FloatingPointModel>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>

View File

@ -17,6 +17,7 @@
#include "Common/File/FileUtil.h"
#include "Common/Log.h"
#include "Common/StringUtils.h"
#include "Core/CoreParameter.h"
#include "Core/System.h"
#include "GPU/Common/GPUDebugInterface.h"
@ -48,36 +49,17 @@ void HeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h) {
gpuDebug->GetCurrentFramebuffer(buffer, GPU_DBG_FRAMEBUF_RENDER);
const std::vector<u32> pixels = TranslateDebugBufferToCompare(&buffer, 512, 272);
std::string error;
double errors = CompareScreenshot(pixels, FRAME_STRIDE, FRAME_WIDTH, FRAME_HEIGHT, comparisonScreenshot_, error);
ScreenshotComparer comparer(pixels, FRAME_STRIDE, FRAME_WIDTH, FRAME_HEIGHT);
double errors = comparer.Compare(comparisonScreenshot_);
if (errors < 0)
SendOrCollectDebugOutput(error + "\n");
SendOrCollectDebugOutput(comparer.GetError() + "\n");
if (errors > 0)
{
char temp[256];
snprintf(temp, sizeof(temp), "Screenshot error: %f%%\n", errors * 100.0f);
SendOrCollectDebugOutput(temp);
}
SendOrCollectDebugOutput(StringFromFormat("Screenshot error: %f%%\n", errors * 100.0f));
if (errors > 0 && !teamCityMode && !getenv("GITHUB_ACTIONS")) {
static const u8 header[14 + 40] = {
0x42, 0x4D, 0x38, 0x80, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x28, 0x00,
0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x10, 0x01,
0x00, 0x00, 0x01, 0x00, 0x20, 0x00, 0x00, 0x00,
0x00, 0x00, 0x02, 0x80, 0x08, 0x00, 0x12, 0x0B,
0x00, 0x00, 0x12, 0x0B, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
};
FILE *saved = File::OpenCFile(Path("__testfailure.bmp"), "wb");
if (saved) {
fwrite(&header, sizeof(header), 1, saved);
fwrite(pixels.data(), sizeof(u32), FRAME_STRIDE * FRAME_HEIGHT, saved);
fclose(saved);
if (comparer.SaveActualBitmap(Path("__testfailure.bmp")))
SendOrCollectDebugOutput("Actual output written to: __testfailure.bmp\n");
}
comparer.SaveVisualComparisonPNG(Path("__testcompare.png"));
}
}