Emulator-Archive/ppsspp
1
0
Fork 0
mirror of https://github.com/hrydgard/ppsspp.git synced 2025-02-26 00:45:49 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #16304 from unknownbrackets/replacement

Browse Source 
Improve texture replacement cache and allow read from zip
This commit is contained in:
Henrik Rydgård 2022-10-31 09:08:25 +01:00 committed by GitHub
commit 3f33cf102c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 354 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

356
Core/TextureReplacer.cpp
View File

@ -21,6 +21,11 @@
#include <cstring>
#include <memory>
#include <png.h>
#ifdef SHARED_LIBZIP
#include <zip.h>
#else
#include "ext/libzip/zip.h"
#endif
#include "ext/xxhash.h"
@ -44,9 +49,11 @@
#include "GPU/Common/TextureDecoder.h"
static const std::string INI_FILENAME = "textures.ini";
static const std::string ZIP_FILENAME = "textures.zip";
static const std::string NEW_TEXTURE_DIR = "new/";
static const int VERSION = 1;
static const int MAX_MIP_LEVELS = 12; // 12 should be plenty, 8 is the max mip levels supported by the PSP.
static const double MAX_CACHE_SIZE = 4.0;
TextureReplacer::TextureReplacer() {
none_.initDone_ = true;
@ -54,6 +61,8 @@ TextureReplacer::TextureReplacer() {
}
TextureReplacer::~TextureReplacer() {
if (zip_)
zip_close(zip_);
}
void TextureReplacer::Init() {
@ -76,8 +85,11 @@ void TextureReplacer::NotifyConfigChanged() {
File::CreateEmptyFile(newTextureDir / ".nomedia");
}
enabled_ = File::Exists(basePath_) && File::IsDirectory(basePath_);
enabled_ = File::IsDirectory(basePath_);
} else if (wasEnabled) {
if (zip_)
zip_close(zip_);
zip_ = nullptr;
Decimate(ReplacerDecimateMode::ALL);
}
@ -86,6 +98,44 @@ void TextureReplacer::NotifyConfigChanged() {
}
}
static struct zip *ZipOpenPath(Path fileName) {
int error = 0;
if (fileName.Type() == PathType::CONTENT_URI) {
int fd = File::OpenFD(fileName, File::OPEN_READ);
return zip_fdopen(fd, 0, &error);
}
return zip_open(fileName.c_str(), 0, &error);
}
static constexpr zip_uint64_t INVALID_ZIP_SIZE = 0xFFFFFFFFFFFFFFFFULL;
static zip_uint64_t ZipFileSize(zip *z, zip_int64_t i) {
zip_stat_t zstat;
if (zip_stat_index(z, i, 0, &zstat) != 0)
return INVALID_ZIP_SIZE;
if ((zstat.valid & ZIP_STAT_SIZE) == 0)
return INVALID_ZIP_SIZE;
return zstat.size;
}
static bool LoadIniZip(IniFile &ini, zip *z, const std::string &filename) {
zip_int64_t i = zip_name_locate(z, filename.c_str(), ZIP_FL_NOCASE);
if (i < 0)
return false;
std::string inistr;
zip_uint64_t sz = ZipFileSize(z, i);
if (sz == INVALID_ZIP_SIZE)
return false;
inistr.resize(sz);
zip_file_t *zf = zip_fopen_index(z, i, 0);
inistr.resize(zip_fread(zf, &inistr[0], inistr.size()));
zip_fclose(zf);
std::stringstream sstream(inistr);
return ini.Load(sstream);
}
bool TextureReplacer::LoadIni() {
// TODO: Use crc32c?
hash_ = ReplacedTextureHash::QUICK;
@ -101,10 +151,32 @@ bool TextureReplacer::LoadIni() {
// Prevents dumping the mipmaps.
ignoreMipmap_ = false;
if (File::Exists(basePath_ / INI_FILENAME)) {
IniFile ini;
ini.LoadFromVFS((basePath_ / INI_FILENAME).ToString());
if (zip_)
zip_close(zip_);
zip_ = nullptr;
IniFile ini;
bool iniLoaded = false;
// First, check for textures.zip, which is used to reduce IO.
zip *z = ZipOpenPath(basePath_ / ZIP_FILENAME);
if (z) {
iniLoaded = LoadIniZip(ini, z, INI_FILENAME);
// Require the zip have textures.ini to use it.
if (iniLoaded) {
iniLoaded = true;
zip_ = z;
} else {
zip_close(z);
z = nullptr;
}
}
if (!iniLoaded) {
iniLoaded = ini.LoadFromVFS((basePath_ / INI_FILENAME).ToString());
}
if (iniLoaded) {
if (!LoadIniValues(ini)) {
return false;
}
@ -113,10 +185,17 @@ bool TextureReplacer::LoadIni() {
std::string overrideFilename;
if (ini.GetOrCreateSection("games")->Get(gameID_.c_str(), &overrideFilename, "")) {
if (!overrideFilename.empty() && overrideFilename != INI_FILENAME) {
INFO_LOG(G3D, "Loading extra texture ini: %s", overrideFilename.c_str());
IniFile overrideIni;
overrideIni.LoadFromVFS((basePath_ / overrideFilename).ToString());
if (zip_)
iniLoaded = LoadIniZip(overrideIni, zip_, overrideFilename);
else
iniLoaded = overrideIni.LoadFromVFS((basePath_ / overrideFilename).ToString());
if (!iniLoaded) {
ERROR_LOG(G3D, "Failed to load extra texture ini: %s", overrideFilename.c_str());
return false;
}
INFO_LOG(G3D, "Loading extra texture ini: %s", overrideFilename.c_str());
if (!LoadIniValues(overrideIni, true)) {
return false;
}
@ -410,7 +489,7 @@ void TextureReplacer::PopulateReplacement(ReplacedTexture *result, u64 cachekey,
for (int i = 0; i < MAX_MIP_LEVELS; ++i) {
const std::string hashfile = LookupHashFile(cachekey, hash, i);
const Path filename = basePath_ / hashfile;
if (hashfile.empty() || !File::Exists(filename)) {
if (hashfile.empty()) {
// Out of valid mip levels. Bail out.
break;
}
@ -418,7 +497,16 @@ void TextureReplacer::PopulateReplacement(ReplacedTexture *result, u64 cachekey,
ReplacedTextureLevel level;
level.fmt = Draw::DataFormat::R8G8B8A8_UNORM;
level.file = filename;
bool good = PopulateLevel(level);
bool good;
bool logError = hashfile != HashName(cachekey, hash, i) + ".png";
if (zip_) {
level.z = zip_;
level.zi = zip_name_locate(zip_, hashfile.c_str(), ZIP_FL_NOCASE);
good = PopulateLevelFromZip(level, !logError);
} else {
good = PopulateLevelFromPath(level, !logError);
}
// We pad files that have been hashrange'd so they are the same texture size.
level.w = (level.w * w) / newW;
@ -439,6 +527,12 @@ void TextureReplacer::PopulateReplacement(ReplacedTexture *result, u64 cachekey,
break;
}
// Populate the level data pointers for each level.
result->levelData_.resize(result->levels_.size());
for (size_t i = 0; i < result->levels_.size(); ++i) {
result->levelData_[i] = &levelCache_[result->levels_[i]];
}
result->prepareDone_ = true;
}
@ -448,12 +542,7 @@ enum class ReplacedImageType {
INVALID,
};
static ReplacedImageType Identify(FILE *fp) {
uint8_t magic[4];
if (fread(magic, 1, 4, fp) != 4)
return ReplacedImageType::INVALID;
rewind(fp);
static ReplacedImageType Identify(const uint8_t magic[4]) {
if (strncmp((const char *)magic, "ZIMG", 4) == 0)
return ReplacedImageType::ZIM;
if (magic[0] == 0x89 && strncmp((const char *)&magic[1], "PNG", 3) == 0)
@ -461,12 +550,30 @@ static ReplacedImageType Identify(FILE *fp) {
return ReplacedImageType::INVALID;
}
bool TextureReplacer::PopulateLevel(ReplacedTextureLevel &level) {
static ReplacedImageType Identify(FILE *fp) {
uint8_t magic[4];
if (fread(magic, 1, 4, fp) != 4)
return ReplacedImageType::INVALID;
rewind(fp);
return Identify(magic);
}
static ReplacedImageType Identify(zip_file_t *zf) {
uint8_t magic[4];
if (zip_fread(zf, magic, 4) != 4)
return ReplacedImageType::INVALID;
return Identify(magic);
}
bool TextureReplacer::PopulateLevelFromPath(ReplacedTextureLevel &level, bool ignoreError) {
bool good = false;
FILE *fp = File::OpenCFile(level.file, "rb");
if (!fp) {
ERROR_LOG(G3D, "Error opening replacement texture file '%s'", level.file.c_str());
if (!ignoreError)
ERROR_LOG(G3D, "Error opening replacement texture file '%s'", level.file.c_str());
return false;
}
@ -499,6 +606,62 @@ bool TextureReplacer::PopulateLevel(ReplacedTextureLevel &level) {
return good;
}
bool TextureReplacer::PopulateLevelFromZip(ReplacedTextureLevel &level, bool ignoreError) {
bool good = false;
if (!level.z || level.zi < 0) {
if (!ignoreError)
ERROR_LOG(G3D, "Error opening replacement texture file '%s' in textures.zip", level.file.c_str());
return false;
}
zip_file_t *zf = zip_fopen_index(level.z, level.zi, 0);
if (!zf)
return false;
auto imageType = Identify(zf);
zip_fclose(zf);
zf = zip_fopen_index(level.z, level.zi, 0);
if (imageType == ReplacedImageType::ZIM) {
uint32_t ignore = 0;
good = zip_fread(zf, &ignore, 4) == 4;
good = good && zip_fread(zf, &level.w, 4) == 4;
good = good && zip_fread(zf, &level.h, 4) == 4;
int flags;
if (good && zip_fread(zf, &flags, 4) == 4) {
good = (flags & ZIM_FORMAT_MASK) == ZIM_RGBA8888;
}
} else if (imageType == ReplacedImageType::PNG) {
png_image png = {};
png.version = PNG_IMAGE_VERSION;
// TODO: Use some way to stream data into libpng. Better than the IO lookups on Android...
zip_uint64_t zsize = ZipFileSize(level.z, level.zi);
std::string pngdata;
if (zsize != INVALID_ZIP_SIZE)
pngdata.resize(zsize);
if (!pngdata.empty()) {
pngdata.resize(zip_fread(zf, pngdata.data(), pngdata.size()));
}
if (png_image_begin_read_from_memory(&png, pngdata.data(), pngdata.size())) {
// We pad files that have been hashrange'd so they are the same texture size.
level.w = png.width;
level.h = png.height;
good = true;
} else {
ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s (zip)", level.file.ToVisualString().c_str(), png.message);
}
png_image_free(&png);
} else {
ERROR_LOG(G3D, "Could not load texture replacement info: %s - unsupported format (zip)", level.file.ToVisualString().c_str());
}
zip_fclose(zf);
return good;
}
static bool WriteTextureToPNG(png_imagep image, const Path &filename, int convert_to_8bit, const void *buffer, png_int_32 row_stride, const void *colormap) {
FILE *fp = File::OpenCFile(filename, "wb");
if (!fp) {
@ -673,15 +836,32 @@ void TextureReplacer::NotifyTextureDecoded(const ReplacedTextureDecodeInfo &repl
void TextureReplacer::Decimate(ReplacerDecimateMode mode) {
// Allow replacements to be cached for a long time, although they're large.
double age = 1800.0;
if (mode == ReplacerDecimateMode::FORCE_PRESSURE)
if (mode == ReplacerDecimateMode::FORCE_PRESSURE) {
age = 90.0;
else if (mode == ReplacerDecimateMode::ALL)
} else if (mode == ReplacerDecimateMode::ALL) {
age = 0.0;
} else if (lastTextureCacheSizeGB_ > 1.0) {
double pressure = std::min(MAX_CACHE_SIZE, lastTextureCacheSizeGB_) / MAX_CACHE_SIZE;
// Get more aggressive the closer we are to the max.
age = 90.0 + (1.0 - pressure) * 1710.0;
}
const double threshold = time_now_d() - age;
for (auto &item : cache_) {
item.second.PurgeIfOlder(threshold);
}
size_t totalSize = 0;
for (auto &item : levelCache_) {
std::lock_guard<std::mutex> guard(item.second.lock);
totalSize += item.second.data.size();
}
double totalSizeGB = totalSize / (1024.0 * 1024.0 * 1024.0);
if (totalSizeGB >= 1.0) {
WARN_LOG(G3D, "Decimated replacements older than %fs, currently using %f GB of RAM", age, totalSizeGB);
}
lastTextureCacheSizeGB_ = totalSizeGB;
}
template <typename Key, typename Value>
@ -821,8 +1001,11 @@ bool ReplacedTexture::IsReady(double budget) {
}
// Loaded already, or not yet on a thread?
if (initDone_ && !levelData_.empty())
if (initDone_ && !levelData_.empty()) {
for (auto &l : levelData_)
l->lastUsed = lastUsed_;
return true;
}
// Let's not even start a new texture if we're already behind.
if (budget < 0.0)
return false;
@ -856,7 +1039,6 @@ void ReplacedTexture::Prepare() {
return;
}
levelData_.resize(levels_.size());
for (int i = 0; i < (int)levels_.size(); ++i) {
if (cancelPrepare_)
break;
@ -870,39 +1052,87 @@ void ReplacedTexture::Prepare() {
void ReplacedTexture::PrepareData(int level) {
_assert_msg_((size_t)level < levels_.size(), "Invalid miplevel");
_assert_msg_(levelData_[level] != nullptr, "Level cache not set for miplevel");
// We must lock around access to levelData_ in case two textures try to load it at once.
std::lock_guard<std::mutex> guard(levelData_[level]->lock);
const ReplacedTextureLevel &info = levels_[level];
std::vector<uint8_t> &out = levelData_[level];
std::vector<uint8_t> &out = levelData_[level]->data;
FILE *fp = File::OpenCFile(info.file, "rb");
if (!fp) {
// Leaving the data sized at zero means failure.
// Already populated from cache.
if (!out.empty())
return;
}
auto imageType = Identify(fp);
if (imageType == ReplacedImageType::ZIM) {
size_t zimSize = File::GetFileSize(fp);
std::unique_ptr<uint8_t[]> zim(new uint8_t[zimSize]);
if (!zim) {
ERROR_LOG(G3D, "Failed to allocate memory for texture replacement");
fclose(fp);
FILE *fp = nullptr;
zip_file_t *zf = nullptr;
ReplacedImageType imageType;
if (info.z) {
zf = zip_fopen_index(info.z, info.zi, 0);
if (!zf)
return;
imageType = Identify(zf);
// Can't assume we can seek. Reopen.
zip_fclose(zf);
zf = zip_fopen_index(info.z, info.zi, 0);
} else {
fp = File::OpenCFile(info.file, "rb");
if (!fp) {
// Leaving the data sized at zero means failure.
return;
}
if (fread(&zim[0], 1, zimSize, fp) != zimSize) {
ERROR_LOG(G3D, "Could not load texture replacement: %s - failed to read ZIM", info.file.c_str());
imageType = Identify(fp);
}
auto cleanup = [&] {
if (zf)
zip_fclose(zf);
if (fp)
fclose(fp);
};
if (imageType == ReplacedImageType::ZIM) {
size_t zimSize;
if (fp) {
zimSize = File::GetFileSize(fp);
} else if (zf) {
zip_uint64_t zsize = ZipFileSize(info.z, info.zi);
zimSize = zsize == INVALID_ZIP_SIZE ? 0 : (size_t)zsize;
} else {
_assert_(false);
}
std::unique_ptr<uint8_t[]> zim(new uint8_t[zimSize]);
if (!zim) {
ERROR_LOG(G3D, "Failed to allocate memory for texture replacement");
cleanup();
return;
}
if (fp) {
if (fread(&zim[0], 1, zimSize, fp) != zimSize) {
ERROR_LOG(G3D, "Could not load texture replacement: %s - failed to read ZIM", info.file.c_str());
cleanup();
return;
}
} else if (zf) {
if (zip_fread(zf, &zim[0], zimSize) != zimSize) {
ERROR_LOG(G3D, "Could not load texture replacement: %s - failed to read ZIM (zip)", info.file.c_str());
cleanup();
return;
}
} else {
_assert_(false);
}
int w, h, f;
uint8_t *image;
if (LoadZIMPtr(&zim[0], zimSize, &w, &h, &f, &image)) {
if (w > info.w || h > info.h) {
ERROR_LOG(G3D, "Texture replacement changed since header read: %s", info.file.c_str());
fclose(fp);
cleanup();
return;
}
@ -925,14 +1155,33 @@ void ReplacedTexture::PrepareData(int level) {
png_image png = {};
png.version = PNG_IMAGE_VERSION;
if (!png_image_begin_read_from_stdio(&png, fp)) {
ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s", info.file.c_str(), png.message);
fclose(fp);
return;
// Needs to survive for a little while, used for zip only.
std::string pngdata;
if (fp) {
if (!png_image_begin_read_from_stdio(&png, fp)) {
ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s", info.file.c_str(), png.message);
cleanup();
return;
}
} else if (zf) {
zip_uint64_t zsize = ZipFileSize(info.z, info.zi);
if (zsize != INVALID_ZIP_SIZE)
pngdata.resize(zsize);
if (!pngdata.empty()) {
pngdata.resize(zip_fread(zf, pngdata.data(), pngdata.size()));
}
if (!png_image_begin_read_from_memory(&png, pngdata.data(), pngdata.size())) {
ERROR_LOG(G3D, "Could not load texture replacement info: %s - %s (zip)", info.file.c_str(), png.message);
cleanup();
return;
}
} else {
_assert_(false);
}
if (png.width > (uint32_t)info.w || png.height > (uint32_t)info.h) {
ERROR_LOG(G3D, "Texture replacement changed since header read: %s", info.file.c_str());
fclose(fp);
cleanup();
return;
}
@ -949,7 +1198,7 @@ void ReplacedTexture::PrepareData(int level) {
out.resize(info.w * info.h * 4);
if (!png_image_finish_read(&png, nullptr, &out[0], info.w * 4, nullptr)) {
ERROR_LOG(G3D, "Could not load texture replacement: %s - %s", info.file.c_str(), png.message);
fclose(fp);
cleanup();
out.resize(0);
return;
}
@ -964,13 +1213,23 @@ void ReplacedTexture::PrepareData(int level) {
}
}
fclose(fp);
cleanup();
}
void ReplacedTexture::PurgeIfOlder(double t) {
if (lastUsed_ < t && (!threadWaitable_ || threadWaitable_->WaitFor(0.0))) {
levelData_.clear();
initDone_ = false;
if (threadWaitable_ && !threadWaitable_->WaitFor(0.0))
return;
if (lastUsed_ >= t)
return;
for (auto &l : levelData_) {
if (l->lastUsed < t) {
// We have to lock since multiple textures might reference this same data.
std::lock_guard<std::mutex> guard(l->lock);
l->data.clear();
// This means we have to reload. If we never purge any, there's no need.
initDone_ = false;
}
}
}
@ -993,8 +1252,13 @@ bool ReplacedTexture::Load(int level, void *out, int rowPitch) {
if (levelData_.empty())
return false;
_assert_msg_(levelData_[level] != nullptr, "Level cache not set for miplevel");
// We probably could avoid this lock, but better to play it safe.
std::lock_guard<std::mutex> guard(levelData_[level]->lock);
const ReplacedTextureLevel &info = levels_[level];
const std::vector<uint8_t> &data = levelData_[level];
const std::vector<uint8_t> &data = levelData_[level]->data;
if (data.empty())
return false;

44
Core/TextureReplacer.h
View File

@ -17,11 +17,13 @@
#pragma once
#include "ppsspp_config.h"
#include <mutex>
#include <string>
#include <unordered_map>
#include <vector>
#include "Common/CommonFuncs.h"
#include "Common/CommonTypes.h"
#include "Common/MemoryUtil.h"
#include "Common/File/Path.h"
@ -35,6 +37,7 @@ class TextureCacheCommon;
class TextureReplacer;
class ReplacedTextureTask;
class LimitedWaitable;
struct zip;
// These must match the constants in TextureCacheCommon.
enum class ReplacedTextureAlpha {
@ -54,6 +57,38 @@ struct ReplacedTextureLevel {
int h;
Draw::DataFormat fmt; // NOTE: Right now, the only supported format is Draw::DataFormat::R8G8B8A8_UNORM.
Path file;
// Can be ignored for hashing/equal, since file has all uniqueness.
// To be able to reload, we need to be able to reopen, unfortunate we can't use zip_file_t.
zip *z = nullptr;
int64_t zi = -1;
bool operator ==(const ReplacedTextureLevel &other) const {
if (w != other.w || h != other.h || fmt != other.fmt)
return false;
return file == other.file;
}
};
namespace std {
template <>
struct hash<ReplacedTextureLevel> {
std::size_t operator()(const ReplacedTextureLevel &k) const {
#if PPSSPP_ARCH(64BIT)
uint64_t v = (uint64_t)k.w | ((uint64_t)k.h << 32);
v = __rotl64(v ^ (uint64_t)k.fmt, 13);
#else
uint32_t v = k.w ^ (uint32_t)k.fmt;
v = __rotl(__rotl(v, 13) ^ k.h, 13);
#endif
return v ^ hash<string>()(k.file.ToString());
}
};
}
struct ReplacedLevelCache {
std::mutex lock;
std::vector<uint8_t> data;
double lastUsed = 0.0;
};
struct ReplacementCacheKey {
@ -169,7 +204,7 @@ protected:
void PurgeIfOlder(double t);
std::vector<ReplacedTextureLevel> levels_;
std::vector<std::vector<uint8_t>> levelData_;
std::vector<ReplacedLevelCache *> levelData_;
ReplacedTextureAlpha alphaStatus_ = ReplacedTextureAlpha::UNKNOWN;
double lastUsed_ = 0.0;
LimitedWaitable *threadWaitable_ = nullptr;
@ -240,7 +275,8 @@ protected:
std::string LookupHashFile(u64 cachekey, u32 hash, int level);
std::string HashName(u64 cachekey, u32 hash, int level);
void PopulateReplacement(ReplacedTexture *result, u64 cachekey, u32 hash, int w, int h);
bool PopulateLevel(ReplacedTextureLevel &level);
bool PopulateLevelFromPath(ReplacedTextureLevel &level, bool ignoreError);
bool PopulateLevelFromZip(ReplacedTextureLevel &level, bool ignoreError);
bool enabled_ = false;
bool allowVideo_ = false;
@ -248,10 +284,13 @@ protected:
bool reduceHash_ = false;
float reduceHashSize = 1.0; // default value with reduceHash to false
float reduceHashGlobalValue = 0.5; // Global value for textures dump pngs of all sizes, 0.5 by default but can be set in textures.ini
double lastTextureCacheSizeGB_ = 0.0;
bool ignoreMipmap_ = false;
std::string gameID_;
Path basePath_;
ReplacedTextureHash hash_ = ReplacedTextureHash::QUICK;
zip *zip_ = nullptr;
typedef std::pair<int, int> WidthHeightPair;
std::unordered_map<u64, WidthHeightPair> hashranges_;
std::unordered_map<u64, float> reducehashranges_;
@ -261,4 +300,5 @@ protected:
ReplacedTexture none_;
std::unordered_map<ReplacementCacheKey, ReplacedTexture> cache_;
std::unordered_map<ReplacementCacheKey, std::pair<ReplacedTextureLevel, double>> savedCache_;
std::unordered_map<ReplacedTextureLevel, ReplacedLevelCache> levelCache_;
};

4
ext/libzip/config.h
View File

@ -27,11 +27,11 @@
#endif
#ifndef MS_UWP
#define HAVE_FILENO
#define HAVE_FSEEKO
#define HAVE_FTELLO
#define HAVE_GETPROGNAME
#endif
#ifndef _WIN32
#define HAVE_FSEEKO
#define HAVE_FTELLO
#define HAVE_LOCALTIME_R
#define HAVE_MKSTEMP 1
#endif