Merge pull request #17072 from hrydgard/more-vfs-work

More VFS work
This commit is contained in:
Henrik Rydgård 2023-03-07 21:58:43 +01:00 committed by GitHub
commit 81817faf1d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 262 additions and 44 deletions

View File

@ -567,9 +567,9 @@ bool IniFile::Load(const Path &path)
return success;
}
bool IniFile::LoadFromVFS(const std::string &filename) {
bool IniFile::LoadFromVFS(VFSInterface &vfs, const std::string &filename) {
size_t size;
uint8_t *data = g_VFS.ReadFile(filename.c_str(), &size);
uint8_t *data = vfs.ReadFile(filename.c_str(), &size);
if (!data)
return false;
std::string str((const char*)data, size);
@ -582,10 +582,10 @@ bool IniFile::LoadFromVFS(const std::string &filename) {
bool IniFile::Load(std::istream &in) {
// Maximum number of letters in a line
static const int MAX_BYTES = 1024*32;
char *templine = new char[MAX_BYTES]; // avoid using up massive stack space
while (!(in.eof() || in.fail()))
{
char templine[MAX_BYTES];
in.getline(templine, MAX_BYTES);
std::string line = templine;
@ -624,6 +624,7 @@ bool IniFile::Load(std::istream &in) {
}
}
delete[] templine;
return true;
}

View File

@ -11,6 +11,8 @@
#include "Common/File/Path.h"
class VFSInterface;
class Section {
friend class IniFile;
@ -86,7 +88,7 @@ public:
bool Load(const Path &path);
bool Load(const std::string &filename) { return Load(Path(filename)); }
bool Load(std::istream &istream);
bool LoadFromVFS(const std::string &filename);
bool LoadFromVFS(VFSInterface &vfs, const std::string &filename);
bool Save(const Path &path);
bool Save(const std::string &filename) { return Save(Path(filename)); }

View File

@ -91,7 +91,7 @@ bool I18NRepo::LoadIni(const std::string &languageID, const Path &overridePath)
iniPath = GetIniPath(languageID);
}
if (!ini.LoadFromVFS(iniPath.ToString()))
if (!ini.LoadFromVFS(g_VFS, iniPath.ToString()))
return false;
Clear();
@ -127,7 +127,7 @@ I18NCategory *I18NRepo::LoadSection(const Section *section, const char *name) {
return cat;
}
// This is a very light touched save variant - it won't overwrite
// This is a very light touched save variant - it won't overwrite
// anything, only create new entries.
void I18NRepo::SaveIni(const std::string &languageID) {
IniFile ini;

View File

@ -29,3 +29,57 @@ bool DirectoryReader::GetFileInfo(const char *path, File::FileInfo *info) {
Path new_path = Path(path).StartsWith(path_) ? Path(path) : path_ / path;
return File::GetFileInfo(new_path, info);
}
class DirectoryReaderFileReference : public VFSFileReference {
public:
Path path;
};
class DirectoryReaderOpenFile : public VFSOpenFile {
public:
FILE *file;
};
VFSFileReference *DirectoryReader::GetFile(const char *path) {
DirectoryReaderFileReference *reference = new DirectoryReaderFileReference();
reference->path = path_ / path;
return reference;
}
bool DirectoryReader::GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) {
DirectoryReaderFileReference *reference = (DirectoryReaderFileReference *)vfsReference;
return File::GetFileInfo(reference->path, fileInfo);
}
void DirectoryReader::ReleaseFile(VFSFileReference *vfsReference) {
DirectoryReaderFileReference *reference = (DirectoryReaderFileReference *)vfsReference;
delete reference;
}
VFSOpenFile *DirectoryReader::OpenFileForRead(VFSFileReference *vfsReference) {
DirectoryReaderFileReference *reference = (DirectoryReaderFileReference *)vfsReference;
FILE *file = File::OpenCFile(reference->path, "rb");
if (!file) {
return nullptr;
}
DirectoryReaderOpenFile *openFile = new DirectoryReaderOpenFile();
openFile->file = file;
return openFile;
}
void DirectoryReader::Rewind(VFSOpenFile *vfsOpenFile) {
DirectoryReaderOpenFile *openFile = (DirectoryReaderOpenFile *)vfsOpenFile;
fseek(openFile->file, 0, SEEK_SET);
}
size_t DirectoryReader::Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) {
DirectoryReaderOpenFile *openFile = (DirectoryReaderOpenFile *)vfsOpenFile;
return fread(buffer, 1, length, openFile->file);
}
void DirectoryReader::CloseFile(VFSOpenFile *vfsOpenFile) {
DirectoryReaderOpenFile *openFile = (DirectoryReaderOpenFile *)vfsOpenFile;
fclose(openFile->file);
delete openFile;
}

View File

@ -9,6 +9,16 @@ public:
explicit DirectoryReader(const Path &path);
// use delete[] on the returned value.
uint8_t *ReadFile(const char *path, size_t *size) override;
VFSFileReference *GetFile(const char *path) override;
bool GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) override;
void ReleaseFile(VFSFileReference *vfsReference) override;
VFSOpenFile *OpenFileForRead(VFSFileReference *vfsReference) override;
void Rewind(VFSOpenFile *vfsOpenFile) override;
size_t Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) override;
void CloseFile(VFSOpenFile *vfsOpenFile) override;
bool GetFileListing(const char *path, std::vector<File::FileInfo> *listing, const char *filter) override;
bool GetFileInfo(const char *path, File::FileInfo *info) override;
std::string toString() const override {
@ -18,4 +28,3 @@ public:
private:
Path path_;
};

View File

@ -8,8 +8,12 @@
VFS g_VFS;
void VFS::Register(const char *prefix, VFSBackend *reader) {
entries_.push_back(VFSEntry{ prefix, reader });
DEBUG_LOG(IO, "Registered VFS for prefix %s: %s", prefix, reader->toString().c_str());
if (reader) {
entries_.push_back(VFSEntry{ prefix, reader });
DEBUG_LOG(IO, "Registered VFS for prefix %s: %s", prefix, reader->toString().c_str());
} else {
ERROR_LOG(IO, "Trying to register null VFS backend for prefix %s", prefix);
}
}
void VFS::Clear() {

View File

@ -14,19 +14,51 @@
// on the system level, like loading assets, and maybe texture packs. Also, as mentioned,
// this one is read-only, so a bit smaller and simpler.
class VFSBackend {
// VFSBackend instances can be used on their own, without the VFS, to serve as an abstraction of
// a single directory or ZIP file.
// The VFSFileReference level of abstraction is there to hold things like zip file indices,
// for fast re-open etc.
class VFSFileReference {
public:
virtual ~VFSBackend() {}
// use delete[] to release the returned memory.
virtual ~VFSFileReference() {}
};
class VFSOpenFile {
public:
virtual ~VFSOpenFile() {}
};
// Common interface parts between VFSBackend and VFS.
// Sometimes you don't need the VFS multiplexing and only have a VFSBackend *, sometimes you do need it,
// and it would be cool to be able to use the same interface, like when loading INI files.
class VFSInterface {
public:
virtual ~VFSInterface() {}
virtual uint8_t *ReadFile(const char *path, size_t *size) = 0;
virtual bool GetFileListing(const char *path, std::vector<File::FileInfo> *listing, const char *filter = nullptr) = 0;
};
class VFSBackend : public VFSInterface {
public:
// use delete[] to release the returned memory.
virtual VFSFileReference *GetFile(const char *path) = 0;
virtual bool GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) = 0;
virtual void ReleaseFile(VFSFileReference *vfsReference) = 0;
virtual VFSOpenFile *OpenFileForRead(VFSFileReference *vfsReference) = 0;
virtual void Rewind(VFSOpenFile *vfsOpenFile) = 0;
virtual size_t Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) = 0;
virtual void CloseFile(VFSOpenFile *vfsOpenFile) = 0;
// Filter support is optional but nice to have
virtual bool GetFileListing(const char *path, std::vector<File::FileInfo> *listing, const char *filter = 0) = 0;
virtual bool GetFileInfo(const char *path, File::FileInfo *info) = 0;
virtual std::string toString() const = 0;
};
class VFS {
class VFS : public VFSInterface {
public:
~VFS() { Clear(); }
void Register(const char *prefix, VFSBackend *reader);
@ -35,9 +67,9 @@ public:
// Use delete [] to release the returned memory.
// Always allocates an extra zero byte at the end, so that it
// can be used for text like shader sources.
uint8_t *ReadFile(const char *filename, size_t *size);
bool GetFileListing(const char *path, std::vector<File::FileInfo> *listing, const char *filter = 0);
uint8_t *ReadFile(const char *filename, size_t *size) override;
bool GetFileInfo(const char *filename, File::FileInfo *fileInfo);
bool GetFileListing(const char *path, std::vector<File::FileInfo> *listing, const char *filter = nullptr) override;
private:
struct VFSEntry {

View File

@ -34,18 +34,29 @@ static uint8_t *ReadFromZip(zip *archive, const char* filename, size_t *size) {
return contents;
}
ZipFileReader::ZipFileReader(const Path &zipFile, const char *inZipPath) {
ZipFileReader *ZipFileReader::Create(const Path &zipFile, const char *inZipPath) {
int error = 0;
zip *zip_file;
if (zipFile.Type() == PathType::CONTENT_URI) {
int fd = File::OpenFD(zipFile, File::OPEN_READ);
zip_file_ = zip_fdopen(fd, 0, &error);
if (!fd) {
ERROR_LOG(IO, "Failed to open FD for %s as zip file", zipFile.c_str());
return nullptr;
}
zip_file = zip_fdopen(fd, 0, &error);
} else {
zip_file_ = zip_open(zipFile.c_str(), 0, &error);
zip_file = zip_open(zipFile.c_str(), 0, &error);
}
truncate_cpy(inZipPath_, inZipPath);
if (!zip_file_) {
if (!zip_file) {
ERROR_LOG(IO, "Failed to open %s as a zip file", zipFile.c_str());
return nullptr;
}
ZipFileReader *reader = new ZipFileReader();
reader->zip_file_ = zip_file;
truncate_cpy(reader->inZipPath_, inZipPath);
return reader;
}
ZipFileReader::~ZipFileReader() {
@ -78,6 +89,7 @@ bool ZipFileReader::GetFileListing(const char *orig_path, std::vector<File::File
filter++;
}
}
if (tmp.size())
filters.insert("." + tmp);
@ -149,12 +161,16 @@ bool ZipFileReader::GetFileInfo(const char *path, File::FileInfo *info) {
struct zip_stat zstat;
char temp_path[1024];
snprintf(temp_path, sizeof(temp_path), "%s%s", inZipPath_, path);
if (0 != zip_stat(zip_file_, temp_path, ZIP_FL_NOCASE | ZIP_FL_UNCHANGED, &zstat)) {
// ZIP files do not have real directories, so we'll end up here if we
// try to stat one. For now that's fine.
info->exists = false;
info->size = 0;
return false;
{
std::lock_guard<std::mutex> guard(lock_);
if (0 != zip_stat(zip_file_, temp_path, ZIP_FL_NOCASE | ZIP_FL_UNCHANGED, &zstat)) {
// ZIP files do not have real directories, so we'll end up here if we
// try to stat one. For now that's fine.
info->exists = false;
info->size = 0;
return false;
}
}
info->fullName = Path(path);
@ -164,3 +180,87 @@ bool ZipFileReader::GetFileInfo(const char *path, File::FileInfo *info) {
info->size = zstat.size;
return true;
}
class ZipFileReaderFileReference : public VFSFileReference {
public:
int zi;
};
class ZipFileReaderOpenFile : public VFSOpenFile {
public:
ZipFileReaderFileReference *reference;
zip_file_t *zf;
};
static constexpr zip_uint64_t INVALID_ZIP_SIZE = 0xFFFFFFFFFFFFFFFFULL;
VFSFileReference *ZipFileReader::GetFile(const char *path) {
std::lock_guard<std::mutex> guard(lock_);
int zi = zip_name_locate(zip_file_, path, ZIP_FL_NOCASE);
if (zi < 0) {
// Not found.
return nullptr;
}
ZipFileReaderFileReference *ref = new ZipFileReaderFileReference();
ref->zi = zi;
return ref;
}
bool ZipFileReader::GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) {
ZipFileReaderFileReference *reference = (ZipFileReaderFileReference *)vfsReference;
// If you crash here, you called this while having the lock held by having the file open.
// Don't do that, check the info before you open the file.
std::lock_guard<std::mutex> guard(lock_);
zip_stat_t zstat;
if (zip_stat_index(zip_file_, reference->zi, 0, &zstat) != 0)
return false;
*fileInfo = File::FileInfo{};
fileInfo->size = 0;
if (zstat.valid & ZIP_STAT_SIZE)
fileInfo->size = zstat.size;
return zstat.size;
}
void ZipFileReader::ReleaseFile(VFSFileReference *vfsReference) {
ZipFileReaderFileReference *reference = (ZipFileReaderFileReference *)vfsReference;
// Don't do anything other than deleting it.
delete reference;
}
VFSOpenFile *ZipFileReader::OpenFileForRead(VFSFileReference *vfsReference) {
ZipFileReaderFileReference *reference = (ZipFileReaderFileReference *)vfsReference;
ZipFileReaderOpenFile *openFile = new ZipFileReaderOpenFile();
openFile->reference = reference;
// We only allow one file to be open for read concurrently. It's possible that this can be improved,
// especially if we only access by index like this.
lock_.lock();
openFile->zf = zip_fopen_index(zip_file_, reference->zi, 0);
if (!openFile->zf) {
WARN_LOG(G3D, "File with index %d not found in zip", reference->zi);
lock_.unlock();
return nullptr;
}
return openFile;
}
void ZipFileReader::Rewind(VFSOpenFile *vfsOpenFile) {
ZipFileReaderOpenFile *openFile = (ZipFileReaderOpenFile *)vfsOpenFile;
// Close and re-open.
zip_fclose(openFile->zf);
openFile->zf = zip_fopen_index(zip_file_, openFile->reference->zi, 0);
}
size_t ZipFileReader::Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) {
ZipFileReaderOpenFile *file = (ZipFileReaderOpenFile *)vfsOpenFile;
return zip_fread(file->zf, buffer, length);
}
void ZipFileReader::CloseFile(VFSOpenFile *vfsOpenFile) {
ZipFileReaderOpenFile *file = (ZipFileReaderOpenFile *)vfsOpenFile;
_dbg_assert_(file->zf != nullptr);
zip_fclose(file->zf);
lock_.unlock();
delete file;
}

View File

@ -16,10 +16,23 @@
class ZipFileReader : public VFSBackend {
public:
ZipFileReader(const Path &zipFile, const char *inZipPath);
static ZipFileReader *Create(const Path &zipFile, const char *inZipPath);
~ZipFileReader();
bool IsValid() const { return zip_file_ != nullptr; }
// use delete[] on the returned value.
uint8_t *ReadFile(const char *path, size_t *size) override;
VFSFileReference *GetFile(const char *path) override;
bool GetFileInfo(VFSFileReference *vfsReference, File::FileInfo *fileInfo) override;
void ReleaseFile(VFSFileReference *vfsReference) override;
VFSOpenFile *OpenFileForRead(VFSFileReference *vfsReference) override;
void Rewind(VFSOpenFile *vfsOpenFile) override;
size_t Read(VFSOpenFile *vfsOpenFile, void *buffer, size_t length) override;
void CloseFile(VFSOpenFile *vfsOpenFile) override;
bool GetFileListing(const char *path, std::vector<File::FileInfo> *listing, const char *filter) override;
bool GetFileInfo(const char *path, File::FileInfo *info) override;
std::string toString() const override {
@ -29,7 +42,7 @@ public:
private:
void GetZipListings(const char *path, std::set<std::string> &files, std::set<std::string> &directories);
zip *zip_file_;
zip *zip_file_ = nullptr;
std::mutex lock_;
char inZipPath_[256];
};

View File

@ -19,6 +19,7 @@
#include "Common/Log.h"
#include "Common/Data/Format/IniFile.h"
#include "Common/File/VFS/VFS.h"
#include "Common/StringUtils.h"
#include "Core/Compatibility.h"
#include "Core/Config.h"
@ -38,7 +39,7 @@ void Compatibility::Load(const std::string &gameID) {
{
IniFile compat;
// This loads from assets.
if (compat.LoadFromVFS("compat.ini")) {
if (compat.LoadFromVFS(g_VFS, "compat.ini")) {
CheckSettings(compat, gameID);
}
}
@ -55,7 +56,7 @@ void Compatibility::Load(const std::string &gameID) {
{
IniFile compat;
// This loads from assets.
if (compat.LoadFromVFS("compatvr.ini")) {
if (compat.LoadFromVFS(g_VFS, "compatvr.ini")) {
CheckVRSettings(compat, gameID);
}
}

View File

@ -39,6 +39,7 @@
#include "Common/Data/Text/Parsers.h"
#include "Common/CPUDetect.h"
#include "Common/File/FileUtil.h"
#include "Common/File/VFS/VFS.h"
#include "Common/LogManager.h"
#include "Common/OSVersion.h"
#include "Common/System/Display.h"
@ -436,7 +437,7 @@ const char *DefaultLangRegion() {
} else if (langRegion.length() >= 3) {
// Don't give up. Let's try a fuzzy match - so nl_BE can match nl_NL.
IniFile mapping;
mapping.LoadFromVFS("langregion.ini");
mapping.LoadFromVFS(g_VFS, "langregion.ini");
std::vector<std::string> keys;
mapping.GetKeys("LangRegionNames", keys);
@ -1222,7 +1223,7 @@ Config::~Config() {
void Config::LoadLangValuesMapping() {
IniFile mapping;
mapping.LoadFromVFS("langregion.ini");
mapping.LoadFromVFS(g_VFS, "langregion.ini");
std::vector<std::string> keys;
mapping.GetKeys("LangRegionNames", keys);
@ -1932,8 +1933,8 @@ bool Config::loadGameConfig(const std::string &pGameId, const std::string &title
});
KeyMap::LoadFromIni(iniFile);
if (!appendedConfigFileName_.ToString().empty() &&
if (!appendedConfigFileName_.ToString().empty() &&
std::find(appendedConfigUpdatedGames_.begin(), appendedConfigUpdatedGames_.end(), pGameId) == appendedConfigUpdatedGames_.end()) {
LoadAppendedConfig();

View File

@ -35,6 +35,7 @@
#include "Common/Data/Text/I18n.h"
#include "Common/Data/Text/Parsers.h"
#include "Common/File/FileUtil.h"
#include "Common/File/VFS/VFS.h"
#include "Common/LogReporting.h"
#include "Common/StringUtils.h"
#include "Common/Thread/ParallelLoop.h"
@ -179,7 +180,7 @@ bool TextureReplacer::LoadIni() {
}
if (!iniLoaded) {
iniLoaded = ini.LoadFromVFS((basePath_ / INI_FILENAME).ToString());
iniLoaded = ini.LoadFromVFS(g_VFS, (basePath_ / INI_FILENAME).ToString());
}
if (iniLoaded) {
@ -196,7 +197,7 @@ bool TextureReplacer::LoadIni() {
std::lock_guard<std::mutex> guard(zip_.lock);
iniLoaded = LoadIniZip(overrideIni, zip_.z, overrideFilename);
} else {
iniLoaded = overrideIni.LoadFromVFS((basePath_ / overrideFilename).ToString());
iniLoaded = overrideIni.LoadFromVFS(g_VFS, (basePath_ / overrideFilename).ToString());
}
if (!iniLoaded) {
ERROR_LOG(G3D, "Failed to load extra texture ini: %s", overrideFilename.c_str());
@ -298,7 +299,7 @@ bool TextureReplacer::LoadIniValues(IniFile &ini, bool isOverride) {
if (ini.HasSection("reducehashranges")) {
auto reducehashranges = ini.GetOrCreateSection("reducehashranges")->ToMap();
// Format: w,h = reducehashvalues
// Format: w,h = reducehashvalues
for (const auto& item : reducehashranges) {
ParseReduceHashRange(item.first, item.second);
}
@ -390,7 +391,7 @@ void TextureReplacer::ParseReduceHashRange(const std::string& key, const std::st
const u64 reducerangeKey = ((u64)forW << 16) | forH;
reducehashranges_[reducerangeKey] = rhashvalue;
}
}
u32 TextureReplacer::ComputeHash(u32 addr, int bufw, int w, int h, GETextureFormat fmt, u16 maxSeenV) {
_dbg_assert_msg_(enabled_, "Replacement not enabled");

View File

@ -90,7 +90,7 @@ void LoadPostShaderInfo(Draw::DrawContext *draw, const std::vector<Path> &direct
if (path.ToString().substr(0, 7) == "assets/")
path = Path(path.ToString().substr(7));
if (ini.LoadFromVFS(name.ToString()) || ini.Load(fileInfo[f].fullName)) {
if (ini.LoadFromVFS(g_VFS, name.ToString()) || ini.Load(fileInfo[f].fullName)) {
success = true;
// vsh load. meh.
}

View File

@ -104,7 +104,7 @@ static void LoadThemeInfo(const std::vector<Path> &directories) {
if (path.ToString().substr(0, 7) == "assets/")
path = Path(path.ToString().substr(7));
if (ini.LoadFromVFS(name.ToString()) || ini.Load(fileInfo[f].fullName)) {
if (ini.LoadFromVFS(g_VFS, name.ToString()) || ini.Load(fileInfo[f].fullName)) {
success = true;
}

View File

@ -696,7 +696,7 @@ extern "C" void Java_org_ppsspp_ppsspp_NativeApp_init
deviceType = jdeviceType;
Path apkPath(GetJavaString(env, japkpath));
g_VFS.Register("", new ZipFileReader(apkPath, "assets/"));
g_VFS.Register("", ZipFileReader::Create(apkPath, "assets/"));
systemName = GetJavaString(env, jmodel);
langRegion = GetJavaString(env, jlangRegion);

View File

@ -483,10 +483,10 @@ int main(int argc, const char* argv[])
#if PPSSPP_PLATFORM(ANDROID)
// For some reason the debugger installs it with this name?
if (File::Exists(Path("/data/app/org.ppsspp.ppsspp-2.apk"))) {
g_VFS.Register("", new ZipFileReader(Path("/data/app/org.ppsspp.ppsspp-2.apk"), "assets/"));
g_VFS.Register("", ZipFileReader::Create(Path("/data/app/org.ppsspp.ppsspp-2.apk"), "assets/"));
}
if (File::Exists(Path("/data/app/org.ppsspp.ppsspp.apk"))) {
g_VFS.Register("", new ZipFileReader(Path("/data/app/org.ppsspp.ppsspp.apk"), "assets/"));
g_VFS.Register("", ZipFileReader::Create(Path("/data/app/org.ppsspp.ppsspp.apk"), "assets/"));
}
#elif !PPSSPP_PLATFORM(WINDOWS)
g_VFS.Register("", new DirectoryReader(g_Config.flash0Directory / ".."));