cemu-project/WudCompress
1
0
Fork 0
mirror of https://github.com/cemu-project/WudCompress.git synced 2024-11-23 03:09:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add files via upload

Browse Source
This commit is contained in:
TeamCemu 2018-06-10 17:23:53 +02:00 committed by GitHub
parent 2d391a8083
commit b0ab5f6b6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 751 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
WudCompress.sln Normal file
View File

@ -0,0 +1,20 @@

Microsoft Visual Studio Solution File, Format Version 10.00
# Visual Studio 2008
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "WudCompress", "WudCompress\WudCompress.vcproj", "{6FC9B74D-FB07-4173-B7CB-86D34867BCD7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
Release|Win32 = Release|Win32
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{6FC9B74D-FB07-4173-B7CB-86D34867BCD7}.Debug|Win32.ActiveCfg = Debug|Win32
{6FC9B74D-FB07-4173-B7CB-86D34867BCD7}.Debug|Win32.Build.0 = Debug|Win32
{6FC9B74D-FB07-4173-B7CB-86D34867BCD7}.Release|Win32.ActiveCfg = Release|Win32
{6FC9B74D-FB07-4173-B7CB-86D34867BCD7}.Release|Win32.Build.0 = Release|Win32
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

193
WudCompress/WudCompress.vcproj Normal file
View File

@ -0,0 +1,193 @@
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="9,00"
Name="WudCompress"
ProjectGUID="{6FC9B74D-FB07-4173-B7CB-86D34867BCD7}"
RootNamespace="WudCompress"
TargetFrameworkVersion="196613"
>
<Platforms>
<Platform
Name="Win32"
/>
</Platforms>
<ToolFiles>
</ToolFiles>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="$(SolutionDir)$(ConfigurationName)"
IntermediateDirectory="$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
WarningLevel="3"
DebugInformationFormat="4"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
GenerateDebugInformation="true"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)$(ConfigurationName)"
IntermediateDirectory="$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
FavorSizeOrSpeed="1"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
WarningLevel="3"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
GenerateDebugInformation="true"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
>
<File
RelativePath=".\main.cpp"
>
</File>
<File
RelativePath=".\wud.cpp"
>
</File>
<File
RelativePath=".\wud.h"
>
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
>
</Filter>
<Filter
Name="Resource Files"
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
>
</Filter>
</Files>
<Globals>
</Globals>
</VisualStudioProject>

349
WudCompress/main.cpp Normal file
View File

@ -0,0 +1,349 @@
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include"wud.h"
/*
* WUX file structure (v1.0):
[Header]
UINT32 magic1 "WUX0"
UINT32 magic2 0x1099d02e
UINT32 sectorSize Size per uncompressed sector (SECTOR_SIZE constant)
UINT64 uncompressedSize Size of the Wii U image before being compressed
UINT32 flags Enable optional parts of the header (not used right now)
[SectorIndexTable]
UINT32[] lookupIndex table of indices for lookup of each sector. To calculate number of entries in this array: sectorCount = (uncompressedSize+sectorSize-1)/sectorSize
[SectorData]
UINT8[] padding Padding until the next field (sectorData) is aligned to sectorSize bytes. You can write whatever data you want here
UINT8[] sectorData Array of unique sectors. Size in bytes: sectorSize * sectorCount
*/
#define SECTOR_SIZE (0x8000)
#define SECTOR_HASH_SIZE (32)
/*
* Hash function used to create a hash of each sector
* The hashes are then compared to find duplicate sectors
*/
void calculateHash256(unsigned char* data, unsigned int length, unsigned char* hashOut)
{
// cheap and simple hash implementation
// you can replace this part with your favorite hash method
memset(hashOut, 0x00, 32);
for(unsigned int i=0; i<length; i++)
{
hashOut[i%32] ^= data[i];
hashOut[(i+7)%32] += data[i];
}
}
/*
* Compare content of two WUD/WUX files
* Used to compare uncompressed and compressed version of a WUD
* Returns false if there is even a single byte of difference
*/
bool validateWUX(char* wud1Path, char* wud2Path)
{
puts("Checking for errors...");
wud_t* wudFile1 = wud_open(wud1Path);
wud_t* wudFile2 = wud_open(wud2Path);
if( wudFile1 == NULL )
{
printf("Failed to open \"%s\"\n", wud1Path);
if( wudFile2 == NULL )
wud_close(wudFile2);
return false;
}
if( wudFile2 == NULL )
{
printf("Failed to open \"%s\"\n", wud2Path);
if( wudFile1 == NULL )
wud_close(wudFile1);
return false;
}
// get and compare sizes
long long wud1Size = wud_getWUDSize(wudFile1);
long long wud2Size = wud_getWUDSize(wudFile2);
if( wud1Size != wud2Size )
{
printf("WUD data size mismatch\n");
return false;
}
// compare data
long long currentValidationOffset = 0;
unsigned int tempBufferSize = 1024*1024+19; // 1MB + some extra bytes to make the number uneven (we want to provoke cross-sector reads)
unsigned char* tempBufferWUD1 = (unsigned char*)malloc(tempBufferSize);
unsigned char* tempBufferWUD2 = (unsigned char*)malloc(tempBufferSize);
bool dataMismatch = false;
int pct = -1;
printf("0%\r");
while( currentValidationOffset < wud1Size )
{
// calculate how many bytes we are reading in this cycle
long long remainingBytes = wud1Size - currentValidationOffset;
unsigned int bytesToRead = tempBufferSize;
if( remainingBytes < (long long)bytesToRead )
bytesToRead = (unsigned int)remainingBytes;
unsigned int readByteCount1 = wud_readData(wudFile1, tempBufferWUD1, bytesToRead, currentValidationOffset);
unsigned int readByteCount2 = wud_readData(wudFile2, tempBufferWUD2, bytesToRead, currentValidationOffset);
if( readByteCount1 != readByteCount2 || bytesToRead != readByteCount1 )
{
printf("Data read size mismatch\n");
dataMismatch = true;
break;
}
// compare buffers
if( memcmp(tempBufferWUD1, tempBufferWUD2, bytesToRead) != 0 )
{
printf("Data mismatch\n");
dataMismatch = true;
break;
}
// progress offset
currentValidationOffset += bytesToRead;
// display current progress
int newPct = (int)(currentValidationOffset*1000LL / wud1Size);
if( newPct != pct )
{
printf("%d.%d%% \r", (newPct/10), (newPct%10));
pct = newPct;
}
}
puts("");
free(tempBufferWUD1);
free(tempBufferWUD2);
wud_close(wudFile1);
wud_close(wudFile2);
return dataMismatch == false;
}
bool compressWUD(wud_t* inputFile, FILE* outputFile, char* outputPath)
{
long long inputSize = wud_getWUDSize(inputFile);
// write header
wuxHeader_t wuxHeader = {0};
wuxHeader.magic0 = WUX_MAGIC_0;
wuxHeader.magic1 = WUX_MAGIC_1;
wuxHeader.sectorSize = SECTOR_SIZE;
wuxHeader.uncompressedSize = inputSize;
wuxHeader.flags = 0;
fwrite(&wuxHeader, sizeof(wuxHeader_t), 1, outputFile);
unsigned int sectorTableEntryCount = (unsigned int)((inputSize+SECTOR_SIZE-1) / (long long)SECTOR_SIZE);
// remember current seek offset, this is where the index table will be written after compression is done
long long offsetIndexTable = _ftelli64(outputFile);
// skip index table and padding
long long offsetSectorArrayStart = (offsetIndexTable + (long long)sectorTableEntryCount*sizeof(unsigned int));
// align to SECTOR_SIZE
offsetSectorArrayStart = (offsetSectorArrayStart + SECTOR_SIZE - 1);
offsetSectorArrayStart = offsetSectorArrayStart - (offsetSectorArrayStart%SECTOR_SIZE);
_fseeki64(outputFile, offsetSectorArrayStart, SEEK_SET);
unsigned int indexTableSize = sizeof(unsigned int) * sectorTableEntryCount;
unsigned int* sectorIndexTable = (unsigned int*)malloc(sizeof(unsigned int) * sectorTableEntryCount);
unsigned char* sectorHashArray = (unsigned char*)malloc(sizeof(unsigned char) * SECTOR_HASH_SIZE * sectorTableEntryCount);
unsigned int uniqueSectorCount = 0;
printf("Compressing %d sectors...\n", sectorTableEntryCount);
unsigned char buffer[SECTOR_SIZE];
unsigned char sectorHash[32];
unsigned int storedSectors = 0;
int currentPct = 0;
long long compressedSize = offsetSectorArrayStart;
for(unsigned int i=0; i<sectorTableEntryCount; i++)
{
// print status
int newPct = ((i+1)*1000)/sectorTableEntryCount;
if( currentPct != newPct )
{
currentPct = newPct;
int compressionRatio = (int)(((long long)i * SECTOR_SIZE)*10 / compressedSize);
printf("\r%d.%d%% Compression ratio: 1:%d.%d \r", currentPct/10, currentPct%10, compressionRatio/10, compressionRatio%10);
}
// read sector and generate hash
wud_readData(inputFile, buffer, SECTOR_SIZE, (long long)i * (long long)SECTOR_SIZE);
calculateHash256(buffer, SECTOR_SIZE, sectorHash);
unsigned int sectorReuseIndex = 0xFFFFFFFF;
// try to locate any previous sector with same hash
for(unsigned int f=0; f<uniqueSectorCount; f++)
{
if( memcmp(sectorHash, sectorHashArray+f*SECTOR_HASH_SIZE, SECTOR_HASH_SIZE) == 0 )
{
sectorReuseIndex = f;
break;
}
}
// if we found a sector then just store the index
if( sectorReuseIndex != 0xFFFFFFFF )
{
sectorIndexTable[i] = sectorReuseIndex;
continue;
}
// else store the sector and append a new index
fwrite(buffer, SECTOR_SIZE, 1, outputFile);
memcpy(sectorHashArray+uniqueSectorCount*SECTOR_HASH_SIZE, sectorHash, SECTOR_HASH_SIZE);
compressedSize += SECTOR_SIZE;
sectorIndexTable[i] = uniqueSectorCount;
uniqueSectorCount++;
storedSectors++;
}
printf("100%% \n");
_fseeki64(outputFile, offsetIndexTable, SEEK_SET);
fwrite(sectorIndexTable, sectorTableEntryCount, sizeof(unsigned int), outputFile);
fclose(outputFile);
puts("done");
return true;
}
bool decompressWUD(wud_t* inputFile, FILE* outputFile, char* outputPath)
{
long long inputSize = wud_getWUDSize(inputFile);
printf("Decompressing...\n");
unsigned char buffer[SECTOR_SIZE];
long long currentIndex = 0;
int currentPct = 0;
while( currentIndex < inputSize )
{
// print status
int newPct = (int)((currentIndex*1000LL)/inputSize);
if( currentPct != newPct )
{
currentPct = newPct;
printf("\r%d.%d%% \r", currentPct/10, currentPct%10);
}
// calculate how many bytes to read
int bytesToRead = SECTOR_SIZE;
if( (inputSize-currentIndex) < SECTOR_SIZE )
bytesToRead = (int)(inputSize-currentIndex);
// read data
wud_readData(inputFile, buffer, bytesToRead, currentIndex);
// write data
fwrite(buffer, bytesToRead, 1, outputFile);
currentIndex += (long long)bytesToRead;
}
printf("100%% \n");
fclose(outputFile);
puts("done");
return true;
}
int main(int argc, char *argv[])
{
if( argc < 2 )
{
puts("Wii U image compression tool v1.0 by Exzap");
puts("Lossless compression and decompression for Wii U dumps.");
puts("");
puts("Usage:");
puts("WudCompress <game.wud/game.wux> [-noverify]");
puts("");
puts("Parameters:");
puts("-noverify Skip the file validation step at the end");
return 0;
}
char* wudPath = argv[1];
// parse options
bool skipVerify = false;
for(int i=2; i<argc; i++)
{
if( stricmp(argv[i], "-noverify") == 0 )
{
skipVerify = true;
}
else
{
printf("Unknown option: %s\n", argv[i]);
return -1;
}
}
// verify path
if( wudPath[0] == '-' )
{
puts("Invalid input file");
return -1;
}
// open input file
wud_t* wud = wud_open(wudPath);
if( wud == NULL )
{
printf("Unable to open input file \"%s\"\n", wudPath);
return -2;
}
// create path of output file by replacing the extension with .wux
char* outputPath = (char*)malloc(strlen(wudPath)+4+1); // allocate space for up to 4 extra characters in case we need to add the .wux extension
strcpy(outputPath, wudPath);
// replace with opposite extension (wux <-> wud)
char* newExtension;
if( wud_isWUXCompressed(wud) )
{
printf("Mode: Decompress\n");
newExtension = ".wud";
}
else
{
printf("Mode: Compress\n");
newExtension = ".wux";
}
bool extensionFound = false;
for(int i=strlen(outputPath)-1; i>=0; i--)
{
if( outputPath[i] == '.' )
{
extensionFound = true;
strcpy(outputPath+i, newExtension);
break;
}
}
if( extensionFound == false )
strcat(outputPath, newExtension);
// make sure the output file doesn't already exist (avoid accidental overwriting)
FILE* outputFile;
outputFile = fopen(outputPath, "r");
if( outputFile != NULL )
{
printf("Output file \"%s\" already exists.\n", outputPath);
wud_close(wud);
return -4;
}
// open output file
outputFile = fopen(outputPath, "wb");
if( outputFile == NULL )
{
printf("Unable to create output file\n");
wud_close(wud);
return -3;
}
printf("Input:\n");
puts(wudPath);
printf("Output:\n");
puts(outputPath);
if( wud_isWUXCompressed(wud) )
{
if( decompressWUD(wud, outputFile, outputPath) == false )
return -1;
}
else
{
if( compressWUD(wud, outputFile, outputPath) == false )
return -1;
}
// verify
if( skipVerify == false )
{
if( validateWUX(wudPath, outputPath) == false )
{
printf("Validation failed. \"%s\" is corrupted.\n", outputPath);
// delete output file
remove(outputPath);
return -5;
}
else
{
printf("Validation successful. No errors detected.\n");
}
}
return 0;
}

157
WudCompress/wud.cpp Normal file
View File

@ -0,0 +1,157 @@
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include"wud.h"
long long wud_getFileSize64(FILE* file)
{
long long prevSeek = _ftelli64(file);
_fseeki64(file, 0, SEEK_END);
long long fileSize = _ftelli64(file);
_fseeki64(file, prevSeek, SEEK_SET);
return fileSize;
}
long long wud_getCurrentSeek64(FILE* file)
{
long long currentSeek = _ftelli64(file);
return currentSeek;
}
void wud_setCurrentSeek64(FILE* file, long long newSeek)
{
_fseeki64(file, newSeek, SEEK_SET);
}
/*
* Open .wud (Wii U image) or .wux (Wii U compressed image) file
*/
wud_t* wud_open(char* path)
{
FILE* inputFile;
inputFile = fopen(path, "rb");
if( inputFile == NULL )
return NULL;
// allocate wud struct
wud_t* wud = (wud_t*)malloc(sizeof(wud_t));
memset(wud, 0x00, sizeof(wud_t));
wud->fileWud = inputFile;
// get size of file
long long inputFileSize = wud_getFileSize64(wud->fileWud);
// determine whether the WUD is compressed or not
wuxHeader_t wuxHeader = {0};
if( fread(&wuxHeader, sizeof(wuxHeader_t), 1, wud->fileWud) != 1 )
{
// file is too short to be either
wud_close(wud);
return NULL;
}
if( wuxHeader.magic0 == WUX_MAGIC_0 && wuxHeader.magic1 == WUX_MAGIC_1 )
{
// this is a compressed file
wud->isCompressed = true;
wud->sectorSize = wuxHeader.sectorSize;
wud->uncompressedSize = wuxHeader.uncompressedSize;
// validate header values
if( wud->sectorSize < 0x100 || wud->sectorSize >= 0x10000000 )
{
wud_close(wud);
return NULL;
}
// calculate offsets and sizes
wud->indexTableEntryCount = (unsigned int)((wud->uncompressedSize+(long long)(wud->sectorSize-1)) / (long long)wud->sectorSize);
wud->offsetIndexTable = wud_getCurrentSeek64(wud->fileWud);
wud->offsetSectorArray = (wud->offsetIndexTable + (long long)wud->indexTableEntryCount*sizeof(unsigned int));
// align to SECTOR_SIZE
wud->offsetSectorArray = (wud->offsetSectorArray + (long long)(wud->sectorSize-1));
wud->offsetSectorArray = wud->offsetSectorArray - (wud->offsetSectorArray%(long long)wud->sectorSize);
// read index table
unsigned int indexTableSize = sizeof(unsigned int) * wud->indexTableEntryCount;
wud->indexTable = (unsigned int*)malloc(sizeof(unsigned int) * wud->indexTableEntryCount);
wud_setCurrentSeek64(wud->fileWud, wud->offsetIndexTable);
if( fread(wud->indexTable, sizeof(unsigned int), wud->indexTableEntryCount, wud->fileWud) != wud->indexTableEntryCount )
{
// could not read index table
wud_close(wud);
return NULL;
}
}
else
{
// uncompressed file
wud->uncompressedSize = inputFileSize;
}
return wud;
}
/*
* Close wud/wux reader
*/
void wud_close(wud_t* wud)
{
fclose(wud->fileWud);
if( wud->indexTable )
free(wud->indexTable);
free(wud);
}
/*
* Read data
* Transparently handles WUX decompression
* Can read up to 4GB-1 at once
*/
unsigned int wud_readData(wud_t* wud, void* buffer, unsigned int length, long long offset)
{
// make sure there is no out-of-bounds read
long long fileBytesLeft = wud->uncompressedSize - offset;
if( fileBytesLeft <= 0 )
return 0;
if( fileBytesLeft < (long long)length )
length = (unsigned int)fileBytesLeft;
// read data
unsigned int readBytes = 0;
if( wud->isCompressed == false )
{
// uncompressed read is just a 1:1 copy
wud_setCurrentSeek64(wud->fileWud, offset);
readBytes = (unsigned int)fread(buffer, 1, length, wud->fileWud);
}
else
{
// compressed read must be handled on a per-sector level
while( length > 0 )
{
unsigned int sectorOffset = (unsigned int)(offset % (long long)wud->sectorSize);
unsigned int remainingSectorBytes = wud->sectorSize - sectorOffset;
unsigned int sectorIndex = (unsigned int)(offset / (long long)wud->sectorSize);
unsigned int bytesToRead = (remainingSectorBytes<length)?remainingSectorBytes:length; // read only up to the end of the current sector
// look up real sector index
sectorIndex = wud->indexTable[sectorIndex];
wud_setCurrentSeek64(wud->fileWud, wud->offsetSectorArray + (long long)sectorIndex*(long long)wud->sectorSize+(long long)sectorOffset);
readBytes += (unsigned int)fread(buffer, 1, bytesToRead, wud->fileWud);
// progress read offset, write pointer and decrease length
buffer = (void*)((char*)buffer + bytesToRead);
length -= bytesToRead;
offset += bytesToRead;
}
}
return readBytes;
}
/*
* Returns true if the file uses .wux compression, false otherwise
*/
bool wud_isWUXCompressed(wud_t* wud)
{
return wud->isCompressed;
}
/*
* Returns size of data in bytes
* For .wud: Size of raw file
* For .wux: Size of uncompressed data
*/
long long wud_getWUDSize(wud_t* wud)
{
return wud->uncompressedSize;
}

32
WudCompress/wud.h Normal file
View File

@ -0,0 +1,32 @@
typedef struct
{
unsigned int magic0;
unsigned int magic1;
unsigned int sectorSize;
unsigned long long uncompressedSize;
unsigned int flags;
}wuxHeader_t;
typedef struct
{
FILE* fileWud;
long long uncompressedSize;
bool isCompressed;
// data only used when compressed
unsigned int sectorSize;
unsigned int indexTableEntryCount;
unsigned int* indexTable;
long long offsetIndexTable;
long long offsetSectorArray;
}wud_t;
#define WUX_MAGIC_0 '0XUW' // "WUX0"
#define WUX_MAGIC_1 0x1099d02e
// wud and wux functions
wud_t* wud_open(char* path); // handles both, compressed and uncompressed files
void wud_close(wud_t* wud);
unsigned int wud_readData(wud_t* wud, void* buffer, unsigned int length, long long offset);
bool wud_isWUXCompressed(wud_t* wud);
long long wud_getWUDSize(wud_t* wud);