Emulator-Archive/ppsspp
1
0
Fork 0
mirror of https://github.com/hrydgard/ppsspp.git synced 2024-11-23 05:19:56 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity
v1.17.7-beta9
ppsspp/GPU/Software/SamplerX86.cpp
Henrik Rydgård e01ca5b057
Logging API change (refactor) (#19324) 
* Rename LogType to Log

* Explicitly use the Log:: enum when logging. Allows for autocomplete when editing.

* Mac/ARM64 buildfix

* Do the same with the hle result log macros

* Rename the log names to mixed case while at it.

* iOS buildfix

* Qt buildfix attempt, ARM32 buildfix
2024-07-14 14:42:59 +02:00

3718 lines
121 KiB
C++
Raw Permalink Blame History

// Copyright (c) 2017- PPSSPP Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0 or later versions.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official git repository and contact information can be found at
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
#include "ppsspp_config.h"
#if PPSSPP_ARCH(X86) || PPSSPP_ARCH(AMD64)
#include <emmintrin.h>
#include "Common/x64Emitter.h"
#include "Common/BitScan.h"
#include "Common/CPUDetect.h"
#include "GPU/GPUState.h"
#include "GPU/Software/Sampler.h"
#include "GPU/ge_constants.h"
using namespace Gen;
using namespace Rasterizer;
namespace Sampler {
FetchFunc SamplerJitCache::CompileFetch(const SamplerID &id) {
_assert_msg_(id.fetch && !id.linear, "Only fetch should be set on sampler id");
regCache_.SetupABI({
RegCache::GEN_ARG_U,
RegCache::GEN_ARG_V,
RegCache::GEN_ARG_TEXPTR,
RegCache::GEN_ARG_BUFW,
RegCache::GEN_ARG_LEVEL,
RegCache::GEN_ARG_ID,
});
regCache_.ChangeReg(RAX, RegCache::GEN_RESULT);
regCache_.ForceRetain(RegCache::GEN_RESULT);
regCache_.ChangeReg(XMM0, RegCache::VEC_RESULT);
BeginWrite(2048);
Describe("Init");
const u8 *start = AlignCode16();
#if PPSSPP_PLATFORM(WINDOWS)
// RET and shadow space.
stackArgPos_ = 8 + 32;
stackIDOffset_ = 8;
stackLevelOffset_ = 0;
#else
stackArgPos_ = 0;
stackIDOffset_ = -1;
stackLevelOffset_ = -1;
#endif
// Early exit on !srcPtr.
FixupBranch zeroSrc;
if (id.hasInvalidPtr) {
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
CMP(PTRBITS, R(srcReg), Imm8(0));
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
FixupBranch nonZeroSrc = J_CC(CC_NZ);
X64Reg vecResultReg = regCache_.Find(RegCache::VEC_RESULT);
PXOR(vecResultReg, R(vecResultReg));
regCache_.Unlock(vecResultReg, RegCache::VEC_RESULT);
zeroSrc = J(true);
SetJumpTarget(nonZeroSrc);
}
// This reads the pixel data into resultReg from the args.
if (!Jit_ReadTextureFormat(id)) {
regCache_.Reset(false);
EndWrite();
ResetCodePtr(GetOffset(start));
ERROR_LOG(Log::G3D, "Failed to compile fetch %s", DescribeSamplerID(id).c_str());
return nullptr;
}
if (regCache_.Has(RegCache::GEN_ARG_LEVEL))
regCache_.ForceRelease(RegCache::GEN_ARG_LEVEL);
if (regCache_.Has(RegCache::GEN_ARG_ID))
regCache_.ForceRelease(RegCache::GEN_ARG_ID);
X64Reg vecResultReg = regCache_.Find(RegCache::VEC_RESULT);
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
MOVD_xmm(vecResultReg, R(resultReg));
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
regCache_.ForceRelease(RegCache::GEN_RESULT);
if (cpu_info.bSSE4_1) {
PMOVZXBD(vecResultReg, R(vecResultReg));
} else {
X64Reg vecTempReg = regCache_.Alloc(RegCache::VEC_TEMP0);
PXOR(vecTempReg, R(vecTempReg));
PUNPCKLBW(vecResultReg, R(vecTempReg));
PUNPCKLWD(vecResultReg, R(vecTempReg));
regCache_.Release(vecTempReg, RegCache::VEC_TEMP0);
}
regCache_.Unlock(vecResultReg, RegCache::VEC_RESULT);
Describe("Init");
if (id.hasInvalidPtr) {
SetJumpTarget(zeroSrc);
}
RET();
regCache_.Reset(true);
EndWrite();
return (FetchFunc)start;
}
NearestFunc SamplerJitCache::CompileNearest(const SamplerID &id) {
_assert_msg_(!id.fetch && !id.linear, "Fetch and linear should be cleared on sampler id");
BeginWrite(2048);
Describe("Init");
// Let's drop some helpful constants here.
WriteConstantPool(id);
const u8 *start = AlignCode16();
regCache_.SetupABI({
RegCache::VEC_ARG_S,
RegCache::VEC_ARG_T,
RegCache::VEC_ARG_COLOR,
RegCache::GEN_ARG_TEXPTR_PTR,
RegCache::GEN_ARG_BUFW_PTR,
RegCache::GEN_ARG_LEVEL,
RegCache::GEN_ARG_LEVELFRAC,
RegCache::GEN_ARG_ID,
});
#if PPSSPP_PLATFORM(WINDOWS)
// RET + shadow space.
stackArgPos_ = 8 + 32;
// Positions: stackArgPos_+0=bufwptr, stackArgPos_+8=level, stackArgPos_+16=levelFrac
stackIDOffset_ = 24;
stackLevelOffset_ = 8;
#else
stackArgPos_ = 0;
// No args on the stack.
stackIDOffset_ = -1;
stackLevelOffset_ = -1;
#endif
// Start out by saving some registers, since we'll need more.
PUSH(R15);
PUSH(R14);
PUSH(R13);
PUSH(R12);
regCache_.Add(R15, RegCache::GEN_INVALID);
regCache_.Add(R14, RegCache::GEN_INVALID);
regCache_.Add(R13, RegCache::GEN_INVALID);
regCache_.Add(R12, RegCache::GEN_INVALID);
stackArgPos_ += 32;
#if PPSSPP_PLATFORM(WINDOWS)
// Use the shadow space to save U1/V1.
stackUV1Offset_ = -8;
#else
// Use the red zone, but account for the R15-R12 we push just below.
stackUV1Offset_ = -stackArgPos_ - 8;
#endif
// We can throw these away right off if there are no mips.
if (!id.hasAnyMips && regCache_.Has(RegCache::GEN_ARG_LEVEL) && id.useSharedClut)
regCache_.ForceRelease(RegCache::GEN_ARG_LEVEL);
if (!id.hasAnyMips && regCache_.Has(RegCache::GEN_ARG_LEVELFRAC))
regCache_.ForceRelease(RegCache::GEN_ARG_LEVELFRAC);
if (regCache_.Has(RegCache::GEN_ARG_LEVELFRAC)) {
// On Linux, RCX is currently levelFrac, but we'll need it for other things.
if (!cpu_info.bBMI2) {
X64Reg levelFracReg = regCache_.Find(RegCache::GEN_ARG_LEVELFRAC);
MOV(64, R(R15), R(levelFracReg));
regCache_.Unlock(levelFracReg, RegCache::GEN_ARG_LEVELFRAC);
regCache_.ForceRelease(RegCache::GEN_ARG_LEVELFRAC);
regCache_.ChangeReg(R15, RegCache::GEN_ARG_LEVELFRAC);
regCache_.ForceRetain(RegCache::GEN_ARG_LEVELFRAC);
}
} else if (!regCache_.Has(RegCache::GEN_ARG_BUFW_PTR)) {
// Let's load bufwptr into regs. RDX is free.
MOV(64, R(RDX), MDisp(RSP, stackArgPos_ + 0));
regCache_.ChangeReg(RDX, RegCache::GEN_ARG_BUFW_PTR);
regCache_.ForceRetain(RegCache::GEN_ARG_BUFW_PTR);
}
// Okay, now lock RCX as a shifting reg.
if (!cpu_info.bBMI2) {
regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
regCache_.ForceRetain(RegCache::GEN_SHIFTVAL);
}
bool success = true;
// Convert S/T + X/Y to U/V (and U1/V1 if appropriate.)
success = success && Jit_GetTexelCoords(id);
// At this point, XMM0 should be free. Swap it to the result.
success = success && regCache_.ChangeReg(XMM0, RegCache::VEC_RESULT);
// Let's also pick a reg for GEN_RESULT - doesn't matter which.
X64Reg resultReg = regCache_.Alloc(RegCache::GEN_RESULT);
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
regCache_.ForceRetain(RegCache::GEN_RESULT);
// Early exit on !srcPtr (either one.)
FixupBranch zeroSrc;
if (id.hasInvalidPtr) {
Describe("NullCheck");
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR_PTR);
if (id.hasAnyMips) {
X64Reg tempReg = regCache_.Alloc(RegCache::GEN_TEMP0);
MOV(64, R(tempReg), MDisp(srcReg, 0));
AND(64, R(tempReg), MDisp(srcReg, 8));
CMP(PTRBITS, R(tempReg), Imm8(0));
regCache_.Release(tempReg, RegCache::GEN_TEMP0);
} else {
CMP(PTRBITS, MatR(srcReg), Imm8(0));
}
FixupBranch nonZeroSrc = J_CC(CC_NZ);
PXOR(XMM0, R(XMM0));
zeroSrc = J(true);
SetJumpTarget(nonZeroSrc);
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR_PTR);
}
auto loadPtrs = [&](bool level1) {
X64Reg bufwReg = regCache_.Alloc(RegCache::GEN_ARG_BUFW);
X64Reg bufwPtrReg = regCache_.Find(RegCache::GEN_ARG_BUFW_PTR);
MOVZX(32, 16, bufwReg, MDisp(bufwPtrReg, level1 ? 2 : 0));
regCache_.Unlock(bufwPtrReg, RegCache::GEN_ARG_BUFW_PTR);
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW);
regCache_.ForceRetain(RegCache::GEN_ARG_BUFW);
X64Reg srcReg = regCache_.Alloc(RegCache::GEN_ARG_TEXPTR);
X64Reg srcPtrReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR_PTR);
MOV(64, R(srcReg), MDisp(srcPtrReg, level1 ? 8 : 0));
regCache_.Unlock(srcPtrReg, RegCache::GEN_ARG_TEXPTR_PTR);
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
regCache_.ForceRetain(RegCache::GEN_ARG_TEXPTR);
};
loadPtrs(false);
success = success && Jit_ReadTextureFormat(id);
// Convert that to 16-bit from 8-bit channels.
X64Reg vecResultReg = regCache_.Find(RegCache::VEC_RESULT);
resultReg = regCache_.Find(RegCache::GEN_RESULT);
MOVD_xmm(vecResultReg, R(resultReg));
if (cpu_info.bSSE4_1) {
PMOVZXBW(vecResultReg, R(vecResultReg));
} else {
X64Reg zeroReg = GetZeroVec();
PUNPCKLBW(vecResultReg, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
}
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
regCache_.Unlock(vecResultReg, RegCache::VEC_RESULT);
if (id.hasAnyMips) {
X64Reg vecResultReg = regCache_.Alloc(RegCache::VEC_RESULT1);
if (regCache_.Has(RegCache::GEN_ARG_LEVELFRAC)) {
X64Reg levelFracReg = regCache_.Find(RegCache::GEN_ARG_LEVELFRAC);
CMP(8, R(levelFracReg), Imm8(0));
regCache_.Unlock(levelFracReg, RegCache::GEN_ARG_LEVELFRAC);
} else {
CMP(8, MDisp(RSP, stackArgPos_ + 16), Imm8(0));
}
FixupBranch skip = J_CC(CC_Z, true);
// Modify the level, so the new level value is used. We don't need the old.
if (regCache_.Has(RegCache::GEN_ARG_LEVEL)) {
X64Reg levelReg = regCache_.Find(RegCache::GEN_ARG_LEVEL);
ADD(32, R(levelReg), Imm8(1));
regCache_.Unlock(levelReg, RegCache::GEN_ARG_LEVEL);
} else {
// It's fine to just modify this in place.
ADD(32, MDisp(RSP, stackArgPos_ + stackLevelOffset_), Imm8(1));
}
// This is inside the conditional, but it's okay because we throw it away after.
loadPtrs(true);
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW_PTR);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR_PTR);
X64Reg uReg = regCache_.Alloc(RegCache::GEN_ARG_U);
MOV(32, R(uReg), MDisp(RSP, stackArgPos_ + stackUV1Offset_ + 0));
regCache_.Unlock(uReg, RegCache::GEN_ARG_U);
regCache_.ForceRetain(RegCache::GEN_ARG_U);
X64Reg vReg = regCache_.Alloc(RegCache::GEN_ARG_V);
MOV(32, R(vReg), MDisp(RSP, stackArgPos_ + stackUV1Offset_ + 4));
regCache_.Unlock(vReg, RegCache::GEN_ARG_V);
regCache_.ForceRetain(RegCache::GEN_ARG_V);
bool hadId = regCache_.Has(RegCache::GEN_ID);
bool hadZero = regCache_.Has(RegCache::VEC_ZERO);
success = success && Jit_ReadTextureFormat(id);
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
MOVD_xmm(vecResultReg, R(resultReg));
if (cpu_info.bSSE4_1) {
PMOVZXBW(vecResultReg, R(vecResultReg));
} else {
X64Reg zeroReg = GetZeroVec();
PUNPCKLBW(vecResultReg, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
}
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
// Since we're inside a conditional, make sure these go away if we allocated them.
if (!hadId && regCache_.Has(RegCache::GEN_ID))
regCache_.ForceRelease(RegCache::GEN_ID);
if (!hadZero && regCache_.Has(RegCache::VEC_ZERO))
regCache_.ForceRelease(RegCache::VEC_ZERO);
SetJumpTarget(skip);
regCache_.Unlock(vecResultReg, RegCache::VEC_RESULT1);
} else {
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW_PTR);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR_PTR);
}
// We're done with these now.
if (regCache_.Has(RegCache::GEN_ARG_TEXPTR_PTR))
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR_PTR);
if (regCache_.Has(RegCache::GEN_ARG_BUFW_PTR))
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW_PTR);
if (regCache_.Has(RegCache::GEN_ARG_LEVEL))
regCache_.ForceRelease(RegCache::GEN_ARG_LEVEL);
if (regCache_.Has(RegCache::GEN_SHIFTVAL))
regCache_.ForceRelease(RegCache::GEN_SHIFTVAL);
regCache_.ForceRelease(RegCache::GEN_RESULT);
if (id.hasAnyMips) {
Describe("BlendMips");
if (!regCache_.Has(RegCache::GEN_ARG_LEVELFRAC)) {
X64Reg levelFracReg = regCache_.Alloc(RegCache::GEN_ARG_LEVELFRAC);
MOVZX(32, 8, levelFracReg, MDisp(RSP, stackArgPos_ + 16));
regCache_.Unlock(levelFracReg, RegCache::GEN_ARG_LEVELFRAC);
regCache_.ForceRetain(RegCache::GEN_ARG_LEVELFRAC);
}
X64Reg levelFracReg = regCache_.Find(RegCache::GEN_ARG_LEVELFRAC);
CMP(8, R(levelFracReg), Imm8(0));
FixupBranch skip = J_CC(CC_Z, true);
// TODO: PMADDWD? Refactor shared?
// First, broadcast the levelFrac value into an XMM.
X64Reg fracReg = regCache_.Alloc(RegCache::VEC_TEMP0);
MOVD_xmm(fracReg, R(levelFracReg));
PSHUFLW(fracReg, R(fracReg), _MM_SHUFFLE(0, 0, 0, 0));
regCache_.Unlock(levelFracReg, RegCache::GEN_ARG_LEVELFRAC);
regCache_.ForceRelease(RegCache::GEN_ARG_LEVELFRAC);
// Multiply level1 color by the fraction.
X64Reg color1Reg = regCache_.Find(RegCache::VEC_RESULT1);
PMULLW(color1Reg, R(fracReg));
// Okay, next we need an inverse for color 0.
X64Reg invFracReg = regCache_.Alloc(RegCache::VEC_TEMP1);
MOVDQA(invFracReg, M(const10All16_));
PSUBW(invFracReg, R(fracReg));
// And multiply.
PMULLW(XMM0, R(invFracReg));
regCache_.Release(fracReg, RegCache::VEC_TEMP0);
regCache_.Release(invFracReg, RegCache::VEC_TEMP1);
// Okay, now sum and divide by 16 (which is what the fraction maxed at.)
PADDW(XMM0, R(color1Reg));
PSRLW(XMM0, 4);
// And now we're done with color1Reg/VEC_RESULT1.
regCache_.Unlock(color1Reg, RegCache::VEC_RESULT1);
regCache_.ForceRelease(RegCache::VEC_RESULT1);
SetJumpTarget(skip);
}
// Finally, it's time to apply the texture function.
success = success && Jit_ApplyTextureFunc(id);
// Last of all, convert to 32-bit channels.
Describe("Init");
if (cpu_info.bSSE4_1) {
PMOVZXWD(XMM0, R(XMM0));
} else {
X64Reg zeroReg = GetZeroVec();
PUNPCKLWD(XMM0, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
}
regCache_.ForceRelease(RegCache::VEC_RESULT);
if (regCache_.Has(RegCache::GEN_ARG_ID))
regCache_.ForceRelease(RegCache::GEN_ARG_ID);
if (!success) {
regCache_.Reset(false);
EndWrite();
ResetCodePtr(GetOffset(start));
ERROR_LOG(Log::G3D, "Failed to compile nearest %s", DescribeSamplerID(id).c_str());
return nullptr;
}
if (id.hasInvalidPtr) {
SetJumpTarget(zeroSrc);
}
POP(R12);
POP(R13);
POP(R14);
POP(R15);
RET();
regCache_.Reset(true);
EndWrite();
return (NearestFunc)start;
}
LinearFunc SamplerJitCache::CompileLinear(const SamplerID &id) {
_assert_msg_(id.linear && !id.fetch, "Only linear should be set on sampler id");
BeginWrite(2048);
Describe("Init");
// We don't use stackArgPos_ here, this is just for DXT.
stackArgPos_ = -1;
// Let's drop some helpful constants here.
WriteConstantPool(id);
const u8 *nearest = nullptr;
if (id.TexFmt() >= GE_TFMT_DXT1) {
regCache_.SetupABI({
RegCache::GEN_ARG_U,
RegCache::GEN_ARG_V,
RegCache::GEN_ARG_TEXPTR,
RegCache::GEN_ARG_BUFW,
RegCache::GEN_ARG_LEVEL,
// Avoid clobber.
RegCache::GEN_ARG_LEVELFRAC,
});
auto lockReg = [&](X64Reg r, RegCache::Purpose p) {
regCache_.ChangeReg(r, p);
regCache_.ForceRetain(p);
};
lockReg(RAX, RegCache::GEN_RESULT);
lockReg(XMM0, RegCache::VEC_ARG_U);
lockReg(XMM1, RegCache::VEC_ARG_V);
lockReg(XMM5, RegCache::VEC_RESULT);
#if !PPSSPP_PLATFORM(WINDOWS)
if (id.hasAnyMips) {
lockReg(XMM6, RegCache::VEC_U1);
lockReg(XMM7, RegCache::VEC_V1);
lockReg(XMM8, RegCache::VEC_RESULT1);
lockReg(XMM12, RegCache::VEC_INDEX1);
}
lockReg(XMM9, RegCache::VEC_ARG_COLOR);
lockReg(XMM10, RegCache::VEC_FRAC);
lockReg(XMM11, RegCache::VEC_INDEX);
#endif
// We'll first write the nearest sampler, which we will CALL.
// This may differ slightly based on the "linear" flag.
nearest = AlignCode16();
if (!Jit_ReadTextureFormat(id)) {
regCache_.Reset(false);
EndWrite();
ResetCodePtr(GetOffset(nearest));
ERROR_LOG(Log::G3D, "Failed to compile linear nearest %s", DescribeSamplerID(id).c_str());
return nullptr;
}
Describe("Init");
RET();
regCache_.ForceRelease(RegCache::GEN_RESULT);
regCache_.ForceRelease(RegCache::VEC_ARG_U);
regCache_.ForceRelease(RegCache::VEC_ARG_V);
regCache_.ForceRelease(RegCache::VEC_RESULT);
auto unlockOptReg = [&](RegCache::Purpose p) {
if (regCache_.Has(p))
regCache_.ForceRelease(p);
};
unlockOptReg(RegCache::GEN_ARG_LEVEL);
unlockOptReg(RegCache::GEN_ARG_LEVELFRAC);
unlockOptReg(RegCache::VEC_U1);
unlockOptReg(RegCache::VEC_V1);
unlockOptReg(RegCache::VEC_RESULT1);
unlockOptReg(RegCache::VEC_ARG_COLOR);
unlockOptReg(RegCache::VEC_FRAC);
unlockOptReg(RegCache::VEC_INDEX);
unlockOptReg(RegCache::VEC_INDEX1);
regCache_.Reset(true);
}
EndWrite();
// Now the actual linear func, which is exposed externally.
const u8 *linearResetPos = GetCodePointer();
Describe("Init");
regCache_.SetupABI({
RegCache::VEC_ARG_S,
RegCache::VEC_ARG_T,
RegCache::VEC_ARG_COLOR,
RegCache::GEN_ARG_TEXPTR_PTR,
RegCache::GEN_ARG_BUFW_PTR,
RegCache::GEN_ARG_LEVEL,
RegCache::GEN_ARG_LEVELFRAC,
RegCache::GEN_ARG_ID,
});
#if PPSSPP_PLATFORM(WINDOWS)
// RET + shadow space.
stackArgPos_ = 8 + 32;
// Free up some more vector regs on Windows too, where we're a bit tight.
stackArgPos_ += WriteProlog(0, { XMM6, XMM7, XMM8, XMM9, XMM10, XMM11, XMM12 }, { R15, R14, R13, R12 });
// Positions: stackArgPos_+0=bufwptr, stackArgPos_+8=level, stackArgPos_+16=levelFrac
stackIDOffset_ = 24;
stackLevelOffset_ = 8;
// If needed, we could store UV1 data in shadow space, but we no longer do.
stackUV1Offset_ = -8;
#else
stackArgPos_ = 0;
stackArgPos_ += WriteProlog(0, {}, { R15, R14, R13, R12 });
stackIDOffset_ = -1;
stackLevelOffset_ = -1;
// Use the red zone.
stackUV1Offset_ = -stackArgPos_ - 8;
#endif
// This is what we'll put in them, anyway...
if (nearest != nullptr) {
regCache_.ChangeReg(XMM10, RegCache::VEC_FRAC);
regCache_.ForceRetain(RegCache::VEC_FRAC);
regCache_.ChangeReg(XMM11, RegCache::VEC_INDEX);
regCache_.ForceRetain(RegCache::VEC_INDEX);
if (id.hasAnyMips) {
regCache_.ChangeReg(XMM12, RegCache::VEC_INDEX1);
regCache_.ForceRetain(RegCache::VEC_INDEX1);
}
}
// Reserve a couple regs that the nearest CALL won't use.
if (id.hasAnyMips) {
regCache_.ChangeReg(XMM6, RegCache::VEC_U1);
regCache_.ChangeReg(XMM7, RegCache::VEC_V1);
regCache_.ForceRetain(RegCache::VEC_U1);
regCache_.ForceRetain(RegCache::VEC_V1);
} else if (regCache_.Has(RegCache::GEN_ARG_LEVELFRAC)) {
regCache_.ForceRelease(RegCache::GEN_ARG_LEVELFRAC);
}
// Save prim color for later in a different XMM too if we're using the nearest helper.
if (nearest != nullptr) {
X64Reg primColorReg = regCache_.Find(RegCache::VEC_ARG_COLOR);
MOVDQA(XMM9, R(primColorReg));
regCache_.Unlock(primColorReg, RegCache::VEC_ARG_COLOR);
regCache_.ForceRelease(RegCache::VEC_ARG_COLOR);
regCache_.ChangeReg(XMM9, RegCache::VEC_ARG_COLOR);
regCache_.ForceRetain(RegCache::VEC_ARG_COLOR);
}
// We also want to save src and bufw for later. Might be in a reg already.
if (regCache_.Has(RegCache::GEN_ARG_TEXPTR_PTR) && regCache_.Has(RegCache::GEN_ARG_BUFW_PTR)) {
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR_PTR);
X64Reg bufwReg = regCache_.Find(RegCache::GEN_ARG_BUFW_PTR);
MOV(64, R(R14), R(srcReg));
MOV(64, R(R15), R(bufwReg));
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR_PTR);
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW_PTR);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR_PTR);
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW_PTR);
} else if (regCache_.Has(RegCache::GEN_ARG_TEXPTR_PTR)) {
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR_PTR);
MOV(64, R(R14), R(srcReg));
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR_PTR);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR_PTR);
MOV(64, R(R15), MDisp(RSP, stackArgPos_ + 0));
} else {
MOV(64, R(R14), MDisp(RSP, stackArgPos_ + 0));
MOV(64, R(R15), MDisp(RSP, stackArgPos_ + 8));
}
// Okay, and now remember we moved to R14/R15.
regCache_.ChangeReg(R14, RegCache::GEN_ARG_TEXPTR_PTR);
regCache_.ForceRetain(RegCache::GEN_ARG_TEXPTR_PTR);
if (!regCache_.Has(RegCache::GEN_ARG_BUFW_PTR)) {
regCache_.ChangeReg(R15, RegCache::GEN_ARG_BUFW_PTR);
regCache_.ForceRetain(RegCache::GEN_ARG_BUFW_PTR);
}
bool success = true;
// Our first goal is to convert S/T and X/Y into U/V and frac_u/frac_v.
success = success && Jit_GetTexelCoordsQuad(id);
// Early exit on !srcPtr (either one.)
FixupBranch zeroSrc;
if (id.hasInvalidPtr) {
Describe("NullCheck");
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR_PTR);
if (id.hasAnyMips) {
X64Reg tempReg = regCache_.Alloc(RegCache::GEN_TEMP0);
MOV(64, R(tempReg), MDisp(srcReg, 0));
AND(64, R(tempReg), MDisp(srcReg, 8));
CMP(PTRBITS, R(tempReg), Imm8(0));
regCache_.Release(tempReg, RegCache::GEN_TEMP0);
} else {
CMP(PTRBITS, MatR(srcReg), Imm8(0));
}
FixupBranch nonZeroSrc = J_CC(CC_NZ);
PXOR(XMM0, R(XMM0));
zeroSrc = J(true);
SetJumpTarget(nonZeroSrc);
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR_PTR);
}
auto prepareDataOffsets = [&](RegCache::Purpose uPurpose, RegCache::Purpose vPurpose, bool level1) {
X64Reg uReg = regCache_.Find(uPurpose);
X64Reg vReg = regCache_.Find(vPurpose);
success = success && Jit_PrepareDataOffsets(id, uReg, vReg, level1);
regCache_.Unlock(uReg, uPurpose);
regCache_.Unlock(vReg, vPurpose);
};
Describe("DataOffsets");
prepareDataOffsets(RegCache::VEC_ARG_U, RegCache::VEC_ARG_V, false);
if (id.hasAnyMips)
prepareDataOffsets(RegCache::VEC_U1, RegCache::VEC_V1, true);
// The data offset goes into V, except in the CLUT4 case and DXT (nearest func) cases.
if (nearest == nullptr && id.TexFmt() != GE_TFMT_CLUT4)
regCache_.ForceRelease(RegCache::VEC_ARG_U);
// Hard allocate results if we're using the func method.
if (nearest != nullptr) {
regCache_.ChangeReg(XMM5, RegCache::VEC_RESULT);
regCache_.ForceRetain(RegCache::VEC_RESULT);
if (id.hasAnyMips) {
regCache_.ChangeReg(XMM8, RegCache::VEC_RESULT1);
regCache_.ForceRetain(RegCache::VEC_RESULT1);
}
}
// This stores the result in an XMM for later processing.
// We map lookups to nearest CALLs, with arg order: u, v, src, bufw, level
auto doNearestCall = [&](int off, bool level1) {
#if PPSSPP_PLATFORM(WINDOWS)
static const X64Reg uArgReg = RCX;
static const X64Reg vArgReg = RDX;
static const X64Reg srcArgReg = R8;
static const X64Reg bufwArgReg = R9;
#else
static const X64Reg uArgReg = RDI;
static const X64Reg vArgReg = RSI;
static const X64Reg srcArgReg = RDX;
static const X64Reg bufwArgReg = RCX;
#endif
static const X64Reg resultReg = RAX;
X64Reg uReg = regCache_.Find(level1 ? RegCache::VEC_U1 : RegCache::VEC_ARG_U);
X64Reg vReg = regCache_.Find(level1 ? RegCache::VEC_V1 : RegCache::VEC_ARG_V);
// Otherwise, we'll overwrite them...
_assert_(level1 || (uReg == XMM0 && vReg == XMM1));
if (cpu_info.bSSE4_1) {
PEXTRD(R(uArgReg), uReg, off / 4);
PEXTRD(R(vArgReg), vReg, off / 4);
} else {
MOVD_xmm(R(uArgReg), uReg);
MOVD_xmm(R(vArgReg), vReg);
PSRLDQ(uReg, 4);
PSRLDQ(vReg, 4);
}
regCache_.Unlock(uReg, level1 ? RegCache::VEC_U1 : RegCache::VEC_ARG_U);
regCache_.Unlock(vReg, level1 ? RegCache::VEC_V1 : RegCache::VEC_ARG_V);
X64Reg indexReg = regCache_.Find(level1 ? RegCache::VEC_INDEX1 : RegCache::VEC_INDEX);
if (cpu_info.bSSE4_1) {
PEXTRD(R(srcArgReg), indexReg, off / 4);
} else {
MOVD_xmm(R(srcArgReg), indexReg);
PSRLDQ(indexReg, 4);
}
regCache_.Unlock(indexReg, level1 ? RegCache::VEC_INDEX1 : RegCache::VEC_INDEX);
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR_PTR);
X64Reg bufwReg = regCache_.Find(RegCache::GEN_ARG_BUFW_PTR);
ADD(64, R(srcArgReg), MDisp(srcReg, level1 ? 8 : 0));
MOVZX(32, 16, bufwArgReg, MDisp(bufwReg, level1 ? 2 : 0));
// Leave level/levelFrac, we just always load from RAM on Windows and lock on POSIX.
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR_PTR);
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW_PTR);
CALL(nearest);
X64Reg vecResultReg = regCache_.Find(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
if (cpu_info.bSSE4_1) {
PINSRD(vecResultReg, R(resultReg), off / 4);
} else if (off == 0) {
MOVD_xmm(vecResultReg, R(resultReg));
} else {
X64Reg tempReg = regCache_.Alloc(RegCache::VEC_TEMP0);
MOVD_xmm(tempReg, R(resultReg));
PSLLDQ(tempReg, off);
POR(vecResultReg, R(tempReg));
regCache_.Release(tempReg, RegCache::VEC_TEMP0);
}
regCache_.Unlock(vecResultReg, level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
};
if (nearest != nullptr) {
Describe("Calls");
doNearestCall(0, false);
doNearestCall(4, false);
doNearestCall(8, false);
doNearestCall(12, false);
// After doing the calls, certain cached things aren't safe.
if (regCache_.Has(RegCache::GEN_ID))
regCache_.ForceRelease(RegCache::GEN_ID);
if (regCache_.Has(RegCache::VEC_ZERO))
regCache_.ForceRelease(RegCache::VEC_ZERO);
} else {
success = success && Jit_FetchQuad(id, false);
}
if (id.hasAnyMips) {
Describe("MipsCalls");
if (regCache_.Has(RegCache::GEN_ARG_LEVELFRAC)) {
X64Reg levelFracReg = regCache_.Find(RegCache::GEN_ARG_LEVELFRAC);
CMP(8, R(levelFracReg), Imm8(0));
regCache_.Unlock(levelFracReg, RegCache::GEN_ARG_LEVELFRAC);
} else {
CMP(8, MDisp(RSP, stackArgPos_ + 16), Imm8(0));
}
FixupBranch skip = J_CC(CC_Z, true);
// Modify the level, so the new level value is used. We don't need the old.
if (regCache_.Has(RegCache::GEN_ARG_LEVEL)) {
X64Reg levelReg = regCache_.Find(RegCache::GEN_ARG_LEVEL);
ADD(32, R(levelReg), Imm8(1));
regCache_.Unlock(levelReg, RegCache::GEN_ARG_LEVEL);
} else {
// It's fine to just modify this in place.
ADD(32, MDisp(RSP, stackArgPos_ + stackLevelOffset_), Imm8(1));
}
if (nearest != nullptr) {
Describe("MipsCalls");
doNearestCall(0, true);
doNearestCall(4, true);
doNearestCall(8, true);
doNearestCall(12, true);
} else {
success = success && Jit_FetchQuad(id, true);
}
SetJumpTarget(skip);
}
// We're done with these now.
if (nearest != nullptr) {
regCache_.ForceRelease(RegCache::VEC_ARG_U);
regCache_.ForceRelease(RegCache::VEC_ARG_V);
regCache_.ForceRelease(RegCache::VEC_INDEX);
}
if (regCache_.Has(RegCache::VEC_INDEX1))
regCache_.ForceRelease(RegCache::VEC_INDEX1);
if (regCache_.Has(RegCache::VEC_U1))
regCache_.ForceRelease(RegCache::VEC_U1);
if (regCache_.Has(RegCache::VEC_V1))
regCache_.ForceRelease(RegCache::VEC_V1);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR_PTR);
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW_PTR);
if (regCache_.Has(RegCache::GEN_ARG_LEVEL))
regCache_.ForceRelease(RegCache::GEN_ARG_LEVEL);
success = success && Jit_DecodeQuad(id, false);
success = success && Jit_BlendQuad(id, false);
if (id.hasAnyMips) {
Describe("BlendMips");
if (!regCache_.Has(RegCache::GEN_ARG_LEVELFRAC)) {
X64Reg levelFracReg = regCache_.Alloc(RegCache::GEN_ARG_LEVELFRAC);
MOVZX(32, 8, levelFracReg, MDisp(RSP, stackArgPos_ + 16));
regCache_.Unlock(levelFracReg, RegCache::GEN_ARG_LEVELFRAC);
regCache_.ForceRetain(RegCache::GEN_ARG_LEVELFRAC);
}
X64Reg levelFracReg = regCache_.Find(RegCache::GEN_ARG_LEVELFRAC);
CMP(8, R(levelFracReg), Imm8(0));
FixupBranch skip = J_CC(CC_Z, true);
success = success && Jit_DecodeQuad(id, true);
success = success && Jit_BlendQuad(id, true);
Describe("BlendMips");
// First, broadcast the levelFrac value into an XMM.
X64Reg fracReg = regCache_.Alloc(RegCache::VEC_TEMP0);
MOVD_xmm(fracReg, R(levelFracReg));
PSHUFLW(fracReg, R(fracReg), _MM_SHUFFLE(0, 0, 0, 0));
regCache_.Unlock(levelFracReg, RegCache::GEN_ARG_LEVELFRAC);
regCache_.ForceRelease(RegCache::GEN_ARG_LEVELFRAC);
// Multiply level1 color by the fraction.
X64Reg color1Reg = regCache_.Find(RegCache::VEC_RESULT1);
PMULLW(color1Reg, R(fracReg));
// Okay, next we need an inverse for color 0.
X64Reg invFracReg = regCache_.Alloc(RegCache::VEC_TEMP1);
MOVDQA(invFracReg, M(const10All16_));
PSUBW(invFracReg, R(fracReg));
// And multiply.
PMULLW(XMM0, R(invFracReg));
regCache_.Release(fracReg, RegCache::VEC_TEMP0);
regCache_.Release(invFracReg, RegCache::VEC_TEMP1);
// Okay, now sum and divide by 16 (which is what the fraction maxed at.)
PADDW(XMM0, R(color1Reg));
PSRLW(XMM0, 4);
// And now we're done with color1Reg/VEC_RESULT1.
regCache_.Unlock(color1Reg, RegCache::VEC_RESULT1);
regCache_.ForceRelease(RegCache::VEC_RESULT1);
SetJumpTarget(skip);
}
if (regCache_.Has(RegCache::VEC_FRAC))
regCache_.ForceRelease(RegCache::VEC_FRAC);
// Finally, it's time to apply the texture function.
success = success && Jit_ApplyTextureFunc(id);
// Last of all, convert to 32-bit channels.
Describe("Init");
if (cpu_info.bSSE4_1) {
PMOVZXWD(XMM0, R(XMM0));
} else {
X64Reg zeroReg = GetZeroVec();
PUNPCKLWD(XMM0, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
}
regCache_.ForceRelease(RegCache::VEC_RESULT);
if (regCache_.Has(RegCache::GEN_ARG_ID))
regCache_.ForceRelease(RegCache::GEN_ARG_ID);
if (!success) {
regCache_.Reset(false);
EndWrite();
ResetCodePtr(GetOffset(nearest ? nearest : linearResetPos));
ERROR_LOG(Log::G3D, "Failed to compile linear %s", DescribeSamplerID(id).c_str());
return nullptr;
}
if (id.hasInvalidPtr) {
SetJumpTarget(zeroSrc);
}
const u8 *start = WriteFinalizedEpilog();
regCache_.Reset(true);
return (LinearFunc)start;
}
void SamplerJitCache::WriteConstantPool(const SamplerID &id) {
// We reuse constants in any pool, because our code space is small.
WriteSimpleConst8x16(const10All16_, 0x10);
WriteSimpleConst16x8(const10All8_, 0x10);
if (const10Low_ == nullptr) {
const10Low_ = AlignCode16();
for (int i = 0; i < 4; ++i)
Write16(0x10);
for (int i = 0; i < 4; ++i)
Write16(0);
}
WriteSimpleConst4x32(constOnes32_, 1);
WriteSimpleConst8x16(constOnes16_, 1);
// This is the mask for clamp or wrap, the max texel in the S or T direction.
WriteSimpleConst4x32(constMaxTexel32_, 511);
if (constUNext_ == nullptr) {
constUNext_ = AlignCode16();
Write32(0); Write32(1); Write32(0); Write32(1);
}
if (constVNext_ == nullptr) {
constVNext_ = AlignCode16();
Write32(0); Write32(0); Write32(1); Write32(1);
}
WriteSimpleConst4x32(const5551Swizzle_, 0x00070707);
WriteSimpleConst4x32(const5650Swizzle_, 0x00070307);
// These are unique to the sampler ID.
if (!id.hasAnyMips) {
float w256f = (1 << id.width0Shift) * 256;
float h256f = (1 << id.height0Shift) * 256;
constWidthHeight256f_ = AlignCode16();
Write32(*(uint32_t *)&w256f);
Write32(*(uint32_t *)&h256f);
Write32(*(uint32_t *)&w256f);
Write32(*(uint32_t *)&h256f);
WriteDynamicConst4x32(constWidthMinus1i_, id.width0Shift > 9 ? 511 : (1 << id.width0Shift) - 1);
WriteDynamicConst4x32(constHeightMinus1i_, id.height0Shift > 9 ? 511 : (1 << id.height0Shift) - 1);
} else {
constWidthHeight256f_ = nullptr;
constWidthMinus1i_ = nullptr;
constHeightMinus1i_ = nullptr;
}
}
RegCache::Reg SamplerJitCache::GetSamplerID() {
if (regCache_.Has(RegCache::GEN_ARG_ID))
return regCache_.Find(RegCache::GEN_ARG_ID);
if (!regCache_.Has(RegCache::GEN_ID)) {
X64Reg r = regCache_.Alloc(RegCache::GEN_ID);
_assert_(stackIDOffset_ != -1);
MOV(PTRBITS, R(r), MDisp(RSP, stackArgPos_ + stackIDOffset_));
return r;
}
return regCache_.Find(RegCache::GEN_ID);
}
void SamplerJitCache::UnlockSamplerID(RegCache::Reg &r) {
if (regCache_.Has(RegCache::GEN_ARG_ID))
regCache_.Unlock(r, RegCache::GEN_ARG_ID);
else
regCache_.Unlock(r, RegCache::GEN_ID);
}
bool SamplerJitCache::Jit_FetchQuad(const SamplerID &id, bool level1) {
bool success = true;
switch (id.TexFmt()) {
case GE_TFMT_5650:
case GE_TFMT_5551:
case GE_TFMT_4444:
success = Jit_GetDataQuad(id, level1, 16);
// Mask away the high bits, if loaded via AVX2.
if (cpu_info.bAVX2) {
X64Reg destReg = regCache_.Find(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
PSLLD(destReg, 16);
PSRLD(destReg, 16);
regCache_.Unlock(destReg, level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
}
break;
case GE_TFMT_8888:
success = Jit_GetDataQuad(id, level1, 32);
break;
case GE_TFMT_CLUT32:
success = Jit_GetDataQuad(id, level1, 32);
if (success)
success = Jit_TransformClutIndexQuad(id, 32);
if (success)
success = Jit_ReadClutQuad(id, level1);
break;
case GE_TFMT_CLUT16:
success = Jit_GetDataQuad(id, level1, 16);
if (success)
success = Jit_TransformClutIndexQuad(id, 16);
if (success)
success = Jit_ReadClutQuad(id, level1);
break;
case GE_TFMT_CLUT8:
success = Jit_GetDataQuad(id, level1, 8);
if (success)
success = Jit_TransformClutIndexQuad(id, 8);
if (success)
success = Jit_ReadClutQuad(id, level1);
break;
case GE_TFMT_CLUT4:
success = Jit_GetDataQuad(id, level1, 4);
if (success)
success = Jit_TransformClutIndexQuad(id, 4);
if (success)
success = Jit_ReadClutQuad(id, level1);
break;
case GE_TFMT_DXT1:
case GE_TFMT_DXT3:
case GE_TFMT_DXT5:
// No SIMD version currently, should use nearest helper path.
success = false;
break;
default:
success = false;
}
return success;
}
bool SamplerJitCache::Jit_GetDataQuad(const SamplerID &id, bool level1, int bitsPerTexel) {
Describe("DataQuad");
bool success = true;
X64Reg baseReg = regCache_.Alloc(RegCache::GEN_ARG_TEXPTR);
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR_PTR);
MOV(64, R(baseReg), MDisp(srcReg, level1 ? 8 : 0));
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR_PTR);
X64Reg destReg = INVALID_REG;
if (id.TexFmt() >= GE_TFMT_CLUT4 && id.TexFmt() <= GE_TFMT_CLUT32)
destReg = regCache_.Alloc(RegCache::VEC_INDEX);
else if (regCache_.Has(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT))
destReg = regCache_.Find(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
else
destReg = regCache_.Alloc(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
X64Reg byteOffsetReg = regCache_.Find(level1 ? RegCache::VEC_V1 : RegCache::VEC_ARG_V);
if (cpu_info.bAVX2 && id.overReadSafe) {
// We have to set a mask for which values to load. Load all 4.
// Note this is overwritten with zeroes by the gather instruction.
X64Reg maskReg = regCache_.Alloc(RegCache::VEC_TEMP0);
PCMPEQD(maskReg, R(maskReg));
VPGATHERDD(128, destReg, MComplex(baseReg, byteOffsetReg, SCALE_1, 0), maskReg);
regCache_.Release(maskReg, RegCache::VEC_TEMP0);
} else {
if (bitsPerTexel != 32)
PXOR(destReg, R(destReg));
// Grab each value separately... try to use the right memory access size.
X64Reg temp2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
if (cpu_info.bSSE4_1) {
for (int i = 0; i < 4; ++i) {
PEXTRD(R(temp2Reg), byteOffsetReg, i);
if (bitsPerTexel <= 8)
PINSRB(destReg, MComplex(baseReg, temp2Reg, SCALE_1, 0), i * 4);
else if (bitsPerTexel == 16)
PINSRW(destReg, MComplex(baseReg, temp2Reg, SCALE_1, 0), i * 2);
else if (bitsPerTexel == 32)
PINSRD(destReg, MComplex(baseReg, temp2Reg, SCALE_1, 0), i);
}
} else {
for (int i = 0; i < 4; ++i) {
MOVD_xmm(R(temp2Reg), byteOffsetReg);
if (i != 3)
PSRLDQ(byteOffsetReg, 4);
if (bitsPerTexel <= 8) {
MOVZX(32, 8, temp2Reg, MComplex(baseReg, temp2Reg, SCALE_1, 0));
PINSRW(destReg, R(temp2Reg), i * 2);
} else if (bitsPerTexel == 16) {
PINSRW(destReg, MComplex(baseReg, temp2Reg, SCALE_1, 0), i * 2);
} else if (bitsPerTexel == 32) {
if (i == 0) {
MOVD_xmm(destReg, MComplex(baseReg, temp2Reg, SCALE_1, 0));
} else {
// Maybe a temporary would be better, but this path should be rare.
PINSRW(destReg, MComplex(baseReg, temp2Reg, SCALE_1, 0), i * 2);
PINSRW(destReg, MComplex(baseReg, temp2Reg, SCALE_1, 2), i * 2 + 1);
}
}
}
}
regCache_.Release(temp2Reg, RegCache::GEN_TEMP2);
}
regCache_.Unlock(byteOffsetReg, level1 ? RegCache::VEC_V1 : RegCache::VEC_ARG_V);
regCache_.ForceRelease(level1 ? RegCache::VEC_V1 : RegCache::VEC_ARG_V);
regCache_.Release(baseReg, RegCache::GEN_ARG_TEXPTR);
if (bitsPerTexel == 4) {
// Take only lowest bit, multiply by 4 with shifting.
X64Reg uReg = regCache_.Find(level1 ? RegCache::VEC_U1 : RegCache::VEC_ARG_U);
// Next, shift away based on the odd U bits.
if (cpu_info.bAVX2) {
// This is really convenient with AVX. Just make the bit into a shift amount.
PSLLD(uReg, 31);
PSRLD(uReg, 29);
VPSRLVD(128, destReg, destReg, R(uReg));
} else {
// This creates a mask - FFFFFFFF to shift, zero otherwise.
PSLLD(uReg, 31);
PSRAD(uReg, 31);
X64Reg unshiftedReg = regCache_.Alloc(RegCache::VEC_TEMP0);
MOVDQA(unshiftedReg, R(destReg));
PSRLD(destReg, 4);
// Mask destReg (shifted) and reverse uReg to unshifted masked.
PAND(destReg, R(uReg));
PANDN(uReg, R(unshiftedReg));
// Now combine.
POR(destReg, R(uReg));
regCache_.Release(unshiftedReg, RegCache::VEC_TEMP0);
}
regCache_.Unlock(uReg, level1 ? RegCache::VEC_U1 : RegCache::VEC_ARG_U);
regCache_.ForceRelease(level1 ? RegCache::VEC_U1 : RegCache::VEC_ARG_U);
}
if (id.TexFmt() >= GE_TFMT_CLUT4 && id.TexFmt() <= GE_TFMT_CLUT32) {
regCache_.Unlock(destReg, RegCache::VEC_INDEX);
} else {
regCache_.Unlock(destReg, level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
regCache_.ForceRetain(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
}
return success;
}
bool SamplerJitCache::Jit_TransformClutIndexQuad(const SamplerID &id, int bitsPerIndex) {
Describe("TrCLUTQuad");
GEPaletteFormat fmt = id.ClutFmt();
if (!id.hasClutShift && !id.hasClutMask && !id.hasClutOffset) {
// This is simple - just mask.
X64Reg indexReg = regCache_.Find(RegCache::VEC_INDEX);
// Mask to 8 bits for CLUT8/16/32, 4 bits for CLUT4.
PSLLD(indexReg, bitsPerIndex >= 8 ? 24 : 28);
PSRLD(indexReg, bitsPerIndex >= 8 ? 24 : 28);
regCache_.Unlock(indexReg, RegCache::VEC_INDEX);
return true;
}
X64Reg indexReg = regCache_.Find(RegCache::VEC_INDEX);
bool maskedIndex = false;
// Okay, first load the actual samplerID clutformat bits we'll use.
X64Reg formatReg = regCache_.Alloc(RegCache::VEC_TEMP0);
X64Reg idReg = GetSamplerID();
if (cpu_info.bAVX2 && !id.hasClutShift)
VPBROADCASTD(128, formatReg, MDisp(idReg, offsetof(SamplerID, cached.clutFormat)));
else
MOVD_xmm(formatReg, MDisp(idReg, offsetof(SamplerID, cached.clutFormat)));
UnlockSamplerID(idReg);
// Shift = (clutformat >> 2) & 0x1F
if (id.hasClutShift) {
// Before shifting, let's mask if needed (we always read 32 bits.)
// We have to do this here, because the bits should be zero even if F is used as a mask.
if (bitsPerIndex < 32) {
PSLLD(indexReg, 32 - bitsPerIndex);
PSRLD(indexReg, 32 - bitsPerIndex);
maskedIndex = true;
}
X64Reg shiftReg = regCache_.Alloc(RegCache::VEC_TEMP1);
// Shift against walls to get 5 bits after the rightmost 2.
PSLLD(shiftReg, formatReg, 32 - 7);
PSRLD(shiftReg, 32 - 5);
// The other lanes are zero, so we can use PSRLD.
PSRLD(indexReg, R(shiftReg));
regCache_.Release(shiftReg, RegCache::VEC_TEMP1);
}
// With shifting done, we need the format in each lane.
if (!cpu_info.bAVX2 || id.hasClutShift)
PSHUFD(formatReg, R(formatReg), _MM_SHUFFLE(0, 0, 0, 0));
// Mask = (clutformat >> 8) & 0xFF
if (id.hasClutMask) {
X64Reg maskReg = regCache_.Alloc(RegCache::VEC_TEMP1);
// If it was CLUT4, grab only 4 bits of the mask.
PSLLD(maskReg, formatReg, bitsPerIndex == 4 ? 20 : 16);
PSRLD(maskReg, bitsPerIndex == 4 ? 28 : 24);
PAND(indexReg, R(maskReg));
regCache_.Release(maskReg, RegCache::VEC_TEMP1);
} else if (!maskedIndex || bitsPerIndex > 8) {
// Apply the fixed 8 bit mask (or the CLUT4 mask if we didn't shift.)
PSLLD(indexReg, maskedIndex || bitsPerIndex >= 8 ? 24 : 28);
PSRLD(indexReg, maskedIndex || bitsPerIndex >= 8 ? 24 : 28);
}
// Offset = (clutformat >> 12) & 0x01F0
if (id.hasClutOffset) {
// Use walls to extract the 5 bits at 16, and then put them shifted left by 4.
int offsetBits = fmt == GE_CMODE_32BIT_ABGR8888 ? 4 : 5;
PSRLD(formatReg, 16);
PSLLD(formatReg, 32 - offsetBits);
PSRLD(formatReg, 32 - offsetBits - 4);
POR(indexReg, R(formatReg));
}
regCache_.Release(formatReg, RegCache::VEC_TEMP0);
regCache_.Unlock(indexReg, RegCache::VEC_INDEX);
return true;
}
bool SamplerJitCache::Jit_ReadClutQuad(const SamplerID &id, bool level1) {
Describe("ReadCLUTQuad");
X64Reg indexReg = regCache_.Find(RegCache::VEC_INDEX);
if (!id.useSharedClut) {
X64Reg vecLevelReg = regCache_.Alloc(RegCache::VEC_TEMP0);
if (regCache_.Has(RegCache::GEN_ARG_LEVEL)) {
X64Reg levelReg = regCache_.Find(RegCache::GEN_ARG_LEVEL);
MOVD_xmm(vecLevelReg, R(levelReg));
regCache_.Unlock(levelReg, RegCache::GEN_ARG_LEVEL);
} else {
#if PPSSPP_PLATFORM(WINDOWS)
if (cpu_info.bAVX2) {
VPBROADCASTD(128, vecLevelReg, MDisp(RSP, stackArgPos_ + stackLevelOffset_));
} else {
MOVD_xmm(vecLevelReg, MDisp(RSP, stackArgPos_ + stackLevelOffset_));
PSHUFD(vecLevelReg, R(vecLevelReg), _MM_SHUFFLE(0, 0, 0, 0));
}
#else
_assert_(false);
#endif
}
// Now we multiply by 16, and add.
PSLLD(vecLevelReg, 4);
PADDD(indexReg, R(vecLevelReg));
regCache_.Release(vecLevelReg, RegCache::VEC_TEMP0);
}
X64Reg idReg = GetSamplerID();
X64Reg clutBaseReg = regCache_.Alloc(RegCache::GEN_TEMP1);
MOV(PTRBITS, R(clutBaseReg), MDisp(idReg, offsetof(SamplerID, cached.clut)));
UnlockSamplerID(idReg);
X64Reg resultReg = INVALID_REG;
if (regCache_.Has(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT))
resultReg = regCache_.Find(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
else
resultReg = regCache_.Alloc(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
X64Reg maskReg = regCache_.Alloc(RegCache::VEC_TEMP0);
if (cpu_info.bAVX2 && id.overReadSafe)
PCMPEQD(maskReg, R(maskReg));
switch (id.ClutFmt()) {
case GE_CMODE_16BIT_BGR5650:
case GE_CMODE_16BIT_ABGR5551:
case GE_CMODE_16BIT_ABGR4444:
if (cpu_info.bAVX2 && id.overReadSafe) {
VPGATHERDD(128, resultReg, MComplex(clutBaseReg, indexReg, SCALE_2, 0), maskReg);
// Clear out the top 16 bits.
PCMPEQD(maskReg, R(maskReg));
PSRLD(maskReg, 16);
PAND(resultReg, R(maskReg));
} else {
PXOR(resultReg, R(resultReg));
X64Reg temp2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
if (cpu_info.bSSE4_1) {
for (int i = 0; i < 4; ++i) {
PEXTRD(R(temp2Reg), indexReg, i);
PINSRW(resultReg, MComplex(clutBaseReg, temp2Reg, SCALE_2, 0), i * 2);
}
} else {
for (int i = 0; i < 4; ++i) {
MOVD_xmm(R(temp2Reg), indexReg);
if (i != 3)
PSRLDQ(indexReg, 4);
PINSRW(resultReg, MComplex(clutBaseReg, temp2Reg, SCALE_2, 0), i * 2);
}
}
regCache_.Release(temp2Reg, RegCache::GEN_TEMP2);
}
break;
case GE_CMODE_32BIT_ABGR8888:
if (cpu_info.bAVX2 && id.overReadSafe) {
VPGATHERDD(128, resultReg, MComplex(clutBaseReg, indexReg, SCALE_4, 0), maskReg);
} else {
X64Reg temp2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
if (cpu_info.bSSE4_1) {
for (int i = 0; i < 4; ++i) {
PEXTRD(R(temp2Reg), indexReg, i);
PINSRD(resultReg, MComplex(clutBaseReg, temp2Reg, SCALE_4, 0), i);
}
} else {
for (int i = 0; i < 4; ++i) {
MOVD_xmm(R(temp2Reg), indexReg);
if (i != 3)
PSRLDQ(indexReg, 4);
if (i == 0) {
MOVD_xmm(resultReg , MComplex(clutBaseReg, temp2Reg, SCALE_4, 0));
} else {
MOVD_xmm(maskReg, MComplex(clutBaseReg, temp2Reg, SCALE_4, 0));
PSLLDQ(maskReg, 4 * i);
POR(resultReg, R(maskReg));
}
}
}
regCache_.Release(temp2Reg, RegCache::GEN_TEMP2);
}
break;
}
regCache_.Release(maskReg, RegCache::VEC_TEMP0);
regCache_.Unlock(resultReg, level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
regCache_.ForceRetain(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
regCache_.Release(clutBaseReg, RegCache::GEN_TEMP1);
regCache_.Release(indexReg, RegCache::VEC_INDEX);
return true;
}
bool SamplerJitCache::Jit_BlendQuad(const SamplerID &id, bool level1) {
Describe(level1 ? "BlendQuadMips" : "BlendQuad");
if (cpu_info.bSSE4_1 && cpu_info.bSSSE3) {
// Let's start by rearranging from TL TR BL BR like this:
// ABCD EFGH IJKL MNOP -> AI BJ CK DL EM FN GO HP -> AIEM BJFN CKGO DLHP
// This way, all the RGBAs are next to each other, and in order TL BL TR BR.
X64Reg quadReg = regCache_.Find(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
X64Reg tempArrangeReg = regCache_.Alloc(RegCache::VEC_TEMP0);
PSHUFD(tempArrangeReg, R(quadReg), _MM_SHUFFLE(3, 2, 3, 2));
PUNPCKLBW(quadReg, R(tempArrangeReg));
// Okay, that's top and bottom interleaved, now for left and right.
PSHUFD(tempArrangeReg, R(quadReg), _MM_SHUFFLE(3, 2, 3, 2));
PUNPCKLWD(quadReg, R(tempArrangeReg));
regCache_.Release(tempArrangeReg, RegCache::VEC_TEMP0);
// Next up, we want to multiply and add using a repeated TB frac pair.
// That's (0x10 - frac_v) in byte 1, frac_v in byte 2, repeating.
X64Reg fracReg = regCache_.Alloc(RegCache::VEC_TEMP0);
X64Reg allFracReg = regCache_.Find(RegCache::VEC_FRAC);
X64Reg zeroReg = GetZeroVec();
if (level1) {
PSHUFLW(fracReg, R(allFracReg), _MM_SHUFFLE(3, 3, 3, 3));
} else {
PSHUFLW(fracReg, R(allFracReg), _MM_SHUFFLE(1, 1, 1, 1));
}
PSHUFB(fracReg, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
regCache_.Unlock(allFracReg, RegCache::VEC_FRAC);
// Now, inverse fracReg, then interleave into the actual multiplier.
// This gives us the repeated TB pairs we wanted.
X64Reg multTBReg = regCache_.Alloc(RegCache::VEC_TEMP1);
MOVDQA(multTBReg, M(const10All8_));
PSUBB(multTBReg, R(fracReg));
PUNPCKLBW(multTBReg, R(fracReg));
regCache_.Release(fracReg, RegCache::VEC_TEMP0);
// Now we can multiply and add paired lanes in one go.
// Note that since T+B=0x10, this gives us exactly 12 bits.
PMADDUBSW(quadReg, R(multTBReg));
regCache_.Release(multTBReg, RegCache::VEC_TEMP1);
// With that done, we need to multiply by LR, or rather 0L0R, and sum again.
// Since RRRR was all next to each other, this gives us a clean total R.
fracReg = regCache_.Alloc(RegCache::VEC_TEMP0);
allFracReg = regCache_.Find(RegCache::VEC_FRAC);
if (level1) {
PSHUFLW(fracReg, R(allFracReg), _MM_SHUFFLE(2, 2, 2, 2));
} else {
// We can ignore the high bits, since we'll interleave those away anyway.
PSHUFLW(fracReg, R(allFracReg), _MM_SHUFFLE(0, 0, 0, 0));
}
regCache_.Unlock(allFracReg, RegCache::VEC_FRAC);
// Again, we're inversing into an interleaved multiplier. L is the inversed one.
// 0L0R is (0x10 - frac_u), frac_u - 2x16 repeated four times.
X64Reg multLRReg = regCache_.Alloc(RegCache::VEC_TEMP1);
MOVDQA(multLRReg, M(const10All16_));
PSUBW(multLRReg, R(fracReg));
PUNPCKLWD(multLRReg, R(fracReg));
regCache_.Release(fracReg, RegCache::VEC_TEMP0);
// This gives us RGBA as dwords, but they're all shifted left by 8 from the multiplies.
PMADDWD(quadReg, R(multLRReg));
PSRLD(quadReg, 8);
regCache_.Release(multLRReg, RegCache::VEC_TEMP1);
// Shrink to 16-bit, it's more convenient for later.
if (level1) {
PACKSSDW(quadReg, R(quadReg));
regCache_.Unlock(quadReg, RegCache::VEC_RESULT1);
} else {
if (cpu_info.bAVX) {
VPACKSSDW(128, XMM0, quadReg, R(quadReg));
} else {
PACKSSDW(quadReg, R(quadReg));
MOVDQA(XMM0, R(quadReg));
}
regCache_.Unlock(quadReg, RegCache::VEC_RESULT);
regCache_.ForceRelease(RegCache::VEC_RESULT);
bool changeSuccess = regCache_.ChangeReg(XMM0, RegCache::VEC_RESULT);
_assert_msg_(changeSuccess, "Unexpected reg locked as destReg");
}
} else {
X64Reg topReg = regCache_.Alloc(RegCache::VEC_TEMP0);
X64Reg bottomReg = regCache_.Alloc(RegCache::VEC_TEMP1);
X64Reg quadReg = regCache_.Find(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
X64Reg zeroReg = GetZeroVec();
PSHUFD(topReg, R(quadReg), _MM_SHUFFLE(0, 0, 1, 0));
PSHUFD(bottomReg, R(quadReg), _MM_SHUFFLE(0, 0, 3, 2));
PUNPCKLBW(topReg, R(zeroReg));
PUNPCKLBW(bottomReg, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
if (!level1) {
regCache_.Unlock(quadReg, RegCache::VEC_RESULT);
regCache_.ForceRelease(RegCache::VEC_RESULT);
}
// Grab frac_u and spread to lower (L) lanes.
X64Reg fracReg = regCache_.Alloc(RegCache::VEC_TEMP2);
X64Reg allFracReg = regCache_.Find(RegCache::VEC_FRAC);
X64Reg fracMulReg = regCache_.Alloc(RegCache::VEC_TEMP3);
if (level1) {
PSHUFLW(fracReg, R(allFracReg), _MM_SHUFFLE(2, 2, 2, 2));
} else {
PSHUFLW(fracReg, R(allFracReg), _MM_SHUFFLE(0, 0, 0, 0));
}
regCache_.Unlock(allFracReg, RegCache::VEC_FRAC);
// Now subtract 0x10 - frac_u in the L lanes only: 00000000 LLLLLLLL.
MOVDQA(fracMulReg, M(const10Low_));
PSUBW(fracMulReg, R(fracReg));
// Then we just put the original frac_u in the upper bits.
PUNPCKLQDQ(fracMulReg, R(fracReg));
regCache_.Release(fracReg, RegCache::VEC_TEMP2);
// Okay, we have 8-bits in the top and bottom rows for the color.
// Multiply by frac to get 12, which we keep for the next stage.
PMULLW(topReg, R(fracMulReg));
PMULLW(bottomReg, R(fracMulReg));
regCache_.Release(fracMulReg, RegCache::VEC_TEMP3);
// Time for frac_v. This time, we want it in all 8 lanes.
fracReg = regCache_.Alloc(RegCache::VEC_TEMP2);
allFracReg = regCache_.Find(RegCache::VEC_FRAC);
X64Reg fracTopReg = regCache_.Alloc(RegCache::VEC_TEMP3);
if (level1) {
PSHUFLW(fracReg, R(allFracReg), _MM_SHUFFLE(3, 3, 3, 3));
} else {
PSHUFLW(fracReg, R(allFracReg), _MM_SHUFFLE(1, 1, 1, 1));
}
PSHUFD(fracReg, R(fracReg), _MM_SHUFFLE(0, 0, 0, 0));
regCache_.Unlock(allFracReg, RegCache::VEC_FRAC);
// Now, inverse fracReg into fracTopReg for the top row.
MOVDQA(fracTopReg, M(const10All16_));
PSUBW(fracTopReg, R(fracReg));
// We had 12, plus 4 frac, that gives us 16.
PMULLW(bottomReg, R(fracReg));
PMULLW(topReg, R(fracTopReg));
regCache_.Release(fracReg, RegCache::VEC_TEMP2);
regCache_.Release(fracTopReg, RegCache::VEC_TEMP3);
// Finally, time to sum them all up and divide by 256 to get back to 8 bits.
PADDUSW(bottomReg, R(topReg));
regCache_.Release(topReg, RegCache::VEC_TEMP0);
if (level1) {
PSHUFD(quadReg, R(bottomReg), _MM_SHUFFLE(3, 2, 3, 2));
PADDUSW(quadReg, R(bottomReg));
PSRLW(quadReg, 8);
regCache_.Release(bottomReg, RegCache::VEC_TEMP1);
regCache_.Unlock(quadReg, RegCache::VEC_RESULT1);
} else {
bool changeSuccess = regCache_.ChangeReg(XMM0, RegCache::VEC_RESULT);
if (!changeSuccess) {
_assert_msg_(XMM0 == bottomReg, "Unexpected other reg locked as destReg");
X64Reg otherReg = regCache_.Alloc(RegCache::VEC_TEMP0);
PSHUFD(otherReg, R(bottomReg), _MM_SHUFFLE(3, 2, 3, 2));
PADDUSW(bottomReg, R(otherReg));
regCache_.Release(otherReg, RegCache::VEC_TEMP0);
regCache_.Release(bottomReg, RegCache::VEC_TEMP1);
// Okay, now it can be changed.
regCache_.ChangeReg(XMM0, RegCache::VEC_RESULT);
} else {
PSHUFD(XMM0, R(bottomReg), _MM_SHUFFLE(3, 2, 3, 2));
PADDUSW(XMM0, R(bottomReg));
regCache_.Release(bottomReg, RegCache::VEC_TEMP1);
}
PSRLW(XMM0, 8);
}
}
return true;
}
bool SamplerJitCache::Jit_ApplyTextureFunc(const SamplerID &id) {
X64Reg resultReg = regCache_.Find(RegCache::VEC_RESULT);
X64Reg primColorReg = regCache_.Find(RegCache::VEC_ARG_COLOR);
X64Reg tempReg = regCache_.Alloc(RegCache::VEC_TEMP0);
auto useAlphaFrom = [&](X64Reg alphaColorReg) {
if (cpu_info.bSSE4_1) {
// Copy only alpha.
PBLENDW(resultReg, R(alphaColorReg), 0x08);
} else {
PSRLDQ(alphaColorReg, 6);
PSLLDQ(alphaColorReg, 6);
// Zero out the result alpha and OR them together.
PSLLDQ(resultReg, 10);
PSRLDQ(resultReg, 10);
POR(resultReg, R(alphaColorReg));
}
};
// Note: color is in DWORDs, but result is in WORDs.
switch (id.TexFunc()) {
case GE_TEXFUNC_MODULATE:
Describe("Modulate");
PACKSSDW(primColorReg, R(primColorReg));
if (cpu_info.bAVX) {
VPADDW(128, tempReg, primColorReg, M(constOnes16_));
// Okay, time to multiply. This produces 16 bits, neatly.
VPMULLW(128, resultReg, tempReg, R(resultReg));
} else {
MOVDQA(tempReg, M(constOnes16_));
PADDW(tempReg, R(primColorReg));
PMULLW(resultReg, R(tempReg));
}
if (id.useColorDoubling)
PSRLW(resultReg, 7);
else
PSRLW(resultReg, 8);
if (!id.useTextureAlpha) {
useAlphaFrom(primColorReg);
} else if (id.useColorDoubling) {
// We still need to finish dividing alpha, it's currently doubled (from the 7 above.)
PSRLW(primColorReg, resultReg, 1);
useAlphaFrom(primColorReg);
}
break;
case GE_TEXFUNC_DECAL:
Describe("Decal");
PACKSSDW(primColorReg, R(primColorReg));
if (id.useTextureAlpha) {
// Get alpha into the tempReg.
PSHUFLW(tempReg, R(resultReg), _MM_SHUFFLE(3, 3, 3, 3));
PADDW(resultReg, M(constOnes16_));
PMULLW(resultReg, R(tempReg));
X64Reg invAlphaReg = regCache_.Alloc(RegCache::VEC_TEMP1);
// Materialize some 255s, and subtract out alpha.
PCMPEQD(invAlphaReg, R(invAlphaReg));
PSRLW(invAlphaReg, 8);
PSUBW(invAlphaReg, R(tempReg));
MOVDQA(tempReg, R(primColorReg));
PADDW(tempReg, M(constOnes16_));
PMULLW(tempReg, R(invAlphaReg));
regCache_.Release(invAlphaReg, RegCache::VEC_TEMP1);
// Now sum, and divide.
PADDW(resultReg, R(tempReg));
if (id.useColorDoubling)
PSRLW(resultReg, 7);
else
PSRLW(resultReg, 8);
} else if (id.useColorDoubling) {
PSLLW(resultReg, 1);
}
useAlphaFrom(primColorReg);
break;
case GE_TEXFUNC_BLEND:
{
Describe("EnvBlend");
PACKSSDW(primColorReg, R(primColorReg));
// First off, let's grab the color value.
X64Reg idReg = GetSamplerID();
X64Reg texEnvReg = regCache_.Alloc(RegCache::VEC_TEMP1);
if (cpu_info.bSSE4_1) {
PMOVZXBW(texEnvReg, MDisp(idReg, offsetof(SamplerID, cached.texBlendColor)));
} else {
MOVD_xmm(texEnvReg, MDisp(idReg, offsetof(SamplerID, cached.texBlendColor)));
X64Reg zeroReg = GetZeroVec();
PUNPCKLBW(texEnvReg, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
}
UnlockSamplerID(idReg);
// Now merge in the prim color so we have them interleaved, texenv low.
PUNPCKLWD(texEnvReg, R(primColorReg));
// Okay, now materialize 255 for inversing resultReg and rounding.
PCMPEQD(tempReg, R(tempReg));
PSRLW(tempReg, 8);
// If alpha is used, we want the roundup and factor to be zero.
if (id.useTextureAlpha)
PSRLDQ(tempReg, 10);
// We're going to lose tempReg, so save the 255s.
X64Reg roundValueReg = regCache_.Alloc(RegCache::VEC_TEMP2);
MOVDQA(roundValueReg, R(tempReg));
// Okay, now inverse, then merge with resultReg low to match texenv low.
PSUBUSW(tempReg, R(resultReg));
PUNPCKLWD(resultReg, R(tempReg));
if (id.useTextureAlpha) {
// Before we multiply, let's include alpha in that multiply.
PADDW(primColorReg, M(constOnes16_));
// Mask off everything but alpha, and move to the second highest short.
PSRLDQ(primColorReg, 6);
PSLLDQ(primColorReg, 12);
// Now simply merge in with texenv.
POR(texEnvReg, R(primColorReg));
}
// Alright, now to multiply and add all in one go. Note this gives us DWORDs.
PMADDWD(resultReg, R(texEnvReg));
regCache_.Release(texEnvReg, RegCache::VEC_TEMP1);
// Now convert back to 16 bit and add the 255s for rounding.
if (cpu_info.bSSE4_1) {
PACKUSDW(resultReg, R(resultReg));
} else {
PSLLD(resultReg, 16);
PSRAD(resultReg, 16);
PACKSSDW(resultReg, R(resultReg));
}
PADDW(resultReg, R(roundValueReg));
regCache_.Release(roundValueReg, RegCache::VEC_TEMP2);
// Okay, divide by 256 or 128 depending on doubling (we want to preserve the precision.)
if (id.useColorDoubling && id.useTextureAlpha) {
// If doubling, we want to still divide alpha by 256.
PSRLW(resultReg, 7);
PSRLW(primColorReg, resultReg, 1);
useAlphaFrom(primColorReg);
} else if (id.useColorDoubling) {
PSRLW(resultReg, 7);
} else {
PSRLW(resultReg, 8);
}
if (!id.useTextureAlpha)
useAlphaFrom(primColorReg);
break;
}
case GE_TEXFUNC_REPLACE:
Describe("Replace");
if (id.useColorDoubling && id.useTextureAlpha) {
// We can abuse primColorReg as a temp.
MOVDQA(primColorReg, R(resultReg));
// Shift to zero out alpha in resultReg.
PSLLDQ(resultReg, 10);
PSRLDQ(resultReg, 10);
// Now simply add them together, restoring alpha and doubling the colors.
PADDW(resultReg, R(primColorReg));
} else if (!id.useTextureAlpha) {
if (id.useColorDoubling) {
// Let's just double using shifting. Ignore alpha.
PSLLW(resultReg, 1);
}
// Now we want prim_color in W, so convert, then shift-mask away the color.
PACKSSDW(primColorReg, R(primColorReg));
useAlphaFrom(primColorReg);
}
break;
case GE_TEXFUNC_ADD:
case GE_TEXFUNC_UNKNOWN1:
case GE_TEXFUNC_UNKNOWN2:
case GE_TEXFUNC_UNKNOWN3:
Describe("Add");
PACKSSDW(primColorReg, R(primColorReg));
if (id.useTextureAlpha) {
MOVDQA(tempReg, M(constOnes16_));
// Add and multiply the alpha (and others, but we'll mask them.)
PADDW(tempReg, R(primColorReg));
PMULLW(tempReg, R(resultReg));
// Now that we've extracted alpha, sum and double as needed.
PADDW(resultReg, R(primColorReg));
if (id.useColorDoubling)
PSLLW(resultReg, 1);
// Divide by 256 to normalize alpha.
PSRLW(tempReg, 8);
useAlphaFrom(tempReg);
} else {
PADDW(resultReg, R(primColorReg));
if (id.useColorDoubling)
PSLLW(resultReg, 1);
useAlphaFrom(primColorReg);
}
break;
}
regCache_.Release(tempReg, RegCache::VEC_TEMP0);
regCache_.Unlock(resultReg, RegCache::VEC_RESULT);
regCache_.Unlock(primColorReg, RegCache::VEC_ARG_COLOR);
regCache_.ForceRelease(RegCache::VEC_ARG_COLOR);
return true;
}
bool SamplerJitCache::Jit_ReadTextureFormat(const SamplerID &id) {
GETextureFormat fmt = id.TexFmt();
bool success = true;
switch (fmt) {
case GE_TFMT_5650:
success = Jit_GetTexData(id, 16);
if (success)
success = Jit_Decode5650(id);
break;
case GE_TFMT_5551:
success = Jit_GetTexData(id, 16);
if (success)
success = Jit_Decode5551(id);
break;
case GE_TFMT_4444:
success = Jit_GetTexData(id, 16);
if (success)
success = Jit_Decode4444(id);
break;
case GE_TFMT_8888:
success = Jit_GetTexData(id, 32);
break;
case GE_TFMT_CLUT32:
success = Jit_GetTexData(id, 32);
if (success)
success = Jit_TransformClutIndex(id, 32);
if (success)
success = Jit_ReadClutColor(id);
break;
case GE_TFMT_CLUT16:
success = Jit_GetTexData(id, 16);
if (success)
success = Jit_TransformClutIndex(id, 16);
if (success)
success = Jit_ReadClutColor(id);
break;
case GE_TFMT_CLUT8:
success = Jit_GetTexData(id, 8);
if (success)
success = Jit_TransformClutIndex(id, 8);
if (success)
success = Jit_ReadClutColor(id);
break;
case GE_TFMT_CLUT4:
success = Jit_GetTexData(id, 4);
if (success)
success = Jit_TransformClutIndex(id, 4);
if (success)
success = Jit_ReadClutColor(id);
break;
case GE_TFMT_DXT1:
success = Jit_GetDXT1Color(id, 8, 255);
break;
case GE_TFMT_DXT3:
success = Jit_GetDXT1Color(id, 16, 0);
if (success)
success = Jit_ApplyDXTAlpha(id);
break;
case GE_TFMT_DXT5:
success = Jit_GetDXT1Color(id, 16, 0);
if (success)
success = Jit_ApplyDXTAlpha(id);
break;
default:
success = false;
}
return success;
}
// Note: afterward, srcReg points at the block, and uReg/vReg have offset into block.
bool SamplerJitCache::Jit_GetDXT1Color(const SamplerID &id, int blockSize, int alpha) {
Describe("DXT1");
// Like Jit_GetTexData, this gets the color into resultReg.
// Note: color low bits are red, high bits are blue.
_assert_msg_(blockSize == 8 || blockSize == 16, "Invalid DXT block size");
X64Reg colorIndexReg = INVALID_REG;
if (!id.linear) {
// First, we need to get the block's offset, which is:
// blockPos = src + (v/4 * bufw/4 + u/4) * blockSize
// We distribute the blockSize constant for convenience:
// blockPos = src + (blockSize*v/4 * bufw/4 + blockSize*u/4)
// Copy u (we'll need it later), and round down to the nearest 4 after scaling.
X64Reg uReg = regCache_.Find(RegCache::GEN_ARG_U);
X64Reg srcBaseReg = regCache_.Alloc(RegCache::GEN_TEMP0);
LEA(32, srcBaseReg, MScaled(uReg, blockSize / 4, 0));
AND(32, R(srcBaseReg), Imm32(blockSize == 8 ? ~7 : ~15));
// Add in srcReg already, since we'll be multiplying soon.
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
ADD(64, R(srcBaseReg), R(srcReg));
X64Reg vReg = regCache_.Find(RegCache::GEN_ARG_V);
X64Reg srcOffsetReg = regCache_.Alloc(RegCache::GEN_TEMP1);
LEA(32, srcOffsetReg, MScaled(vReg, blockSize / 4, 0));
AND(32, R(srcOffsetReg), Imm32(blockSize == 8 ? ~7 : ~15));
// Modify bufw in place and then multiply.
X64Reg bufwReg = regCache_.Find(RegCache::GEN_ARG_BUFW);
SHR(32, R(bufwReg), Imm8(2));
IMUL(32, srcOffsetReg, R(bufwReg));
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW);
// We no longer need bufwReg.
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW);
// And now let's chop off the offset for u and v.
AND(32, R(uReg), Imm32(3));
AND(32, R(vReg), Imm32(3));
// Okay, at this point srcBaseReg + srcOffsetReg = blockPos. To free up regs, put back in srcReg.
LEA(64, srcReg, MRegSum(srcBaseReg, srcOffsetReg));
regCache_.Release(srcBaseReg, RegCache::GEN_TEMP0);
regCache_.Release(srcOffsetReg, RegCache::GEN_TEMP1);
// Make sure we don't grab this as colorIndexReg.
if (uReg != ECX && !cpu_info.bBMI2)
regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
// The colorIndex is simply the 2 bits at blockPos + (v & 3), shifted right by (u & 3) twice.
colorIndexReg = regCache_.Alloc(RegCache::GEN_TEMP0);
MOVZX(32, 8, colorIndexReg, MRegSum(srcReg, vReg));
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
regCache_.Unlock(vReg, RegCache::GEN_ARG_V);
// Only DXT3/5 need this reg later.
if (id.TexFmt() == GE_TFMT_DXT1)
regCache_.ForceRelease(RegCache::GEN_ARG_V);
if (uReg == ECX) {
SHR(32, R(colorIndexReg), R(CL));
SHR(32, R(colorIndexReg), R(CL));
} else if (cpu_info.bBMI2) {
SHRX(32, colorIndexReg, R(colorIndexReg), uReg);
SHRX(32, colorIndexReg, R(colorIndexReg), uReg);
} else {
bool hasRCX = regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
_assert_(hasRCX);
LEA(32, ECX, MScaled(uReg, SCALE_2, 0));
SHR(32, R(colorIndexReg), R(CL));
}
regCache_.Unlock(uReg, RegCache::GEN_ARG_U);
// If DXT1, there's no alpha and we can toss this reg.
if (id.TexFmt() == GE_TFMT_DXT1)
regCache_.ForceRelease(RegCache::GEN_ARG_U);
} else {
// For linear, we already precalculated the block pos into srcReg.
// uReg is the shift for the color index fomr the 32 bits of color index data.
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW);
// If we don't have alpha, we don't need vReg.
if (id.TexFmt() == GE_TFMT_DXT1)
regCache_.ForceRelease(RegCache::GEN_ARG_V);
// Make sure we don't grab this as colorIndexReg.
X64Reg uReg = regCache_.Find(RegCache::GEN_ARG_U);
if (uReg != ECX && !cpu_info.bBMI2)
regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
// Shift and mask out the 2 bits we need into colorIndexReg.
colorIndexReg = regCache_.Alloc(RegCache::GEN_TEMP0);
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
if (cpu_info.bBMI2) {
SHRX(32, colorIndexReg, MatR(srcReg), uReg);
} else {
MOV(32, R(colorIndexReg), MatR(srcReg));
if (uReg != RCX) {
bool hasRCX = regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
_assert_(hasRCX);
MOV(32, R(RCX), R(uReg));
}
SHR(32, R(colorIndexReg), R(CL));
}
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
// We're done with U now.
regCache_.Unlock(uReg, RegCache::GEN_ARG_U);
regCache_.ForceRelease(RegCache::GEN_ARG_U);
}
// Mask out the value.
AND(32, R(colorIndexReg), Imm32(3));
X64Reg color1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
X64Reg color2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
// For colorIndex 0 or 1, we'll simply take the 565 color and convert.
CMP(32, R(colorIndexReg), Imm32(1));
FixupBranch handleSimple565 = J_CC(CC_BE);
// Otherwise, it depends if color1 or color2 is higher, so fetch them.
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
MOVZX(32, 16, color1Reg, MDisp(srcReg, 4));
MOVZX(32, 16, color2Reg, MDisp(srcReg, 6));
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
CMP(32, R(color1Reg), R(color2Reg));
FixupBranch handleMix23 = J_CC(CC_A, true);
// If we're still here, then colorIndex is either 3 for 0 (easy) or 2 for 50% mix.
XOR(32, R(resultReg), R(resultReg));
CMP(32, R(colorIndexReg), Imm32(3));
FixupBranch finishZero = J_CC(CC_E, true);
// At this point, resultReg, colorIndexReg, and maybe R12/R13 can be used as temps.
// We'll add, then shift from 565 a bit less to "divide" by 2 for a 50/50 mix.
if (cpu_info.bBMI2_fast) {
// Expand everything out to 0BGR at 8888, but halved.
MOV(32, R(colorIndexReg), Imm32(0x007C7E7C));
PDEP(32, color1Reg, color1Reg, R(colorIndexReg));
PDEP(32, color2Reg, color2Reg, R(colorIndexReg));
// Now let's sum them together (this undoes our halving.)
LEA(32, resultReg, MRegSum(color1Reg, color2Reg));
// Time to swap into order. Luckily we can ignore alpha.
BSWAP(32, resultReg);
SHR(32, R(resultReg), Imm8(8));
} else {
// We'll need more regs. Grab two more.
PUSH(R12);
PUSH(R13);
// Start with summing R, then shift into position.
MOV(32, R(resultReg), R(color1Reg));
AND(32, R(resultReg), Imm32(0x0000F800));
MOV(32, R(colorIndexReg), R(color2Reg));
AND(32, R(colorIndexReg), Imm32(0x0000F800));
LEA(32, R12, MRegSum(resultReg, colorIndexReg));
// The position is 9, instead of 8, due to doubling.
SHR(32, R(R12), Imm8(9));
// For G, summing leaves it 4 right (doubling made it not need more.)
MOV(32, R(resultReg), R(color1Reg));
AND(32, R(resultReg), Imm32(0x000007E0));
MOV(32, R(colorIndexReg), R(color2Reg));
AND(32, R(colorIndexReg), Imm32(0x000007E0));
LEA(32, resultReg, MRegSum(resultReg, colorIndexReg));
SHL(32, R(resultReg), Imm8(5 - 1));
// Now add G and R together.
OR(32, R(resultReg), R(R12));
// At B, we're free to modify the regs in place, finally.
AND(32, R(color1Reg), Imm32(0x0000001F));
AND(32, R(color2Reg), Imm32(0x0000001F));
LEA(32, colorIndexReg, MRegSum(color1Reg, color2Reg));
// We shift left 2 into position (not 3 due to doubling), then 16 more into the B slot.
SHL(32, R(colorIndexReg), Imm8(16 + 2));
// And combine into the result.
OR(32, R(resultReg), R(colorIndexReg));
POP(R13);
POP(R12);
}
FixupBranch finishMix50 = J(true);
// Simply load the 565 color, and convert to 0888.
SetJumpTarget(handleSimple565);
srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
MOVZX(32, 16, colorIndexReg, MComplex(srcReg, colorIndexReg, SCALE_2, 4));
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
// DXT1 is done with this reg.
if (id.TexFmt() == GE_TFMT_DXT1)
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR);
if (cpu_info.bBMI2_fast) {
// We're only grabbing the high bits, no swizzle here.
MOV(32, R(resultReg), Imm32(0x00F8FCF8));
PDEP(32, resultReg, colorIndexReg, R(resultReg));
BSWAP(32, resultReg);
SHR(32, R(resultReg), Imm8(8));
} else {
// Start with R, shifting it into place.
MOV(32, R(resultReg), R(colorIndexReg));
AND(32, R(resultReg), Imm32(0x0000F800));
SHR(32, R(resultReg), Imm8(8));
// Then take G and shift it too.
MOV(32, R(color2Reg), R(colorIndexReg));
AND(32, R(color2Reg), Imm32(0x000007E0));
SHL(32, R(color2Reg), Imm8(5));
// And now combine with R, shifting that in the process.
OR(32, R(resultReg), R(color2Reg));
// Modify B in place and OR in.
AND(32, R(colorIndexReg), Imm32(0x0000001F));
SHL(32, R(colorIndexReg), Imm8(16 + 3));
OR(32, R(resultReg), R(colorIndexReg));
}
FixupBranch finish565 = J(true);
// Here we'll mix color1 and color2 by 2/3 (which gets the 2 depends on colorIndexReg.)
SetJumpTarget(handleMix23);
// If colorIndexReg is 2, it's color1Reg * 2 + color2Reg, but if colorIndexReg is 3, it's reversed.
// Let's swap the regs in that case.
CMP(32, R(colorIndexReg), Imm32(2));
FixupBranch skipSwap23 = J_CC(CC_E);
XCHG(32, R(color2Reg), R(color1Reg));
SetJumpTarget(skipSwap23);
if (cpu_info.bBMI2_fast) {
// Gather B, G, and R and space them apart by 14 or 15 bits.
MOV(64, R(colorIndexReg), Imm64(0x00001F0003F0001FULL));
PDEP(64, color1Reg, color1Reg, R(colorIndexReg));
PDEP(64, color2Reg, color2Reg, R(colorIndexReg));
LEA(64, resultReg, MComplex(color2Reg, color1Reg, SCALE_2, 0));
// Now multiply all of them by a special constant to divide by 3.
// This constant is (1 << 13) / 3, which is importantly less than 14 or 15.
IMUL(64, resultReg, R(resultReg), Imm32(0x00000AAB));
// Now extract the BGR values to 8 bits each.
// We subtract 3 from 13 to get 8 from 5 bits, then 2 from 20 + 13, and 3 from 40 + 13.
MOV(64, R(colorIndexReg), Imm64((0xFFULL << 10) | (0xFFULL << 31) | (0xFFULL << 50)));
PEXT(64, resultReg, resultReg, R(colorIndexReg));
// Finally swap B and R.
BSWAP(32, resultReg);
SHR(32, R(resultReg), Imm8(8));
} else {
// We'll need more regs. Grab two more to keep the stack aligned.
PUSH(R12);
PUSH(R13);
// Start off with R, adding together first...
MOV(32, R(resultReg), R(color1Reg));
AND(32, R(resultReg), Imm32(0x0000F800));
MOV(32, R(colorIndexReg), R(color2Reg));
AND(32, R(colorIndexReg), Imm32(0x0000F800));
LEA(32, resultReg, MComplex(colorIndexReg, resultReg, SCALE_2, 0));
// We'll overflow if we divide here, so shift into place already.
SHR(32, R(resultReg), Imm8(8));
// Now we divide that by 3, by actually multiplying by AAAB and shifting off.
IMUL(32, R12, R(resultReg), Imm32(0x0000AAAB));
// Now we SHR off the extra bits we added on.
SHR(32, R(R12), Imm8(17));
// Now add up G. We leave this in place and shift right more.
MOV(32, R(resultReg), R(color1Reg));
AND(32, R(resultReg), Imm32(0x000007E0));
MOV(32, R(colorIndexReg), R(color2Reg));
AND(32, R(colorIndexReg), Imm32(0x000007E0));
LEA(32, resultReg, MComplex(colorIndexReg, resultReg, SCALE_2, 0));
// Again, multiply and now we use AAAB, this time masking.
IMUL(32, resultReg, R(resultReg), Imm32(0x0000AAAB));
SHR(32, R(resultReg), Imm8(17 - 5));
AND(32, R(resultReg), Imm32(0x0000FF00));
// Let's combine R in already.
OR(32, R(resultReg), R(R12));
// Now for B, it starts in the lowest place so we'll need to mask.
AND(32, R(color1Reg), Imm32(0x0000001F));
AND(32, R(color2Reg), Imm32(0x0000001F));
LEA(32, colorIndexReg, MComplex(color2Reg, color1Reg, SCALE_2, 0));
// Instead of shifting left, though, we multiply by a bit more.
IMUL(32, colorIndexReg, R(colorIndexReg), Imm32(0x0002AAAB));
AND(32, R(colorIndexReg), Imm32(0x00FF0000));
OR(32, R(resultReg), R(colorIndexReg));
POP(R13);
POP(R12);
}
regCache_.Release(colorIndexReg, RegCache::GEN_TEMP0);
regCache_.Release(color1Reg, RegCache::GEN_TEMP1);
regCache_.Release(color2Reg, RegCache::GEN_TEMP2);
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
SetJumpTarget(finishMix50);
SetJumpTarget(finish565);
// In all these cases, it's time to add in alpha. Zero doesn't get it.
if (alpha != 0) {
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
OR(32, R(resultReg), Imm32(alpha << 24));
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
}
SetJumpTarget(finishZero);
return true;
}
bool SamplerJitCache::Jit_ApplyDXTAlpha(const SamplerID &id) {
GETextureFormat fmt = id.TexFmt();
// At this point, srcReg points at the block, and u/v are offsets inside it.
bool success = false;
if (fmt == GE_TFMT_DXT3) {
Describe("DXT3A");
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
X64Reg vReg = regCache_.Find(RegCache::GEN_ARG_V);
if (id.linear) {
// We precalculated the shift for the 64 bits of alpha data in vReg.
if (!cpu_info.bBMI2) {
regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
_assert_(regCache_.Has(RegCache::GEN_SHIFTVAL));
}
if (cpu_info.bBMI2) {
SHRX(64, srcReg, MDisp(srcReg, 8), vReg);
} else {
MOV(64, R(srcReg), MDisp(srcReg, 8));
MOV(32, R(RCX), R(vReg));
SHR(64, R(srcReg), R(CL));
}
// This will mask the 4 bits we want using a wall also.
SHL(32, R(srcReg), Imm8(28));
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
OR(32, R(resultReg), R(srcReg));
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
success = true;
} else {
X64Reg uReg = regCache_.Find(RegCache::GEN_ARG_U);
if (uReg != RCX && !cpu_info.bBMI2) {
regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
_assert_(regCache_.Has(RegCache::GEN_SHIFTVAL));
}
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
MOVZX(32, 16, temp1Reg, MComplex(srcReg, vReg, SCALE_2, 8));
if (cpu_info.bBMI2) {
LEA(32, uReg, MScaled(uReg, SCALE_4, 0));
SHRX(32, temp1Reg, R(temp1Reg), uReg);
} else {
// Still depending on it being GEN_SHIFTVAL or GEN_ARG_U above.
LEA(32, RCX, MScaled(uReg, SCALE_4, 0));
SHR(32, R(temp1Reg), R(CL));
}
SHL(32, R(temp1Reg), Imm8(28));
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
OR(32, R(resultReg), R(temp1Reg));
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
success = true;
regCache_.Unlock(uReg, RegCache::GEN_ARG_U);
regCache_.ForceRelease(RegCache::GEN_ARG_U);
}
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR);
regCache_.Unlock(vReg, RegCache::GEN_ARG_V);
regCache_.ForceRelease(RegCache::GEN_ARG_V);
} else if (fmt == GE_TFMT_DXT5) {
Describe("DXT5A");
X64Reg vReg = regCache_.Find(RegCache::GEN_ARG_V);
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
X64Reg alphaIndexReg = INVALID_REG;
if (id.linear) {
// We precalculated the shift for the 64 bits of alpha data in vReg.
if (cpu_info.bBMI2) {
alphaIndexReg = regCache_.Alloc(RegCache::GEN_TEMP0);
SHRX(64, alphaIndexReg, MDisp(srcReg, 8), vReg);
} else {
regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
alphaIndexReg = regCache_.Alloc(RegCache::GEN_TEMP0);
MOV(64, R(alphaIndexReg), MDisp(srcReg, 8));
MOV(32, R(RCX), R(vReg));
SHR(64, R(alphaIndexReg), R(CL));
}
regCache_.Unlock(vReg, RegCache::GEN_ARG_V);
regCache_.ForceRelease(RegCache::GEN_ARG_V);
} else {
X64Reg uReg = regCache_.Find(RegCache::GEN_ARG_U);
if (uReg != RCX && !cpu_info.bBMI2)
regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
alphaIndexReg = regCache_.Alloc(RegCache::GEN_TEMP0);
// Let's figure out the alphaIndex bit offset so we can read the right byte.
// bitOffset = (u + v * 4) * 3;
LEA(32, uReg, MComplex(uReg, vReg, SCALE_4, 0));
LEA(32, uReg, MComplex(uReg, uReg, SCALE_2, 0));
regCache_.Unlock(vReg, RegCache::GEN_ARG_V);
regCache_.ForceRelease(RegCache::GEN_ARG_V);
if (cpu_info.bBMI2) {
SHRX(64, alphaIndexReg, MDisp(srcReg, 8), uReg);
} else {
// And now the byte offset and bit from there, from those.
MOV(32, R(alphaIndexReg), R(uReg));
SHR(32, R(alphaIndexReg), Imm8(3));
AND(32, R(uReg), Imm32(7));
// Load 16 bits and mask, in case it straddles bytes.
MOVZX(32, 16, alphaIndexReg, MComplex(srcReg, alphaIndexReg, SCALE_1, 8));
// If not, it's in what was bufwReg.
if (uReg != RCX) {
_assert_(regCache_.Has(RegCache::GEN_SHIFTVAL));
MOV(32, R(RCX), R(uReg));
}
SHR(32, R(alphaIndexReg), R(CL));
}
regCache_.Unlock(uReg, RegCache::GEN_ARG_U);
regCache_.ForceRelease(RegCache::GEN_ARG_U);
}
X64Reg alpha1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
X64Reg alpha2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
AND(32, R(alphaIndexReg), Imm32(7));
X64Reg temp3Reg = regCache_.Alloc(RegCache::GEN_TEMP3);
// Okay, now check for 0 or 1 alphaIndex in alphaIndexReg, those are simple.
CMP(32, R(alphaIndexReg), Imm32(1));
FixupBranch handleSimple = J_CC(CC_BE, true);
// Now load a1 and a2, since the rest depend on those values. Frees up srcReg.
MOVZX(32, 8, alpha1Reg, MDisp(srcReg, 14));
MOVZX(32, 8, alpha2Reg, MDisp(srcReg, 15));
CMP(32, R(alpha1Reg), R(alpha2Reg));
FixupBranch handleLerp8 = J_CC(CC_A);
// Okay, check for zero or full alpha, at alphaIndex 6 or 7.
CMP(32, R(alphaIndexReg), Imm32(6));
FixupBranch finishZero = J_CC(CC_E, true);
// Remember, MOV doesn't affect flags.
MOV(32, R(srcReg), Imm32(0xFF));
FixupBranch finishFull = J_CC(CC_A, true);
// At this point, we're handling a 6-step lerp between alpha1 and alpha2.
SHL(32, R(alphaIndexReg), Imm8(8));
// Prepare a multiplier in temp3Reg and multiply alpha1 by it.
MOV(32, R(temp3Reg), Imm32(6 << 8));
SUB(32, R(temp3Reg), R(alphaIndexReg));
IMUL(32, alpha1Reg, R(temp3Reg));
// And now the same for alpha2, using alphaIndexReg.
SUB(32, R(alphaIndexReg), Imm32(1 << 8));
IMUL(32, alpha2Reg, R(alphaIndexReg));
// Let's skip a step and sum before dividing by 5, also adding the 31.
LEA(32, srcReg, MComplex(alpha1Reg, alpha2Reg, SCALE_1, 5 * 31));
// To divide by 5, we will actually multiply by 0x3334 and shift.
IMUL(32, srcReg, Imm32(0x3334));
SHR(32, R(srcReg), Imm8(24));
FixupBranch finishLerp6 = J(true);
// This will be a 8-step lerp between alpha1 and alpha2.
SetJumpTarget(handleLerp8);
SHL(32, R(alphaIndexReg), Imm8(8));
// Prepare a multiplier in temp3Reg and multiply alpha1 by it.
MOV(32, R(temp3Reg), Imm32(8 << 8));
SUB(32, R(temp3Reg), R(alphaIndexReg));
IMUL(32, alpha1Reg, R(temp3Reg));
// And now the same for alpha2, using alphaIndexReg.
SUB(32, R(alphaIndexReg), Imm32(1 << 8));
IMUL(32, alpha2Reg, R(alphaIndexReg));
// And divide by 7 together here too, also adding the 31.
LEA(32, srcReg, MComplex(alpha1Reg, alpha2Reg, SCALE_1, 7 * 31));
// Our magic constant here is 0x124A, but it's a bit more complex than just a shift.
IMUL(32, alpha1Reg, R(srcReg), Imm32(0x124A));
SHR(32, R(alpha1Reg), Imm8(15));
SUB(32, R(srcReg), R(alpha1Reg));
SHR(32, R(srcReg), Imm8(1));
ADD(32, R(srcReg), R(alpha1Reg));
SHR(32, R(srcReg), Imm8(10));
FixupBranch finishLerp8 = J();
SetJumpTarget(handleSimple);
// Just load the specified alpha byte.
MOVZX(32, 8, srcReg, MComplex(srcReg, alphaIndexReg, SCALE_1, 14));
regCache_.Release(alphaIndexReg, RegCache::GEN_TEMP0);
regCache_.Release(alpha1Reg, RegCache::GEN_TEMP1);
regCache_.Release(alpha2Reg, RegCache::GEN_TEMP2);
regCache_.Release(temp3Reg, RegCache::GEN_TEMP3);
SetJumpTarget(finishFull);
SetJumpTarget(finishLerp6);
SetJumpTarget(finishLerp8);
SHL(32, R(srcReg), Imm8(24));
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
OR(32, R(resultReg), R(srcReg));
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
success = true;
SetJumpTarget(finishZero);
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR);
}
_dbg_assert_(success);
return success;
}
bool SamplerJitCache::Jit_GetTexData(const SamplerID &id, int bitsPerTexel) {
if (id.swizzle) {
return Jit_GetTexDataSwizzled(id, bitsPerTexel);
}
_assert_msg_(!id.linear, "Should not use this path for linear")
Describe("TexData");
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
X64Reg temp2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
// srcReg might be EDX, so let's copy and uReg that before we multiply.
X64Reg uReg = regCache_.Find(RegCache::GEN_ARG_U);
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
bool success = true;
switch (bitsPerTexel) {
case 32:
case 16:
case 8:
LEA(64, temp1Reg, MComplex(srcReg, uReg, bitsPerTexel / 8, 0));
break;
case 4: {
if (cpu_info.bBMI2_fast)
MOV(32, R(temp2Reg), Imm32(0x0F));
else
XOR(32, R(temp2Reg), R(temp2Reg));
SHR(32, R(uReg), Imm8(1));
FixupBranch skip = J_CC(CC_NC);
// Track whether we shifted a 1 off or not.
if (cpu_info.bBMI2_fast)
SHL(32, R(temp2Reg), Imm8(4));
else
MOV(32, R(temp2Reg), Imm32(4));
SetJumpTarget(skip);
LEA(64, temp1Reg, MRegSum(srcReg, uReg));
break;
}
default:
success = false;
break;
}
// All done with u and texptr.
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR);
regCache_.Unlock(uReg, RegCache::GEN_ARG_U);
regCache_.ForceRelease(RegCache::GEN_ARG_U);
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
X64Reg vReg = regCache_.Find(RegCache::GEN_ARG_V);
MOV(32, R(resultReg), R(vReg));
regCache_.Unlock(vReg, RegCache::GEN_ARG_V);
regCache_.ForceRelease(RegCache::GEN_ARG_V);
X64Reg bufwReg = regCache_.Find(RegCache::GEN_ARG_BUFW);
IMUL(32, resultReg, R(bufwReg));
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW);
// We can throw bufw away, now.
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW);
if (bitsPerTexel == 4 && !cpu_info.bBMI2) {
bool hasRCX = regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
_assert_(hasRCX);
}
switch (bitsPerTexel) {
case 32:
case 16:
case 8:
MOVZX(32, bitsPerTexel, resultReg, MComplex(temp1Reg, resultReg, bitsPerTexel / 8, 0));
break;
case 4: {
SHR(32, R(resultReg), Imm8(1));
if (cpu_info.bBMI2_fast) {
MOV(8, R(resultReg), MRegSum(temp1Reg, resultReg));
PEXT(32, resultReg, resultReg, R(temp2Reg));
} else if (cpu_info.bBMI2) {
SHRX(32, resultReg, MRegSum(temp1Reg, resultReg), temp2Reg);
AND(32, R(resultReg), Imm8(0x0F));
} else {
MOV(8, R(resultReg), MRegSum(temp1Reg, resultReg));
// RCX is now free.
MOV(8, R(RCX), R(temp2Reg));
SHR(8, R(resultReg), R(RCX));
// Zero out any bits not shifted off.
AND(32, R(resultReg), Imm8(0x0F));
}
break;
}
default:
success = false;
break;
}
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
regCache_.Release(temp2Reg, RegCache::GEN_TEMP2);
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
return success;
}
bool SamplerJitCache::Jit_GetTexDataSwizzled4(const SamplerID &id) {
Describe("TexDataS4");
_assert_msg_(!id.linear, "Should not use this path for linear")
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
X64Reg temp2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
X64Reg uReg = regCache_.Find(RegCache::GEN_ARG_U);
X64Reg vReg = regCache_.Find(RegCache::GEN_ARG_V);
// Get the horizontal tile pos into temp1Reg.
LEA(32, temp1Reg, MScaled(uReg, SCALE_4, 0));
// Note: imm8 sign extends negative.
AND(32, R(temp1Reg), Imm8(~127));
// Add vertical offset inside tile to temp1Reg.
LEA(32, temp2Reg, MScaled(vReg, SCALE_4, 0));
AND(32, R(temp2Reg), Imm8(31));
LEA(32, temp1Reg, MComplex(temp1Reg, temp2Reg, SCALE_4, 0));
// Add srcReg, since we'll need it at some point.
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
ADD(64, R(temp1Reg), R(srcReg));
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR);
// Now find the vertical tile pos, and add to temp1Reg.
SHR(32, R(vReg), Imm8(3));
X64Reg bufwReg = regCache_.Find(RegCache::GEN_ARG_BUFW);
LEA(32, temp2Reg, MScaled(bufwReg, SCALE_4, 0));
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW);
// We can throw bufw away, now.
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW);
IMUL(32, temp2Reg, R(vReg));
ADD(64, R(temp1Reg), R(temp2Reg));
// We no longer have a good value in vReg.
regCache_.Unlock(vReg, RegCache::GEN_ARG_V);
regCache_.ForceRelease(RegCache::GEN_ARG_V);
// Last and possible also least, the horizontal offset inside the tile.
AND(32, R(uReg), Imm8(31));
SHR(32, R(uReg), Imm8(1));
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
MOV(8, R(resultReg), MRegSum(temp1Reg, uReg));
FixupBranch skipNonZero = J_CC(CC_NC);
// If the horizontal offset was odd, take the upper 4.
SHR(8, R(resultReg), Imm8(4));
SetJumpTarget(skipNonZero);
// Zero out the rest of the bits.
AND(32, R(resultReg), Imm8(0x0F));
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
// This destroyed u as well.
regCache_.Unlock(uReg, RegCache::GEN_ARG_U);
regCache_.ForceRelease(RegCache::GEN_ARG_U);
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
regCache_.Release(temp2Reg, RegCache::GEN_TEMP2);
return true;
}
bool SamplerJitCache::Jit_GetTexDataSwizzled(const SamplerID &id, int bitsPerTexel) {
if (bitsPerTexel == 4) {
// Specialized implementation.
return Jit_GetTexDataSwizzled4(id);
}
bool success = true;
_assert_msg_(!id.linear, "Should not use this path for linear")
Describe("TexDataS");
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
X64Reg temp2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
X64Reg uReg = regCache_.Find(RegCache::GEN_ARG_U);
X64Reg vReg = regCache_.Find(RegCache::GEN_ARG_V);
LEA(32, temp1Reg, MScaled(vReg, SCALE_4, 0));
AND(32, R(temp1Reg), Imm8(31));
AND(32, R(vReg), Imm8(~7));
MOV(32, R(temp2Reg), R(uReg));
MOV(32, R(resultReg), R(uReg));
switch (bitsPerTexel) {
case 32:
SHR(32, R(resultReg), Imm8(2));
break;
case 16:
SHR(32, R(vReg), Imm8(1));
SHR(32, R(temp2Reg), Imm8(1));
SHR(32, R(resultReg), Imm8(3));
break;
case 8:
SHR(32, R(vReg), Imm8(2));
SHR(32, R(temp2Reg), Imm8(2));
SHR(32, R(resultReg), Imm8(4));
break;
default:
success = false;
break;
}
AND(32, R(temp2Reg), Imm8(3));
SHL(32, R(resultReg), Imm8(5));
ADD(32, R(temp1Reg), R(temp2Reg));
ADD(32, R(temp1Reg), R(resultReg));
// We may clobber srcReg in the multiply, so let's grab it now.
X64Reg srcReg = regCache_.Find(RegCache::GEN_ARG_TEXPTR);
LEA(64, temp1Reg, MComplex(srcReg, temp1Reg, SCALE_4, 0));
regCache_.Unlock(srcReg, RegCache::GEN_ARG_TEXPTR);
regCache_.ForceRelease(RegCache::GEN_ARG_TEXPTR);
X64Reg bufwReg = regCache_.Find(RegCache::GEN_ARG_BUFW);
LEA(32, resultReg, MScaled(bufwReg, SCALE_4, 0));
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW);
// We can throw bufw away, now.
regCache_.ForceRelease(RegCache::GEN_ARG_BUFW);
IMUL(32, resultReg, R(vReg));
// We no longer have a good value in vReg.
regCache_.Unlock(vReg, RegCache::GEN_ARG_V);
regCache_.ForceRelease(RegCache::GEN_ARG_V);
switch (bitsPerTexel) {
case 32:
MOV(bitsPerTexel, R(resultReg), MRegSum(temp1Reg, resultReg));
break;
case 16:
AND(32, R(uReg), Imm8(1));
LEA(32, resultReg, MComplex(resultReg, uReg, SCALE_2, 0));
MOVZX(32, bitsPerTexel, resultReg, MRegSum(temp1Reg, resultReg));
break;
case 8:
AND(32, R(uReg), Imm8(3));
ADD(32, R(resultReg), R(uReg));
MOVZX(32, bitsPerTexel, resultReg, MRegSum(temp1Reg, resultReg));
break;
default:
success = false;
break;
}
regCache_.Unlock(uReg, RegCache::GEN_ARG_U);
regCache_.ForceRelease(RegCache::GEN_ARG_U);
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
regCache_.Release(temp2Reg, RegCache::GEN_TEMP2);
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
return success;
}
bool SamplerJitCache::Jit_GetTexelCoords(const SamplerID &id) {
Describe("Texel");
X64Reg uReg = regCache_.Alloc(RegCache::GEN_ARG_U);
X64Reg vReg = regCache_.Alloc(RegCache::GEN_ARG_V);
X64Reg sReg = regCache_.Find(RegCache::VEC_ARG_S);
X64Reg tReg = regCache_.Find(RegCache::VEC_ARG_T);
if (id.hasAnyMips) {
// We have to figure out levels and the proper width, ugh.
X64Reg idReg = GetSamplerID();
X64Reg tempReg = regCache_.Alloc(RegCache::GEN_TEMP0);
X64Reg levelReg = INVALID_REG;
if (regCache_.Has(RegCache::GEN_ARG_LEVEL)) {
levelReg = regCache_.Find(RegCache::GEN_ARG_LEVEL);
} else {
levelReg = regCache_.Alloc(RegCache::GEN_ARG_LEVEL);
MOV(32, R(levelReg), MDisp(RSP, stackArgPos_ + stackLevelOffset_));
}
// We'll multiply these at the same time, so it's nice to put together.
UNPCKLPS(sReg, R(tReg));
SHUFPS(sReg, R(sReg), _MM_SHUFFLE(1, 0, 1, 0));
X64Reg sizesReg = regCache_.Alloc(RegCache::VEC_TEMP0);
if (cpu_info.bSSE4_1) {
PMOVZXWD(sizesReg, MComplex(idReg, levelReg, SCALE_4, offsetof(SamplerID, cached.sizes[0].w)));
} else {
MOVQ_xmm(sizesReg, MComplex(idReg, levelReg, SCALE_4, offsetof(SamplerID, cached.sizes[0].w)));
X64Reg zeroReg = GetZeroVec();
PUNPCKLWD(sizesReg, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
}
// We just want this value as a float, times 256.
PSLLD(sizesReg, 8);
CVTDQ2PS(sizesReg, R(sizesReg));
// Okay, we can multiply now, and convert back to integer.
MULPS(sReg, R(sizesReg));
CVTTPS2DQ(sReg, R(sReg));
regCache_.Release(sizesReg, RegCache::VEC_TEMP0);
PSRAD(sReg, 8);
// Reuse tempXYReg for the level1 values.
if (!cpu_info.bSSE4_1)
PSHUFD(tReg, R(sReg), _MM_SHUFFLE(3, 2, 3, 2));
auto applyClampWrap = [&](X64Reg dest, bool clamp, bool isY, bool isLevel1) {
int offset = offsetof(SamplerID, cached.sizes[0].w) + (isY ? 2 : 0) + (isLevel1 ? 4 : 0);
// Grab the size, already pre-shifted for us.
MOVZX(32, 16, tempReg, MComplex(idReg, levelReg, SCALE_4, offset));
// Grab the size from the multiply.
if (cpu_info.bSSE4_1) {
if (isY || isLevel1)
PEXTRD(R(dest), sReg, (isY ? 1 : 0) + (isLevel1 ? 2 : 0));
else
MOVD_xmm(R(dest), sReg);
} else {
X64Reg srcReg = isLevel1 ? tReg : sReg;
MOVD_xmm(R(dest), srcReg);
if (!isY)
PSRLDQ(srcReg, 4);
}
SUB(32, R(tempReg), Imm8(1));
AND(32, R(tempReg), Imm32(0x000001FF));
if (clamp) {
CMP(32, R(dest), R(tempReg));
CMOVcc(32, dest, R(tempReg), CC_G);
XOR(32, R(tempReg), R(tempReg));
CMP(32, R(dest), R(tempReg));
CMOVcc(32, dest, R(tempReg), CC_L);
} else {
AND(32, R(dest), R(tempReg));
}
};
// Do the next level first, so we can save them and reuse the regs.
// Note: for non-SSE4, this must be in S/T order.
applyClampWrap(uReg, id.clampS, false, true);
applyClampWrap(vReg, id.clampT, true, true);
// Okay, now stuff them on the stack - we'll load them again later.
MOV(32, MDisp(RSP, stackArgPos_ + stackUV1Offset_ + 0), R(uReg));
MOV(32, MDisp(RSP, stackArgPos_ + stackUV1Offset_ + 4), R(vReg));
// And then the given level.
// Note: for non-SSE4, this must be in S/T order.
applyClampWrap(uReg, id.clampS, false, false);
applyClampWrap(vReg, id.clampT, true, false);
UnlockSamplerID(idReg);
regCache_.Release(tempReg, RegCache::GEN_TEMP0);
regCache_.Unlock(levelReg, RegCache::GEN_ARG_LEVEL);
} else {
// Multiply, then convert to integer...
UNPCKLPS(sReg, R(tReg));
MULPS(sReg, M(constWidthHeight256f_));
CVTTPS2DQ(sReg, R(sReg));
// Great, shift out the fraction.
PSRAD(sReg, 8);
// Square textures are kinda common.
bool clampApplied = false;
if (id.width0Shift == id.height0Shift) {
if (!id.clampS && !id.clampT) {
PAND(sReg, M(constWidthMinus1i_));
clampApplied = true;
} else if (id.clampS && id.clampT && cpu_info.bSSE4_1) {
X64Reg zeroReg = GetZeroVec();
PMINSD(sReg, M(constWidthMinus1i_));
PMAXSD(sReg, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
clampApplied = true;
}
}
// Now extract to do the clamping (unless we already did it.)
MOVQ_xmm(R(uReg), sReg);
MOV(64, R(vReg), R(uReg));
SHR(64, R(vReg), Imm8(32));
// Strip off the top bits.
AND(32, R(uReg), R(uReg));
auto applyClampWrap = [this](X64Reg dest, bool clamp, uint8_t shift) {
// Clamp and wrap both max out at 512.
if (shift > 9)
shift = 9;
if (clamp) {
X64Reg tempReg = regCache_.Alloc(RegCache::GEN_TEMP0);
MOV(32, R(tempReg), Imm32((1 << shift) - 1));
CMP(32, R(dest), R(tempReg));
CMOVcc(32, dest, R(tempReg), CC_G);
XOR(32, R(tempReg), R(tempReg));
CMP(32, R(dest), R(tempReg));
CMOVcc(32, dest, R(tempReg), CC_L);
regCache_.Release(tempReg, RegCache::GEN_TEMP0);
} else {
AND(32, R(dest), Imm32((1 << shift) - 1));
}
};
// Now apply clamp/wrap.
if (!clampApplied) {
applyClampWrap(uReg, id.clampS, id.width0Shift);
applyClampWrap(vReg, id.clampT, id.height0Shift);
}
}
regCache_.Unlock(uReg, RegCache::GEN_ARG_U);
regCache_.Unlock(vReg, RegCache::GEN_ARG_V);
regCache_.ForceRetain(RegCache::GEN_ARG_U);
regCache_.ForceRetain(RegCache::GEN_ARG_V);
// And get rid of S and T, we're done with them now.
regCache_.Unlock(sReg, RegCache::VEC_ARG_S);
regCache_.Unlock(tReg, RegCache::VEC_ARG_T);
regCache_.ForceRelease(RegCache::VEC_ARG_S);
regCache_.ForceRelease(RegCache::VEC_ARG_T);
return true;
}
bool SamplerJitCache::Jit_GetTexelCoordsQuad(const SamplerID &id) {
Describe("TexelQuad");
X64Reg sReg = regCache_.Find(RegCache::VEC_ARG_S);
X64Reg tReg = regCache_.Find(RegCache::VEC_ARG_T);
// We use this if there are mips later, to apply wrap/clamp.
X64Reg sizesReg = INVALID_REG;
// Start by multiplying with the width/height... which might be complex with mips.
if (id.hasAnyMips) {
// We have to figure out levels and the proper width, ugh.
X64Reg idReg = GetSamplerID();
X64Reg levelReg = INVALID_REG;
// To avoid ABI problems, we don't hold onto level.
bool releaseLevelReg = !regCache_.Has(RegCache::GEN_ARG_LEVEL);
if (!releaseLevelReg) {
levelReg = regCache_.Find(RegCache::GEN_ARG_LEVEL);
} else {
releaseLevelReg = true;
levelReg = regCache_.Alloc(RegCache::GEN_ARG_LEVEL);
MOV(32, R(levelReg), MDisp(RSP, stackArgPos_ + stackLevelOffset_));
}
// This will load the current and next level's sizes, 16x4.
sizesReg = regCache_.Alloc(RegCache::VEC_TEMP5);
// We actually want this in 32-bit, though, so extend.
if (cpu_info.bSSE4_1) {
PMOVZXWD(sizesReg, MComplex(idReg, levelReg, SCALE_4, offsetof(SamplerID, cached.sizes[0].w)));
} else {
MOVQ_xmm(sizesReg, MComplex(idReg, levelReg, SCALE_4, offsetof(SamplerID, cached.sizes[0].w)));
X64Reg zeroReg = GetZeroVec();
PUNPCKLWD(sizesReg, R(zeroReg));
regCache_.Unlock(zeroReg, RegCache::VEC_ZERO);
}
if (releaseLevelReg)
regCache_.Release(levelReg, RegCache::GEN_ARG_LEVEL);
else
regCache_.Unlock(levelReg, RegCache::GEN_ARG_LEVEL);
UnlockSamplerID(idReg);
// Now make a float version of sizesReg, times 256.
X64Reg sizes256Reg = regCache_.Alloc(RegCache::VEC_TEMP0);
PSLLD(sizes256Reg, sizesReg, 8);
CVTDQ2PS(sizes256Reg, R(sizes256Reg));
// Next off, move S and T into a single reg, which will become U0 V0 U1 V1.
UNPCKLPS(sReg, R(tReg));
SHUFPS(sReg, R(sReg), _MM_SHUFFLE(1, 0, 1, 0));
// And multiply by the sizes, all lined up already.
MULPS(sReg, R(sizes256Reg));
regCache_.Release(sizes256Reg, RegCache::VEC_TEMP0);
// For wrap/clamp purposes, we want width or height minus one. Do that now.
PSUBD(sizesReg, M(constOnes32_));
PAND(sizesReg, M(constMaxTexel32_));
} else {
// Easy mode.
UNPCKLPS(sReg, R(tReg));
MULPS(sReg, M(constWidthHeight256f_));
}
// And now, convert to integers for all later processing.
CVTPS2DQ(sReg, R(sReg));
// Now adjust X and Y...
X64Reg tempXYReg = regCache_.Alloc(RegCache::VEC_TEMP0);
// Product a -128 constant.
PCMPEQD(tempXYReg, R(tempXYReg));
PSLLD(tempXYReg, 7);
PADDD(sReg, R(tempXYReg));
regCache_.Release(tempXYReg, RegCache::VEC_TEMP0);
// We do want the fraction, though, so extract that to an XMM for later.
X64Reg allFracReg = INVALID_REG;
if (regCache_.Has(RegCache::VEC_FRAC))
allFracReg = regCache_.Find(RegCache::VEC_FRAC);
else
allFracReg = regCache_.Alloc(RegCache::VEC_FRAC);
// We only want the four bits after the first four, though.
PSLLD(allFracReg, sReg, 24);
PSRLD(allFracReg, 28);
// It's convenient later if this is in the low words only.
PACKSSDW(allFracReg, R(allFracReg));
regCache_.Unlock(allFracReg, RegCache::VEC_FRAC);
regCache_.ForceRetain(RegCache::VEC_FRAC);
// With those extracted, we can now get rid of the fractional bits.
PSRAD(sReg, 8);
// Now it's time to separate the lanes into separate registers and add next UV offsets.
if (id.hasAnyMips) {
X64Reg u1Reg = regCache_.Find(RegCache::VEC_U1);
X64Reg v1Reg = regCache_.Find(RegCache::VEC_V1);
PSHUFD(u1Reg, R(sReg), _MM_SHUFFLE(2, 2, 2, 2));
PSHUFD(v1Reg, R(sReg), _MM_SHUFFLE(3, 3, 3, 3));
PADDD(u1Reg, M(constUNext_));
PADDD(v1Reg, M(constVNext_));
regCache_.Unlock(u1Reg, RegCache::VEC_U1);
regCache_.Unlock(v1Reg, RegCache::VEC_V1);
}
PSHUFD(tReg, R(sReg), _MM_SHUFFLE(1, 1, 1, 1));
PSHUFD(sReg, R(sReg), _MM_SHUFFLE(0, 0, 0, 0));
PADDD(tReg, M(constVNext_));
PADDD(sReg, M(constUNext_));
X64Reg temp0ClampReg = regCache_.Alloc(RegCache::VEC_TEMP0);
bool temp0ClampZero = false;
auto doClamp = [&](bool clamp, X64Reg stReg, const OpArg &bound) {
if (!clamp) {
// Wrapping is easy.
PAND(stReg, bound);
return;
}
if (!temp0ClampZero)
PXOR(temp0ClampReg, R(temp0ClampReg));
temp0ClampZero = true;
if (cpu_info.bSSE4_1) {
PMINSD(stReg, bound);
PMAXSD(stReg, R(temp0ClampReg));
} else {
temp0ClampZero = false;
// Set temp to max(0, stReg) = AND(NOT(0 > stReg), stReg).
PCMPGTD(temp0ClampReg, R(stReg));
PANDN(temp0ClampReg, R(stReg));
// Now make a mask where bound is greater than the ST value in temp0ClampReg.
if (cpu_info.bAVX && bound.IsSimpleReg()) {
VPCMPGTD(128, stReg, bound.GetSimpleReg(), R(temp0ClampReg));
} else {
MOVDQA(stReg, bound);
PCMPGTD(stReg, R(temp0ClampReg));
}
// Throw away the values that are greater in our temp0ClampReg in progress result.
PAND(temp0ClampReg, R(stReg));
// Now, set bound only where ST was too high.
PANDN(stReg, bound);
// And put in the values that were fine.
POR(stReg, R(temp0ClampReg));
}
};
if (id.hasAnyMips) {
// We'll spread sizes out into a temp.
X64Reg spreadSizeReg = regCache_.Alloc(RegCache::VEC_TEMP1);
PSHUFD(spreadSizeReg, R(sizesReg), _MM_SHUFFLE(0, 0, 0, 0));
doClamp(id.clampS, sReg, R(spreadSizeReg));
PSHUFD(spreadSizeReg, R(sizesReg), _MM_SHUFFLE(1, 1, 1, 1));
doClamp(id.clampT, tReg, R(spreadSizeReg));
X64Reg u1Reg = regCache_.Find(RegCache::VEC_U1);
X64Reg v1Reg = regCache_.Find(RegCache::VEC_V1);
PSHUFD(spreadSizeReg, R(sizesReg), _MM_SHUFFLE(2, 2, 2, 2));
doClamp(id.clampS, u1Reg, R(spreadSizeReg));
PSHUFD(spreadSizeReg, R(sizesReg), _MM_SHUFFLE(3, 3, 3, 3));
doClamp(id.clampT, v1Reg, R(spreadSizeReg));
regCache_.Unlock(u1Reg, RegCache::VEC_U1);
regCache_.Unlock(v1Reg, RegCache::VEC_V1);
regCache_.Release(spreadSizeReg, RegCache::VEC_TEMP1);
} else {
doClamp(id.clampS, sReg, M(constWidthMinus1i_));
doClamp(id.clampT, tReg, M(constHeightMinus1i_));
}
if (sizesReg != INVALID_REG)
regCache_.Release(sizesReg, RegCache::VEC_TEMP5);
regCache_.Release(temp0ClampReg, RegCache::VEC_TEMP0);
regCache_.Unlock(sReg, RegCache::VEC_ARG_S);
regCache_.Unlock(tReg, RegCache::VEC_ARG_T);
regCache_.Change(RegCache::VEC_ARG_S, RegCache::VEC_ARG_U);
regCache_.Change(RegCache::VEC_ARG_T, RegCache::VEC_ARG_V);
return true;
}
bool SamplerJitCache::Jit_PrepareDataOffsets(const SamplerID &id, RegCache::Reg uReg, RegCache::Reg vReg, bool level1) {
_assert_(id.linear);
bool success = true;
int bits = 0;
switch (id.TexFmt()) {
case GE_TFMT_5650:
case GE_TFMT_5551:
case GE_TFMT_4444:
case GE_TFMT_CLUT16:
bits = 16;
break;
case GE_TFMT_8888:
case GE_TFMT_CLUT32:
bits = 32;
break;
case GE_TFMT_CLUT8:
bits = 8;
break;
case GE_TFMT_CLUT4:
bits = 4;
break;
case GE_TFMT_DXT1:
bits = -8;
break;
case GE_TFMT_DXT3:
case GE_TFMT_DXT5:
bits = -16;
break;
default:
success = false;
}
if (success && bits != 0) {
if (bits < 0) {
success = Jit_PrepareDataDXTOffsets(id, uReg, vReg, level1, -bits);
} else if (id.swizzle) {
success = Jit_PrepareDataSwizzledOffsets(id, uReg, vReg, level1, bits);
} else {
success = Jit_PrepareDataDirectOffsets(id, uReg, vReg, level1, bits);
}
}
return success;
}
bool SamplerJitCache::Jit_PrepareDataDirectOffsets(const SamplerID &id, RegCache::Reg uReg, RegCache::Reg vReg, bool level1, int bitsPerTexel) {
Describe("DataOff");
X64Reg bufwVecReg = regCache_.Alloc(RegCache::VEC_TEMP0);
if (!id.useStandardBufw || id.hasAnyMips) {
// Spread bufw into each lane.
X64Reg bufwReg = regCache_.Find(RegCache::GEN_ARG_BUFW_PTR);
if (cpu_info.bSSE4_1) {
PMOVZXWD(bufwVecReg, MDisp(bufwReg, level1 ? 2 : 0));
} else {
PXOR(bufwVecReg, R(bufwVecReg));
PINSRW(bufwVecReg, MDisp(bufwReg, level1 ? 2 : 0), 0);
}
PSHUFD(bufwVecReg, R(bufwVecReg), _MM_SHUFFLE(0, 0, 0, 0));
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW_PTR);
if (bitsPerTexel == 4)
PSRLD(bufwVecReg, 1);
else if (bitsPerTexel == 16)
PSLLD(bufwVecReg, 1);
else if (bitsPerTexel == 32)
PSLLD(bufwVecReg, 2);
}
if (id.useStandardBufw && !id.hasAnyMips) {
int amt = id.width0Shift;
if (bitsPerTexel == 4)
amt -= 1;
else if (bitsPerTexel == 16)
amt += 1;
else if (bitsPerTexel == 32)
amt += 2;
// It's aligned to 16 bytes, so must at least be 16.
PSLLD(vReg, std::max(4, amt));
} else if (cpu_info.bSSE4_1) {
// And now multiply. This is slow, but not worse than the SSE2 version...
PMULLD(vReg, R(bufwVecReg));
} else {
// Copy that into another temp for multiply.
X64Reg vOddLaneReg = regCache_.Alloc(RegCache::VEC_TEMP1);
MOVDQA(vOddLaneReg, R(vReg));
// Okay, first, multiply to get XXXX CCCC XXXX AAAA.
PMULUDQ(vReg, R(bufwVecReg));
PSRLDQ(vOddLaneReg, 4);
PSRLDQ(bufwVecReg, 4);
// And now get XXXX DDDD XXXX BBBB.
PMULUDQ(vOddLaneReg, R(bufwVecReg));
// We know everything is positive, so XXXX must be zero. Let's combine.
PSLLDQ(vOddLaneReg, 4);
POR(vReg, R(vOddLaneReg));
regCache_.Release(vOddLaneReg, RegCache::VEC_TEMP1);
}
regCache_.Release(bufwVecReg, RegCache::VEC_TEMP0);
if (bitsPerTexel == 4) {
// Need to keep uvec for the odd bit.
X64Reg uCopyReg = regCache_.Alloc(RegCache::VEC_TEMP0);
MOVDQA(uCopyReg, R(uReg));
PSRLD(uCopyReg, 1);
PADDD(vReg, R(uCopyReg));
regCache_.Release(uCopyReg, RegCache::VEC_TEMP0);
} else {
// Destroy uvec, we won't use it again.
if (bitsPerTexel == 16)
PSLLD(uReg, 1);
else if (bitsPerTexel == 32)
PSLLD(uReg, 2);
PADDD(vReg, R(uReg));
}
return true;
}
bool SamplerJitCache::Jit_PrepareDataSwizzledOffsets(const SamplerID &id, RegCache::Reg uReg, RegCache::Reg vReg, bool level1, int bitsPerTexel) {
Describe("DataOffS");
// See Jit_GetTexDataSwizzled() for usage of this offset.
X64Reg bufwVecReg = regCache_.Alloc(RegCache::VEC_TEMP0);
if (!id.useStandardBufw || id.hasAnyMips) {
// Spread bufw into each lane.
X64Reg bufwReg = regCache_.Find(RegCache::GEN_ARG_BUFW_PTR);
if (cpu_info.bSSE4_1) {
PMOVZXWD(bufwVecReg, MDisp(bufwReg, level1 ? 2 : 0));
} else {
PXOR(bufwVecReg, R(bufwVecReg));
PINSRW(bufwVecReg, MDisp(bufwReg, level1 ? 2 : 0), 0);
}
PSHUFD(bufwVecReg, R(bufwVecReg), _MM_SHUFFLE(0, 0, 0, 0));
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW_PTR);
}
// Divide vvec by 8 in a temp.
X64Reg vMultReg = regCache_.Alloc(RegCache::VEC_TEMP1);
PSRLD(vMultReg, vReg, 3);
// And now multiply by bufw. May be able to use a shift in a common case.
int shiftAmount = 32 - clz32_nonzero(bitsPerTexel - 1);
if (id.useStandardBufw && !id.hasAnyMips) {
int amt = id.width0Shift;
// Account for 16 byte minimum.
amt = std::max(7 - shiftAmount, amt);
shiftAmount += amt;
} else if (cpu_info.bSSE4_1) {
// And now multiply. This is slow, but not worse than the SSE2 version...
PMULLD(vMultReg, R(bufwVecReg));
} else {
// Copy that into another temp for multiply.
X64Reg vOddLaneReg = regCache_.Alloc(RegCache::VEC_TEMP2);
MOVDQA(vOddLaneReg, R(vMultReg));
// Okay, first, multiply to get XXXX CCCC XXXX AAAA.
PMULUDQ(vMultReg, R(bufwVecReg));
PSRLDQ(vOddLaneReg, 4);
PSRLDQ(bufwVecReg, 4);
// And now get XXXX DDDD XXXX BBBB.
PMULUDQ(vOddLaneReg, R(bufwVecReg));
// We know everything is positive, so XXXX must be zero. Let's combine.
PSLLDQ(vOddLaneReg, 4);
POR(vMultReg, R(vOddLaneReg));
regCache_.Release(vOddLaneReg, RegCache::VEC_TEMP2);
}
regCache_.Release(bufwVecReg, RegCache::VEC_TEMP0);
// Multiply the result by bitsPerTexel using a shift.
PSLLD(vMultReg, shiftAmount);
// Now we're adding (v & 7) * 16. Use a 16-bit wall.
PSLLW(vReg, 13);
PSRLD(vReg, 9);
PADDD(vReg, R(vMultReg));
regCache_.Release(vMultReg, RegCache::VEC_TEMP1);
// Now get ((uvec / texels_per_tile) / 4) * 32 * 4 aka (uvec / (128 / bitsPerTexel)) << 7.
X64Reg uCopyReg = regCache_.Alloc(RegCache::VEC_TEMP0);
PSRLD(uCopyReg, uReg, 7 + clz32_nonzero(bitsPerTexel - 1) - 32);
PSLLD(uCopyReg, 7);
// Add it in to our running total.
PADDD(vReg, R(uCopyReg));
if (bitsPerTexel == 4) {
// Finally, we want (uvec & 31) / 2. Use a 16-bit wall.
PSLLW(uCopyReg, uReg, 11);
PSRLD(uCopyReg, 12);
// With that, this is our byte offset. uvec & 1 has which half.
PADDD(vReg, R(uCopyReg));
} else {
// We can destroy uvec in this path. Clear all but 2 bits for 32, 3 for 16, or 4 for 8.
PSLLW(uReg, 32 - clz32_nonzero(bitsPerTexel - 1) + 9);
// Now that it's at the top of the 16 bits, we always shift that to the top of 4 bits.
PSRLD(uReg, 12);
PADDD(vReg, R(uReg));
}
regCache_.Release(uCopyReg, RegCache::VEC_TEMP0);
return true;
}
bool SamplerJitCache::Jit_PrepareDataDXTOffsets(const SamplerID &id, Rasterizer::RegCache::Reg uReg, Rasterizer::RegCache::Reg vReg, bool level1, int blockSize) {
Describe("DataOffDXT");
// Wwe need to get the block's offset, which is:
// blockPos = src + (v/4 * bufw/4 + u/4) * blockSize
// We distribute the blockSize constant for convenience:
// blockPos = src + (blockSize*v/4 * bufw/4 + blockSize*u/4)
X64Reg baseVReg = regCache_.Find(level1 ? RegCache::VEC_INDEX1 : RegCache::VEC_INDEX);
// This gives us the V factor for the block, which we multiply by bufw.
PSRLD(baseVReg, vReg, 2);
PSLLD(baseVReg, blockSize == 16 ? 4 : 3);
X64Reg bufwVecReg = regCache_.Alloc(RegCache::VEC_TEMP0);
if (!id.useStandardBufw || id.hasAnyMips) {
// Spread bufw into each lane.
X64Reg bufwReg = regCache_.Find(RegCache::GEN_ARG_BUFW_PTR);
if (cpu_info.bSSE4_1) {
PMOVZXWD(bufwVecReg, MDisp(bufwReg, level1 ? 2 : 0));
} else {
PXOR(bufwVecReg, R(bufwVecReg));
PINSRW(bufwVecReg, MDisp(bufwReg, level1 ? 2 : 0), 0);
}
PSHUFD(bufwVecReg, R(bufwVecReg), _MM_SHUFFLE(0, 0, 0, 0));
regCache_.Unlock(bufwReg, RegCache::GEN_ARG_BUFW_PTR);
// Divide by 4 before the multiply.
PSRLD(bufwVecReg, 2);
}
if (id.useStandardBufw && !id.hasAnyMips) {
int amt = id.width0Shift - 2;
if (amt < 0)
PSRLD(baseVReg, -amt);
else if (amt > 0)
PSLLD(baseVReg, amt);
} else if (cpu_info.bSSE4_1) {
// And now multiply. This is slow, but not worse than the SSE2 version...
PMULLD(baseVReg, R(bufwVecReg));
} else {
// Copy that into another temp for multiply.
X64Reg vOddLaneReg = regCache_.Alloc(RegCache::VEC_TEMP1);
MOVDQA(vOddLaneReg, R(baseVReg));
// Okay, first, multiply to get XXXX CCCC XXXX AAAA.
PMULUDQ(baseVReg, R(bufwVecReg));
PSRLDQ(vOddLaneReg, 4);
PSRLDQ(bufwVecReg, 4);
// And now get XXXX DDDD XXXX BBBB.
PMULUDQ(vOddLaneReg, R(bufwVecReg));
// We know everything is positive, so XXXX must be zero. Let's combine.
PSLLDQ(vOddLaneReg, 4);
POR(baseVReg, R(vOddLaneReg));
regCache_.Release(vOddLaneReg, RegCache::VEC_TEMP1);
}
regCache_.Release(bufwVecReg, RegCache::VEC_TEMP0);
// Now add in the U factor for the block.
X64Reg baseUReg = regCache_.Alloc(RegCache::VEC_TEMP0);
PSRLD(baseUReg, uReg, 2);
PSLLD(baseUReg, blockSize == 16 ? 4 : 3);
PADDD(baseVReg, R(baseUReg));
regCache_.Release(baseUReg, RegCache::VEC_TEMP0);
// Okay, the base index (block byte offset from src) is ready.
regCache_.Unlock(baseVReg, level1 ? RegCache::VEC_INDEX1 : RegCache::VEC_INDEX);
regCache_.ForceRetain(level1 ? RegCache::VEC_INDEX1 : RegCache::VEC_INDEX);
// For everything else, we only want the low two bits of U and V.
PSLLD(uReg, 30);
PSLLD(vReg, 30);
X64Reg alphaTempRegU = regCache_.Alloc(RegCache::VEC_TEMP0);
if (id.TexFmt() == GE_TFMT_DXT3 || id.TexFmt() == GE_TFMT_DXT5)
PSRLD(alphaTempRegU, uReg, 30);
PSRLD(uReg, 30 - 1);
PSRLD(vReg, 30 - 3);
// At this point, uReg is now the bit offset of the color index.
PADDD(uReg, R(vReg));
// Grab the alpha index into vReg next.
if (id.TexFmt() == GE_TFMT_DXT3 || id.TexFmt() == GE_TFMT_DXT5) {
PSRLD(vReg, 1);
PADDD(vReg, R(alphaTempRegU));
if (id.TexFmt() == GE_TFMT_DXT3) {
PSLLD(vReg, 2);
} else if (id.TexFmt() == GE_TFMT_DXT5) {
// Multiply by 3.
PSLLD(alphaTempRegU, vReg, 1);
PADDD(vReg, R(alphaTempRegU));
}
}
regCache_.Release(alphaTempRegU, RegCache::VEC_TEMP0);
return true;
}
bool SamplerJitCache::Jit_DecodeQuad(const SamplerID &id, bool level1) {
GETextureFormat decodeFmt = id.TexFmt();
switch (id.TexFmt()) {
case GE_TFMT_CLUT32:
case GE_TFMT_CLUT16:
case GE_TFMT_CLUT8:
case GE_TFMT_CLUT4:
// The values match, so just use the clut fmt.
decodeFmt = (GETextureFormat)id.ClutFmt();
break;
default:
// We'll decode below.
break;
}
bool success = true;
X64Reg quadReg = regCache_.Find(level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
switch (decodeFmt) {
case GE_TFMT_5650:
success = Jit_Decode5650Quad(id, quadReg);
break;
case GE_TFMT_5551:
success = Jit_Decode5551Quad(id, quadReg);
break;
case GE_TFMT_4444:
success = Jit_Decode4444Quad(id, quadReg);
break;
default:
// Doesn't need decoding.
break;
}
regCache_.Unlock(quadReg, level1 ? RegCache::VEC_RESULT1 : RegCache::VEC_RESULT);
return success;
}
bool SamplerJitCache::Jit_Decode5650Quad(const SamplerID &id, Rasterizer::RegCache::Reg quadReg) {
Describe("5650Quad");
X64Reg temp1Reg = regCache_.Alloc(RegCache::VEC_TEMP1);
X64Reg temp2Reg = regCache_.Alloc(RegCache::VEC_TEMP2);
// Filter out red only into temp1. We do this by shifting into a wall.
PSLLD(temp1Reg, quadReg, 32 - 5);
// Move it right to the top of the 8 bits.
PSRLD(temp1Reg, 24);
// Now we bring in blue, since it's also 5 like red.
// Luckily, we know the top 16 bits are zero. Shift right into a wall.
PSRLD(temp2Reg, quadReg, 11);
// Shift blue into place at 19, and merge back to temp1.
PSLLD(temp2Reg, 19);
POR(temp1Reg, R(temp2Reg));
// Make a copy back in temp2, and shift left 1 so we can swizzle together with G.
PSLLD(temp2Reg, temp1Reg, 1);
// We go to green last because it's the different one. Shift off red and blue.
PSRLD(quadReg, 5);
// Use a word shift to put a wall just at the right place, top 6 bits of second byte.
PSLLW(quadReg, 10);
// Combine with temp2 (for swizzling), then merge in temp1 (R+B pre-swizzle.)
POR(temp2Reg, R(quadReg));
POR(quadReg, R(temp1Reg));
// Now shift and mask temp2 for swizzle.
PSRLD(temp2Reg, 6);
PAND(temp2Reg, M(const5650Swizzle_));
// And then OR that in too. Only alpha left now.
POR(quadReg, R(temp2Reg));
if (id.useTextureAlpha) {
// Just put a fixed FF in. Maybe we could even avoid this and act like it's FF later...
PCMPEQD(temp2Reg, R(temp2Reg));
PSLLD(temp2Reg, 24);
POR(quadReg, R(temp2Reg));
}
regCache_.Release(temp1Reg, RegCache::VEC_TEMP1);
regCache_.Release(temp2Reg, RegCache::VEC_TEMP2);
return true;
}
bool SamplerJitCache::Jit_Decode5650(const SamplerID &id) {
Describe("5650");
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
X64Reg temp2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
if (cpu_info.bBMI2_fast) {
// Start off with the high bits.
MOV(32, R(temp1Reg), Imm32(0x00F8FCF8));
PDEP(32, temp1Reg, resultReg, R(temp1Reg));
if (id.useTextureAlpha || id.fetch)
OR(32, R(temp1Reg), Imm32(0xFF000000));
// Now grab the low bits (they end up packed.)
MOV(32, R(temp2Reg), Imm32(0x0000E61C));
PEXT(32, resultReg, resultReg, R(temp2Reg));
// And spread them back out.
MOV(32, R(temp2Reg), Imm32(0x00070307));
PDEP(32, resultReg, resultReg, R(temp2Reg));
// Finally put the high bits in, we're done.
OR(32, R(resultReg), R(temp1Reg));
} else {
MOV(32, R(temp2Reg), R(resultReg));
AND(32, R(temp2Reg), Imm32(0x0000001F));
// B (we do R and B at the same time, they're both 5.)
MOV(32, R(temp1Reg), R(resultReg));
AND(32, R(temp1Reg), Imm32(0x0000F800));
SHL(32, R(temp1Reg), Imm8(5));
OR(32, R(temp2Reg), R(temp1Reg));
// Expand 5 -> 8. At this point we have 00BB00RR.
MOV(32, R(temp1Reg), R(temp2Reg));
SHL(32, R(temp2Reg), Imm8(3));
SHR(32, R(temp1Reg), Imm8(2));
OR(32, R(temp2Reg), R(temp1Reg));
AND(32, R(temp2Reg), Imm32(0x00FF00FF));
// Now's as good a time to put in A as any.
if (id.useTextureAlpha || id.fetch)
OR(32, R(temp2Reg), Imm32(0xFF000000));
// Last, we need to align, extract, and expand G.
// 3 to align to G, and then 2 to expand to 8.
SHL(32, R(resultReg), Imm8(3 + 2));
AND(32, R(resultReg), Imm32(0x0000FC00));
MOV(32, R(temp1Reg), R(resultReg));
// 2 to account for resultReg being preshifted, 4 for expansion.
SHR(32, R(temp1Reg), Imm8(2 + 4));
OR(32, R(resultReg), R(temp1Reg));
AND(32, R(resultReg), Imm32(0x0000FF00));
OR(32, R(resultReg), R(temp2Reg));
}
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
regCache_.Release(temp2Reg, RegCache::GEN_TEMP2);
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
return true;
}
bool SamplerJitCache::Jit_Decode5551Quad(const SamplerID &id, Rasterizer::RegCache::Reg quadReg) {
Describe("5551Quad");
X64Reg temp1Reg = regCache_.Alloc(RegCache::VEC_TEMP1);
X64Reg temp2Reg = regCache_.Alloc(RegCache::VEC_TEMP2);
// Filter out red only into temp1. We do this by shifting into a wall.
PSLLD(temp1Reg, quadReg, 32 - 5);
// Move it right to the top of the 8 bits.
PSRLD(temp1Reg, 24);
// Add in green and shift into place (top 5 bits of byte 2.)
PSRLD(temp2Reg, quadReg, 5);
PSLLW(temp2Reg, 11);
POR(temp1Reg, R(temp2Reg));
// First, extend alpha using an arithmetic shift.
// We use 10 to meanwhile get rid of green too. The extra alpha bits are fine.
PSRAW(quadReg, 10);
// This gets rid of those extra alpha bits and puts blue in place too.
PSLLD(quadReg, 19);
// Combine both together, we still need to swizzle.
POR(quadReg, R(temp1Reg));
PSRLD(temp1Reg, quadReg, 5);
// Now for swizzle, we'll mask carefully to avoid overflow.
PAND(temp1Reg, M(const5551Swizzle_));
// Then finally merge in the swizzle bits.
POR(quadReg, R(temp1Reg));
regCache_.Release(temp1Reg, RegCache::VEC_TEMP1);
regCache_.Release(temp2Reg, RegCache::VEC_TEMP2);
return true;
}
bool SamplerJitCache::Jit_Decode5551(const SamplerID &id) {
Describe("5551");
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
X64Reg temp2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
if (cpu_info.bBMI2_fast) {
// First, grab the top bits.
bool keepAlpha = id.useTextureAlpha || id.fetch;
MOV(32, R(temp1Reg), Imm32(keepAlpha ? 0x01F8F8F8 : 0x00F8F8F8));
PDEP(32, resultReg, resultReg, R(temp1Reg));
// Now make the swizzle bits.
MOV(32, R(temp2Reg), R(resultReg));
SHR(32, R(temp2Reg), Imm8(5));
AND(32, R(temp2Reg), Imm32(0x00070707));
if (keepAlpha) {
// Sign extend the alpha bit to 8 bits.
SHL(32, R(resultReg), Imm8(7));
SAR(32, R(resultReg), Imm8(7));
}
OR(32, R(resultReg), R(temp2Reg));
} else {
MOV(32, R(temp2Reg), R(resultReg));
MOV(32, R(temp1Reg), R(resultReg));
AND(32, R(temp2Reg), Imm32(0x0000001F));
AND(32, R(temp1Reg), Imm32(0x000003E0));
SHL(32, R(temp1Reg), Imm8(3));
OR(32, R(temp2Reg), R(temp1Reg));
MOV(32, R(temp1Reg), R(resultReg));
AND(32, R(temp1Reg), Imm32(0x00007C00));
SHL(32, R(temp1Reg), Imm8(6));
OR(32, R(temp2Reg), R(temp1Reg));
// Expand 5 -> 8. After this is just A.
MOV(32, R(temp1Reg), R(temp2Reg));
SHL(32, R(temp2Reg), Imm8(3));
SHR(32, R(temp1Reg), Imm8(2));
// Chop off the bits that were shifted out.
AND(32, R(temp1Reg), Imm32(0x00070707));
OR(32, R(temp2Reg), R(temp1Reg));
if (id.useTextureAlpha || id.fetch) {
// For A, we sign extend to get either 16 1s or 0s of alpha.
SAR(16, R(resultReg), Imm8(15));
// Now, shift left by 24 to get the lowest 8 of those at the top.
SHL(32, R(resultReg), Imm8(24));
OR(32, R(resultReg), R(temp2Reg));
} else {
MOV(32, R(resultReg), R(temp2Reg));
}
}
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
regCache_.Release(temp2Reg, RegCache::GEN_TEMP2);
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
return true;
}
bool SamplerJitCache::Jit_Decode4444Quad(const SamplerID &id, Rasterizer::RegCache::Reg quadReg) {
Describe("4444Quad");
X64Reg temp1Reg = regCache_.Alloc(RegCache::VEC_TEMP1);
X64Reg temp2Reg = regCache_.Alloc(RegCache::VEC_TEMP2);
// Mask and move red into position within temp1.
PSLLD(temp1Reg, quadReg, 28);
PSRLD(temp1Reg, 24);
// Green is easy too, we use a word shift to get a free wall.
PSRLD(temp2Reg, quadReg, 4);
PSLLW(temp2Reg, 12);
POR(temp1Reg, R(temp2Reg));
// Blue isn't last this time, but it's next.
PSRLD(temp2Reg, quadReg, 8);
PSLLD(temp2Reg, 28);
PSRLD(temp2Reg, 8);
POR(temp1Reg, R(temp2Reg));
if (id.useTextureAlpha) {
// Last but not least, alpha.
PSRLW(quadReg, 12);
PSLLD(quadReg, 28);
POR(quadReg, R(temp1Reg));
// Masking isn't necessary here since everything is 4 wide.
PSRLD(temp1Reg, quadReg, 4);
POR(quadReg, R(temp1Reg));
} else {
// Overwrite quadReg (we need temp1 as a copy anyway.)
PSRLD(quadReg, temp1Reg, 4);
POR(quadReg, R(temp1Reg));
}
regCache_.Release(temp1Reg, RegCache::VEC_TEMP1);
regCache_.Release(temp2Reg, RegCache::VEC_TEMP2);
return true;
}
alignas(16) static const u32 color4444mask[4] = { 0xf00ff00f, 0xf00ff00f, 0xf00ff00f, 0xf00ff00f, };
bool SamplerJitCache::Jit_Decode4444(const SamplerID &id) {
Describe("4444");
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
if (cpu_info.bBMI2_fast) {
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
// First, spread the bits out with spaces.
MOV(32, R(temp1Reg), Imm32(0xF0F0F0F0));
PDEP(32, resultReg, resultReg, R(temp1Reg));
// Now swizzle the low bits in.
MOV(32, R(temp1Reg), R(resultReg));
SHR(32, R(temp1Reg), Imm8(4));
OR(32, R(resultReg), R(temp1Reg));
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
} else {
X64Reg vecTemp1Reg = regCache_.Alloc(RegCache::VEC_TEMP1);
X64Reg vecTemp2Reg = regCache_.Alloc(RegCache::VEC_TEMP2);
X64Reg vecTemp3Reg = regCache_.Alloc(RegCache::VEC_TEMP3);
MOVD_xmm(vecTemp1Reg, R(resultReg));
PUNPCKLBW(vecTemp1Reg, R(vecTemp1Reg));
if (RipAccessible(color4444mask)) {
PAND(vecTemp1Reg, M(color4444mask));
} else {
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
MOV(PTRBITS, R(temp1Reg), ImmPtr(color4444mask));
PAND(vecTemp1Reg, MatR(temp1Reg));
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
}
MOVSS(vecTemp2Reg, R(vecTemp1Reg));
MOVSS(vecTemp3Reg, R(vecTemp1Reg));
PSRLW(vecTemp2Reg, 4);
PSLLW(vecTemp3Reg, 4);
POR(vecTemp1Reg, R(vecTemp2Reg));
POR(vecTemp1Reg, R(vecTemp3Reg));
MOVD_xmm(R(resultReg), vecTemp1Reg);
regCache_.Release(vecTemp1Reg, RegCache::VEC_TEMP1);
regCache_.Release(vecTemp2Reg, RegCache::VEC_TEMP2);
regCache_.Release(vecTemp3Reg, RegCache::VEC_TEMP3);
}
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
return true;
}
bool SamplerJitCache::Jit_TransformClutIndex(const SamplerID &id, int bitsPerIndex) {
Describe("TrCLUT");
GEPaletteFormat fmt = id.ClutFmt();
if (!id.hasClutShift && !id.hasClutMask && !id.hasClutOffset) {
// This is simple - just mask if necessary.
if (bitsPerIndex > 8) {
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
AND(32, R(resultReg), Imm32(0x000000FF));
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
}
return true;
}
if (!cpu_info.bBMI2) {
bool hasRCX = regCache_.ChangeReg(RCX, RegCache::GEN_SHIFTVAL);
_assert_msg_(hasRCX, "Could not obtain RCX, locked?");
}
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
X64Reg idReg = GetSamplerID();
MOV(32, R(temp1Reg), MDisp(idReg, offsetof(SamplerID, cached.clutFormat)));
UnlockSamplerID(idReg);
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
int shiftedToSoFar = 0;
// Shift = (clutformat >> 2) & 0x1F
if (id.hasClutShift) {
SHR(32, R(temp1Reg), Imm8(2 - shiftedToSoFar));
shiftedToSoFar = 2;
if (cpu_info.bBMI2) {
SHRX(32, resultReg, R(resultReg), temp1Reg);
} else {
_assert_(regCache_.Has(RegCache::GEN_SHIFTVAL));
MOV(32, R(RCX), R(temp1Reg));
SHR(32, R(resultReg), R(RCX));
}
}
// Mask = (clutformat >> 8) & 0xFF
if (id.hasClutMask) {
SHR(32, R(temp1Reg), Imm8(8 - shiftedToSoFar));
shiftedToSoFar = 8;
AND(32, R(resultReg), R(temp1Reg));
}
// We need to wrap any entries beyond the first 1024 bytes.
u32 offsetMask = fmt == GE_CMODE_32BIT_ABGR8888 ? 0x00FF : 0x01FF;
// We must mask to 0xFF before ORing 0x100 in 16 bit CMODEs.
// But skip if we'll mask 0xFF after offset anyway.
if (bitsPerIndex > 8 && (!id.hasClutOffset || offsetMask != 0x00FF)) {
AND(32, R(resultReg), Imm32(0x000000FF));
}
// Offset = (clutformat >> 12) & 0x01F0
if (id.hasClutOffset) {
SHR(32, R(temp1Reg), Imm8(16 - shiftedToSoFar));
SHL(32, R(temp1Reg), Imm8(4));
OR(32, R(resultReg), R(temp1Reg));
AND(32, R(resultReg), Imm32(offsetMask));
}
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
return true;
}
bool SamplerJitCache::Jit_ReadClutColor(const SamplerID &id) {
Describe("ReadCLUT");
X64Reg resultReg = regCache_.Find(RegCache::GEN_RESULT);
_assert_msg_(!id.linear, "Should not use this path for linear");
if (!id.useSharedClut) {
X64Reg temp2Reg = regCache_.Alloc(RegCache::GEN_TEMP2);
if (regCache_.Has(RegCache::GEN_ARG_LEVEL)) {
X64Reg levelReg = regCache_.Find(RegCache::GEN_ARG_LEVEL);
// We need to multiply by 16 and add, LEA allows us to copy too.
LEA(32, temp2Reg, MScaled(levelReg, SCALE_4, 0));
regCache_.Unlock(levelReg, RegCache::GEN_ARG_LEVEL);
if (id.fetch)
regCache_.ForceRelease(RegCache::GEN_ARG_LEVEL);
} else {
_assert_(stackLevelOffset_ != -1);
// The argument was saved on the stack.
MOV(32, R(temp2Reg), MDisp(RSP, stackArgPos_ + stackLevelOffset_));
LEA(32, temp2Reg, MScaled(temp2Reg, SCALE_4, 0));
}
// Second step of the multiply by 16 (since we only multiplied by 4 before.)
LEA(64, resultReg, MComplex(resultReg, temp2Reg, SCALE_4, 0));
regCache_.Release(temp2Reg, RegCache::GEN_TEMP2);
}
X64Reg idReg = GetSamplerID();
X64Reg temp1Reg = regCache_.Alloc(RegCache::GEN_TEMP1);
MOV(PTRBITS, R(temp1Reg), MDisp(idReg, offsetof(SamplerID, cached.clut)));
UnlockSamplerID(idReg);
switch (id.ClutFmt()) {
case GE_CMODE_16BIT_BGR5650:
case GE_CMODE_16BIT_ABGR5551:
case GE_CMODE_16BIT_ABGR4444:
MOVZX(32, 16, resultReg, MComplex(temp1Reg, resultReg, SCALE_2, 0));
break;
case GE_CMODE_32BIT_ABGR8888:
MOV(32, R(resultReg), MComplex(temp1Reg, resultReg, SCALE_4, 0));
break;
}
regCache_.Release(temp1Reg, RegCache::GEN_TEMP1);
regCache_.Unlock(resultReg, RegCache::GEN_RESULT);
switch (id.ClutFmt()) {
case GE_CMODE_16BIT_BGR5650:
return Jit_Decode5650(id);
case GE_CMODE_16BIT_ABGR5551:
return Jit_Decode5551(id);
case GE_CMODE_16BIT_ABGR4444:
return Jit_Decode4444(id);
case GE_CMODE_32BIT_ABGR8888:
return true;
default:
return false;
}
}
};
#endif
Reference in New Issue View Git Blame Copy Permalink