/*
	This file is part of Flycast.

    Flycast is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 2 of the License, or
    (at your option) any later version.

    Flycast is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with Flycast.  If not, see <https://www.gnu.org/licenses/>.
*/
#include "TexCache.h"
#include "xbrz/xbrz.h"
#include "hw/pvr/pvr_mem.h"
#include "hw/mem/addrspace.h"

#include <mutex>
#include <new>
#include <type_traits>
#include <xxhash.h>

#if HOST_CPU == CPU_ARM64
#include <arm_neon.h>
#endif

#ifdef _OPENMP
#include <omp.h>
#endif

extern bool pal_needs_update;

// Rough approximation of LoD bias from D adjust param, only used to increase LoD
const std::array<f32, 16> D_Adjust_LoD_Bias = {
		0.f, -4.f, -2.f, -1.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f
};

static std::vector<vram_block*> *VramLocks;

static inline void initVramLocks() {
	if (VramLocks == nullptr)
		VramLocks = new std::vector<vram_block*>[VRAM_SIZE_MAX / PAGE_SIZE];
}

//List functions
//
static void vramlock_list_remove(vram_block* block)
{
	u32 base = block->start / PAGE_SIZE;
	u32 end = block->end / PAGE_SIZE;

	for (u32 i = base; i <= end; i++)
	{
		std::vector<vram_block*>& list = VramLocks[i];
		for (auto& lock : list)
		{
			if (lock == block)
				lock = nullptr;
		}
	}
}
 
static void vramlock_list_add(vram_block* block)
{
	u32 base = block->start / PAGE_SIZE;
	u32 end = block->end / PAGE_SIZE;

	for (u32 i = base; i <= end; i++)
	{
		std::vector<vram_block*>& list = VramLocks[i];
		// If the list is empty then we need to protect vram, otherwise it's already been done
		if (list.empty() || std::all_of(list.begin(), list.end(), [](vram_block *block) { return block == nullptr; }))
			addrspace::protectVram(i * PAGE_SIZE, PAGE_SIZE);
		auto it = std::find(list.begin(), list.end(), nullptr);
		if (it != list.end())
			*it = block;
		else
			list.push_back(block);
	}
}
 
static std::mutex vramlist_lock;

bool VramLockedWriteOffset(size_t offset)
{
	if (offset >= VRAM_SIZE || VramLocks == nullptr)
		return false;

	size_t addr_hash = offset / PAGE_SIZE;
	std::vector<vram_block *>& list = VramLocks[addr_hash];

	{
		std::lock_guard<std::mutex> lockguard(vramlist_lock);

		for (auto& lock : list)
		{
			if (lock != nullptr)
			{
				lock->texture->invalidate();

				if (lock != nullptr)
				{
					ERROR_LOG(PVR, "Error : pvr is supposed to remove lock");
					die("Invalid state");
				}
			}
		}
		list.clear();

		addrspace::unprotectVram((u32)(offset & ~PAGE_MASK), PAGE_SIZE);
	}

	return true;
}

bool VramLockedWrite(u8* address)
{
	u32 offset = addrspace::getVramOffset(address);
	if (offset == (u32)-1)
		return false;
	return VramLockedWriteOffset(offset);
}

//unlocks mem
//also frees the handle
static void libCore_vramlock_Unlock_block_wb(vram_block* block)
{
	vramlock_list_remove(block);
	delete block;
}

#ifdef _OPENMP
static inline int getThreadCount()
{
	int tcount = omp_get_num_procs() - 1;
	if (tcount < 1)
		tcount = 1;
	return std::min(tcount, (int)config::MaxThreads);
}

template<typename Func>
void parallelize(Func func, int start, int end)
{
	int tcount = getThreadCount();
#pragma omp parallel num_threads(tcount)
	{
		int num_threads = omp_get_num_threads();
		int thread = omp_get_thread_num();
		int chunk = (end - start) / num_threads;
		func(start + chunk * thread,
				num_threads == thread + 1 ? end
						: (start + chunk * (thread + 1)));
	}
}
#endif

static struct xbrz::ScalerCfg xbrz_cfg;

void UpscalexBRZ(int factor, u32* source, u32* dest, int width, int height, bool has_alpha)
{
#ifdef _OPENMP
	parallelize([=](int start, int end) {
		xbrz::scale(factor, source, dest, width, height, has_alpha ? xbrz::ColorFormat::ARGB : xbrz::ColorFormat::RGB,
				xbrz_cfg, start, end);
	}, 0, height);
#else
	xbrz::scale(factor, source, dest, width, height, has_alpha ? xbrz::ColorFormat::ARGB : xbrz::ColorFormat::RGB, xbrz_cfg);
#endif
}

extern const u32 VQMipPoint[11] =
{
	VQ_CODEBOOK_SIZE + 0x00000, // 1
	VQ_CODEBOOK_SIZE + 0x00001, // 2
	VQ_CODEBOOK_SIZE + 0x00002, // 4
	VQ_CODEBOOK_SIZE + 0x00006, // 8
	VQ_CODEBOOK_SIZE + 0x00016, // 16
	VQ_CODEBOOK_SIZE + 0x00056, // 32
	VQ_CODEBOOK_SIZE + 0x00156, // 64
	VQ_CODEBOOK_SIZE + 0x00556, // 128
	VQ_CODEBOOK_SIZE + 0x01556, // 256
	VQ_CODEBOOK_SIZE + 0x05556, // 512
	VQ_CODEBOOK_SIZE + 0x15556  // 1024
};
extern const u32 OtherMipPoint[11] =
{
	0x00003,//1
	0x00004,//2
	0x00008,//4
	0x00018,//8
	0x00058,//16
	0x00158,//32
	0x00558,//64
	0x01558,//128
	0x05558,//256
	0x15558,//512
	0x55558//1024
};

static const TextureType PAL_TYPE[4] = {
	TextureType::_5551, TextureType::_565, TextureType::_4444, TextureType::_8888
};

void BaseTextureCacheData::PrintTextureName()
{
#if !defined(NDEBUG) || defined(DEBUGFAST)
	char str[512];
	snprintf(str, sizeof(str), "Texture: %s", GetPixelFormatName());

	if (tcw.VQ_Comp)
		strcat(str, " VQ");
	else if (tcw.ScanOrder == 0 || IsPaletted())
		strcat(str, " TW");
	else if (tcw.StrideSel == 1 && !IsPaletted())
		strcat(str, " Stride");

	if (tcw.ScanOrder == 0 && tcw.MipMapped)
		strcat(str, " MM");
	if (tsp.FilterMode != 0)
		strcat(str, " Bilinear");

	size_t len = strlen(str);
	snprintf(str + len, sizeof(str) - len, " %dx%d @ 0x%X", 8 << tsp.TexU, 8 << tsp.TexV, tcw.TexAddr << 3);
	std::string id = GetId();
	len = strlen(str);
	snprintf(str + len, sizeof(str) - len, " id=%s", id.c_str());
	DEBUG_LOG(RENDERER, "%s", str);
#endif
}

//true if : dirty or paletted texture and hashes don't match
bool BaseTextureCacheData::NeedsUpdate() {
	bool rc = dirty != 0;
	if (tex_type != TextureType::_8)
	{
		if (tcw.PixelFmt == PixelPal4 && palette_hash != pal_hash_16[tcw.PalSelect])
			rc = true;
		else if (tcw.PixelFmt == PixelPal8 && palette_hash != pal_hash_256[tcw.PalSelect >> 4])
			rc = true;
	}

	return rc;
}

void BaseTextureCacheData::protectVRam()
{
	u32 end = mmStartAddress + size - 1;
	if (end >= VRAM_SIZE)
	{
		WARN_LOG(PVR, "protectVRam: end >= VRAM_SIZE. Tried to lock area out of vram");
		end = VRAM_SIZE - 1;
	}

	if (startAddress > end)
	{
		WARN_LOG(PVR, "vramlock_Lock: startAddress > end. Tried to lock negative block");
		return;
	}

	vram_block *block = new vram_block();
	block->end = end;
	block->start = startAddress;
	block->texture = this;

	{
		std::lock_guard<std::mutex> lock(vramlist_lock);

		if (lock_block == nullptr)
		{
			// This also protects vram if needed
			vramlock_list_add(block);
			lock_block = block;
		}
		else
			delete block;
	}
}

void BaseTextureCacheData::unprotectVRam()
{
	std::lock_guard<std::mutex> lock(vramlist_lock);
	if (lock_block)
		libCore_vramlock_Unlock_block_wb(lock_block);
	lock_block = nullptr;
}

bool BaseTextureCacheData::Delete()
{
	unprotectVRam();
	custom_texture.cancelRequest(customRequestId);
	customRequestId = {};
	customPayload.reset();
	gpuPreloadedTexture.reset();
	usingGpuPreloadedTexture = false;
	customMipLevels = 0;

	return true;
}

BaseTextureCacheData::BaseTextureCacheData(TSP tsp, TCW tcw, int area)
{
	initVramLocks();

	if (tcw.VQ_Comp == 1 && tcw.MipMapped == 1)
		// Star Wars Demolition
		tcw.ScanOrder = 0;
	this->tsp = tsp;
	this->tcw = tcw;
	this->area = area;

	//Reset state info ..
	Updates = 0;
	dirty = FrameCount;
	lock_block = nullptr;
	customPayload.reset();
	gpuPreloadedTexture.reset();
	customRequestId = {};
	gpuPalette = false;
	is_custom_replaced = false;
	usingGpuPreloadedTexture = false;
	customMipLevels = 0;

	//decode info from tsp/tcw into the texture struct
	tex = &pvrTexInfo[tcw.PixelFmt == PixelReserved ? Pixel1555 : tcw.PixelFmt];	//texture format table entry

	startAddress = (tcw.TexAddr << 3) & VRAM_MASK;	// texture start address
	mmStartAddress = startAddress;					// data texture start address (modified for MIPs, as needed)
	width = 8 << tsp.TexU;						//tex width
	height = 8 << tsp.TexV;						//tex height

	texconv8 = nullptr;

	if (tcw.ScanOrder && tex->PL32 != nullptr)
	{
		//Texture is stored 'planar' in memory, no deswizzle is needed
		if (tcw.MipMapped != 0)
		{
			WARN_LOG(RENDERER, "Warning: planar texture with mipmaps (invalid)");
			this->tcw.MipMapped = 0;
		}

		//Planar textures support stride selection, mostly used for non power of 2 textures (videos)
		int stride = width;
		if (tcw.StrideSel)
		{
			stride = (TEXT_CONTROL & 31) * 32;
			if (stride == 0)
				stride = width;
		}

		//Call the format specific conversion code
		texconv = nullptr;
		if (tcw.VQ_Comp != 0)
		{
			// VQ
			texconv32 = tex->PLVQ32;
			mmStartAddress += VQ_CODEBOOK_SIZE;
			size = stride * height / 4;
		}
		else
		{
			// Normal
			texconv32 = tex->PL32;
			//calculate the size, in bytes, for the locking
			size = stride * height * tex->bpp / 8;
		}
	}
	else
	{
		if (!IsPaletted())
		{
			this->tcw.ScanOrder = 0;
			this->tcw.StrideSel = 0;
		}
		// Quake 3 Arena uses one
		if (tcw.MipMapped)
			// Mipmapped texture must be square and TexV is ignored
			height = width;

		if (tcw.VQ_Comp)
		{
			verify(tex->VQ != NULL || tex->VQ32 != NULL);
			if (tcw.MipMapped)
				mmStartAddress += VQMipPoint[tsp.TexU + 3];
			else
				mmStartAddress += VQ_CODEBOOK_SIZE;
			texconv = tex->VQ;
			texconv32 = tex->VQ32;
			size = width * height / 4;
		}
		else
		{
			verify(tex->TW != NULL || tex->TW32 != NULL);
			if (tcw.MipMapped)
				mmStartAddress += OtherMipPoint[tsp.TexU + 3] * tex->bpp / 8;
			texconv = tex->TW;
			texconv32 = tex->TW32;
			size = width * height * tex->bpp / 8;
			texconv8 = tex->TW8;
		}
	}
}

void BaseTextureCacheData::ComputeHash()
{
	// Include everything but texaddr, reserved and stride. Palette textures don't have ScanOrder
	const u32 tcwMask = IsPaletted() ? 0xF8000000 : 0xFC000000;
	if (tcw.VQ_Comp)
	{
		// The size for VQ textures wasn't correctly calculated.
		// We use the old size to compute the hash for backward-compatibility
		// with existing custom texture packs.
		int oldSize = width * height / 8;
		old_vqtexture_hash = XXH32(&vram[mmStartAddress - VQ_CODEBOOK_SIZE], oldSize, 7);
		if (IsPaletted())
			old_vqtexture_hash ^= palette_hash;
		old_texture_hash = old_vqtexture_hash;
		old_vqtexture_hash ^= tcw.full & tcwMask;
		// New hash
	    XXH32_state_t *state = XXH32_createState();
	    XXH32_reset(state, 7);
	    // hash vq codebook
	    XXH32_update(state, &vram[startAddress], VQ_CODEBOOK_SIZE);
	    // hash texture
	    XXH32_update(state, &vram[mmStartAddress], size);
	    texture_hash = XXH32_digest(state);
	    XXH32_freeState(state);
		if (IsPaletted())
			texture_hash ^= palette_hash;
		texture_hash ^= tcw.full & tcwMask;
	}
	else
	{
		old_vqtexture_hash = 0;
		texture_hash = XXH32(&vram[mmStartAddress], size, 7);
		if (IsPaletted())
			texture_hash ^= palette_hash;
		old_texture_hash = texture_hash;
		texture_hash ^= tcw.full & tcwMask;
	}
}

bool BaseTextureCacheData::Update()
{
	//texture state tracking stuff
	Updates++;
	dirty = 0;
	gpuPalette = false;
	tex_type = tex->type;

	const u8 *vqCodebook = nullptr;
	u32 paletteIndex = 0;
	bool has_alpha = false;
	if (IsPaletted())
	{
		if (IsGpuHandledPaletted(tsp, tcw, area))
		{
			tex_type = TextureType::_8;
			gpuPalette = true;
		}
		else
		{
			tex_type = PAL_TYPE[PAL_RAM_CTRL&3];
			if (tex_type != TextureType::_565)
				has_alpha = true;
		}

		// Get the palette hash to check for future updates
		if (tcw.PixelFmt == PixelPal4)
		{
			palette_hash = pal_hash_16[tcw.PalSelect];
			paletteIndex = tcw.PalSelect << 4;
		}
		else
		{
			palette_hash = pal_hash_256[tcw.PalSelect >> 4];
			paletteIndex = (tcw.PalSelect >> 4) << 8;
		}
	}

	if (tcw.VQ_Comp)
		vqCodebook = &vram[startAddress];

	//texture conversion work
	u32 stride = width;

	if (tcw.StrideSel && tcw.ScanOrder && tex->PL32 != nullptr)
	{
		stride = (TEXT_CONTROL & 31) * 32;
		if (stride == 0)
			stride = width;
	}

	u32 heightLimit = height;
	const u32 originalSize = size;
	if (startAddress > VRAM_SIZE || mmStartAddress + size > VRAM_SIZE)
	{
		heightLimit = 0;
		if (mmStartAddress < VRAM_SIZE && mmStartAddress + size > VRAM_SIZE && tcw.ScanOrder)
		{
			// Shenmue Space Harrier mini-arcade loads a texture that goes beyond the end of VRAM
			// but only uses the top portion of it
			heightLimit = (VRAM_SIZE - mmStartAddress) * 8 / stride / tex->bpp;
			size = stride * heightLimit * tex->bpp/8;
		}
		if (heightLimit == 0)
		{
			size = originalSize;
			WARN_LOG(RENDERER, "Warning: invalid texture. Address %08X %08X size %d", startAddress, mmStartAddress, size);
			dirty = 1;
			unprotectVRam();
			return false;
		}
	}
	bool dumpReplacedTexture = false;
	if (custom_texture.enabled())
	{
		u32 oldHash = texture_hash;
		ComputeHash();
		if (Updates > 1 && oldHash == texture_hash)
		{
			// Texture hasn't changed so skip the update.
			if (is_custom_replaced)
			{
				tex_type = TextureType::_8888;
				gpuPalette = false;
			}
			protectVRam();
			size = originalSize;
			return true;
		}
		gpuPreloadedTexture = renderer == nullptr ? nullptr
				: renderer->findGpuPreloadedTexture(texture_hash,
						old_vqtexture_hash, old_texture_hash);
		if (gpuPreloadedTexture)
		{
			custom_texture.cancelRequest(customRequestId);
			customRequestId = {};
			customPayload.reset();
		}
		else
			custom_texture.loadCustomTextureAsync(this);
		if (gpuPreloadedTexture || customPayload)
		{
			dumpReplacedTexture = config::DumpTextures && config::DumpReplacedTextures.get();
			if (!dumpReplacedTexture)
			{
				const TextureType originalTextureType = tex_type;
				if (CheckCustomTexture())
				{
					protectVRam();
					PrintTextureName();
					size = originalSize;
					return true;
				}
				tex_type = originalTextureType;
			}
		}
	}
	is_custom_replaced = false;
	customMipLevels = 0;

	void *temp_tex_buffer = nullptr;
	u32 upscaled_w = width;
	u32 upscaled_h = height;

	PixelBuffer<u16> pb16;
	PixelBuffer<u32> pb32;
	PixelBuffer<u8> pb8;

	// Figure out if we really need to use a 32-bit pixel buffer
	bool textureUpscaling = config::TextureUpscale > 1
			// Don't process textures that are too big
			&& (int)(width * height) <= config::MaxFilteredTextureSize * config::MaxFilteredTextureSize
			// Don't process YUV textures
			&& tcw.PixelFmt != PixelYUV;
	bool need_32bit_buffer = true;
	if (!textureUpscaling
		&& (!IsPaletted() || tex_type != TextureType::_8888)
		&& texconv != NULL
		&& !Force32BitTexture(tex_type))
		need_32bit_buffer = false;
	// TODO avoid upscaling/depost. textures that change too often

	bool mipmapped = IsMipmapped() && !config::DumpTextures;

	if (texconv32 != NULL && need_32bit_buffer)
	{
		if (textureUpscaling)
			// don't use mipmaps if upscaling
			mipmapped = false;
		// Force the texture type since that's the only 32-bit one we know
		tex_type = TextureType::_8888;
		pb32.setVQCodebook(vqCodebook);
		pb32.setPaletteIndex(paletteIndex);

		if (mipmapped)
		{
			pb32.init(width, height, true);
			for (u32 i = 0; i <= tsp.TexU + 3u; i++)
			{
				pb32.set_mipmap(i);
				u32 vram_addr;
				if (tcw.VQ_Comp)
				{
					vram_addr = startAddress + VQMipPoint[i];
					if (i == 0)
					{
						PixelBuffer<u32> pb0;
						pb0.init(2, 2 ,false);
						pb0.setVQCodebook(vqCodebook);
						pb0.setPaletteIndex(paletteIndex);
						if (tcw.PixelFmt == PixelYUV)
							// Use higher LoD mipmap
							vram_addr = startAddress + VQMipPoint[1];
						texconv32(&pb0, &vram[vram_addr], 2, 2);
						*pb32.data() = *pb0.data(1, 1);
						continue;
					}
				}
				else
					vram_addr = startAddress + OtherMipPoint[i] * tex->bpp / 8;
				if (tcw.PixelFmt == PixelYUV && i == 0)
					// Special case for YUV at 1x1 LoD
					pvrTexInfo[Pixel565].TW32(&pb32, &vram[vram_addr], 1, 1);
				else
					texconv32(&pb32, &vram[vram_addr], 1 << i, 1 << i);
			}
			pb32.set_mipmap(0);
		}
		else
		{
			pb32.init(width, height);
			texconv32(&pb32, (u8*)&vram[mmStartAddress], stride, heightLimit);

			// xBRZ scaling
			if (textureUpscaling)
			{
				// xBRZ only supports scale factors 2 to 6. Out of range, scale() returns
				// without writing the destination buffer.
				const int factor = std::clamp((int)config::TextureUpscale, 2, 6);
				PixelBuffer<u32> tmp_buf;
				tmp_buf.init(width * factor, height * factor);

				if (tcw.PixelFmt == Pixel1555 || tcw.PixelFmt == Pixel4444)
					// Alpha channel formats. Palettes with alpha are already handled
					has_alpha = true;
				UpscalexBRZ(factor, pb32.data(), tmp_buf.data(), width, height, has_alpha);
				pb32.steal_data(tmp_buf);
				upscaled_w *= factor;
				upscaled_h *= factor;
			}
		}
		temp_tex_buffer = pb32.data();
	}
	else if (texconv8 != NULL && tex_type == TextureType::_8)
	{
		pb8.setVQCodebook(vqCodebook);
		pb8.setPaletteIndex(paletteIndex);

		if (mipmapped)
		{
			// This shouldn't happen since mipmapped palette textures are converted to rgba
			pb8.init(width, height, true);
			for (u32 i = 0; i <= tsp.TexU + 3u; i++)
			{
				pb8.set_mipmap(i);
				u32 vram_addr = startAddress + OtherMipPoint[i] * tex->bpp / 8;
				texconv8(&pb8, &vram[vram_addr], 1 << i, 1 << i);
			}
			pb8.set_mipmap(0);
		}
		else
		{
			pb8.init(width, height);
			texconv8(&pb8, &vram[mmStartAddress], stride, height);
		}
		temp_tex_buffer = pb8.data();
	}
	else if (texconv != NULL)
	{
		pb16.setVQCodebook(vqCodebook);
		pb16.setPaletteIndex(paletteIndex);

		if (mipmapped)
		{
			pb16.init(width, height, true);
			for (u32 i = 0; i <= tsp.TexU + 3u; i++)
			{
				pb16.set_mipmap(i);
				u32 vram_addr;
				if (tcw.VQ_Comp)
				{
					vram_addr = startAddress + VQMipPoint[i];
					if (i == 0)
					{
						PixelBuffer<u16> pb0;
						pb0.init(2, 2 ,false);
						pb0.setVQCodebook(vqCodebook);
						pb0.setPaletteIndex(paletteIndex);
						texconv(&pb0, (u8*)&vram[vram_addr], 2, 2);
						*pb16.data() = *pb0.data(1, 1);
						continue;
					}
				}
				else
					vram_addr = startAddress + OtherMipPoint[i] * tex->bpp / 8;
				texconv(&pb16, (u8*)&vram[vram_addr], 1 << i, 1 << i);
			}
			pb16.set_mipmap(0);
		}
		else
		{
			pb16.init(width, height);
			texconv(&pb16, (u8*)&vram[mmStartAddress], stride, heightLimit);
		}
		temp_tex_buffer = pb16.data();
	}
	else
	{
		//fill it in with a temp color
		WARN_LOG(RENDERER, "UNHANDLED TEXTURE");
		pb16.init(width, height);
		memset(pb16.data(), 0x80, width * height * 2);
		temp_tex_buffer = pb16.data();
		mipmapped = false;
	}
	//lock the texture to detect changes in it
	protectVRam();

	bool customTextureUploaded = false;
	if (dumpReplacedTexture)
	{
		ComputeHash();
		custom_texture.dumpTexture(this, upscaled_w, upscaled_h, temp_tex_buffer);
		NOTICE_LOG(RENDERER, "Dumped texture %x.png. Old hash %x", texture_hash, old_texture_hash);
		const TextureType originalTextureType = tex_type;
		customTextureUploaded = CheckCustomTexture();
		if (!customTextureUploaded)
			tex_type = originalTextureType;
	}
	if (!customTextureUploaded)
		UploadToGPU(upscaled_w, upscaled_h, (const u8 *)temp_tex_buffer, IsMipmapped(), mipmapped);
	if (config::DumpTextures && !dumpReplacedTexture)
	{
		ComputeHash();
		custom_texture.dumpTexture(this, upscaled_w, upscaled_h, temp_tex_buffer);
		NOTICE_LOG(RENDERER, "Dumped texture %x.png. Old hash %x", texture_hash, old_texture_hash);
	}
	PrintTextureName();
	// Restore the original texture size if it was constrained to VRAM limits above
	size = originalSize;

	return true;
}

bool BaseTextureCacheData::uploadCustomTexture(const PreparedCustomTexture& texture, bool mipmapped)
{
	if (texture.nativeFormat != NativeTextureFormat::Rgba8Unorm || texture.levels.size() != 1)
		return false;
	tex_type = TextureType::_8888;
	UploadToGPU(texture.width, texture.height, texture.bytes.data(),
			mipmapped && texture.generateMipmaps, false);
	return true;
}

bool BaseTextureCacheData::CheckCustomTexture()
{
	if (gpuPreloadedTexture)
	{
		if (!useGpuPreloadedTexture(gpuPreloadedTexture))
		{
			WARN_LOG(RENDERER, "Could not use GPU-preloaded custom texture");
			custom_texture.reportError(CustomTextureException::Error::Upload);
			custom_texture.showErrorNotification();
			gpuPreloadedTexture.reset();
			return false;
		}
		tex_type = TextureType::_8888;
		gpuPalette = false;
		is_custom_replaced = true;
		usingGpuPreloadedTexture = true;
		customMipLevels = gpuPreloadedTexture->mipLevels;
		return true;
	}
	if (!customPayload && customRequestId)
	{
		bool failed = false;
		customPayload = custom_texture.takePreparedTexture(customRequestId, failed);
		if (failed)
			customRequestId = {};
	}
	if (!customPayload)
		return false;
	const CustomTextureCapabilities capabilities = custom_texture.getCapabilities();
	if (!capabilities.canUpload(customPayload->nativeFormat, customPayload->width,
			customPayload->height, static_cast<u32>(customPayload->levels.size())))
	{
		WARN_LOG(RENDERER, "Discarding incompatible prepared custom texture %08x",
				customPayload->replacementHash);
		custom_texture.reportError(customPayload->width > capabilities.max2DWidth
				|| customPayload->height > capabilities.max2DHeight
				? CustomTextureException::Error::TextureTooLarge
				: CustomTextureException::Error::Upload);
		custom_texture.showErrorNotification();
		customPayload.reset();
		customRequestId = {};
		return false;
	}
	bool uploaded = false;
	bool uploadFailureLogged = false;
	const bool mipmapped = tcw.MipMapped != 0 && tcw.ScanOrder == 0 && config::UseMipmaps;
	try
	{
		uploaded = uploadCustomTexture(*customPayload, mipmapped);
	}
	catch (const FlycastException& exception)
	{
		WARN_LOG(RENDERER, "Custom texture validation failed for %08x (%s): %s",
				customPayload->replacementHash,
				nativeTextureFormatName(customPayload->nativeFormat), exception.what());
		uploadFailureLogged = true;
	}
	catch (const std::bad_alloc& exception)
	{
		WARN_LOG(RENDERER, "Custom texture upload allocation failed for %08x (%s): %s",
				customPayload->replacementHash,
				nativeTextureFormatName(customPayload->nativeFormat), exception.what());
		uploadFailureLogged = true;
	}
	if (!uploaded)
	{
		if (!uploadFailureLogged)
			WARN_LOG(RENDERER, "Custom texture upload failed for %08x (%s)",
					customPayload->replacementHash, nativeTextureFormatName(customPayload->nativeFormat));
		custom_texture.reportError(CustomTextureException::Error::Upload);
		custom_texture.showErrorNotification();
		customPayload.reset();
		customRequestId = {};
		return false;
	}
	tex_type = TextureType::_8888;
	gpuPalette = false;
	is_custom_replaced = true;
	customMipLevels = static_cast<u8>(mipmapped && customPayload->generateMipmaps
			? mipmapLevelCount(customPayload->width, customPayload->height)
			: customPayload->levels.size());
	customPayload.reset();
	customRequestId = {};
	return true;
}

void BaseTextureCacheData::SetDirectXColorOrder(bool enabled) {
	pvrTexInfo = enabled ? directx::pvrTexInfo : opengl::pvrTexInfo;
	pal_needs_update = true;
}

#if HOST_CPU == CPU_ARM64
static inline bool pvr32SpanIsContiguous(u32 addr, u32 logicalBytes)
{
	const u32 maskedAddr = addr & VRAM_MASK;
	if ((maskedAddr & 3) != 0
			|| logicalBytes > VRAM_32_BANK_BIT
			|| (maskedAddr & (VRAM_32_BANK_BIT - 1)) > VRAM_32_BANK_BIT - logicalBytes
			|| maskedAddr > VRAM_MASK - (logicalBytes - 1))
		return false;

	// The 32-bit bus doubles the offset and adds bank * 4. NEON deinterleaving
	// loads therefore access twice the logical span, including the other bank.
	return (u64)pvr_map32(maskedAddr) + (u64)logicalBytes * 2 <= (u64)vram.getSize();
}

template<typename Packer>
static inline void storeFramebufferRgba(u32 *dst, uint8x16_t red, uint8x16_t green, uint8x16_t blue)
{
	uint8x16x4_t rgba;
	if constexpr (std::is_same_v<Packer, RGBAPacker>)
	{
		rgba.val[0] = red;
		rgba.val[2] = blue;
	}
	else
	{
		rgba.val[0] = blue;
		rgba.val[2] = red;
	}
	rgba.val[1] = green;
	rgba.val[3] = vdupq_n_u8(0xff);
	vst4q_u8((u8 *)dst, rgba);
}

template<typename Packer, bool RGB565>
static void readFramebuffer16Line(u32 addr, u32 *dst, int width, u32 fb_concat)
{
	int x = 0;
	for (; x < width && (addr & 3) != 0; x++, addr += 2)
	{
		const u16 src = pvr_read32p<u16>(addr);
		if constexpr (RGB565)
			dst[x] = Packer::pack((((src >> 11) & 0x1f) << 3) | fb_concat,
					(((src >> 5) & 0x3f) << 2) | (fb_concat & 3),
					((src & 0x1f) << 3) | fb_concat, 0xff);
		else
			dst[x] = Packer::pack((((src >> 10) & 0x1f) << 3) | fb_concat,
					(((src >> 5) & 0x1f) << 3) | fb_concat,
					((src & 0x1f) << 3) | fb_concat, 0xff);
	}

	for (; x + 16 <= width && pvr32SpanIsContiguous(addr, 32); x += 16, addr += 32)
	{
		const u8 *mapped = &vram[pvr_map32(addr)];
		const uint16x8_t src0 = vreinterpretq_u16_u32(vld2q_u32((const u32 *)mapped).val[0]);
		const uint16x8_t src1 = vreinterpretq_u16_u32(vld2q_u32((const u32 *)(mapped + 32)).val[0]);
		const uint16x8_t concat5 = vdupq_n_u16(fb_concat);

		uint16x8_t red0;
		uint16x8_t red1;
		uint16x8_t green0;
		uint16x8_t green1;
		if constexpr (RGB565)
		{
			red0 = vorrq_u16(vshlq_n_u16(vshrq_n_u16(src0, 11), 3), concat5);
			red1 = vorrq_u16(vshlq_n_u16(vshrq_n_u16(src1, 11), 3), concat5);
			const uint16x8_t concat6 = vdupq_n_u16(fb_concat & 3);
			green0 = vorrq_u16(vshlq_n_u16(vandq_u16(vshrq_n_u16(src0, 5), vdupq_n_u16(0x3f)), 2), concat6);
			green1 = vorrq_u16(vshlq_n_u16(vandq_u16(vshrq_n_u16(src1, 5), vdupq_n_u16(0x3f)), 2), concat6);
		}
		else
		{
			red0 = vorrq_u16(vshlq_n_u16(vandq_u16(vshrq_n_u16(src0, 10), vdupq_n_u16(0x1f)), 3), concat5);
			red1 = vorrq_u16(vshlq_n_u16(vandq_u16(vshrq_n_u16(src1, 10), vdupq_n_u16(0x1f)), 3), concat5);
			green0 = vorrq_u16(vshlq_n_u16(vandq_u16(vshrq_n_u16(src0, 5), vdupq_n_u16(0x1f)), 3), concat5);
			green1 = vorrq_u16(vshlq_n_u16(vandq_u16(vshrq_n_u16(src1, 5), vdupq_n_u16(0x1f)), 3), concat5);
		}
		const uint16x8_t blue0 = vorrq_u16(vshlq_n_u16(vandq_u16(src0, vdupq_n_u16(0x1f)), 3), concat5);
		const uint16x8_t blue1 = vorrq_u16(vshlq_n_u16(vandq_u16(src1, vdupq_n_u16(0x1f)), 3), concat5);
		storeFramebufferRgba<Packer>(dst + x,
				vcombine_u8(vmovn_u16(red0), vmovn_u16(red1)),
				vcombine_u8(vmovn_u16(green0), vmovn_u16(green1)),
				vcombine_u8(vmovn_u16(blue0), vmovn_u16(blue1)));
	}

	for (; x < width; x++, addr += 2)
	{
		const u16 src = pvr_read32p<u16>(addr);
		if constexpr (RGB565)
			dst[x] = Packer::pack((((src >> 11) & 0x1f) << 3) | fb_concat,
					(((src >> 5) & 0x3f) << 2) | (fb_concat & 3),
					((src & 0x1f) << 3) | fb_concat, 0xff);
		else
			dst[x] = Packer::pack((((src >> 10) & 0x1f) << 3) | fb_concat,
					(((src >> 5) & 0x1f) << 3) | fb_concat,
					((src & 0x1f) << 3) | fb_concat, 0xff);
	}
}

template<typename Packer>
static void readFramebufferC888Line(u32 addr, u32 *dst, int width)
{
	int x = 0;
	for (; x + 8 <= width && pvr32SpanIsContiguous(addr, 32); x += 8, addr += 32)
	{
		const u8 *mapped = &vram[pvr_map32(addr)];
		const uint32x4_t src0 = vld2q_u32((const u32 *)mapped).val[0];
		const uint32x4_t src1 = vld2q_u32((const u32 *)(mapped + 32)).val[0];
		const uint8x8_t red = vmovn_u16(vcombine_u16(
				vmovn_u32(vshrq_n_u32(src0, 16)), vmovn_u32(vshrq_n_u32(src1, 16))));
		const uint8x8_t green = vmovn_u16(vcombine_u16(
				vmovn_u32(vshrq_n_u32(src0, 8)), vmovn_u32(vshrq_n_u32(src1, 8))));
		const uint8x8_t blue = vmovn_u16(vcombine_u16(vmovn_u32(src0), vmovn_u32(src1)));
		uint8x8x4_t rgba;
		if constexpr (std::is_same_v<Packer, RGBAPacker>)
		{
			rgba.val[0] = red;
			rgba.val[2] = blue;
		}
		else
		{
			rgba.val[0] = blue;
			rgba.val[2] = red;
		}
		rgba.val[1] = green;
		rgba.val[3] = vdup_n_u8(0xff);
		vst4_u8((u8 *)(dst + x), rgba);
	}
	for (; x < width; x++, addr += 4)
	{
		const u32 src = pvr_read32p<u32>(addr);
		dst[x] = Packer::pack(src >> 16, src >> 8, src, 0xff);
	}
}
#endif

template<typename Packer>
void ReadFramebuffer(const FramebufferInfo& info, PixelBuffer<u32>& pb, int& width, int& height)
{
	width = (info.fb_r_size.fb_x_size + 1) * 2;     // in 16-bit words
	height = info.fb_r_size.fb_y_size + 1;
	int modulus = (info.fb_r_size.fb_modulus - 1) * 2;

	int bpp;
	switch (info.fb_r_ctrl.fb_depth)
	{
		case fbde_0555:
		case fbde_565:
			bpp = 2;
			break;
		case fbde_888:
			bpp = 3;
			width = (width * 2) / 3;		// in pixels
			modulus = (modulus * 2) / 3;	// in pixels
			break;
		case fbde_C888:
			bpp = 4;
			width /= 2;             // in pixels
			modulus /= 2;           // in pixels
			break;
		default:
			die("Invalid framebuffer format\n");
			bpp = 4;
			break;
	}

	u32 addr = info.fb_r_sof1;
	if (info.spg_control.interlace)
	{
		if (width == modulus && info.fb_r_sof2 == info.fb_r_sof1 + width * bpp)
		{
			// Typical case alternating even and odd lines -> take the whole buffer at once
			modulus = 0;
			height *= 2;
		}
		else
		{
			addr = info.spg_status.fieldnum ? info.fb_r_sof2 : info.fb_r_sof1;
		}
	}
	else if (info.fb_r_ctrl.vclk_div == 0) {
		// 240p if PAL or NTSC
		height = std::min(height, 240);
	}

	pb.init(width, height);
	u32 *dst = (u32 *)pb.data();
	const u32 fb_concat = info.fb_r_ctrl.fb_concat;

	switch (info.fb_r_ctrl.fb_depth)
	{
		case fbde_0555:    // 555 RGB
			for (int y = 0; y < height; y++)
			{
#if HOST_CPU == CPU_ARM64
				readFramebuffer16Line<Packer, false>(addr, dst, width, fb_concat);
				dst += width;
				addr += (width + modulus) * bpp;
#else
				for (int i = 0; i < width; i++)
				{
					u16 src = pvr_read32p<u16>(addr);
					*dst++ = Packer::pack(
							(((src >> 10) & 0x1F) << 3) | fb_concat,
							(((src >> 5) & 0x1F) << 3) | fb_concat,
							(((src >> 0) & 0x1F) << 3) | fb_concat,
							0xff);
					addr += bpp;
				}
				addr += modulus * bpp;
#endif
			}
			break;

		case fbde_565:    // 565 RGB
			for (int y = 0; y < height; y++)
			{
#if HOST_CPU == CPU_ARM64
				readFramebuffer16Line<Packer, true>(addr, dst, width, fb_concat);
				dst += width;
				addr += (width + modulus) * bpp;
#else
				for (int i = 0; i < width; i++)
				{
					u16 src = pvr_read32p<u16>(addr);
					*dst++ = Packer::pack(
							(((src >> 11) & 0x1F) << 3) | fb_concat,
							(((src >> 5) & 0x3F) << 2) | (fb_concat & 3),
							(((src >> 0) & 0x1F) << 3) | fb_concat,
							0xFF);
					addr += bpp;
				}
				addr += modulus * bpp;
#endif
			}
			break;
		case fbde_888:		// 888 RGB
			for (int y = 0; y < height; y++)
			{
				for (int i = 0; i < width; i += 4)
				{
					u32 src = pvr_read32p<u32>(addr);
					*dst++ = Packer::pack(src >> 16, src >> 8, src, 0xff);
					addr += 4;
					if (i + 1 >= width)
						break;
					u32 src2 = pvr_read32p<u32>(addr);
					*dst++ = Packer::pack(src2 >> 8, src2, src >> 24, 0xff);
					addr += 4;
					if (i + 2 >= width)
						break;
					u32 src3 = pvr_read32p<u32>(addr);
					*dst++ = Packer::pack(src3, src2 >> 24, src2 >> 16, 0xff);
					addr += 4;
					if (i + 3 >= width)
						break;
					*dst++ = Packer::pack(src3 >> 24, src3 >> 16, src3 >> 8, 0xff);
				}
				addr += modulus * bpp;
			}
			break;
		case fbde_C888:     // 0888 RGB
			for (int y = 0; y < height; y++)
			{
#if HOST_CPU == CPU_ARM64
				readFramebufferC888Line<Packer>(addr, dst, width);
				dst += width;
				addr += (width + modulus) * bpp;
#else
				for (int i = 0; i < width; i++)
				{
					u32 src = pvr_read32p<u32>(addr);
					*dst++ = Packer::pack(src >> 16, src >> 8, src, 0xff);
					addr += bpp;
				}
				addr += modulus * bpp;
#endif
			}
			break;
	}
}
template void ReadFramebuffer<RGBAPacker>(const FramebufferInfo& info, PixelBuffer<u32>& pb, int& width, int& height);
template void ReadFramebuffer<BGRAPacker>(const FramebufferInfo& info, PixelBuffer<u32>& pb, int& width, int& height);

template<int bits>
static inline u8 roundColor(u8 in)
{
	u8 out = in >> (8 - bits);
	if (out != 0xffu >> (8 - bits))
		out += (in >> (8 - bits - 1)) & 1;
	return out;
}

// write to 32-bit vram area (framebuffer)
class FBPixelWriter
{
public:
	FBPixelWriter(u32 dstAddr) : dstAddr(dstAddr) {}

	template<typename T>
	void write(T pixel) {
		pvr_write32p<T, true>(dstAddr, pixel);
		dstAddr += sizeof(T);
	}

	void advance(int bytes) {
		dstAddr += bytes;
	}

private:
	u32 dstAddr;
};

// write to 64-bit vram area (render to texture)
class TexPixelWriter
{
public:
	TexPixelWriter(u16 *dest) : dest(dest) {}

	void write(u16 pixel) {
		*dest++ = pixel;
	}

	void advance(int bytes) {
		(u8 *&)dest += bytes;
	}

private:
	u16 *dest;
};

// 0555 KRGB 16 bit  (default)	Bit 15 is the value of fb_kval[7].
template<int Red, int Green, int Blue, int Alpha, typename PixelWriter, bool Round = false>
class FBLineWriter0555
{
public:
	FBLineWriter0555(FB_W_CTRL_type fb_w_ctrl, PixelWriter& pixWriter) : pixWriter(pixWriter) {
		kval_bit = (fb_w_ctrl.fb_kval & 0x80) << 8;
	}

	void write(int xmin, int xmax, const u8 *& pixel)
	{
		for (int c = xmin; c < xmax; c++)
		{
			u8 red, green, blue;
			if constexpr (Round)
			{
				red = roundColor<5>(pixel[Red]);
				green = roundColor<5>(pixel[Green]);
				blue = roundColor<5>(pixel[Blue]);
			}
			else
			{
				red = pixel[Red] >> 3;
				green = pixel[Green] >> 3;
				blue = pixel[Blue] >> 3;
			}
			pixWriter.write((u16)((red << 10) | (green << 5) | blue | kval_bit));
			pixel += 4;
		}
	}

	static constexpr int BytesPerPixel = 2;

private:
	u16 kval_bit;
	PixelWriter& pixWriter;
};

// 565 RGB 16 bit
template<int Red, int Green, int Blue, int Alpha, typename PixelWriter, bool Round = false>
class FBLineWriter565
{
public:
	FBLineWriter565(FB_W_CTRL_type fb_w_ctrl, PixelWriter& pixWriter) : pixWriter(pixWriter) {}

	void write(int xmin, int xmax, const u8 *& pixel)
	{
		for (int c = xmin; c < xmax; c++)
		{
			u8 red, green, blue;
			if constexpr (Round)
			{
				red = roundColor<5>(pixel[Red]);
				green = roundColor<6>(pixel[Green]);
				blue = roundColor<5>(pixel[Blue]);
			}
			else
			{
				red = pixel[Red] >> 3;
				green = pixel[Green] >> 2;
				blue = pixel[Blue] >> 3;
			}
			pixWriter.write((u16)((red << 11) | (green << 5) | blue));
			pixel += 4;
		}
	}

	static constexpr int BytesPerPixel = 2;

private:
	PixelWriter& pixWriter;
};

// 4444 ARGB 16 bit
template<int Red, int Green, int Blue, int Alpha, typename PixelWriter, bool Round = false>
class FBLineWriter4444
{
public:
	FBLineWriter4444(FB_W_CTRL_type fb_w_ctrl, PixelWriter& pixWriter) : pixWriter(pixWriter) {}

	void write(int xmin, int xmax, const u8 *& pixel)
	{
		for (int c = xmin; c < xmax; c++)
		{
			u8 red, green, blue, alpha;
			if constexpr (Round)
			{
				red = roundColor<4>(pixel[Red]);
				green = roundColor<4>(pixel[Green]);
				blue = roundColor<4>(pixel[Blue]);
				alpha = roundColor<4>(pixel[Alpha]);
			}
			else
			{
				red = pixel[Red] >> 4;
				green = pixel[Green] >> 4;
				blue = pixel[Blue] >> 4;
				alpha = pixel[Alpha] >> 4;
			}
			pixWriter.write((u16)((red << 8) | (green << 4) | blue | (alpha << 12)));
			pixel += 4;
		}
	}

	static constexpr int BytesPerPixel = 2;

private:
	PixelWriter& pixWriter;
};

// 1555 ARGB 16 bit    The alpha value is determined by comparison with the value of fb_alpha_threshold.
template<int Red, int Green, int Blue, int Alpha, typename PixelWriter, bool Round = false>
class FBLineWriter1555
{
public:
	FBLineWriter1555(FB_W_CTRL_type fb_w_ctrl, PixelWriter& pixWriter) : pixWriter(pixWriter) {
		fb_alpha_threshold = fb_w_ctrl.fb_alpha_threshold;
	}

	void write(int xmin, int xmax, const u8 *& pixel)
	{
		for (int c = xmin; c < xmax; c++)
		{
			u8 red, green, blue;
			if constexpr (Round)
			{
				red = roundColor<5>(pixel[Red]);
				green = roundColor<5>(pixel[Green]);
				blue = roundColor<5>(pixel[Blue]);
			}
			else
			{
				red = pixel[Red] >> 3;
				green = pixel[Green] >> 3;
				blue = pixel[Blue] >> 3;
			}
			pixWriter.write((u16)((red << 10) | (green << 5) | blue
					| (pixel[Alpha] >= fb_alpha_threshold ? 0x8000 : 0)));
			pixel += 4;
		}
	}

	static constexpr int BytesPerPixel = 2;

private:
	PixelWriter& pixWriter;
	u8 fb_alpha_threshold;
};

// 888 RGB 24 bit packed
template<int Red, int Green, int Blue, int Alpha, typename PixelWriter>
class FBLineWriter888
{
public:
	FBLineWriter888(FB_W_CTRL_type fb_w_ctrl, PixelWriter& pixWriter) : pixWriter(pixWriter) {}

	void write(int xmin, int xmax, const u8 *& pixel)
	{
		for (int c = xmin; c < xmax; c++)
		{
			pixWriter.write(pixel[Blue]);
			pixWriter.write(pixel[Green]);
			pixWriter.write(pixel[Red]);
			pixel += 4;
		}
	}

	static constexpr int BytesPerPixel = 3;

private:
	PixelWriter& pixWriter;
};

// 0888 KRGB 32 bit (K is the value of fb_kval.)
template<int Red, int Green, int Blue, int Alpha, typename PixelWriter>
class FBLineWriter0888
{
public:
	FBLineWriter0888(FB_W_CTRL_type fb_w_ctrl, PixelWriter& pixWriter) : pixWriter(pixWriter) {
		fb_kval = fb_w_ctrl.fb_kval << 24;
	}

	void write(int xmin, int xmax, const u8 *& pixel)
	{
		for (int c = xmin; c < xmax; c++)
		{
			pixWriter.write((u32)((pixel[Red] << 16) | (pixel[Green] << 8) | pixel[Blue] | fb_kval));
			pixel += 4;
		}
	}

	static constexpr int BytesPerPixel = 4;

private:
	u32 fb_kval;
	PixelWriter& pixWriter;
};

// 8888 ARGB 32 bit
template<int Red, int Green, int Blue, int Alpha, typename PixelWriter>
class FBLineWriter8888
{
public:
	FBLineWriter8888(FB_W_CTRL_type fb_w_ctrl, PixelWriter& pixWriter) : pixWriter(pixWriter) {}

	void write(int xmin, int xmax, const u8 *& pixel)
	{
		for (int c = xmin; c < xmax; c++)
		{
			pixWriter.write((u32)((pixel[Red] << 16) | (pixel[Green] << 8) | pixel[Blue] | (pixel[Alpha] << 24)));
			pixel += 4;
		}
	}

	static constexpr int BytesPerPixel = 4;

private:
	PixelWriter& pixWriter;
};

template<typename FBLineWriter>
static void writeTexture(u32 width, u32 height, const u8 *data, u16 *dst, FB_W_CTRL_type fb_w_ctrl, u32 linestride, const Rect& clip)
{
	int srcPadding = 0;
	int destPadding = 0;
	if (linestride == 0)
		linestride = width * 2;
	else if (linestride > width * 2u)
		destPadding = linestride - width * 2;
	else
		srcPadding = (width * 2 - linestride) * 2;
	height = std::min<u32>(height, clip.bottomRight().y + 1);

	const int xmin = clip.origin.x;
	int xmax = std::min<int>(clip.bottomRight().x + 1, width);
	xmax = std::min<int>(xmax, linestride / 2);
	TexPixelWriter pixWriter(dst + clip.origin.y * (width + destPadding / 2) + xmin);
	data += clip.origin.y * (width * 2 + srcPadding) + xmin * 2;
	const int deltaPix = xmin + std::min(linestride / 2, width) - xmax;
	destPadding += deltaPix * 2;
	srcPadding += deltaPix * 4;
	FBLineWriter lineWriter(fb_w_ctrl, pixWriter);
	for (u32 l = clip.origin.y; l < height; l++)
	{
		lineWriter.write(xmin, xmax, data);
		pixWriter.advance(destPadding);
		data += srcPadding;
	}
}

template<int Red, int Green, int Blue, int Alpha>
void WriteTextureToVRam(u32 width, u32 height, const u8 *data, u16 *dst, FB_W_CTRL_type fb_w_ctrl, u32 linestride, const Rect& clip)
{
	bool dither = fb_w_ctrl.fb_dither && config::EmulateFramebuffer;
	switch (fb_w_ctrl.fb_packmode)
	{
	case 0: // 0555 KRGB 16 bit  (default)
		if (dither)
			writeTexture<FBLineWriter0555<Red, Green, Blue, Alpha, TexPixelWriter, false>>(width, height, data, dst, fb_w_ctrl, linestride, clip);
		else
			writeTexture<FBLineWriter0555<Red, Green, Blue, Alpha, TexPixelWriter, true>>(width, height, data, dst, fb_w_ctrl, linestride, clip);
		break;
	case 1: // 565 RGB 16 bit
		if (dither)
			writeTexture<FBLineWriter565<Red, Green, Blue, Alpha, TexPixelWriter, false>>(width, height, data, dst, fb_w_ctrl, linestride, clip);
		else
			writeTexture<FBLineWriter565<Red, Green, Blue, Alpha, TexPixelWriter, true>>(width, height, data, dst, fb_w_ctrl, linestride, clip);
		break;
	case 2: // 4444 ARGB 16 bit
		if (dither)
			writeTexture<FBLineWriter4444<Red, Green, Blue, Alpha, TexPixelWriter, false>>(width, height, data, dst, fb_w_ctrl, linestride, clip);
		else
			writeTexture<FBLineWriter4444<Red, Green, Blue, Alpha, TexPixelWriter, true>>(width, height, data, dst, fb_w_ctrl, linestride, clip);
		break;
	case 3: // 1555 ARGB 16 bit
		if (dither)
			writeTexture<FBLineWriter1555<Red, Green, Blue, Alpha, TexPixelWriter, false>>(width, height, data, dst, fb_w_ctrl, linestride, clip);
		else
			writeTexture<FBLineWriter1555<Red, Green, Blue, Alpha, TexPixelWriter, true>>(width, height, data, dst, fb_w_ctrl, linestride, clip);
		break;
	default:
		ERROR_LOG(PVR, "Invalid/unsupported texture format: %d", fb_w_ctrl.fb_packmode);
		break;
	}
}
template void WriteTextureToVRam<0, 1, 2, 3>(u32 width, u32 height, const u8 *data, u16 *dst, FB_W_CTRL_type fb_w_ctrl, u32 linestride, const Rect& clip);
template void WriteTextureToVRam<2, 1, 0, 3>(u32 width, u32 height, const u8 *data, u16 *dst, FB_W_CTRL_type fb_w_ctrl, u32 linestride, const Rect& clip);

template<typename FBLineWriter>
static void writeFramebufferLW(u32 width, u32 height, const u8 *data, u32 dstAddr, FB_W_CTRL_type fb_w_ctrl, u32 linestride, const Rect& clip)
{
	int bpp = FBLineWriter::BytesPerPixel;

	u32 padding = linestride;
	if (padding > width * bpp)
		padding = padding - width * bpp;
	else
		padding = 0;

	const u8 *p = data + 4 * clip.origin.y * width;
	dstAddr += bpp * clip.origin.y * (width + padding / bpp);

	const u32 clipWidth = std::min<u32>(width, clip.origin.x + clip.size.x);
	height = std::min<u32>(height, clip.origin.y + clip.size.y);

	FBPixelWriter pixWriter(dstAddr);
	FBLineWriter lineWriter(fb_w_ctrl, pixWriter);

	const unsigned dp1 = 4 * clip.origin.x;
	const unsigned adv1 = bpp * clip.origin.x;
	const unsigned adv2 = padding + (width - clipWidth) * bpp;
	const unsigned dp2 = (width - clipWidth) * 4;

	for (u32 l = clip.origin.y; l < height; l++)
	{
		p += dp1;
		pixWriter.advance(adv1);

		lineWriter.write(clip.origin.x, clipWidth, p);

		pixWriter.advance(adv2);
		p += dp2;
	}
}

#if HOST_CPU == CPU_ARM64
template<int PackMode, int Red, int Green, int Blue, int Alpha>
static inline u16 packFramebuffer16Pixel(const u8 *pixel, FB_W_CTRL_type fb_w_ctrl)
{
	const u16 red = pixel[Red];
	const u16 green = pixel[Green];
	const u16 blue = pixel[Blue];
	if constexpr (PackMode == 0)
		return ((red >> 3) << 10) | ((green >> 3) << 5) | (blue >> 3)
				| ((fb_w_ctrl.fb_kval & 0x80) << 8);
	else if constexpr (PackMode == 1)
		return ((red >> 3) << 11) | ((green >> 2) << 5) | (blue >> 3);
	else if constexpr (PackMode == 2)
		return ((pixel[Alpha] >> 4) << 12) | ((red >> 4) << 8)
				| ((green >> 4) << 4) | (blue >> 4);
	else
		return (pixel[Alpha] >= fb_w_ctrl.fb_alpha_threshold ? 0x8000 : 0)
				| ((red >> 3) << 10) | ((green >> 3) << 5) | (blue >> 3);
}

template<int PackMode>
static inline uint16x8_t packFramebuffer16Pixels(uint8x8_t red, uint8x8_t green,
		uint8x8_t blue, uint8x8_t alpha, FB_W_CTRL_type fb_w_ctrl)
{
	const uint16x8_t red16 = vmovl_u8(red);
	const uint16x8_t green16 = vmovl_u8(green);
	const uint16x8_t blue16 = vmovl_u8(blue);
	if constexpr (PackMode == 0)
	{
		uint16x8_t packed = vshlq_n_u16(vshrq_n_u16(red16, 3), 10);
		packed = vorrq_u16(packed, vshlq_n_u16(vshrq_n_u16(green16, 3), 5));
		packed = vorrq_u16(packed, vshrq_n_u16(blue16, 3));
		return vorrq_u16(packed, vdupq_n_u16((fb_w_ctrl.fb_kval & 0x80) << 8));
	}
	else if constexpr (PackMode == 1)
	{
		uint16x8_t packed = vshlq_n_u16(vshrq_n_u16(red16, 3), 11);
		packed = vorrq_u16(packed, vshlq_n_u16(vshrq_n_u16(green16, 2), 5));
		return vorrq_u16(packed, vshrq_n_u16(blue16, 3));
	}
	else if constexpr (PackMode == 2)
	{
		uint16x8_t packed = vshlq_n_u16(vshrq_n_u16(vmovl_u8(alpha), 4), 12);
		packed = vorrq_u16(packed, vshlq_n_u16(vshrq_n_u16(red16, 4), 8));
		packed = vorrq_u16(packed, vshlq_n_u16(vshrq_n_u16(green16, 4), 4));
		return vorrq_u16(packed, vshrq_n_u16(blue16, 4));
	}
	else
	{
		uint16x8_t packed = vshlq_n_u16(vshrq_n_u16(red16, 3), 10);
		packed = vorrq_u16(packed, vshlq_n_u16(vshrq_n_u16(green16, 3), 5));
		packed = vorrq_u16(packed, vshrq_n_u16(blue16, 3));
		const uint16x8_t alphaBit = vandq_u16(
				vshlq_n_u16(vmovl_u8(vcge_u8(alpha, vdup_n_u8(fb_w_ctrl.fb_alpha_threshold))), 8),
				vdupq_n_u16(0x8000));
		return vorrq_u16(packed, alphaBit);
	}
}

// 8 words to 8 interleaved 32-bit VRAM locations
static inline void storeFramebufferWords(u8 *mapped, uint32x4_t words0, uint32x4_t words1)
{
	vst1q_lane_u32((u32 *)(mapped + 0), words0, 0);
	vst1q_lane_u32((u32 *)(mapped + 8), words0, 1);
	vst1q_lane_u32((u32 *)(mapped + 16), words0, 2);
	vst1q_lane_u32((u32 *)(mapped + 24), words0, 3);
	vst1q_lane_u32((u32 *)(mapped + 32), words1, 0);
	vst1q_lane_u32((u32 *)(mapped + 40), words1, 1);
	vst1q_lane_u32((u32 *)(mapped + 48), words1, 2);
	vst1q_lane_u32((u32 *)(mapped + 56), words1, 3);
}

static inline void storeFramebuffer16Pixels(u8 *mapped, uint16x8_t pixels0, uint16x8_t pixels1)
{
	storeFramebufferWords(mapped, vreinterpretq_u32_u16(pixels0), vreinterpretq_u32_u16(pixels1));
}

template<int PackMode, int Red, int Green, int Blue, int Alpha>
static void writeFramebuffer16Neon(u32 width, u32 height, const u8 *data, u32 dstAddr,
		FB_W_CTRL_type fb_w_ctrl, u32 linestride, const Rect& clip)
{
	u32 padding = linestride;
	if (padding > width * 2)
		padding -= width * 2;
	else
		padding = 0;

	const int clipWidth = std::min<int>(width, clip.origin.x + clip.size.x);
	height = std::min<u32>(height, clip.origin.y + clip.size.y);
	if (clip.origin.x >= clipWidth || clip.origin.y >= (int)height)
		return;

	const u32 lineStride = width * 2 + padding;
	const int count = clipWidth - clip.origin.x;
	for (u32 y = clip.origin.y; y < height; y++)
	{
		const u8 *pixel = data + ((size_t)y * width + clip.origin.x) * 4;
		u32 addr = dstAddr + y * lineStride + clip.origin.x * 2;
		int x = 0;
		for (; x < count && (addr & 3) != 0; x++, addr += 2, pixel += 4)
			pvr_write32p<u16, true>(addr, packFramebuffer16Pixel<PackMode, Red, Green, Blue, Alpha>(pixel, fb_w_ctrl));

		for (; x + 16 <= count && pvr32SpanIsContiguous(addr, 32); x += 16, addr += 32, pixel += 64)
		{
			const uint8x16x4_t rgba = vld4q_u8(pixel);
			const uint16x8_t packed0 = packFramebuffer16Pixels<PackMode>(
					vget_low_u8(rgba.val[Red]), vget_low_u8(rgba.val[Green]),
					vget_low_u8(rgba.val[Blue]), vget_low_u8(rgba.val[Alpha]), fb_w_ctrl);
			const uint16x8_t packed1 = packFramebuffer16Pixels<PackMode>(
					vget_high_u8(rgba.val[Red]), vget_high_u8(rgba.val[Green]),
					vget_high_u8(rgba.val[Blue]), vget_high_u8(rgba.val[Alpha]), fb_w_ctrl);
			storeFramebuffer16Pixels(&vram[pvr_map32(addr)], packed0, packed1);
		}

		for (; x < count; x++, addr += 2, pixel += 4)
			pvr_write32p<u16, true>(addr, packFramebuffer16Pixel<PackMode, Red, Green, Blue, Alpha>(pixel, fb_w_ctrl));
	}
}

// 0888 KRGB and 8888 ARGB: one pixel is one VRAM word, so 8 pixels fill the
// same 8 interleaved words as 16 16-bit pixels.
// 888 (24-bit packed) stays scalar: its pixels straddle VRAM words.
template<int PackMode, int Red, int Green, int Blue, int Alpha>
static inline u32 packFramebuffer32Pixel(const u8 *pixel, u32 kval)
{
	const u32 high = PackMode == 5 ? kval : ((u32)pixel[Alpha] << 24);
	return ((u32)pixel[Red] << 16) | ((u32)pixel[Green] << 8) | pixel[Blue] | high;
}

template<int PackMode>
static inline void packFramebuffer32Pixels(uint8x8_t red, uint8x8_t green, uint8x8_t blue,
		uint8x8_t alpha, u32 kval, uint32x4_t& words0, uint32x4_t& words1)
{
	// word = B | G << 8 | R << 16 | (K or A) << 24, built as two 16-bit halves
	const uint16x8_t low = vorrq_u16(vmovl_u8(blue), vshlq_n_u16(vmovl_u8(green), 8));
	uint16x8_t high;
	if constexpr (PackMode == 5)
		high = vorrq_u16(vmovl_u8(red), vdupq_n_u16((u16)(kval >> 16)));
	else
		high = vorrq_u16(vmovl_u8(red), vshlq_n_u16(vmovl_u8(alpha), 8));
	words0 = vorrq_u32(vmovl_u16(vget_low_u16(low)), vshlq_n_u32(vmovl_u16(vget_low_u16(high)), 16));
	words1 = vorrq_u32(vmovl_u16(vget_high_u16(low)), vshlq_n_u32(vmovl_u16(vget_high_u16(high)), 16));
}

// Same address walk as writeFramebufferLW(), so the output is identical to the
// scalar path, including when padding isn't a multiple of 4.
template<int PackMode, int Red, int Green, int Blue, int Alpha>
static void writeFramebuffer32Neon(u32 width, u32 height, const u8 *data, u32 dstAddr,
		FB_W_CTRL_type fb_w_ctrl, u32 linestride, const Rect& clip)
{
	constexpr u32 bpp = 4;
	u32 padding = linestride;
	if (padding > width * bpp)
		padding -= width * bpp;
	else
		padding = 0;

	const u32 kval = fb_w_ctrl.fb_kval << 24;
	const u8 *p = data + 4 * clip.origin.y * width;
	u32 addr = dstAddr + bpp * clip.origin.y * (width + padding / bpp);

	const u32 clipWidth = std::min<u32>(width, clip.origin.x + clip.size.x);
	height = std::min<u32>(height, clip.origin.y + clip.size.y);
	const int count = clipWidth > (u32)clip.origin.x ? clipWidth - clip.origin.x : 0;

	for (u32 l = clip.origin.y; l < height; l++)
	{
		p += 4 * clip.origin.x;
		addr += bpp * clip.origin.x;

		int x = 0;
		for (; x + 8 <= count && pvr32SpanIsContiguous(addr, 32); x += 8, addr += 32, p += 32)
		{
			const uint8x8x4_t rgba = vld4_u8(p);
			uint32x4_t words0, words1;
			packFramebuffer32Pixels<PackMode>(rgba.val[Red], rgba.val[Green], rgba.val[Blue],
					rgba.val[Alpha], kval, words0, words1);
			storeFramebufferWords(&vram[pvr_map32(addr)], words0, words1);
		}
		for (; x < count; x++, addr += bpp, p += 4)
			pvr_write32p<u32, true>(addr, packFramebuffer32Pixel<PackMode, Red, Green, Blue, Alpha>(p, kval));

		addr += padding + (width - clipWidth) * bpp;
		p += (width - clipWidth) * 4;
	}
}
#endif

template<int Red, int Green, int Blue, int Alpha>
void WriteFramebuffer(u32 width, u32 height, const u8 *data, u32 dstAddr, FB_W_CTRL_type fb_w_ctrl, u32 linestride, const Rect& clip)
{
	switch (fb_w_ctrl.fb_packmode)
	{
	case 0: // 0555 KRGB 16 bit
#if HOST_CPU == CPU_ARM64
		writeFramebuffer16Neon<0, Red, Green, Blue, Alpha>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#else
		writeFramebufferLW<FBLineWriter0555<Red, Green, Blue, Alpha, FBPixelWriter>>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#endif
		break;
	case 1: // 565 RGB 16 bit
#if HOST_CPU == CPU_ARM64
		writeFramebuffer16Neon<1, Red, Green, Blue, Alpha>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#else
		writeFramebufferLW<FBLineWriter565<Red, Green, Blue, Alpha, FBPixelWriter>>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#endif
		break;
	case 2: // 4444 ARGB 16 bit
#if HOST_CPU == CPU_ARM64
		writeFramebuffer16Neon<2, Red, Green, Blue, Alpha>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#else
		writeFramebufferLW<FBLineWriter4444<Red, Green, Blue, Alpha, FBPixelWriter>>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#endif
		break;
	case 3: // 1555 ARGB 16 bit
#if HOST_CPU == CPU_ARM64
		writeFramebuffer16Neon<3, Red, Green, Blue, Alpha>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#else
		writeFramebufferLW<FBLineWriter1555<Red, Green, Blue, Alpha, FBPixelWriter>>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#endif
		break;
	case 4: // 888 RGB 24 bit packed
		writeFramebufferLW<FBLineWriter888<Red, Green, Blue, Alpha, FBPixelWriter>>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
		break;
	case 5: // 0888 KRGB 32 bit
#if HOST_CPU == CPU_ARM64
		writeFramebuffer32Neon<5, Red, Green, Blue, Alpha>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#else
		writeFramebufferLW<FBLineWriter0888<Red, Green, Blue, Alpha, FBPixelWriter>>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#endif
		break;
	case 6: // 8888 ARGB 32 bit
#if HOST_CPU == CPU_ARM64
		writeFramebuffer32Neon<6, Red, Green, Blue, Alpha>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#else
		writeFramebufferLW<FBLineWriter8888<Red, Green, Blue, Alpha, FBPixelWriter>>(width, height, data, dstAddr, fb_w_ctrl, linestride, clip);
#endif
		break;
	default:
		ERROR_LOG(PVR, "Invalid framebuffer format: %d", fb_w_ctrl.fb_packmode);
		break;
	}
}
template void WriteFramebuffer<0, 1, 2, 3>(u32 width, u32 height, const u8 *data, u32 dstAddr, FB_W_CTRL_type fb_w_ctrl,
		u32 linestride, const Rect& clip);
template void WriteFramebuffer<2, 1, 0, 3>(u32 width, u32 height, const u8 *data, u32 dstAddr, FB_W_CTRL_type fb_w_ctrl,
		u32 linestride, const Rect& clip);

void BaseTextureCacheData::invalidate()
{
	dirty = FrameCount;

	libCore_vramlock_Unlock_block_wb(lock_block);
	lock_block = nullptr;
}

#ifdef TEST_AUTOMATION
#include <stb_image_write.h>

void dump_screenshot(u8 *buffer, u32 width, u32 height, bool alpha, u32 rowPitch, bool invertY)
{
	stbi_flip_vertically_on_write((int)invertY);
	stbi_write_png("screenshot.png", width, height, alpha ? 4 : 3, buffer, rowPitch);
}
#endif