From cb27df02f37edf6d182ac08b3606464d16a2201b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Thu, 21 Nov 2024 23:15:17 +0100 Subject: [PATCH] Implement new fast queue data structure CharQueue --- CMakeLists.txt | 1 + Common/Common.vcxproj | 1 + Common/Common.vcxproj.filters | 3 + Common/Data/Collections/CharQueue.h | 210 ++++++++++++++++++++++++++++ Common/Data/Collections/Slice.h | 1 + unittest/UnitTest.cpp | 57 ++++++++ unittest/UnitTest.h | 2 + 7 files changed, 275 insertions(+) create mode 100644 Common/Data/Collections/CharQueue.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 69b2bf0c29..534b951dfb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -653,6 +653,7 @@ add_library(Common STATIC Common/Data/Collections/Hashmaps.h Common/Data/Collections/TinySet.h Common/Data/Collections/FastVec.h + Common/Data/Collections/CharQueue.h Common/Data/Collections/CyclicBuffer.h Common/Data/Collections/ThreadSafeList.h Common/Data/Color/RGBAUtil.cpp diff --git a/Common/Common.vcxproj b/Common/Common.vcxproj index f0870fdba6..1eea117903 100644 --- a/Common/Common.vcxproj +++ b/Common/Common.vcxproj @@ -456,6 +456,7 @@ + diff --git a/Common/Common.vcxproj.filters b/Common/Common.vcxproj.filters index 055a82c5d7..f7d6ab2b95 100644 --- a/Common/Common.vcxproj.filters +++ b/Common/Common.vcxproj.filters @@ -671,6 +671,9 @@ ext\lua + + Data\Collections + diff --git a/Common/Data/Collections/CharQueue.h b/Common/Data/Collections/CharQueue.h new file mode 100644 index 0000000000..50fb455cb0 --- /dev/null +++ b/Common/Data/Collections/CharQueue.h @@ -0,0 +1,210 @@ +#pragma once + +#include "Common/Log.h" +#include "Common/Data/Collections/Slice.h" + +#include +#include + +// Queue with a dynamic size, optimized for bulk inserts and retrievals - and optimized +// to be fast in debug builds, hence it's pretty much C internally. +class CharQueue { +public: + explicit CharQueue(size_t blockSize = 16384) : blockSize_(blockSize) { + head_ = new Block{}; + tail_ = head_; + head_->data = (char *)malloc(blockSize_); + head_->size = (int)blockSize_; + head_->next = nullptr; + } + + // Remove copy constructors. + CharQueue(const CharQueue &) = delete; + void operator=(const CharQueue &) = delete; + + // But let's have a move constructor. + CharQueue(CharQueue &&src) noexcept { + // Steal the data from the other queue. + blockSize_ = src.blockSize_; + head_ = src.head_; + tail_ = src.tail_; + // Give the old queue a new block. Could probably also leave it in an invalid state and get rid of it. + src.head_ = new Block{}; + src.tail_ = src.head_; + src.head_->data = (char *)malloc(src.blockSize_); + src.head_->size = (int)src.blockSize_; + } + + ~CharQueue() { + clear(); + _dbg_assert_(head_ == tail_); + _dbg_assert_(head_->size == blockSize_); + _dbg_assert_(size() == 0); + // delete the final block + delete head_; + } + + char *push_back_write(size_t size) { + int remain = tail_->size - tail_->tail; + _dbg_assert_(remain >= 0); + if (remain >= (int)size) { + char *retval = tail_->data + tail_->tail; + tail_->tail += (int)size; + return retval; + } else { + // Can't fit? Just allocate a new block and fill it up with the new data. + int bsize = (int)blockSize_; + if (size > bsize) { + bsize = (int)size; + } + Block *b = new Block{}; + b->head = 0; + b->tail = (int)size; + b->size = bsize; + b->data = (char *)malloc(bsize); + tail_->next = b; + tail_ = b; + return tail_->data; + } + } + + void push_back(const char *data, size_t size) { + memcpy(push_back_write(size), data, size); + } + + void push_back(std::string_view chars) { + memcpy(push_back_write(chars.size()), chars.data(), chars.size()); + } + + // For debugging, mainly. + size_t block_count() const { + int count = 0; + Block *b = head_; + do { + count++; + b = b->next; + } while (b); + return count; + } + + size_t size() const { + size_t s = 0; + Block *b = head_; + do { + s += b->tail - b->head; + b = b->next; + } while (b); + return s; + } + + char peek(size_t peekOff) { + Block *b = head_; + do { + int remain = b->tail - b->head; + if (remain > peekOff) { + return b->data[b->head + peekOff]; + } else { + peekOff -= remain; + } + b = b->next; + } while (b); + // Ran out of data. + _dbg_assert_(false); + return 0; + } + + bool empty() const { + return size() == 0; + } + + // Pass in a lambda that takes each partial buffer as char*, size_t. + template + bool iterate_blocks(Func callback) const { + Block *b = head_; + do { + if (b->tail > b->head) { + if (!callback(b->data + b->head, b->tail - b->head)) { + return false; + } + } + b = b->next; + } while (b); + return true; + } + + size_t pop_front_bulk(char *dest, size_t size) { + int popSize = (int)size; + int writeOff = 0; + while (popSize > 0) { + int remain = head_->tail - head_->head; + int readSize = popSize; + if (readSize > remain) { + readSize = remain; + } + if (dest) { + memcpy(dest + writeOff, head_->data + head_->head, readSize); + } + writeOff += readSize; + head_->head += readSize; + popSize -= readSize; + if (head_->head == head_->tail) { + // Ran out of data in this block. Let's hope there's more... + if (head_ == tail_) { + // Can't read any more, bail. + break; + } + Block *next = head_->next; + delete head_; + head_ = next; + } + } + return (int)size - popSize; + } + + size_t skip(size_t size) { + return pop_front_bulk(nullptr, size); + } + + void clear() { + Block *b = head_; + // Delete all blocks except the last. + while (b != tail_) { + Block *next = b->next; + delete b; + b = next; + } + if (b->size != blockSize_) { + // Restore the remaining block to default size. + free(b->data); + b->data = (char *)malloc(blockSize_); + b->size = (int)blockSize_; + } + b->head = 0; + b->tail = 0; + // head and tail are now equal. + head_ = b; + } + +private: + struct Block { + ~Block() { + if (data) { + free(data); + data = 0; + } + size = 0; + } + Block *next; + char *data; + int size; // Can be bigger than the default block size if a push is very large. + // Internal head and tail inside the block. + int head; + int tail; + }; + + // There's always at least one block, initialized in the constructor. + Block *head_; + Block *tail_; + // Default min block size for new blocks. + size_t blockSize_; +}; diff --git a/Common/Data/Collections/Slice.h b/Common/Data/Collections/Slice.h index 88a4c9f199..de1bb0bbc5 100644 --- a/Common/Data/Collections/Slice.h +++ b/Common/Data/Collections/Slice.h @@ -1,6 +1,7 @@ #pragma once #include +#include // Like a const begin/end pair, just more convenient to use (and can only be used for linear array data). // Inspired by Rust's slices and Google's StringPiece. diff --git a/unittest/UnitTest.cpp b/unittest/UnitTest.cpp index 2502bb235d..f800437934 100644 --- a/unittest/UnitTest.cpp +++ b/unittest/UnitTest.cpp @@ -48,10 +48,12 @@ #include "Common/Data/Collections/TinySet.h" #include "Common/Data/Collections/FastVec.h" +#include "Common/Data/Collections/CharQueue.h" #include "Common/Data/Convert/SmallDataConvert.h" #include "Common/Data/Text/Parsers.h" #include "Common/Data/Text/WrapText.h" #include "Common/Data/Encoding/Utf8.h" +#include "Common/Buffer.h" #include "Common/File/Path.h" #include "Common/Input/InputState.h" #include "Common/Math/math_util.h" @@ -1034,6 +1036,59 @@ bool TestColorConv() { return true; } +CharQueue GetQueue() { + CharQueue queue(5); + return queue; +} + +bool TestCharQueue() { + // We use a tiny block size for testing. + CharQueue queue = std::move(GetQueue()); + + // Add 16 chars. + queue.push_back("abcdefghijkl"); + queue.push_back("mnop"); + + std::string testStr; + queue.iterate_blocks([&](const char *buf, size_t sz) { + testStr.append(buf, sz); + return true; + }); + EXPECT_EQ_STR(testStr, std::string("abcdefghijklmnop")); + + EXPECT_EQ_CHAR(queue.peek(11), 'l'); + EXPECT_EQ_CHAR(queue.peek(12), 'm'); + EXPECT_EQ_CHAR(queue.peek(15), 'p'); + EXPECT_EQ_INT(queue.block_count(), 3); // Didn't fit in the first block, so the two pushes above should have each created one additional block. + EXPECT_EQ_INT(queue.size(), 16); + char dest[15]; + EXPECT_EQ_INT(queue.pop_front_bulk(dest, 4), 4); + EXPECT_EQ_INT(queue.size(), 12); + EXPECT_EQ_MEM(dest, "abcd", 4); + EXPECT_EQ_INT(queue.pop_front_bulk(dest, 6), 6); + EXPECT_EQ_INT(queue.size(), 6); + EXPECT_EQ_MEM(dest, "efghij", 6); + queue.push_back("qr"); + EXPECT_EQ_INT(queue.pop_front_bulk(dest, 4), 4); // should pop off klmn + EXPECT_EQ_MEM(dest, "klmn", 4); + EXPECT_EQ_INT(queue.size(), 4); + EXPECT_EQ_CHAR(queue.peek(3), 'r'); + queue.pop_front_bulk(dest, 4); + EXPECT_EQ_MEM(dest, "opqr", 4); + EXPECT_TRUE(queue.empty()); + return true; +} + +bool TestBuffer() { + Buffer b = Buffer::Void(); + b.Append("hello"); + b.Append("world"); + std::string temp; + b.Take(10, &temp); + EXPECT_EQ_STR(temp, std::string("helloworld")); + return true; +} + typedef bool (*TestFunc)(); struct TestItem { const char *name; @@ -1094,6 +1149,8 @@ TestItem availableTests[] = { TEST_ITEM(Substitutions), TEST_ITEM(IniFile), TEST_ITEM(ColorConv), + TEST_ITEM(CharQueue), + TEST_ITEM(Buffer), }; int main(int argc, const char *argv[]) { diff --git a/unittest/UnitTest.h b/unittest/UnitTest.h index 42bd8b3665..53e189cbfa 100644 --- a/unittest/UnitTest.h +++ b/unittest/UnitTest.h @@ -17,10 +17,12 @@ inline bool rel_equal(float a, float b, float precision) { #define EXPECT_TRUE(a) if (!(a)) { printf("%s:%i: Test Fail\n", __FUNCTION__, __LINE__); return false; } #define EXPECT_FALSE(a) if ((a)) { printf("%s:%i: Test Fail\n", __FUNCTION__, __LINE__); return false; } #define EXPECT_EQ_INT(a, b) if ((a) != (b)) { printf("%s:%i: Test Fail\n%d\nvs\n%d\n", __FUNCTION__, __LINE__, (int)(a), (int)(b)); return false; } +#define EXPECT_EQ_CHAR(a, b) if ((a) != (b)) { printf("%s:%i: Test Fail\n%c\nvs\n%c\n", __FUNCTION__, __LINE__, (int)(a), (int)(b)); return false; } #define EXPECT_EQ_HEX(a, b) if ((a) != (b)) { printf("%s:%i: Test Fail\n%x\nvs\n%x\n", __FUNCTION__, __LINE__, a, b); return false; } #define EXPECT_EQ_FLOAT(a, b) if ((a) != (b)) { printf("%s:%i: Test Fail\n%0.7f\nvs\n%0.7f\n", __FUNCTION__, __LINE__, a, b); return false; } #define EXPECT_APPROX_EQ_FLOAT(a, b) if (fabsf((a)-(b))>0.00001f) { printf("%s:%i: Test Fail\n%f\nvs\n%f\n", __FUNCTION__, __LINE__, a, b); /*return false;*/ } #define EXPECT_REL_EQ_FLOAT(a, b, precision) if (!rel_equal(a, b, precision)) { printf("%s:%i: Test Fail\n%0.9f\nvs\n%0.9f\n", __FUNCTION__, __LINE__, a, b); /*return false;*/ } #define EXPECT_EQ_STR(a, b) if (a != b) { printf("%s: Test Fail\n%s\nvs\n%s\n", __FUNCTION__, a.c_str(), b.c_str()); return false; } +#define EXPECT_EQ_MEM(a, b, sz) if (memcmp(a, b, sz) != 0) { printf("%s: Test Fail\n%.*s\nvs\n%.*s\n", __FUNCTION__, (int)sz, a, (int)sz, b); return false; } #define RET(a) if (!(a)) { return false; }