资源编译优化

Signed-off-by: liduo <liduo29@huawei.com>
This commit is contained in:
liduo
2024-08-26 21:59:52 +08:00
parent 1ee4042244
commit 6c5c2e9ae3
13 changed files with 416 additions and 86 deletions
+2
View File
@@ -31,6 +31,7 @@ ohos_executable("restool") {
"src/id_worker.cpp",
"src/json_compiler.cpp",
"src/key_parser.cpp",
"src/rawfile_resfile_packer.cpp",
"src/reference_parser.cpp",
"src/resconfig_parser.cpp",
"src/resource_append.cpp",
@@ -45,6 +46,7 @@ ohos_executable("restool") {
"src/restool.cpp",
"src/select_compile_parse.cpp",
"src/task_handle.cpp",
"src/thread_pool.cpp",
"src/translatable_parser.cpp",
]
+3
View File
@@ -17,6 +17,7 @@
#define OHOS_RESTOOL_GENERIC_COMPILER_H
#include "i_resource_compiler.h"
#include <mutex>
namespace OHOS {
namespace Global {
@@ -27,11 +28,13 @@ public:
virtual ~GenericCompiler();
protected:
uint32_t CompileSingleFile(const FileInfo &fileInfo) override;
uint32_t CompileFiles(const std::vector<FileInfo> &fileInfos) override;
bool PostMediaFile(const FileInfo &fileInfo, const std::string &output);
private:
std::string GetOutputFilePath(const FileInfo &fileInfo) const;
bool IsIgnore(const FileInfo &fileInfo);
bool CopyMediaFile(const FileInfo &fileInfo, std::string &output);
std::mutex mutex_;
};
}
}
+1
View File
@@ -36,6 +36,7 @@ public:
protected:
virtual uint32_t CompileSingleFile(const FileInfo &fileInfo);
virtual uint32_t CompileFiles(const std::vector<FileInfo> &fileInfos);
bool MergeResourceItem(const ResourceItem &resourceItem);
std::string GetOutputFolder(const DirectoryInfo &directoryInfo) const;
ResType type_;
+45
View File
@@ -0,0 +1,45 @@
/*
* Copyright (c) 2021-2024 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef OHOS_RESTOOL_RAWFILE_RESFILE_PACKER_H
#define OHOS_RESTOOL_RAWFILE_RESFILE_PACKER_H
#include "cmd_parser.h"
#include "resource_util.h"
#include "thread_pool.h"
namespace OHOS {
namespace Global {
namespace Restool {
class RawFileResFilePacker {
public:
explicit RawFileResFilePacker(const PackageParser &packageParser, const std::string &moduleName);
~RawFileResFilePacker();
std::future<uint32_t> CopyRawFileOrResFileAsync(const std::vector<std::string> &inputs);
private:
uint32_t CopyRawFileOrResFile(const std::vector<std::string> &inputs);
uint32_t CopyRawFileOrResFile(const std::string &filePath, const std::string &fileType);
uint32_t CopyRawFileOrResFileImpl(const std::string &src, const std::string &dst);
uint32_t CopySingleFile(const std::string &path, std::string &subPath);
PackageParser packageParser_;
std::string moduleName_;
ThreadPool threadPool_;
std::vector<std::future<uint32_t>> copyResults_;
};
} // namespace Restool
} // namespace Global
} // namespace OHOS
#endif
+3 -1
View File
@@ -16,6 +16,7 @@
#ifndef OHOS_RESTOOL_RESOURCE_DATA_H
#define OHOS_RESTOOL_RESOURCE_DATA_H
#include <cstdint>
#include <map>
#include <set>
#include <stdint.h>
@@ -48,9 +49,10 @@ const static std::string SOLUTIONS_ARROW = "> ";
const static std::string LONG_PATH_HEAD = "\\\\?\\";
const static int32_t VERSION_MAX_LEN = 128;
const static int32_t INT_TO_BYTES = sizeof(uint32_t);
static const int8_t RESTOOL_VERSION[VERSION_MAX_LEN] = { "Restool 5.010" };
static const int8_t RESTOOL_VERSION[VERSION_MAX_LEN] = { "Restool 5.011" };
const static int32_t TAG_LEN = 4;
static std::set<std::string> g_resourceSet;
const static int8_t THREAD_POOL_SIZE = 2;
enum class KeyType {
LANGUAGE = 0,
-3
View File
@@ -43,9 +43,6 @@ private:
uint32_t GenerateTextHeader(const std::string &headerPath) const;
uint32_t GenerateCplusHeader(const std::string &headerPath) const;
uint32_t GenerateJsHeader(const std::string &headerPath) const;
uint32_t CopyRawFileOrResFile(const std::string &filePath, const std::string &fileType);
uint32_t CopyRawFileOrResFile(const std::vector<std::string> &inputs);
uint32_t CopyRawFileOrResFileImpl(const std::string &src, const std::string &dst);
uint32_t GenerateConfigJson();
uint32_t ScanResources(const std::vector<std::string> &inputs, const std::string &output);
uint32_t PackNormal();
+94
View File
@@ -0,0 +1,94 @@
/*
* Copyright (c) 2021-2024 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef OHOS_RESTOOL_THREAD_POOL_H
#define OHOS_RESTOOL_THREAD_POOL_H
#include "no_copy_able.h"
#include <condition_variable>
#include <functional>
#include <future>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <vector>
namespace OHOS {
namespace Global {
namespace Restool {
class ThreadPool : public NoCopyable {
public:
/**
* @brief Creates a thread pool with specify thread count
* @param threadCount the count of threads to be created
*/
explicit ThreadPool(const size_t threadCount);
~ThreadPool() override;
/**
* @brief Start the thread pool
*/
uint32_t Start();
/**
* @brief Stop the thread pool
*/
void Stop();
/**
* @brief Enqueue a task to queue of thread pool
* @param f the function to execute
* @param args the args of the function
*/
template <class F, class... Args>
auto Enqueue(F &&f, Args &&...args) -> std::future<typename std::result_of<F(Args...)>::type>;
private:
void WorkInThread();
std::vector<std::thread> workerThreads_;
std::queue<std::function<void()>> tasks_;
std::mutex queueMutex_;
std::condition_variable condition_;
bool running_;
size_t threadCount_;
};
template <typename F, typename... Args>
auto ThreadPool::Enqueue(F &&f, Args &&...args) -> std::future<typename std::result_of<F(Args...)>::type>
{
using return_type = typename std::result_of<F(Args...)>::type;
using p_task = std::packaged_task<return_type()>;
auto task = std::make_shared<p_task>(std::bind(std::forward<F>(f), std::forward<Args>(args)...));
std::future<return_type> res = task->get_future();
if (!running_ || workerThreads_.empty()) {
// If the pool is not running or there are no worker threads, execute the task immediately
(*task)();
return res;
}
{
std::unique_lock<std::mutex> lock(queueMutex_);
tasks_.emplace([task]() { (*task)(); });
}
condition_.notify_one();
return res;
}
} // namespace Restool
} // namespace Global
} // namespace OHOS
#endif
+22
View File
@@ -19,6 +19,7 @@
#include "resource_util.h"
#include "restool_errors.h"
#include "compression_parser.h"
#include "thread_pool.h"
namespace OHOS {
namespace Global {
@@ -32,6 +33,24 @@ GenericCompiler::~GenericCompiler()
{
}
uint32_t GenericCompiler::CompileFiles(const std::vector<FileInfo> &fileInfos)
{
cout << "Info: GenericCompiler::CompileFiles" << endl;
ThreadPool pool(THREAD_POOL_SIZE);
pool.Start();
std::vector<std::future<uint32_t>> results;
for (const auto &fileInfo : fileInfos) {
auto taskFunc = [this](const FileInfo &fileInfo) { return this->CompileSingleFile(fileInfo); };
results.push_back(pool.Enqueue(taskFunc, fileInfo));
}
for (auto &ret : results) {
if (ret.get() != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
}
return RESTOOL_SUCCESS;
}
uint32_t GenericCompiler::CompileSingleFile(const FileInfo &fileInfo)
{
if (IsIgnore(fileInfo)) {
@@ -51,6 +70,7 @@ uint32_t GenericCompiler::CompileSingleFile(const FileInfo &fileInfo)
bool GenericCompiler::PostMediaFile(const FileInfo &fileInfo, const std::string &output)
{
std::lock_guard<std::mutex> lock(mutex_);
ResourceItem resourceItem(fileInfo.filename, fileInfo.keyParams, type_);
resourceItem.SetFilePath(fileInfo.filePath);
resourceItem.SetLimitKey(fileInfo.limitKey);
@@ -79,6 +99,7 @@ string GenericCompiler::GetOutputFilePath(const FileInfo &fileInfo) const
bool GenericCompiler::IsIgnore(const FileInfo &fileInfo)
{
std::lock_guard<std::mutex> lock(mutex_);
string output = GetOutputFilePath(fileInfo);
if (!g_resourceSet.emplace(output).second) {
cerr << "Warning: '" << fileInfo.filePath << "' is defined repeatedly." << endl;
@@ -91,6 +112,7 @@ bool GenericCompiler::CopyMediaFile(const FileInfo &fileInfo, std::string &outpu
{
string outputFolder = GetOutputFolder(fileInfo);
if (!ResourceUtil::CreateDirs(outputFolder)) {
cerr << "Error: CopyMediaFile create dirs failed." << NEW_LINE_PATH << outputFolder << endl;
return false;
}
output = GetOutputFilePath(fileInfo);
+9 -1
View File
@@ -65,12 +65,20 @@ uint32_t IResourceCompiler::Compile(const vector<DirectoryInfo> &directoryInfos)
sort(fileInfos.begin(), fileInfos.end(), [](const auto &a, const auto &b) {
return a.filePath < b.filePath;
});
if (CompileFiles(fileInfos) != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
return PostCommit();
}
uint32_t IResourceCompiler::CompileFiles(const std::vector<FileInfo> &fileInfos)
{
for (const auto &fileInfo : fileInfos) {
if (CompileSingleFile(fileInfo) != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
}
return PostCommit();
return RESTOOL_SUCCESS;
}
const map<int64_t, vector<ResourceItem>> &IResourceCompiler::GetResult() const
+139
View File
@@ -0,0 +1,139 @@
/*
* Copyright (c) 2021-2024 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "rawfile_resfile_packer.h"
#include "compression_parser.h"
#include "restool_errors.h"
namespace OHOS {
namespace Global {
namespace Restool {
using namespace std;
RawFileResFilePacker::RawFileResFilePacker(const PackageParser &packageParser, const std::string &moduleName)
: packageParser_(packageParser), moduleName_(moduleName), threadPool_(ThreadPool(THREAD_POOL_SIZE))
{
threadPool_.Start();
}
RawFileResFilePacker::~RawFileResFilePacker()
{
threadPool_.Stop();
}
std::future<uint32_t> RawFileResFilePacker::CopyRawFileOrResFileAsync(const std::vector<std::string> &inputs)
{
auto func = [this](const std::vector<std::string> &inputs) { return this->CopyRawFileOrResFile(inputs); };
std::future<uint32_t> res = threadPool_.Enqueue(func, inputs);
return res;
}
uint32_t RawFileResFilePacker::CopyRawFileOrResFile(const vector<string> &inputs)
{
for (const auto &input : inputs) {
string rawfilePath = FileEntry::FilePath(input).Append(RAW_FILE_DIR).GetPath();
if (CopyRawFileOrResFile(rawfilePath, RAW_FILE_DIR) == RESTOOL_ERROR) {
cerr << "Error: copy raw file failed." << NEW_LINE_PATH << rawfilePath << endl;
return RESTOOL_ERROR;
}
string resfilePath = FileEntry::FilePath(input).Append(RES_FILE_DIR).GetPath();
if (CopyRawFileOrResFile(resfilePath, RES_FILE_DIR) == RESTOOL_ERROR) {
cerr << "Error: copy res file failed." << NEW_LINE_PATH << resfilePath << endl;
return RESTOOL_ERROR;
}
}
for (auto &res : copyResults_) {
uint32_t ret = res.get();
if (ret != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
}
return RESTOOL_SUCCESS;
}
uint32_t RawFileResFilePacker::CopyRawFileOrResFile(const string &filePath, const string &fileType)
{
if (!ResourceUtil::FileExist(filePath)) {
return RESTOOL_SUCCESS;
}
if (!FileEntry::IsDirectory(filePath)) {
cerr << "Error: '" << filePath << "' not directory." << endl;
return RESTOOL_ERROR;
}
string dst = FileEntry::FilePath(packageParser_.GetOutput()).Append(RESOURCES_DIR).Append(fileType).GetPath();
if (CopyRawFileOrResFileImpl(filePath, dst) != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
return RESTOOL_SUCCESS;
}
uint32_t RawFileResFilePacker::CopyRawFileOrResFileImpl(const string &src, const string &dst)
{
if (!ResourceUtil::CreateDirs(dst)) {
cerr << "Error: copy rawfile of resfile, create dirs failed." << NEW_LINE_PATH << dst << endl;
return RESTOOL_ERROR;
}
FileEntry f(src);
if (!f.Init()) {
return RESTOOL_ERROR;
}
for (const auto &entry : f.GetChilds()) {
string filename = entry->GetFilePath().GetFilename();
if (ResourceUtil::IsIgnoreFile(filename, entry->IsFile())) {
continue;
}
string subPath = FileEntry::FilePath(dst).Append(filename).GetPath();
if (!entry->IsFile()) {
if (CopyRawFileOrResFileImpl(entry->GetFilePath().GetPath(), subPath) != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
continue;
}
if (!g_resourceSet.emplace(subPath).second) {
cerr << "Warning: '" << entry->GetFilePath().GetPath() << "' is defined repeatedly." << endl;
continue;
}
string path = entry->GetFilePath().GetPath();
auto copyFunc = [this](const string path, string subPath) { return this->CopySingleFile(path, subPath); };
std::future<uint32_t> res = threadPool_.Enqueue(copyFunc, path, subPath);
copyResults_.push_back(std::move(res));
}
return RESTOOL_SUCCESS;
}
uint32_t RawFileResFilePacker::CopySingleFile(const std::string &path, std::string &subPath)
{
if (moduleName_ == "har" || CompressionParser::GetCompressionParser()->GetDefaultCompress()) {
if (!ResourceUtil::CopyFileInner(path, subPath)) {
cerr << "Error: copy rawfile or resfile failed." << NEW_LINE_PATH << path << endl;
return RESTOOL_ERROR;
}
return RESTOOL_SUCCESS;
}
if (!CompressionParser::GetCompressionParser()->CopyAndTranscode(path, subPath, true)) {
cerr << "Error: copy rawfile or resfile, CopyAndTranscode failed." << NEW_LINE_PATH << endl;
return RESTOOL_ERROR;
}
return RESTOOL_SUCCESS;
}
} // namespace Restool
} // namespace Global
} // namespace OHOS
+6 -81
View File
@@ -23,6 +23,7 @@
#include "resource_merge.h"
#include "resource_table.h"
#include "compression_parser.h"
#include "rawfile_resfile_packer.h"
namespace OHOS {
namespace Global {
@@ -251,82 +252,6 @@ uint32_t ResourcePack::GenerateJsHeader(const std::string &headerPath) const
return result;
}
uint32_t ResourcePack::CopyRawFileOrResFile(const string &filePath, const string &fileType)
{
if (!ResourceUtil::FileExist(filePath)) {
return RESTOOL_SUCCESS;
}
if (!FileEntry::IsDirectory(filePath)) {
cerr << "Error: '" << filePath << "' not directory." << endl;
return RESTOOL_ERROR;
}
string dst = FileEntry::FilePath(packageParser_.GetOutput())
.Append(RESOURCES_DIR).Append(fileType).GetPath();
if (CopyRawFileOrResFileImpl(filePath, dst) != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
return RESTOOL_SUCCESS;
}
uint32_t ResourcePack::CopyRawFileOrResFile(const vector<string> &inputs)
{
for (const auto &input : inputs) {
string rawfilePath = FileEntry::FilePath(input).Append(RAW_FILE_DIR).GetPath();
if (CopyRawFileOrResFile(rawfilePath, RAW_FILE_DIR) == RESTOOL_ERROR) {
return RESTOOL_ERROR;
}
string resfilePath = FileEntry::FilePath(input).Append(RES_FILE_DIR).GetPath();
if (CopyRawFileOrResFile(resfilePath, RES_FILE_DIR) == RESTOOL_ERROR) {
return RESTOOL_ERROR;
}
}
return RESTOOL_SUCCESS;
}
uint32_t ResourcePack::CopyRawFileOrResFileImpl(const string &src, const string &dst)
{
if (!ResourceUtil::CreateDirs(dst)) {
return RESTOOL_ERROR;
}
FileEntry f(src);
if (!f.Init()) {
return RESTOOL_ERROR;
}
for (const auto &entry : f.GetChilds()) {
string filename = entry->GetFilePath().GetFilename();
if (ResourceUtil::IsIgnoreFile(filename, entry->IsFile())) {
continue;
}
string subPath = FileEntry::FilePath(dst).Append(filename).GetPath();
if (!entry->IsFile()) {
if (CopyRawFileOrResFileImpl(entry->GetFilePath().GetPath(), subPath) != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
continue;
}
if (!g_resourceSet.emplace(subPath).second) {
cerr << "Warning: '" << entry->GetFilePath().GetPath() << "' is defined repeatedly." << endl;
continue;
}
string path = entry->GetFilePath().GetPath();
if (moduleName_ == "har" || CompressionParser::GetCompressionParser()->GetDefaultCompress()) {
if (!ResourceUtil::CopyFileInner(path, subPath)) {
return RESTOOL_ERROR;
}
continue;
}
if (!CompressionParser::GetCompressionParser()->CopyAndTranscode(path, subPath, true)) {
return RESTOOL_ERROR;
}
}
return RESTOOL_SUCCESS;
}
uint32_t ResourcePack::GenerateConfigJson()
{
if (configJson_.ParseRefence() != RESTOOL_SUCCESS) {
@@ -368,6 +293,9 @@ uint32_t ResourcePack::PackNormal()
return RESTOOL_ERROR;
}
RawFileResFilePacker rawFilePacker(packageParser_, moduleName_);
std::future<uint32_t> copyFuture = rawFilePacker.CopyRawFileOrResFileAsync(resourceMerge.GetInputs());
if (ScanResources(resourceMerge.GetInputs(), packageParser_.GetOutput()) != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
@@ -376,10 +304,6 @@ uint32_t ResourcePack::PackNormal()
return RESTOOL_ERROR;
}
if (CopyRawFileOrResFile(resourceMerge.GetInputs()) != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
if (GenerateConfigJson() != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
@@ -401,7 +325,8 @@ uint32_t ResourcePack::PackNormal()
if (resourceTable.CreateResourceTable() != RESTOOL_SUCCESS) {
return RESTOOL_ERROR;
}
return RESTOOL_SUCCESS;
uint32_t copyRet = copyFuture.get();
return copyRet == RESTOOL_SUCCESS ? RESTOOL_SUCCESS : RESTOOL_ERROR;
}
uint32_t ResourcePack::HandleFeature()
+4
View File
@@ -17,6 +17,7 @@
#include <algorithm>
#include <cstdlib>
#include <fstream>
#include <mutex>
#include <iostream>
#include <iomanip>
#include <regex>
@@ -40,6 +41,8 @@ const map<string, ResourceUtil::IgnoreType> ResourceUtil::IGNORE_FILE_REGEX = {
{ ".+~", IgnoreType::IGNORE_ALL }
};
static std::mutex fileMutex_;
void ResourceUtil::Split(const string &str, vector<string> &out, const string &splitter)
{
string::size_type len = str.size();
@@ -216,6 +219,7 @@ bool ResourceUtil::CopyFileInner(const string &src, const string &dst)
bool ResourceUtil::CreateDirs(const string &filePath)
{
std::lock_guard<std::mutex> lock(fileMutex_);
if (FileExist(filePath)) {
return true;
}
+88
View File
@@ -0,0 +1,88 @@
/*
* Copyright (c) 2021-2024 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "thread_pool.h"
#include "restool_errors.h"
#include <iostream>
#include <string>
namespace OHOS {
namespace Global {
namespace Restool {
using namespace std;
ThreadPool::ThreadPool(size_t threadCount) : threadCount_(threadCount)
{}
uint32_t ThreadPool::Start()
{
if (!workerThreads_.empty() || threadCount_ <= 0) {
cerr << "Error: ThreadPool start failed." << endl;
return RESTOOL_ERROR;
}
running_ = true;
workerThreads_.reserve(threadCount_);
for (size_t i = 0; i < threadCount_; ++i) {
workerThreads_.emplace_back([this] { this->WorkInThread(); });
}
cout << "Info: thread pool is started" << endl;
return RESTOOL_SUCCESS;
}
void ThreadPool::Stop()
{
{
std::unique_lock<std::mutex> lock(queueMutex_);
running_ = false;
}
condition_.notify_all();
for (std::thread &worker : workerThreads_) { worker.join(); }
cout << "Info: thread pool is stopped" << endl;
}
ThreadPool::~ThreadPool()
{
if (running_) {
Stop();
}
}
void ThreadPool::WorkInThread()
{
while (this->running_) {
std::function<void()> task;
{
std::unique_lock<std::mutex> lock(this->queueMutex_);
// wake up when there's a task or when the pool is stopped
this->condition_.wait(lock, [this] { return !this->running_ || !this->tasks_.empty(); });
if (!this->running_) {
// exit thread when the pool is stopped
return;
}
if (!this->tasks_.empty()) {
task = std::move(this->tasks_.front());
this->tasks_.pop();
}
}
if (task) {
task();
}
}
}
} // namespace Restool
} // namespace Global
} // namespace OHOS