diff --git a/BUILD.gn b/BUILD.gn index 6c092b6dce..0ded17f2b9 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -189,6 +189,9 @@ group("ark_runtime_host_unittest") { # quickfix test deps += [ "$js_root/test/quickfix:ark_quickfix_test" ] + if (!ark_standalone_build && run_regress_test) { + deps += [ "$js_root/test/regresstest:ark_regress_test" ] + } # ts aot test and asm test if (!run_with_asan) { diff --git a/ecmascript/builtins/builtins_ark_tools.cpp b/ecmascript/builtins/builtins_ark_tools.cpp index b88b25834d..30ca0e756b 100644 --- a/ecmascript/builtins/builtins_ark_tools.cpp +++ b/ecmascript/builtins/builtins_ark_tools.cpp @@ -332,4 +332,80 @@ JSTaggedValue BuiltinsArkTools::TimeInUs([[maybe_unused]] EcmaRuntimeCallInfo *i ClockScope scope; return JSTaggedValue(scope.GetCurTime()); } -} // namespace panda::ecmascript::builtins +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::PrepareFunctionForOptimization([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter PrepareFunctionForOptimization()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::OptimizeFunctionOnNextCall([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter OptimizeFunctionOnNextCall()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::OptimizeMaglevOnNextCall([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter OptimizeMaglevOnNextCall()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::DeoptimizeFunction([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter DeoptimizeFunction()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::OptimizeOsr([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter OptimizeOsr()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::NeverOptimizeFunction([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter NeverOptimizeFunction()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::HeapObjectVerify([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter HeapObjectVerify()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::DisableOptimizationFinalization([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter DisableOptimizationFinalization()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::DeoptimizeNow([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter DeoptimizeNow()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::WaitForBackgroundOptimization([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter WaitForBackgroundOptimization()"; + return JSTaggedValue::Undefined(); +} + +// empty function for regress-xxx test cases +JSTaggedValue BuiltinsArkTools::Gc([[maybe_unused]] EcmaRuntimeCallInfo *info) +{ + LOG_ECMA(INFO) << "Enter Gc()"; + return JSTaggedValue::Undefined(); +} +} // namespace panda::ecmascript::builtins \ No newline at end of file diff --git a/ecmascript/builtins/builtins_ark_tools.h b/ecmascript/builtins/builtins_ark_tools.h index 762557d746..0fa9e79a2c 100644 --- a/ecmascript/builtins/builtins_ark_tools.h +++ b/ecmascript/builtins/builtins_ark_tools.h @@ -40,6 +40,20 @@ V("removeAOTFlag", RemoveAOTFlag, 1, INVALID) \ V("timeInUs", TimeInUs, 0, INVALID) +#define BUILTIN_ARK_TOOLS_FUNCTIONS_REGRESS(V) \ + V("prepareFunctionForOptimization", PrepareFunctionForOptimization, 1, INVALID) \ + V("optimizeFunctionOnNextCall", OptimizeFunctionOnNextCall, 1, INVALID) \ + V("optimizeMaglevOnNextCall", OptimizeMaglevOnNextCall, 1, INVALID) \ + V("deoptimizeFunction", DeoptimizeFunction, 1, INVALID) \ + V("optimizeOsr", OptimizeOsr, 1, INVALID) \ + V("neverOptimizeFunction", NeverOptimizeFunction, 1, INVALID) \ + V("heapObjectVerify", HeapObjectVerify, 1, INVALID) \ + V("disableOptimizationFinalization", DisableOptimizationFinalization, 0, INVALID) \ + V("deoptimizeNow", DeoptimizeNow, 0, INVALID) \ + V("deoptimize_now", DeoptimizeNow, 0, INVALID) \ + V("waitForBackgroundOptimization", WaitForBackgroundOptimization, 0, INVALID) \ + V("gc", Gc, 0, INVALID) + #ifdef ECMASCRIPT_SUPPORT_CPUPROFILER #define BUILTIN_ARK_TOOLS_FUNCTIONS_CPUPROFILER(V) \ V("startCpuProf", StartCpuProfiler, 0, INVALID) \ @@ -50,7 +64,8 @@ #define BUILTIN_ARK_TOOLS_FUNCTIONS(V) \ BUILTIN_ARK_TOOLS_FUNCTIONS_COMMON(V) \ - BUILTIN_ARK_TOOLS_FUNCTIONS_CPUPROFILER(V) + BUILTIN_ARK_TOOLS_FUNCTIONS_CPUPROFILER(V) \ + BUILTIN_ARK_TOOLS_FUNCTIONS_REGRESS(V) namespace panda::ecmascript::builtins { class BuiltinsArkTools : public base::BuiltinsBase { @@ -98,6 +113,28 @@ public: static JSTaggedValue TimeInUs(EcmaRuntimeCallInfo *info); + static JSTaggedValue PrepareFunctionForOptimization(EcmaRuntimeCallInfo *info); + + static JSTaggedValue OptimizeFunctionOnNextCall(EcmaRuntimeCallInfo *info); + + static JSTaggedValue OptimizeMaglevOnNextCall(EcmaRuntimeCallInfo *info); + + static JSTaggedValue DeoptimizeFunction(EcmaRuntimeCallInfo *info); + + static JSTaggedValue OptimizeOsr(EcmaRuntimeCallInfo *info); + + static JSTaggedValue NeverOptimizeFunction(EcmaRuntimeCallInfo *info); + + static JSTaggedValue HeapObjectVerify(EcmaRuntimeCallInfo *info); + + static JSTaggedValue DisableOptimizationFinalization(EcmaRuntimeCallInfo *info); + + static JSTaggedValue DeoptimizeNow(EcmaRuntimeCallInfo *info); + + static JSTaggedValue WaitForBackgroundOptimization(EcmaRuntimeCallInfo *info); + + static JSTaggedValue Gc(EcmaRuntimeCallInfo *info); + static Span GetArkToolsFunctions() { return Span(ARK_TOOLS_FUNCTIONS); @@ -114,4 +151,4 @@ private: }; } // namespace panda::ecmascript::builtins -#endif // ECMASCRIPT_BUILTINS_BUILTINS_ARK_TOOLS_H +#endif // ECMASCRIPT_BUILTINS_BUILTINS_ARK_TOOLS_H \ No newline at end of file diff --git a/js_runtime_config.gni b/js_runtime_config.gni index 0b82a47b01..9336b89a75 100644 --- a/js_runtime_config.gni +++ b/js_runtime_config.gni @@ -22,6 +22,7 @@ declare_args() { # If true, use icu to implement Intl APIs. # If false, use @ohos.intl APIs to implement Intl APIs. enable_ark_intl = true + run_regress_test = false } if (!ark_standalone_build) { diff --git a/test/regresstest/BUILD.gn b/test/regresstest/BUILD.gn new file mode 100644 index 0000000000..69d177e4d0 --- /dev/null +++ b/test/regresstest/BUILD.gn @@ -0,0 +1,39 @@ +# Copyright (c) 2023 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("../test_helper.gni") + +action("ark_regress_test") { + _host_test_target_ = ":ark_regress_test(${host_toolchain})" + _root_out_dir_ = get_label_info(_host_test_target_, "root_out_dir") + ark_js_runtime_dir = rebase_path("${_root_out_dir_}/arkcompiler/ets_runtime") + icu_dir = rebase_path("${_root_out_dir_}/thirdparty/icu") + llvm_lib_dir = rebase_path("//prebuilts/clang/ohos/linux-x86_64/llvm/lib/") + zlib_dir = rebase_path("${_root_out_dir_}/thirdparty/zlib") + ld_library_path = + "${ark_js_runtime_dir}:${icu_dir}:${llvm_lib_dir}:${zlib_dir}" + script = "$js_root/test/regresstest/run_regress_test.py" + _target_out_dir_ = "$target_out_dir" + args = [ + "--ark-tool", + rebase_path(_root_out_dir_) + "/arkcompiler/ets_runtime/ark_js_vm", + "--ark-frontend-binary", + rebase_path(_root_out_dir_) + "/arkcompiler/ets_frontend/es2abc", + "--LD_LIBRARY_PATH", + ld_library_path, + "--out-dir", + rebase_path(_target_out_dir_), + ] + inputs = [] + outputs = [ "$target_out_dir/regresstest/" ] +} diff --git a/test/regresstest/regress_skip_force_gc.json b/test/regresstest/regress_skip_force_gc.json new file mode 100644 index 0000000000..5617ccfbab --- /dev/null +++ b/test/regresstest/regress_skip_force_gc.json @@ -0,0 +1,236 @@ +[ + { + "reason": "Force GC TimeOut", + "files": [ + "intl/regress-1003748.js", + "intl/regress-10437.js", + "intl/regress-10528.js", + "intl/regress-10529.js", + "intl/regress-10573.js", + "intl/regress-1107661.js", + "intl/regress-1108810.js", + "intl/regress-11295.js", + "intl/regress-1170305.js", + "intl/regress-1347690.js", + "intl/regress-1427932.js", + "intl/regress-1451943.js", + "intl/regress-527926.js", + "intl/regress-7481.js", + "intl/regress-8030.js", + "intl/regress-8031.js", + "intl/regress-8432.js", + "intl/regress-8469.js", + "intl/regress-8604.js", + "intl/regress-9035.js", + "intl/regress-9084.js", + "intl/regress-925216.js", + "intl/regress-930304.js", + "intl/regress-9408.js", + "intl/regress-9475.js", + "intl/regress-9513.js", + "intl/regress-9786.js", + "intl/regress-9812.js", + "intl/regress-9849.js", + "intl/regress-992694.js", + "intl/regress-997401.js", + "mjsunit/compiler/regress-1125145.js", + "mjsunit/compiler/regress-1158049.js", + "mjsunit/compiler/regress-1161357.js", + "mjsunit/compiler/regress-1196185.js", + "mjsunit/compiler/regress-808472.js", + "mjsunit/compiler/regress-890057.js", + "mjsunit/compiler/regress-902608.js", + "mjsunit/compiler/regress-9087.js", + "mjsunit/compiler/regress-958350.js", + "mjsunit/compiler/regress-crbug-1211215.js", + "mjsunit/compiler/regress-crbug-1399627.js", + "mjsunit/compiler/regress-crbug-1463334.js", + "mjsunit/compiler/regress-crbug-1464516.js", + "mjsunit/compiler/regress-serialized-slots.js", + "mjsunit/es6/regress/regress-cr895860.js", + "mjsunit/es9/regress/regress-904167.js", + "mjsunit/harmony/bigint/regress-tostring-2.js", + "mjsunit/maglev/regress-1407606.js", + "mjsunit/maglev/regress-1466928.js", + "mjsunit/maglev/regress/regress-1364074.js", + "mjsunit/maglev/regress/regress-crbug-1394036.js", + "mjsunit/regress/regress-1015.js", + "mjsunit/regress/regress-1105746.js", + "mjsunit/regress/regress-11115.js", + "mjsunit/regress/regress-1139782.js", + "mjsunit/regress/regress-1146013.js", + "mjsunit/regress/regress-115100.js", + "mjsunit/regress/regress-1167.js", + "mjsunit/regress/regress-1200351.js", + "mjsunit/regress/regress-1208805.js", + "mjsunit/regress/regress-1243989.js", + "mjsunit/regress/regress-1376784.js", + "mjsunit/regress/regress-1407349.js", + "mjsunit/regress/regress-1416697.js", + "mjsunit/regress/regress-1432470.js", + "mjsunit/regress/regress-1439135.js", + "mjsunit/regress/regress-1442603.js", + "mjsunit/regress/regress-1449567.js", + "mjsunit/regress/regress-145201.js", + "mjsunit/regress/regress-1455706.js", + "mjsunit/regress/regress-1457747.js", + "mjsunit/regress/regress-1748.js", + "mjsunit/regress/regress-201.js", + "mjsunit/regress/regress-2172.js", + "mjsunit/regress/regress-233.js", + "mjsunit/regress/regress-241344.js", + "mjsunit/regress/regress-2419.js", + "mjsunit/regress/regress-2435.js", + "mjsunit/regress/regress-2646.js", + "mjsunit/regress/regress-3247124.js", + "mjsunit/regress/regress-3252443.js", + "mjsunit/regress/regress-353551.js", + "mjsunit/regress/regress-379770.js", + "mjsunit/regress/regress-455207.js", + "mjsunit/regress/regress-4595.js", + "mjsunit/regress/regress-475705.js", + "mjsunit/regress/regress-484544.js", + "mjsunit/regress/regress-487981.js", + "mjsunit/regress/regress-5181.js", + "mjsunit/regress/regress-5783.js", + "mjsunit/regress/regress-631050.js", + "mjsunit/regress/regress-708247.js", + "mjsunit/regress/regress-732.js", + "mjsunit/regress/regress-740784.js", + "mjsunit/regress/regress-78270.js", + "mjsunit/regress/regress-804096.js", + "mjsunit/regress/regress-821137.js", + "mjsunit/regress/regress-851.js", + "mjsunit/regress/regress-869735.js", + "mjsunit/regress/regress-897366.js", + "mjsunit/regress/regress-897815.js", + "mjsunit/regress/regress-901633.js", + "mjsunit/regress/regress-904275.js", + "mjsunit/regress/regress-91008.js", + "mjsunit/regress/regress-9781.js", + "mjsunit/regress/regress-builtinbust-7.js", + "mjsunit/regress/regress-crbug-100859.js", + "mjsunit/regress/regress-crbug-1163499.js", + "mjsunit/regress/regress-crbug-119926.js", + "mjsunit/regress/regress-crbug-1203122-1.js", + "mjsunit/regress/regress-crbug-1203122-2.js", + "mjsunit/regress/regress-crbug-1335445.js", + "mjsunit/regress/regress-crbug-1374995.js", + "mjsunit/regress/regress-crbug-1414292.js", + "mjsunit/regress/regress-crbug-1420860.js", + "mjsunit/regress/regress-crbug-142087.js", + "mjsunit/regress/regress-crbug-450960.js", + "mjsunit/regress/regress-crbug-451013.js", + "mjsunit/regress/regress-crbug-476477-1.js", + "mjsunit/regress/regress-crbug-616709-1.js", + "mjsunit/regress/regress-crbug-616709-2.js", + "mjsunit/regress/regress-crbug-627934.js", + "mjsunit/regress/regress-crbug-644689-1.js", + "mjsunit/regress/regress-crbug-644689-2.js", + "mjsunit/regress/regress-crbug-664974.js", + "mjsunit/regress/regress-crbug-669411.js", + "mjsunit/regress/regress-crbug-677757.js", + "mjsunit/regress/regress-crbug-685506.js", + "mjsunit/regress/regress-crbug-731193.js", + "mjsunit/regress/regress-crbug-732169.js", + "mjsunit/regress/regress-crbug-737645.js", + "mjsunit/regress/regress-1139782.js", + "mjsunit/regress/regress-453481.js", + "mjsunit/regress/regress-45469.js", + "mjsunit/regress/regress-crbug-1105383.js", + "mjsunit/regress/regress-crbug-1191886.js", + "mjsunit/regress/regress-crbug-1249941.js", + "mjsunit/regress/regress-crbug-1264013.js", + "mjsunit/regress/regress-crbug-134055.js", + "mjsunit/regress/regress-crbug-1424486.js", + "mjsunit/regress/regress-crbug-1443133.js", + "mjsunit/regress/regress-crbug-1448545.js", + "mjsunit/regress/regress-crbug-1455707.js", + "mjsunit/regress/regress-crbug-157019.js", + "mjsunit/regress/regress-crbug-178790.js", + "mjsunit/regress/regress-crbug-181422.js", + "mjsunit/regress/regress-crbug-196583.js", + "mjsunit/regress/regress-crbug-242924.js", + "mjsunit/regress/regress-crbug-347903.js", + "mjsunit/regress/regress-crbug-3867.js", + "mjsunit/regress/regress-crbug-416558.js", + "mjsunit/regress/regress-crbug-469768.js", + "mjsunit/regress/regress-crbug-471659.js", + "mjsunit/regress/regress-crbug-527364.js", + "mjsunit/regress/regress-crbug-554946.js", + "mjsunit/regress/regress-crbug-580584.js", + "mjsunit/regress/regress-crbug-593282.js", + "mjsunit/regress/regress-crbug-595657.js", + "mjsunit/regress/regress-crbug-650404.js", + "mjsunit/regress/regress-crbug-658185.js", + "mjsunit/regress/regress-crbug-664802.js", + "mjsunit/regress/regress-crbug-665793.js", + "mjsunit/regress/regress-crbug-665886.js", + "mjsunit/regress/regress-crbug-668414.js", + "mjsunit/regress/regress-crbug-671576.js", + "mjsunit/regress/regress-crbug-673008.js", + "mjsunit/regress/regress-crbug-679202.js", + "mjsunit/regress/regress-crbug-686427.js", + "mjsunit/regress/regress-crbug-687063.js", + "mjsunit/regress/regress-crbug-696622.js", + "mjsunit/regress/regress-crbug-697017.js", + "mjsunit/regress/regress-crbug-699282.js", + "mjsunit/regress/regress-crbug-700678.js", + "mjsunit/regress/regress-crbug-702058-1.js", + "mjsunit/regress/regress-crbug-702058-2.js", + "mjsunit/regress/regress-crbug-702058-3.js", + "mjsunit/regress/regress-crbug-703610.js", + "mjsunit/regress/regress-crbug-707580.js", + "mjsunit/regress/regress-crbug-708050-1.js", + "mjsunit/regress/regress-crbug-708050-2.js", + "mjsunit/regress/regress-crbug-709537.js", + "mjsunit/regress/regress-crbug-709753.js", + "mjsunit/regress/regress-crbug-715404.js", + "mjsunit/regress/regress-crbug-716804.js", + "mjsunit/regress/regress-crbug-719479.js", + "mjsunit/regress/regress-crbug-722756.js", + "mjsunit/regress/regress-crbug-723132.js", + "mjsunit/regress/regress-crbug-723455.js", + "mjsunit/regress/regress-crbug-724608.js", + "mjsunit/regress/regress-crbug-725201.js", + "mjsunit/regress/regress-crbug-729573-1.js", + "mjsunit/regress/regress-crbug-729573-2.js", + "mjsunit/regress/regress-crbug-734051.js", + "mjsunit/regress/regress-crbug-736575.js", + "mjsunit/regress/regress-crbug-736633.js", + "mjsunit/regress/regress-crbug-738763.js", + "mjsunit/regress/regress-crbug-740116.js", + "mjsunit/regress/regress-crbug-740591.js", + "mjsunit/regress/regress-crbug-740803.js", + "mjsunit/regress/regress-crbug-741078.js", + "mjsunit/regress/regress-crbug-743154.js", + "mjsunit/regress/regress-crbug-747979.js", + "mjsunit/regress/regress-crbug-748539.js", + "mjsunit/regress/regress-crbug-751109.js", + "mjsunit/regress/regress-crbug-751715.js", + "mjsunit/regress/regress-crbug-752481.js", + "mjsunit/regress/regress-crbug-752712.js", + "mjsunit/regress/regress-crbug-752826.js", + "mjsunit/regress/regress-crbug-752846.js", + "mjsunit/regress/regress-crbug-754177.js", + "mjsunit/regress/regress-crbug-756332.js", + "mjsunit/regress/regress-crbug-762472.js", + "mjsunit/regress/regress-crbug-763683.js", + "mjsunit/regress/regress-crbug-768080.js", + "mjsunit/regress/regress-crbug-768367.js", + "mjsunit/regress/regress-crbug-769852.js", + "mjsunit/regress/regress-crbug-771971.js", + "mjsunit/regress/regress-crbug-772610.js", + "mjsunit/regress/regress-crbug-772672.js", + "mjsunit/regress/regress-crbug-772689.js", + "mjsunit/regress/regress-crbug-772720.js", + "mjsunit/regress/regress-crbug-772897.js", + "mjsunit/regress/regress-crbug-774459.js", + "mjsunit/regress/regress-crbug-774994.js", + "mjsunit/regress/regress-crbug-776511.js", + "mjsunit/regress/regress-crbug-778952.js", + "mjsunit/regress/regress-crbug-779367.js", + "mjsunit/regress/regress-crbug-781116-1.js" + ] + } +] diff --git a/test/regresstest/regress_test_config.py b/test/regresstest/regress_test_config.py new file mode 100644 index 0000000000..b38770b1bb --- /dev/null +++ b/test/regresstest/regress_test_config.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Copyright (c) 2023 Huawei Device Co., Ltd. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Description: run regress test case config +""" +import os + + +class RegressTestConfig: + DEFAULT_TIMEOUT = 10 + CURRENT_PATH = os.getcwd() + TEST_TOOL_FILE_DIR = os.path.dirname(__file__) + CODE_ROOT = os.path.abspath(os.path.join(TEST_TOOL_FILE_DIR, "../../../..")) + PROJECT_ROOT = os.path.abspath(os.path.join(TEST_TOOL_FILE_DIR, "../..")) + REGRESS_GIT_REPO = "ark-regress" + TEST_TOOL_FILE_JS_NAME = 'mjsunit.js' + TEST_TOOL_FILE_NAME = 'mjsunit.abc' + REGRESS_GIT_URL = "https://gitee.com/chenjx-huawei/ark-regress.git" + REGRESS_GIT_HASH = "5c3c866d46f3334c1f974f35be2ea078ca1b99d5" + PROJECT_BASE_OUT_DIR = os.path.join(PROJECT_ROOT, "out") + REGRESS_BASE_TEST_DIR = os.path.join(PROJECT_ROOT, "test", "regresstest") + REGRESS_TEST_CASE_DIR = os.path.join(REGRESS_BASE_TEST_DIR, REGRESS_GIT_REPO) + REGRESS_TEST_TOOL_DIR = os.path.join(REGRESS_TEST_CASE_DIR, TEST_TOOL_FILE_JS_NAME) + SKIP_LIST_FILE = os.path.join(REGRESS_BASE_TEST_DIR, "regresstest_skip_tests.json") + FORCE_GC_FILE_LIST = os.path.join(REGRESS_BASE_TEST_DIR, "regress_skip_force_gc.json") + DEFAULT_LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s' + ICU_PATH = f"{CODE_ROOT}/third_party/icu/ohos_icu4j/data" + LLVM_DIR = f"{CODE_ROOT}/prebuilts/clang/ohos/linux-x86_64/llvm/lib/" + DEFAULT_LIBS_DIR = f"{ICU_PATH}:{LLVM_DIR}" diff --git a/test/regresstest/regresstest_skip_tests.json b/test/regresstest/regresstest_skip_tests.json new file mode 100644 index 0000000000..974af784fe --- /dev/null +++ b/test/regresstest/regresstest_skip_tests.json @@ -0,0 +1,36 @@ +[ + { + "reason": "Not support eval()", + "files": [ + ] + }, + { + "reason": "not support d8", + "files": [ + ] + }, + { + "reason": "not support async_hooks", + "files": [ + ] + }, + { + "reason": "not support Worker", + "files": [ + ] + }, + { + "reason": "Not support WebAssembly", + "files": [ + ] + }, + { + "reason": "Not a use case file", + "files": [ + ".gitignore", + "LICENSE", + "README.en.md", + "README.md" + ] + } +] diff --git a/test/regresstest/run_regress_test.py b/test/regresstest/run_regress_test.py new file mode 100755 index 0000000000..177b939373 --- /dev/null +++ b/test/regresstest/run_regress_test.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Copyright (c) 2023 Huawei Device Co., Ltd. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Description: run regress test case +""" +import argparse +import datetime +import json +import logging +import os +import platform +import shutil +import signal +import stat +import subprocess +import sys + +from regress_test_config import RegressTestConfig + + +def init_log_file(args): + logging.basicConfig(filename=args.out_log, format=RegressTestConfig.DEFAULT_LOG_FORMAT, level=logging.INFO) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--test-dir', metavar='DIR', + help='Directory to test ') + parser.add_argument('--test-file', metavar='FILE', + help='File to test') + parser.add_argument('--timeout', default=RegressTestConfig.DEFAULT_TIMEOUT, type=int, + help='Set a custom test timeout in milliseconds !!!\n') + parser.add_argument('--ark-tool', + help="ark's binary tool") + parser.add_argument('--ark-frontend-binary', + help="ark frontend conversion binary tool") + parser.add_argument('--LD_LIBRARY_PATH', + dest='ld_library_path', default=None, help='LD_LIBRARY_PATH') + parser.add_argument('--out-dir', + default=None, help='target out dir') + return parser.parse_args() + + +def check_args(args): + current_dir = os.getcwd() + if args.ark_frontend_binary is None: + print('ark_frontend_binary is required, please add this parameter') + return False + else: + current_frontend_binary = os.path.join(current_dir, args.ark_frontend_binary) + test_tool_frontend_binary = os.path.join(RegressTestConfig.TEST_TOOL_FILE_DIR, args.ark_frontend_binary) + if not os.path.exists(current_frontend_binary) and not os.path.exists(test_tool_frontend_binary): + print('entered ark_frontend_binary does not exist. please confirm') + return False + else: + args.ark_frontend_binary = current_frontend_binary if os.path.exists( + current_frontend_binary) else test_tool_frontend_binary + + if args.ark_tool is None: + print('ark_tool is required, please add this parameter') + return False + else: + current_ark_tool = os.path.join(current_dir, args.ark_tool) + test_tool_ark_tool = os.path.join(RegressTestConfig.TEST_TOOL_FILE_DIR, args.ark_tool) + if not os.path.exists(current_ark_tool) and not os.path.exists(test_tool_ark_tool): + print('entered ark_tool does not exist. please confirm') + return False + else: + args.ark_tool = current_ark_tool if os.path.exists(current_ark_tool) else test_tool_ark_tool + if args.ld_library_path is None: + args.ld_library_path = RegressTestConfig.DEFAULT_LIBS_DIR + if args.out_dir is None: + args.out_dir = RegressTestConfig.PROJECT_BASE_OUT_DIR + else: + args.out_dir = os.path.join(RegressTestConfig.CURRENT_PATH, args.out_dir) + args.regress_out_dir = os.path.join(args.out_dir, "regresstest") + args.out_result = os.path.join(args.regress_out_dir, 'result.txt') + args.out_log = os.path.join(args.regress_out_dir, 'test.log') + args.test_case_out_dir = os.path.join(args.regress_out_dir, RegressTestConfig.REGRESS_GIT_REPO) + return True + + +def get_skip_test_cases(): + skip_tests_list = [] + with os.fdopen(os.open(RegressTestConfig.SKIP_LIST_FILE, os.O_RDONLY, stat.S_IRUSR), "r") as file_object: + json_data = json.load(file_object) + for key in json_data: + skip_tests_list.extend(key["files"]) + return skip_tests_list + + +def remove_dir(path): + if os.path.exists(path): + shutil.rmtree(path) + + +def git_clone(git_url, code_dir): + cmds = ['git', 'clone', git_url, code_dir] + result = True + with subprocess.Popen(cmds) as proc: + ret = proc.wait() + if ret: + print(f"\n error: Cloning '{git_url}' failed.") + result = False + return result + + +def output(msg): + print(str(msg)) + logging.info(str(msg)) + + +def out_put_std(ret_code, cmds, msg): + error_messages = { + 0: msg, + -6: f'{cmds}:{msg}\nAborted (core dumped)', + -4: f'{cmds}:{msg}\nAborted (core dumped)', + -11: f'{cmds}:{msg}\nSegmentation fault (core dumped)' + } + error_message = error_messages.get(ret_code, f'{cmds}: Unknown Error: {str(ret_code)}') + if error_message != '': + output(str(error_message)) + + +def exec_command(cmd_args, timeout=RegressTestConfig.DEFAULT_TIMEOUT): + code_format = 'utf-8' + if platform.system() == "Windows": + code_format = 'gbk' + code = 0 + msg = "" + try: + with subprocess.Popen(cmd_args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True, + start_new_session=True) as process: + cmd_string = " ".join(cmd_args) + output_res, errs = process.communicate(timeout=timeout) + ret_code = process.poll() + if errs.decode(code_format, 'ignore'): + msg += errs.decode(code_format, 'ignore') + if ret_code and ret_code != 1: + code = ret_code + msg += f"Command {cmd_string}:\n error: {errs.decode(code_format, 'ignore')}" + else: + msg += output_res.decode(code_format, 'ignore') + except subprocess.TimeoutExpired: + process.kill() + process.terminate() + os.kill(process.pid, signal.SIGTERM) + code = 1 + msg += f"Timeout: '{cmd_string}' timed out after {timeout} seconds" + except Exception as error: + code = 1 + msg += f"{cmd_string}: unknown error: {error}" + out_put_std(code, cmd_args, msg) + + +class RegressTestPrepare: + def __init__(self, args, skip_test_cases): + self.args = args + self.out_dir = args.regress_out_dir + self.skil_test = skip_test_cases + + def run(self): + self.get_test_case() + self.prepare_clean_data() + test_list = self.get_regress_test_files() + self.gen_test_tool() + self.gen_abc_files(test_list) + + def get_test_case(self): + if not os.path.isdir(os.path.join(RegressTestConfig.REGRESS_TEST_CASE_DIR, '.git')): + git_clone(RegressTestConfig.REGRESS_GIT_URL, RegressTestConfig.REGRESS_TEST_CASE_DIR) + return self.git_checkout(RegressTestConfig.REGRESS_GIT_HASH, RegressTestConfig.REGRESS_TEST_CASE_DIR) + return True + + @staticmethod + def change_extension(path): + base_path, ext = os.path.splitext(path) + if ext: + new_path = base_path + ".abc" + else: + new_path = path + ".abc" + return new_path + + def gen_test_tool(self): + self.gen_abc_files([RegressTestConfig.REGRESS_TEST_TOOL_DIR]) + + def gen_abc_files(self, test_list): + for file_path in test_list: + input_file_path = file_path + start_index = input_file_path.find(RegressTestConfig.REGRESS_GIT_REPO) + if start_index != -1: + test_case_path = input_file_path[start_index + len(RegressTestConfig.REGRESS_GIT_REPO) + 1:] + if test_case_path in self.skil_test: + continue + src_dir = RegressTestConfig.REGRESS_TEST_CASE_DIR + out_dir = self.args.test_case_out_dir + self.mk_dst_dir(input_file_path, src_dir, out_dir) + output_test_case_path = self.change_extension(test_case_path) + output_file = os.path.join(self.args.test_case_out_dir, output_test_case_path) + command = [self.args.ark_frontend_binary, input_file_path, f'--output={output_file}'] + exec_command(command) + + def mk_dst_dir(self, file, src_dir, dist_dir): + idx = file.rfind(src_dir) + fpath, _ = os.path.split(file[idx:]) + fpath = fpath.replace(src_dir, dist_dir) + self.mk_dir(fpath) + + @staticmethod + def mk_dir(path): + if not os.path.exists(path): + os.makedirs(path) + + def get_regress_test_files(self): + result = [] + if self.args.test_file is not None and len(self.args.test_file) > 0: + test_file_list = os.path.join(RegressTestConfig.REGRESS_TEST_CASE_DIR, self.args.test_file) + result.append(test_file_list) + return result + elif self.args.test_dir is not None and len(self.args.test_dir) > 0: + test_file_list = os.path.join(RegressTestConfig.REGRESS_TEST_CASE_DIR, self.args.test_dir) + else: + test_file_list = RegressTestConfig.REGRESS_TEST_CASE_DIR + for dir_path, path, filenames in os.walk(test_file_list): + if dir_path.find(".git") != -1: + continue + for filename in filenames: + result.append(os.path.join(dir_path, filename)) + return result + + def prepare_clean_data(self): + self.git_clean(RegressTestConfig.REGRESS_TEST_CASE_DIR) + self.git_pull(RegressTestConfig.REGRESS_TEST_CASE_DIR) + self.git_checkout(RegressTestConfig.REGRESS_GIT_HASH, RegressTestConfig.REGRESS_TEST_CASE_DIR) + + @staticmethod + def git_checkout(checkout_options, check_out_dir=os.getcwd()): + cmds = ['git', 'checkout', checkout_options] + result = True + with subprocess.Popen(cmds, cwd=check_out_dir) as proc: + ret = proc.wait() + if ret: + print(f"\n error: git checkout '{checkout_options}' failed.") + result = False + return result + + @staticmethod + def git_pull(check_out_dir=os.getcwd()): + cmds = ['git', 'pull', '--rebase'] + with subprocess.Popen(cmds, cwd=check_out_dir) as proc: + proc.wait() + + @staticmethod + def git_clean(clean_dir=os.getcwd()): + cmds = ['git', 'checkout', '--', '.'] + with subprocess.Popen(cmds, cwd=clean_dir) as proc: + proc.wait() + + +def run_regress_test_prepare(args): + remove_dir(args.regress_out_dir) + remove_dir(RegressTestConfig.REGRESS_TEST_CASE_DIR) + os.makedirs(args.regress_out_dir) + init_log_file(args) + skip_tests_case = get_skip_test_cases() + test_prepare = RegressTestPrepare(args, skip_tests_case) + test_prepare.run() + + +def read_expect_file(expect_file, test_case_file): + with os.fdopen(os.open(expect_file, os.O_RDWR, stat.S_IRUSR), "r+") as file_object: + lines = file_object.readlines() + lines = [line for line in lines if not line.strip().startswith('#')] + expect_output = ''.join(lines) + expect_output_str = expect_output.replace('*%(basename)s', test_case_file) + return expect_output_str + + +def open_write_file(file_path, append): + if append: + args = os.O_RDWR | os.O_CREAT | os.O_APPEND + else: + args = os.O_RDWR | os.O_CREAT + file_descriptor = os.open(file_path, args, stat.S_IRUSR | stat.S_IWUSR) + file_object = os.fdopen(file_descriptor, "w+") + return file_object + + +def run_test_case_with_expect(command, test_case_file, expect_file, result_file, timeout): + expect_output_str = read_expect_file(expect_file, test_case_file) + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as process: + try: + out, err = process.communicate(timeout=timeout) + out_str = out.decode('UTF-8') + err_str = err.decode('UTF-8') + ret_code = process.poll() + if ret_code == 0 and (out_str == expect_output_str or err_str == expect_output_str): + write_result_file(f'PASS {test_case_file} \n', result_file) + out_put_std(ret_code, command, f'PASS: {test_case_file}') + return True + else: + msg = f'FAIL: {test_case_file} \nexpect: [{expect_output_str}]\nbut got: [{out_str}]' + out_put_std(ret_code, command, msg) + write_result_file(f'FAIL {test_case_file} \n', result_file) + return False + except subprocess.TimeoutExpired as exception: + process.kill() + process.terminate() + os.kill(process.pid, signal.SIGTERM) + out_put_std(ret_code, command, f'FAIL: {test_case_file},err:{exception}') + write_result_file(f'FAIL {test_case_file} \n', result_file) + return False + + +def run_test_case_with_assert(command, test_case_file, result_file, timeout): + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as process: + try: + out, err = process.communicate(timeout=timeout) + ret_code = process.poll() + if ret_code != 0 or err != b'': + out_put_std(ret_code, command, f'FAIL: {test_case_file} \nerr: {str(err)}') + write_result_file(f'FAIL {test_case_file} \n', result_file) + return False + else: + out_put_std(ret_code, command, f'PASS: {test_case_file}') + write_result_file(f'PASS {test_case_file} \n', result_file) + return True + except subprocess.TimeoutExpired as exception: + process.kill() + process.terminate() + os.kill(process.pid, signal.SIGTERM) + out_put_std(ret_code, command, f'FAIL: {test_case_file},err:{exception}') + write_result_file(f'FAIL {test_case_file} \n', result_file) + return False + + +def run_test_case_file(command, test_case_file, expect_file, result_file, timeout): + expect_file_exits = os.path.exists(expect_file) + if expect_file_exits: + return run_test_case_with_expect(command, test_case_file, expect_file, result_file, timeout) + else: + return run_test_case_with_assert(command, test_case_file, result_file, timeout) + + +def run_test_case_dir(args, test_abc_files, force_gc_files, timeout=RegressTestConfig.DEFAULT_TIMEOUT): + pass_count = 0 + fail_count = 0 + result_file = open_write_file(args.out_result, False) + for index, file in enumerate(test_abc_files): + test_file = file + if test_file.endswith(RegressTestConfig.TEST_TOOL_FILE_NAME): + continue + test_case_file = test_file.replace('abc', 'js').replace(f'{args.out_dir}', '') + assert_file = os.path.join(args.test_case_out_dir, RegressTestConfig.TEST_TOOL_FILE_NAME) + test_case = f'{assert_file}:{file}' + ld_library_path = args.ld_library_path + os.environ["LD_LIBRARY_PATH"] = ld_library_path + unforced_gc = False + force_gc_file = test_case_file.replace('regresstest/ark-regress/', '') + if force_gc_file in force_gc_files: + unforced_gc = True + asm_arg1 = "--enable-force-gc=true" + if unforced_gc: + asm_arg1 = "--enable-force-gc=false" + command = [args.ark_tool, asm_arg1, test_case] + expect_file = test_file.replace('.abc', '.out') + test_res = run_test_case_file(command, test_case_file, expect_file, result_file, timeout) + if test_res: + pass_count += 1 + else: + fail_count += 1 + result_file.close() + return pass_count, fail_count + + +def run_regress_test_case(args): + test_cast_list = get_regress_test_files(args.test_case_out_dir, 'abc') + force_gc_files = get_regress_force_gc_files() + return run_test_case_dir(args, test_cast_list, force_gc_files) + + +def get_regress_force_gc_files(): + skip_force_gc_tests_list = [] + with os.fdopen(os.open(RegressTestConfig.FORCE_GC_FILE_LIST, os.O_RDONLY, stat.S_IRUSR), "r") as file_object: + json_data = json.load(file_object) + for key in json_data: + skip_force_gc_tests_list.extend(key["files"]) + return skip_force_gc_tests_list + + +def get_regress_test_files(start_dir, suffix): + result = [] + for dir_path, dir_names, filenames in os.walk(start_dir): + for filename in filenames: + if filename.endswith(suffix): + result.append(os.path.join(dir_path, filename)) + return result + + +def write_result_file(msg, result_file): + result_file.write(f'{str(msg)}\n') + + +def main(args): + if not check_args(args): + return + print("\nStart conducting regress test cases........\n") + start_time = datetime.datetime.now() + run_regress_test_prepare(args) + pass_count, fail_count = run_regress_test_case(args) + end_time = datetime.datetime.now() + print_result(args, pass_count, fail_count, end_time, start_time) + + +def print_result(args, pass_count, fail_count, end_time, start_time): + result_file = open_write_file(args.out_result, True) + msg = f'\npass count: {pass_count}' + write_result_file(msg, result_file) + print(msg) + msg = f'fail count: {fail_count}' + write_result_file(msg, result_file) + print(msg) + msg = f'total count: {fail_count + pass_count}' + write_result_file(msg, result_file) + print(msg) + msg = f'used time is: {str(end_time - start_time)}' + write_result_file(msg, result_file) + print(msg) + result_file.close() + + +if __name__ == "__main__": + sys.exit(main(parse_args()))