From 919d78ce803e0f939413ef3fbc8885243ae7a015 Mon Sep 17 00:00:00 2001 From: Rafael Espindola Date: Wed, 17 May 2017 18:20:01 +0000 Subject: [PATCH] Always use the multiprocess module. This seems to work on freebsd and openbsd these days. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@303280 91177308-0d34-0410-b5e6-96231b3b80d8 --- utils/lit/lit/main.py | 15 +----- utils/lit/lit/run.py | 103 +----------------------------------------- 2 files changed, 3 insertions(+), 115 deletions(-) diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py index 10cd7775060..72ba6359a7e 100755 --- a/utils/lit/lit/main.py +++ b/utils/lit/lit/main.py @@ -282,15 +282,6 @@ def main_with_tmp(builtinParameters): debug_group.add_argument("--show-tests", dest="showTests", help="Show all discovered tests", action="store_true", default=False) - debug_group.add_argument("--use-process-pool", dest="executionStrategy", - help="Run tests in parallel with a process pool", - action="store_const", const="PROCESS_POOL") - debug_group.add_argument("--use-processes", dest="executionStrategy", - help="Run tests in parallel with processes (not threads)", - action="store_const", const="PROCESSES") - debug_group.add_argument("--use-threads", dest="executionStrategy", - help="Run tests in parallel with threads (not processes)", - action="store_const", const="THREADS") opts = parser.parse_args() args = opts.test_paths @@ -305,9 +296,6 @@ def main_with_tmp(builtinParameters): if opts.numThreads is None: opts.numThreads = lit.util.detectCPUs() - if opts.executionStrategy is None: - opts.executionStrategy = 'PROCESS_POOL' - if opts.maxFailures == 0: parser.error("Setting --max-failures to 0 does not have any effect.") @@ -490,8 +478,7 @@ def main_with_tmp(builtinParameters): startTime = time.time() display = TestingProgressDisplay(opts, len(run.tests), progressBar) try: - run.execute_tests(display, opts.numThreads, opts.maxTime, - opts.executionStrategy) + run.execute_tests(display, opts.numThreads, opts.maxTime) except KeyboardInterrupt: sys.exit(2) display.finish() diff --git a/utils/lit/lit/run.py b/utils/lit/lit/run.py index 27c7a9e59f8..aa4fdc18b87 100644 --- a/utils/lit/lit/run.py +++ b/utils/lit/lit/run.py @@ -13,11 +13,7 @@ try: except ImportError: win32api = None -try: - import multiprocessing -except ImportError: - multiprocessing = None - +import multiprocessing import lit.Test def abort_now(): @@ -227,8 +223,7 @@ class Run(object): def execute_test(self, test): return execute_test(test, self.lit_config, self.parallelism_semaphores) - def execute_tests(self, display, jobs, max_time=None, - execution_strategy=None): + def execute_tests(self, display, jobs, max_time=None): """ execute_tests(display, jobs, [max_time]) @@ -249,100 +244,6 @@ class Run(object): computed. Tests which were not actually executed (for any reason) will be given an UNRESOLVED result. """ - - if execution_strategy == 'PROCESS_POOL': - self.execute_tests_with_mp_pool(display, jobs, max_time) - return - # FIXME: Standardize on the PROCESS_POOL execution strategy and remove - # the other two strategies. - - use_processes = execution_strategy == 'PROCESSES' - - # Choose the appropriate parallel execution implementation. - consumer = None - if jobs != 1 and use_processes and multiprocessing: - try: - task_impl = multiprocessing.Process - queue_impl = multiprocessing.Queue - sem_impl = multiprocessing.Semaphore - canceled_flag = multiprocessing.Value('i', 0) - consumer = MultiprocessResultsConsumer(self, display, jobs) - except: - # multiprocessing fails to initialize with certain OpenBSD and - # FreeBSD Python versions: http://bugs.python.org/issue3770 - # Unfortunately the error raised also varies by platform. - self.lit_config.note('failed to initialize multiprocessing') - consumer = None - if not consumer: - task_impl = threading.Thread - queue_impl = queue.Queue - sem_impl = threading.Semaphore - canceled_flag = LockedValue(0) - consumer = ThreadResultsConsumer(display) - - self.parallelism_semaphores = {k: sem_impl(v) - for k, v in self.lit_config.parallelism_groups.items()} - - # Create the test provider. - provider = TestProvider(queue_impl, canceled_flag) - handleFailures(provider, consumer, self.lit_config.maxFailures) - - # Putting tasks into the threading or multiprocessing Queue may block, - # so do it in a separate thread. - # https://docs.python.org/2/library/multiprocessing.html - # e.g: On Mac OS X, we will hang if we put 2^15 elements in the queue - # without taking any out. - queuer = task_impl(target=provider.queue_tests, args=(self.tests, jobs)) - queuer.start() - - # Install a console-control signal handler on Windows. - if win32api is not None: - def console_ctrl_handler(type): - provider.cancel() - return True - win32api.SetConsoleCtrlHandler(console_ctrl_handler, True) - - # Install a timeout handler, if requested. - if max_time is not None: - def timeout_handler(): - provider.cancel() - timeout_timer = threading.Timer(max_time, timeout_handler) - timeout_timer.start() - - # If not using multiple tasks, just run the tests directly. - if jobs == 1: - run_one_tester(self, provider, consumer) - else: - # Otherwise, execute the tests in parallel - self._execute_tests_in_parallel(task_impl, provider, consumer, jobs) - - queuer.join() - - # Cancel the timeout handler. - if max_time is not None: - timeout_timer.cancel() - - # Update results for any tests which weren't run. - for test in self.tests: - if test.result is None: - test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0)) - - def _execute_tests_in_parallel(self, task_impl, provider, consumer, jobs): - # Start all of the tasks. - tasks = [task_impl(target=run_one_tester, - args=(self, provider, consumer)) - for i in range(jobs)] - for t in tasks: - t.start() - - # Allow the consumer to handle results, if necessary. - consumer.handle_results() - - # Wait for all the tasks to complete. - for t in tasks: - t.join() - - def execute_tests_with_mp_pool(self, display, jobs, max_time=None): # Don't do anything if we aren't going to run any tests. if not self.tests or jobs == 0: return