Bug 1428713 [mozprocess] Add support for Python 3 r=ahal

Differential Revision: https://phabricator.services.mozilla.com/D16552

--HG--
extra : moz-landing-system : lando
This commit is contained in:
AndreiH 2019-01-30 11:19:54 +00:00
parent 7d9ec0823f
commit add8b50260
11 changed files with 133 additions and 115 deletions

View File

@ -10,12 +10,16 @@ import signal
import subprocess
import sys
import threading
import time
import traceback
from Queue import Queue, Empty
from datetime import datetime
import six
import time
if six.PY2:
from Queue import Queue, Empty # Python 2
else:
from queue import Queue, Empty # Python 3
__all__ = ['ProcessHandlerMixin', 'ProcessHandler', 'LogOutput',
'StoreOutput', 'StreamOutput']
@ -23,7 +27,6 @@ __all__ = ['ProcessHandlerMixin', 'ProcessHandler', 'LogOutput',
# Set the MOZPROCESS_DEBUG environment variable to 1 to see some debugging output
MOZPROCESS_DEBUG = os.getenv("MOZPROCESS_DEBUG")
INTERVAL_PROCESS_ALIVE_CHECK = 0.02
# We dont use mozinfo because it is expensive to import, see bug 933558.
@ -33,8 +36,8 @@ isPosix = os.name == "posix" # includes MacOS X
if isWin:
from ctypes import sizeof, addressof, c_ulong, byref, WinError, c_longlong
from . import winprocess
from .qijo import JobObjectAssociateCompletionPortInformation,\
JOBOBJECT_ASSOCIATE_COMPLETION_PORT, JobObjectExtendedLimitInformation,\
from .qijo import JobObjectAssociateCompletionPortInformation, \
JOBOBJECT_ASSOCIATE_COMPLETION_PORT, JobObjectExtendedLimitInformation, \
JOBOBJECT_BASIC_LIMIT_INFORMATION, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, IO_COUNTERS
@ -109,6 +112,7 @@ class ProcessHandlerMixin(object):
# child processes, TODO: Ideally, find a way around this
def setpgidfn():
os.setpgid(0, 0)
preexec_fn = setpgidfn
try:
@ -127,14 +131,15 @@ class ProcessHandlerMixin(object):
thread = threading.current_thread().name
print("DBG::MOZPROC PID:{} ({}) | {}".format(self.pid, thread, msg))
def __del__(self, _maxint=sys.maxint):
def __del__(self):
if isWin:
if six.PY2:
_maxint = sys.maxint
else:
_maxint = sys.maxsize
handle = getattr(self, '_handle', None)
if handle:
if hasattr(self, '_internal_poll'):
self._internal_poll(_deadstate=_maxint)
else:
self.poll(_deadstate=sys.maxint)
self._internal_poll(_deadstate=_maxint)
if handle or self._job or self._io_port:
self._cleanup()
else:
@ -230,8 +235,16 @@ class ProcessHandlerMixin(object):
if isWin:
# Redefine the execute child so that we can track process groups
def _execute_child(self, *args_tuple):
if six.PY3:
(args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session) = args_tuple
# workaround for bug 950894
if sys.hexversion < 0x02070600: # prior to 2.7.6
elif sys.hexversion < 0x02070600: # prior to 2.7.6
(args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines, startupinfo,
creationflags, shell,
@ -246,7 +259,7 @@ class ProcessHandlerMixin(object):
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = args_tuple
if not isinstance(args, basestring):
if not isinstance(args, six.string_types):
args = subprocess.list2cmdline(args)
# Always or in the create new process group
@ -343,11 +356,11 @@ class ProcessHandlerMixin(object):
iocntr = IO_COUNTERS()
jeli = JOBOBJECT_EXTENDED_LIMIT_INFORMATION(
jbli, # basic limit info struct
iocntr, # io_counters (ignored)
0, # process mem limit (ignored)
0, # job mem limit (ignored)
0, # peak process limit (ignored)
0) # peak job limit (ignored)
iocntr, # io_counters (ignored)
0, # process mem limit (ignored)
0, # job mem limit (ignored)
0, # peak process limit (ignored)
0) # peak job limit (ignored)
winprocess.SetInformationJobObject(self._job,
JobObjectExtendedLimitInformation,
@ -489,7 +502,7 @@ falling back to not using job objects for managing child processes""", file=sys.
countdowntokill = datetime.now()
elif pid.value in self._spawned_procs:
# Child Process died remove from list
del(self._spawned_procs[pid.value])
del (self._spawned_procs[pid.value])
elif msgid.value == winprocess.JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS:
# One process existed abnormally
self.debug("process id %s exited abnormally" % pid.value)
@ -589,7 +602,7 @@ falling back to not using job objects for managing child processes""", file=sys.
self._job = None
if getattr(self, '_io_port', None) and \
self._io_port != winprocess.INVALID_HANDLE_VALUE:
self._io_port != winprocess.INVALID_HANDLE_VALUE:
self._io_port.Close()
self._io_port = None
else:
@ -708,6 +721,7 @@ falling back to not using job objects for managing child processes""", file=sys.
self.didOutputTimeout = self.reader.didOutputTimeout
if kill_on_timeout:
self.kill()
onTimeout.insert(0, on_timeout)
self._stderr = subprocess.STDOUT
@ -1081,7 +1095,7 @@ class StreamOutput(object):
def __call__(self, line):
try:
self.stream.write(line + '\n')
self.stream.write(line + '\n'.encode('utf8'))
except UnicodeDecodeError:
# TODO: Workaround for bug #991866 to make sure we can display when
# when normal UTF-8 display is failing

View File

@ -16,9 +16,10 @@ from ctypes import (
c_size_t,
c_ulong
)
from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LARGE_INTEGER
import six
LPVOID = c_void_p
LPDWORD = POINTER(DWORD)
SIZE_T = c_size_t
@ -99,7 +100,7 @@ class JobObjectInfo(object):
}
def __init__(self, _class):
if isinstance(_class, basestring):
if isinstance(_class, six.string_types):
assert _class in self.mapping, \
'Class should be one of %s; you gave %s' % (self.mapping, _class)
_class = self.mapping[_class]
@ -110,12 +111,12 @@ class JobObjectInfo(object):
QueryInformationJobObjectProto = WINFUNCTYPE(
BOOL, # Return type
HANDLE, # hJob
DWORD, # JobObjectInfoClass
LPVOID, # lpJobObjectInfo
DWORD, # cbJobObjectInfoLength
LPDWORD # lpReturnLength
BOOL, # Return type
HANDLE, # hJob
DWORD, # JobObjectInfoClass
LPVOID, # lpJobObjectInfo
DWORD, # cbJobObjectInfoLength
LPDWORD # lpReturnLength
)
QueryInformationJobObjectFlags = (

View File

@ -36,11 +36,11 @@
from __future__ import absolute_import, unicode_literals, print_function
import sys
import subprocess
import sys
from ctypes import c_void_p, POINTER, sizeof, Structure, windll, WinError, WINFUNCTYPE, c_ulong
from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LPCWSTR, LPWSTR, UINT, WORD
from .qijo import QueryInformationJobObject
LPVOID = c_void_p
@ -85,6 +85,7 @@ def ErrCheckHandle(result, func, args):
raise WinError()
return AutoHANDLE(result)
# PROCESS_INFORMATION structure
@ -102,6 +103,7 @@ class PROCESS_INFORMATION(Structure):
LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
# STARTUPINFO structure
@ -141,6 +143,7 @@ STARTF_FORCEONFEEDBACK = 0x40
STARTF_FORCEOFFFEEDBACK = 0x80
STARTF_USESTDHANDLES = 0x100
# EnvironmentBlock
@ -154,7 +157,7 @@ class EnvironmentBlock:
else:
values = []
fs_encoding = sys.getfilesystemencoding() or 'mbcs'
for k, v in env.iteritems():
for k, v in env.items():
if isinstance(k, bytes):
k = k.decode(fs_encoding, 'replace')
if isinstance(v, bytes):
@ -180,16 +183,16 @@ GetLastError = GetLastErrorProto(("GetLastError", windll.kernel32), GetLastError
# CreateProcess()
CreateProcessProto = WINFUNCTYPE(BOOL, # Return type
LPCWSTR, # lpApplicationName
LPWSTR, # lpCommandLine
LPVOID, # lpProcessAttributes
LPVOID, # lpThreadAttributes
BOOL, # bInheritHandles
DWORD, # dwCreationFlags
LPVOID, # lpEnvironment
LPCWSTR, # lpCurrentDirectory
LPSTARTUPINFO, # lpStartupInfo
CreateProcessProto = WINFUNCTYPE(BOOL, # Return type
LPCWSTR, # lpApplicationName
LPWSTR, # lpCommandLine
LPVOID, # lpProcessAttributes
LPVOID, # lpThreadAttributes
BOOL, # bInheritHandles
DWORD, # dwCreationFlags
LPVOID, # lpEnvironment
LPCWSTR, # lpCurrentDirectory
LPSTARTUPINFO, # lpStartupInfo
LPPROCESS_INFORMATION # lpProcessInformation
)
@ -264,9 +267,9 @@ PROCESS_VM_READ = 0x0010
OpenProcessProto = WINFUNCTYPE(
HANDLE, # Return type
DWORD, # dwDesiredAccess
BOOL, # bInheritHandle
DWORD, # dwProcessId
DWORD, # dwDesiredAccess
BOOL, # bInheritHandle
DWORD, # dwProcessId
)
OpenProcessFlags = (
@ -288,13 +291,13 @@ OpenProcess.errcheck = ErrCheckOpenProcess
# GetQueuedCompletionPortStatus -
# http://msdn.microsoft.com/en-us/library/aa364986%28v=vs.85%29.aspx
GetQueuedCompletionStatusProto = WINFUNCTYPE(BOOL, # Return Type
HANDLE, # Completion Port
LPDWORD, # Msg ID
LPULONG, # Completion Key
GetQueuedCompletionStatusProto = WINFUNCTYPE(BOOL, # Return Type
HANDLE, # Completion Port
LPDWORD, # Msg ID
LPULONG, # Completion Key
# PID Returned from the call (may be null)
LPULONG,
DWORD) # milliseconds to wait
DWORD) # milliseconds to wait
GetQueuedCompletionStatusFlags = ((1, "CompletionPort", INVALID_HANDLE_VALUE),
(1, "lpNumberOfBytes", None),
(1, "lpCompletionKey", None),
@ -306,11 +309,11 @@ GetQueuedCompletionStatus = GetQueuedCompletionStatusProto(("GetQueuedCompletion
# CreateIOCompletionPort
# Note that the completion key is just a number, not a pointer.
CreateIoCompletionPortProto = WINFUNCTYPE(HANDLE, # Return Type
HANDLE, # File Handle
HANDLE, # Existing Completion Port
c_ulong, # Completion Key
DWORD) # Number of Threads
CreateIoCompletionPortProto = WINFUNCTYPE(HANDLE, # Return Type
HANDLE, # File Handle
HANDLE, # Existing Completion Port
c_ulong, # Completion Key
DWORD) # Number of Threads
CreateIoCompletionPortFlags = ((1, "FileHandle", INVALID_HANDLE_VALUE),
(1, "ExistingCompletionPort", 0),
@ -322,11 +325,11 @@ CreateIoCompletionPort = CreateIoCompletionPortProto(("CreateIoCompletionPort",
CreateIoCompletionPort.errcheck = ErrCheckHandle
# SetInformationJobObject
SetInformationJobObjectProto = WINFUNCTYPE(BOOL, # Return Type
HANDLE, # Job Handle
DWORD, # Type of Class next param is
LPVOID, # Job Object Class
DWORD) # Job Object Class Length
SetInformationJobObjectProto = WINFUNCTYPE(BOOL, # Return Type
HANDLE, # Job Handle
DWORD, # Type of Class next param is
LPVOID, # Job Object Class
DWORD) # Job Object Class Length
SetInformationJobObjectProtoFlags = ((1, "hJob", None),
(1, "JobObjectInfoClass", None),
@ -338,9 +341,9 @@ SetInformationJobObject = SetInformationJobObjectProto(("SetInformationJobObject
SetInformationJobObject.errcheck = ErrCheckBool
# CreateJobObject()
CreateJobObjectProto = WINFUNCTYPE(HANDLE, # Return type
LPVOID, # lpJobAttributes
LPCWSTR # lpName
CreateJobObjectProto = WINFUNCTYPE(HANDLE, # Return type
LPVOID, # lpJobAttributes
LPCWSTR # lpName
)
CreateJobObjectFlags = ((1, "lpJobAttributes", None),
@ -352,9 +355,9 @@ CreateJobObject.errcheck = ErrCheckHandle
# AssignProcessToJobObject()
AssignProcessToJobObjectProto = WINFUNCTYPE(BOOL, # Return type
HANDLE, # hJob
HANDLE # hProcess
AssignProcessToJobObjectProto = WINFUNCTYPE(BOOL, # Return type
HANDLE, # hJob
HANDLE # hProcess
)
AssignProcessToJobObjectFlags = ((1, "hJob"),
(1, "hProcess"))
@ -365,7 +368,7 @@ AssignProcessToJobObject.errcheck = ErrCheckBool
# GetCurrentProcess()
# because os.getPid() is way too easy
GetCurrentProcessProto = WINFUNCTYPE(HANDLE # Return type
GetCurrentProcessProto = WINFUNCTYPE(HANDLE # Return type
)
GetCurrentProcessFlags = ()
GetCurrentProcess = GetCurrentProcessProto(
@ -375,10 +378,10 @@ GetCurrentProcess.errcheck = ErrCheckHandle
# IsProcessInJob()
try:
IsProcessInJobProto = WINFUNCTYPE(BOOL, # Return type
HANDLE, # Process Handle
HANDLE, # Job Handle
LPBOOL # Result
IsProcessInJobProto = WINFUNCTYPE(BOOL, # Return type
HANDLE, # Process Handle
HANDLE, # Job Handle
LPBOOL # Result
)
IsProcessInJobFlags = ((1, "ProcessHandle"),
(1, "JobHandle", HANDLE(0)),
@ -402,8 +405,8 @@ def ErrCheckResumeThread(result, func, args):
return args
ResumeThreadProto = WINFUNCTYPE(DWORD, # Return type
HANDLE # hThread
ResumeThreadProto = WINFUNCTYPE(DWORD, # Return type
HANDLE # hThread
)
ResumeThreadFlags = ((1, "hThread"),)
ResumeThread = ResumeThreadProto(("ResumeThread", windll.kernel32),
@ -412,9 +415,9 @@ ResumeThread.errcheck = ErrCheckResumeThread
# TerminateProcess()
TerminateProcessProto = WINFUNCTYPE(BOOL, # Return type
TerminateProcessProto = WINFUNCTYPE(BOOL, # Return type
HANDLE, # hProcess
UINT # uExitCode
UINT # uExitCode
)
TerminateProcessFlags = ((1, "hProcess"),
(1, "uExitCode", 127))
@ -425,9 +428,9 @@ TerminateProcess.errcheck = ErrCheckBool
# TerminateJobObject()
TerminateJobObjectProto = WINFUNCTYPE(BOOL, # Return type
TerminateJobObjectProto = WINFUNCTYPE(BOOL, # Return type
HANDLE, # hJob
UINT # uExitCode
UINT # uExitCode
)
TerminateJobObjectFlags = ((1, "hJob"),
(1, "uExitCode", 127))
@ -438,9 +441,9 @@ TerminateJobObject.errcheck = ErrCheckBool
# WaitForSingleObject()
WaitForSingleObjectProto = WINFUNCTYPE(DWORD, # Return type
WaitForSingleObjectProto = WINFUNCTYPE(DWORD, # Return type
HANDLE, # hHandle
DWORD, # dwMilliseconds
DWORD, # dwMilliseconds
)
WaitForSingleObjectFlags = ((1, "hHandle"),
(1, "dwMilliseconds", -1))
@ -462,8 +465,8 @@ ERROR_CONTROL_C_EXIT = 0x23c
# GetExitCodeProcess()
GetExitCodeProcessProto = WINFUNCTYPE(BOOL, # Return type
HANDLE, # hProcess
GetExitCodeProcessProto = WINFUNCTYPE(BOOL, # Return type
HANDLE, # hProcess
LPDWORD, # lpExitCode
)
GetExitCodeProcessFlags = ((1, "hProcess"),
@ -484,6 +487,7 @@ def CanCreateJobObject():
else:
return True
# testing functions

View File

@ -0,0 +1,2 @@
[bdist_wheel]
universal = 1

View File

@ -6,7 +6,7 @@ from __future__ import absolute_import
from setuptools import setup
PACKAGE_VERSION = '0.26'
PACKAGE_VERSION = '1.0.0'
setup(name='mozprocess',
version=PACKAGE_VERSION,
@ -17,7 +17,8 @@ setup(name='mozprocess',
'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
'Natural Language :: English',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.5',
'Topic :: Software Development :: Libraries :: Python Modules',
],
keywords='mozilla',

View File

@ -1,6 +1,7 @@
[DEFAULT]
subsuite = mozbase
skip-if = python == 3
# Python bug https://bugs.python.org/issue32745
skip-if = python == 3 && os == "win" # Bug 1428713 for more info
[test_detached.py]
skip-if = os == "win" # Bug 1493796
[test_kill.py]

View File

@ -4,15 +4,15 @@ from __future__ import absolute_import, print_function
import argparse
import collections
import ConfigParser
import multiprocessing
import time
from six.moves import configparser
ProcessNode = collections.namedtuple('ProcessNode', ['maxtime', 'children'])
class ProcessLauncher(object):
""" Create and Launch process trees specified by a '.ini' file
Typical .ini file accepted by this class :
@ -90,7 +90,7 @@ class ProcessLauncher(object):
# Where each child process is a list of type: [count to run, name of child]
self.children = {}
cfgparser = ConfigParser.ConfigParser()
cfgparser = configparser.ConfigParser()
if not cfgparser.read(manifest):
raise IOError('The manifest %s could not be found/opened', manifest)
@ -100,7 +100,7 @@ class ProcessLauncher(object):
# Maxtime is a mandatory option
# ConfigParser.NoOptionError is raised if maxtime does not exist
if '*' in section or ',' in section:
raise ConfigParser.ParsingError(
raise configparser.ParsingError(
"%s is not a valid section name. "
"Section names cannot contain a '*' or ','." % section)
m_time = cfgparser.get(section, 'maxtime')
@ -130,13 +130,13 @@ class ProcessLauncher(object):
children[i][0] = int(child[0])
if children[i][1] not in sections:
raise ConfigParser.ParsingError(
raise configparser.ParsingError(
'No section corresponding to child %s' % child[1])
except ValueError:
raise ValueError(
'Expected process count to be an integer, specified %s' % child[0])
except ConfigParser.NoOptionError:
except configparser.NoOptionError:
children = None
pn = ProcessNode(maxtime=m_time,
children=children)
@ -193,7 +193,6 @@ class ProcessLauncher(object):
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("manifest", help="Specify the configuration .ini file")
args = parser.parse_args()

View File

@ -6,7 +6,6 @@ import io
import os
import mozunit
import proctest
from mozprocess import processhandler
@ -61,10 +60,10 @@ class ProcTestOutput(proctest.ProcTest):
p.run()
p.wait()
for i in range(5, 10):
stream.write(str(i) + '\n')
stream.write(str(i).encode('utf8') + '\n'.encode('utf8'))
buf.flush()
self.assertEquals(stream.getvalue().strip(), expected)
self.assertEquals(stream.getvalue().strip().decode('utf8'), expected)
# make sure mozprocess doesn't close the stream
# since mozprocess didn't create it

View File

@ -5,16 +5,13 @@ from __future__ import absolute_import
import os
import signal
import sys
import time
import unittest
import mozinfo
import mozunit
from mozprocess import processhandler
import proctest
import time
from mozprocess import processhandler
here = os.path.dirname(os.path.abspath(__file__))
@ -116,7 +113,7 @@ class ProcTestPoll(proctest.ProcTest):
os.kill(p.pid, signal.SIGTERM)
# Allow the output reader thread to finish processing remaining data
for i in xrange(0, 100):
for i in range(0, 100):
time.sleep(processhandler.INTERVAL_PROCESS_ALIVE_CHECK)
returncode = p.poll()
if returncode is not None:

View File

@ -1,11 +1,10 @@
from __future__ import absolute_import
import unittest
import subprocess
import sys
import unittest
import mozunit
from mozprocess.processhandler import ProcessReader, StoreOutput
@ -23,21 +22,23 @@ class TestProcessReader(unittest.TestCase):
def on_finished():
self.finished = True
self.timeout = False
def on_timeout():
self.timeout = True
self.reader = ProcessReader(stdout_callback=self.out,
stderr_callback=self.err,
finished_callback=on_finished,
timeout_callback=on_timeout)
def test_stdout_callback(self):
proc = run_python('print 1; print 2')
proc = run_python('print(1); print(2)')
self.reader.start(proc)
self.reader.thread.join()
self.assertEqual(self.out.output, ['1', '2'])
self.assertEqual([x.decode('utf8') for x in self.out.output], ['1', '2'])
self.assertEqual(self.err.output, [])
def test_stderr_callback(self):
@ -46,15 +47,15 @@ class TestProcessReader(unittest.TestCase):
self.reader.thread.join()
self.assertEqual(self.out.output, [])
self.assertEqual(self.err.output, ['hello world'])
self.assertEqual([x.decode('utf8') for x in self.err.output], ['hello world'])
def test_stdout_and_stderr_callbacks(self):
proc = run_python('import sys; sys.stderr.write("hello world\\n"); print 1; print 2')
proc = run_python('import sys; sys.stderr.write("hello world\\n"); print(1); print(2)')
self.reader.start(proc)
self.reader.thread.join()
self.assertEqual(self.out.output, ['1', '2'])
self.assertEqual(self.err.output, ['hello world'])
self.assertEqual([x.decode('utf8') for x in self.out.output], ['1', '2'])
self.assertEqual([x.decode('utf8') for x in self.err.output], ['hello world'])
def test_finished_callback(self):
self.assertFalse(self.finished)
@ -85,21 +86,22 @@ class TestProcessReader(unittest.TestCase):
proc = run_python('import sys; sys.stdout.write("1")')
self.reader.start(proc)
self.reader.thread.join()
self.assertEqual(self.out.output, ['1'])
self.assertEqual([x.decode('utf8') for x in self.out.output], ['1'])
def test_read_with_strange_eol(self):
proc = run_python('import sys; sys.stdout.write("1\\r\\r\\r\\n")')
self.reader.start(proc)
self.reader.thread.join()
self.assertEqual(self.out.output, ['1'])
self.assertEqual([x.decode('utf8') for x in self.out.output], ['1'])
def test_mixed_stdout_stderr(self):
proc = run_python('import sys; sys.stderr.write("hello world\\n"); print 1; print 2',
proc = run_python('import sys; sys.stderr.write("hello world\\n"); print(1); print(2)',
stderr=subprocess.STDOUT)
self.reader.start(proc)
self.reader.thread.join()
self.assertEqual(sorted(self.out.output), sorted(['1', '2', 'hello world']))
self.assertEqual(sorted([x.decode('utf8') for x in self.out.output]),
sorted(['1', '2', 'hello world']))
self.assertEqual(self.err.output, [])

View File

@ -7,10 +7,8 @@ import signal
import mozinfo
import mozunit
from mozprocess import processhandler
import proctest
from mozprocess import processhandler
here = os.path.dirname(os.path.abspath(__file__))