Bug 1890817 - Run ipdl.py in distributed r=nika

Slightly restructure the script to be compatible with multiprocessing,
but in a non-intrusive way: guard execution by __main__ and use shared
containers for shared states.

Avoid a redundant parser call (even if cached).

Differential Revision: https://phabricator.services.mozilla.com/D207180
This commit is contained in:
serge-sans-paille 2024-05-03 08:08:07 +00:00
parent 8c60fc5a5c
commit 967c226657
2 changed files with 313 additions and 232 deletions

View File

@ -6,214 +6,291 @@ import os
import sys
from configparser import RawConfigParser
from io import StringIO
from multiprocessing import Manager
import ipdl
from ipdl.ast import SYNC
def log(minv, fmt, *args):
if _verbosity >= minv:
print(fmt % args)
class WorkerPool:
per_process_context = None
def __init__(
self,
manager,
files,
asts,
headersdir,
cppdir,
segmentCapacityDict,
allmessages,
allprotocols,
allmessageprognames,
allsyncmessages,
*,
processes=None
):
if processes is None:
processes = os.cpu_count() or 1
processes = min(processes, 8)
self.n = len(files)
self.asts = asts
self.pool = manager.Pool(
initializer=WorkerPool._init_worker,
initargs=(
files,
asts,
headersdir,
cppdir,
segmentCapacityDict,
allmessages,
allprotocols,
allmessageprognames,
allsyncmessages,
),
processes=processes,
)
def run(self):
self.pool.map(WorkerPool._run_worker, range(self.n))
@staticmethod
def _init_worker(*state):
WorkerPool.per_process_context = state
@staticmethod
def _run_worker(index):
(
files,
asts,
headersdir,
cppdir,
segmentCapacityDict,
allmessages,
allprotocols,
allmessageprognames,
allsyncmessages,
) = WorkerPool.per_process_context
ast = asts[index]
ipdl.gencxx(files[index], ast, headersdir, cppdir, segmentCapacityDict)
if ast.protocol:
allmessages[ast.protocol.name] = ipdl.genmsgenum(ast)
allprotocols.append(ast.protocol.name)
# e.g. PContent::RequestMemoryReport (not prefixed or suffixed.)
for md in ast.protocol.messageDecls:
allmessageprognames.append("%s::%s" % (md.namespace, md.decl.progname))
if md.sendSemantics is SYNC:
allsyncmessages.append(
"%s__%s" % (ast.protocol.name, md.prettyMsgName())
)
# process command line
def main():
def log(minv, fmt, *args):
if _verbosity >= minv:
print(fmt % args)
# process command line
op = optparse.OptionParser(usage="ipdl.py [options] IPDLfiles...")
op.add_option(
"-I",
"--include",
dest="includedirs",
default=[],
action="append",
help="Additional directory to search for included protocol specifications",
)
op.add_option(
"-s",
"--sync-msg-list",
dest="syncMsgList",
default="sync-messages.ini",
help="Config file listing allowed sync messages",
)
op.add_option(
"-m",
"--msg-metadata",
dest="msgMetadata",
default="message-metadata.ini",
help="Predicted message sizes for reducing serialization malloc overhead.",
)
op.add_option(
"-v",
"--verbose",
dest="verbosity",
default=1,
action="count",
help="Verbose logging (specify -vv or -vvv for very verbose logging)",
)
op.add_option(
"-q",
"--quiet",
dest="verbosity",
action="store_const",
const=0,
help="Suppress logging output",
)
op.add_option(
"-d",
"--outheaders-dir",
dest="headersdir",
default=".",
help="""Directory into which C++ headers will be generated.
A protocol Foo in the namespace bar will cause the headers
dir/bar/Foo.h, dir/bar/FooParent.h, and dir/bar/FooParent.h
to be generated""",
)
op.add_option(
"-o",
"--outcpp-dir",
dest="cppdir",
default=".",
help="""Directory into which C++ sources will be generated
A protocol Foo in the namespace bar will cause the sources
cppdir/FooParent.cpp, cppdir/FooChild.cpp
to be generated""",
)
op.add_option(
"-F",
"--file-list",
dest="file_list_file",
default=None,
help="""A file containing IPDL files to parse. This will be
merged with files provided on the commandline.""",
)
op = optparse.OptionParser(usage="ipdl.py [options] IPDLfiles...")
op.add_option(
"-I",
"--include",
dest="includedirs",
default=[],
action="append",
help="Additional directory to search for included protocol specifications",
)
op.add_option(
"-s",
"--sync-msg-list",
dest="syncMsgList",
default="sync-messages.ini",
help="Config file listing allowed sync messages",
)
op.add_option(
"-m",
"--msg-metadata",
dest="msgMetadata",
default="message-metadata.ini",
help="Predicted message sizes for reducing serialization malloc overhead.",
)
op.add_option(
"-v",
"--verbose",
dest="verbosity",
default=1,
action="count",
help="Verbose logging (specify -vv or -vvv for very verbose logging)",
)
op.add_option(
"-q",
"--quiet",
dest="verbosity",
action="store_const",
const=0,
help="Suppress logging output",
)
op.add_option(
"-d",
"--outheaders-dir",
dest="headersdir",
default=".",
help="""Directory into which C++ headers will be generated.
A protocol Foo in the namespace bar will cause the headers
dir/bar/Foo.h, dir/bar/FooParent.h, and dir/bar/FooParent.h
to be generated""",
)
op.add_option(
"-o",
"--outcpp-dir",
dest="cppdir",
default=".",
help="""Directory into which C++ sources will be generated
A protocol Foo in the namespace bar will cause the sources
cppdir/FooParent.cpp, cppdir/FooChild.cpp
to be generated""",
)
op.add_option(
"-F",
"--file-list",
dest="file_list_file",
default=None,
help="""A file containing IPDL files to parse. This will be
merged with files provided on the commandline.""",
)
options, cmdline_files = op.parse_args()
_verbosity = options.verbosity
syncMsgList = options.syncMsgList
msgMetadata = options.msgMetadata
headersdir = options.headersdir
cppdir = options.cppdir
includedirs = [os.path.abspath(incdir) for incdir in options.includedirs]
options, cmdline_files = op.parse_args()
_verbosity = options.verbosity
syncMsgList = options.syncMsgList
headersdir = options.headersdir
cppdir = options.cppdir
includedirs = [os.path.abspath(incdir) for incdir in options.includedirs]
files = []
files = []
if options.file_list_file is not None:
with open(options.file_list_file) as f:
files.extend(f.read().splitlines())
if options.file_list_file is not None:
with open(options.file_list_file) as f:
files.extend(f.read().splitlines())
files.extend(cmdline_files)
files.extend(cmdline_files)
if not len(files):
op.error("No IPDL files specified")
if not len(files):
op.error("No IPDL files specified")
ipcmessagestartpath = os.path.join(headersdir, "IPCMessageStart.h")
ipc_msgtype_name_path = os.path.join(cppdir, "IPCMessageTypeName.cpp")
ipcmessagestartpath = os.path.join(headersdir, "IPCMessageStart.h")
ipc_msgtype_name_path = os.path.join(cppdir, "IPCMessageTypeName.cpp")
log(2, 'Generated C++ headers will be generated relative to "%s"', headersdir)
log(2, 'Generated C++ sources will be generated in "%s"', cppdir)
log(2, 'Generated C++ headers will be generated relative to "%s"', headersdir)
log(2, 'Generated C++ sources will be generated in "%s"', cppdir)
allmessages = {}
allsyncmessages = []
allmessageprognames = []
allprotocols = []
def normalizedFilename(f):
if f == "-":
return "<stdin>"
return f
log(2, "Reading sync message list")
parser = RawConfigParser()
parser.read_file(open(options.syncMsgList))
syncMsgList = parser.sections()
def normalizedFilename(f):
if f == "-":
return "<stdin>"
return f
for section in syncMsgList:
if not parser.get(section, "description"):
print(
"Error: Sync message %s lacks a description" % section, file=sys.stderr
)
sys.exit(1)
# Read message metadata. Right now we only have 'segment_capacity'
# for the standard segment size used for serialization.
log(2, "Reading message metadata...")
msgMetadataConfig = RawConfigParser()
msgMetadataConfig.read_file(open(options.msgMetadata))
log(2, "Reading sync message list")
parser = RawConfigParser()
parser.read_file(open(options.syncMsgList))
syncMsgList = parser.sections()
manager = Manager()
segmentCapacityDict = manager.dict()
allmessages = manager.dict()
allsyncmessages = manager.list()
allmessageprognames = manager.list()
allprotocols = manager.list()
for section in syncMsgList:
if not parser.get(section, "description"):
print("Error: Sync message %s lacks a description" % section, file=sys.stderr)
for msgName in msgMetadataConfig.sections():
if msgMetadataConfig.has_option(msgName, "segment_capacity"):
capacity = msgMetadataConfig.get(msgName, "segment_capacity")
segmentCapacityDict[msgName] = capacity
# First pass: parse and type-check all protocols
asts = []
for f in files:
log(2, os.path.basename(f))
filename = normalizedFilename(f)
if f == "-":
fd = sys.stdin
else:
fd = open(f)
specstring = fd.read()
fd.close()
ast = ipdl.parse(specstring, filename, includedirs=includedirs)
if ast is None:
print("Specification could not be parsed.", file=sys.stderr)
sys.exit(1)
log(2, "checking types")
if not ipdl.typecheck(ast):
print("Specification is not well typed.", file=sys.stderr)
sys.exit(1)
if not ipdl.checkSyncMessage(ast, syncMsgList):
print(
"Error: New sync IPC messages must be reviewed by an IPC peer and recorded in %s"
% options.syncMsgList,
file=sys.stderr,
) # NOQA: E501
sys.exit(1)
asts.append(ast)
if not ipdl.checkFixedSyncMessages(parser):
# Errors have alraedy been printed to stderr, just exit
sys.exit(1)
# Read message metadata. Right now we only have 'segment_capacity'
# for the standard segment size used for serialization.
log(2, "Reading message metadata...")
msgMetadataConfig = RawConfigParser()
msgMetadataConfig.read_file(open(options.msgMetadata))
# Second pass: generate code
pool = WorkerPool(
manager,
files,
asts,
headersdir,
cppdir,
segmentCapacityDict,
allmessages,
allprotocols,
allmessageprognames,
allsyncmessages,
)
pool.run()
segmentCapacityDict = {}
for msgName in msgMetadataConfig.sections():
if msgMetadataConfig.has_option(msgName, "segment_capacity"):
capacity = msgMetadataConfig.get(msgName, "segment_capacity")
segmentCapacityDict[msgName] = capacity
allprotocols.sort()
allsyncmessages.sort()
# First pass: parse and type-check all protocols
for f in files:
log(2, os.path.basename(f))
filename = normalizedFilename(f)
if f == "-":
fd = sys.stdin
else:
fd = open(f)
specstring = fd.read()
fd.close()
ast = ipdl.parse(specstring, filename, includedirs=includedirs)
if ast is None:
print("Specification could not be parsed.", file=sys.stderr)
sys.exit(1)
log(2, "checking types")
if not ipdl.typecheck(ast):
print("Specification is not well typed.", file=sys.stderr)
sys.exit(1)
if not ipdl.checkSyncMessage(ast, syncMsgList):
# Check if we have undefined message names in segmentCapacityDict.
# This is a fool-proof of the 'message-metadata.ini' file.
undefinedMessages = set(segmentCapacityDict.keys()) - set(allmessageprognames)
if len(undefinedMessages) > 0:
print(
"Error: New sync IPC messages must be reviewed by an IPC peer and recorded in %s"
% options.syncMsgList,
file=sys.stderr,
) # NOQA: E501
"Error: Undefined message names in message-metadata.ini:", file=sys.stderr
)
print(undefinedMessages, file=sys.stderr)
sys.exit(1)
if not ipdl.checkFixedSyncMessages(parser):
# Errors have alraedy been printed to stderr, just exit
sys.exit(1)
ipcmsgstart = StringIO()
# Second pass: generate code
for f in files:
# Read from parser cache
filename = normalizedFilename(f)
ast = ipdl.parse(None, filename, includedirs=includedirs)
ipdl.gencxx(filename, ast, headersdir, cppdir, segmentCapacityDict)
if ast.protocol:
allmessages[ast.protocol.name] = ipdl.genmsgenum(ast)
allprotocols.append(ast.protocol.name)
# e.g. PContent::RequestMemoryReport (not prefixed or suffixed.)
for md in ast.protocol.messageDecls:
allmessageprognames.append("%s::%s" % (md.namespace, md.decl.progname))
if md.sendSemantics is SYNC:
allsyncmessages.append(
"%s__%s" % (ast.protocol.name, md.prettyMsgName())
)
allprotocols.sort()
# Check if we have undefined message names in segmentCapacityDict.
# This is a fool-proof of the 'message-metadata.ini' file.
undefinedMessages = set(segmentCapacityDict.keys()) - set(allmessageprognames)
if len(undefinedMessages) > 0:
print("Error: Undefined message names in message-metadata.ini:", file=sys.stderr)
print(undefinedMessages, file=sys.stderr)
sys.exit(1)
ipcmsgstart = StringIO()
print(
"""
print(
"""
// CODE GENERATED by ipdl.py. Do not edit.
#ifndef IPCMessageStart_h
@ -221,14 +298,14 @@ print(
enum IPCMessageStart {
""",
file=ipcmsgstart,
)
file=ipcmsgstart,
)
for name in allprotocols:
print(" %sMsgStart," % name, file=ipcmsgstart)
for name in allprotocols:
print(" %sMsgStart," % name, file=ipcmsgstart)
print(
"""
print(
"""
LastMsgIndex
};
@ -236,12 +313,12 @@ static_assert(LastMsgIndex <= 65536, "need to update IPC_MESSAGE_MACRO");
#endif // ifndef IPCMessageStart_h
""",
file=ipcmsgstart,
)
file=ipcmsgstart,
)
ipc_msgtype_name = StringIO()
print(
"""
ipc_msgtype_name = StringIO()
print(
"""
// CODE GENERATED by ipdl.py. Do not edit.
#include <cstdint>
@ -253,19 +330,19 @@ using std::uint32_t;
namespace {
enum IPCMessages {
""",
file=ipc_msgtype_name,
)
""",
file=ipc_msgtype_name,
)
for protocol in sorted(allmessages.keys()):
for msg, num in allmessages[protocol].idnums:
if num:
print(" %s = %s," % (msg, num), file=ipc_msgtype_name)
elif not msg.endswith("End"):
print(" %s__%s," % (protocol, msg), file=ipc_msgtype_name)
for protocol in sorted(allmessages.keys()):
for msg, num in allmessages[protocol].idnums:
if num:
print(" %s = %s," % (msg, num), file=ipc_msgtype_name)
elif not msg.endswith("End"):
print(" %s__%s," % (protocol, msg), file=ipc_msgtype_name)
print(
"""
print(
"""
};
} // anonymous namespace
@ -276,14 +353,14 @@ bool IPCMessageTypeIsSync(uint32_t aMessageType)
{
switch (aMessageType) {
""",
file=ipc_msgtype_name,
)
file=ipc_msgtype_name,
)
for msg in allsyncmessages:
print(" case %s:" % msg, file=ipc_msgtype_name)
for msg in allsyncmessages:
print(" case %s:" % msg, file=ipc_msgtype_name)
print(
""" return true;
print(
""" return true;
default:
return false;
}
@ -292,24 +369,24 @@ print(
const char* StringFromIPCMessageType(uint32_t aMessageType)
{
switch (aMessageType) {
""",
file=ipc_msgtype_name,
)
""",
file=ipc_msgtype_name,
)
for protocol in sorted(allmessages.keys()):
for msg, num in allmessages[protocol].idnums:
if num or msg.endswith("End"):
continue
print(
"""
for protocol in sorted(allmessages.keys()):
for msg, num in allmessages[protocol].idnums:
if num or msg.endswith("End"):
continue
print(
"""
case %s__%s:
return "%s::%s";"""
% (protocol, msg, protocol, msg),
file=ipc_msgtype_name,
)
% (protocol, msg, protocol, msg),
file=ipc_msgtype_name,
)
print(
"""
print(
"""
case DATA_PIPE_CLOSED_MESSAGE_TYPE:
return "DATA_PIPE_CLOSED_MESSAGE";
case DATA_PIPE_BYTES_CONSUMED_MESSAGE_TYPE:
@ -353,15 +430,15 @@ namespace ipc {
const char* ProtocolIdToName(IPCMessageStart aId) {
switch (aId) {
""",
file=ipc_msgtype_name,
)
file=ipc_msgtype_name,
)
for name in allprotocols:
print(" case %sMsgStart:" % name, file=ipc_msgtype_name)
print(' return "%s";' % name, file=ipc_msgtype_name)
for name in allprotocols:
print(" case %sMsgStart:" % name, file=ipc_msgtype_name)
print(' return "%s";' % name, file=ipc_msgtype_name)
print(
"""
print(
"""
default:
return "<unknown protocol id>";
}
@ -370,8 +447,12 @@ print(
} // namespace ipc
} // namespace mozilla
""",
file=ipc_msgtype_name,
)
file=ipc_msgtype_name,
)
ipdl.writeifmodified(ipcmsgstart.getvalue(), ipcmessagestartpath)
ipdl.writeifmodified(ipc_msgtype_name.getvalue(), ipc_msgtype_name_path)
ipdl.writeifmodified(ipcmsgstart.getvalue(), ipcmessagestartpath)
ipdl.writeifmodified(ipc_msgtype_name.getvalue(), ipc_msgtype_name_path)
if __name__ == "__main__":
main()

View File

@ -85,7 +85,7 @@ def genmsgenum(ast):
def writeifmodified(contents, file):
contents = contents.encode("utf-8")
dir = os.path.dirname(file)
os.path.exists(dir) or os.makedirs(dir)
os.makedirs(dir, exist_ok=True)
oldcontents = None
if os.path.exists(file):