From 748befe904124c59b094cfda2436458a13797535 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Wed, 18 Dec 2019 02:53:34 +0000 Subject: [PATCH] scripts/genbuildplan.py: drop --reorder; cleanup args handling, and other processing. --- config/multithread | 2 +- scripts/genbuildplan.py | 87 +++++++++++++++++------------------------ tools/viewplan | 2 +- 3 files changed, 37 insertions(+), 54 deletions(-) diff --git a/config/multithread b/config/multithread index f9ab7d026d..edea6f4888 100644 --- a/config/multithread +++ b/config/multithread @@ -91,7 +91,7 @@ start_multithread_build() { cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \ ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \ - ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --with-json "${THREAD_CONTROL}"/plan.json \ + ${SCRIPTS}/genbuildplan.py --show-wants --with-json "${THREAD_CONTROL}"/plan.json \ --build ${@} > "${THREAD_CONTROL}"/plan || result=1 if [ ${result} -eq 0 ]; then diff --git a/scripts/genbuildplan.py b/scripts/genbuildplan.py index 722da4567b..84adf8925d 100755 --- a/scripts/genbuildplan.py +++ b/scripts/genbuildplan.py @@ -109,7 +109,7 @@ class Node: self.edges.append(node) def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) + print(*args, file=sys.stderr, **kwargs) # Read a JSON list of all possible packages from stdin def loadPackages(): @@ -178,7 +178,7 @@ def findbuildpos(node, list): return list.index(candidate) + 1 if candidate else -1 # Resolve dependencies for a node -def dep_resolve(node, resolved, unresolved, noreorder): +def dep_resolve(node, resolved, unresolved): unresolved.append(node) for edge in node.edges: @@ -186,19 +186,15 @@ def dep_resolve(node, resolved, unresolved, noreorder): if edge in unresolved: raise Exception('Circular reference detected: %s -> %s\nRemove %s from %s package.mk::PKG_DEPENDS_%s' % \ (node.fqname, edge.commonName(), edge.commonName(), node.name, node.target.upper())) - dep_resolve(edge, resolved, unresolved, noreorder) + dep_resolve(edge, resolved, unresolved) if node not in resolved: - pos = -1 if noreorder else findbuildpos(node, resolved) - if pos != -1: - resolved.insert(pos, node) - else: - resolved.append(node) + resolved.append(node) unresolved.remove(node) # Return a list of build steps for the trigger packages -def get_build_steps(args, nodes, trigger_pkgs, built_pkgs): +def get_build_steps(args, nodes): resolved = [] unresolved = [] @@ -210,12 +206,12 @@ def get_build_steps(args, nodes, trigger_pkgs, built_pkgs): # install = True if "image" in args.build else False - for pkgname in [x for x in trigger_pkgs if x]: + for pkgname in [x for x in args.build if x]: if pkgname.find(":") == -1: pkgname = "%s:target" % pkgname if pkgname in nodes: - dep_resolve(nodes[pkgname], resolved, unresolved, args.no_reorder) + dep_resolve(nodes[pkgname], resolved, unresolved) # Abort if any references remain unresolved if unresolved != []: @@ -226,14 +222,12 @@ def get_build_steps(args, nodes, trigger_pkgs, built_pkgs): # Output list of resolved dependencies for pkg in resolved: - if pkg.fqname not in built_pkgs: - built_pkgs.append(pkg.fqname) - task = "build" if pkg.fqname.endswith(":host") or pkg.fqname.endswith(":init") or not install else "install" - yield(task, pkg.fqname) + task = "build" if pkg.fqname.endswith(":host") or pkg.fqname.endswith(":init") or not install else "install" + yield(task, pkg.fqname) # Reduce the complete list of packages to a map of those packages that will # be needed for the build. -def processPackages(args, packages, build): +def processPackages(args, packages): # Add dummy package to ensure build/install dependencies are not culled pkg = { "name": ROOT_PKG, @@ -241,8 +235,8 @@ def processPackages(args, packages, build): "hierarchy": "global", "bootstrap": "", "init": "", - "host": " ".join(get_packages_by_target("host", build)), - "target": " ".join(get_packages_by_target("target", build)) + "host": " ".join(get_packages_by_target("host", args.build)), + "target": " ".join(get_packages_by_target("target", args.build)) } packages[pkg["name"]] = initPackage(pkg) @@ -277,16 +271,17 @@ def processPackages(args, packages, build): needed_map[pkgname] = pkg # Validate package dependency references - for pkgname in needed_map: - pkg = needed_map[pkgname] - for t in pkg.deps: - for d in pkg.deps[t]: - if split_package(d)[0] not in needed_map and not args.ignore_invalid: - msg = 'Invalid package reference: dependency %s in package %s::PKG_DEPENDS_%s is not valid' % (d, pkgname, t.upper()) - if args.warn_invalid: - eprint("WARNING: %s" % msg) - else: - raise Exception(msg) + if not args.ignore_invalid: + for pkgname in needed_map: + pkg = needed_map[pkgname] + for t in pkg.deps: + for d in pkg.deps[t]: + if split_package(d)[0] not in needed_map: + msg = 'Invalid package reference: dependency %s in package %s::PKG_DEPENDS_%s is not valid' % (d, pkgname, t.upper()) + if args.warn_invalid: + eprint("WARNING: %s" % msg) + else: + raise Exception(msg) node_map = {} @@ -336,24 +331,18 @@ parser = argparse.ArgumentParser(description="Generate package dependency list f parser.add_argument("-b", "--build", nargs="+", metavar="PACKAGE", required=True, \ help="Space-separated list of build trigger packages, either for host or target. Required property - specify at least one package.") -parser.add_argument("--warn-invalid", action="store_true", \ +parser.add_argument("--warn-invalid", action="store_true", default=False, \ help="Warn about invalid/missing dependency packages, perhaps excluded by a PKG_ARCH incompatability. Default is to abort.") -parser.add_argument("--no-reorder", action="store_true", default="True", \ - help="Do not resequence steps based on dependencies. This is the default.") - -parser.add_argument("--reorder", action="store_false", dest="no_reorder", \ - help="Disable --no-reorder and resequence packages to try and reduce stalls etc.") - -parser.add_argument("--show-wants", action="store_true", \ - help="Output \"wants\" dependencies for each step.") - -parser.add_argument("--hide-wants", action="store_false", dest="show_wants", default="True", \ - help="Disable --show-wants.") - -parser.add_argument("--ignore-invalid", action="store_true", \ +parser.add_argument("--ignore-invalid", action="store_true", default=False, \ help="Ignore invalid packages.") +group = parser.add_mutually_exclusive_group() +group.add_argument("--show-wants", action="store_true", \ + help="Output \"wants\" dependencies for each step.") +group.add_argument("--hide-wants", action="store_false", dest="show_wants", default=True, \ + help="Disable --show-wants. This is the default.") + parser.add_argument("--with-json", metavar="FILE", \ help="File into which JSON formatted plan will be written.") @@ -363,14 +352,10 @@ ALL_PACKAGES = loadPackages() loaded = len(ALL_PACKAGES) -REQUIRED_PKGS = processPackages(args, ALL_PACKAGES, args.build) +REQUIRED_PKGS = processPackages(args, ALL_PACKAGES) -# Output list of packages to build/install -built_pkgs = [] -steps = [] - -for step in get_build_steps(args, REQUIRED_PKGS, args.build, built_pkgs): - steps.append(step) +# Identify list of packages to build/install +steps = [step for step in get_build_steps(args, REQUIRED_PKGS)] eprint("Packages loaded : %d" % loaded) eprint("Build trigger(s): %d [%s]" % (len(args.build), " ".join(args.build))) @@ -391,10 +376,8 @@ if args.with_json: # Output build/install steps if args.show_wants: for step in steps: - wants = [] node = (REQUIRED_PKGS[step[1]]) - for e in node.edges: - wants.append(e.fqname) + wants = [edge.fqname for edge in node.edges] print("%-7s %-25s (wants: %s)" % (step[0], step[1].replace(":target",""), ", ".join(wants).replace(":target",""))) else: for step in steps: diff --git a/tools/viewplan b/tools/viewplan index 09fdc889b6..26233ca3ae 100755 --- a/tools/viewplan +++ b/tools/viewplan @@ -25,5 +25,5 @@ set -o pipefail cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \ ${PARALLEL_BIN} --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \ - ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@:-image} --warn-invalid ${GENFLAGS} || \ + ${SCRIPTS}/genbuildplan.py --show-wants --build ${@:-image} --warn-invalid ${GENFLAGS} || \ die "FAILURE: Unable to generate plan"