From 0ff2536f46afa5ce27d3040f25a9c9ef4b072346 Mon Sep 17 00:00:00 2001 From: liangxinyan Date: Sat, 25 May 2024 09:35:03 +0800 Subject: [PATCH] IssueNo: https://gitee.com/openharmony/third_party_ninja/issues/I9RW2O Signed-off-by: liangxinyan --- src/build.cc | 165 ++++--- src/build.h | 492 ++++++++++----------- src/build_log.cc | 992 +++++++++++++++++++++--------------------- src/depfile_parser.cc | 623 +++++++++++++------------- src/disk_interface.h | 210 ++++----- src/graph.h | 756 ++++++++++++++++---------------- src/status.cc | 57 ++- 7 files changed, 1646 insertions(+), 1649 deletions(-) diff --git a/src/build.cc b/src/build.cc index fb32ad9..1b504a1 100755 --- a/src/build.cc +++ b/src/build.cc @@ -52,8 +52,8 @@ struct DryRunCommandRunner : public CommandRunner { virtual bool StartCommand(Edge* edge); virtual bool WaitForCommand(Result* result); - private: - queue finished_; + private: + queue finished_; }; size_t DryRunCommandRunner::CanRunMore() const { @@ -66,13 +66,13 @@ bool DryRunCommandRunner::StartCommand(Edge* edge) { } bool DryRunCommandRunner::WaitForCommand(Result* result) { - if (finished_.empty()) - return false; + if (finished_.empty()) + return false; - result->status = ExitSuccess; - result->edge = finished_.front(); - finished_.pop(); - return true; + result->status = ExitSuccess; + result->edge = finished_.front(); + finished_.pop(); + return true; } } // namespace @@ -98,19 +98,19 @@ bool Plan::AddSubTarget(const Node* node, const Node* dependent, string* err, set* dyndep_walk) { Edge* edge = node->in_edge(); if (!edge) { - // Leaf node, this can be either a regular input from the manifest - // (e.g. a source file), or an implicit input from a depfile or dyndep - // file. In the first case, a dirty flag means the file is missing, - // and the build should stop. In the second, do not do anything here - // since there is no producing edge to add to the plan. - if (node->dirty() && !node->generated_by_dep_loader()) { - string referenced; - if (dependent) - referenced = ", needed by '" + dependent->path() + "',"; - *err = "'" + node->path() + "'" + referenced + - " missing and no known rule to make it"; - } - return false; + // Leaf node, this can be either a regular input from the manifest + // (e.g. a source file), or an implicit input from a depfile or dyndep + // file. In the first case, a dirty flag means the file is missing, + // and the build should stop. In the second, do not do anything here + // since there is no producing edge to add to the plan. + if (node->dirty() && !node->generated_by_dep_loader()) { + string referenced; + if (dependent) + referenced = ", needed by '" + dependent->path() + "',"; + *err = "'" + node->path() + "'" + referenced + + " missing and no known rule to make it"; + } + return false; } if (edge->outputs_ready()) @@ -141,7 +141,7 @@ bool Plan::AddSubTarget(const Node* node, const Node* dependent, string* err, return true; // We've already processed the inputs. for (vector::iterator i = edge->inputs_.begin(); - i != edge->inputs_.end(); ++i) { + i != edge->inputs_.end(); ++i) { if (!AddSubTarget(*i, node, err, dyndep_walk) && !err->empty()) return false; } @@ -211,7 +211,7 @@ bool Plan::EdgeFinished(Edge* edge, EdgeResult result, string* err) { // Check off any nodes we were waiting for with this edge. for (vector::iterator o = edge->outputs_.begin(); - o != edge->outputs_.end(); ++o) { + o != edge->outputs_.end(); ++o) { if (!NodeFinished(*o, err)) return false; } @@ -229,7 +229,7 @@ bool Plan::NodeFinished(Node* node, string* err) { // See if we we want any edges from this node. for (vector::const_iterator oe = node->out_edges().begin(); - oe != node->out_edges().end(); ++oe) { + oe != node->out_edges().end(); ++oe) { map::iterator want_e = want_.find(*oe); if (want_e == want_.end()) continue; @@ -260,7 +260,7 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { node->set_dirty(false); for (vector::const_iterator oe = node->out_edges().begin(); - oe != node->out_edges().end(); ++oe) { + oe != node->out_edges().end(); ++oe) { // Don't process edges that we don't actually want. map::iterator want_e = want_.find(*oe); if (want_e == want_.end() || want_e->second == kWantNothing) @@ -273,8 +273,8 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { // If all non-order-only inputs for this edge are now clean, // we might have changed the dirty state of the outputs. vector::iterator - begin = (*oe)->inputs_.begin(), - end = (*oe)->inputs_.end() - (*oe)->order_only_deps_; + begin = (*oe)->inputs_.begin(), + end = (*oe)->inputs_.end() - (*oe)->order_only_deps_; #if __cplusplus < 201703L #define MEM_FN mem_fun #else @@ -293,12 +293,12 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { // wanted. bool outputs_dirty = false; if (!scan->RecomputeOutputsDirty(*oe, most_recent_input, - &outputs_dirty, err)) { + &outputs_dirty, err)) { return false; } if (!outputs_dirty) { for (vector::iterator o = (*oe)->outputs_.begin(); - o != (*oe)->outputs_.end(); ++o) { + o != (*oe)->outputs_.end(); ++o) { if (!CleanNode(scan, *o, err)) return false; } @@ -354,7 +354,7 @@ bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node, oei = dyndep_roots.begin(); oei != dyndep_roots.end(); ++oei) { DyndepFile::const_iterator oe = *oei; for (vector::const_iterator i = oe->second.implicit_inputs_.begin(); - i != oe->second.implicit_inputs_.end(); ++i) { + i != oe->second.implicit_inputs_.end(); ++i) { if (!AddSubTarget(*i, oe->first->outputs_[0], err, &dyndep_walk) && !err->empty()) return false; @@ -364,7 +364,7 @@ bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node, // Add out edges from this node that are in the plan (just as // Plan::NodeFinished would have without taking the dyndep code path). for (vector::const_iterator oe = node->out_edges().begin(); - oe != node->out_edges().end(); ++oe) { + oe != node->out_edges().end(); ++oe) { map::iterator want_e = want_.find(*oe); if (want_e == want_.end()) continue; @@ -373,7 +373,7 @@ bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node, // See if any encountered edges are now ready. for (set::iterator wi = dyndep_walk.begin(); - wi != dyndep_walk.end(); ++wi) { + wi != dyndep_walk.end(); ++wi) { map::iterator want_e = want_.find(*wi); if (want_e == want_.end()) continue; @@ -394,7 +394,7 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node, // Update the dirty state of all dependents and check if their edges // have become wanted. for (set::iterator i = dependents.begin(); - i != dependents.end(); ++i) { + i != dependents.end(); ++i) { Node* n = *i; // Check if this dependent node is now dirty. Also checks for new cycles. @@ -405,7 +405,7 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node, // Add any validation nodes found during RecomputeDirty as new top level // targets. for (std::vector::iterator v = validation_nodes.begin(); - v != validation_nodes.end(); ++v) { + v != validation_nodes.end(); ++v) { if (Edge* in_edge = (*v)->in_edge()) { if (!in_edge->outputs_ready() && !AddTarget(*v, err)) { @@ -433,7 +433,7 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node, void Plan::UnmarkDependents(const Node* node, set* dependents) { for (vector::const_iterator oe = node->out_edges().begin(); - oe != node->out_edges().end(); ++oe) { + oe != node->out_edges().end(); ++oe) { Edge* edge = *oe; map::iterator want_e = want_.find(edge); @@ -443,7 +443,7 @@ void Plan::UnmarkDependents(const Node* node, set* dependents) { if (edge->mark_ != Edge::VisitNone) { edge->mark_ = Edge::VisitNone; for (vector::iterator o = edge->outputs_.begin(); - o != edge->outputs_.end(); ++o) { + o != edge->outputs_.end(); ++o) { if (dependents->insert(*o).second) UnmarkDependents(*o, dependents); } @@ -478,7 +478,7 @@ struct RealCommandRunner : public CommandRunner { vector RealCommandRunner::GetActiveEdges() { vector edges; for (map::iterator e = subproc_to_edge_.begin(); - e != subproc_to_edge_.end(); ++e) + e != subproc_to_edge_.end(); ++e) edges.push_back(e->second); return edges; } @@ -489,7 +489,7 @@ void RealCommandRunner::Abort() { size_t RealCommandRunner::CanRunMore() const { size_t subproc_number = - subprocs_.running_.size() + subprocs_.finished_.size(); + subprocs_.running_.size() + subprocs_.finished_.size(); int64_t capacity = config_.parallelism - subproc_number; @@ -545,7 +545,7 @@ Builder::Builder(State* state, const BuildConfig& config, : state_(state), config_(config), plan_(this), status_(status), start_time_millis_(start_time_millis), disk_interface_(disk_interface), scan_(state, build_log, deps_log, disk_interface, - &config_.depfile_parser_options) { + &config_.depfile_parser_options) { lock_file_path_ = ".ninja_lock"; string build_dir = state_->bindings_.LookupVariable("builddir"); if (!build_dir.empty()) @@ -562,10 +562,10 @@ void Builder::Cleanup() { command_runner_->Abort(); for (vector::iterator e = active_edges.begin(); - e != active_edges.end(); ++e) { + e != active_edges.end(); ++e) { string depfile = (*e)->GetUnescapedDepfile(); for (vector::iterator o = (*e)->outputs_.begin(); - o != (*e)->outputs_.end(); ++o) { + o != (*e)->outputs_.end(); ++o) { // Only delete this output if it was actually modified. This is // important for things like the generator where we don't want to // delete the manifest file if we can avoid it. But if the rule @@ -616,10 +616,10 @@ bool Builder::AddTarget(Node* target, string* err) { // Also add any validation nodes found during RecomputeDirty as top level // targets. for (std::vector::iterator n = validation_nodes.begin(); - n != validation_nodes.end(); ++n) { + n != validation_nodes.end(); ++n) { if (Edge* validation_in_edge = (*n)->in_edge()) { if (!validation_in_edge->outputs_ready() && - !plan_.AddTarget(*n, err)) { + !plan_.AddTarget(*n, err)) { return false; } } @@ -630,32 +630,32 @@ bool Builder::AddTarget(Node* target, string* err) { static std::string &Trim(std::string &s) { - if (s.empty()) { - return s; - } - s.erase(0, s.find_first_not_of(" \t\r\n")); - s.erase(s.find_last_not_of(" \t\r\n") + 1); + if (s.empty()) { return s; + } + s.erase(0, s.find_first_not_of(" \t\r\n")); + s.erase(s.find_last_not_of(" \t\r\n") + 1); + return s; } static std::vector SplitStringBySpace(std::string content) { - std::string space_delimiter = " "; - std::vector words{}; - size_t pos = 0; - std::string temp_content = content; - while ((pos = temp_content.find(space_delimiter)) != string::npos) { - std::string sub_str = temp_content.substr(0, pos); - std::string tmp = Trim(sub_str); - if (!tmp.empty()) { - words.push_back(tmp); - } - temp_content = temp_content.substr(pos + 1); + std::string space_delimiter = " "; + std::vector words{}; + size_t pos = 0; + std::string temp_content = content; + while ((pos = temp_content.find(space_delimiter)) != string::npos) { + std::string sub_str = temp_content.substr(0, pos); + std::string tmp = Trim(sub_str); + if (!tmp.empty()) { + words.push_back(tmp); } - std::string tmp_last = Trim(temp_content); - if (!tmp_last.empty()) { - words.push_back(tmp_last); - } - return words; + temp_content = temp_content.substr(pos + 1); + } + std::string tmp_last = Trim(temp_content); + if (!tmp_last.empty()) { + words.push_back(tmp_last); + } + return words; } static std::string SplicingWholeContent(std::string content, std::string whole_content, bool is_whole_archive) { @@ -665,19 +665,19 @@ static std::string SplicingWholeContent(std::string content, std::string whole_c std::string temp_content = content; if (is_whole_archive) { - temp_content += " -Wl,--whole-archive"; + temp_content += " -Wl,--whole-archive"; } std::vector whole_list = SplitStringBySpace(whole_content); std::vector content_list = SplitStringBySpace(temp_content); for (const std::string &word : whole_list) { - auto it = std::find_if(content_list.begin(), content_list.end(), [&](const std::string& s) { - return s.find(word) != std::string::npos; - }); - if (it != content_list.end()) { - content_list.push_back(*it); - content_list.erase(it); - } + auto it = std::find_if(content_list.begin(), content_list.end(), [&](const std::string& s) { + return s.find(word) != std::string::npos; + }); + if (it != content_list.end()) { + content_list.push_back(*it); + content_list.erase(it); + } } std::string result = ""; @@ -766,9 +766,9 @@ bool Builder::Build(string* err) { } } - // We are finished with all work items and have no pending - // commands. Therefore, break out of the main loop. - if (pending_commands == 0 && !plan_.more_to_do()) break; + // We are finished with all work items and have no pending + // commands. Therefore, break out of the main loop. + if (pending_commands == 0 && !plan_.more_to_do()) break; } // See if we can reap any finished commands. @@ -833,7 +833,7 @@ bool Builder::StartEdge(Edge* edge, string* err) { // filesystem mtime to record later // XXX: this will block; do we care? for (vector::iterator o = edge->outputs_.begin(); - o != edge->outputs_.end(); ++o) { + o != edge->outputs_.end(); ++o) { if (!disk_interface_->MakeDirs((*o)->path())) return false; if (build_start == -1) { @@ -880,8 +880,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { if (!deps_type.empty()) { string extract_err; if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes, - &extract_err) && - result->success()) { + &extract_err) && result->success()) { if (!result->output.empty()) result->output.append("\n"); result->output.append(extract_err); @@ -918,7 +917,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { // log. if (record_mtime == 0 || restat || generator) { for (vector::iterator o = edge->outputs_.begin(); - o != edge->outputs_.end(); ++o) { + o != edge->outputs_.end(); ++o) { TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err); if (new_mtime == -1) return false; @@ -949,7 +948,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { if (scan_.build_log()) { if (!scan_.build_log()->RecordCommand(edge, start_time_millis, - end_time_millis, record_mtime)) { + end_time_millis, record_mtime)) { *err = string("Error writing to build log: ") + strerror(errno); return false; } @@ -958,7 +957,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { if (!deps_type.empty() && !config_.dry_run) { assert(!edge->outputs_.empty() && "should have been rejected by parser"); for (std::vector::const_iterator o = edge->outputs_.begin(); - o != edge->outputs_.end(); ++o) { + o != edge->outputs_.end(); ++o) { TimeStamp deps_mtime = disk_interface_->Stat((*o)->path(), err); if (deps_mtime == -1) return false; @@ -983,7 +982,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, return false; result->output = output; for (set::iterator i = parser.includes_.begin(); - i != parser.includes_.end(); ++i) { + i != parser.includes_.end(); ++i) { // ~0 is assuming that with MSVC-parsed headers, it's ok to always make // all backslashes (as some of the slashes will certainly be backslashes // anyway). This could be fixed if necessary with some additional @@ -1018,7 +1017,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, // XXX check depfile matches expected output. deps_nodes->reserve(deps.ins_.size()); for (vector::iterator i = deps.ins_.begin(); - i != deps.ins_.end(); ++i) { + i != deps.ins_.end(); ++i) { uint64_t slash_bits; CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits); deps_nodes->push_back(state_->GetNode(*i, slash_bits)); diff --git a/src/build.h b/src/build.h index d5d0b8b..248acd8 100755 --- a/src/build.h +++ b/src/build.h @@ -1,246 +1,246 @@ -// Copyright 2011 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef NINJA_BUILD_H_ -#define NINJA_BUILD_H_ - -#include -#include -#include -#include -#include -#include - -#include "depfile_parser.h" -#include "graph.h" // XXX needed for DependencyScan; should rearrange. -#include "graph.h" -#include "exit_status.h" -#include "util.h" // int64_t - -struct BuildLog; -struct Builder; -struct DiskInterface; -struct Edge; -struct Node; -struct State; -struct Status; - -/// Plan stores the state of a build plan: what we intend to build, -/// which steps we're ready to execute. -struct Plan { - Plan(Builder* builder = NULL); - - /// Add a target to our plan (including all its dependencies). - /// Returns false if we don't need to build this target; may - /// fill in |err| with an error message if there's a problem. - bool AddTarget(const Node* target, std::string* err); - - // Pop a ready edge off the queue of edges to build. - // Returns NULL if there's no work to do. - Edge* FindWork(); - - /// Returns true if there's more work to be done. - bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; } - - /// Dumps the current state of the plan. - void Dump() const; - - enum EdgeResult { - kEdgeFailed, - kEdgeSucceeded - }; - - /// Mark an edge as done building (whether it succeeded or failed). - /// If any of the edge's outputs are dyndep bindings of their dependents, - /// this loads dynamic dependencies from the nodes' paths. - /// Returns 'false' if loading dyndep info fails and 'true' otherwise. - bool EdgeFinished(Edge* edge, EdgeResult result, std::string* err); - - /// Clean the given node during the build. - /// Return false on error. - bool CleanNode(DependencyScan* scan, Node* node, std::string* err); - - /// Number of edges with commands to run. - int command_edge_count() const { return command_edges_; } - - /// Reset state. Clears want and ready sets. - void Reset(); - - /// Update the build plan to account for modifications made to the graph - /// by information loaded from a dyndep file. - bool DyndepsLoaded(DependencyScan* scan, const Node* node, - const DyndepFile& ddf, std::string* err); - -private: - bool RefreshDyndepDependents(DependencyScan* scan, const Node* node, std::string* err); - void UnmarkDependents(const Node* node, std::set* dependents); - bool AddSubTarget(const Node* node, const Node* dependent, std::string* err, - std::set* dyndep_walk); - - /// Update plan with knowledge that the given node is up to date. - /// If the node is a dyndep binding on any of its dependents, this - /// loads dynamic dependencies from the node's path. - /// Returns 'false' if loading dyndep info fails and 'true' otherwise. - bool NodeFinished(Node* node, std::string* err); - - /// Enumerate possible steps we want for an edge. - enum Want - { - /// We do not want to build the edge, but we might want to build one of - /// its dependents. - kWantNothing, - /// We want to build the edge, but have not yet scheduled it. - kWantToStart, - /// We want to build the edge, have scheduled it, and are waiting - /// for it to complete. - kWantToFinish - }; - - void EdgeWanted(const Edge* edge); - bool EdgeMaybeReady(std::map::iterator want_e, std::string* err); - - /// Submits a ready edge as a candidate for execution. - /// The edge may be delayed from running, for example if it's a member of a - /// currently-full pool. - void ScheduleWork(std::map::iterator want_e); - - /// Keep track of which edges we want to build in this plan. If this map does - /// not contain an entry for an edge, we do not want to build the entry or its - /// dependents. If it does contain an entry, the enumeration indicates what - /// we want for the edge. - std::map want_; - - EdgeSet ready_; - - Builder* builder_; - - /// Total number of edges that have commands (not phony). - int command_edges_; - - /// Total remaining number of wanted edges. - int wanted_edges_; -}; - -/// CommandRunner is an interface that wraps running the build -/// subcommands. This allows tests to abstract out running commands. -/// RealCommandRunner is an implementation that actually runs commands. -struct CommandRunner { - virtual ~CommandRunner() {} - virtual size_t CanRunMore() const = 0; - virtual bool StartCommand(Edge* edge) = 0; - - /// The result of waiting for a command. - struct Result { - Result() : edge(NULL) {} - Edge* edge; - ExitStatus status; - std::string output; - bool success() const { return status == ExitSuccess; } - }; - /// Wait for a command to complete, or return false if interrupted. - virtual bool WaitForCommand(Result* result) = 0; - - virtual std::vector GetActiveEdges() { return std::vector(); } - virtual void Abort() {} -}; - -/// Options (e.g. verbosity, parallelism) passed to a build. -struct BuildConfig { - BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), - failures_allowed(1), max_load_average(-0.0f) {} - - enum Verbosity { - QUIET, // No output -- used when testing. - NO_STATUS_UPDATE, // just regular output but suppress status update - NORMAL, // regular output and status update - VERBOSE - }; - Verbosity verbosity; - bool dry_run; - int parallelism; - int failures_allowed; - /// The maximum load average we must not exceed. A negative value - /// means that we do not have any limit. - double max_load_average; - DepfileParserOptions depfile_parser_options; -}; - -/// Builder wraps the build process: starting commands, updating status. -struct Builder { - Builder(State* state, const BuildConfig& config, - BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface, Status* status, - int64_t start_time_millis); - ~Builder(); - - /// Clean up after interrupted commands by deleting output files. - void Cleanup(); - - Node* AddTarget(const std::string& name, std::string* err); - - /// Add a target to the build, scanning dependencies. - /// @return false on error. - bool AddTarget(Node* target, std::string* err); - - /// Returns true if the build targets are already up to date. - bool AlreadyUpToDate() const; - - /// Run the build. Returns false on error. - /// It is an error to call this function when AlreadyUpToDate() is true. - bool Build(std::string* err); - - bool StartEdge(Edge* edge, std::string* err); - - std::string GetContent(Edge* edge); - - /// Update status ninja logs following a command termination. - /// @return false if the build can not proceed further due to a fatal error. - bool FinishCommand(CommandRunner::Result* result, std::string* err); - - /// Used for tests. - void SetBuildLog(BuildLog* log) { - scan_.set_build_log(log); - } - - /// Load the dyndep information provided by the given node. - bool LoadDyndeps(Node* node, std::string* err); - - State* state_; - const BuildConfig& config_; - Plan plan_; - std::unique_ptr command_runner_; - Status* status_; - - private: - bool ExtractDeps(CommandRunner::Result* result, const std::string& deps_type, - const std::string& deps_prefix, - std::vector* deps_nodes, std::string* err); - - /// Map of running edge to time the edge started running. - typedef std::map RunningEdgeMap; - RunningEdgeMap running_edges_; - - /// Time the build started. - int64_t start_time_millis_; - - std::string lock_file_path_; - DiskInterface* disk_interface_; - DependencyScan scan_; - - // Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr. - Builder(const Builder &other); // DO NOT IMPLEMENT - void operator=(const Builder &other); // DO NOT IMPLEMENT -}; - -#endif // NINJA_BUILD_H_ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_BUILD_H_ +#define NINJA_BUILD_H_ + +#include +#include +#include +#include +#include +#include + +#include "depfile_parser.h" +#include "graph.h" // XXX needed for DependencyScan; should rearrange. +#include "graph.h" +#include "exit_status.h" +#include "util.h" // int64_t + +struct BuildLog; +struct Builder; +struct DiskInterface; +struct Edge; +struct Node; +struct State; +struct Status; + +/// Plan stores the state of a build plan: what we intend to build, +/// which steps we're ready to execute. +struct Plan { + Plan(Builder* builder = NULL); + + /// Add a target to our plan (including all its dependencies). + /// Returns false if we don't need to build this target; may + /// fill in |err| with an error message if there's a problem. + bool AddTarget(const Node* target, std::string* err); + + // Pop a ready edge off the queue of edges to build. + // Returns NULL if there's no work to do. + Edge* FindWork(); + + /// Returns true if there's more work to be done. + bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; } + + /// Dumps the current state of the plan. + void Dump() const; + + enum EdgeResult { + kEdgeFailed, + kEdgeSucceeded + }; + + /// Mark an edge as done building (whether it succeeded or failed). + /// If any of the edge's outputs are dyndep bindings of their dependents, + /// this loads dynamic dependencies from the nodes' paths. + /// Returns 'false' if loading dyndep info fails and 'true' otherwise. + bool EdgeFinished(Edge* edge, EdgeResult result, std::string* err); + + /// Clean the given node during the build. + /// Return false on error. + bool CleanNode(DependencyScan* scan, Node* node, std::string* err); + + /// Number of edges with commands to run. + int command_edge_count() const { return command_edges_; } + + /// Reset state. Clears want and ready sets. + void Reset(); + + /// Update the build plan to account for modifications made to the graph + /// by information loaded from a dyndep file. + bool DyndepsLoaded(DependencyScan* scan, const Node* node, + const DyndepFile& ddf, std::string* err); + +private: + bool RefreshDyndepDependents(DependencyScan* scan, const Node* node, std::string* err); + void UnmarkDependents(const Node* node, std::set* dependents); + bool AddSubTarget(const Node* node, const Node* dependent, std::string* err, + std::set* dyndep_walk); + + /// Update plan with knowledge that the given node is up to date. + /// If the node is a dyndep binding on any of its dependents, this + /// loads dynamic dependencies from the node's path. + /// Returns 'false' if loading dyndep info fails and 'true' otherwise. + bool NodeFinished(Node* node, std::string* err); + + /// Enumerate possible steps we want for an edge. + enum Want + { + /// We do not want to build the edge, but we might want to build one of + /// its dependents. + kWantNothing, + /// We want to build the edge, but have not yet scheduled it. + kWantToStart, + /// We want to build the edge, have scheduled it, and are waiting + /// for it to complete. + kWantToFinish + }; + + void EdgeWanted(const Edge* edge); + bool EdgeMaybeReady(std::map::iterator want_e, std::string* err); + + /// Submits a ready edge as a candidate for execution. + /// The edge may be delayed from running, for example if it's a member of a + /// currently-full pool. + void ScheduleWork(std::map::iterator want_e); + + /// Keep track of which edges we want to build in this plan. If this map does + /// not contain an entry for an edge, we do not want to build the entry or its + /// dependents. If it does contain an entry, the enumeration indicates what + /// we want for the edge. + std::map want_; + + EdgeSet ready_; + + Builder* builder_; + + /// Total number of edges that have commands (not phony). + int command_edges_; + + /// Total remaining number of wanted edges. + int wanted_edges_; +}; + +/// CommandRunner is an interface that wraps running the build +/// subcommands. This allows tests to abstract out running commands. +/// RealCommandRunner is an implementation that actually runs commands. +struct CommandRunner { + virtual ~CommandRunner() {} + virtual size_t CanRunMore() const = 0; + virtual bool StartCommand(Edge* edge) = 0; + + /// The result of waiting for a command. + struct Result { + Result() : edge(NULL) {} + Edge* edge; + ExitStatus status; + std::string output; + bool success() const { return status == ExitSuccess; } + }; + /// Wait for a command to complete, or return false if interrupted. + virtual bool WaitForCommand(Result* result) = 0; + + virtual std::vector GetActiveEdges() { return std::vector(); } + virtual void Abort() {} +}; + +/// Options (e.g. verbosity, parallelism) passed to a build. +struct BuildConfig { + BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), + failures_allowed(1), max_load_average(-0.0f) {} + + enum Verbosity { + QUIET, // No output -- used when testing. + NO_STATUS_UPDATE, // just regular output but suppress status update + NORMAL, // regular output and status update + VERBOSE + }; + Verbosity verbosity; + bool dry_run; + int parallelism; + int failures_allowed; + /// The maximum load average we must not exceed. A negative value + /// means that we do not have any limit. + double max_load_average; + DepfileParserOptions depfile_parser_options; +}; + +/// Builder wraps the build process: starting commands, updating status. +struct Builder { + Builder(State* state, const BuildConfig& config, + BuildLog* build_log, DepsLog* deps_log, + DiskInterface* disk_interface, Status* status, + int64_t start_time_millis); + ~Builder(); + + /// Clean up after interrupted commands by deleting output files. + void Cleanup(); + + Node* AddTarget(const std::string& name, std::string* err); + + /// Add a target to the build, scanning dependencies. + /// @return false on error. + bool AddTarget(Node* target, std::string* err); + + /// Returns true if the build targets are already up to date. + bool AlreadyUpToDate() const; + + /// Run the build. Returns false on error. + /// It is an error to call this function when AlreadyUpToDate() is true. + bool Build(std::string* err); + + bool StartEdge(Edge* edge, std::string* err); + + std::string GetContent(Edge* edge); + + /// Update status ninja logs following a command termination. + /// @return false if the build can not proceed further due to a fatal error. + bool FinishCommand(CommandRunner::Result* result, std::string* err); + + /// Used for tests. + void SetBuildLog(BuildLog* log) { + scan_.set_build_log(log); + } + + /// Load the dyndep information provided by the given node. + bool LoadDyndeps(Node* node, std::string* err); + + State* state_; + const BuildConfig& config_; + Plan plan_; + std::unique_ptr command_runner_; + Status* status_; + + private: + bool ExtractDeps(CommandRunner::Result* result, const std::string& deps_type, + const std::string& deps_prefix, + std::vector* deps_nodes, std::string* err); + + /// Map of running edge to time the edge started running. + typedef std::map RunningEdgeMap; + RunningEdgeMap running_edges_; + + /// Time the build started. + int64_t start_time_millis_; + + std::string lock_file_path_; + DiskInterface* disk_interface_; + DependencyScan scan_; + + // Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr. + Builder(const Builder &other); // DO NOT IMPLEMENT + void operator=(const Builder &other); // DO NOT IMPLEMENT +}; + +#endif // NINJA_BUILD_H_ diff --git a/src/build_log.cc b/src/build_log.cc index 792d1a3..0f2d0bd 100755 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -1,496 +1,496 @@ -// Copyright 2011 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// On AIX, inttypes.h gets indirectly included by build_log.h. -// It's easiest just to ask for the printf format macros right away. -#ifndef _WIN32 -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif -#endif - -#include "build_log.h" -#include "disk_interface.h" - -#include -#include -#include -#include - -#ifndef _WIN32 -#include -#include -#endif - -#include "build.h" -#include "graph.h" -#include "metrics.h" -#include "util.h" -#if defined(_MSC_VER) && (_MSC_VER < 1800) -#define strtoll _strtoi64 -#endif - -using namespace std; - -// Implementation details: -// Each run's log appends to the log file. -// To load, we run through all log entries in series, throwing away -// older runs. -// Once the number of redundant entries exceeds a threshold, we write -// out a new file and replace the existing one with it. - -namespace { - -const char kFileSignature[] = "# ninja log v%d\n"; -const int kOldestSupportedVersion = 6; -const int kCurrentVersion = 6; - -// 64bit MurmurHash2, by Austin Appleby -#if defined(_MSC_VER) -#define BIG_CONSTANT(x) (x) -#else // defined(_MSC_VER) -#define BIG_CONSTANT(x) (x##LLU) -#endif // !defined(_MSC_VER) -inline -uint64_t MurmurHash64A(const void* key, size_t len) { - static const uint64_t seed = 0xDECAFBADDECAFBADull; - const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); - const int r = 47; - uint64_t h = seed ^ (len * m); - const unsigned char* data = (const unsigned char*)key; - while (len >= 8) { - uint64_t k; - memcpy(&k, data, sizeof k); - k *= m; - k ^= k >> r; - k *= m; - h ^= k; - h *= m; - data += 8; - len -= 8; - } - switch (len & 7) - { - case 7: h ^= uint64_t(data[6]) << 48; - NINJA_FALLTHROUGH; - case 6: h ^= uint64_t(data[5]) << 40; - NINJA_FALLTHROUGH; - case 5: h ^= uint64_t(data[4]) << 32; - NINJA_FALLTHROUGH; - case 4: h ^= uint64_t(data[3]) << 24; - NINJA_FALLTHROUGH; - case 3: h ^= uint64_t(data[2]) << 16; - NINJA_FALLTHROUGH; - case 2: h ^= uint64_t(data[1]) << 8; - NINJA_FALLTHROUGH; - case 1: h ^= uint64_t(data[0]); - h *= m; - }; - h ^= h >> r; - h *= m; - h ^= h >> r; - return h; -} -#undef BIG_CONSTANT - - -} // namespace - -// static -uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { - return MurmurHash64A(command.str_, command.len_); -} - -BuildLog::LogEntry::LogEntry(const string& output) - : output(output) {} - -BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash, - int start_time, int end_time, TimeStamp mtime) - : output(output), command_hash(command_hash), - start_time(start_time), end_time(end_time), mtime(mtime) -{} - -BuildLog::BuildLog() - : log_file_(NULL), needs_recompaction_(false) {} - -BuildLog::~BuildLog() { - Close(); -} - -bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user, - string* err) { - if (needs_recompaction_) { - if (!Recompact(path, user, err)) - return false; - } - - assert(!log_file_); - log_file_path_ = path; // we don't actually open the file right now, but will - // do so on the first write attempt - return true; -} - -bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, - TimeStamp mtime) { - string command = edge->EvaluateCommand(true); - uint64_t command_hash = LogEntry::HashCommand(command); - for (vector::iterator out = edge->outputs_.begin(); - out != edge->outputs_.end(); ++out) { - const string& path = (*out)->path(); - Entries::iterator i = entries_.find(path); - LogEntry* log_entry; - if (i != entries_.end()) { - log_entry = i->second; - } else { - log_entry = new LogEntry(path); - entries_.insert(Entries::value_type(log_entry->output, log_entry)); - } - log_entry->command_hash = command_hash; - log_entry->start_time = start_time; - log_entry->end_time = end_time; - log_entry->mtime = mtime; - - if (!OpenForWriteIfNeeded()) { - return false; - } - if (log_file_) { - if (!WriteEntry(log_file_, *log_entry)) - return false; - if (fflush(log_file_) != 0) { - return false; - } - } - } - return true; -} - -void BuildLog::Close() { - OpenForWriteIfNeeded(); // create the file even if nothing has been recorded - if (log_file_) - fclose(log_file_); - log_file_ = NULL; -} - -bool BuildLog::OpenForWriteIfNeeded() { - if (log_file_ || log_file_path_.empty()) { - return true; - } - log_file_ = fopen(log_file_path_.c_str(), "ab"); - if (!log_file_) { - return false; - } - if (setvbuf(log_file_, NULL, _IOLBF, BUFSIZ) != 0) { - return false; - } - SetCloseOnExec(fileno(log_file_)); - - // Opening a file in append mode doesn't set the file pointer to the file's - // end on Windows. Do that explicitly. - fseek(log_file_, 0, SEEK_END); - - if (ftell(log_file_) == 0) { - if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) { - return false; - } - } - return true; -} - -struct LineReader { - explicit LineReader(FILE* file) - : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) { - memset(buf_, 0, sizeof(buf_)); - } - - // Reads a \n-terminated line from the file passed to the constructor. - // On return, *line_start points to the beginning of the next line, and - // *line_end points to the \n at the end of the line. If no newline is seen - // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF. - bool ReadLine(char** line_start, char** line_end) { - if (line_start_ >= buf_end_ || !line_end_) { - // Buffer empty, refill. - size_t size_read = fread(buf_, 1, sizeof(buf_), file_); - if (!size_read) - return false; - line_start_ = buf_; - buf_end_ = buf_ + size_read; - } else { - // Advance to next line in buffer. - line_start_ = line_end_ + 1; - } - - line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); - if (!line_end_) { - // No newline. Move rest of data to start of buffer, fill rest. - size_t already_consumed = line_start_ - buf_; - size_t size_rest = (buf_end_ - buf_) - already_consumed; - memmove(buf_, line_start_, size_rest); - - size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_); - buf_end_ = buf_ + size_rest + read; - line_start_ = buf_; - line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); - } - - *line_start = line_start_; - *line_end = line_end_; - return true; - } - - private: - FILE* file_; - char buf_[256 << 10]; - char* buf_end_; // Points one past the last valid byte in |buf_|. - - char* line_start_; - // Points at the next \n in buf_ after line_start, or NULL. - char* line_end_; -}; - -LoadStatus BuildLog::Load(const string& path, string* err) { - METRIC_RECORD(".ninja_log load"); - FILE* file = fopen(path.c_str(), "r"); - if (!file) { - if (errno == ENOENT) - return LOAD_NOT_FOUND; - *err = strerror(errno); - return LOAD_ERROR; - } - - int log_version = 0; - int unique_entry_count = 0; - int total_entry_count = 0; - - LineReader reader(file); - char* line_start = 0; - char* line_end = 0; - while (reader.ReadLine(&line_start, &line_end)) { - if (!log_version) { - sscanf(line_start, kFileSignature, &log_version); - - bool invalid_log_version = false; - if (log_version < kOldestSupportedVersion) { - invalid_log_version = true; - *err = "build log version is too old; starting over"; - - } else if (log_version > kCurrentVersion) { - invalid_log_version = true; - *err = "build log version is too new; starting over"; - } - if (invalid_log_version) { - fclose(file); - unlink(path.c_str()); - // Don't report this as a failure. A missing build log will cause - // us to rebuild the outputs anyway. - return LOAD_NOT_FOUND; - } - } - - // If no newline was found in this chunk, read the next. - if (!line_end) - continue; - - const char kFieldSeparator = '\t'; - - char* start = line_start; - char* end = (char*)memchr(start, kFieldSeparator, line_end - start); - if (!end) - continue; - *end = 0; - - int start_time = 0, end_time = 0; - TimeStamp mtime = 0; - - start_time = atoi(start); - start = end + 1; - - end = (char*)memchr(start, kFieldSeparator, line_end - start); - if (!end) - continue; - *end = 0; - end_time = atoi(start); - start = end + 1; - - end = (char*)memchr(start, kFieldSeparator, line_end - start); - if (!end) - continue; - *end = 0; - mtime = strtoll(start, NULL, 10); - start = end + 1; - - end = (char*)memchr(start, kFieldSeparator, line_end - start); - if (!end) - continue; - string output = string(start, end - start); - - start = end + 1; - end = line_end; - - LogEntry* entry; - Entries::iterator i = entries_.find(output); - if (i != entries_.end()) { - entry = i->second; - } else { - entry = new LogEntry(output); - entries_.insert(Entries::value_type(entry->output, entry)); - ++unique_entry_count; - } - ++total_entry_count; - - entry->start_time = start_time; - entry->end_time = end_time; - entry->mtime = mtime; - char c = *end; *end = '\0'; - entry->command_hash = (uint64_t)strtoull(start, NULL, 16); - *end = c; - } - fclose(file); - - if (!line_start) { - return LOAD_SUCCESS; // file was empty - } - - // Decide whether it's time to rebuild the log: - // - if we're upgrading versions - // - if it's getting large - int kMinCompactionEntryCount = 100; - int kCompactionRatio = 3; - if (log_version < kCurrentVersion) { - needs_recompaction_ = true; - } else if (total_entry_count > kMinCompactionEntryCount && - total_entry_count > unique_entry_count * kCompactionRatio) { - needs_recompaction_ = true; - } - - return LOAD_SUCCESS; -} - -BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { - Entries::iterator i = entries_.find(path); - if (i != entries_.end()) - return i->second; - return NULL; -} - -bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { - return fprintf(f, "%d\t%d\t%" PRId64 "\t%s\t%" PRIx64 "\n", - entry.start_time, entry.end_time, entry.mtime, - entry.output.c_str(), entry.command_hash) > 0; -} - -bool BuildLog::Recompact(const string& path, const BuildLogUser& user, - string* err) { - METRIC_RECORD(".ninja_log recompact"); - - Close(); - string temp_path = path + ".recompact"; - FILE* f = fopen(temp_path.c_str(), "wb"); - if (!f) { - *err = strerror(errno); - return false; - } - - if (fprintf(f, kFileSignature, kCurrentVersion) < 0) { - *err = strerror(errno); - fclose(f); - return false; - } - - vector dead_outputs; - for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { - if (user.IsPathDead(i->first)) { - dead_outputs.push_back(i->first); - continue; - } - - if (!WriteEntry(f, *i->second)) { - *err = strerror(errno); - fclose(f); - return false; - } - } - - for (size_t i = 0; i < dead_outputs.size(); ++i) - entries_.erase(dead_outputs[i]); - - fclose(f); - if (unlink(path.c_str()) < 0) { - *err = strerror(errno); - return false; - } - - if (rename(temp_path.c_str(), path.c_str()) < 0) { - *err = strerror(errno); - return false; - } - - return true; -} - -bool BuildLog::Restat(const StringPiece path, - const DiskInterface& disk_interface, - const int output_count, char** outputs, - std::string* const err) { - METRIC_RECORD(".ninja_log restat"); - - Close(); - std::string temp_path = path.AsString() + ".restat"; - FILE* f = fopen(temp_path.c_str(), "wb"); - if (!f) { - *err = strerror(errno); - return false; - } - - if (fprintf(f, kFileSignature, kCurrentVersion) < 0) { - *err = strerror(errno); - fclose(f); - return false; - } - for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { - bool skip = output_count > 0; - for (int j = 0; j < output_count; ++j) { - if (i->second->output == outputs[j]) { - skip = false; - break; - } - } - if (!skip) { - const TimeStamp mtime = disk_interface.Stat(i->second->output, err); - if (mtime == -1) { - fclose(f); - return false; - } - i->second->mtime = mtime; - } - - if (!WriteEntry(f, *i->second)) { - *err = strerror(errno); - fclose(f); - return false; - } - } - - fclose(f); - if (unlink(path.str_) < 0) { - *err = strerror(errno); - return false; - } - - if (rename(temp_path.c_str(), path.str_) < 0) { - *err = strerror(errno); - return false; - } - - return true; -} +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// On AIX, inttypes.h gets indirectly included by build_log.h. +// It's easiest just to ask for the printf format macros right away. +#ifndef _WIN32 +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif +#endif + +#include "build_log.h" +#include "disk_interface.h" + +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#endif + +#include "build.h" +#include "graph.h" +#include "metrics.h" +#include "util.h" +#if defined(_MSC_VER) && (_MSC_VER < 1800) +#define strtoll _strtoi64 +#endif + +using namespace std; + +// Implementation details: +// Each run's log appends to the log file. +// To load, we run through all log entries in series, throwing away +// older runs. +// Once the number of redundant entries exceeds a threshold, we write +// out a new file and replace the existing one with it. + +namespace { + +const char kFileSignature[] = "# ninja log v%d\n"; +const int kOldestSupportedVersion = 6; +const int kCurrentVersion = 6; + +// 64bit MurmurHash2, by Austin Appleby +#if defined(_MSC_VER) +#define BIG_CONSTANT(x) (x) +#else // defined(_MSC_VER) +#define BIG_CONSTANT(x) (x##LLU) +#endif // !defined(_MSC_VER) +inline +uint64_t MurmurHash64A(const void* key, size_t len) { + static const uint64_t seed = 0xDECAFBADDECAFBADull; + const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); + const int r = 47; + uint64_t h = seed ^ (len * m); + const unsigned char* data = (const unsigned char*)key; + while (len >= 8) { + uint64_t k; + memcpy(&k, data, sizeof k); + k *= m; + k ^= k >> r; + k *= m; + h ^= k; + h *= m; + data += 8; + len -= 8; + } + switch (len & 7) + { + case 7: h ^= uint64_t(data[6]) << 48; + NINJA_FALLTHROUGH; + case 6: h ^= uint64_t(data[5]) << 40; + NINJA_FALLTHROUGH; + case 5: h ^= uint64_t(data[4]) << 32; + NINJA_FALLTHROUGH; + case 4: h ^= uint64_t(data[3]) << 24; + NINJA_FALLTHROUGH; + case 3: h ^= uint64_t(data[2]) << 16; + NINJA_FALLTHROUGH; + case 2: h ^= uint64_t(data[1]) << 8; + NINJA_FALLTHROUGH; + case 1: h ^= uint64_t(data[0]); + h *= m; + }; + h ^= h >> r; + h *= m; + h ^= h >> r; + return h; +} +#undef BIG_CONSTANT + + +} // namespace + +// static +uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { + return MurmurHash64A(command.str_, command.len_); +} + +BuildLog::LogEntry::LogEntry(const string& output) + : output(output) {} + +BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash, + int start_time, int end_time, TimeStamp mtime) + : output(output), command_hash(command_hash), + start_time(start_time), end_time(end_time), mtime(mtime) +{} + +BuildLog::BuildLog() + : log_file_(NULL), needs_recompaction_(false) {} + +BuildLog::~BuildLog() { + Close(); +} + +bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user, + string* err) { + if (needs_recompaction_) { + if (!Recompact(path, user, err)) + return false; + } + + assert(!log_file_); + log_file_path_ = path; // we don't actually open the file right now, but will + // do so on the first write attempt + return true; +} + +bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, + TimeStamp mtime) { + string command = edge->EvaluateCommand(true); + uint64_t command_hash = LogEntry::HashCommand(command); + for (vector::iterator out = edge->outputs_.begin(); + out != edge->outputs_.end(); ++out) { + const string& path = (*out)->path(); + Entries::iterator i = entries_.find(path); + LogEntry* log_entry; + if (i != entries_.end()) { + log_entry = i->second; + } else { + log_entry = new LogEntry(path); + entries_.insert(Entries::value_type(log_entry->output, log_entry)); + } + log_entry->command_hash = command_hash; + log_entry->start_time = start_time; + log_entry->end_time = end_time; + log_entry->mtime = mtime; + + if (!OpenForWriteIfNeeded()) { + return false; + } + if (log_file_) { + if (!WriteEntry(log_file_, *log_entry)) + return false; + if (fflush(log_file_) != 0) { + return false; + } + } + } + return true; +} + +void BuildLog::Close() { + OpenForWriteIfNeeded(); // create the file even if nothing has been recorded + if (log_file_) + fclose(log_file_); + log_file_ = NULL; +} + +bool BuildLog::OpenForWriteIfNeeded() { + if (log_file_ || log_file_path_.empty()) { + return true; + } + log_file_ = fopen(log_file_path_.c_str(), "ab"); + if (!log_file_) { + return false; + } + if (setvbuf(log_file_, NULL, _IOLBF, BUFSIZ) != 0) { + return false; + } + SetCloseOnExec(fileno(log_file_)); + + // Opening a file in append mode doesn't set the file pointer to the file's + // end on Windows. Do that explicitly. + fseek(log_file_, 0, SEEK_END); + + if (ftell(log_file_) == 0) { + if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) { + return false; + } + } + return true; +} + +struct LineReader { + explicit LineReader(FILE* file) + : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) { + memset(buf_, 0, sizeof(buf_)); + } + + // Reads a \n-terminated line from the file passed to the constructor. + // On return, *line_start points to the beginning of the next line, and + // *line_end points to the \n at the end of the line. If no newline is seen + // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF. + bool ReadLine(char** line_start, char** line_end) { + if (line_start_ >= buf_end_ || !line_end_) { + // Buffer empty, refill. + size_t size_read = fread(buf_, 1, sizeof(buf_), file_); + if (!size_read) + return false; + line_start_ = buf_; + buf_end_ = buf_ + size_read; + } else { + // Advance to next line in buffer. + line_start_ = line_end_ + 1; + } + + line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); + if (!line_end_) { + // No newline. Move rest of data to start of buffer, fill rest. + size_t already_consumed = line_start_ - buf_; + size_t size_rest = (buf_end_ - buf_) - already_consumed; + memmove(buf_, line_start_, size_rest); + + size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_); + buf_end_ = buf_ + size_rest + read; + line_start_ = buf_; + line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); + } + + *line_start = line_start_; + *line_end = line_end_; + return true; + } + + private: + FILE* file_; + char buf_[256 << 10]; + char* buf_end_; // Points one past the last valid byte in |buf_|. + + char* line_start_; + // Points at the next \n in buf_ after line_start, or NULL. + char* line_end_; +}; + +LoadStatus BuildLog::Load(const string& path, string* err) { + METRIC_RECORD(".ninja_log load"); + FILE* file = fopen(path.c_str(), "r"); + if (!file) { + if (errno == ENOENT) + return LOAD_NOT_FOUND; + *err = strerror(errno); + return LOAD_ERROR; + } + + int log_version = 0; + int unique_entry_count = 0; + int total_entry_count = 0; + + LineReader reader(file); + char* line_start = 0; + char* line_end = 0; + while (reader.ReadLine(&line_start, &line_end)) { + if (!log_version) { + sscanf(line_start, kFileSignature, &log_version); + + bool invalid_log_version = false; + if (log_version < kOldestSupportedVersion) { + invalid_log_version = true; + *err = "build log version is too old; starting over"; + + } else if (log_version > kCurrentVersion) { + invalid_log_version = true; + *err = "build log version is too new; starting over"; + } + if (invalid_log_version) { + fclose(file); + unlink(path.c_str()); + // Don't report this as a failure. A missing build log will cause + // us to rebuild the outputs anyway. + return LOAD_NOT_FOUND; + } + } + + // If no newline was found in this chunk, read the next. + if (!line_end) + continue; + + const char kFieldSeparator = '\t'; + + char* start = line_start; + char* end = (char*)memchr(start, kFieldSeparator, line_end - start); + if (!end) + continue; + *end = 0; + + int start_time = 0, end_time = 0; + TimeStamp mtime = 0; + + start_time = atoi(start); + start = end + 1; + + end = (char*)memchr(start, kFieldSeparator, line_end - start); + if (!end) + continue; + *end = 0; + end_time = atoi(start); + start = end + 1; + + end = (char*)memchr(start, kFieldSeparator, line_end - start); + if (!end) + continue; + *end = 0; + mtime = strtoll(start, NULL, 10); + start = end + 1; + + end = (char*)memchr(start, kFieldSeparator, line_end - start); + if (!end) + continue; + string output = string(start, end - start); + + start = end + 1; + end = line_end; + + LogEntry* entry; + Entries::iterator i = entries_.find(output); + if (i != entries_.end()) { + entry = i->second; + } else { + entry = new LogEntry(output); + entries_.insert(Entries::value_type(entry->output, entry)); + ++unique_entry_count; + } + ++total_entry_count; + + entry->start_time = start_time; + entry->end_time = end_time; + entry->mtime = mtime; + char c = *end; *end = '\0'; + entry->command_hash = (uint64_t)strtoull(start, NULL, 16); + *end = c; + } + fclose(file); + + if (!line_start) { + return LOAD_SUCCESS; // file was empty + } + + // Decide whether it's time to rebuild the log: + // - if we're upgrading versions + // - if it's getting large + int kMinCompactionEntryCount = 100; + int kCompactionRatio = 3; + if (log_version < kCurrentVersion) { + needs_recompaction_ = true; + } else if (total_entry_count > kMinCompactionEntryCount && + total_entry_count > unique_entry_count * kCompactionRatio) { + needs_recompaction_ = true; + } + + return LOAD_SUCCESS; +} + +BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { + Entries::iterator i = entries_.find(path); + if (i != entries_.end()) + return i->second; + return NULL; +} + +bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { + return fprintf(f, "%d\t%d\t%" PRId64 "\t%s\t%" PRIx64 "\n", + entry.start_time, entry.end_time, entry.mtime, + entry.output.c_str(), entry.command_hash) > 0; +} + +bool BuildLog::Recompact(const string& path, const BuildLogUser& user, + string* err) { + METRIC_RECORD(".ninja_log recompact"); + + Close(); + string temp_path = path + ".recompact"; + FILE* f = fopen(temp_path.c_str(), "wb"); + if (!f) { + *err = strerror(errno); + return false; + } + + if (fprintf(f, kFileSignature, kCurrentVersion) < 0) { + *err = strerror(errno); + fclose(f); + return false; + } + + vector dead_outputs; + for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { + if (user.IsPathDead(i->first)) { + dead_outputs.push_back(i->first); + continue; + } + + if (!WriteEntry(f, *i->second)) { + *err = strerror(errno); + fclose(f); + return false; + } + } + + for (size_t i = 0; i < dead_outputs.size(); ++i) + entries_.erase(dead_outputs[i]); + + fclose(f); + if (unlink(path.c_str()) < 0) { + *err = strerror(errno); + return false; + } + + if (rename(temp_path.c_str(), path.c_str()) < 0) { + *err = strerror(errno); + return false; + } + + return true; +} + +bool BuildLog::Restat(const StringPiece path, + const DiskInterface& disk_interface, + const int output_count, char** outputs, + std::string* const err) { + METRIC_RECORD(".ninja_log restat"); + + Close(); + std::string temp_path = path.AsString() + ".restat"; + FILE* f = fopen(temp_path.c_str(), "wb"); + if (!f) { + *err = strerror(errno); + return false; + } + + if (fprintf(f, kFileSignature, kCurrentVersion) < 0) { + *err = strerror(errno); + fclose(f); + return false; + } + for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { + bool skip = output_count > 0; + for (int j = 0; j < output_count; ++j) { + if (i->second->output == outputs[j]) { + skip = false; + break; + } + } + if (!skip) { + const TimeStamp mtime = disk_interface.Stat(i->second->output, err); + if (mtime == -1) { + fclose(f); + return false; + } + i->second->mtime = mtime; + } + + if (!WriteEntry(f, *i->second)) { + *err = strerror(errno); + fclose(f); + return false; + } + } + + fclose(f); + if (unlink(path.str_) < 0) { + *err = strerror(errno); + return false; + } + + if (rename(temp_path.c_str(), path.str_) < 0) { + *err = strerror(errno); + return false; + } + + return true; +} diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 7ce7290..b41f1cf 100755 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -21,7 +21,7 @@ using namespace std; DepfileParser::DepfileParser(DepfileParserOptions options) - : options_(options) + : options_(options) { } @@ -46,328 +46,327 @@ DepfileParser::DepfileParser(DepfileParserOptions options) // If anyone actually has depfiles that rely on the more complicated // behavior we can adjust this. bool DepfileParser::Parse(string* content, string* err) { - // in: current parser input point. - // end: end of input. - // parsing_targets: whether we are parsing targets or dependencies. - char* in = &(*content)[0]; - char* end = in + content->size(); - bool have_target = false; - bool parsing_targets = true; - bool poisoned_input = false; - bool is_empty = true; - while (in < end) { - bool have_newline = false; - // out: current output point (typically same as in, but can fall behind - // as we de-escape backslashes). - char* out = in; - // filename: start of the current parsed filename. - char* filename = out; - for (;;) { - // start: beginning of the current parsed span. - const char* start = in; - char* yymarker = NULL; - - { - unsigned char yych; - static const unsigned char yybm[] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 128, 0, 0, 0, 128, 0, 0, - 128, 128, 0, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 0, 0, 128, 0, 0, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 0, 128, 0, 128, - 0, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 0, 128, 128, 0, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - }; - yych = *in; - if (yybm[0+yych] & 128) { - goto yy9; - } - if (yych <= '\r') { - if (yych <= '\t') { - if (yych >= 0x01) goto yy4; - } else { - if (yych <= '\n') goto yy6; - if (yych <= '\f') goto yy4; - goto yy8; - } - } else { - if (yych <= '$') { - if (yych <= '#') goto yy4; - goto yy12; - } else { - if (yych <= '?') goto yy4; - if (yych <= '\\') goto yy13; - goto yy4; - } - } - ++in; - { - break; - } -yy4: - ++in; -yy5: - { - // For any other character (e.g. whitespace), swallow it here, - // allowing the outer logic to loop around again. - break; - } -yy6: - ++in; - { - // A newline ends the current file name and the current rule. - have_newline = true; - break; - } -yy8: - yych = *++in; - if (yych == '\n') goto yy6; - goto yy5; -yy9: - yych = *++in; - if (yybm[0+yych] & 128) { - goto yy9; - } -yy11: - { - // Got a span of plain text. - int len = (int)(in - start); - // Need to shift it over if we're overwriting backslashes. - if (out < start) - memmove(out, start, len); - out += len; - continue; - } -yy12: - yych = *++in; - if (yych == '$') goto yy14; - goto yy5; -yy13: - yych = *(yymarker = ++in); - if (yych <= ' ') { - if (yych <= '\n') { - if (yych <= 0x00) goto yy5; - if (yych <= '\t') goto yy16; - goto yy17; - } else { - if (yych == '\r') goto yy19; - if (yych <= 0x1F) goto yy16; - goto yy21; - } - } else { - if (yych <= '9') { - if (yych == '#') goto yy23; - goto yy16; - } else { - if (yych <= ':') goto yy25; - if (yych == '\\') goto yy27; - goto yy16; - } - } -yy14: - ++in; - { - // De-escape dollar character. - *out++ = '$'; - continue; - } -yy16: - ++in; - goto yy11; -yy17: - ++in; - { - // A line continuation ends the current file name. - break; - } -yy19: - yych = *++in; - if (yych == '\n') goto yy17; - in = yymarker; - goto yy5; -yy21: - ++in; - { - // 2N+1 backslashes plus space -> N backslashes plus space. - int len = (int)(in - start); - int n = len / 2 - 1; - if (out < start) - memset(out, '\\', n); - out += n; - *out++ = ' '; - continue; - } -yy23: - ++in; - { - // De-escape hash sign, but preserve other leading backslashes. - int len = (int)(in - start); - if (len > 2 && out < start) - memset(out, '\\', len - 2); - out += len - 2; - *out++ = '#'; - continue; - } -yy25: - yych = *++in; - if (yych <= '\f') { - if (yych <= 0x00) goto yy28; - if (yych <= 0x08) goto yy26; - if (yych <= '\n') goto yy28; - } else { - if (yych <= '\r') goto yy28; - if (yych == ' ') goto yy28; - } -yy26: - { - // De-escape colon sign, but preserve other leading backslashes. - // Regular expression uses lookahead to make sure that no whitespace - // nor EOF follows. In that case it'd be the : at the end of a target - int len = (int)(in - start); - if (len > 2 && out < start) - memset(out, '\\', len - 2); - out += len - 2; - *out++ = ':'; - continue; - } -yy27: - yych = *++in; - if (yych <= ' ') { - if (yych <= '\n') { - if (yych <= 0x00) goto yy11; - if (yych <= '\t') goto yy16; - goto yy11; - } else { - if (yych == '\r') goto yy11; - if (yych <= 0x1F) goto yy16; - goto yy30; - } - } else { - if (yych <= '9') { - if (yych == '#') goto yy23; - goto yy16; - } else { - if (yych <= ':') goto yy25; - if (yych == '\\') goto yy32; - goto yy16; - } - } -yy28: - ++in; - { - // Backslash followed by : and whitespace. - // It is therefore normal text and not an escaped colon - int len = (int)(in - start - 1); - // Need to shift it over if we're overwriting backslashes. - if (out < start) - memmove(out, start, len); - out += len; - if (*(in - 1) == '\n') - have_newline = true; - break; - } -yy30: - ++in; - { - // 2N backslashes plus space -> 2N backslashes, end of filename. - int len = (int)(in - start); - if (out < start) - memset(out, '\\', len - 1); - out += len - 1; - break; - } -yy32: - yych = *++in; - if (yych <= ' ') { - if (yych <= '\n') { - if (yych <= 0x00) goto yy11; - if (yych <= '\t') goto yy16; - goto yy11; - } else { - if (yych == '\r') goto yy11; - if (yych <= 0x1F) goto yy16; - goto yy21; - } - } else { - if (yych <= '9') { - if (yych == '#') goto yy23; - goto yy16; - } else { - if (yych <= ':') goto yy25; - if (yych == '\\') goto yy27; - goto yy16; - } - } - } + // in: current parser input point. + // end: end of input. + // parsing_targets: whether we are parsing targets or dependencies. + char* in = &(*content)[0]; + char* end = in + content->size(); + bool have_target = false; + bool parsing_targets = true; + bool poisoned_input = false; + bool is_empty = true; + while (in < end) { + bool have_newline = false; + // out: current output point (typically same as in, but can fall behind + // as we de-escape backslashes). + char* out = in; + // filename: start of the current parsed filename. + char* filename = out; + for (;;) { + // start: beginning of the current parsed span. + const char* start = in; + char* yymarker = NULL; + { + unsigned char yych; + static const unsigned char yybm[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 128, 0, 0, 0, 128, 0, 0, + 128, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 0, 0, 128, 0, 0, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 0, 128, 0, 128, + 0, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 0, 128, 128, 0, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + }; + yych = *in; + if (yybm[0+yych] & 128) { + goto yy9; + } + if (yych <= '\r') { + if (yych <= '\t') { + if (yych >= 0x01) goto yy4; + } else { + if (yych <= '\n') goto yy6; + if (yych <= '\f') goto yy4; + goto yy8; + } + } else { + if (yych <= '$') { + if (yych <= '#') goto yy4; + goto yy12; + } else { + if (yych <= '?') goto yy4; + if (yych <= '\\') goto yy13; + goto yy4; + } + } + ++in; + { + break; + } + yy4: + ++in; + yy5: + { + // For any other character (e.g. whitespace), swallow it here, + // allowing the outer logic to loop around again. + break; + } + yy6: + ++in; + { + // A newline ends the current file name and the current rule. + have_newline = true; + break; + } + yy8: + yych = *++in; + if (yych == '\n') goto yy6; + goto yy5; + yy9: + yych = *++in; + if (yybm[0+yych] & 128) { + goto yy9; + } + yy11: + { + // Got a span of plain text. + int len = (int)(in - start); + // Need to shift it over if we're overwriting backslashes. + if (out < start) + memmove(out, start, len); + out += len; + continue; + } + yy12: + yych = *++in; + if (yych == '$') goto yy14; + goto yy5; + yy13: + yych = *(yymarker = ++in); + if (yych <= ' ') { + if (yych <= '\n') { + if (yych <= 0x00) goto yy5; + if (yych <= '\t') goto yy16; + goto yy17; + } else { + if (yych == '\r') goto yy19; + if (yych <= 0x1F) goto yy16; + goto yy21; + } + } else { + if (yych <= '9') { + if (yych == '#') goto yy23; + goto yy16; + } else { + if (yych <= ':') goto yy25; + if (yych == '\\') goto yy27; + goto yy16; + } + } + yy14: + ++in; + { + // De-escape dollar character. + *out++ = '$'; + continue; + } + yy16: + ++in; + goto yy11; + yy17: + ++in; + { + // A line continuation ends the current file name. + break; + } + yy19: + yych = *++in; + if (yych == '\n') goto yy17; + in = yymarker; + goto yy5; + yy21: + ++in; + { + // 2N+1 backslashes plus space -> N backslashes plus space. + int len = (int)(in - start); + int n = len / 2 - 1; + if (out < start) + memset(out, '\\', n); + out += n; + *out++ = ' '; + continue; + } + yy23: + ++in; + { + // De-escape hash sign, but preserve other leading backslashes. + int len = (int)(in - start); + if (len > 2 && out < start) + memset(out, '\\', len - 2); + out += len - 2; + *out++ = '#'; + continue; + } + yy25: + yych = *++in; + if (yych <= '\f') { + if (yych <= 0x00) goto yy28; + if (yych <= 0x08) goto yy26; + if (yych <= '\n') goto yy28; + } else { + if (yych <= '\r') goto yy28; + if (yych == ' ') goto yy28; + } + yy26: + { + // De-escape colon sign, but preserve other leading backslashes. + // Regular expression uses lookahead to make sure that no whitespace + // nor EOF follows. In that case it'd be the : at the end of a target + int len = (int)(in - start); + if (len > 2 && out < start) + memset(out, '\\', len - 2); + out += len - 2; + *out++ = ':'; + continue; + } + yy27: + yych = *++in; + if (yych <= ' ') { + if (yych <= '\n') { + if (yych <= 0x00) goto yy11; + if (yych <= '\t') goto yy16; + goto yy11; + } else { + if (yych == '\r') goto yy11; + if (yych <= 0x1F) goto yy16; + goto yy30; + } + } else { + if (yych <= '9') { + if (yych == '#') goto yy23; + goto yy16; + } else { + if (yych <= ':') goto yy25; + if (yych == '\\') goto yy32; + goto yy16; + } + } + yy28: + ++in; + { + // Backslash followed by : and whitespace. + // It is therefore normal text and not an escaped colon + int len = (int)(in - start - 1); + // Need to shift it over if we're overwriting backslashes. + if (out < start) + memmove(out, start, len); + out += len; + if (*(in - 1) == '\n') + have_newline = true; + break; + } + yy30: + ++in; + { + // 2N backslashes plus space -> 2N backslashes, end of filename. + int len = (int)(in - start); + if (out < start) + memset(out, '\\', len - 1); + out += len - 1; + break; + } + yy32: + yych = *++in; + if (yych <= ' ') { + if (yych <= '\n') { + if (yych <= 0x00) goto yy11; + if (yych <= '\t') goto yy16; + goto yy11; + } else { + if (yych == '\r') goto yy11; + if (yych <= 0x1F) goto yy16; + goto yy21; + } + } else { + if (yych <= '9') { + if (yych == '#') goto yy23; + goto yy16; + } else { + if (yych <= ':') goto yy25; + if (yych == '\\') goto yy27; + goto yy16; + } + } } +} int len = (int)(out - filename); const bool is_dependency = !parsing_targets; if (len > 0 && filename[len - 1] == ':') { - len--; // Strip off trailing colon, if any. - parsing_targets = false; - have_target = true; + len--; // Strip off trailing colon, if any. + parsing_targets = false; + have_target = true; } if (len > 0) { - is_empty = false; - StringPiece piece = StringPiece(filename, len); - // If we've seen this as an input before, skip it. - std::vector::iterator pos = std::find(ins_.begin(), ins_.end(), piece); - if (pos == ins_.end()) { - if (is_dependency) { - if (poisoned_input) { - *err = "inputs may not also have inputs"; - return false; - } - // New input. - ins_.push_back(piece); - } else { - // Check for a new output. - if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end()) - outs_.push_back(piece); - } - } else if (!is_dependency) { - // We've passed an input on the left side; reject new inputs. - poisoned_input = true; - } + is_empty = false; + StringPiece piece = StringPiece(filename, len); + // If we've seen this as an input before, skip it. + std::vector::iterator pos = std::find(ins_.begin(), ins_.end(), piece); + if (pos == ins_.end()) { + if (is_dependency) { + if (poisoned_input) { + *err = "inputs may not also have inputs"; + return false; + } + // New input. + ins_.push_back(piece); + } else { + // Check for a new output. + if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end()) + outs_.push_back(piece); + } + } else if (!is_dependency) { + // We've passed an input on the left side; reject new inputs. + poisoned_input = true; + } } if (have_newline) { - // A newline ends a rule so the next filename will be a new target. - parsing_targets = true; - poisoned_input = false; + // A newline ends a rule so the next filename will be a new target. + parsing_targets = true; + poisoned_input = false; } - } - if (!have_target && !is_empty) { - *err = "expected ':' in depfile"; - return false; - } - return true; +} + if (!have_target && !is_empty) { + *err = "expected ':' in depfile"; + return false; + } + return true; } diff --git a/src/disk_interface.h b/src/disk_interface.h index 74200b8..25b357d 100755 --- a/src/disk_interface.h +++ b/src/disk_interface.h @@ -1,105 +1,105 @@ -// Copyright 2011 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef NINJA_DISK_INTERFACE_H_ -#define NINJA_DISK_INTERFACE_H_ - -#include -#include - -#include "timestamp.h" - -/// Interface for reading files from disk. See DiskInterface for details. -/// This base offers the minimum interface needed just to read files. -struct FileReader { - virtual ~FileReader() {} - - /// Result of ReadFile. - enum Status { - Okay, - NotFound, - OtherError - }; - - /// Read and store in given string. On success, return Okay. - /// On error, return another Status and fill |err|. - virtual Status ReadFile(const std::string& path, std::string* contents, - std::string* err) = 0; -}; - -/// Interface for accessing the disk. -/// -/// Abstract so it can be mocked out for tests. The real implementation -/// is RealDiskInterface. -struct DiskInterface: public FileReader { - /// stat() a file, returning the mtime, or 0 if missing and -1 on - /// other errors. - virtual TimeStamp Stat(const std::string& path, std::string* err) const = 0; - - /// Create a directory, returning false on failure. - virtual bool MakeDir(const std::string& path) = 0; - - /// Create a file, with the specified name and contents - /// Returns true on success, false on failure - virtual bool WriteFile(const std::string& path, - const std::string& contents) = 0; - - /// Remove the file named @a path. It behaves like 'rm -f path' so no errors - /// are reported if it does not exists. - /// @returns 0 if the file has been removed, - /// 1 if the file does not exist, and - /// -1 if an error occurs. - virtual int RemoveFile(const std::string& path) = 0; - - /// Create all the parent directories for path; like mkdir -p - /// `basename path`. - bool MakeDirs(const std::string& path); -}; - -/// Implementation of DiskInterface that actually hits the disk. -struct RealDiskInterface : public DiskInterface { - RealDiskInterface(); - virtual ~RealDiskInterface() {} - virtual TimeStamp Stat(const std::string& path, std::string* err) const; - virtual bool MakeDir(const std::string& path); - virtual bool WriteFile(const std::string& path, const std::string& contents); - virtual Status ReadFile(const std::string& path, std::string* contents, - std::string* err); - virtual int RemoveFile(const std::string& path); - - /// Whether stat information can be cached. Only has an effect on Windows. - void AllowStatCache(bool allow); - -#ifdef _WIN32 - /// Whether long paths are enabled. Only has an effect on Windows. - bool AreLongPathsEnabled() const; -#endif - - private: -#ifdef _WIN32 - /// Whether stat information can be cached. - bool use_cache_; - - /// Whether long paths are enabled. - bool long_paths_enabled_; - - typedef std::map DirCache; - // TODO: Neither a map nor a hashmap seems ideal here. If the statcache - // works out, come up with a better data structure. - typedef std::map Cache; - mutable Cache cache_; -#endif -}; - -#endif // NINJA_DISK_INTERFACE_H_ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_DISK_INTERFACE_H_ +#define NINJA_DISK_INTERFACE_H_ + +#include +#include + +#include "timestamp.h" + +/// Interface for reading files from disk. See DiskInterface for details. +/// This base offers the minimum interface needed just to read files. +struct FileReader { + virtual ~FileReader() {} + + /// Result of ReadFile. + enum Status { + Okay, + NotFound, + OtherError + }; + + /// Read and store in given string. On success, return Okay. + /// On error, return another Status and fill |err|. + virtual Status ReadFile(const std::string& path, std::string* contents, + std::string* err) = 0; +}; + +/// Interface for accessing the disk. +/// +/// Abstract so it can be mocked out for tests. The real implementation +/// is RealDiskInterface. +struct DiskInterface: public FileReader { + /// stat() a file, returning the mtime, or 0 if missing and -1 on + /// other errors. + virtual TimeStamp Stat(const std::string& path, std::string* err) const = 0; + + /// Create a directory, returning false on failure. + virtual bool MakeDir(const std::string& path) = 0; + + /// Create a file, with the specified name and contents + /// Returns true on success, false on failure + virtual bool WriteFile(const std::string& path, + const std::string& contents) = 0; + + /// Remove the file named @a path. It behaves like 'rm -f path' so no errors + /// are reported if it does not exists. + /// @returns 0 if the file has been removed, + /// 1 if the file does not exist, and + /// -1 if an error occurs. + virtual int RemoveFile(const std::string& path) = 0; + + /// Create all the parent directories for path; like mkdir -p + /// `basename path`. + bool MakeDirs(const std::string& path); +}; + +/// Implementation of DiskInterface that actually hits the disk. +struct RealDiskInterface : public DiskInterface { + RealDiskInterface(); + virtual ~RealDiskInterface() {} + virtual TimeStamp Stat(const std::string& path, std::string* err) const; + virtual bool MakeDir(const std::string& path); + virtual bool WriteFile(const std::string& path, const std::string& contents); + virtual Status ReadFile(const std::string& path, std::string* contents, + std::string* err); + virtual int RemoveFile(const std::string& path); + + /// Whether stat information can be cached. Only has an effect on Windows. + void AllowStatCache(bool allow); + +#ifdef _WIN32 + /// Whether long paths are enabled. Only has an effect on Windows. + bool AreLongPathsEnabled() const; +#endif + + private: +#ifdef _WIN32 + /// Whether stat information can be cached. + bool use_cache_; + + /// Whether long paths are enabled. + bool long_paths_enabled_; + + typedef std::map DirCache; + // TODO: Neither a map nor a hashmap seems ideal here. If the statcache + // works out, come up with a better data structure. + typedef std::map Cache; + mutable Cache cache_; +#endif +}; + +#endif // NINJA_DISK_INTERFACE_H_ diff --git a/src/graph.h b/src/graph.h index 433b7e3..a687412 100755 --- a/src/graph.h +++ b/src/graph.h @@ -1,378 +1,378 @@ -// Copyright 2011 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef NINJA_GRAPH_H_ -#define NINJA_GRAPH_H_ - -#include -#include -#include -#include - -#include "dyndep.h" -#include "eval_env.h" -#include "timestamp.h" -#include "util.h" - -struct BuildLog; -struct DepfileParserOptions; -struct DiskInterface; -struct DepsLog; -struct Edge; -struct Node; -struct Pool; -struct State; - -/// Information about a node in the dependency graph: the file, whether -/// it's dirty, mtime, etc. -struct Node { - Node(const std::string& path, uint64_t slash_bits) - : path_(path), slash_bits_(slash_bits) {} - - /// Return false on error. - bool Stat(DiskInterface* disk_interface, std::string* err); - - /// If the file doesn't exist, set the mtime_ from its dependencies - void UpdatePhonyMtime(TimeStamp mtime); - - /// Return false on error. - bool StatIfNecessary(DiskInterface* disk_interface, std::string* err) { - if (status_known()) - return true; - return Stat(disk_interface, err); - } - - /// Mark as not-yet-stat()ed and not dirty. - void ResetState() { - mtime_ = -1; - exists_ = ExistenceStatusUnknown; - dirty_ = false; - } - - /// Mark the Node as already-stat()ed and missing. - void MarkMissing() { - if (mtime_ == -1) { - mtime_ = 0; - } - exists_ = ExistenceStatusMissing; - } - - bool exists() const { - return exists_ == ExistenceStatusExists; - } - - bool status_known() const { - return exists_ != ExistenceStatusUnknown; - } - - const std::string& path() const { return path_; } - /// Get |path()| but use slash_bits to convert back to original slash styles. - std::string PathDecanonicalized() const { - return PathDecanonicalized(path_, slash_bits_); - } - static std::string PathDecanonicalized(const std::string& path, - uint64_t slash_bits); - uint64_t slash_bits() const { return slash_bits_; } - - TimeStamp mtime() const { return mtime_; } - - bool dirty() const { return dirty_; } - void set_dirty(bool dirty) { dirty_ = dirty; } - void MarkDirty() { dirty_ = true; } - - bool dyndep_pending() const { return dyndep_pending_; } - void set_dyndep_pending(bool pending) { dyndep_pending_ = pending; } - - Edge* in_edge() const { return in_edge_; } - void set_in_edge(Edge* edge) { in_edge_ = edge; } - - /// Indicates whether this node was generated from a depfile or dyndep file, - /// instead of being a regular input or output from the Ninja manifest. - bool generated_by_dep_loader() const { return generated_by_dep_loader_; } - - void set_generated_by_dep_loader(bool value) { - generated_by_dep_loader_ = value; - } - - int id() const { return id_; } - void set_id(int id) { id_ = id; } - - const std::vector& out_edges() const { return out_edges_; } - const std::vector& validation_out_edges() const { return validation_out_edges_; } - void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); } - void AddValidationOutEdge(Edge* edge) { validation_out_edges_.push_back(edge); } - - void Dump(const char* prefix="") const; - -private: - std::string path_; - - /// Set bits starting from lowest for backslashes that were normalized to - /// forward slashes by CanonicalizePath. See |PathDecanonicalized|. - uint64_t slash_bits_ = 0; - - /// Possible values of mtime_: - /// -1: file hasn't been examined - /// 0: we looked, and file doesn't exist - /// >0: actual file's mtime, or the latest mtime of its dependencies if it doesn't exist - TimeStamp mtime_ = -1; - - enum ExistenceStatus { - /// The file hasn't been examined. - ExistenceStatusUnknown, - /// The file doesn't exist. mtime_ will be the latest mtime of its dependencies. - ExistenceStatusMissing, - /// The path is an actual file. mtime_ will be the file's mtime. - ExistenceStatusExists - }; - ExistenceStatus exists_ = ExistenceStatusUnknown; - - /// Dirty is true when the underlying file is out-of-date. - /// But note that Edge::outputs_ready_ is also used in judging which - /// edges to build. - bool dirty_ = false; - - /// Store whether dyndep information is expected from this node but - /// has not yet been loaded. - bool dyndep_pending_ = false; - - /// Set to true when this node comes from a depfile, a dyndep file or the - /// deps log. If it does not have a producing edge, the build should not - /// abort if it is missing (as for regular source inputs). By default - /// all nodes have this flag set to true, since the deps and build logs - /// can be loaded before the manifest. - bool generated_by_dep_loader_ = true; - - /// The Edge that produces this Node, or NULL when there is no - /// known edge to produce it. - Edge* in_edge_ = nullptr; - - /// All Edges that use this Node as an input. - std::vector out_edges_; - - /// All Edges that use this Node as a validation. - std::vector validation_out_edges_; - - /// A dense integer id for the node, assigned and used by DepsLog. - int id_ = -1; -}; - -/// An edge in the dependency graph; links between Nodes using Rules. -struct Edge { - enum VisitMark { - VisitNone, - VisitInStack, - VisitDone - }; - - Edge() - : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), mark_(VisitNone), - id_(0), outputs_ready_(false), deps_loaded_(false), - deps_missing_(false), generated_by_dep_loader_(false), - command_start_time_(0), implicit_deps_(0), order_only_deps_(0), - implicit_outs_(0) {} - - /// Return true if all inputs' in-edges are ready. - bool AllInputsReady() const; - - /// Expand all variables in a command and return it as a string. - /// If incl_rsp_file is enabled, the string will also contain the - /// full contents of a response file (if applicable) - std::string EvaluateCommand(bool incl_rsp_file = false) const; - - /// Returns the shell-escaped value of |key|. - std::string GetBinding(const std::string& key) const; - bool GetBindingBool(const std::string& key) const; - - /// Like GetBinding("depfile"), but without shell escaping. - std::string GetUnescapedDepfile() const; - /// Like GetBinding("dyndep"), but without shell escaping. - std::string GetUnescapedDyndep() const; - /// Like GetBinding("rspfile"), but without shell escaping. - std::string GetUnescapedRspfile() const; - - void Dump(const char* prefix="") const; - - // Append all edge explicit inputs to |*out|. Possibly with shell escaping. - void CollectInputs(bool shell_escape, std::vector* out) const; - - const Rule* rule_ = nullptr; - Pool* pool_ = nullptr; - std::vector inputs_; - std::vector outputs_; - std::vector validations_; - Node* dyndep_ = nullptr; - BindingEnv* env_ = nullptr; - VisitMark mark_ = VisitNone; - size_t id_ = 0; - bool outputs_ready_ = false; - bool deps_loaded_ = false; - bool deps_missing_ = false; - bool generated_by_dep_loader_ = false; - TimeStamp command_start_time_ = 0; - - const Rule& rule() const { return *rule_; } - Pool* pool() const { return pool_; } - int weight() const { return 1; } - bool outputs_ready() const { return outputs_ready_; } - - // There are three types of inputs. - // 1) explicit deps, which show up as $in on the command line; - // 2) implicit deps, which the target depends on implicitly (e.g. C headers), - // and changes in them cause the target to rebuild; - // 3) order-only deps, which are needed before the target builds but which - // don't cause the target to rebuild. - // These are stored in inputs_ in that order, and we keep counts of - // #2 and #3 when we need to access the various subsets. - int implicit_deps_ = 0; - int order_only_deps_ = 0; - bool is_implicit(size_t index) { - return index >= inputs_.size() - order_only_deps_ - implicit_deps_ && - !is_order_only(index); - } - bool is_order_only(size_t index) { - return index >= inputs_.size() - order_only_deps_; - } - - // There are two types of outputs. - // 1) explicit outs, which show up as $out on the command line; - // 2) implicit outs, which the target generates but are not part of $out. - // These are stored in outputs_ in that order, and we keep a count of - // #2 to use when we need to access the various subsets. - int implicit_outs_ = 0; - bool is_implicit_out(size_t index) const { - return index >= outputs_.size() - implicit_outs_; - } - - bool is_phony() const; - bool use_console() const; - bool maybe_phonycycle_diagnostic() const; - - // Historical info: how long did this edge take last time, - // as per .ninja_log, if known? Defaults to -1 if unknown. - int64_t prev_elapsed_time_millis = -1; -}; - -struct EdgeCmp { - bool operator()(const Edge* a, const Edge* b) const { - return a->id_ < b->id_; - } -}; - -typedef std::set EdgeSet; - -/// ImplicitDepLoader loads implicit dependencies, as referenced via the -/// "depfile" attribute in build files. -struct ImplicitDepLoader { - ImplicitDepLoader(State* state, DepsLog* deps_log, - DiskInterface* disk_interface, - DepfileParserOptions const* depfile_parser_options) - : state_(state), disk_interface_(disk_interface), deps_log_(deps_log), - depfile_parser_options_(depfile_parser_options) {} - - /// Load implicit dependencies for \a edge. - /// @return false on error (without filling \a err if info is just missing - // or out of date). - bool LoadDeps(Edge* edge, std::string* err); - - DepsLog* deps_log() const { - return deps_log_; - } - - protected: - /// Process loaded implicit dependencies for \a edge and update the graph - /// @return false on error (without filling \a err if info is just missing) - virtual bool ProcessDepfileDeps(Edge* edge, - std::vector* depfile_ins, - std::string* err); - - /// Load implicit dependencies for \a edge from a depfile attribute. - /// @return false on error (without filling \a err if info is just missing). - bool LoadDepFile(Edge* edge, const std::string& path, std::string* err); - - /// Load implicit dependencies for \a edge from the DepsLog. - /// @return false on error (without filling \a err if info is just missing). - bool LoadDepsFromLog(Edge* edge, std::string* err); - - /// Preallocate \a count spaces in the input array on \a edge, returning - /// an iterator pointing at the first new space. - std::vector::iterator PreallocateSpace(Edge* edge, int count); - - State* state_; - DiskInterface* disk_interface_; - DepsLog* deps_log_; - DepfileParserOptions const* depfile_parser_options_; -}; - - -/// DependencyScan manages the process of scanning the files in a graph -/// and updating the dirty/outputs_ready state of all the nodes and edges. -struct DependencyScan { - DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface, - DepfileParserOptions const* depfile_parser_options) - : build_log_(build_log), - disk_interface_(disk_interface), - dep_loader_(state, deps_log, disk_interface, depfile_parser_options), - dyndep_loader_(state, disk_interface) {} - - /// Update the |dirty_| state of the given nodes by transitively inspecting - /// their input edges. - /// Examine inputs, outputs, and command lines to judge whether an edge - /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| - /// state accordingly. - /// Appends any validation nodes found to the nodes parameter. - /// Returns false on failure. - bool RecomputeDirty(Node* node, std::vector* validation_nodes, std::string* err); - - /// Recompute whether any output of the edge is dirty, if so sets |*dirty|. - /// Returns false on failure. - bool RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, - bool* dirty, std::string* err); - - BuildLog* build_log() const { - return build_log_; - } - void set_build_log(BuildLog* log) { - build_log_ = log; - } - - DepsLog* deps_log() const { - return dep_loader_.deps_log(); - } - - /// Load a dyndep file from the given node's path and update the - /// build graph with the new information. One overload accepts - /// a caller-owned 'DyndepFile' object in which to store the - /// information loaded from the dyndep file. - bool LoadDyndeps(Node* node, std::string* err) const; - bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const; - - private: - bool RecomputeNodeDirty(Node* node, std::vector* stack, - std::vector* validation_nodes, std::string* err); - bool VerifyDAG(Node* node, std::vector* stack, std::string* err); - - /// Recompute whether a given single output should be marked dirty. - /// Returns true if so. - bool RecomputeOutputDirty(const Edge* edge, const Node* most_recent_input, - const std::string& command, Node* output); - - BuildLog* build_log_; - DiskInterface* disk_interface_; - ImplicitDepLoader dep_loader_; - DyndepLoader dyndep_loader_; -}; - -#endif // NINJA_GRAPH_H_ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_GRAPH_H_ +#define NINJA_GRAPH_H_ + +#include +#include +#include +#include + +#include "dyndep.h" +#include "eval_env.h" +#include "timestamp.h" +#include "util.h" + +struct BuildLog; +struct DepfileParserOptions; +struct DiskInterface; +struct DepsLog; +struct Edge; +struct Node; +struct Pool; +struct State; + +/// Information about a node in the dependency graph: the file, whether +/// it's dirty, mtime, etc. +struct Node { + Node(const std::string& path, uint64_t slash_bits) + : path_(path), slash_bits_(slash_bits) {} + + /// Return false on error. + bool Stat(DiskInterface* disk_interface, std::string* err); + + /// If the file doesn't exist, set the mtime_ from its dependencies + void UpdatePhonyMtime(TimeStamp mtime); + + /// Return false on error. + bool StatIfNecessary(DiskInterface* disk_interface, std::string* err) { + if (status_known()) + return true; + return Stat(disk_interface, err); + } + + /// Mark as not-yet-stat()ed and not dirty. + void ResetState() { + mtime_ = -1; + exists_ = ExistenceStatusUnknown; + dirty_ = false; + } + + /// Mark the Node as already-stat()ed and missing. + void MarkMissing() { + if (mtime_ == -1) { + mtime_ = 0; + } + exists_ = ExistenceStatusMissing; + } + + bool exists() const { + return exists_ == ExistenceStatusExists; + } + + bool status_known() const { + return exists_ != ExistenceStatusUnknown; + } + + const std::string& path() const { return path_; } + /// Get |path()| but use slash_bits to convert back to original slash styles. + std::string PathDecanonicalized() const { + return PathDecanonicalized(path_, slash_bits_); + } + static std::string PathDecanonicalized(const std::string& path, + uint64_t slash_bits); + uint64_t slash_bits() const { return slash_bits_; } + + TimeStamp mtime() const { return mtime_; } + + bool dirty() const { return dirty_; } + void set_dirty(bool dirty) { dirty_ = dirty; } + void MarkDirty() { dirty_ = true; } + + bool dyndep_pending() const { return dyndep_pending_; } + void set_dyndep_pending(bool pending) { dyndep_pending_ = pending; } + + Edge* in_edge() const { return in_edge_; } + void set_in_edge(Edge* edge) { in_edge_ = edge; } + + /// Indicates whether this node was generated from a depfile or dyndep file, + /// instead of being a regular input or output from the Ninja manifest. + bool generated_by_dep_loader() const { return generated_by_dep_loader_; } + + void set_generated_by_dep_loader(bool value) { + generated_by_dep_loader_ = value; + } + + int id() const { return id_; } + void set_id(int id) { id_ = id; } + + const std::vector& out_edges() const { return out_edges_; } + const std::vector& validation_out_edges() const { return validation_out_edges_; } + void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); } + void AddValidationOutEdge(Edge* edge) { validation_out_edges_.push_back(edge); } + + void Dump(const char* prefix="") const; + +private: + std::string path_; + + /// Set bits starting from lowest for backslashes that were normalized to + /// forward slashes by CanonicalizePath. See |PathDecanonicalized|. + uint64_t slash_bits_ = 0; + + /// Possible values of mtime_: + /// -1: file hasn't been examined + /// 0: we looked, and file doesn't exist + /// >0: actual file's mtime, or the latest mtime of its dependencies if it doesn't exist + TimeStamp mtime_ = -1; + + enum ExistenceStatus { + /// The file hasn't been examined. + ExistenceStatusUnknown, + /// The file doesn't exist. mtime_ will be the latest mtime of its dependencies. + ExistenceStatusMissing, + /// The path is an actual file. mtime_ will be the file's mtime. + ExistenceStatusExists + }; + ExistenceStatus exists_ = ExistenceStatusUnknown; + + /// Dirty is true when the underlying file is out-of-date. + /// But note that Edge::outputs_ready_ is also used in judging which + /// edges to build. + bool dirty_ = false; + + /// Store whether dyndep information is expected from this node but + /// has not yet been loaded. + bool dyndep_pending_ = false; + + /// Set to true when this node comes from a depfile, a dyndep file or the + /// deps log. If it does not have a producing edge, the build should not + /// abort if it is missing (as for regular source inputs). By default + /// all nodes have this flag set to true, since the deps and build logs + /// can be loaded before the manifest. + bool generated_by_dep_loader_ = true; + + /// The Edge that produces this Node, or NULL when there is no + /// known edge to produce it. + Edge* in_edge_ = nullptr; + + /// All Edges that use this Node as an input. + std::vector out_edges_; + + /// All Edges that use this Node as a validation. + std::vector validation_out_edges_; + + /// A dense integer id for the node, assigned and used by DepsLog. + int id_ = -1; +}; + +/// An edge in the dependency graph; links between Nodes using Rules. +struct Edge { + enum VisitMark { + VisitNone, + VisitInStack, + VisitDone + }; + + Edge() + : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), mark_(VisitNone), + id_(0), outputs_ready_(false), deps_loaded_(false), + deps_missing_(false), generated_by_dep_loader_(false), + command_start_time_(0), implicit_deps_(0), order_only_deps_(0), + implicit_outs_(0) {} + + /// Return true if all inputs' in-edges are ready. + bool AllInputsReady() const; + + /// Expand all variables in a command and return it as a string. + /// If incl_rsp_file is enabled, the string will also contain the + /// full contents of a response file (if applicable) + std::string EvaluateCommand(bool incl_rsp_file = false) const; + + /// Returns the shell-escaped value of |key|. + std::string GetBinding(const std::string& key) const; + bool GetBindingBool(const std::string& key) const; + + /// Like GetBinding("depfile"), but without shell escaping. + std::string GetUnescapedDepfile() const; + /// Like GetBinding("dyndep"), but without shell escaping. + std::string GetUnescapedDyndep() const; + /// Like GetBinding("rspfile"), but without shell escaping. + std::string GetUnescapedRspfile() const; + + void Dump(const char* prefix="") const; + + // Append all edge explicit inputs to |*out|. Possibly with shell escaping. + void CollectInputs(bool shell_escape, std::vector* out) const; + + const Rule* rule_ = nullptr; + Pool* pool_ = nullptr; + std::vector inputs_; + std::vector outputs_; + std::vector validations_; + Node* dyndep_ = nullptr; + BindingEnv* env_ = nullptr; + VisitMark mark_ = VisitNone; + size_t id_ = 0; + bool outputs_ready_ = false; + bool deps_loaded_ = false; + bool deps_missing_ = false; + bool generated_by_dep_loader_ = false; + TimeStamp command_start_time_ = 0; + + const Rule& rule() const { return *rule_; } + Pool* pool() const { return pool_; } + int weight() const { return 1; } + bool outputs_ready() const { return outputs_ready_; } + + // There are three types of inputs. + // 1) explicit deps, which show up as $in on the command line; + // 2) implicit deps, which the target depends on implicitly (e.g. C headers), + // and changes in them cause the target to rebuild; + // 3) order-only deps, which are needed before the target builds but which + // don't cause the target to rebuild. + // These are stored in inputs_ in that order, and we keep counts of + // #2 and #3 when we need to access the various subsets. + int implicit_deps_ = 0; + int order_only_deps_ = 0; + bool is_implicit(size_t index) { + return index >= inputs_.size() - order_only_deps_ - implicit_deps_ && + !is_order_only(index); + } + bool is_order_only(size_t index) { + return index >= inputs_.size() - order_only_deps_; + } + + // There are two types of outputs. + // 1) explicit outs, which show up as $out on the command line; + // 2) implicit outs, which the target generates but are not part of $out. + // These are stored in outputs_ in that order, and we keep a count of + // #2 to use when we need to access the various subsets. + int implicit_outs_ = 0; + bool is_implicit_out(size_t index) const { + return index >= outputs_.size() - implicit_outs_; + } + + bool is_phony() const; + bool use_console() const; + bool maybe_phonycycle_diagnostic() const; + + // Historical info: how long did this edge take last time, + // as per .ninja_log, if known? Defaults to -1 if unknown. + int64_t prev_elapsed_time_millis = -1; +}; + +struct EdgeCmp { + bool operator()(const Edge* a, const Edge* b) const { + return a->id_ < b->id_; + } +}; + +typedef std::set EdgeSet; + +/// ImplicitDepLoader loads implicit dependencies, as referenced via the +/// "depfile" attribute in build files. +struct ImplicitDepLoader { + ImplicitDepLoader(State* state, DepsLog* deps_log, + DiskInterface* disk_interface, + DepfileParserOptions const* depfile_parser_options) + : state_(state), disk_interface_(disk_interface), deps_log_(deps_log), + depfile_parser_options_(depfile_parser_options) {} + + /// Load implicit dependencies for \a edge. + /// @return false on error (without filling \a err if info is just missing + // or out of date). + bool LoadDeps(Edge* edge, std::string* err); + + DepsLog* deps_log() const { + return deps_log_; + } + + protected: + /// Process loaded implicit dependencies for \a edge and update the graph + /// @return false on error (without filling \a err if info is just missing) + virtual bool ProcessDepfileDeps(Edge* edge, + std::vector* depfile_ins, + std::string* err); + + /// Load implicit dependencies for \a edge from a depfile attribute. + /// @return false on error (without filling \a err if info is just missing). + bool LoadDepFile(Edge* edge, const std::string& path, std::string* err); + + /// Load implicit dependencies for \a edge from the DepsLog. + /// @return false on error (without filling \a err if info is just missing). + bool LoadDepsFromLog(Edge* edge, std::string* err); + + /// Preallocate \a count spaces in the input array on \a edge, returning + /// an iterator pointing at the first new space. + std::vector::iterator PreallocateSpace(Edge* edge, int count); + + State* state_; + DiskInterface* disk_interface_; + DepsLog* deps_log_; + DepfileParserOptions const* depfile_parser_options_; +}; + + +/// DependencyScan manages the process of scanning the files in a graph +/// and updating the dirty/outputs_ready state of all the nodes and edges. +struct DependencyScan { + DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log, + DiskInterface* disk_interface, + DepfileParserOptions const* depfile_parser_options) + : build_log_(build_log), + disk_interface_(disk_interface), + dep_loader_(state, deps_log, disk_interface, depfile_parser_options), + dyndep_loader_(state, disk_interface) {} + + /// Update the |dirty_| state of the given nodes by transitively inspecting + /// their input edges. + /// Examine inputs, outputs, and command lines to judge whether an edge + /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| + /// state accordingly. + /// Appends any validation nodes found to the nodes parameter. + /// Returns false on failure. + bool RecomputeDirty(Node* node, std::vector* validation_nodes, std::string* err); + + /// Recompute whether any output of the edge is dirty, if so sets |*dirty|. + /// Returns false on failure. + bool RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, + bool* dirty, std::string* err); + + BuildLog* build_log() const { + return build_log_; + } + void set_build_log(BuildLog* log) { + build_log_ = log; + } + + DepsLog* deps_log() const { + return dep_loader_.deps_log(); + } + + /// Load a dyndep file from the given node's path and update the + /// build graph with the new information. One overload accepts + /// a caller-owned 'DyndepFile' object in which to store the + /// information loaded from the dyndep file. + bool LoadDyndeps(Node* node, std::string* err) const; + bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const; + + private: + bool RecomputeNodeDirty(Node* node, std::vector* stack, + std::vector* validation_nodes, std::string* err); + bool VerifyDAG(Node* node, std::vector* stack, std::string* err); + + /// Recompute whether a given single output should be marked dirty. + /// Returns true if so. + bool RecomputeOutputDirty(const Edge* edge, const Node* most_recent_input, + const std::string& command, Node* output); + + BuildLog* build_log_; + DiskInterface* disk_interface_; + ImplicitDepLoader dep_loader_; + DyndepLoader dyndep_loader_; +}; + +#endif // NINJA_GRAPH_H_ diff --git a/src/status.cc b/src/status.cc index f8e60ca..1768165 100755 --- a/src/status.cc +++ b/src/status.cc @@ -107,20 +107,20 @@ void StatusPrinter::RecalculateProgressPrediction() { // that is, if we have took at least 15 sec AND finished at least 5% of edges, // we can check whether our performance so far matches the previous one. if (use_previous_times && total_edges_ && finished_edges_ && - (time_millis_ >= 15 * 1e3) && - (((double)finished_edges_ / total_edges_) >= 0.05)) { + (time_millis_ >= 15 * 1e3) && + (((double)finished_edges_ / total_edges_) >= 0.05)) { // Over the edges we've just run, how long did they take on average? double actual_average_cpu_time_millis = - (double)cpu_time_millis_ / finished_edges_; + (double)cpu_time_millis_ / finished_edges_; // What is the previous average, for the edges with such knowledge? double previous_average_cpu_time_millis = - (double)eta_predictable_cpu_time_total_millis_ / - eta_predictable_edges_total_; + (double)eta_predictable_cpu_time_total_millis_ / + eta_predictable_edges_total_; double ratio = std::max(previous_average_cpu_time_millis, - actual_average_cpu_time_millis) / - std::min(previous_average_cpu_time_millis, - actual_average_cpu_time_millis); + actual_average_cpu_time_millis) / + std::min(previous_average_cpu_time_millis, + actual_average_cpu_time_millis); // Let's say that the average times should differ by less than 10x use_previous_times = ratio < 10; @@ -133,8 +133,8 @@ void StatusPrinter::RecalculateProgressPrediction() { return; int edges_with_unknown_runtime = use_previous_times - ? eta_unpredictable_edges_remaining_ - : (total_edges_ - finished_edges_); + ? eta_unpredictable_edges_remaining_ + : (total_edges_ - finished_edges_); // Given the time elapsed on the edges we've just run, // and the runtime of the edges for which we know previous runtime, @@ -142,10 +142,10 @@ void StatusPrinter::RecalculateProgressPrediction() { int64_t edges_known_runtime_total_millis = cpu_time_millis_; if (use_previous_times) edges_known_runtime_total_millis += - eta_predictable_cpu_time_remaining_millis_; + eta_predictable_cpu_time_remaining_millis_; double average_cpu_time_millis = - (double)edges_known_runtime_total_millis / edges_with_known_runtime; + (double)edges_known_runtime_total_millis / edges_with_known_runtime; // For the edges for which we do not have the previous runtime, // let's assume that their average runtime is the same as for the other edges, @@ -155,12 +155,12 @@ void StatusPrinter::RecalculateProgressPrediction() { // And therefore we can predict the remaining and total runtimes. double total_cpu_time_remaining_millis = - unpredictable_cpu_time_remaining_millis; + unpredictable_cpu_time_remaining_millis; if (use_previous_times) total_cpu_time_remaining_millis += - eta_predictable_cpu_time_remaining_millis_; + eta_predictable_cpu_time_remaining_millis_; double total_cpu_time_millis = - cpu_time_millis_ + total_cpu_time_remaining_millis; + cpu_time_millis_ + total_cpu_time_remaining_millis; if (total_cpu_time_millis == 0.0) return; @@ -181,7 +181,7 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis, if (edge->prev_elapsed_time_millis != -1) { --eta_predictable_edges_remaining_; eta_predictable_cpu_time_remaining_millis_ -= - edge->prev_elapsed_time_millis; + edge->prev_elapsed_time_millis; } else --eta_unpredictable_edges_remaining_; @@ -200,13 +200,13 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis, if (!success) { string outputs; for (vector::const_iterator o = edge->outputs_.begin(); - o != edge->outputs_.end(); ++o) + o != edge->outputs_.end(); ++o) outputs += (*o)->path() + " "; if (printer_.supports_color()) { - printer_.PrintOnNewLine("\x1B[31m" "FAILED: " "\x1B[0m" + outputs + "\n"); + printer_.PrintOnNewLine("\x1B[31m" "FAILED: " "\x1B[0m" + outputs + "\n"); } else { - printer_.PrintOnNewLine("FAILED: " + outputs + "\n"); + printer_.PrintOnNewLine("FAILED: " + outputs + "\n"); } printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n"); } @@ -230,14 +230,14 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis, final_output = output; #ifdef _WIN32 - // Fix extra CR being added on Windows, writing out CR CR LF (#773) - _setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix + // Fix extra CR being added on Windows, writing out CR CR LF (#773) + _setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix #endif - printer_.PrintOnNewLine(final_output); + printer_.PrintOnNewLine(final_output); #ifdef _WIN32 - _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix + _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix #endif } } @@ -356,7 +356,7 @@ string StatusPrinter::FormatProgressStatus(const char* progress_status_format, } const bool print_with_hours = - elapsed_sec >= 60 * 60 || eta_sec >= 60 * 60; + elapsed_sec >= 60 * 60 || eta_sec >= 60 * 60; double sec = -1; switch (*s) { @@ -394,7 +394,7 @@ string StatusPrinter::FormatProgressStatus(const char* progress_status_format, // Percentage of time spent out of the predicted time total case 'P': { snprintf(buf, sizeof(buf), "%3i%%", - (int)(100. * time_predicted_percentage_)); + (int)(100. * time_predicted_percentage_)); out += buf; break; } @@ -413,7 +413,7 @@ string StatusPrinter::FormatProgressStatus(const char* progress_status_format, void StatusPrinter::PrintStatus(const Edge* edge, int64_t time_millis) { if (config_.verbosity == BuildConfig::QUIET - || config_.verbosity == BuildConfig::NO_STATUS_UPDATE) + || config_.verbosity == BuildConfig::NO_STATUS_UPDATE) return; RecalculateProgressPrediction(); @@ -425,10 +425,9 @@ void StatusPrinter::PrintStatus(const Edge* edge, int64_t time_millis) { to_print = edge->GetBinding("command"); to_print = FormatProgressStatus(progress_status_format_, time_millis) - + to_print; + + to_print; - printer_.Print(to_print, - force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); + printer_.Print(to_print,force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); } void StatusPrinter::Warning(const char* msg, ...) {