openharmony/third_party_ninja
1
0
Fork 0
mirror of https://gitee.com/openharmony/third_party_ninja synced 2024-11-23 07:20:07 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

IssueNo: https://gitee.com/openharmony/third_party_ninja/issues/I9RW2O

Browse Source 
Signed-off-by: liangxinyan <liangxinyan2@huawei.com>
This commit is contained in:
liangxinyan 2024-05-25 09:35:03 +08:00
parent 03745fb315
commit 0ff2536f46
7 changed files with 1646 additions and 1649 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
src/build.cc
View File

@ -52,8 +52,8 @@ struct DryRunCommandRunner : public CommandRunner {
virtual bool StartCommand(Edge* edge);
virtual bool WaitForCommand(Result* result);
private:
queue<Edge*> finished_;
private:
queue<Edge*> finished_;
};
size_t DryRunCommandRunner::CanRunMore() const {
@ -66,13 +66,13 @@ bool DryRunCommandRunner::StartCommand(Edge* edge) {
}
bool DryRunCommandRunner::WaitForCommand(Result* result) {
if (finished_.empty())
return false;
if (finished_.empty())
return false;
result->status = ExitSuccess;
result->edge = finished_.front();
finished_.pop();
return true;
result->status = ExitSuccess;
result->edge = finished_.front();
finished_.pop();
return true;
}
} // namespace
@ -98,19 +98,19 @@ bool Plan::AddSubTarget(const Node* node, const Node* dependent, string* err,
set<Edge*>* dyndep_walk) {
Edge* edge = node->in_edge();
if (!edge) {
// Leaf node, this can be either a regular input from the manifest
// (e.g. a source file), or an implicit input from a depfile or dyndep
// file. In the first case, a dirty flag means the file is missing,
// and the build should stop. In the second, do not do anything here
// since there is no producing edge to add to the plan.
if (node->dirty() && !node->generated_by_dep_loader()) {
string referenced;
if (dependent)
referenced = ", needed by '" + dependent->path() + "',";
*err = "'" + node->path() + "'" + referenced +
" missing and no known rule to make it";
}
return false;
// Leaf node, this can be either a regular input from the manifest
// (e.g. a source file), or an implicit input from a depfile or dyndep
// file. In the first case, a dirty flag means the file is missing,
// and the build should stop. In the second, do not do anything here
// since there is no producing edge to add to the plan.
if (node->dirty() && !node->generated_by_dep_loader()) {
string referenced;
if (dependent)
referenced = ", needed by '" + dependent->path() + "',";
*err = "'" + node->path() + "'" + referenced +
" missing and no known rule to make it";
}
return false;
}
if (edge->outputs_ready())
@ -141,7 +141,7 @@ bool Plan::AddSubTarget(const Node* node, const Node* dependent, string* err,
return true; // We've already processed the inputs.
for (vector<Node*>::iterator i = edge->inputs_.begin();
i != edge->inputs_.end(); ++i) {
i != edge->inputs_.end(); ++i) {
if (!AddSubTarget(*i, node, err, dyndep_walk) && !err->empty())
return false;
}
@ -211,7 +211,7 @@ bool Plan::EdgeFinished(Edge* edge, EdgeResult result, string* err) {
// Check off any nodes we were waiting for with this edge.
for (vector<Node*>::iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
o != edge->outputs_.end(); ++o) {
if (!NodeFinished(*o, err))
return false;
}
@ -229,7 +229,7 @@ bool Plan::NodeFinished(Node* node, string* err) {
// See if we we want any edges from this node.
for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
oe != node->out_edges().end(); ++oe) {
oe != node->out_edges().end(); ++oe) {
map<Edge*, Want>::iterator want_e = want_.find(*oe);
if (want_e == want_.end())
continue;
@ -260,7 +260,7 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) {
node->set_dirty(false);
for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
oe != node->out_edges().end(); ++oe) {
oe != node->out_edges().end(); ++oe) {
// Don't process edges that we don't actually want.
map<Edge*, Want>::iterator want_e = want_.find(*oe);
if (want_e == want_.end() || want_e->second == kWantNothing)
@ -273,8 +273,8 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) {
// If all non-order-only inputs for this edge are now clean,
// we might have changed the dirty state of the outputs.
vector<Node*>::iterator
begin = (*oe)->inputs_.begin(),
end = (*oe)->inputs_.end() - (*oe)->order_only_deps_;
begin = (*oe)->inputs_.begin(),
end = (*oe)->inputs_.end() - (*oe)->order_only_deps_;
#if __cplusplus < 201703L
#define MEM_FN mem_fun
#else
@ -293,12 +293,12 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) {
// wanted.
bool outputs_dirty = false;
if (!scan->RecomputeOutputsDirty(*oe, most_recent_input,
&outputs_dirty, err)) {
&outputs_dirty, err)) {
return false;
}
if (!outputs_dirty) {
for (vector<Node*>::iterator o = (*oe)->outputs_.begin();
o != (*oe)->outputs_.end(); ++o) {
o != (*oe)->outputs_.end(); ++o) {
if (!CleanNode(scan, *o, err))
return false;
}
@ -354,7 +354,7 @@ bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node,
oei = dyndep_roots.begin(); oei != dyndep_roots.end(); ++oei) {
DyndepFile::const_iterator oe = *oei;
for (vector<Node*>::const_iterator i = oe->second.implicit_inputs_.begin();
i != oe->second.implicit_inputs_.end(); ++i) {
i != oe->second.implicit_inputs_.end(); ++i) {
if (!AddSubTarget(*i, oe->first->outputs_[0], err, &dyndep_walk) &&
!err->empty())
return false;
@ -364,7 +364,7 @@ bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node,
// Add out edges from this node that are in the plan (just as
// Plan::NodeFinished would have without taking the dyndep code path).
for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
oe != node->out_edges().end(); ++oe) {
oe != node->out_edges().end(); ++oe) {
map<Edge*, Want>::iterator want_e = want_.find(*oe);
if (want_e == want_.end())
continue;
@ -373,7 +373,7 @@ bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node,
// See if any encountered edges are now ready.
for (set<Edge*>::iterator wi = dyndep_walk.begin();
wi != dyndep_walk.end(); ++wi) {
wi != dyndep_walk.end(); ++wi) {
map<Edge*, Want>::iterator want_e = want_.find(*wi);
if (want_e == want_.end())
continue;
@ -394,7 +394,7 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node,
// Update the dirty state of all dependents and check if their edges
// have become wanted.
for (set<Node*>::iterator i = dependents.begin();
i != dependents.end(); ++i) {
i != dependents.end(); ++i) {
Node* n = *i;
// Check if this dependent node is now dirty. Also checks for new cycles.
@ -405,7 +405,7 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node,
// Add any validation nodes found during RecomputeDirty as new top level
// targets.
for (std::vector<Node*>::iterator v = validation_nodes.begin();
v != validation_nodes.end(); ++v) {
v != validation_nodes.end(); ++v) {
if (Edge* in_edge = (*v)->in_edge()) {
if (!in_edge->outputs_ready() &&
!AddTarget(*v, err)) {
@ -433,7 +433,7 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node,
void Plan::UnmarkDependents(const Node* node, set<Node*>* dependents) {
for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
oe != node->out_edges().end(); ++oe) {
oe != node->out_edges().end(); ++oe) {
Edge* edge = *oe;
map<Edge*, Want>::iterator want_e = want_.find(edge);
@ -443,7 +443,7 @@ void Plan::UnmarkDependents(const Node* node, set<Node*>* dependents) {
if (edge->mark_ != Edge::VisitNone) {
edge->mark_ = Edge::VisitNone;
for (vector<Node*>::iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
o != edge->outputs_.end(); ++o) {
if (dependents->insert(*o).second)
UnmarkDependents(*o, dependents);
}
@ -478,7 +478,7 @@ struct RealCommandRunner : public CommandRunner {
vector<Edge*> RealCommandRunner::GetActiveEdges() {
vector<Edge*> edges;
for (map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.begin();
e != subproc_to_edge_.end(); ++e)
e != subproc_to_edge_.end(); ++e)
edges.push_back(e->second);
return edges;
}
@ -489,7 +489,7 @@ void RealCommandRunner::Abort() {
size_t RealCommandRunner::CanRunMore() const {
size_t subproc_number =
subprocs_.running_.size() + subprocs_.finished_.size();
subprocs_.running_.size() + subprocs_.finished_.size();
int64_t capacity = config_.parallelism - subproc_number;
@ -545,7 +545,7 @@ Builder::Builder(State* state, const BuildConfig& config,
: state_(state), config_(config), plan_(this), status_(status),
start_time_millis_(start_time_millis), disk_interface_(disk_interface),
scan_(state, build_log, deps_log, disk_interface,
&config_.depfile_parser_options) {
&config_.depfile_parser_options) {
lock_file_path_ = ".ninja_lock";
string build_dir = state_->bindings_.LookupVariable("builddir");
if (!build_dir.empty())
@ -562,10 +562,10 @@ void Builder::Cleanup() {
command_runner_->Abort();
for (vector<Edge*>::iterator e = active_edges.begin();
e != active_edges.end(); ++e) {
e != active_edges.end(); ++e) {
string depfile = (*e)->GetUnescapedDepfile();
for (vector<Node*>::iterator o = (*e)->outputs_.begin();
o != (*e)->outputs_.end(); ++o) {
o != (*e)->outputs_.end(); ++o) {
// Only delete this output if it was actually modified. This is
// important for things like the generator where we don't want to
// delete the manifest file if we can avoid it. But if the rule
@ -616,10 +616,10 @@ bool Builder::AddTarget(Node* target, string* err) {
// Also add any validation nodes found during RecomputeDirty as top level
// targets.
for (std::vector<Node*>::iterator n = validation_nodes.begin();
n != validation_nodes.end(); ++n) {
n != validation_nodes.end(); ++n) {
if (Edge* validation_in_edge = (*n)->in_edge()) {
if (!validation_in_edge->outputs_ready() &&
!plan_.AddTarget(*n, err)) {
!plan_.AddTarget(*n, err)) {
return false;
}
}
@ -630,32 +630,32 @@ bool Builder::AddTarget(Node* target, string* err) {
static std::string &Trim(std::string &s)
{
if (s.empty()) {
return s;
}
s.erase(0, s.find_first_not_of(" \t\r\n"));
s.erase(s.find_last_not_of(" \t\r\n") + 1);
if (s.empty()) {
return s;
}
s.erase(0, s.find_first_not_of(" \t\r\n"));
s.erase(s.find_last_not_of(" \t\r\n") + 1);
return s;
}
static std::vector<std::string> SplitStringBySpace(std::string content) {
std::string space_delimiter = " ";
std::vector<std::string> words{};
size_t pos = 0;
std::string temp_content = content;
while ((pos = temp_content.find(space_delimiter)) != string::npos) {
std::string sub_str = temp_content.substr(0, pos);
std::string tmp = Trim(sub_str);
if (!tmp.empty()) {
words.push_back(tmp);
}
temp_content = temp_content.substr(pos + 1);
std::string space_delimiter = " ";
std::vector<std::string> words{};
size_t pos = 0;
std::string temp_content = content;
while ((pos = temp_content.find(space_delimiter)) != string::npos) {
std::string sub_str = temp_content.substr(0, pos);
std::string tmp = Trim(sub_str);
if (!tmp.empty()) {
words.push_back(tmp);
}
std::string tmp_last = Trim(temp_content);
if (!tmp_last.empty()) {
words.push_back(tmp_last);
}
return words;
temp_content = temp_content.substr(pos + 1);
}
std::string tmp_last = Trim(temp_content);
if (!tmp_last.empty()) {
words.push_back(tmp_last);
}
return words;
}
static std::string SplicingWholeContent(std::string content, std::string whole_content, bool is_whole_archive) {
@ -665,19 +665,19 @@ static std::string SplicingWholeContent(std::string content, std::string whole_c
std::string temp_content = content;
if (is_whole_archive) {
temp_content += " -Wl,--whole-archive";
temp_content += " -Wl,--whole-archive";
}
std::vector<std::string> whole_list = SplitStringBySpace(whole_content);
std::vector<std::string> content_list = SplitStringBySpace(temp_content);
for (const std::string &word : whole_list) {
auto it = std::find_if(content_list.begin(), content_list.end(), [&](const std::string& s) {
return s.find(word) != std::string::npos;
});
if (it != content_list.end()) {
content_list.push_back(*it);
content_list.erase(it);
}
auto it = std::find_if(content_list.begin(), content_list.end(), [&](const std::string& s) {
return s.find(word) != std::string::npos;
});
if (it != content_list.end()) {
content_list.push_back(*it);
content_list.erase(it);
}
}
std::string result = "";
@ -766,9 +766,9 @@ bool Builder::Build(string* err) {
}
}
// We are finished with all work items and have no pending
// commands. Therefore, break out of the main loop.
if (pending_commands == 0 && !plan_.more_to_do()) break;
// We are finished with all work items and have no pending
// commands. Therefore, break out of the main loop.
if (pending_commands == 0 && !plan_.more_to_do()) break;
}
// See if we can reap any finished commands.
@ -833,7 +833,7 @@ bool Builder::StartEdge(Edge* edge, string* err) {
// filesystem mtime to record later
// XXX: this will block; do we care?
for (vector<Node*>::iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
o != edge->outputs_.end(); ++o) {
if (!disk_interface_->MakeDirs((*o)->path()))
return false;
if (build_start == -1) {
@ -880,8 +880,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
if (!deps_type.empty()) {
string extract_err;
if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes,
&extract_err) &&
result->success()) {
&extract_err) && result->success()) {
if (!result->output.empty())
result->output.append("\n");
result->output.append(extract_err);
@ -918,7 +917,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
// log.
if (record_mtime == 0 || restat || generator) {
for (vector<Node*>::iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
o != edge->outputs_.end(); ++o) {
TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err);
if (new_mtime == -1)
return false;
@ -949,7 +948,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
if (scan_.build_log()) {
if (!scan_.build_log()->RecordCommand(edge, start_time_millis,
end_time_millis, record_mtime)) {
end_time_millis, record_mtime)) {
*err = string("Error writing to build log: ") + strerror(errno);
return false;
}
@ -958,7 +957,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
if (!deps_type.empty() && !config_.dry_run) {
assert(!edge->outputs_.empty() && "should have been rejected by parser");
for (std::vector<Node*>::const_iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
o != edge->outputs_.end(); ++o) {
TimeStamp deps_mtime = disk_interface_->Stat((*o)->path(), err);
if (deps_mtime == -1)
return false;
@ -983,7 +982,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result,
return false;
result->output = output;
for (set<string>::iterator i = parser.includes_.begin();
i != parser.includes_.end(); ++i) {
i != parser.includes_.end(); ++i) {
// ~0 is assuming that with MSVC-parsed headers, it's ok to always make
// all backslashes (as some of the slashes will certainly be backslashes
// anyway). This could be fixed if necessary with some additional
@ -1018,7 +1017,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result,
// XXX check depfile matches expected output.
deps_nodes->reserve(deps.ins_.size());
for (vector<StringPiece>::iterator i = deps.ins_.begin();
i != deps.ins_.end(); ++i) {
i != deps.ins_.end(); ++i) {
uint64_t slash_bits;
CanonicalizePath(const_cast<char*>(i->str_), &i->len_, &slash_bits);
deps_nodes->push_back(state_->GetNode(*i, slash_bits));

492
src/build.h
View File

@ -1,246 +1,246 @@
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef NINJA_BUILD_H_
#define NINJA_BUILD_H_
#include <cstdio>
#include <map>
#include <memory>
#include <string>
#include <vector>
#include <queue>
#include "depfile_parser.h"
#include "graph.h" // XXX needed for DependencyScan; should rearrange.
#include "graph.h"
#include "exit_status.h"
#include "util.h" // int64_t
struct BuildLog;
struct Builder;
struct DiskInterface;
struct Edge;
struct Node;
struct State;
struct Status;
/// Plan stores the state of a build plan: what we intend to build,
/// which steps we're ready to execute.
struct Plan {
Plan(Builder* builder = NULL);
/// Add a target to our plan (including all its dependencies).
/// Returns false if we don't need to build this target; may
/// fill in |err| with an error message if there's a problem.
bool AddTarget(const Node* target, std::string* err);
// Pop a ready edge off the queue of edges to build.
// Returns NULL if there's no work to do.
Edge* FindWork();
/// Returns true if there's more work to be done.
bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; }
/// Dumps the current state of the plan.
void Dump() const;
enum EdgeResult {
kEdgeFailed,
kEdgeSucceeded
};
/// Mark an edge as done building (whether it succeeded or failed).
/// If any of the edge's outputs are dyndep bindings of their dependents,
/// this loads dynamic dependencies from the nodes' paths.
/// Returns 'false' if loading dyndep info fails and 'true' otherwise.
bool EdgeFinished(Edge* edge, EdgeResult result, std::string* err);
/// Clean the given node during the build.
/// Return false on error.
bool CleanNode(DependencyScan* scan, Node* node, std::string* err);
/// Number of edges with commands to run.
int command_edge_count() const { return command_edges_; }
/// Reset state. Clears want and ready sets.
void Reset();
/// Update the build plan to account for modifications made to the graph
/// by information loaded from a dyndep file.
bool DyndepsLoaded(DependencyScan* scan, const Node* node,
const DyndepFile& ddf, std::string* err);
private:
bool RefreshDyndepDependents(DependencyScan* scan, const Node* node, std::string* err);
void UnmarkDependents(const Node* node, std::set<Node*>* dependents);
bool AddSubTarget(const Node* node, const Node* dependent, std::string* err,
std::set<Edge*>* dyndep_walk);
/// Update plan with knowledge that the given node is up to date.
/// If the node is a dyndep binding on any of its dependents, this
/// loads dynamic dependencies from the node's path.
/// Returns 'false' if loading dyndep info fails and 'true' otherwise.
bool NodeFinished(Node* node, std::string* err);
/// Enumerate possible steps we want for an edge.
enum Want
{
/// We do not want to build the edge, but we might want to build one of
/// its dependents.
kWantNothing,
/// We want to build the edge, but have not yet scheduled it.
kWantToStart,
/// We want to build the edge, have scheduled it, and are waiting
/// for it to complete.
kWantToFinish
};
void EdgeWanted(const Edge* edge);
bool EdgeMaybeReady(std::map<Edge*, Want>::iterator want_e, std::string* err);
/// Submits a ready edge as a candidate for execution.
/// The edge may be delayed from running, for example if it's a member of a
/// currently-full pool.
void ScheduleWork(std::map<Edge*, Want>::iterator want_e);
/// Keep track of which edges we want to build in this plan. If this map does
/// not contain an entry for an edge, we do not want to build the entry or its
/// dependents. If it does contain an entry, the enumeration indicates what
/// we want for the edge.
std::map<Edge*, Want> want_;
EdgeSet ready_;
Builder* builder_;
/// Total number of edges that have commands (not phony).
int command_edges_;
/// Total remaining number of wanted edges.
int wanted_edges_;
};
/// CommandRunner is an interface that wraps running the build
/// subcommands. This allows tests to abstract out running commands.
/// RealCommandRunner is an implementation that actually runs commands.
struct CommandRunner {
virtual ~CommandRunner() {}
virtual size_t CanRunMore() const = 0;
virtual bool StartCommand(Edge* edge) = 0;
/// The result of waiting for a command.
struct Result {
Result() : edge(NULL) {}
Edge* edge;
ExitStatus status;
std::string output;
bool success() const { return status == ExitSuccess; }
};
/// Wait for a command to complete, or return false if interrupted.
virtual bool WaitForCommand(Result* result) = 0;
virtual std::vector<Edge*> GetActiveEdges() { return std::vector<Edge*>(); }
virtual void Abort() {}
};
/// Options (e.g. verbosity, parallelism) passed to a build.
struct BuildConfig {
BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
failures_allowed(1), max_load_average(-0.0f) {}
enum Verbosity {
QUIET, // No output -- used when testing.
NO_STATUS_UPDATE, // just regular output but suppress status update
NORMAL, // regular output and status update
VERBOSE
};
Verbosity verbosity;
bool dry_run;
int parallelism;
int failures_allowed;
/// The maximum load average we must not exceed. A negative value
/// means that we do not have any limit.
double max_load_average;
DepfileParserOptions depfile_parser_options;
};
/// Builder wraps the build process: starting commands, updating status.
struct Builder {
Builder(State* state, const BuildConfig& config,
BuildLog* build_log, DepsLog* deps_log,
DiskInterface* disk_interface, Status* status,
int64_t start_time_millis);
~Builder();
/// Clean up after interrupted commands by deleting output files.
void Cleanup();
Node* AddTarget(const std::string& name, std::string* err);
/// Add a target to the build, scanning dependencies.
/// @return false on error.
bool AddTarget(Node* target, std::string* err);
/// Returns true if the build targets are already up to date.
bool AlreadyUpToDate() const;
/// Run the build. Returns false on error.
/// It is an error to call this function when AlreadyUpToDate() is true.
bool Build(std::string* err);
bool StartEdge(Edge* edge, std::string* err);
std::string GetContent(Edge* edge);
/// Update status ninja logs following a command termination.
/// @return false if the build can not proceed further due to a fatal error.
bool FinishCommand(CommandRunner::Result* result, std::string* err);
/// Used for tests.
void SetBuildLog(BuildLog* log) {
scan_.set_build_log(log);
}
/// Load the dyndep information provided by the given node.
bool LoadDyndeps(Node* node, std::string* err);
State* state_;
const BuildConfig& config_;
Plan plan_;
std::unique_ptr<CommandRunner> command_runner_;
Status* status_;
private:
bool ExtractDeps(CommandRunner::Result* result, const std::string& deps_type,
const std::string& deps_prefix,
std::vector<Node*>* deps_nodes, std::string* err);
/// Map of running edge to time the edge started running.
typedef std::map<const Edge*, int> RunningEdgeMap;
RunningEdgeMap running_edges_;
/// Time the build started.
int64_t start_time_millis_;
std::string lock_file_path_;
DiskInterface* disk_interface_;
DependencyScan scan_;
// Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr.
Builder(const Builder &other); // DO NOT IMPLEMENT
void operator=(const Builder &other); // DO NOT IMPLEMENT
};
#endif // NINJA_BUILD_H_
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef NINJA_BUILD_H_
#define NINJA_BUILD_H_
#include <cstdio>
#include <map>
#include <memory>
#include <string>
#include <vector>
#include <queue>
#include "depfile_parser.h"
#include "graph.h" // XXX needed for DependencyScan; should rearrange.
#include "graph.h"
#include "exit_status.h"
#include "util.h" // int64_t
struct BuildLog;
struct Builder;
struct DiskInterface;
struct Edge;
struct Node;
struct State;
struct Status;
/// Plan stores the state of a build plan: what we intend to build,
/// which steps we're ready to execute.
struct Plan {
Plan(Builder* builder = NULL);
/// Add a target to our plan (including all its dependencies).
/// Returns false if we don't need to build this target; may
/// fill in |err| with an error message if there's a problem.
bool AddTarget(const Node* target, std::string* err);
// Pop a ready edge off the queue of edges to build.
// Returns NULL if there's no work to do.
Edge* FindWork();
/// Returns true if there's more work to be done.
bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; }
/// Dumps the current state of the plan.
void Dump() const;
enum EdgeResult {
kEdgeFailed,
kEdgeSucceeded
};
/// Mark an edge as done building (whether it succeeded or failed).
/// If any of the edge's outputs are dyndep bindings of their dependents,
/// this loads dynamic dependencies from the nodes' paths.
/// Returns 'false' if loading dyndep info fails and 'true' otherwise.
bool EdgeFinished(Edge* edge, EdgeResult result, std::string* err);
/// Clean the given node during the build.
/// Return false on error.
bool CleanNode(DependencyScan* scan, Node* node, std::string* err);
/// Number of edges with commands to run.
int command_edge_count() const { return command_edges_; }
/// Reset state. Clears want and ready sets.
void Reset();
/// Update the build plan to account for modifications made to the graph
/// by information loaded from a dyndep file.
bool DyndepsLoaded(DependencyScan* scan, const Node* node,
const DyndepFile& ddf, std::string* err);
private:
bool RefreshDyndepDependents(DependencyScan* scan, const Node* node, std::string* err);
void UnmarkDependents(const Node* node, std::set<Node*>* dependents);
bool AddSubTarget(const Node* node, const Node* dependent, std::string* err,
std::set<Edge*>* dyndep_walk);
/// Update plan with knowledge that the given node is up to date.
/// If the node is a dyndep binding on any of its dependents, this
/// loads dynamic dependencies from the node's path.
/// Returns 'false' if loading dyndep info fails and 'true' otherwise.
bool NodeFinished(Node* node, std::string* err);
/// Enumerate possible steps we want for an edge.
enum Want
{
/// We do not want to build the edge, but we might want to build one of
/// its dependents.
kWantNothing,
/// We want to build the edge, but have not yet scheduled it.
kWantToStart,
/// We want to build the edge, have scheduled it, and are waiting
/// for it to complete.
kWantToFinish
};
void EdgeWanted(const Edge* edge);
bool EdgeMaybeReady(std::map<Edge*, Want>::iterator want_e, std::string* err);
/// Submits a ready edge as a candidate for execution.
/// The edge may be delayed from running, for example if it's a member of a
/// currently-full pool.
void ScheduleWork(std::map<Edge*, Want>::iterator want_e);
/// Keep track of which edges we want to build in this plan. If this map does
/// not contain an entry for an edge, we do not want to build the entry or its
/// dependents. If it does contain an entry, the enumeration indicates what
/// we want for the edge.
std::map<Edge*, Want> want_;
EdgeSet ready_;
Builder* builder_;
/// Total number of edges that have commands (not phony).
int command_edges_;
/// Total remaining number of wanted edges.
int wanted_edges_;
};
/// CommandRunner is an interface that wraps running the build
/// subcommands. This allows tests to abstract out running commands.
/// RealCommandRunner is an implementation that actually runs commands.
struct CommandRunner {
virtual ~CommandRunner() {}
virtual size_t CanRunMore() const = 0;
virtual bool StartCommand(Edge* edge) = 0;
/// The result of waiting for a command.
struct Result {
Result() : edge(NULL) {}
Edge* edge;
ExitStatus status;
std::string output;
bool success() const { return status == ExitSuccess; }
};
/// Wait for a command to complete, or return false if interrupted.
virtual bool WaitForCommand(Result* result) = 0;
virtual std::vector<Edge*> GetActiveEdges() { return std::vector<Edge*>(); }
virtual void Abort() {}
};
/// Options (e.g. verbosity, parallelism) passed to a build.
struct BuildConfig {
BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
failures_allowed(1), max_load_average(-0.0f) {}
enum Verbosity {
QUIET, // No output -- used when testing.
NO_STATUS_UPDATE, // just regular output but suppress status update
NORMAL, // regular output and status update
VERBOSE
};
Verbosity verbosity;
bool dry_run;
int parallelism;
int failures_allowed;
/// The maximum load average we must not exceed. A negative value
/// means that we do not have any limit.
double max_load_average;
DepfileParserOptions depfile_parser_options;
};
/// Builder wraps the build process: starting commands, updating status.
struct Builder {
Builder(State* state, const BuildConfig& config,
BuildLog* build_log, DepsLog* deps_log,
DiskInterface* disk_interface, Status* status,
int64_t start_time_millis);
~Builder();
/// Clean up after interrupted commands by deleting output files.
void Cleanup();
Node* AddTarget(const std::string& name, std::string* err);
/// Add a target to the build, scanning dependencies.
/// @return false on error.
bool AddTarget(Node* target, std::string* err);
/// Returns true if the build targets are already up to date.
bool AlreadyUpToDate() const;
/// Run the build. Returns false on error.
/// It is an error to call this function when AlreadyUpToDate() is true.
bool Build(std::string* err);
bool StartEdge(Edge* edge, std::string* err);
std::string GetContent(Edge* edge);
/// Update status ninja logs following a command termination.
/// @return false if the build can not proceed further due to a fatal error.
bool FinishCommand(CommandRunner::Result* result, std::string* err);
/// Used for tests.
void SetBuildLog(BuildLog* log) {
scan_.set_build_log(log);
}
/// Load the dyndep information provided by the given node.
bool LoadDyndeps(Node* node, std::string* err);
State* state_;
const BuildConfig& config_;
Plan plan_;
std::unique_ptr<CommandRunner> command_runner_;
Status* status_;
private:
bool ExtractDeps(CommandRunner::Result* result, const std::string& deps_type,
const std::string& deps_prefix,
std::vector<Node*>* deps_nodes, std::string* err);
/// Map of running edge to time the edge started running.
typedef std::map<const Edge*, int> RunningEdgeMap;
RunningEdgeMap running_edges_;
/// Time the build started.
int64_t start_time_millis_;
std::string lock_file_path_;
DiskInterface* disk_interface_;
DependencyScan scan_;
// Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr.
Builder(const Builder &other); // DO NOT IMPLEMENT
void operator=(const Builder &other); // DO NOT IMPLEMENT
};
#endif // NINJA_BUILD_H_

992
src/build_log.cc
View File

@ -1,496 +1,496 @@
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// On AIX, inttypes.h gets indirectly included by build_log.h.
// It's easiest just to ask for the printf format macros right away.
#ifndef _WIN32
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#endif
#include "build_log.h"
#include "disk_interface.h"
#include <cassert>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#ifndef _WIN32
#include <inttypes.h>
#include <unistd.h>
#endif
#include "build.h"
#include "graph.h"
#include "metrics.h"
#include "util.h"
#if defined(_MSC_VER) && (_MSC_VER < 1800)
#define strtoll _strtoi64
#endif
using namespace std;
// Implementation details:
// Each run's log appends to the log file.
// To load, we run through all log entries in series, throwing away
// older runs.
// Once the number of redundant entries exceeds a threshold, we write
// out a new file and replace the existing one with it.
namespace {
const char kFileSignature[] = "# ninja log v%d\n";
const int kOldestSupportedVersion = 6;
const int kCurrentVersion = 6;
// 64bit MurmurHash2, by Austin Appleby
#if defined(_MSC_VER)
#define BIG_CONSTANT(x) (x)
#else // defined(_MSC_VER)
#define BIG_CONSTANT(x) (x##LLU)
#endif // !defined(_MSC_VER)
inline
uint64_t MurmurHash64A(const void* key, size_t len) {
static const uint64_t seed = 0xDECAFBADDECAFBADull;
const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
const int r = 47;
uint64_t h = seed ^ (len * m);
const unsigned char* data = (const unsigned char*)key;
while (len >= 8) {
uint64_t k;
memcpy(&k, data, sizeof k);
k *= m;
k ^= k >> r;
k *= m;
h ^= k;
h *= m;
data += 8;
len -= 8;
}
switch (len & 7)
{
case 7: h ^= uint64_t(data[6]) << 48;
NINJA_FALLTHROUGH;
case 6: h ^= uint64_t(data[5]) << 40;
NINJA_FALLTHROUGH;
case 5: h ^= uint64_t(data[4]) << 32;
NINJA_FALLTHROUGH;
case 4: h ^= uint64_t(data[3]) << 24;
NINJA_FALLTHROUGH;
case 3: h ^= uint64_t(data[2]) << 16;
NINJA_FALLTHROUGH;
case 2: h ^= uint64_t(data[1]) << 8;
NINJA_FALLTHROUGH;
case 1: h ^= uint64_t(data[0]);
h *= m;
};
h ^= h >> r;
h *= m;
h ^= h >> r;
return h;
}
#undef BIG_CONSTANT
} // namespace
// static
uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) {
return MurmurHash64A(command.str_, command.len_);
}
BuildLog::LogEntry::LogEntry(const string& output)
: output(output) {}
BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
int start_time, int end_time, TimeStamp mtime)
: output(output), command_hash(command_hash),
start_time(start_time), end_time(end_time), mtime(mtime)
{}
BuildLog::BuildLog()
: log_file_(NULL), needs_recompaction_(false) {}
BuildLog::~BuildLog() {
Close();
}
bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user,
string* err) {
if (needs_recompaction_) {
if (!Recompact(path, user, err))
return false;
}
assert(!log_file_);
log_file_path_ = path; // we don't actually open the file right now, but will
// do so on the first write attempt
return true;
}
bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
TimeStamp mtime) {
string command = edge->EvaluateCommand(true);
uint64_t command_hash = LogEntry::HashCommand(command);
for (vector<Node*>::iterator out = edge->outputs_.begin();
out != edge->outputs_.end(); ++out) {
const string& path = (*out)->path();
Entries::iterator i = entries_.find(path);
LogEntry* log_entry;
if (i != entries_.end()) {
log_entry = i->second;
} else {
log_entry = new LogEntry(path);
entries_.insert(Entries::value_type(log_entry->output, log_entry));
}
log_entry->command_hash = command_hash;
log_entry->start_time = start_time;
log_entry->end_time = end_time;
log_entry->mtime = mtime;
if (!OpenForWriteIfNeeded()) {
return false;
}
if (log_file_) {
if (!WriteEntry(log_file_, *log_entry))
return false;
if (fflush(log_file_) != 0) {
return false;
}
}
}
return true;
}
void BuildLog::Close() {
OpenForWriteIfNeeded(); // create the file even if nothing has been recorded
if (log_file_)
fclose(log_file_);
log_file_ = NULL;
}
bool BuildLog::OpenForWriteIfNeeded() {
if (log_file_ || log_file_path_.empty()) {
return true;
}
log_file_ = fopen(log_file_path_.c_str(), "ab");
if (!log_file_) {
return false;
}
if (setvbuf(log_file_, NULL, _IOLBF, BUFSIZ) != 0) {
return false;
}
SetCloseOnExec(fileno(log_file_));
// Opening a file in append mode doesn't set the file pointer to the file's
// end on Windows. Do that explicitly.
fseek(log_file_, 0, SEEK_END);
if (ftell(log_file_) == 0) {
if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) {
return false;
}
}
return true;
}
struct LineReader {
explicit LineReader(FILE* file)
: file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
memset(buf_, 0, sizeof(buf_));
}
// Reads a \n-terminated line from the file passed to the constructor.
// On return, *line_start points to the beginning of the next line, and
// *line_end points to the \n at the end of the line. If no newline is seen
// in a fixed buffer size, *line_end is set to NULL. Returns false on EOF.
bool ReadLine(char** line_start, char** line_end) {
if (line_start_ >= buf_end_ || !line_end_) {
// Buffer empty, refill.
size_t size_read = fread(buf_, 1, sizeof(buf_), file_);
if (!size_read)
return false;
line_start_ = buf_;
buf_end_ = buf_ + size_read;
} else {
// Advance to next line in buffer.
line_start_ = line_end_ + 1;
}
line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
if (!line_end_) {
// No newline. Move rest of data to start of buffer, fill rest.
size_t already_consumed = line_start_ - buf_;
size_t size_rest = (buf_end_ - buf_) - already_consumed;
memmove(buf_, line_start_, size_rest);
size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_);
buf_end_ = buf_ + size_rest + read;
line_start_ = buf_;
line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
}
*line_start = line_start_;
*line_end = line_end_;
return true;
}
private:
FILE* file_;
char buf_[256 << 10];
char* buf_end_; // Points one past the last valid byte in |buf_|.
char* line_start_;
// Points at the next \n in buf_ after line_start, or NULL.
char* line_end_;
};
LoadStatus BuildLog::Load(const string& path, string* err) {
METRIC_RECORD(".ninja_log load");
FILE* file = fopen(path.c_str(), "r");
if (!file) {
if (errno == ENOENT)
return LOAD_NOT_FOUND;
*err = strerror(errno);
return LOAD_ERROR;
}
int log_version = 0;
int unique_entry_count = 0;
int total_entry_count = 0;
LineReader reader(file);
char* line_start = 0;
char* line_end = 0;
while (reader.ReadLine(&line_start, &line_end)) {
if (!log_version) {
sscanf(line_start, kFileSignature, &log_version);
bool invalid_log_version = false;
if (log_version < kOldestSupportedVersion) {
invalid_log_version = true;
*err = "build log version is too old; starting over";
} else if (log_version > kCurrentVersion) {
invalid_log_version = true;
*err = "build log version is too new; starting over";
}
if (invalid_log_version) {
fclose(file);
unlink(path.c_str());
// Don't report this as a failure. A missing build log will cause
// us to rebuild the outputs anyway.
return LOAD_NOT_FOUND;
}
}
// If no newline was found in this chunk, read the next.
if (!line_end)
continue;
const char kFieldSeparator = '\t';
char* start = line_start;
char* end = (char*)memchr(start, kFieldSeparator, line_end - start);
if (!end)
continue;
*end = 0;
int start_time = 0, end_time = 0;
TimeStamp mtime = 0;
start_time = atoi(start);
start = end + 1;
end = (char*)memchr(start, kFieldSeparator, line_end - start);
if (!end)
continue;
*end = 0;
end_time = atoi(start);
start = end + 1;
end = (char*)memchr(start, kFieldSeparator, line_end - start);
if (!end)
continue;
*end = 0;
mtime = strtoll(start, NULL, 10);
start = end + 1;
end = (char*)memchr(start, kFieldSeparator, line_end - start);
if (!end)
continue;
string output = string(start, end - start);
start = end + 1;
end = line_end;
LogEntry* entry;
Entries::iterator i = entries_.find(output);
if (i != entries_.end()) {
entry = i->second;
} else {
entry = new LogEntry(output);
entries_.insert(Entries::value_type(entry->output, entry));
++unique_entry_count;
}
++total_entry_count;
entry->start_time = start_time;
entry->end_time = end_time;
entry->mtime = mtime;
char c = *end; *end = '\0';
entry->command_hash = (uint64_t)strtoull(start, NULL, 16);
*end = c;
}
fclose(file);
if (!line_start) {
return LOAD_SUCCESS; // file was empty
}
// Decide whether it's time to rebuild the log:
// - if we're upgrading versions
// - if it's getting large
int kMinCompactionEntryCount = 100;
int kCompactionRatio = 3;
if (log_version < kCurrentVersion) {
needs_recompaction_ = true;
} else if (total_entry_count > kMinCompactionEntryCount &&
total_entry_count > unique_entry_count * kCompactionRatio) {
needs_recompaction_ = true;
}
return LOAD_SUCCESS;
}
BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) {
Entries::iterator i = entries_.find(path);
if (i != entries_.end())
return i->second;
return NULL;
}
bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
return fprintf(f, "%d\t%d\t%" PRId64 "\t%s\t%" PRIx64 "\n",
entry.start_time, entry.end_time, entry.mtime,
entry.output.c_str(), entry.command_hash) > 0;
}
bool BuildLog::Recompact(const string& path, const BuildLogUser& user,
string* err) {
METRIC_RECORD(".ninja_log recompact");
Close();
string temp_path = path + ".recompact";
FILE* f = fopen(temp_path.c_str(), "wb");
if (!f) {
*err = strerror(errno);
return false;
}
if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
*err = strerror(errno);
fclose(f);
return false;
}
vector<StringPiece> dead_outputs;
for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
if (user.IsPathDead(i->first)) {
dead_outputs.push_back(i->first);
continue;
}
if (!WriteEntry(f, *i->second)) {
*err = strerror(errno);
fclose(f);
return false;
}
}
for (size_t i = 0; i < dead_outputs.size(); ++i)
entries_.erase(dead_outputs[i]);
fclose(f);
if (unlink(path.c_str()) < 0) {
*err = strerror(errno);
return false;
}
if (rename(temp_path.c_str(), path.c_str()) < 0) {
*err = strerror(errno);
return false;
}
return true;
}
bool BuildLog::Restat(const StringPiece path,
const DiskInterface& disk_interface,
const int output_count, char** outputs,
std::string* const err) {
METRIC_RECORD(".ninja_log restat");
Close();
std::string temp_path = path.AsString() + ".restat";
FILE* f = fopen(temp_path.c_str(), "wb");
if (!f) {
*err = strerror(errno);
return false;
}
if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
*err = strerror(errno);
fclose(f);
return false;
}
for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
bool skip = output_count > 0;
for (int j = 0; j < output_count; ++j) {
if (i->second->output == outputs[j]) {
skip = false;
break;
}
}
if (!skip) {
const TimeStamp mtime = disk_interface.Stat(i->second->output, err);
if (mtime == -1) {
fclose(f);
return false;
}
i->second->mtime = mtime;
}
if (!WriteEntry(f, *i->second)) {
*err = strerror(errno);
fclose(f);
return false;
}
}
fclose(f);
if (unlink(path.str_) < 0) {
*err = strerror(errno);
return false;
}
if (rename(temp_path.c_str(), path.str_) < 0) {
*err = strerror(errno);
return false;
}
return true;
}
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// On AIX, inttypes.h gets indirectly included by build_log.h.
// It's easiest just to ask for the printf format macros right away.
#ifndef _WIN32
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#endif
#include "build_log.h"
#include "disk_interface.h"
#include <cassert>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#ifndef _WIN32
#include <inttypes.h>
#include <unistd.h>
#endif
#include "build.h"
#include "graph.h"
#include "metrics.h"
#include "util.h"
#if defined(_MSC_VER) && (_MSC_VER < 1800)
#define strtoll _strtoi64
#endif
using namespace std;
// Implementation details:
// Each run's log appends to the log file.
// To load, we run through all log entries in series, throwing away
// older runs.
// Once the number of redundant entries exceeds a threshold, we write
// out a new file and replace the existing one with it.
namespace {
const char kFileSignature[] = "# ninja log v%d\n";
const int kOldestSupportedVersion = 6;
const int kCurrentVersion = 6;
// 64bit MurmurHash2, by Austin Appleby
#if defined(_MSC_VER)
#define BIG_CONSTANT(x) (x)
#else // defined(_MSC_VER)
#define BIG_CONSTANT(x) (x##LLU)
#endif // !defined(_MSC_VER)
inline
uint64_t MurmurHash64A(const void* key, size_t len) {
static const uint64_t seed = 0xDECAFBADDECAFBADull;
const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
const int r = 47;
uint64_t h = seed ^ (len * m);
const unsigned char* data = (const unsigned char*)key;
while (len >= 8) {
uint64_t k;
memcpy(&k, data, sizeof k);
k *= m;
k ^= k >> r;
k *= m;
h ^= k;
h *= m;
data += 8;
len -= 8;
}
switch (len & 7)
{
case 7: h ^= uint64_t(data[6]) << 48;
NINJA_FALLTHROUGH;
case 6: h ^= uint64_t(data[5]) << 40;
NINJA_FALLTHROUGH;
case 5: h ^= uint64_t(data[4]) << 32;
NINJA_FALLTHROUGH;
case 4: h ^= uint64_t(data[3]) << 24;
NINJA_FALLTHROUGH;
case 3: h ^= uint64_t(data[2]) << 16;
NINJA_FALLTHROUGH;
case 2: h ^= uint64_t(data[1]) << 8;
NINJA_FALLTHROUGH;
case 1: h ^= uint64_t(data[0]);
h *= m;
};
h ^= h >> r;
h *= m;
h ^= h >> r;
return h;
}
#undef BIG_CONSTANT
} // namespace
// static
uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) {
return MurmurHash64A(command.str_, command.len_);
}
BuildLog::LogEntry::LogEntry(const string& output)
: output(output) {}
BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
int start_time, int end_time, TimeStamp mtime)
: output(output), command_hash(command_hash),
start_time(start_time), end_time(end_time), mtime(mtime)
{}
BuildLog::BuildLog()
: log_file_(NULL), needs_recompaction_(false) {}
BuildLog::~BuildLog() {
Close();
}
bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user,
string* err) {
if (needs_recompaction_) {
if (!Recompact(path, user, err))
return false;
}
assert(!log_file_);
log_file_path_ = path; // we don't actually open the file right now, but will
// do so on the first write attempt
return true;
}
bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
TimeStamp mtime) {
string command = edge->EvaluateCommand(true);
uint64_t command_hash = LogEntry::HashCommand(command);
for (vector<Node*>::iterator out = edge->outputs_.begin();
out != edge->outputs_.end(); ++out) {
const string& path = (*out)->path();
Entries::iterator i = entries_.find(path);
LogEntry* log_entry;
if (i != entries_.end()) {
log_entry = i->second;
} else {
log_entry = new LogEntry(path);
entries_.insert(Entries::value_type(log_entry->output, log_entry));
}
log_entry->command_hash = command_hash;
log_entry->start_time = start_time;
log_entry->end_time = end_time;
log_entry->mtime = mtime;
if (!OpenForWriteIfNeeded()) {
return false;
}
if (log_file_) {
if (!WriteEntry(log_file_, *log_entry))
return false;
if (fflush(log_file_) != 0) {
return false;
}
}
}
return true;
}
void BuildLog::Close() {
OpenForWriteIfNeeded(); // create the file even if nothing has been recorded
if (log_file_)
fclose(log_file_);
log_file_ = NULL;
}
bool BuildLog::OpenForWriteIfNeeded() {
if (log_file_ || log_file_path_.empty()) {
return true;
}
log_file_ = fopen(log_file_path_.c_str(), "ab");
if (!log_file_) {
return false;
}
if (setvbuf(log_file_, NULL, _IOLBF, BUFSIZ) != 0) {
return false;
}
SetCloseOnExec(fileno(log_file_));
// Opening a file in append mode doesn't set the file pointer to the file's
// end on Windows. Do that explicitly.
fseek(log_file_, 0, SEEK_END);
if (ftell(log_file_) == 0) {
if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) {
return false;
}
}
return true;
}
struct LineReader {
explicit LineReader(FILE* file)
: file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
memset(buf_, 0, sizeof(buf_));
}
// Reads a \n-terminated line from the file passed to the constructor.
// On return, *line_start points to the beginning of the next line, and
// *line_end points to the \n at the end of the line. If no newline is seen
// in a fixed buffer size, *line_end is set to NULL. Returns false on EOF.
bool ReadLine(char** line_start, char** line_end) {
if (line_start_ >= buf_end_ || !line_end_) {
// Buffer empty, refill.
size_t size_read = fread(buf_, 1, sizeof(buf_), file_);
if (!size_read)
return false;
line_start_ = buf_;
buf_end_ = buf_ + size_read;
} else {
// Advance to next line in buffer.
line_start_ = line_end_ + 1;
}
line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
if (!line_end_) {
// No newline. Move rest of data to start of buffer, fill rest.
size_t already_consumed = line_start_ - buf_;
size_t size_rest = (buf_end_ - buf_) - already_consumed;
memmove(buf_, line_start_, size_rest);
size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_);
buf_end_ = buf_ + size_rest + read;
line_start_ = buf_;
line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
}
*line_start = line_start_;
*line_end = line_end_;
return true;
}
private:
FILE* file_;
char buf_[256 << 10];
char* buf_end_; // Points one past the last valid byte in |buf_|.
char* line_start_;
// Points at the next \n in buf_ after line_start, or NULL.
char* line_end_;
};
LoadStatus BuildLog::Load(const string& path, string* err) {
METRIC_RECORD(".ninja_log load");
FILE* file = fopen(path.c_str(), "r");
if (!file) {
if (errno == ENOENT)
return LOAD_NOT_FOUND;
*err = strerror(errno);
return LOAD_ERROR;
}
int log_version = 0;
int unique_entry_count = 0;
int total_entry_count = 0;
LineReader reader(file);
char* line_start = 0;
char* line_end = 0;
while (reader.ReadLine(&line_start, &line_end)) {
if (!log_version) {
sscanf(line_start, kFileSignature, &log_version);
bool invalid_log_version = false;
if (log_version < kOldestSupportedVersion) {
invalid_log_version = true;
*err = "build log version is too old; starting over";
} else if (log_version > kCurrentVersion) {
invalid_log_version = true;
*err = "build log version is too new; starting over";
}
if (invalid_log_version) {
fclose(file);
unlink(path.c_str());
// Don't report this as a failure. A missing build log will cause
// us to rebuild the outputs anyway.
return LOAD_NOT_FOUND;
}
}
// If no newline was found in this chunk, read the next.
if (!line_end)
continue;
const char kFieldSeparator = '\t';
char* start = line_start;
char* end = (char*)memchr(start, kFieldSeparator, line_end - start);
if (!end)
continue;
*end = 0;
int start_time = 0, end_time = 0;
TimeStamp mtime = 0;
start_time = atoi(start);
start = end + 1;
end = (char*)memchr(start, kFieldSeparator, line_end - start);
if (!end)
continue;
*end = 0;
end_time = atoi(start);
start = end + 1;
end = (char*)memchr(start, kFieldSeparator, line_end - start);
if (!end)
continue;
*end = 0;
mtime = strtoll(start, NULL, 10);
start = end + 1;
end = (char*)memchr(start, kFieldSeparator, line_end - start);
if (!end)
continue;
string output = string(start, end - start);
start = end + 1;
end = line_end;
LogEntry* entry;
Entries::iterator i = entries_.find(output);
if (i != entries_.end()) {
entry = i->second;
} else {
entry = new LogEntry(output);
entries_.insert(Entries::value_type(entry->output, entry));
++unique_entry_count;
}
++total_entry_count;
entry->start_time = start_time;
entry->end_time = end_time;
entry->mtime = mtime;
char c = *end; *end = '\0';
entry->command_hash = (uint64_t)strtoull(start, NULL, 16);
*end = c;
}
fclose(file);
if (!line_start) {
return LOAD_SUCCESS; // file was empty
}
// Decide whether it's time to rebuild the log:
// - if we're upgrading versions
// - if it's getting large
int kMinCompactionEntryCount = 100;
int kCompactionRatio = 3;
if (log_version < kCurrentVersion) {
needs_recompaction_ = true;
} else if (total_entry_count > kMinCompactionEntryCount &&
total_entry_count > unique_entry_count * kCompactionRatio) {
needs_recompaction_ = true;
}
return LOAD_SUCCESS;
}
BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) {
Entries::iterator i = entries_.find(path);
if (i != entries_.end())
return i->second;
return NULL;
}
bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
return fprintf(f, "%d\t%d\t%" PRId64 "\t%s\t%" PRIx64 "\n",
entry.start_time, entry.end_time, entry.mtime,
entry.output.c_str(), entry.command_hash) > 0;
}
bool BuildLog::Recompact(const string& path, const BuildLogUser& user,
string* err) {
METRIC_RECORD(".ninja_log recompact");
Close();
string temp_path = path + ".recompact";
FILE* f = fopen(temp_path.c_str(), "wb");
if (!f) {
*err = strerror(errno);
return false;
}
if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
*err = strerror(errno);
fclose(f);
return false;
}
vector<StringPiece> dead_outputs;
for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
if (user.IsPathDead(i->first)) {
dead_outputs.push_back(i->first);
continue;
}
if (!WriteEntry(f, *i->second)) {
*err = strerror(errno);
fclose(f);
return false;
}
}
for (size_t i = 0; i < dead_outputs.size(); ++i)
entries_.erase(dead_outputs[i]);
fclose(f);
if (unlink(path.c_str()) < 0) {
*err = strerror(errno);
return false;
}
if (rename(temp_path.c_str(), path.c_str()) < 0) {
*err = strerror(errno);
return false;
}
return true;
}
bool BuildLog::Restat(const StringPiece path,
const DiskInterface& disk_interface,
const int output_count, char** outputs,
std::string* const err) {
METRIC_RECORD(".ninja_log restat");
Close();
std::string temp_path = path.AsString() + ".restat";
FILE* f = fopen(temp_path.c_str(), "wb");
if (!f) {
*err = strerror(errno);
return false;
}
if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
*err = strerror(errno);
fclose(f);
return false;
}
for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
bool skip = output_count > 0;
for (int j = 0; j < output_count; ++j) {
if (i->second->output == outputs[j]) {
skip = false;
break;
}
}
if (!skip) {
const TimeStamp mtime = disk_interface.Stat(i->second->output, err);
if (mtime == -1) {
fclose(f);
return false;
}
i->second->mtime = mtime;
}
if (!WriteEntry(f, *i->second)) {
*err = strerror(errno);
fclose(f);
return false;
}
}
fclose(f);
if (unlink(path.str_) < 0) {
*err = strerror(errno);
return false;
}
if (rename(temp_path.c_str(), path.str_) < 0) {
*err = strerror(errno);
return false;
}
return true;
}

623
src/depfile_parser.cc
View File

@ -21,7 +21,7 @@
using namespace std;
DepfileParser::DepfileParser(DepfileParserOptions options)
: options_(options)
: options_(options)
{
}
@ -46,328 +46,327 @@ DepfileParser::DepfileParser(DepfileParserOptions options)
// If anyone actually has depfiles that rely on the more complicated
// behavior we can adjust this.
bool DepfileParser::Parse(string* content, string* err) {
// in: current parser input point.
// end: end of input.
// parsing_targets: whether we are parsing targets or dependencies.
char* in = &(*content)[0];
char* end = in + content->size();
bool have_target = false;
bool parsing_targets = true;
bool poisoned_input = false;
bool is_empty = true;
while (in < end) {
bool have_newline = false;
// out: current output point (typically same as in, but can fall behind
// as we de-escape backslashes).
char* out = in;
// filename: start of the current parsed filename.
char* filename = out;
for (;;) {
// start: beginning of the current parsed span.
const char* start = in;
char* yymarker = NULL;
{
unsigned char yych;
static const unsigned char yybm[] = {
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 128, 0, 0, 0, 128, 0, 0,
128, 128, 0, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 0, 0, 128, 0, 0,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 0, 128,
0, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 128, 0,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
};
yych = *in;
if (yybm[0+yych] & 128) {
goto yy9;
}
if (yych <= '\r') {
if (yych <= '\t') {
if (yych >= 0x01) goto yy4;
} else {
if (yych <= '\n') goto yy6;
if (yych <= '\f') goto yy4;
goto yy8;
}
} else {
if (yych <= '$') {
if (yych <= '#') goto yy4;
goto yy12;
} else {
if (yych <= '?') goto yy4;
if (yych <= '\\') goto yy13;
goto yy4;
}
}
++in;
{
break;
}
yy4:
++in;
yy5:
{
// For any other character (e.g. whitespace), swallow it here,
// allowing the outer logic to loop around again.
break;
}
yy6:
++in;
{
// A newline ends the current file name and the current rule.
have_newline = true;
break;
}
yy8:
yych = *++in;
if (yych == '\n') goto yy6;
goto yy5;
yy9:
yych = *++in;
if (yybm[0+yych] & 128) {
goto yy9;
}
yy11:
{
// Got a span of plain text.
int len = (int)(in - start);
// Need to shift it over if we're overwriting backslashes.
if (out < start)
memmove(out, start, len);
out += len;
continue;
}
yy12:
yych = *++in;
if (yych == '$') goto yy14;
goto yy5;
yy13:
yych = *(yymarker = ++in);
if (yych <= ' ') {
if (yych <= '\n') {
if (yych <= 0x00) goto yy5;
if (yych <= '\t') goto yy16;
goto yy17;
} else {
if (yych == '\r') goto yy19;
if (yych <= 0x1F) goto yy16;
goto yy21;
}
} else {
if (yych <= '9') {
if (yych == '#') goto yy23;
goto yy16;
} else {
if (yych <= ':') goto yy25;
if (yych == '\\') goto yy27;
goto yy16;
}
}
yy14:
++in;
{
// De-escape dollar character.
*out++ = '$';
continue;
}
yy16:
++in;
goto yy11;
yy17:
++in;
{
// A line continuation ends the current file name.
break;
}
yy19:
yych = *++in;
if (yych == '\n') goto yy17;
in = yymarker;
goto yy5;
yy21:
++in;
{
// 2N+1 backslashes plus space -> N backslashes plus space.
int len = (int)(in - start);
int n = len / 2 - 1;
if (out < start)
memset(out, '\\', n);
out += n;
*out++ = ' ';
continue;
}
yy23:
++in;
{
// De-escape hash sign, but preserve other leading backslashes.
int len = (int)(in - start);
if (len > 2 && out < start)
memset(out, '\\', len - 2);
out += len - 2;
*out++ = '#';
continue;
}
yy25:
yych = *++in;
if (yych <= '\f') {
if (yych <= 0x00) goto yy28;
if (yych <= 0x08) goto yy26;
if (yych <= '\n') goto yy28;
} else {
if (yych <= '\r') goto yy28;
if (yych == ' ') goto yy28;
}
yy26:
{
// De-escape colon sign, but preserve other leading backslashes.
// Regular expression uses lookahead to make sure that no whitespace
// nor EOF follows. In that case it'd be the : at the end of a target
int len = (int)(in - start);
if (len > 2 && out < start)
memset(out, '\\', len - 2);
out += len - 2;
*out++ = ':';
continue;
}
yy27:
yych = *++in;
if (yych <= ' ') {
if (yych <= '\n') {
if (yych <= 0x00) goto yy11;
if (yych <= '\t') goto yy16;
goto yy11;
} else {
if (yych == '\r') goto yy11;
if (yych <= 0x1F) goto yy16;
goto yy30;
}
} else {
if (yych <= '9') {
if (yych == '#') goto yy23;
goto yy16;
} else {
if (yych <= ':') goto yy25;
if (yych == '\\') goto yy32;
goto yy16;
}
}
yy28:
++in;
{
// Backslash followed by : and whitespace.
// It is therefore normal text and not an escaped colon
int len = (int)(in - start - 1);
// Need to shift it over if we're overwriting backslashes.
if (out < start)
memmove(out, start, len);
out += len;
if (*(in - 1) == '\n')
have_newline = true;
break;
}
yy30:
++in;
{
// 2N backslashes plus space -> 2N backslashes, end of filename.
int len = (int)(in - start);
if (out < start)
memset(out, '\\', len - 1);
out += len - 1;
break;
}
yy32:
yych = *++in;
if (yych <= ' ') {
if (yych <= '\n') {
if (yych <= 0x00) goto yy11;
if (yych <= '\t') goto yy16;
goto yy11;
} else {
if (yych == '\r') goto yy11;
if (yych <= 0x1F) goto yy16;
goto yy21;
}
} else {
if (yych <= '9') {
if (yych == '#') goto yy23;
goto yy16;
} else {
if (yych <= ':') goto yy25;
if (yych == '\\') goto yy27;
goto yy16;
}
}
}
// in: current parser input point.
// end: end of input.
// parsing_targets: whether we are parsing targets or dependencies.
char* in = &(*content)[0];
char* end = in + content->size();
bool have_target = false;
bool parsing_targets = true;
bool poisoned_input = false;
bool is_empty = true;
while (in < end) {
bool have_newline = false;
// out: current output point (typically same as in, but can fall behind
// as we de-escape backslashes).
char* out = in;
// filename: start of the current parsed filename.
char* filename = out;
for (;;) {
// start: beginning of the current parsed span.
const char* start = in;
char* yymarker = NULL;
{
unsigned char yych;
static const unsigned char yybm[] = {
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 128, 0, 0, 0, 128, 0, 0,
128, 128, 0, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 0, 0, 128, 0, 0,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 0, 128,
0, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 0, 128, 128, 0,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
};
yych = *in;
if (yybm[0+yych] & 128) {
goto yy9;
}
if (yych <= '\r') {
if (yych <= '\t') {
if (yych >= 0x01) goto yy4;
} else {
if (yych <= '\n') goto yy6;
if (yych <= '\f') goto yy4;
goto yy8;
}
} else {
if (yych <= '$') {
if (yych <= '#') goto yy4;
goto yy12;
} else {
if (yych <= '?') goto yy4;
if (yych <= '\\') goto yy13;
goto yy4;
}
}
++in;
{
break;
}
yy4:
++in;
yy5:
{
// For any other character (e.g. whitespace), swallow it here,
// allowing the outer logic to loop around again.
break;
}
yy6:
++in;
{
// A newline ends the current file name and the current rule.
have_newline = true;
break;
}
yy8:
yych = *++in;
if (yych == '\n') goto yy6;
goto yy5;
yy9:
yych = *++in;
if (yybm[0+yych] & 128) {
goto yy9;
}
yy11:
{
// Got a span of plain text.
int len = (int)(in - start);
// Need to shift it over if we're overwriting backslashes.
if (out < start)
memmove(out, start, len);
out += len;
continue;
}
yy12:
yych = *++in;
if (yych == '$') goto yy14;
goto yy5;
yy13:
yych = *(yymarker = ++in);
if (yych <= ' ') {
if (yych <= '\n') {
if (yych <= 0x00) goto yy5;
if (yych <= '\t') goto yy16;
goto yy17;
} else {
if (yych == '\r') goto yy19;
if (yych <= 0x1F) goto yy16;
goto yy21;
}
} else {
if (yych <= '9') {
if (yych == '#') goto yy23;
goto yy16;
} else {
if (yych <= ':') goto yy25;
if (yych == '\\') goto yy27;
goto yy16;
}
}
yy14:
++in;
{
// De-escape dollar character.
*out++ = '$';
continue;
}
yy16:
++in;
goto yy11;
yy17:
++in;
{
// A line continuation ends the current file name.
break;
}
yy19:
yych = *++in;
if (yych == '\n') goto yy17;
in = yymarker;
goto yy5;
yy21:
++in;
{
// 2N+1 backslashes plus space -> N backslashes plus space.
int len = (int)(in - start);
int n = len / 2 - 1;
if (out < start)
memset(out, '\\', n);
out += n;
*out++ = ' ';
continue;
}
yy23:
++in;
{
// De-escape hash sign, but preserve other leading backslashes.
int len = (int)(in - start);
if (len > 2 && out < start)
memset(out, '\\', len - 2);
out += len - 2;
*out++ = '#';
continue;
}
yy25:
yych = *++in;
if (yych <= '\f') {
if (yych <= 0x00) goto yy28;
if (yych <= 0x08) goto yy26;
if (yych <= '\n') goto yy28;
} else {
if (yych <= '\r') goto yy28;
if (yych == ' ') goto yy28;
}
yy26:
{
// De-escape colon sign, but preserve other leading backslashes.
// Regular expression uses lookahead to make sure that no whitespace
// nor EOF follows. In that case it'd be the : at the end of a target
int len = (int)(in - start);
if (len > 2 && out < start)
memset(out, '\\', len - 2);
out += len - 2;
*out++ = ':';
continue;
}
yy27:
yych = *++in;
if (yych <= ' ') {
if (yych <= '\n') {
if (yych <= 0x00) goto yy11;
if (yych <= '\t') goto yy16;
goto yy11;
} else {
if (yych == '\r') goto yy11;
if (yych <= 0x1F) goto yy16;
goto yy30;
}
} else {
if (yych <= '9') {
if (yych == '#') goto yy23;
goto yy16;
} else {
if (yych <= ':') goto yy25;
if (yych == '\\') goto yy32;
goto yy16;
}
}
yy28:
++in;
{
// Backslash followed by : and whitespace.
// It is therefore normal text and not an escaped colon
int len = (int)(in - start - 1);
// Need to shift it over if we're overwriting backslashes.
if (out < start)
memmove(out, start, len);
out += len;
if (*(in - 1) == '\n')
have_newline = true;
break;
}
yy30:
++in;
{
// 2N backslashes plus space -> 2N backslashes, end of filename.
int len = (int)(in - start);
if (out < start)
memset(out, '\\', len - 1);
out += len - 1;
break;
}
yy32:
yych = *++in;
if (yych <= ' ') {
if (yych <= '\n') {
if (yych <= 0x00) goto yy11;
if (yych <= '\t') goto yy16;
goto yy11;
} else {
if (yych == '\r') goto yy11;
if (yych <= 0x1F) goto yy16;
goto yy21;
}
} else {
if (yych <= '9') {
if (yych == '#') goto yy23;
goto yy16;
} else {
if (yych <= ':') goto yy25;
if (yych == '\\') goto yy27;
goto yy16;
}
}
}
}
int len = (int)(out - filename);
const bool is_dependency = !parsing_targets;
if (len > 0 && filename[len - 1] == ':') {
len--; // Strip off trailing colon, if any.
parsing_targets = false;
have_target = true;
len--; // Strip off trailing colon, if any.
parsing_targets = false;
have_target = true;
}
if (len > 0) {
is_empty = false;
StringPiece piece = StringPiece(filename, len);
// If we've seen this as an input before, skip it.
std::vector<StringPiece>::iterator pos = std::find(ins_.begin(), ins_.end(), piece);
if (pos == ins_.end()) {
if (is_dependency) {
if (poisoned_input) {
*err = "inputs may not also have inputs";
return false;
}
// New input.
ins_.push_back(piece);
} else {
// Check for a new output.
if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end())
outs_.push_back(piece);
}
} else if (!is_dependency) {
// We've passed an input on the left side; reject new inputs.
poisoned_input = true;
}
is_empty = false;
StringPiece piece = StringPiece(filename, len);
// If we've seen this as an input before, skip it.
std::vector<StringPiece>::iterator pos = std::find(ins_.begin(), ins_.end(), piece);
if (pos == ins_.end()) {
if (is_dependency) {
if (poisoned_input) {
*err = "inputs may not also have inputs";
return false;
}
// New input.
ins_.push_back(piece);
} else {
// Check for a new output.
if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end())
outs_.push_back(piece);
}
} else if (!is_dependency) {
// We've passed an input on the left side; reject new inputs.
poisoned_input = true;
}
}
if (have_newline) {
// A newline ends a rule so the next filename will be a new target.
parsing_targets = true;
poisoned_input = false;
// A newline ends a rule so the next filename will be a new target.
parsing_targets = true;
poisoned_input = false;
}
}
if (!have_target && !is_empty) {
*err = "expected ':' in depfile";
return false;
}
return true;
}
if (!have_target && !is_empty) {
*err = "expected ':' in depfile";
return false;
}
return true;
}

210
src/disk_interface.h
View File

@ -1,105 +1,105 @@
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef NINJA_DISK_INTERFACE_H_
#define NINJA_DISK_INTERFACE_H_
#include <map>
#include <string>
#include "timestamp.h"
/// Interface for reading files from disk. See DiskInterface for details.
/// This base offers the minimum interface needed just to read files.
struct FileReader {
virtual ~FileReader() {}
/// Result of ReadFile.
enum Status {
Okay,
NotFound,
OtherError
};
/// Read and store in given string. On success, return Okay.
/// On error, return another Status and fill |err|.
virtual Status ReadFile(const std::string& path, std::string* contents,
std::string* err) = 0;
};
/// Interface for accessing the disk.
///
/// Abstract so it can be mocked out for tests. The real implementation
/// is RealDiskInterface.
struct DiskInterface: public FileReader {
/// stat() a file, returning the mtime, or 0 if missing and -1 on
/// other errors.
virtual TimeStamp Stat(const std::string& path, std::string* err) const = 0;
/// Create a directory, returning false on failure.
virtual bool MakeDir(const std::string& path) = 0;
/// Create a file, with the specified name and contents
/// Returns true on success, false on failure
virtual bool WriteFile(const std::string& path,
const std::string& contents) = 0;
/// Remove the file named @a path. It behaves like 'rm -f path' so no errors
/// are reported if it does not exists.
/// @returns 0 if the file has been removed,
/// 1 if the file does not exist, and
/// -1 if an error occurs.
virtual int RemoveFile(const std::string& path) = 0;
/// Create all the parent directories for path; like mkdir -p
/// `basename path`.
bool MakeDirs(const std::string& path);
};
/// Implementation of DiskInterface that actually hits the disk.
struct RealDiskInterface : public DiskInterface {
RealDiskInterface();
virtual ~RealDiskInterface() {}
virtual TimeStamp Stat(const std::string& path, std::string* err) const;
virtual bool MakeDir(const std::string& path);
virtual bool WriteFile(const std::string& path, const std::string& contents);
virtual Status ReadFile(const std::string& path, std::string* contents,
std::string* err);
virtual int RemoveFile(const std::string& path);
/// Whether stat information can be cached. Only has an effect on Windows.
void AllowStatCache(bool allow);
#ifdef _WIN32
/// Whether long paths are enabled. Only has an effect on Windows.
bool AreLongPathsEnabled() const;
#endif
private:
#ifdef _WIN32
/// Whether stat information can be cached.
bool use_cache_;
/// Whether long paths are enabled.
bool long_paths_enabled_;
typedef std::map<std::string, TimeStamp> DirCache;
// TODO: Neither a map nor a hashmap seems ideal here. If the statcache
// works out, come up with a better data structure.
typedef std::map<std::string, DirCache> Cache;
mutable Cache cache_;
#endif
};
#endif // NINJA_DISK_INTERFACE_H_
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef NINJA_DISK_INTERFACE_H_
#define NINJA_DISK_INTERFACE_H_
#include <map>
#include <string>
#include "timestamp.h"
/// Interface for reading files from disk. See DiskInterface for details.
/// This base offers the minimum interface needed just to read files.
struct FileReader {
virtual ~FileReader() {}
/// Result of ReadFile.
enum Status {
Okay,
NotFound,
OtherError
};
/// Read and store in given string. On success, return Okay.
/// On error, return another Status and fill |err|.
virtual Status ReadFile(const std::string& path, std::string* contents,
std::string* err) = 0;
};
/// Interface for accessing the disk.
///
/// Abstract so it can be mocked out for tests. The real implementation
/// is RealDiskInterface.
struct DiskInterface: public FileReader {
/// stat() a file, returning the mtime, or 0 if missing and -1 on
/// other errors.
virtual TimeStamp Stat(const std::string& path, std::string* err) const = 0;
/// Create a directory, returning false on failure.
virtual bool MakeDir(const std::string& path) = 0;
/// Create a file, with the specified name and contents
/// Returns true on success, false on failure
virtual bool WriteFile(const std::string& path,
const std::string& contents) = 0;
/// Remove the file named @a path. It behaves like 'rm -f path' so no errors
/// are reported if it does not exists.
/// @returns 0 if the file has been removed,
/// 1 if the file does not exist, and
/// -1 if an error occurs.
virtual int RemoveFile(const std::string& path) = 0;
/// Create all the parent directories for path; like mkdir -p
/// `basename path`.
bool MakeDirs(const std::string& path);
};
/// Implementation of DiskInterface that actually hits the disk.
struct RealDiskInterface : public DiskInterface {
RealDiskInterface();
virtual ~RealDiskInterface() {}
virtual TimeStamp Stat(const std::string& path, std::string* err) const;
virtual bool MakeDir(const std::string& path);
virtual bool WriteFile(const std::string& path, const std::string& contents);
virtual Status ReadFile(const std::string& path, std::string* contents,
std::string* err);
virtual int RemoveFile(const std::string& path);
/// Whether stat information can be cached. Only has an effect on Windows.
void AllowStatCache(bool allow);
#ifdef _WIN32
/// Whether long paths are enabled. Only has an effect on Windows.
bool AreLongPathsEnabled() const;
#endif
private:
#ifdef _WIN32
/// Whether stat information can be cached.
bool use_cache_;
/// Whether long paths are enabled.
bool long_paths_enabled_;
typedef std::map<std::string, TimeStamp> DirCache;
// TODO: Neither a map nor a hashmap seems ideal here. If the statcache
// works out, come up with a better data structure.
typedef std::map<std::string, DirCache> Cache;
mutable Cache cache_;
#endif
};
#endif // NINJA_DISK_INTERFACE_H_

756
src/graph.h
View File

@ -1,378 +1,378 @@
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef NINJA_GRAPH_H_
#define NINJA_GRAPH_H_
#include <algorithm>
#include <set>
#include <string>
#include <vector>
#include "dyndep.h"
#include "eval_env.h"
#include "timestamp.h"
#include "util.h"
struct BuildLog;
struct DepfileParserOptions;
struct DiskInterface;
struct DepsLog;
struct Edge;
struct Node;
struct Pool;
struct State;
/// Information about a node in the dependency graph: the file, whether
/// it's dirty, mtime, etc.
struct Node {
Node(const std::string& path, uint64_t slash_bits)
: path_(path), slash_bits_(slash_bits) {}
/// Return false on error.
bool Stat(DiskInterface* disk_interface, std::string* err);
/// If the file doesn't exist, set the mtime_ from its dependencies
void UpdatePhonyMtime(TimeStamp mtime);
/// Return false on error.
bool StatIfNecessary(DiskInterface* disk_interface, std::string* err) {
if (status_known())
return true;
return Stat(disk_interface, err);
}
/// Mark as not-yet-stat()ed and not dirty.
void ResetState() {
mtime_ = -1;
exists_ = ExistenceStatusUnknown;
dirty_ = false;
}
/// Mark the Node as already-stat()ed and missing.
void MarkMissing() {
if (mtime_ == -1) {
mtime_ = 0;
}
exists_ = ExistenceStatusMissing;
}
bool exists() const {
return exists_ == ExistenceStatusExists;
}
bool status_known() const {
return exists_ != ExistenceStatusUnknown;
}
const std::string& path() const { return path_; }
/// Get |path()| but use slash_bits to convert back to original slash styles.
std::string PathDecanonicalized() const {
return PathDecanonicalized(path_, slash_bits_);
}
static std::string PathDecanonicalized(const std::string& path,
uint64_t slash_bits);
uint64_t slash_bits() const { return slash_bits_; }
TimeStamp mtime() const { return mtime_; }
bool dirty() const { return dirty_; }
void set_dirty(bool dirty) { dirty_ = dirty; }
void MarkDirty() { dirty_ = true; }
bool dyndep_pending() const { return dyndep_pending_; }
void set_dyndep_pending(bool pending) { dyndep_pending_ = pending; }
Edge* in_edge() const { return in_edge_; }
void set_in_edge(Edge* edge) { in_edge_ = edge; }
/// Indicates whether this node was generated from a depfile or dyndep file,
/// instead of being a regular input or output from the Ninja manifest.
bool generated_by_dep_loader() const { return generated_by_dep_loader_; }
void set_generated_by_dep_loader(bool value) {
generated_by_dep_loader_ = value;
}
int id() const { return id_; }
void set_id(int id) { id_ = id; }
const std::vector<Edge*>& out_edges() const { return out_edges_; }
const std::vector<Edge*>& validation_out_edges() const { return validation_out_edges_; }
void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); }
void AddValidationOutEdge(Edge* edge) { validation_out_edges_.push_back(edge); }
void Dump(const char* prefix="") const;
private:
std::string path_;
/// Set bits starting from lowest for backslashes that were normalized to
/// forward slashes by CanonicalizePath. See |PathDecanonicalized|.
uint64_t slash_bits_ = 0;
/// Possible values of mtime_:
/// -1: file hasn't been examined
/// 0: we looked, and file doesn't exist
/// >0: actual file's mtime, or the latest mtime of its dependencies if it doesn't exist
TimeStamp mtime_ = -1;
enum ExistenceStatus {
/// The file hasn't been examined.
ExistenceStatusUnknown,
/// The file doesn't exist. mtime_ will be the latest mtime of its dependencies.
ExistenceStatusMissing,
/// The path is an actual file. mtime_ will be the file's mtime.
ExistenceStatusExists
};
ExistenceStatus exists_ = ExistenceStatusUnknown;
/// Dirty is true when the underlying file is out-of-date.
/// But note that Edge::outputs_ready_ is also used in judging which
/// edges to build.
bool dirty_ = false;
/// Store whether dyndep information is expected from this node but
/// has not yet been loaded.
bool dyndep_pending_ = false;
/// Set to true when this node comes from a depfile, a dyndep file or the
/// deps log. If it does not have a producing edge, the build should not
/// abort if it is missing (as for regular source inputs). By default
/// all nodes have this flag set to true, since the deps and build logs
/// can be loaded before the manifest.
bool generated_by_dep_loader_ = true;
/// The Edge that produces this Node, or NULL when there is no
/// known edge to produce it.
Edge* in_edge_ = nullptr;
/// All Edges that use this Node as an input.
std::vector<Edge*> out_edges_;
/// All Edges that use this Node as a validation.
std::vector<Edge*> validation_out_edges_;
/// A dense integer id for the node, assigned and used by DepsLog.
int id_ = -1;
};
/// An edge in the dependency graph; links between Nodes using Rules.
struct Edge {
enum VisitMark {
VisitNone,
VisitInStack,
VisitDone
};
Edge()
: rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), mark_(VisitNone),
id_(0), outputs_ready_(false), deps_loaded_(false),
deps_missing_(false), generated_by_dep_loader_(false),
command_start_time_(0), implicit_deps_(0), order_only_deps_(0),
implicit_outs_(0) {}
/// Return true if all inputs' in-edges are ready.
bool AllInputsReady() const;
/// Expand all variables in a command and return it as a string.
/// If incl_rsp_file is enabled, the string will also contain the
/// full contents of a response file (if applicable)
std::string EvaluateCommand(bool incl_rsp_file = false) const;
/// Returns the shell-escaped value of |key|.
std::string GetBinding(const std::string& key) const;
bool GetBindingBool(const std::string& key) const;
/// Like GetBinding("depfile"), but without shell escaping.
std::string GetUnescapedDepfile() const;
/// Like GetBinding("dyndep"), but without shell escaping.
std::string GetUnescapedDyndep() const;
/// Like GetBinding("rspfile"), but without shell escaping.
std::string GetUnescapedRspfile() const;
void Dump(const char* prefix="") const;
// Append all edge explicit inputs to |*out|. Possibly with shell escaping.
void CollectInputs(bool shell_escape, std::vector<std::string>* out) const;
const Rule* rule_ = nullptr;
Pool* pool_ = nullptr;
std::vector<Node*> inputs_;
std::vector<Node*> outputs_;
std::vector<Node*> validations_;
Node* dyndep_ = nullptr;
BindingEnv* env_ = nullptr;
VisitMark mark_ = VisitNone;
size_t id_ = 0;
bool outputs_ready_ = false;
bool deps_loaded_ = false;
bool deps_missing_ = false;
bool generated_by_dep_loader_ = false;
TimeStamp command_start_time_ = 0;
const Rule& rule() const { return *rule_; }
Pool* pool() const { return pool_; }
int weight() const { return 1; }
bool outputs_ready() const { return outputs_ready_; }
// There are three types of inputs.
// 1) explicit deps, which show up as $in on the command line;
// 2) implicit deps, which the target depends on implicitly (e.g. C headers),
// and changes in them cause the target to rebuild;
// 3) order-only deps, which are needed before the target builds but which
// don't cause the target to rebuild.
// These are stored in inputs_ in that order, and we keep counts of
// #2 and #3 when we need to access the various subsets.
int implicit_deps_ = 0;
int order_only_deps_ = 0;
bool is_implicit(size_t index) {
return index >= inputs_.size() - order_only_deps_ - implicit_deps_ &&
!is_order_only(index);
}
bool is_order_only(size_t index) {
return index >= inputs_.size() - order_only_deps_;
}
// There are two types of outputs.
// 1) explicit outs, which show up as $out on the command line;
// 2) implicit outs, which the target generates but are not part of $out.
// These are stored in outputs_ in that order, and we keep a count of
// #2 to use when we need to access the various subsets.
int implicit_outs_ = 0;
bool is_implicit_out(size_t index) const {
return index >= outputs_.size() - implicit_outs_;
}
bool is_phony() const;
bool use_console() const;
bool maybe_phonycycle_diagnostic() const;
// Historical info: how long did this edge take last time,
// as per .ninja_log, if known? Defaults to -1 if unknown.
int64_t prev_elapsed_time_millis = -1;
};
struct EdgeCmp {
bool operator()(const Edge* a, const Edge* b) const {
return a->id_ < b->id_;
}
};
typedef std::set<Edge*, EdgeCmp> EdgeSet;
/// ImplicitDepLoader loads implicit dependencies, as referenced via the
/// "depfile" attribute in build files.
struct ImplicitDepLoader {
ImplicitDepLoader(State* state, DepsLog* deps_log,
DiskInterface* disk_interface,
DepfileParserOptions const* depfile_parser_options)
: state_(state), disk_interface_(disk_interface), deps_log_(deps_log),
depfile_parser_options_(depfile_parser_options) {}
/// Load implicit dependencies for \a edge.
/// @return false on error (without filling \a err if info is just missing
// or out of date).
bool LoadDeps(Edge* edge, std::string* err);
DepsLog* deps_log() const {
return deps_log_;
}
protected:
/// Process loaded implicit dependencies for \a edge and update the graph
/// @return false on error (without filling \a err if info is just missing)
virtual bool ProcessDepfileDeps(Edge* edge,
std::vector<StringPiece>* depfile_ins,
std::string* err);
/// Load implicit dependencies for \a edge from a depfile attribute.
/// @return false on error (without filling \a err if info is just missing).
bool LoadDepFile(Edge* edge, const std::string& path, std::string* err);
/// Load implicit dependencies for \a edge from the DepsLog.
/// @return false on error (without filling \a err if info is just missing).
bool LoadDepsFromLog(Edge* edge, std::string* err);
/// Preallocate \a count spaces in the input array on \a edge, returning
/// an iterator pointing at the first new space.
std::vector<Node*>::iterator PreallocateSpace(Edge* edge, int count);
State* state_;
DiskInterface* disk_interface_;
DepsLog* deps_log_;
DepfileParserOptions const* depfile_parser_options_;
};
/// DependencyScan manages the process of scanning the files in a graph
/// and updating the dirty/outputs_ready state of all the nodes and edges.
struct DependencyScan {
DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log,
DiskInterface* disk_interface,
DepfileParserOptions const* depfile_parser_options)
: build_log_(build_log),
disk_interface_(disk_interface),
dep_loader_(state, deps_log, disk_interface, depfile_parser_options),
dyndep_loader_(state, disk_interface) {}
/// Update the |dirty_| state of the given nodes by transitively inspecting
/// their input edges.
/// Examine inputs, outputs, and command lines to judge whether an edge
/// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_|
/// state accordingly.
/// Appends any validation nodes found to the nodes parameter.
/// Returns false on failure.
bool RecomputeDirty(Node* node, std::vector<Node*>* validation_nodes, std::string* err);
/// Recompute whether any output of the edge is dirty, if so sets |*dirty|.
/// Returns false on failure.
bool RecomputeOutputsDirty(Edge* edge, Node* most_recent_input,
bool* dirty, std::string* err);
BuildLog* build_log() const {
return build_log_;
}
void set_build_log(BuildLog* log) {
build_log_ = log;
}
DepsLog* deps_log() const {
return dep_loader_.deps_log();
}
/// Load a dyndep file from the given node's path and update the
/// build graph with the new information. One overload accepts
/// a caller-owned 'DyndepFile' object in which to store the
/// information loaded from the dyndep file.
bool LoadDyndeps(Node* node, std::string* err) const;
bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const;
private:
bool RecomputeNodeDirty(Node* node, std::vector<Node*>* stack,
std::vector<Node*>* validation_nodes, std::string* err);
bool VerifyDAG(Node* node, std::vector<Node*>* stack, std::string* err);
/// Recompute whether a given single output should be marked dirty.
/// Returns true if so.
bool RecomputeOutputDirty(const Edge* edge, const Node* most_recent_input,
const std::string& command, Node* output);
BuildLog* build_log_;
DiskInterface* disk_interface_;
ImplicitDepLoader dep_loader_;
DyndepLoader dyndep_loader_;
};
#endif // NINJA_GRAPH_H_
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef NINJA_GRAPH_H_
#define NINJA_GRAPH_H_
#include <algorithm>
#include <set>
#include <string>
#include <vector>
#include "dyndep.h"
#include "eval_env.h"
#include "timestamp.h"
#include "util.h"
struct BuildLog;
struct DepfileParserOptions;
struct DiskInterface;
struct DepsLog;
struct Edge;
struct Node;
struct Pool;
struct State;
/// Information about a node in the dependency graph: the file, whether
/// it's dirty, mtime, etc.
struct Node {
Node(const std::string& path, uint64_t slash_bits)
: path_(path), slash_bits_(slash_bits) {}
/// Return false on error.
bool Stat(DiskInterface* disk_interface, std::string* err);
/// If the file doesn't exist, set the mtime_ from its dependencies
void UpdatePhonyMtime(TimeStamp mtime);
/// Return false on error.
bool StatIfNecessary(DiskInterface* disk_interface, std::string* err) {
if (status_known())
return true;
return Stat(disk_interface, err);
}
/// Mark as not-yet-stat()ed and not dirty.
void ResetState() {
mtime_ = -1;
exists_ = ExistenceStatusUnknown;
dirty_ = false;
}
/// Mark the Node as already-stat()ed and missing.
void MarkMissing() {
if (mtime_ == -1) {
mtime_ = 0;
}
exists_ = ExistenceStatusMissing;
}
bool exists() const {
return exists_ == ExistenceStatusExists;
}
bool status_known() const {
return exists_ != ExistenceStatusUnknown;
}
const std::string& path() const { return path_; }
/// Get |path()| but use slash_bits to convert back to original slash styles.
std::string PathDecanonicalized() const {
return PathDecanonicalized(path_, slash_bits_);
}
static std::string PathDecanonicalized(const std::string& path,
uint64_t slash_bits);
uint64_t slash_bits() const { return slash_bits_; }
TimeStamp mtime() const { return mtime_; }
bool dirty() const { return dirty_; }
void set_dirty(bool dirty) { dirty_ = dirty; }
void MarkDirty() { dirty_ = true; }
bool dyndep_pending() const { return dyndep_pending_; }
void set_dyndep_pending(bool pending) { dyndep_pending_ = pending; }
Edge* in_edge() const { return in_edge_; }
void set_in_edge(Edge* edge) { in_edge_ = edge; }
/// Indicates whether this node was generated from a depfile or dyndep file,
/// instead of being a regular input or output from the Ninja manifest.
bool generated_by_dep_loader() const { return generated_by_dep_loader_; }
void set_generated_by_dep_loader(bool value) {
generated_by_dep_loader_ = value;
}
int id() const { return id_; }
void set_id(int id) { id_ = id; }
const std::vector<Edge*>& out_edges() const { return out_edges_; }
const std::vector<Edge*>& validation_out_edges() const { return validation_out_edges_; }
void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); }
void AddValidationOutEdge(Edge* edge) { validation_out_edges_.push_back(edge); }
void Dump(const char* prefix="") const;
private:
std::string path_;
/// Set bits starting from lowest for backslashes that were normalized to
/// forward slashes by CanonicalizePath. See |PathDecanonicalized|.
uint64_t slash_bits_ = 0;
/// Possible values of mtime_:
/// -1: file hasn't been examined
/// 0: we looked, and file doesn't exist
/// >0: actual file's mtime, or the latest mtime of its dependencies if it doesn't exist
TimeStamp mtime_ = -1;
enum ExistenceStatus {
/// The file hasn't been examined.
ExistenceStatusUnknown,
/// The file doesn't exist. mtime_ will be the latest mtime of its dependencies.
ExistenceStatusMissing,
/// The path is an actual file. mtime_ will be the file's mtime.
ExistenceStatusExists
};
ExistenceStatus exists_ = ExistenceStatusUnknown;
/// Dirty is true when the underlying file is out-of-date.
/// But note that Edge::outputs_ready_ is also used in judging which
/// edges to build.
bool dirty_ = false;
/// Store whether dyndep information is expected from this node but
/// has not yet been loaded.
bool dyndep_pending_ = false;
/// Set to true when this node comes from a depfile, a dyndep file or the
/// deps log. If it does not have a producing edge, the build should not
/// abort if it is missing (as for regular source inputs). By default
/// all nodes have this flag set to true, since the deps and build logs
/// can be loaded before the manifest.
bool generated_by_dep_loader_ = true;
/// The Edge that produces this Node, or NULL when there is no
/// known edge to produce it.
Edge* in_edge_ = nullptr;
/// All Edges that use this Node as an input.
std::vector<Edge*> out_edges_;
/// All Edges that use this Node as a validation.
std::vector<Edge*> validation_out_edges_;
/// A dense integer id for the node, assigned and used by DepsLog.
int id_ = -1;
};
/// An edge in the dependency graph; links between Nodes using Rules.
struct Edge {
enum VisitMark {
VisitNone,
VisitInStack,
VisitDone
};
Edge()
: rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), mark_(VisitNone),
id_(0), outputs_ready_(false), deps_loaded_(false),
deps_missing_(false), generated_by_dep_loader_(false),
command_start_time_(0), implicit_deps_(0), order_only_deps_(0),
implicit_outs_(0) {}
/// Return true if all inputs' in-edges are ready.
bool AllInputsReady() const;
/// Expand all variables in a command and return it as a string.
/// If incl_rsp_file is enabled, the string will also contain the
/// full contents of a response file (if applicable)
std::string EvaluateCommand(bool incl_rsp_file = false) const;
/// Returns the shell-escaped value of |key|.
std::string GetBinding(const std::string& key) const;
bool GetBindingBool(const std::string& key) const;
/// Like GetBinding("depfile"), but without shell escaping.
std::string GetUnescapedDepfile() const;
/// Like GetBinding("dyndep"), but without shell escaping.
std::string GetUnescapedDyndep() const;
/// Like GetBinding("rspfile"), but without shell escaping.
std::string GetUnescapedRspfile() const;
void Dump(const char* prefix="") const;
// Append all edge explicit inputs to |*out|. Possibly with shell escaping.
void CollectInputs(bool shell_escape, std::vector<std::string>* out) const;
const Rule* rule_ = nullptr;
Pool* pool_ = nullptr;
std::vector<Node*> inputs_;
std::vector<Node*> outputs_;
std::vector<Node*> validations_;
Node* dyndep_ = nullptr;
BindingEnv* env_ = nullptr;
VisitMark mark_ = VisitNone;
size_t id_ = 0;
bool outputs_ready_ = false;
bool deps_loaded_ = false;
bool deps_missing_ = false;
bool generated_by_dep_loader_ = false;
TimeStamp command_start_time_ = 0;
const Rule& rule() const { return *rule_; }
Pool* pool() const { return pool_; }
int weight() const { return 1; }
bool outputs_ready() const { return outputs_ready_; }
// There are three types of inputs.
// 1) explicit deps, which show up as $in on the command line;
// 2) implicit deps, which the target depends on implicitly (e.g. C headers),
// and changes in them cause the target to rebuild;
// 3) order-only deps, which are needed before the target builds but which
// don't cause the target to rebuild.
// These are stored in inputs_ in that order, and we keep counts of
// #2 and #3 when we need to access the various subsets.
int implicit_deps_ = 0;
int order_only_deps_ = 0;
bool is_implicit(size_t index) {
return index >= inputs_.size() - order_only_deps_ - implicit_deps_ &&
!is_order_only(index);
}
bool is_order_only(size_t index) {
return index >= inputs_.size() - order_only_deps_;
}
// There are two types of outputs.
// 1) explicit outs, which show up as $out on the command line;
// 2) implicit outs, which the target generates but are not part of $out.
// These are stored in outputs_ in that order, and we keep a count of
// #2 to use when we need to access the various subsets.
int implicit_outs_ = 0;
bool is_implicit_out(size_t index) const {
return index >= outputs_.size() - implicit_outs_;
}
bool is_phony() const;
bool use_console() const;
bool maybe_phonycycle_diagnostic() const;
// Historical info: how long did this edge take last time,
// as per .ninja_log, if known? Defaults to -1 if unknown.
int64_t prev_elapsed_time_millis = -1;
};
struct EdgeCmp {
bool operator()(const Edge* a, const Edge* b) const {
return a->id_ < b->id_;
}
};
typedef std::set<Edge*, EdgeCmp> EdgeSet;
/// ImplicitDepLoader loads implicit dependencies, as referenced via the
/// "depfile" attribute in build files.
struct ImplicitDepLoader {
ImplicitDepLoader(State* state, DepsLog* deps_log,
DiskInterface* disk_interface,
DepfileParserOptions const* depfile_parser_options)
: state_(state), disk_interface_(disk_interface), deps_log_(deps_log),
depfile_parser_options_(depfile_parser_options) {}
/// Load implicit dependencies for \a edge.
/// @return false on error (without filling \a err if info is just missing
// or out of date).
bool LoadDeps(Edge* edge, std::string* err);
DepsLog* deps_log() const {
return deps_log_;
}
protected:
/// Process loaded implicit dependencies for \a edge and update the graph
/// @return false on error (without filling \a err if info is just missing)
virtual bool ProcessDepfileDeps(Edge* edge,
std::vector<StringPiece>* depfile_ins,
std::string* err);
/// Load implicit dependencies for \a edge from a depfile attribute.
/// @return false on error (without filling \a err if info is just missing).
bool LoadDepFile(Edge* edge, const std::string& path, std::string* err);
/// Load implicit dependencies for \a edge from the DepsLog.
/// @return false on error (without filling \a err if info is just missing).
bool LoadDepsFromLog(Edge* edge, std::string* err);
/// Preallocate \a count spaces in the input array on \a edge, returning
/// an iterator pointing at the first new space.
std::vector<Node*>::iterator PreallocateSpace(Edge* edge, int count);
State* state_;
DiskInterface* disk_interface_;
DepsLog* deps_log_;
DepfileParserOptions const* depfile_parser_options_;
};
/// DependencyScan manages the process of scanning the files in a graph
/// and updating the dirty/outputs_ready state of all the nodes and edges.
struct DependencyScan {
DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log,
DiskInterface* disk_interface,
DepfileParserOptions const* depfile_parser_options)
: build_log_(build_log),
disk_interface_(disk_interface),
dep_loader_(state, deps_log, disk_interface, depfile_parser_options),
dyndep_loader_(state, disk_interface) {}
/// Update the |dirty_| state of the given nodes by transitively inspecting
/// their input edges.
/// Examine inputs, outputs, and command lines to judge whether an edge
/// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_|
/// state accordingly.
/// Appends any validation nodes found to the nodes parameter.
/// Returns false on failure.
bool RecomputeDirty(Node* node, std::vector<Node*>* validation_nodes, std::string* err);
/// Recompute whether any output of the edge is dirty, if so sets |*dirty|.
/// Returns false on failure.
bool RecomputeOutputsDirty(Edge* edge, Node* most_recent_input,
bool* dirty, std::string* err);
BuildLog* build_log() const {
return build_log_;
}
void set_build_log(BuildLog* log) {
build_log_ = log;
}
DepsLog* deps_log() const {
return dep_loader_.deps_log();
}
/// Load a dyndep file from the given node's path and update the
/// build graph with the new information. One overload accepts
/// a caller-owned 'DyndepFile' object in which to store the
/// information loaded from the dyndep file.
bool LoadDyndeps(Node* node, std::string* err) const;
bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const;
private:
bool RecomputeNodeDirty(Node* node, std::vector<Node*>* stack,
std::vector<Node*>* validation_nodes, std::string* err);
bool VerifyDAG(Node* node, std::vector<Node*>* stack, std::string* err);
/// Recompute whether a given single output should be marked dirty.
/// Returns true if so.
bool RecomputeOutputDirty(const Edge* edge, const Node* most_recent_input,
const std::string& command, Node* output);
BuildLog* build_log_;
DiskInterface* disk_interface_;
ImplicitDepLoader dep_loader_;
DyndepLoader dyndep_loader_;
};
#endif // NINJA_GRAPH_H_

57
src/status.cc
View File

@ -107,20 +107,20 @@ void StatusPrinter::RecalculateProgressPrediction() {
// that is, if we have took at least 15 sec AND finished at least 5% of edges,
// we can check whether our performance so far matches the previous one.
if (use_previous_times && total_edges_ && finished_edges_ &&
(time_millis_ >= 15 * 1e3) &&
(((double)finished_edges_ / total_edges_) >= 0.05)) {
(time_millis_ >= 15 * 1e3) &&
(((double)finished_edges_ / total_edges_) >= 0.05)) {
// Over the edges we've just run, how long did they take on average?
double actual_average_cpu_time_millis =
(double)cpu_time_millis_ / finished_edges_;
(double)cpu_time_millis_ / finished_edges_;
// What is the previous average, for the edges with such knowledge?
double previous_average_cpu_time_millis =
(double)eta_predictable_cpu_time_total_millis_ /
eta_predictable_edges_total_;
(double)eta_predictable_cpu_time_total_millis_ /
eta_predictable_edges_total_;
double ratio = std::max(previous_average_cpu_time_millis,
actual_average_cpu_time_millis) /
std::min(previous_average_cpu_time_millis,
actual_average_cpu_time_millis);
actual_average_cpu_time_millis) /
std::min(previous_average_cpu_time_millis,
actual_average_cpu_time_millis);
// Let's say that the average times should differ by less than 10x
use_previous_times = ratio < 10;
@ -133,8 +133,8 @@ void StatusPrinter::RecalculateProgressPrediction() {
return;
int edges_with_unknown_runtime = use_previous_times
? eta_unpredictable_edges_remaining_
: (total_edges_ - finished_edges_);
? eta_unpredictable_edges_remaining_
: (total_edges_ - finished_edges_);
// Given the time elapsed on the edges we've just run,
// and the runtime of the edges for which we know previous runtime,
@ -142,10 +142,10 @@ void StatusPrinter::RecalculateProgressPrediction() {
int64_t edges_known_runtime_total_millis = cpu_time_millis_;
if (use_previous_times)
edges_known_runtime_total_millis +=
eta_predictable_cpu_time_remaining_millis_;
eta_predictable_cpu_time_remaining_millis_;
double average_cpu_time_millis =
(double)edges_known_runtime_total_millis / edges_with_known_runtime;
(double)edges_known_runtime_total_millis / edges_with_known_runtime;
// For the edges for which we do not have the previous runtime,
// let's assume that their average runtime is the same as for the other edges,
@ -155,12 +155,12 @@ void StatusPrinter::RecalculateProgressPrediction() {
// And therefore we can predict the remaining and total runtimes.
double total_cpu_time_remaining_millis =
unpredictable_cpu_time_remaining_millis;
unpredictable_cpu_time_remaining_millis;
if (use_previous_times)
total_cpu_time_remaining_millis +=
eta_predictable_cpu_time_remaining_millis_;
eta_predictable_cpu_time_remaining_millis_;
double total_cpu_time_millis =
cpu_time_millis_ + total_cpu_time_remaining_millis;
cpu_time_millis_ + total_cpu_time_remaining_millis;
if (total_cpu_time_millis == 0.0)
return;
@ -181,7 +181,7 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
if (edge->prev_elapsed_time_millis != -1) {
--eta_predictable_edges_remaining_;
eta_predictable_cpu_time_remaining_millis_ -=
edge->prev_elapsed_time_millis;
edge->prev_elapsed_time_millis;
} else
--eta_unpredictable_edges_remaining_;
@ -200,13 +200,13 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
if (!success) {
string outputs;
for (vector<Node*>::const_iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o)
o != edge->outputs_.end(); ++o)
outputs += (*o)->path() + " ";
if (printer_.supports_color()) {
printer_.PrintOnNewLine("\x1B[31m" "FAILED: " "\x1B[0m" + outputs + "\n");
printer_.PrintOnNewLine("\x1B[31m" "FAILED: " "\x1B[0m" + outputs + "\n");
} else {
printer_.PrintOnNewLine("FAILED: " + outputs + "\n");
printer_.PrintOnNewLine("FAILED: " + outputs + "\n");
}
printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n");
}
@ -230,14 +230,14 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
final_output = output;
#ifdef _WIN32
// Fix extra CR being added on Windows, writing out CR CR LF (#773)
_setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix
// Fix extra CR being added on Windows, writing out CR CR LF (#773)
_setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix
#endif
printer_.PrintOnNewLine(final_output);
printer_.PrintOnNewLine(final_output);
#ifdef _WIN32
_setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix
_setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix
#endif
}
}
@ -356,7 +356,7 @@ string StatusPrinter::FormatProgressStatus(const char* progress_status_format,
}
const bool print_with_hours =
elapsed_sec >= 60 * 60 || eta_sec >= 60 * 60;
elapsed_sec >= 60 * 60 || eta_sec >= 60 * 60;
double sec = -1;
switch (*s) {
@ -394,7 +394,7 @@ string StatusPrinter::FormatProgressStatus(const char* progress_status_format,
// Percentage of time spent out of the predicted time total
case 'P': {
snprintf(buf, sizeof(buf), "%3i%%",
(int)(100. * time_predicted_percentage_));
(int)(100. * time_predicted_percentage_));
out += buf;
break;
}
@ -413,7 +413,7 @@ string StatusPrinter::FormatProgressStatus(const char* progress_status_format,
void StatusPrinter::PrintStatus(const Edge* edge, int64_t time_millis) {
if (config_.verbosity == BuildConfig::QUIET
|| config_.verbosity == BuildConfig::NO_STATUS_UPDATE)
|| config_.verbosity == BuildConfig::NO_STATUS_UPDATE)
return;
RecalculateProgressPrediction();
@ -425,10 +425,9 @@ void StatusPrinter::PrintStatus(const Edge* edge, int64_t time_millis) {
to_print = edge->GetBinding("command");
to_print = FormatProgressStatus(progress_status_format_, time_millis)
+ to_print;
+ to_print;
printer_.Print(to_print,
force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE);
printer_.Print(to_print,force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE);
}
void StatusPrinter::Warning(const char* msg, ...) {