Let direct mode create dependency file too

This commit is contained in:
Joel Rosdahl 2009-11-22 21:13:32 +01:00
parent 37739bc9e1
commit ba93474268
4 changed files with 242 additions and 59 deletions

155
ccache.c
View File

@ -88,6 +88,12 @@ static time_t time_of_compilation;
*/
static struct hashtable *included_files;
/* is gcc being asked to output dependencies? */
static int generating_dependencies;
/* the path to the dependency file (implicit or specified with -MF) */
static char *dependency_path;
/* the extension of the file after pre-processing */
static const char *i_extension;
@ -203,7 +209,7 @@ char *format_file_hash(struct file_hash *file_hash)
* sublevels if needed. Caller frees.
*/
static char *get_path_in_cache(const char *name, const char *suffix,
int nlevels)
int nlevels)
{
int i;
char *path;
@ -334,7 +340,7 @@ static void process_preprocessed_file(struct mdfour *hash, const char *path)
if (enable_direct) {
included_files = create_hashtable(1000, hash_from_string,
strings_equal);
strings_equal);
}
/* Bytes between p and q are pending to be hashed. */
@ -404,6 +410,8 @@ static void to_cache(ARGS *args)
} else {
args_add(args, i_tmpfile);
}
cc_log("Running real compiler\n");
status = execute(args->argv, tmp_stdout, tmp_stderr);
args_pop(args, 3);
@ -476,7 +484,7 @@ static void to_cache(ARGS *args)
}
#endif
cc_log("Placed %s into cache\n", output_file);
cc_log("Placed object file into the cache\n");
stats_tocache(file_size(&st1) + file_size(&st2));
free(tmp_hashname);
@ -513,9 +521,9 @@ get_object_name_from_cpp(ARGS *args, struct mdfour *hash)
/* now the run */
x_asprintf(&path_stdout, "%s/%s.tmp.%s.%s", temp_dir,
input_base, tmp_string(), i_extension);
input_base, tmp_string(), i_extension);
x_asprintf(&path_stderr, "%s/tmp.cpp_stderr.%s", temp_dir,
tmp_string());
tmp_string());
time_of_compilation = time(NULL);
@ -603,7 +611,7 @@ static int find_hash(ARGS *args, enum findhash_call_mode mode)
break;
case FINDHASH_CPP_MODE:
cc_log("Trying to run preprocessor\n");
cc_log("Running preprocessor\n");
break;
}
@ -710,19 +718,22 @@ static int find_hash(ARGS *args, enum findhash_call_mode mode)
}
manifest_name = hash_result(&hash);
manifest_path = get_path_in_cache(manifest_name, ".manifest",
nlevels);
nlevels);
object_hash = manifest_get(manifest_path);
if (object_hash) {
cc_log("Got object hash from manifest\n");
cc_log("Got object file hash from manifest\n");
} else {
cc_log("Did not find object hash in manifest\n");
cc_log("Did not find object file hash in manifest\n");
return 0;
}
break;
case FINDHASH_CPP_MODE:
object_hash = get_object_name_from_cpp(args, &hash);
cc_log("Got object hash from preprocessor\n");
cc_log("Got object file hash from preprocessor\n");
if (generating_dependencies) {
cc_log("Preprocessor created %s\n", dependency_path);
}
break;
}
@ -741,25 +752,48 @@ static void from_cache(enum fromcache_call_mode mode)
{
int fd_stderr, fd_cpp_stderr;
char *stderr_file;
char *dep_file;
int ret;
struct stat st;
int produce_dep_file;
/*
* (If mode != FROMCACHE_DIRECT_MODE, the dependency file is created by
* gcc.)
*/
produce_dep_file = \
generating_dependencies && mode == FROMCACHE_DIRECT_MODE;
x_asprintf(&stderr_file, "%s.stderr", object_path);
x_asprintf(&dep_file, "%s.d", object_path);
fd_stderr = open(stderr_file, O_RDONLY | O_BINARY);
if (fd_stderr == -1) {
/* it isn't in cache ... */
cc_log("Did not find object file in cache\n");
free(stderr_file);
return;
}
/* make sure the output is there too */
if (stat(object_path, &st) != 0) {
cc_log("Object file missing in cache\n");
close(fd_stderr);
unlink(stderr_file);
free(stderr_file);
return;
}
/* ...and the dependency file, if wanted */
if (produce_dep_file && stat(dep_file, &st) != 0) {
cc_log("Dependency file missing in cache\n");
close(fd_stderr);
unlink(stderr_file);
unlink(object_path);
free(stderr_file);
return;
}
/* the user might be disabling cache hits */
if ((mode != FROMCACHE_COMPILED_MODE && getenv("CCACHE_RECACHE"))
#ifndef ENABLE_ZLIB
@ -769,6 +803,7 @@ static void from_cache(enum fromcache_call_mode mode)
) {
close(fd_stderr);
unlink(stderr_file);
unlink(dep_file);
free(stderr_file);
return;
}
@ -778,9 +813,15 @@ static void from_cache(enum fromcache_call_mode mode)
#ifdef HAVE_UTIMES
utimes(object_path, NULL);
utimes(stderr_file, NULL);
if (produce_dep_file) {
utimes(dep_file, NULL);
}
#else
utime(object_path, NULL);
utime(stderr_file, NULL);
if (produce_dep_file) {
utime(dep_file, NULL);
}
#endif
if (strcmp(output_file, "/dev/null") == 0) {
@ -796,24 +837,65 @@ static void from_cache(enum fromcache_call_mode mode)
}
}
/* the hash file might have been deleted by some external process */
if (ret == -1 && errno == ENOENT) {
cc_log("hashfile missing for %s\n", output_file);
stats_update(STATS_MISSING);
close(fd_stderr);
unlink(stderr_file);
return;
}
free(stderr_file);
if (ret == -1) {
ret = copy_file(object_path, output_file);
if (ret == -1) {
cc_log("failed to copy %s -> %s (%s)\n",
if (errno == ENOENT) {
cc_log("Object file missing for %s\n", output_file);
stats_update(STATS_MISSING);
} else {
cc_log("Failed to copy/link %s -> %s (%s)\n",
object_path, output_file, strerror(errno));
stats_update(STATS_ERROR);
failed();
}
close(fd_stderr);
unlink(stderr_file);
free(stderr_file);
return;
} else {
cc_log("Created %s\n", output_file);
}
if (produce_dep_file) {
unlink(dependency_path);
/* only make a hardlink if the cache file is uncompressed */
if (getenv("CCACHE_HARDLINK") &&
test_if_compressed(dep_file) == 0) {
ret = link(dep_file, dependency_path);
} else {
ret = copy_file(dep_file, dependency_path);
}
if (ret == -1) {
if (errno == ENOENT) {
cc_log("dependency file missing for %s\n",
output_file);
stats_update(STATS_MISSING);
} else {
cc_log("failed to copy/link %s -> %s (%s)\n",
dep_file, dependency_path,
strerror(errno));
stats_update(STATS_ERROR);
failed();
}
close(fd_stderr);
unlink(stderr_file);
free(stderr_file);
unlink(object_path);
return;
} else {
cc_log("Created %s\n", dependency_path);
}
}
if (generating_dependencies && mode != FROMCACHE_DIRECT_MODE) {
/* Store the dependency file in the cache. */
ret = copy_file(dependency_path, dep_file);
if (ret == -1) {
cc_log("Failed to copy %s -> %s\n", dependency_path,
dep_file);
/* Continue despite the error. */
} else {
cc_log("Placed dependency file into the cache\n");
}
}
/* get rid of the intermediate preprocessor file */
@ -844,7 +926,7 @@ static void from_cache(enum fromcache_call_mode mode)
/* Create or update the manifest file. */
if (enable_direct && mode != FROMCACHE_DIRECT_MODE) {
if (manifest_put(manifest_path, object_hash, included_files)) {
cc_log("Added object hash to manifest\n");
cc_log("Added object file hash to manifest\n");
/* Update timestamp for LRU cleanup. */
#ifdef HAVE_UTIMES
utimes(manifest_path, NULL);
@ -852,7 +934,7 @@ static void from_cache(enum fromcache_call_mode mode)
utime(manifest_path, NULL);
#endif
} else {
cc_log("Failed to add object hash to manifest\n");
cc_log("Failed to add object file hash to manifest\n");
}
}
@ -952,14 +1034,11 @@ static void process_args(int argc, char **argv)
int found_c_opt = 0;
int found_S_opt = 0;
struct stat st;
/* is gcc being asked to output dependencies? */
int generating_dependencies = 0;
/* is the dependency makefile name overridden with -MF? */
int dependency_filename_specified = 0;
/* is the dependency makefile target name specified with -MQ or -MF? */
/* is the dependency makefile target name specified with -MT or -MQ? */
int dependency_target_specified = 0;
stripped_args = args_init(0, NULL);
args_add(stripped_args, argv[0]);
@ -1040,13 +1119,17 @@ static void process_args(int argc, char **argv)
/* These options require special handling, because they
behave differently with gcc -E, when the output
file is not specified. */
if (strcmp(argv[i], "-MD") == 0 || strcmp(argv[i], "-MMD") == 0) {
generating_dependencies = 1;
} else if (strcmp(argv[i], "-MF") == 0) {
dependency_filename_specified = 1;
} else if (strcmp(argv[i], "-MQ") == 0 || strcmp(argv[i], "-MT") == 0) {
dependency_target_specified = 1;
}
if (i < argc - 1) {
if (strcmp(argv[i], "-MF") == 0) {
dependency_filename_specified = 1;
dependency_path = x_strdup(argv[i + 1]);
} else if (strcmp(argv[i], "-MQ") == 0
|| strcmp(argv[i], "-MT") == 0) {
dependency_target_specified = 1;
}
}
/* options that take an argument */
@ -1188,6 +1271,7 @@ static void process_args(int argc, char **argv)
strcat(default_depfile_name, ".d");
args_add(stripped_args, "-MF");
args_add(stripped_args, default_depfile_name);
dependency_path = x_strdup(default_depfile_name);
}
if (!dependency_target_specified) {
@ -1256,6 +1340,9 @@ static void ccache(int argc, char *argv[])
process_args(orig_args->argc, orig_args->argv);
cc_log("Source file: %s\n", input_file);
if (generating_dependencies) {
cc_log("Dependency file: %s\n", dependency_path);
}
cc_log("Object file: %s\n", output_file);
/* try to find the hash using the manifest */

View File

@ -37,7 +37,7 @@ print "File infos:"
n = get_uint16()
for i in range(n):
print " %d:" % i
print " Include path index: %d" % get_uint16()
print " Path index: %d" % get_uint16()
print " Hash: %s" % get_md4()
print " Size: %d" % get_uint32()

View File

@ -332,7 +332,7 @@ error:
}
static int verify_object(struct manifest *mf, struct object *obj,
struct hashtable *hashed_files)
struct hashtable *hashed_files)
{
uint32_t i;
struct file_info *fi;
@ -354,8 +354,8 @@ static int verify_object(struct manifest *mf, struct object *obj,
hash_result_as_bytes(&hash, actual->hash);
actual->size = hash.totalN;
hashtable_insert(hashed_files,
x_strdup(mf->files[fi->index]),
actual);
x_strdup(mf->files[fi->index]),
actual);
}
if (memcmp(fi->hash, actual->hash, 16) != 0
|| fi->size != actual->size) {
@ -382,7 +382,7 @@ static struct hashtable *create_string_index_map(char **strings, uint32_t len)
}
static struct hashtable *create_file_info_index_map(struct file_info *infos,
uint32_t len)
uint32_t len)
{
uint32_t i;
struct hashtable *h;
@ -401,8 +401,8 @@ static struct hashtable *create_file_info_index_map(struct file_info *infos,
}
static uint32_t get_include_file_index(struct manifest *mf,
char *path,
struct hashtable *mf_files)
char *path,
struct hashtable *mf_files)
{
uint32_t *index;
uint32_t n;
@ -421,10 +421,10 @@ static uint32_t get_include_file_index(struct manifest *mf,
}
static uint32 get_file_hash_index(struct manifest *mf,
char *path,
struct file_hash *file_hash,
struct hashtable *mf_files,
struct hashtable *mf_file_infos)
char *path,
struct file_hash *file_hash,
struct hashtable *mf_files,
struct hashtable *mf_file_infos)
{
struct file_info fi;
uint32_t *fi_index;
@ -441,7 +441,7 @@ static uint32 get_file_hash_index(struct manifest *mf,
n = mf->n_file_infos;
mf->file_infos = x_realloc(mf->file_infos,
(n + 1) * sizeof(*mf->file_infos));
(n + 1) * sizeof(*mf->file_infos));
mf->n_file_infos++;
mf->file_infos[n] = fi;
@ -450,7 +450,7 @@ static uint32 get_file_hash_index(struct manifest *mf,
static void
add_file_info_indexes(uint32_t *indexes, uint32_t size,
struct manifest *mf, struct hashtable *included_files)
struct manifest *mf, struct hashtable *included_files)
{
struct hashtable_itr *iter;
uint32_t i;
@ -465,14 +465,14 @@ add_file_info_indexes(uint32_t *indexes, uint32_t size,
mf_files = create_string_index_map(mf->files, mf->n_files);
mf_file_infos = create_file_info_index_map(mf->file_infos,
mf->n_file_infos);
mf->n_file_infos);
iter = hashtable_iterator(included_files);
i = 0;
do {
path = hashtable_iterator_key(iter);
file_hash = hashtable_iterator_value(iter);
indexes[i] = get_file_hash_index(mf, path, file_hash, mf_files,
mf_file_infos);
mf_file_infos);
i++;
} while (hashtable_iterator_advance(iter));
assert(i == size);
@ -482,8 +482,8 @@ add_file_info_indexes(uint32_t *indexes, uint32_t size,
}
static void add_object_entry(struct manifest *mf,
struct file_hash *object_hash,
struct hashtable *included_files)
struct file_hash *object_hash,
struct hashtable *included_files)
{
struct object *obj;
uint32_t n;
@ -563,7 +563,7 @@ out:
* Returns 1 on success, otherwise 0.
*/
int manifest_put(const char *manifest_path, struct file_hash *object_hash,
struct hashtable *included_files)
struct hashtable *included_files)
{
int ret = 0;
int fd1;

110
test.sh
View File

@ -48,16 +48,16 @@ checkstat() {
expected_value="$2"
value=`getstat "$stat"`
if [ "$expected_value" != "$value" ]; then
test_failed "SUITE: $testsuite, TEST: $testname - Expected $stat to be $expected_value, got $value"
test_failed "SUITE: $testsuite, TEST: \"$testname\" - Expected $stat to be $expected_value, got $value"
fi
}
checkfile() {
if [ ! -f $1 ]; then
test_failed "SUITE: $testsuite, TEST: $testname - $1 not found"
test_failed "SUITE: $testsuite, TEST: \"$testname\" - $1 not found"
fi
if [ `cat $1` != "$2" ]; then
test_failed "SUITE: $testsuite, TEST: $testname - Bad content of $2.\nExpected: $2\nActual: `cat $1`"
if [ "`cat $1`" != "$2" ]; then
test_failed "SUITE: $testsuite, TEST: \"$testname\" - Bad content of $2.\nExpected: $2\nActual: `cat $1`"
fi
}
@ -285,7 +285,7 @@ EOF
##################################################################
# First compilation is a miss.
testname="first compilation"
$CCACHE -z >/dev/null
$CCACHE -C >/dev/null
$CCACHE $COMPILER -c test.c
checkstat 'cache hit (direct)' 0
checkstat 'cache hit (preprocessed)' 0
@ -331,12 +331,13 @@ EOF
testname="missing header file"
$CCACHE -z >/dev/null
mv test1.h test1.h.saved
mv test3.h test3.h.saved
cat <<EOF >test1.h
/* No more include of test3.h */
int test1;
EOF
sleep 1 # Sleep to make the include file trusted.
rm -f test3.h
$CCACHE $COMPILER -c test.c
checkstat 'cache hit (direct)' 0
@ -348,9 +349,102 @@ EOF
checkstat 'cache hit (preprocessed)' 0
checkstat 'cache miss' 1
# Restore
mv test1.h.saved test1.h
mv test3.h.saved test3.h
sleep 1 # Sleep to make the include files trusted.
##################################################################
# Reset CCACHE_NODIRECT again.
# Test some header modifications to get multiple objects in the manifest.
testname="several objects"
$CCACHE -z >/dev/null
for i in 0 1 2 3 4; do
echo "int test1_$i;" >>test1.h
sleep 1 # Sleep to make the include file trusted.
$CCACHE $COMPILER -c test.c
$CCACHE $COMPILER -c test.c
done
checkstat 'cache hit (direct)' 5
checkstat 'cache hit (preprocessed)' 0
checkstat 'cache miss' 5
##################################################################
# Check that -MD works.
testname="-MD"
$CCACHE -z >/dev/null
$CCACHE $COMPILER -c -MD test.c
checkstat 'cache hit (direct)' 0
checkstat 'cache hit (preprocessed)' 0
checkstat 'cache miss' 1
checkfile test.d "test.o: test.c test1.h test3.h test2.h"
rm -f test.d
$CCACHE $COMPILER -c -MD test.c
checkstat 'cache hit (direct)' 1
checkstat 'cache hit (preprocessed)' 0
checkstat 'cache miss' 1
checkfile test.d "test.o: test.c test1.h test3.h test2.h"
##################################################################
# Check the scenario of running a ccache with direct mode on a cache
# built up by a ccache without direct mode support.
testname="direct mode on old cache"
$CCACHE -z >/dev/null
$CCACHE -C >/dev/null
CCACHE_NODIRECT=1 $CCACHE $COMPILER -c -MD test.c
checkstat 'cache hit (direct)' 0
checkstat 'cache hit (preprocessed)' 0
checkstat 'cache miss' 1
checkfile test.d "test.o: test.c test1.h test3.h test2.h"
rm -f test.d
CCACHE_NODIRECT=1 $CCACHE $COMPILER -c -MD test.c
checkstat 'cache hit (direct)' 0
checkstat 'cache hit (preprocessed)' 1
checkstat 'cache miss' 1
checkfile test.d "test.o: test.c test1.h test3.h test2.h"
rm -f test.d
$CCACHE $COMPILER -c -MD test.c
checkstat 'cache hit (direct)' 0
checkstat 'cache hit (preprocessed)' 2
checkstat 'cache miss' 1
checkfile test.d "test.o: test.c test1.h test3.h test2.h"
rm -f test.d
$CCACHE $COMPILER -c -MD test.c
checkstat 'cache hit (direct)' 1
checkstat 'cache hit (preprocessed)' 2
checkstat 'cache miss' 1
checkfile test.d "test.o: test.c test1.h test3.h test2.h"
##################################################################
# Check that -MF works.
testname="-MF"
$CCACHE -z >/dev/null
$CCACHE $COMPILER -c -MD -MF other.d test.c
checkstat 'cache hit (direct)' 0
checkstat 'cache hit (preprocessed)' 0
checkstat 'cache miss' 1
checkfile other.d "test.o: test.c test1.h test3.h test2.h"
rm -f other.d
$CCACHE $COMPILER -c -MD -MF other.d test.c
checkstat 'cache hit (direct)' 1
checkstat 'cache hit (preprocessed)' 0
checkstat 'cache miss' 1
checkfile other.d "test.o: test.c test1.h test3.h test2.h"
##################################################################
# Reset things.
CCACHE_NODIRECT=1
export CCACHE_NODIRECT
$CCACHE -C >/dev/null
}
######
@ -361,6 +455,8 @@ cd $TESTDIR || exit 1
mkdir .ccache
CCACHE_DIR=.ccache
export CCACHE_DIR
CCACHE_LOGFILE=ccache.log
export CCACHE_LOGFILE
CCACHE_NODIRECT=1
export CCACHE_NODIRECT