Fix statistics and cache cleanup for files generated by direct mode

The statistics counters "files in cache" and "cache size" now only include
object files. It's not worth the effort to keep track of the .manifest, .d and
.stderr files since the cache size is dominated by the object files, and it's
unlikely that anyone is bothered by du(1) having a different opinion (which
also was the case before anyway). Consequently, the "max file" and "max cache
size" settings now specify thresholds for object files count and size.

ccache.c

@@ -501,17 +501,18 @@ static void to_cache(ARGS *args)
 		failed();
 	}
 
-	/* do an extra stat on the cache files for
-	   the size statistics */
-	if (stat(path_stderr, &st1) != 0
-	    || stat(object_path, &st2) != 0) {
-		cc_log("Failed to stat cache files - %s\n", strerror(errno));
+	/*
+	 * Do an extra stat on the potentially compressed object file for the
+	 * size statistics.
+	 */
+	if (stat(object_path, &st2) != 0) {
+		cc_log("Failed to stat %s\n", strerror(errno));
 		stats_update(STATS_ERROR);
 		failed();
 	}
 
 	cc_log("Placed object file into the cache\n");
-	stats_tocache(file_size(&st1) + file_size(&st2));
+	stats_tocache(file_size(&st2));
 
 	free(tmp_hashname);
 	free(tmp_stderr);
@@ -1731,7 +1732,7 @@ int main(int argc, char *argv[])
 	}
 
 	/* make sure the cache dir exists */
-	if (cache_dir && (create_dir(cache_dir) != 0)) {
+	if (cache_dir && (create_dir(cache_dir)) != 0) {
 		fprintf(stderr,"ccache: failed to create %s (%s)\n",
 			cache_dir, strerror(errno));
 		exit(1);

cleanup.c

@@ -28,10 +28,24 @@ static struct files {
 } **files;
 static unsigned allocated;
 static unsigned num_files;
-static size_t total_size;
-static size_t total_files;
-static size_t size_threshold;
-static size_t files_threshold;
+static size_t total_object_size;
+static size_t total_object_files;
+static size_t object_size_threshold;
+static size_t object_files_threshold;
+
+static int is_object_file(const char *fname)
+{
+	int i;
+
+	for (i = strlen(fname) - 1; i >= 0; i--) {
+		if (fname[i] == '.') {
+			return 0;
+		} else if (fname[i] == '-') {
+			return 1;
+		}
+	}
+	return 1;
+}
 
 /* file comparison function to try to delete the oldest files first */
 static int files_compare(struct files **f1, struct files **f2)
@@ -72,15 +86,18 @@ static void traverse_fn(const char *fname, struct stat *st)
 
 	if (num_files == allocated) {
 		allocated = 10000 + num_files*2;
-		files = (struct files **)x_realloc(files,
-						   sizeof(struct files *)*allocated);
+		files = (struct files **)x_realloc(
+			files, sizeof(struct files *)*allocated);
 	}
 
 	files[num_files] = (struct files *)x_malloc(sizeof(struct files));
 	files[num_files]->fname = x_strdup(fname);
 	files[num_files]->mtime = st->st_mtime;
 	files[num_files]->size = file_size(st) / 1024;
-	total_size += files[num_files]->size;
+	if (is_object_file(fname)) {
+		total_object_files += 1;
+		total_object_size += files[num_files]->size;
+	}
 	num_files++;
 }
 
@@ -97,9 +114,13 @@ static void sort_and_clean(void)
 	}
 
 	/* delete enough files to bring us below the threshold */
-	for (i=0;i<num_files; i++) {
-		if ((size_threshold==0 || total_size < size_threshold) &&
-		    (files_threshold==0 || (num_files-i) < files_threshold)) break;
+	for (i = 0; i < num_files; i++) {
+		if ((object_size_threshold == 0
+		     || total_object_size < object_size_threshold)
+		    && (object_files_threshold == 0
+			|| (num_files-i) < object_files_threshold)) {
+			break;
+		}
 
 		if (unlink(files[i]->fname) != 0 && errno != ENOENT) {
 			fprintf(stderr, "unlink %s - %s\n",
@@ -107,10 +128,11 @@ static void sort_and_clean(void)
 			continue;
 		}
 
-		total_size -= files[i]->size;
+		if (is_object_file(files[i]->fname)) {
+			total_object_files -= 1;
+			total_object_size -= files[i]->size;
+		}
 	}
-
-	total_files = num_files - i;
 }
 
 /* cleanup in one cache subdir */
@@ -118,11 +140,12 @@ void cleanup_dir(const char *dir, size_t maxfiles, size_t maxsize)
 {
 	unsigned i;
 
-	size_threshold = maxsize * LIMIT_MULTIPLE;
-	files_threshold = maxfiles * LIMIT_MULTIPLE;
+	object_size_threshold = maxsize * LIMIT_MULTIPLE;
+	object_files_threshold = maxfiles * LIMIT_MULTIPLE;
 
 	num_files = 0;
-	total_size = 0;
+	total_object_files = 0;
+	total_object_size = 0;
 
 	/* build a list of files */
 	traverse(dir, traverse_fn);
@@ -130,20 +153,23 @@ void cleanup_dir(const char *dir, size_t maxfiles, size_t maxsize)
 	/* clean the cache */
 	sort_and_clean();
 
-	stats_set_sizes(dir, total_files, total_size);
+	stats_set_sizes(dir, total_object_files, total_object_size);
 
 	/* free it up */
-	for (i=0;i<num_files;i++) {
+	for (i = 0; i < num_files; i++) {
 		free(files[i]->fname);
 		free(files[i]);
 		files[i] = NULL;
 	}
-	if (files) free(files);
+	if (files) {
+		free(files);
+	}
 	allocated = 0;
 	files = NULL;
 
 	num_files = 0;
-	total_size = 0;
+	total_object_files = 0;
+	total_object_size = 0;
 }
 
 /* cleanup in all cache subdirs */
@@ -153,7 +179,7 @@ void cleanup_all(const char *dir)
 	char *dname, *sfile;
 	int i;
 
-	for (i=0;i<=0xF;i++) {
+	for (i = 0; i <= 0xF; i++) {
 		x_asprintf(&dname, "%s/%1x", dir, i);
 		x_asprintf(&sfile, "%s/%1x/stats", dir, i);
 
@@ -168,7 +194,6 @@ void cleanup_all(const char *dir)
 	}
 }
 
-
 /* traverse function for wiping files */
 static void wipe_fn(const char *fname, struct stat *st)
 {
@@ -186,14 +211,13 @@ static void wipe_fn(const char *fname, struct stat *st)
 	unlink(fname);
 }
 
-
 /* wipe all cached files in all subdirs */
 void wipe_all(const char *dir)
 {
 	char *dname;
 	int i;
 
-	for (i=0;i<=0xF;i++) {
+	for (i = 0; i <= 0xF; i++) {
 		x_asprintf(&dname, "%s/%1x", dir, i);
 		traverse(dir, wipe_fn);
 		free(dname);

stats.c

@@ -149,7 +149,7 @@ static void stats_update_size(enum stats stat, size_t size)
 
 	/* on a cache miss we up the file count and size */
 	if (stat == STATS_TOCACHE) {
-		counters[STATS_NUMFILES] += 2;
+		counters[STATS_NUMFILES] += 1;
 		counters[STATS_TOTALSIZE] += size;
 	}
 

test.sh

@@ -65,6 +65,7 @@ basetests() {
     rm -rf $CCACHE_DIR
     checkstat 'cache hit (preprocessed)' 0
     checkstat 'cache miss' 0
+    checkstat 'files in cache' 0
 
     j=1
     rm -f *.c
@@ -77,16 +78,19 @@ basetests() {
     $CCACHE_COMPILE -c test1.c
     checkstat 'cache hit (preprocessed)' 0
     checkstat 'cache miss' 1
+    checkstat 'files in cache' 1
 
     testname="BASIC2"
     $CCACHE_COMPILE -c test1.c
     checkstat 'cache hit (preprocessed)' 1
     checkstat 'cache miss' 1
+    checkstat 'files in cache' 1
 
     testname="debug"
     $CCACHE_COMPILE -c test1.c -g
     checkstat 'cache hit (preprocessed)' 1
     checkstat 'cache miss' 2
+    checkstat 'files in cache' 2
 
     testname="debug2"
     $CCACHE_COMPILE -c test1.c -g
@@ -137,7 +141,6 @@ basetests() {
     $CCACHE_COMPILE -c -O2 2> /dev/null
     checkstat 'no input file' 1
 
-
     testname="CCACHE_DISABLE"
     CCACHE_DISABLE=1 $CCACHE_COMPILE -c test1.c 2> /dev/null
     checkstat 'cache hit (preprocessed)' 3
@@ -163,11 +166,10 @@ basetests() {
     checkstat 'cache hit (preprocessed)' 5
     checkstat 'cache miss' 4
 
-    # strictly speaking should be 6 - RECACHE causes a double counting!
-    checkstat 'files in cache' 8
+    # strictly speaking should be 3 - RECACHE causes a double counting!
+    checkstat 'files in cache' 4
     $CCACHE -c > /dev/null
-    checkstat 'files in cache' 6
-
+    checkstat 'files in cache' 3
 
     testname="CCACHE_HASHDIR"
     CCACHE_HASHDIR=1 $CCACHE_COMPILE -c test1.c -O -O
@@ -177,8 +179,7 @@ basetests() {
     CCACHE_HASHDIR=1 $CCACHE_COMPILE -c test1.c -O -O
     checkstat 'cache hit (preprocessed)' 6
     checkstat 'cache miss' 5
-
-    checkstat 'files in cache' 8
+    checkstat 'files in cache' 4
 
     testname="comments"
     echo '/* a silly comment */' > test1-comment.c
@@ -206,9 +207,10 @@ basetests() {
     done
     checkstat 'cache hit (preprocessed)' 8
     checkstat 'cache miss' 37
-    checkstat 'files in cache' 72
-    $CCACHE -F 48 -c > /dev/null
-    if [ `getstat 'files in cache'` -gt 48 ]; then
+    checkstat 'files in cache' 36
+
+    $CCACHE -F 32 -c > /dev/null
+    if [ `getstat 'files in cache'` -gt 32 ]; then
         test_failed '-F test failed'
     fi