cleaned up the xattr_impl branch and changed output of xattr values to base64 instead of hex since values may be large blobs.

option_parser.ggo

@@ -21,6 +21,8 @@ option  "output"     o   "where to write output" string  typestr="FILE" optional
 option  "input"      i   "output from a previous run to check for uniqueness"
     string  typestr="FILE"  optional    multiple
 
+option  "xattr"      x   "where to write eXtended ATTRibutes" string typestr="FILE" optional
+
 option  "owner"      O   "username to chown new files/dirs" 
     string  typestr="USER"  optional
     

src/impl/process_regular.c

@@ -106,7 +106,7 @@ static int copy_fd(int dirfd, const char *pathname, struct stream *stream,
  *
  * @return          number of bytes that are valid in buf, -1 on error
  */
-static ssize_t cache_n_digest(digesterset_t *set, int fd, void *buf, 
+static ssize_t cache_n_digest(digesterset_t *set, int fd, void *buf,
         size_t blen);
 
 /**
@@ -173,8 +173,10 @@ int process_regular(file_t *newdir, const char *newpath, const char *oldpath,
     /* ensure we create the hash needed for the and index */
     digesterset_create(&dgstset, opts->digests | idxkeytype);
 
-    /* there is no index to check against, just copy and digest at the same
-     * time */
+    /*
+     * there is no index to check against, just copy and digest at the same
+     * time
+     */
     if (opts->index == NULL)
     {
         valid_len = copy_n_digest(newdir->fd, newpath, opts->uid, opts->gid,
@@ -205,7 +207,6 @@ int process_regular(file_t *newdir, const char *newpath, const char *oldpath,
     }
     else
     {
-
         /* read in the file and calculate the desired digests */
         if ((valid_len = cache_n_digest(&dgstset, s, opts->buffer,
                 opts->buffer_size)) == -1)
@@ -229,8 +230,8 @@ int process_regular(file_t *newdir, const char *newpath, const char *oldpath,
                 ret = -1;
                 goto cleanup;
 
+            /* we have seen this file, skip it */
             case INDEX_SUCCESS:
-                /* we have seen this file, skip it */
                 ret = 0;        /* set ret to success */
                 goto cleanup;
 
@@ -283,7 +284,7 @@ int process_regular(file_t *newdir, const char *newpath, const char *oldpath,
 /* Private Impl ***************************************************************/
 
 
-int copy_fd(int dirfd, const char *pathname, struct stream *stream, uid_t uid, 
+int copy_fd(int dirfd, const char *pathname, struct stream *stream, uid_t uid,
         gid_t gid)
 {
     int d;

src/io/io_entry.c

@@ -18,7 +18,7 @@
 #include <time.h>
 #include <linux/limits.h>
 
-#include "jansson.h"
+#include <jansson.h>
 
 #include "io_entry.h"
 #include "../entry.h"
@@ -311,15 +311,16 @@ int io_entry_read(entry_t *entry, FILE *in, size_t *line)
 }
 
 
-/* while jansson can be used for creating a json structure then printing it
+/*
+ * while jansson can be used for creating a json structure then printing it
  * out, there is a large overhead cost. Since we control the data only use
- * jansson to escape the strings we cannot control. */
+ * jansson to escape the strings we cannot control.
+ */
 int io_entry_write_fields(const char *state, const char *path,
-        const struct stat *st, const void *pathmd5, const char *symlinkpath, const void *md5,
-        const void *sha1, const void *sha256, const void *sha512,
-        long elapsed, FILE *stream)
+        const struct stat *st, const void *pathmd5, const char *symlinkpath,
+        const void *md5, const void *sha1, const void *sha256,
+        const void *sha512, long elapsed, FILE *stream)
 {
-
     enum { MAX_LENGTH = PATH_MAX * 4 };
 
     int ret;
@@ -458,7 +459,7 @@ int io_entry_write_fields(const char *state, const char *path,
             goto cleanup;
         }
         unpack(buf, path, len);
-        fprintf(stream, "\"pathhex\":\"%s\",", buf);
+        fprintf(stream, "\"pathhex\":\"%s\"", buf);
     }
 
 

src/io/io_xattr.c

@@ -7,15 +7,18 @@
  *
  * Implementation of the io_xattr.h interface. Entries in dcp input and output
  * are an entry in the filesystem that was copied/duplicated. For each
- * file and symbolic link we log xattr information about each as a single line json
- * object.
+ * file and symbolic link we log xattr information about each as a single line
+ * json object.
  */
 #include <stdio.h>
 #include <string.h>
 #include <stddef.h>
 #include <stdint.h>
 #include <time.h>
-#include <linux/limits.h>
+
+#include <jansson.h>
+#include <openssl/bio.h>
+#include <openssl/evp.h>
 
 #include "io_xattr.h"
 #include "../digest.h"
@@ -26,42 +29,63 @@
 /* Public Impl ****************************************************************/
 
 
-/* while jansson can be used for creating a json structure then printing it
- * out, there is a large overhead cost. Since we control the data only use
- * jansson to escape the strings we cannot control. */
-int io_entry_write_xattr_fields(const void *pathmd5, const char *name, void *valuebuffer, ssize_t valuesize, FILE *stream)
+int io_entry_write_xattr_fields(const void *pathmd5, const char *name,
+        void *valuebuffer, ssize_t valuesize, FILE *stream)
 {
-
-    enum { MAX_LENGTH = PATH_MAX * 4 };
-
     int ret;
+    int w;
 
-    /* used to hold hex strings for the hashes and the path if it contains
-     * non utf-8 characters */
-    char buf[MAX_LENGTH];
+    /* where to store the hex representation of the path md5 sum */
+    char buf[(MD5_DIGEST_LENGTH * 2) + 1];
 
+    /* we store the xattr value as base64 */
+    BIO *bio, *b64;
+
+    /* used to properly escape the utf-8 name of an xattr */
+    json_t *escaped;
 
     ret = 0;
+    b64 = NULL;
 
-    /* we are writing a json object on the line*/
+    /* we are writing a json object on the line */
     fputc('{', stream);
 
     unpack(buf, pathmd5, MD5_DIGEST_LENGTH);
     fprintf(stream, "\"pathmd5\":\"%s\",", buf);
 
     if (name != NULL)
-    	fprintf(stream, "\"xattrName\":\"%s\",",name);
+    {
+        if ((escaped = json_string(name)) == NULL)
+        {
+            log_errorx("non valid utf-8 xattr string '%s'", name);
+            ret = -1;
+            goto cleanup;
+        }
+        fputs("\"xattrName\":", stream);
+        json_dumpf(escaped, stream, JSON_ENCODE_ANY);
+        json_decref(escaped);
+    }
 
     if (valuesize > 0)
     {
-    	if (valuebuffer != NULL) {
-    		char valuehex[8193]; /* 4096 * 2 + 1 */
-    		memset(valuehex, 0, sizeof(valuehex));
-    		unpack(valuehex, valuebuffer, valuesize);
-    		fprintf(stream, "\"xattrValue\":\"%s\",",valuehex);
-    	}
+        fputs(",\"xattrValue\":\"", stream);
+        /*
+         * base64 encoding
+         * https://www.openssl.org/docs/manmaster/crypto/BIO_f_base64.html
+         */
+        b64 = BIO_new(BIO_f_base64());
+        BIO_set_flags(b64, BIO_FLAGS_BASE64_NO_NL); /* do not append newline */
+        bio = BIO_new_fp(stream, BIO_NOCLOSE);      /* do not close stream */
+        BIO_push(b64, bio);
+        if ((w = BIO_write(b64, valuebuffer, valuesize)) != valuesize)
+        {
+            log_errorx("cannot base64 encode xattr value for attr: '%s'", name);
+            ret = -1;
+        }
+        BIO_flush(b64);
+        BIO_free_all(b64);
+        fputc('"', stream);
     }
-    goto cleanup;
 
 
 cleanup:

src/io/io_xattr.h

@@ -31,7 +31,8 @@
  *
  * @return              0 on success, -1 on error
  */
-int io_entry_write_xattr_fields(const void *pathmd5, const char *name, void *valuebuffer, ssize_t valuesize, FILE *stream);
+int io_entry_write_xattr_fields(const void *pathmd5, const char *name,
+        void *valuebuffer, ssize_t valuesize, FILE *stream);
 
 
 #endif

src/io_dcp_processor.c

@@ -12,6 +12,7 @@
 #include <string.h>
 #include <sys/stat.h>
 #include <sys/xattr.h>
+#include <linux/limits.h>
 
 #include "io_dcp_processor.h"
 #include "entry.h"
@@ -32,9 +33,13 @@ struct io_dcp_processor_ctx {
     FILE *xattrout; /**< where to write xattr values for paths */
 };
 
+
 /* Private API ****************************************************************/
+
+
 static int process_xattrs(const void *pathmd5, const char *filepath, FILE *out);
 
+
 /* Public Impl ****************************************************************/
 
 
@@ -52,7 +57,8 @@ int io_dcp_processor(dcp_state_t state, const void *pathmd5,
 }
 
 
-int io_dcp_processor_ctx_create(io_dcp_processor_ctx_t **ctx, FILE *stream, FILE *xattrstream)
+int io_dcp_processor_ctx_create(io_dcp_processor_ctx_t **ctx, FILE *stream,
+        FILE *xattrstream)
 {
     if (ctx != NULL)
     {
@@ -61,7 +67,7 @@ int io_dcp_processor_ctx_create(io_dcp_processor_ctx_t **ctx, FILE *stream, FILE
         (*ctx)->xattrout = xattrstream;
         return 0;
     }
-    return 0;
+    return -1;
 }
 
 
@@ -72,81 +78,46 @@ int io_dcp_processor_ctx_free(io_dcp_processor_ctx_t *ctx)
     return 0;
 }
 
+
 /* Private Impl **************************************************************/
+
+
 int process_xattrs(const void *pathmd5, const char *filepath, FILE *out)
 {
-	char *keys;
-	uint8_t valuebuffer[4096];
-	size_t valuebuffersize = sizeof(valuebuffer);
-	ssize_t bufsize, bufsize2, valuesize;
-	char *next, *end;
-	int retval;
+    char keys[XATTR_LIST_MAX];
+    size_t keysbuffersize = XATTR_LIST_MAX;
+    uint8_t valuebuffer[XATTR_SIZE_MAX];
+    size_t valuebuffersize = XATTR_SIZE_MAX;
+    ssize_t bufsize, valuesize;
+    char *next, *end;
 
-	if (filepath == NULL)
-	{
-		return 0; /* Nothing to do */
-	}
+    if (filepath == NULL)
+        return 0; /* Nothing to do */
 
-	/* Use listxattrs to get a list of */
-	bufsize = llistxattr(filepath, NULL, 0);
-	if (bufsize == -1)
-	{
-		if (errno == ENOTSUP)
-		{
-			/* xattrs not supported or disabled for this file, success */
-			return 0;
-		}
+    if ((bufsize = llistxattr(filepath, keys, keysbuffersize)) == 0)
+        return 0;
+    else if (bufsize < 0)
+    {
+        /* xattrs not supported or disabled for this file, success */
+        if (errno == ENOTSUP)
+            return 0;
+        log_error("llistxattr");
+        return -1;
+    }
 
-		perror("llistxattr");
-		return -1;
-	}
+    end = keys + bufsize; /* pointer to the end of the list */
 
-	if (bufsize == 0)
-	{
-		/* no xattrs to process */
-		return 0;
-	}
+    /* keys is one large string with '\0' delimited names */
+    for (next = keys; next < end; next += (strlen(next) + 1))
+    {
+        memset(valuebuffer, 0, valuebuffersize);
+        valuesize = lgetxattr(filepath, next, valuebuffer, valuebuffersize);
+        if (valuesize < 0)
+            log_error("lgetxattr");
+        else
+            io_entry_write_xattr_fields(pathmd5,next,valuebuffer,valuesize,out);
+    }
 
-	/* NOTE: Not sure if listxattr returns size including the last null byte */
-	keys = NULL;
-	keys = calloc(sizeof(char), bufsize + 1);
-	if (keys == NULL)
-	{
-			perror("calloc");
-			return -1;
-	}
-
-	retval = 0;
-	bufsize2 = llistxattr(filepath, keys, bufsize);
-	if (bufsize2 != bufsize)
-	{
-		/* possible race condition or just an error */
-		perror("llistxattr");
-		retval = -1;
-		goto cleanup;
-	}
-
-	end = keys + bufsize; /* pointer to the end of the list */
-
-	/* Pointer math */
-	for (next = keys; next != end; next += (strlen(next) + 1))
-	{
-		memset(valuebuffer, 0, valuebuffersize); /* zero out the value buffer*/
-		valuesize = lgetxattr(filepath, next, valuebuffer, valuebuffersize);
-		if (valuesize < 0)
-		{
-			perror("lgetxattr");
-		}
-
-		else
-		{
-			io_entry_write_xattr_fields(pathmd5, next, valuebuffer, valuesize, out);
-		}
-	}
-
-	cleanup:
-		free(keys);
-		fflush(out);
-
-	return retval;
+    fflush(out);
+    return 0;
 }

src/main.c

@@ -205,10 +205,6 @@ FILE *parse_outputstream(const struct cmdline_info *info, char **outfilename)
         if ((stream = fopen(info->output_arg, "w")) == NULL)
             log_crit(EXIT_FAILURE,"failed to open output file '%s'",
                     info->output_arg);
-
-        if (fchown(fileno(stream), parse_owner(info, NULL), parse_group(info, NULL)) == -1)
-                	warnx("Unable to change owner of DCP out file.");
-
         *outfilename = strdup(info->output_arg);
     }
 
@@ -225,10 +221,6 @@ FILE *parse_outputstream(const struct cmdline_info *info, char **outfilename)
             else
                 log_crit(EXIT_FAILURE, "cannot create output file '%s'", name);
         }
-
-        if (fchown(fd, parse_owner(info, NULL), parse_group(info, NULL)) == -1)
-        	warnx("Unable to change owner of DCP out file.");
-
         if ((stream = fdopen(fd, "a")) == NULL)
             log_crit(EXIT_FAILURE, "cannot create output stream");
         *outfilename = strdup(name);
@@ -249,10 +241,6 @@ FILE *parse_xattroutputstream(const struct cmdline_info *info, char **outfilenam
         if ((stream = fopen(info->xattr_arg, "w")) == NULL)
             log_crit(EXIT_FAILURE,"failed to open output file '%s'",
                     info->xattr_arg);
-
-        if (fchown(fileno(stream), parse_owner(info, NULL), parse_group(info, NULL)) == -1)
-                	warnx("Unable to change owner of DCP out file.");
-
         *outfilename = strdup(info->xattr_arg);
     }
 
@@ -269,10 +257,6 @@ FILE *parse_xattroutputstream(const struct cmdline_info *info, char **outfilenam
             else
                 log_crit(EXIT_FAILURE, "cannot create output file '%s'", name);
         }
-
-        if (fchown(fd, parse_owner(info, NULL), parse_group(info, NULL)) == -1)
-        	warnx("Unable to change owner of DCP XATTR out file.");
-
         if ((stream = fdopen(fd, "a")) == NULL)
             log_crit(EXIT_FAILURE, "cannot create output stream");
         *outfilename = strdup(name);
@@ -441,10 +425,10 @@ index_t *build_index(int digests, const char *paths[], size_t count)
     digest_t type;
     size_t i;
 
-	/* assign type to the first valid one we find, checking md5 then sha1 ... */
-	if (  !(type = digests & DGST_MD5)    && !(type = digests & DGST_SHA1) &&
-		  !(type = digests & DGST_SHA256) && !(type = digests & DGST_SHA512))
-		log_critx(EXIT_FAILURE, "corrput parsing of digest types from inputs");
+    /* assign type to the first valid one we find, checking md5 then sha1 ... */
+    if (  !(type = digests & DGST_MD5)    && !(type = digests & DGST_SHA1) &&
+          !(type = digests & DGST_SHA256) && !(type = digests & DGST_SHA512))
+        log_critx(EXIT_FAILURE, "corrput parsing of digest types from inputs");
 
     if (index_create(&idx, type) != 0)
         log_critx(EXIT_FAILURE, "cannot create index");
@@ -498,8 +482,8 @@ int prepare(const char **files, size_t count, const char *dest, char **newdest)
                 {
                     name = basename(path);
                     /* build the dest/name path including the slash if needed */
-                    if(asprintf(&fulldest, "%s%s%s",
-                            dest, dest[strlen(dest)-1] == '/'? "" : "/", name) < 0)
+                    if(asprintf(&fulldest, "%s%s%s", dest,
+                        dest[strlen(dest)-1] == '/'? "" : "/", name) < 0)
                     {
                         fulldest = NULL;
                     }
@@ -550,7 +534,7 @@ int print_metadata(FILE *out, const char *version, int argc,
 
     /* current working directory */
     if ((cwd = getcwd(NULL, 0)) == NULL)
-    	log_warn("cannot retrieve current working directory");
+        log_warn("cannot retrieve current working directory");
 
     /* hostname */
     gethostname(hostname, sizeof(hostname));
@@ -565,13 +549,13 @@ int print_metadata(FILE *out, const char *version, int argc,
 
     if (cwd != NULL)
     {
-    	io_metadata_put_json("cwd        ", 1, (const char **) &cwd, out);
-    	free(cwd);
+        io_metadata_put_json("cwd        ", 1, (const char **) &cwd, out);
+        free(cwd);
     }
 
     io_metadata_put_json("sources    ", opts->filecount, opts->files, out);
-    io_metadata_put_json("destination", 1, (const char **) &opts->dest, out);
-    io_metadata_put_json("output     ", 1,(const char**)&opts->outfilename,out);
+    io_metadata_put_json("destination",1,(const char **) &opts->dest, out);
+    io_metadata_put_json("output     ",1,(const char**)&opts->outfilename,out);
 
     if (opts->username != NULL)
         io_metadata_put( "data_owner ", opts->username, out);