diff --git a/option_parser.ggo b/option_parser.ggo index 9b2caa0..3a0a755 100644 --- a/option_parser.ggo +++ b/option_parser.ggo @@ -21,6 +21,8 @@ option "output" o "where to write output" string typestr="FILE" optional option "input" i "output from a previous run to check for uniqueness" string typestr="FILE" optional multiple +option "xattr" x "where to write eXtended ATTRibutes" string typestr="FILE" optional + option "owner" O "username to chown new files/dirs" string typestr="USER" optional diff --git a/src/impl/process_regular.c b/src/impl/process_regular.c index c85c125..b84aaf1 100644 --- a/src/impl/process_regular.c +++ b/src/impl/process_regular.c @@ -106,7 +106,7 @@ static int copy_fd(int dirfd, const char *pathname, struct stream *stream, * * @return number of bytes that are valid in buf, -1 on error */ -static ssize_t cache_n_digest(digesterset_t *set, int fd, void *buf, +static ssize_t cache_n_digest(digesterset_t *set, int fd, void *buf, size_t blen); /** @@ -173,8 +173,10 @@ int process_regular(file_t *newdir, const char *newpath, const char *oldpath, /* ensure we create the hash needed for the and index */ digesterset_create(&dgstset, opts->digests | idxkeytype); - /* there is no index to check against, just copy and digest at the same - * time */ + /* + * there is no index to check against, just copy and digest at the same + * time + */ if (opts->index == NULL) { valid_len = copy_n_digest(newdir->fd, newpath, opts->uid, opts->gid, @@ -205,7 +207,6 @@ int process_regular(file_t *newdir, const char *newpath, const char *oldpath, } else { - /* read in the file and calculate the desired digests */ if ((valid_len = cache_n_digest(&dgstset, s, opts->buffer, opts->buffer_size)) == -1) @@ -229,8 +230,8 @@ int process_regular(file_t *newdir, const char *newpath, const char *oldpath, ret = -1; goto cleanup; + /* we have seen this file, skip it */ case INDEX_SUCCESS: - /* we have seen this file, skip it */ ret = 0; /* set ret to success */ goto cleanup; @@ -283,7 +284,7 @@ int process_regular(file_t *newdir, const char *newpath, const char *oldpath, /* Private Impl ***************************************************************/ -int copy_fd(int dirfd, const char *pathname, struct stream *stream, uid_t uid, +int copy_fd(int dirfd, const char *pathname, struct stream *stream, uid_t uid, gid_t gid) { int d; diff --git a/src/io/io_entry.c b/src/io/io_entry.c index 304b05f..459629a 100644 --- a/src/io/io_entry.c +++ b/src/io/io_entry.c @@ -18,7 +18,7 @@ #include #include -#include "jansson.h" +#include #include "io_entry.h" #include "../entry.h" @@ -311,15 +311,16 @@ int io_entry_read(entry_t *entry, FILE *in, size_t *line) } -/* while jansson can be used for creating a json structure then printing it +/* + * while jansson can be used for creating a json structure then printing it * out, there is a large overhead cost. Since we control the data only use - * jansson to escape the strings we cannot control. */ + * jansson to escape the strings we cannot control. + */ int io_entry_write_fields(const char *state, const char *path, - const struct stat *st, const void *pathmd5, const char *symlinkpath, const void *md5, - const void *sha1, const void *sha256, const void *sha512, - long elapsed, FILE *stream) + const struct stat *st, const void *pathmd5, const char *symlinkpath, + const void *md5, const void *sha1, const void *sha256, + const void *sha512, long elapsed, FILE *stream) { - enum { MAX_LENGTH = PATH_MAX * 4 }; int ret; @@ -458,7 +459,7 @@ int io_entry_write_fields(const char *state, const char *path, goto cleanup; } unpack(buf, path, len); - fprintf(stream, "\"pathhex\":\"%s\",", buf); + fprintf(stream, "\"pathhex\":\"%s\"", buf); } diff --git a/src/io/io_xattr.c b/src/io/io_xattr.c index 27cfee3..9f42e4e 100644 --- a/src/io/io_xattr.c +++ b/src/io/io_xattr.c @@ -7,15 +7,18 @@ * * Implementation of the io_xattr.h interface. Entries in dcp input and output * are an entry in the filesystem that was copied/duplicated. For each - * file and symbolic link we log xattr information about each as a single line json - * object. + * file and symbolic link we log xattr information about each as a single line + * json object. */ #include #include #include #include #include -#include + +#include +#include +#include #include "io_xattr.h" #include "../digest.h" @@ -26,42 +29,63 @@ /* Public Impl ****************************************************************/ -/* while jansson can be used for creating a json structure then printing it - * out, there is a large overhead cost. Since we control the data only use - * jansson to escape the strings we cannot control. */ -int io_entry_write_xattr_fields(const void *pathmd5, const char *name, void *valuebuffer, ssize_t valuesize, FILE *stream) +int io_entry_write_xattr_fields(const void *pathmd5, const char *name, + void *valuebuffer, ssize_t valuesize, FILE *stream) { - - enum { MAX_LENGTH = PATH_MAX * 4 }; - int ret; + int w; - /* used to hold hex strings for the hashes and the path if it contains - * non utf-8 characters */ - char buf[MAX_LENGTH]; + /* where to store the hex representation of the path md5 sum */ + char buf[(MD5_DIGEST_LENGTH * 2) + 1]; + /* we store the xattr value as base64 */ + BIO *bio, *b64; + + /* used to properly escape the utf-8 name of an xattr */ + json_t *escaped; ret = 0; + b64 = NULL; - /* we are writing a json object on the line*/ + /* we are writing a json object on the line */ fputc('{', stream); unpack(buf, pathmd5, MD5_DIGEST_LENGTH); fprintf(stream, "\"pathmd5\":\"%s\",", buf); if (name != NULL) - fprintf(stream, "\"xattrName\":\"%s\",",name); + { + if ((escaped = json_string(name)) == NULL) + { + log_errorx("non valid utf-8 xattr string '%s'", name); + ret = -1; + goto cleanup; + } + fputs("\"xattrName\":", stream); + json_dumpf(escaped, stream, JSON_ENCODE_ANY); + json_decref(escaped); + } if (valuesize > 0) { - if (valuebuffer != NULL) { - char valuehex[8193]; /* 4096 * 2 + 1 */ - memset(valuehex, 0, sizeof(valuehex)); - unpack(valuehex, valuebuffer, valuesize); - fprintf(stream, "\"xattrValue\":\"%s\",",valuehex); - } + fputs(",\"xattrValue\":\"", stream); + /* + * base64 encoding + * https://www.openssl.org/docs/manmaster/crypto/BIO_f_base64.html + */ + b64 = BIO_new(BIO_f_base64()); + BIO_set_flags(b64, BIO_FLAGS_BASE64_NO_NL); /* do not append newline */ + bio = BIO_new_fp(stream, BIO_NOCLOSE); /* do not close stream */ + BIO_push(b64, bio); + if ((w = BIO_write(b64, valuebuffer, valuesize)) != valuesize) + { + log_errorx("cannot base64 encode xattr value for attr: '%s'", name); + ret = -1; + } + BIO_flush(b64); + BIO_free_all(b64); + fputc('"', stream); } - goto cleanup; cleanup: diff --git a/src/io/io_xattr.h b/src/io/io_xattr.h index f848820..5792a32 100644 --- a/src/io/io_xattr.h +++ b/src/io/io_xattr.h @@ -31,7 +31,8 @@ * * @return 0 on success, -1 on error */ -int io_entry_write_xattr_fields(const void *pathmd5, const char *name, void *valuebuffer, ssize_t valuesize, FILE *stream); +int io_entry_write_xattr_fields(const void *pathmd5, const char *name, + void *valuebuffer, ssize_t valuesize, FILE *stream); #endif diff --git a/src/io_dcp_processor.c b/src/io_dcp_processor.c index 3b26a00..7d09706 100644 --- a/src/io_dcp_processor.c +++ b/src/io_dcp_processor.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "io_dcp_processor.h" #include "entry.h" @@ -32,9 +33,13 @@ struct io_dcp_processor_ctx { FILE *xattrout; /**< where to write xattr values for paths */ }; + /* Private API ****************************************************************/ + + static int process_xattrs(const void *pathmd5, const char *filepath, FILE *out); + /* Public Impl ****************************************************************/ @@ -52,7 +57,8 @@ int io_dcp_processor(dcp_state_t state, const void *pathmd5, } -int io_dcp_processor_ctx_create(io_dcp_processor_ctx_t **ctx, FILE *stream, FILE *xattrstream) +int io_dcp_processor_ctx_create(io_dcp_processor_ctx_t **ctx, FILE *stream, + FILE *xattrstream) { if (ctx != NULL) { @@ -61,7 +67,7 @@ int io_dcp_processor_ctx_create(io_dcp_processor_ctx_t **ctx, FILE *stream, FILE (*ctx)->xattrout = xattrstream; return 0; } - return 0; + return -1; } @@ -72,81 +78,46 @@ int io_dcp_processor_ctx_free(io_dcp_processor_ctx_t *ctx) return 0; } + /* Private Impl **************************************************************/ + + int process_xattrs(const void *pathmd5, const char *filepath, FILE *out) { - char *keys; - uint8_t valuebuffer[4096]; - size_t valuebuffersize = sizeof(valuebuffer); - ssize_t bufsize, bufsize2, valuesize; - char *next, *end; - int retval; + char keys[XATTR_LIST_MAX]; + size_t keysbuffersize = XATTR_LIST_MAX; + uint8_t valuebuffer[XATTR_SIZE_MAX]; + size_t valuebuffersize = XATTR_SIZE_MAX; + ssize_t bufsize, valuesize; + char *next, *end; - if (filepath == NULL) - { - return 0; /* Nothing to do */ - } + if (filepath == NULL) + return 0; /* Nothing to do */ - /* Use listxattrs to get a list of */ - bufsize = llistxattr(filepath, NULL, 0); - if (bufsize == -1) - { - if (errno == ENOTSUP) - { - /* xattrs not supported or disabled for this file, success */ - return 0; - } + if ((bufsize = llistxattr(filepath, keys, keysbuffersize)) == 0) + return 0; + else if (bufsize < 0) + { + /* xattrs not supported or disabled for this file, success */ + if (errno == ENOTSUP) + return 0; + log_error("llistxattr"); + return -1; + } - perror("llistxattr"); - return -1; - } + end = keys + bufsize; /* pointer to the end of the list */ - if (bufsize == 0) - { - /* no xattrs to process */ - return 0; - } + /* keys is one large string with '\0' delimited names */ + for (next = keys; next < end; next += (strlen(next) + 1)) + { + memset(valuebuffer, 0, valuebuffersize); + valuesize = lgetxattr(filepath, next, valuebuffer, valuebuffersize); + if (valuesize < 0) + log_error("lgetxattr"); + else + io_entry_write_xattr_fields(pathmd5,next,valuebuffer,valuesize,out); + } - /* NOTE: Not sure if listxattr returns size including the last null byte */ - keys = NULL; - keys = calloc(sizeof(char), bufsize + 1); - if (keys == NULL) - { - perror("calloc"); - return -1; - } - - retval = 0; - bufsize2 = llistxattr(filepath, keys, bufsize); - if (bufsize2 != bufsize) - { - /* possible race condition or just an error */ - perror("llistxattr"); - retval = -1; - goto cleanup; - } - - end = keys + bufsize; /* pointer to the end of the list */ - - /* Pointer math */ - for (next = keys; next != end; next += (strlen(next) + 1)) - { - memset(valuebuffer, 0, valuebuffersize); /* zero out the value buffer*/ - valuesize = lgetxattr(filepath, next, valuebuffer, valuebuffersize); - if (valuesize < 0) - { - perror("lgetxattr"); - } - - else - { - io_entry_write_xattr_fields(pathmd5, next, valuebuffer, valuesize, out); - } - } - - cleanup: - free(keys); - fflush(out); - - return retval; + fflush(out); + return 0; } diff --git a/src/main.c b/src/main.c index 7af61a4..84bbfbc 100644 --- a/src/main.c +++ b/src/main.c @@ -205,10 +205,6 @@ FILE *parse_outputstream(const struct cmdline_info *info, char **outfilename) if ((stream = fopen(info->output_arg, "w")) == NULL) log_crit(EXIT_FAILURE,"failed to open output file '%s'", info->output_arg); - - if (fchown(fileno(stream), parse_owner(info, NULL), parse_group(info, NULL)) == -1) - warnx("Unable to change owner of DCP out file."); - *outfilename = strdup(info->output_arg); } @@ -225,10 +221,6 @@ FILE *parse_outputstream(const struct cmdline_info *info, char **outfilename) else log_crit(EXIT_FAILURE, "cannot create output file '%s'", name); } - - if (fchown(fd, parse_owner(info, NULL), parse_group(info, NULL)) == -1) - warnx("Unable to change owner of DCP out file."); - if ((stream = fdopen(fd, "a")) == NULL) log_crit(EXIT_FAILURE, "cannot create output stream"); *outfilename = strdup(name); @@ -249,10 +241,6 @@ FILE *parse_xattroutputstream(const struct cmdline_info *info, char **outfilenam if ((stream = fopen(info->xattr_arg, "w")) == NULL) log_crit(EXIT_FAILURE,"failed to open output file '%s'", info->xattr_arg); - - if (fchown(fileno(stream), parse_owner(info, NULL), parse_group(info, NULL)) == -1) - warnx("Unable to change owner of DCP out file."); - *outfilename = strdup(info->xattr_arg); } @@ -269,10 +257,6 @@ FILE *parse_xattroutputstream(const struct cmdline_info *info, char **outfilenam else log_crit(EXIT_FAILURE, "cannot create output file '%s'", name); } - - if (fchown(fd, parse_owner(info, NULL), parse_group(info, NULL)) == -1) - warnx("Unable to change owner of DCP XATTR out file."); - if ((stream = fdopen(fd, "a")) == NULL) log_crit(EXIT_FAILURE, "cannot create output stream"); *outfilename = strdup(name); @@ -441,10 +425,10 @@ index_t *build_index(int digests, const char *paths[], size_t count) digest_t type; size_t i; - /* assign type to the first valid one we find, checking md5 then sha1 ... */ - if ( !(type = digests & DGST_MD5) && !(type = digests & DGST_SHA1) && - !(type = digests & DGST_SHA256) && !(type = digests & DGST_SHA512)) - log_critx(EXIT_FAILURE, "corrput parsing of digest types from inputs"); + /* assign type to the first valid one we find, checking md5 then sha1 ... */ + if ( !(type = digests & DGST_MD5) && !(type = digests & DGST_SHA1) && + !(type = digests & DGST_SHA256) && !(type = digests & DGST_SHA512)) + log_critx(EXIT_FAILURE, "corrput parsing of digest types from inputs"); if (index_create(&idx, type) != 0) log_critx(EXIT_FAILURE, "cannot create index"); @@ -498,8 +482,8 @@ int prepare(const char **files, size_t count, const char *dest, char **newdest) { name = basename(path); /* build the dest/name path including the slash if needed */ - if(asprintf(&fulldest, "%s%s%s", - dest, dest[strlen(dest)-1] == '/'? "" : "/", name) < 0) + if(asprintf(&fulldest, "%s%s%s", dest, + dest[strlen(dest)-1] == '/'? "" : "/", name) < 0) { fulldest = NULL; } @@ -550,7 +534,7 @@ int print_metadata(FILE *out, const char *version, int argc, /* current working directory */ if ((cwd = getcwd(NULL, 0)) == NULL) - log_warn("cannot retrieve current working directory"); + log_warn("cannot retrieve current working directory"); /* hostname */ gethostname(hostname, sizeof(hostname)); @@ -565,13 +549,13 @@ int print_metadata(FILE *out, const char *version, int argc, if (cwd != NULL) { - io_metadata_put_json("cwd ", 1, (const char **) &cwd, out); - free(cwd); + io_metadata_put_json("cwd ", 1, (const char **) &cwd, out); + free(cwd); } io_metadata_put_json("sources ", opts->filecount, opts->files, out); - io_metadata_put_json("destination", 1, (const char **) &opts->dest, out); - io_metadata_put_json("output ", 1,(const char**)&opts->outfilename,out); + io_metadata_put_json("destination",1,(const char **) &opts->dest, out); + io_metadata_put_json("output ",1,(const char**)&opts->outfilename,out); if (opts->username != NULL) io_metadata_put( "data_owner ", opts->username, out);