From b2f9f6fd8bb233c3b26a972c8b9c4071d6d974ec Mon Sep 17 00:00:00 2001 From: Simon Fraser Date: Wed, 21 Feb 2018 11:54:21 +0000 Subject: [PATCH] Bug 1439860 update-packaging linting fixes and shell style updates r=bhearsum MozReview-Commit-ID: 2vMbmonkVwD --HG-- extra : rebase_source : 05e49e5c3013c0dd726c93f5e48e1b81879b97a9 --- tools/lint/shellcheck.yml | 1 + tools/update-packaging/common.sh | 168 +++++++++--------- tools/update-packaging/make_full_update.sh | 26 +-- .../make_incremental_update.sh | 164 +++++++---------- tools/update-packaging/test/buildrefmars.sh | 12 +- tools/update-packaging/test/catmanifest.sh | 6 +- tools/update-packaging/test/common.sh | 168 ++++++++++-------- tools/update-packaging/test/diffmar.sh | 22 +-- .../update-packaging/test/make_full_update.sh | 30 ++-- 9 files changed, 300 insertions(+), 297 deletions(-) diff --git a/tools/lint/shellcheck.yml b/tools/lint/shellcheck.yml index df93627aa6a8..3088271648ec 100644 --- a/tools/lint/shellcheck.yml +++ b/tools/lint/shellcheck.yml @@ -3,6 +3,7 @@ shellcheck: description: Shell script linter include: - taskcluster/docker/ + - tools/update-packaging/ exclude: [] # 1090: https://github.com/koalaman/shellcheck/wiki/SC1090 # 'Can't follow a non-constant source' diff --git a/tools/update-packaging/common.sh b/tools/update-packaging/common.sh index 42fbf52574d8..03fefa90de17 100755 --- a/tools/update-packaging/common.sh +++ b/tools/update-packaging/common.sh @@ -13,24 +13,23 @@ MAR=${MAR:-mar} MBSDIFF=${MBSDIFF:-mbsdiff} if [[ -z "${MAR_OLD_FORMAT}" ]]; then - XZ=${XZ:-xz} - $XZ --version > /dev/null 2>&1 - if [ $? -ne 0 ]; then - # If $XZ is not set and not found on the path then this is probably - # running on a windows buildbot. Some of the Windows build systems have - # xz.exe in topsrcdir/xz/. Look in the places this would be in both a - # mozilla-central and comm-central build. - XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe" - $XZ --version > /dev/null 2>&1 - if [ $? -ne 0 ]; then - XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe" - $XZ --version > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "xz was not found on this system!" - echo "exiting" - exit 1 - fi + while read -r XZ + do + if ${XZ} --version > /dev/null 2>&1 + then + break fi + done << EOM +${XZ:-xz} +$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe +$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe +EOM + + if [ -z "${XZ}" ] + then + echo "xz was not found on this system!" + echo "exiting" + exit 1 fi else MAR_OLD_FORMAT=1 @@ -45,8 +44,7 @@ notice() { } get_file_size() { - info=($(ls -ln "$1")) - echo ${info[4]} + stat -f"%z" "$1" } copy_perm() { @@ -71,27 +69,27 @@ make_add_instruction() { filev3="$3" # Used to log to the console - if [ $4 ]; then + if [ "$4" ]; then forced=" (forced)" else forced= fi - is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') - if [ $is_extension = "1" ]; then + if [[ "$f" =~ distribution/extensions/.*/ ]] + then # Use the subdirectory of the extensions folder as the file to test # before performing this add instruction. testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') notice " add-if \"$testdir\" \"$f\"" - echo "add-if \"$testdir\" \"$f\"" >> "$filev2" - if [ ! $filev3 = "" ]; then - echo "add-if \"$testdir\" \"$f\"" >> "$filev3" + printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev2}" + if [ -n "${filev3}" ]; then + printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev3}" fi else notice " add \"$f\"$forced" - echo "add \"$f\"" >> "$filev2" + printf 'add "%s"\n' "${f}" >> "${filev2}" if [ ! "$filev3" = "" ]; then - echo "add \"$f\"" >> "$filev3" + printf 'add "%s"\n' "${f}" >> "${filev3}" fi fi } @@ -99,23 +97,21 @@ make_add_instruction() { check_for_add_if_not_update() { add_if_not_file_chk="$1" - if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ - `basename $add_if_not_file_chk` = "update-settings.ini" ]; then - ## "true" *giggle* + if [ "$(basename "$add_if_not_file_chk")" = "channel-prefs.js" ] || \ + [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ] + then return 0; fi - ## 'false'... because this is bash. Oh yay! return 1; } check_for_add_to_manifestv2() { add_if_not_file_chk="$1" - if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then - ## "true" *giggle* + if [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ] + then return 0; fi - ## 'false'... because this is bash. Oh yay! return 1; } @@ -124,7 +120,7 @@ make_add_if_not_instruction() { filev3="$2" notice " add-if-not \"$f\" \"$f\"" - echo "add-if-not \"$f\" \"$f\"" >> "$filev3" + printf 'add-if-not "%s" "%s"\n' "${f}" "${f}" >> "${filev3}" } make_patch_instruction() { @@ -132,18 +128,18 @@ make_patch_instruction() { filev2="$2" filev3="$3" - is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') - if [ $is_extension = "1" ]; then + if [[ "$f" =~ distribution/extensions/.*/ ]] + then # Use the subdirectory of the extensions folder as the file to test # before performing this add instruction. testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" - echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev2" - echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3" + printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev2}" + printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev3}" else notice " patch \"$f.patch\" \"$f\"" - echo "patch \"$f.patch\" \"$f\"" >> "$filev2" - echo "patch \"$f.patch\" \"$f\"" >> "$filev3" + printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev2}" + printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev3}" fi } @@ -152,50 +148,55 @@ append_remove_instructions() { filev2="$2" filev3="$3" - if [ -f "$dir/removed-files" ]; then + if [ -f "$dir/removed-files" ] + then listfile="$dir/removed-files" - elif [ -f "$dir/Contents/Resources/removed-files" ]; then + elif [ -f "$dir/Contents/Resources/removed-files" ] + then listfile="$dir/Contents/Resources/removed-files" fi - if [ -n "$listfile" ]; then - # Map spaces to pipes so that we correctly handle filenames with spaces. - files=($(cat "$listfile" | tr " " "|" | sort -r)) - num_files=${#files[*]} - for ((i=0; $i<$num_files; i=$i+1)); do - # Map pipes back to whitespace and remove carriage returns - f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') - # Trim whitespace - f=$(echo $f) - # Exclude blank lines. - if [ -n "$f" ]; then - # Exclude comments - if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then - if [ $(echo "$f" | grep -c '\/$') = 1 ]; then - notice " rmdir \"$f\"" - echo "rmdir \"$f\"" >> "$filev2" - echo "rmdir \"$f\"" >> "$filev3" - elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then - # Remove the * - f=$(echo "$f" | sed -e 's:\*$::') - notice " rmrfdir \"$f\"" - echo "rmrfdir \"$f\"" >> "$filev2" - echo "rmrfdir \"$f\"" >> "$filev3" - else - notice " remove \"$f\"" - echo "remove \"$f\"" >> "$filev2" - echo "remove \"$f\"" >> "$filev3" - fi - fi - fi - done + if [ -z "$listfile" ]; then + return fi + + while read -r f + do + # skip blank lines + if [ -z "$f" ] + then + continue + fi + # skip comments + if [[ "$f" =~ ^\#.* ]] + then + continue + fi + + if [[ "$f" =~ .*/$ ]] + then + notice " rmdir \"$f\"" + printf 'rmdir "%s"\n' "${f}" >> "${filev2}" + printf 'rmdir "%s"\n' "${f}" >> "${filev3}" + elif [[ "$f" =~ .*/\*$ ]] + then + # Remove the * + f=${f%\*} + notice " rmrfdir \"$f\"" + printf 'rmrfdir "%s"\n' "${f}" >> "${filev2}" + printf 'rmrfdir "%s"\n' "${f}" >> "${filev3}" + else + notice " remove \"$f\"" + printf 'remove "%s"\n' "${f}" >> "${filev2}" + printf 'remove "%s"\n' "${f}" >> "${filev3}" + fi + done <"$listfile" } # List all files in the current directory, stripping leading "./" # Pass a variable name and it will be filled as an array. list_files() { count=0 - + tmpfile="$(mktemp)" find . -type f \ ! -name "update.manifest" \ ! -name "updatev2.manifest" \ @@ -203,26 +204,27 @@ list_files() { ! -name "temp-dirlist" \ ! -name "temp-filelist" \ | sed 's/\.\/\(.*\)/\1/' \ - | sort -r > "temp-filelist" - while read file; do + | sort -r > "${tmpfile}" + while read -r file; do eval "${1}[$count]=\"$file\"" (( count++ )) - done < "temp-filelist" - rm "temp-filelist" + done <"${tmpfile}" + rm -f "${tmpfile}" } # List all directories in the current directory, stripping leading "./" list_dirs() { count=0 + tmpfile="$(mktemp)" find . -type d \ ! -name "." \ ! -name ".." \ | sed 's/\.\/\(.*\)/\1/' \ - | sort -r > "temp-dirlist" - while read dir; do + | sort -r > "${tmpfile}" + while read -r dir; do eval "${1}[$count]=\"$dir\"" (( count++ )) - done < "temp-dirlist" - rm "temp-dirlist" + done <"${tmpfile}" + rm -f "${tmpfile}" } diff --git a/tools/update-packaging/make_full_update.sh b/tools/update-packaging/make_full_update.sh index 5ff098e01b33..4f096faa1a20 100755 --- a/tools/update-packaging/make_full_update.sh +++ b/tools/update-packaging/make_full_update.sh @@ -8,12 +8,13 @@ # Author: Darin Fisher # -. $(dirname "$0")/common.sh +# shellcheck disable=SC1090 +. "$(dirname "$0")/common.sh" # ----------------------------------------------------------------------------- print_usage() { - notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" + notice "Usage: $(basename "$0") [OPTIONS] ARCHIVE DIRECTORY" } if [ $# = 0 ]; then @@ -21,7 +22,7 @@ if [ $# = 0 ]; then exit 1 fi -if [ $1 = -h ]; then +if [ "$1" = -h ]; then print_usage notice "" notice "The contents of DIRECTORY will be stored in ARCHIVE." @@ -38,9 +39,9 @@ archive="$1" targetdir="$2" # Prevent the workdir from being inside the targetdir so it isn't included in # the update mar. -if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then +if [ "$(echo "$targetdir" | grep -c '\/$')" = 1 ]; then # Remove the / - targetdir=$(echo "$targetdir" | sed -e 's:\/$::') + targetdir=${targetdir%/} fi workdir="$targetdir.work" updatemanifestv2="$workdir/updatev2.manifest" @@ -50,7 +51,7 @@ targetfiles="updatev2.manifest updatev3.manifest" mkdir -p "$workdir" # Generate a list of all files in the target directory. -pushd "$targetdir" +pushd "$targetdir" || exit 1 if test $? -ne 0 ; then exit 1 fi @@ -62,13 +63,14 @@ if [ ! -f "precomplete" ]; then fi fi +declare -a files list_files files -popd +popd || exit 1 # Add the type of update to the beginning of the update manifests. -> "$updatemanifestv2" -> "$updatemanifestv3" +: > "$updatemanifestv2" +: > "$updatemanifestv3" notice "" notice "Adding type instruction to update manifests" notice " type complete" @@ -77,11 +79,9 @@ echo "type \"complete\"" >> "$updatemanifestv3" notice "" notice "Adding file add instructions to update manifests" -num_files=${#files[*]} - -for ((i=0; $i<$num_files; i=$i+1)); do - f="${files[$i]}" +for f in "${files[@]}" +do if check_for_add_if_not_update "$f"; then make_add_if_not_instruction "$f" "$updatemanifestv3" if check_for_add_to_manifestv2 "$f"; then diff --git a/tools/update-packaging/make_incremental_update.sh b/tools/update-packaging/make_incremental_update.sh index d44388afcaab..ce5712a6b782 100755 --- a/tools/update-packaging/make_incremental_update.sh +++ b/tools/update-packaging/make_incremental_update.sh @@ -8,12 +8,13 @@ # Author: Darin Fisher # -. $(dirname "$0")/common.sh +# shellcheck disable=SC1090 +. "$(dirname "$0")/common.sh" # ----------------------------------------------------------------------------- print_usage() { - notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" + notice "Usage: $(basename "$0") [OPTIONS] ARCHIVE FROMDIR TODIR" notice "" notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." notice "" @@ -31,48 +32,39 @@ check_for_forced_update() { local f if [ "$forced_file_chk" = "precomplete" ]; then - ## "true" *giggle* return 0; fi if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then - ## "true" *giggle* return 0; fi if [ "$forced_file_chk" = "removed-files" ]; then - ## "true" *giggle* return 0; fi if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then - ## "true" *giggle* return 0; fi if [ "$forced_file_chk" = "chrome.manifest" ]; then - ## "true" *giggle* return 0; fi if [ "$forced_file_chk" = "Contents/Resources/chrome.manifest" ]; then - ## "true" *giggle* return 0; fi if [ "${forced_file_chk##*.}" = "chk" ]; then - ## "true" *giggle* return 0; fi for f in $force_list; do #echo comparing $forced_file_chk to $f if [ "$forced_file_chk" = "$f" ]; then - ## "true" *giggle* return 0; fi done - ## 'false'... because this is bash. Oh yay! return 1; } @@ -97,7 +89,7 @@ done # ----------------------------------------------------------------------------- -let arg_start=$OPTIND-1 +arg_start=$((OPTIND-1)) shift $arg_start archive="$1" @@ -105,10 +97,10 @@ olddir="$2" newdir="$3" # Prevent the workdir from being inside the targetdir so it isn't included in # the update mar. -if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then - # Remove the / - newdir=$(echo "$newdir" | sed -e 's:\/$::') -fi + +# Remove the / +newdir="${newdir%/}" + workdir="$newdir.work" updatemanifestv2="$workdir/updatev2.manifest" updatemanifestv3="$workdir/updatev3.manifest" @@ -117,58 +109,51 @@ archivefiles="updatev2.manifest updatev3.manifest" mkdir -p "$workdir" # Generate a list of all files in the target directory. -pushd "$olddir" -if test $? -ne 0 ; then - exit 1 -fi +pushd "$olddir" || exit 1 +declare -a oldfiles list_files oldfiles +declare -a olddirs list_dirs olddirs -popd +popd || exit 1 -pushd "$newdir" -if test $? -ne 0 ; then +pushd "$newdir" || exit 1 + +if [ ! -f "precomplete" ] && [ ! -f "Contents/Resources/precomplete" ] +then + notice "precomplete file is missing!" exit 1 fi -if [ ! -f "precomplete" ]; then - if [ ! -f "Contents/Resources/precomplete" ]; then - notice "precomplete file is missing!" - exit 1 - fi -fi - -list_dirs newdirs +declare -a newfiles list_files newfiles -popd +popd || exit 1 # Add the type of update to the beginning of the update manifests. notice "" notice "Adding type instruction to update manifests" -> $updatemanifestv2 -> $updatemanifestv3 +: > "$updatemanifestv2" +: > "$updatemanifestv3" notice " type partial" -echo "type \"partial\"" >> $updatemanifestv2 -echo "type \"partial\"" >> $updatemanifestv3 +echo 'type "partial"' >> "$updatemanifestv2" +echo 'type "partial"' >> "$updatemanifestv3" notice "" notice "Adding file patch and add instructions to update manifests" -num_oldfiles=${#oldfiles[*]} -remove_array= +declare -a remove_array num_removes=0 -for ((i=0; $i<$num_oldfiles; i=$i+1)); do - f="${oldfiles[$i]}" - +for f in "${oldfiles[@]}" +do # If this file exists in the new directory as well, then check if it differs. if [ -f "$newdir/$f" ]; then if check_for_add_if_not_update "$f"; then # The full workdir may not exist yet, so create it if necessary. - mkdir -p `dirname "$workdir/$f"` + mkdir -p "$(dirname "$workdir/$f")" if [[ -n $MAR_OLD_FORMAT ]]; then $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" else @@ -182,7 +167,7 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do if check_for_forced_update "$requested_forced_updates" "$f"; then # The full workdir may not exist yet, so create it if necessary. - mkdir -p `dirname "$workdir/$f"` + mkdir -p "$(dirname "$workdir/$f")" if [[ -n $MAR_OLD_FORMAT ]]; then $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" else @@ -212,51 +197,41 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do # # Note: patches are bzipped or xz stashed in funsize to gain more speed - # if service is not enabled then default to old behavior - if [ -z "$MBSDIFF_HOOK" ]; then + if [[ -n $MAR_OLD_FORMAT ]] + then + COMPRESS="$BZIP2 -z9" + EXT=".bz2" + STDOUT="-c" + else + COMPRESS="$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force" + EXT=".xz" + STDOUT="--stdout" + fi + + if [ -z "$MBSDIFF_HOOK" ] + then $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" - if [[ -n $MAR_OLD_FORMAT ]]; then - $BZIP2 -z9 "$workdir/$f.patch" - else - $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" - fi + $COMPRESS "$workdir/$f.patch" else - # if service enabled then check patch existence for retrieval - if [[ -n $MAR_OLD_FORMAT ]]; then - if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then - notice "file \"$f\" found in funsize, diffing skipped" - else - # if not found already - compute it and cache it for future use - $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" - $BZIP2 -z9 "$workdir/$f.patch" - $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2" - fi + if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch${EXT}" + then + notice "file \"$f\" found in funsize, diffing skipped" else - if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then - notice "file \"$f\" found in funsize, diffing skipped" - else - # if not found already - compute it and cache it for future use - $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" - $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" - $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz" - fi + # if not found already - compute it and cache it for future use + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $COMPRESS "$workdir/$f.patch" + $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch${EXT}" fi fi - if [[ -n $MAR_OLD_FORMAT ]]; then - $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" - else - $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" - fi + $COMPRESS $STDOUT "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" - if [[ -n $MAR_OLD_FORMAT ]]; then - patchfile="$workdir/$f.patch.bz2" - else - patchfile="$workdir/$f.patch.xz" - fi + patchfile="$workdir/$f.patch${EXT}" + patchsize=$(get_file_size "$patchfile") fullsize=$(get_file_size "$workdir/$f") - if [ $patchsize -lt $fullsize ]; then + if [ "$patchsize" -lt "$fullsize" ]; then make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" mv -f "$patchfile" "$workdir/$f.patch" rm -f "$workdir/$f" @@ -278,18 +253,16 @@ done # Newly added files notice "" notice "Adding file add instructions to update manifests" -num_newfiles=${#newfiles[*]} -for ((i=0; $i<$num_newfiles; i=$i+1)); do - f="${newfiles[$i]}" - - # If we've already tested this file, then skip it - for ((j=0; $j<$num_oldfiles; j=$j+1)); do - if [ "$f" = "${oldfiles[j]}" ]; then +for f in "${newfiles[@]}" +do + for j in "${oldfiles[@]}" + do + if [ "$f" = "$j" ] + then continue 2 fi done - dir=$(dirname "$workdir/$f") mkdir -p "$dir" @@ -312,11 +285,11 @@ done notice "" notice "Adding file remove instructions to update manifests" -for ((i=0; $i<$num_removes; i=$i+1)); do - f="${remove_array[$i]}" +for f in "${remove_array[@]}" +do notice " remove \"$f\"" - echo "remove \"$f\"" >> $updatemanifestv2 - echo "remove \"$f\"" >> $updatemanifestv3 + echo "remove \"$f\"" >> "$updatemanifestv2" + echo "remove \"$f\"" >> "$updatemanifestv3" done # Add remove instructions for any dead files. @@ -326,15 +299,14 @@ append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3" notice "" notice "Adding directory remove instructions for directories that no longer exist" -num_olddirs=${#olddirs[*]} -for ((i=0; $i<$num_olddirs; i=$i+1)); do - f="${olddirs[$i]}" +for f in "${olddirs[@]}" +do # If this dir doesn't exist in the new directory remove it. if [ ! -d "$newdir/$f" ]; then notice " rmdir $f/" - echo "rmdir \"$f/\"" >> $updatemanifestv2 - echo "rmdir \"$f/\"" >> $updatemanifestv3 + echo "rmdir \"$f/\"" >> "$updatemanifestv2" + echo "rmdir \"$f/\"" >> "$updatemanifestv3" fi done diff --git a/tools/update-packaging/test/buildrefmars.sh b/tools/update-packaging/test/buildrefmars.sh index fd2d5384d777..a20f888337c0 100755 --- a/tools/update-packaging/test/buildrefmars.sh +++ b/tools/update-packaging/test/buildrefmars.sh @@ -8,8 +8,8 @@ if [ -f "ref-mac.mar" ]; then rm "ref-mac.mar" fi - ../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to - ../make_incremental_update.sh ref-mac.mar `pwd`/from-mac `pwd`/to-mac + ../make_incremental_update.sh ref.mar "$(pwd)/from" "$(pwd)/to" + ../make_incremental_update.sh ref-mac.mar "$(pwd)/from-mac" "$(pwd)/to-mac" if [ -f "product-1.0.lang.platform.complete.mar" ]; then rm "product-1.0.lang.platform.complete.mar" @@ -21,7 +21,7 @@ if [ -f "product-2.0.lang.mac.complete.mar" ]; then rm "product-2.0.lang.mac.complete.mar" fi -./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from" -./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to" -./make_full_update.sh product-1.0.lang.mac.complete.mar "`pwd`/from-mac" -./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac" +./make_full_update.sh product-1.0.lang.platform.complete.mar "$(pwd)/from" +./make_full_update.sh product-2.0.lang.platform.complete.mar "$(pwd)/to" +./make_full_update.sh product-1.0.lang.mac.complete.mar "$(pwd)/from-mac" +./make_full_update.sh product-2.0.lang.mac.complete.mar "$(pwd)/to-mac" diff --git a/tools/update-packaging/test/catmanifest.sh b/tools/update-packaging/test/catmanifest.sh index e6349d47a3b6..ad0fc6a545e3 100755 --- a/tools/update-packaging/test/catmanifest.sh +++ b/tools/update-packaging/test/catmanifest.sh @@ -6,9 +6,9 @@ workdir="/tmp/catmanifest" rm -rf "$workdir" mkdir -p "$workdir" -cp "$1" "$workdir" -cd "$workdir" -mar -x "$1" +cp "$mar" "$workdir" +cd "$workdir" || exit 1 +mar -x "$mar" mv updatev2.manifest updatev2.manifest.xz xz -d updatev2.manifest.xz cat updatev2.manifest diff --git a/tools/update-packaging/test/common.sh b/tools/update-packaging/test/common.sh index ad88da2203af..03fefa90de17 100755 --- a/tools/update-packaging/test/common.sh +++ b/tools/update-packaging/test/common.sh @@ -7,13 +7,34 @@ # Code shared by update packaging scripts. # Author: Darin Fisher # -# In here to use the local common.sh to allow the full mars to have unfiltered files # ----------------------------------------------------------------------------- # By default just assume that these tools exist on our path MAR=${MAR:-mar} -XZ=${XZ:-xz} MBSDIFF=${MBSDIFF:-mbsdiff} +if [[ -z "${MAR_OLD_FORMAT}" ]]; then + while read -r XZ + do + if ${XZ} --version > /dev/null 2>&1 + then + break + fi + done << EOM +${XZ:-xz} +$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe +$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe +EOM + + if [ -z "${XZ}" ] + then + echo "xz was not found on this system!" + echo "exiting" + exit 1 + fi +else + MAR_OLD_FORMAT=1 + BZIP2=${BZIP2:-bzip2} +fi # ----------------------------------------------------------------------------- # Helper routines @@ -23,8 +44,7 @@ notice() { } get_file_size() { - info=($(ls -ln "$1")) - echo ${info[4]} + stat -f"%z" "$1" } copy_perm() { @@ -49,27 +69,27 @@ make_add_instruction() { filev3="$3" # Used to log to the console - if [ $4 ]; then + if [ "$4" ]; then forced=" (forced)" else forced= fi - is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') - if [ $is_extension = "1" ]; then + if [[ "$f" =~ distribution/extensions/.*/ ]] + then # Use the subdirectory of the extensions folder as the file to test # before performing this add instruction. testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') notice " add-if \"$testdir\" \"$f\"" - echo "add-if \"$testdir\" \"$f\"" >> $filev2 - if [ ! $filev3 = "" ]; then - echo "add-if \"$testdir\" \"$f\"" >> $filev3 + printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev2}" + if [ -n "${filev3}" ]; then + printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev3}" fi else notice " add \"$f\"$forced" - echo "add \"$f\"" >> $filev2 - if [ ! $filev3 = "" ]; then - echo "add \"$f\"" >> $filev3 + printf 'add "%s"\n' "${f}" >> "${filev2}" + if [ ! "$filev3" = "" ]; then + printf 'add "%s"\n' "${f}" >> "${filev3}" fi fi } @@ -77,23 +97,21 @@ make_add_instruction() { check_for_add_if_not_update() { add_if_not_file_chk="$1" - if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ - `basename $add_if_not_file_chk` = "update-settings.ini" ]; then - ## "true" *giggle* + if [ "$(basename "$add_if_not_file_chk")" = "channel-prefs.js" ] || \ + [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ] + then return 0; fi - ## 'false'... because this is bash. Oh yay! return 1; } check_for_add_to_manifestv2() { add_if_not_file_chk="$1" - if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then - ## "true" *giggle* + if [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ] + then return 0; fi - ## 'false'... because this is bash. Oh yay! return 1; } @@ -102,7 +120,7 @@ make_add_if_not_instruction() { filev3="$2" notice " add-if-not \"$f\" \"$f\"" - echo "add-if-not \"$f\" \"$f\"" >> $filev3 + printf 'add-if-not "%s" "%s"\n' "${f}" "${f}" >> "${filev3}" } make_patch_instruction() { @@ -110,18 +128,18 @@ make_patch_instruction() { filev2="$2" filev3="$3" - is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') - if [ $is_extension = "1" ]; then + if [[ "$f" =~ distribution/extensions/.*/ ]] + then # Use the subdirectory of the extensions folder as the file to test # before performing this add instruction. testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" - echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 - echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3 + printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev2}" + printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev3}" else notice " patch \"$f.patch\" \"$f\"" - echo "patch \"$f.patch\" \"$f\"" >> $filev2 - echo "patch \"$f.patch\" \"$f\"" >> $filev3 + printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev2}" + printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev3}" fi } @@ -130,73 +148,83 @@ append_remove_instructions() { filev2="$2" filev3="$3" - if [ -f "$dir/removed-files" ]; then + if [ -f "$dir/removed-files" ] + then listfile="$dir/removed-files" - elif [ -f "$dir/Contents/Resources/removed-files" ]; then + elif [ -f "$dir/Contents/Resources/removed-files" ] + then listfile="$dir/Contents/Resources/removed-files" fi - if [ -n "$listfile" ]; then - # Map spaces to pipes so that we correctly handle filenames with spaces. - files=($(cat "$listfile" | tr " " "|" | sort -r)) - num_files=${#files[*]} - for ((i=0; $i<$num_files; i=$i+1)); do - # Map pipes back to whitespace and remove carriage returns - f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') - # Trim whitespace - f=$(echo $f) - # Exclude blank lines. - if [ -n "$f" ]; then - # Exclude comments - if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then - if [ $(echo "$f" | grep -c '\/$') = 1 ]; then - notice " rmdir \"$f\"" - echo "rmdir \"$f\"" >> $filev2 - echo "rmdir \"$f\"" >> $filev3 - elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then - # Remove the * - f=$(echo "$f" | sed -e 's:\*$::') - notice " rmrfdir \"$f\"" - echo "rmrfdir \"$f\"" >> $filev2 - echo "rmrfdir \"$f\"" >> $filev3 - else - notice " remove \"$f\"" - echo "remove \"$f\"" >> $filev2 - echo "remove \"$f\"" >> $filev3 - fi - fi - fi - done + if [ -z "$listfile" ]; then + return fi + + while read -r f + do + # skip blank lines + if [ -z "$f" ] + then + continue + fi + # skip comments + if [[ "$f" =~ ^\#.* ]] + then + continue + fi + + if [[ "$f" =~ .*/$ ]] + then + notice " rmdir \"$f\"" + printf 'rmdir "%s"\n' "${f}" >> "${filev2}" + printf 'rmdir "%s"\n' "${f}" >> "${filev3}" + elif [[ "$f" =~ .*/\*$ ]] + then + # Remove the * + f=${f%\*} + notice " rmrfdir \"$f\"" + printf 'rmrfdir "%s"\n' "${f}" >> "${filev2}" + printf 'rmrfdir "%s"\n' "${f}" >> "${filev3}" + else + notice " remove \"$f\"" + printf 'remove "%s"\n' "${f}" >> "${filev2}" + printf 'remove "%s"\n' "${f}" >> "${filev3}" + fi + done <"$listfile" } # List all files in the current directory, stripping leading "./" # Pass a variable name and it will be filled as an array. list_files() { count=0 - - # Removed the exclusion cases here to allow for generation of testing mars + tmpfile="$(mktemp)" find . -type f \ + ! -name "update.manifest" \ + ! -name "updatev2.manifest" \ + ! -name "updatev3.manifest" \ + ! -name "temp-dirlist" \ + ! -name "temp-filelist" \ | sed 's/\.\/\(.*\)/\1/' \ - | sort -r > "$workdir/temp-filelist" - while read file; do + | sort -r > "${tmpfile}" + while read -r file; do eval "${1}[$count]=\"$file\"" (( count++ )) - done < "$workdir/temp-filelist" - rm "$workdir/temp-filelist" + done <"${tmpfile}" + rm -f "${tmpfile}" } # List all directories in the current directory, stripping leading "./" list_dirs() { count=0 + tmpfile="$(mktemp)" find . -type d \ ! -name "." \ ! -name ".." \ | sed 's/\.\/\(.*\)/\1/' \ - | sort -r > "$workdir/temp-dirlist" - while read dir; do + | sort -r > "${tmpfile}" + while read -r dir; do eval "${1}[$count]=\"$dir\"" (( count++ )) - done < "$workdir/temp-dirlist" - rm "$workdir/temp-dirlist" + done <"${tmpfile}" + rm -f "${tmpfile}" } diff --git a/tools/update-packaging/test/diffmar.sh b/tools/update-packaging/test/diffmar.sh index f1dfc7fbdd43..6dbaf0fad89d 100755 --- a/tools/update-packaging/test/diffmar.sh +++ b/tools/update-packaging/test/diffmar.sh @@ -12,10 +12,10 @@ todir="$workdir/1" # the fromdir and todir due to them being extracted synchronously so use # time-style and exclude seconds from the creation time. lsargs="-algR" -unamestr=`uname` +unamestr=$(uname) if [ ! "$unamestr" = 'Darwin' ]; then - unamestr=`uname -o` - if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then + unamestr=$(uname -o) + if [ "$unamestr" = 'Msys' ] || [ "$unamestr" = "Cygwin" ]; then lsargs="-algR --time-style=+%Y-%m-%d-%H:%M" fi fi @@ -27,23 +27,23 @@ mkdir -p "$todir" cp "$1" "$fromdir" cp "$2" "$todir" -cd "$fromdir" -mar -x "$1" -rm "$1" +cd "$fromdir" || exit 1 +mar -x "$marA" +rm "$marA" mv updatev2.manifest updatev2.manifest.xz xz -d updatev2.manifest.xz mv updatev3.manifest updatev3.manifest.xz xz -d updatev3.manifest.xz -ls $lsargs > files.txt +ls "$lsargs" > files.txt -cd "$todir" -mar -x "$2" -rm "$2" +cd "$todir" || exit 1 +mar -x "$marB" +rm "$marB" mv updatev2.manifest updatev2.manifest.xz xz -d updatev2.manifest.xz mv updatev3.manifest updatev3.manifest.xz xz -d updatev3.manifest.xz -ls $lsargs > files.txt +ls "$lsargs" > files.txt echo "diffing $fromdir and $todir" echo "on linux shell sort and python sort return different results" diff --git a/tools/update-packaging/test/make_full_update.sh b/tools/update-packaging/test/make_full_update.sh index d6a01dcb7607..47d8420ed2c7 100755 --- a/tools/update-packaging/test/make_full_update.sh +++ b/tools/update-packaging/test/make_full_update.sh @@ -9,12 +9,13 @@ # # In here to use the local common.sh to allow the full mars to have unfiltered files -. $(dirname "$0")/common.sh +# shellcheck disable=SC1090 +. "$(dirname "$0")/common.sh" # ----------------------------------------------------------------------------- print_usage() { - notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" + notice "Usage: $(basename "$0") [OPTIONS] ARCHIVE DIRECTORY" } if [ $# = 0 ]; then @@ -22,7 +23,7 @@ if [ $# = 0 ]; then exit 1 fi -if [ $1 = -h ]; then +if [ "$1" = -h ]; then print_usage notice "" notice "The contents of DIRECTORY will be stored in ARCHIVE." @@ -39,9 +40,9 @@ archive="$1" targetdir="$2" # Prevent the workdir from being inside the targetdir so it isn't included in # the update mar. -if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then +if [ "$(echo "$targetdir" | grep -c '\/$')" = 1 ]; then # Remove the / - targetdir=$(echo "$targetdir" | sed -e 's:\/$::') + targetdir=${targetdir%/} fi workdir="$targetdir.work" updatemanifestv2="$workdir/updatev2.manifest" @@ -51,7 +52,7 @@ targetfiles="updatev2.manifest updatev3.manifest" mkdir -p "$workdir" # Generate a list of all files in the target directory. -pushd "$targetdir" +pushd "$targetdir" || exit 1 if test $? -ne 0 ; then exit 1 fi @@ -63,26 +64,25 @@ if [ ! -f "precomplete" ]; then fi fi +declare -a files list_files files -popd +popd || exit 1 # Add the type of update to the beginning of the update manifests. -> $updatemanifestv2 -> $updatemanifestv3 +: > "$updatemanifestv2" +: > "$updatemanifestv3" notice "" notice "Adding type instruction to update manifests" notice " type complete" -echo "type \"complete\"" >> $updatemanifestv2 -echo "type \"complete\"" >> $updatemanifestv3 +echo "type \"complete\"" >> "$updatemanifestv2" +echo "type \"complete\"" >> "$updatemanifestv3" notice "" notice "Adding file add instructions to update manifests" -num_files=${#files[*]} - -for ((i=0; $i<$num_files; i=$i+1)); do - f="${files[$i]}" +for f in "${files[@]}" +do if check_for_add_if_not_update "$f"; then make_add_if_not_instruction "$f" "$updatemanifestv3" if check_for_add_to_manifestv2 "$f"; then