Bug 1439860 update-packaging linting fixes and shell style updates r=bhearsum

MozReview-Commit-ID: 2vMbmonkVwD

--HG--
extra : rebase_source : 05e49e5c3013c0dd726c93f5e48e1b81879b97a9
This commit is contained in:
Simon Fraser 2018-02-21 11:54:21 +00:00
parent 8f11c1f438
commit b2f9f6fd8b
9 changed files with 300 additions and 297 deletions

View File

@ -3,6 +3,7 @@ shellcheck:
description: Shell script linter
include:
- taskcluster/docker/
- tools/update-packaging/
exclude: []
# 1090: https://github.com/koalaman/shellcheck/wiki/SC1090
# 'Can't follow a non-constant source'

View File

@ -13,25 +13,24 @@
MAR=${MAR:-mar}
MBSDIFF=${MBSDIFF:-mbsdiff}
if [[ -z "${MAR_OLD_FORMAT}" ]]; then
XZ=${XZ:-xz}
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
# If $XZ is not set and not found on the path then this is probably
# running on a windows buildbot. Some of the Windows build systems have
# xz.exe in topsrcdir/xz/. Look in the places this would be in both a
# mozilla-central and comm-central build.
XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe"
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe"
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
while read -r XZ
do
if ${XZ} --version > /dev/null 2>&1
then
break
fi
done << EOM
${XZ:-xz}
$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe
$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe
EOM
if [ -z "${XZ}" ]
then
echo "xz was not found on this system!"
echo "exiting"
exit 1
fi
fi
fi
else
MAR_OLD_FORMAT=1
BZIP2=${BZIP2:-bzip2}
@ -45,8 +44,7 @@ notice() {
}
get_file_size() {
info=($(ls -ln "$1"))
echo ${info[4]}
stat -f"%z" "$1"
}
copy_perm() {
@ -71,27 +69,27 @@ make_add_instruction() {
filev3="$3"
# Used to log to the console
if [ $4 ]; then
if [ "$4" ]; then
forced=" (forced)"
else
forced=
fi
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
if [[ "$f" =~ distribution/extensions/.*/ ]]
then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " add-if \"$testdir\" \"$f\""
echo "add-if \"$testdir\" \"$f\"" >> "$filev2"
if [ ! $filev3 = "" ]; then
echo "add-if \"$testdir\" \"$f\"" >> "$filev3"
printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev2}"
if [ -n "${filev3}" ]; then
printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev3}"
fi
else
notice " add \"$f\"$forced"
echo "add \"$f\"" >> "$filev2"
printf 'add "%s"\n' "${f}" >> "${filev2}"
if [ ! "$filev3" = "" ]; then
echo "add \"$f\"" >> "$filev3"
printf 'add "%s"\n' "${f}" >> "${filev3}"
fi
fi
}
@ -99,23 +97,21 @@ make_add_instruction() {
check_for_add_if_not_update() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
`basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
if [ "$(basename "$add_if_not_file_chk")" = "channel-prefs.js" ] || \
[ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]
then
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
check_for_add_to_manifestv2() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
if [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]
then
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
@ -124,7 +120,7 @@ make_add_if_not_instruction() {
filev3="$2"
notice " add-if-not \"$f\" \"$f\""
echo "add-if-not \"$f\" \"$f\"" >> "$filev3"
printf 'add-if-not "%s" "%s"\n' "${f}" "${f}" >> "${filev3}"
}
make_patch_instruction() {
@ -132,18 +128,18 @@ make_patch_instruction() {
filev2="$2"
filev3="$3"
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
if [[ "$f" =~ distribution/extensions/.*/ ]]
then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev2"
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3"
printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev2}"
printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev3}"
else
notice " patch \"$f.patch\" \"$f\""
echo "patch \"$f.patch\" \"$f\"" >> "$filev2"
echo "patch \"$f.patch\" \"$f\"" >> "$filev3"
printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev2}"
printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev3}"
fi
}
@ -152,50 +148,55 @@ append_remove_instructions() {
filev2="$2"
filev3="$3"
if [ -f "$dir/removed-files" ]; then
if [ -f "$dir/removed-files" ]
then
listfile="$dir/removed-files"
elif [ -f "$dir/Contents/Resources/removed-files" ]; then
elif [ -f "$dir/Contents/Resources/removed-files" ]
then
listfile="$dir/Contents/Resources/removed-files"
fi
if [ -n "$listfile" ]; then
# Map spaces to pipes so that we correctly handle filenames with spaces.
files=($(cat "$listfile" | tr " " "|" | sort -r))
num_files=${#files[*]}
for ((i=0; $i<$num_files; i=$i+1)); do
# Map pipes back to whitespace and remove carriage returns
f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
# Trim whitespace
f=$(echo $f)
# Exclude blank lines.
if [ -n "$f" ]; then
# Exclude comments
if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
if [ -z "$listfile" ]; then
return
fi
while read -r f
do
# skip blank lines
if [ -z "$f" ]
then
continue
fi
# skip comments
if [[ "$f" =~ ^\#.* ]]
then
continue
fi
if [[ "$f" =~ .*/$ ]]
then
notice " rmdir \"$f\""
echo "rmdir \"$f\"" >> "$filev2"
echo "rmdir \"$f\"" >> "$filev3"
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
printf 'rmdir "%s"\n' "${f}" >> "${filev2}"
printf 'rmdir "%s"\n' "${f}" >> "${filev3}"
elif [[ "$f" =~ .*/\*$ ]]
then
# Remove the *
f=$(echo "$f" | sed -e 's:\*$::')
f=${f%\*}
notice " rmrfdir \"$f\""
echo "rmrfdir \"$f\"" >> "$filev2"
echo "rmrfdir \"$f\"" >> "$filev3"
printf 'rmrfdir "%s"\n' "${f}" >> "${filev2}"
printf 'rmrfdir "%s"\n' "${f}" >> "${filev3}"
else
notice " remove \"$f\""
echo "remove \"$f\"" >> "$filev2"
echo "remove \"$f\"" >> "$filev3"
fi
fi
fi
done
printf 'remove "%s"\n' "${f}" >> "${filev2}"
printf 'remove "%s"\n' "${f}" >> "${filev3}"
fi
done <"$listfile"
}
# List all files in the current directory, stripping leading "./"
# Pass a variable name and it will be filled as an array.
list_files() {
count=0
tmpfile="$(mktemp)"
find . -type f \
! -name "update.manifest" \
! -name "updatev2.manifest" \
@ -203,26 +204,27 @@ list_files() {
! -name "temp-dirlist" \
! -name "temp-filelist" \
| sed 's/\.\/\(.*\)/\1/' \
| sort -r > "temp-filelist"
while read file; do
| sort -r > "${tmpfile}"
while read -r file; do
eval "${1}[$count]=\"$file\""
(( count++ ))
done < "temp-filelist"
rm "temp-filelist"
done <"${tmpfile}"
rm -f "${tmpfile}"
}
# List all directories in the current directory, stripping leading "./"
list_dirs() {
count=0
tmpfile="$(mktemp)"
find . -type d \
! -name "." \
! -name ".." \
| sed 's/\.\/\(.*\)/\1/' \
| sort -r > "temp-dirlist"
while read dir; do
| sort -r > "${tmpfile}"
while read -r dir; do
eval "${1}[$count]=\"$dir\""
(( count++ ))
done < "temp-dirlist"
rm "temp-dirlist"
done <"${tmpfile}"
rm -f "${tmpfile}"
}

View File

@ -8,12 +8,13 @@
# Author: Darin Fisher
#
. $(dirname "$0")/common.sh
# shellcheck disable=SC1090
. "$(dirname "$0")/common.sh"
# -----------------------------------------------------------------------------
print_usage() {
notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY"
notice "Usage: $(basename "$0") [OPTIONS] ARCHIVE DIRECTORY"
}
if [ $# = 0 ]; then
@ -21,7 +22,7 @@ if [ $# = 0 ]; then
exit 1
fi
if [ $1 = -h ]; then
if [ "$1" = -h ]; then
print_usage
notice ""
notice "The contents of DIRECTORY will be stored in ARCHIVE."
@ -38,9 +39,9 @@ archive="$1"
targetdir="$2"
# Prevent the workdir from being inside the targetdir so it isn't included in
# the update mar.
if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
if [ "$(echo "$targetdir" | grep -c '\/$')" = 1 ]; then
# Remove the /
targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
targetdir=${targetdir%/}
fi
workdir="$targetdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
@ -50,7 +51,7 @@ targetfiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
# Generate a list of all files in the target directory.
pushd "$targetdir"
pushd "$targetdir" || exit 1
if test $? -ne 0 ; then
exit 1
fi
@ -62,13 +63,14 @@ if [ ! -f "precomplete" ]; then
fi
fi
declare -a files
list_files files
popd
popd || exit 1
# Add the type of update to the beginning of the update manifests.
> "$updatemanifestv2"
> "$updatemanifestv3"
: > "$updatemanifestv2"
: > "$updatemanifestv3"
notice ""
notice "Adding type instruction to update manifests"
notice " type complete"
@ -77,11 +79,9 @@ echo "type \"complete\"" >> "$updatemanifestv3"
notice ""
notice "Adding file add instructions to update manifests"
num_files=${#files[*]}
for ((i=0; $i<$num_files; i=$i+1)); do
f="${files[$i]}"
for f in "${files[@]}"
do
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
if check_for_add_to_manifestv2 "$f"; then

View File

@ -8,12 +8,13 @@
# Author: Darin Fisher
#
. $(dirname "$0")/common.sh
# shellcheck disable=SC1090
. "$(dirname "$0")/common.sh"
# -----------------------------------------------------------------------------
print_usage() {
notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR"
notice "Usage: $(basename "$0") [OPTIONS] ARCHIVE FROMDIR TODIR"
notice ""
notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE."
notice ""
@ -31,48 +32,39 @@ check_for_forced_update() {
local f
if [ "$forced_file_chk" = "precomplete" ]; then
## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then
## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "removed-files" ]; then
## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then
## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "chrome.manifest" ]; then
## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "Contents/Resources/chrome.manifest" ]; then
## "true" *giggle*
return 0;
fi
if [ "${forced_file_chk##*.}" = "chk" ]; then
## "true" *giggle*
return 0;
fi
for f in $force_list; do
#echo comparing $forced_file_chk to $f
if [ "$forced_file_chk" = "$f" ]; then
## "true" *giggle*
return 0;
fi
done
## 'false'... because this is bash. Oh yay!
return 1;
}
@ -97,7 +89,7 @@ done
# -----------------------------------------------------------------------------
let arg_start=$OPTIND-1
arg_start=$((OPTIND-1))
shift $arg_start
archive="$1"
@ -105,10 +97,10 @@ olddir="$2"
newdir="$3"
# Prevent the workdir from being inside the targetdir so it isn't included in
# the update mar.
if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
# Remove the /
newdir=$(echo "$newdir" | sed -e 's:\/$::')
fi
newdir="${newdir%/}"
workdir="$newdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest"
@ -117,58 +109,51 @@ archivefiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
# Generate a list of all files in the target directory.
pushd "$olddir"
if test $? -ne 0 ; then
exit 1
fi
pushd "$olddir" || exit 1
declare -a oldfiles
list_files oldfiles
declare -a olddirs
list_dirs olddirs
popd
popd || exit 1
pushd "$newdir"
if test $? -ne 0 ; then
exit 1
fi
pushd "$newdir" || exit 1
if [ ! -f "precomplete" ]; then
if [ ! -f "Contents/Resources/precomplete" ]; then
if [ ! -f "precomplete" ] && [ ! -f "Contents/Resources/precomplete" ]
then
notice "precomplete file is missing!"
exit 1
fi
fi
list_dirs newdirs
declare -a newfiles
list_files newfiles
popd
popd || exit 1
# Add the type of update to the beginning of the update manifests.
notice ""
notice "Adding type instruction to update manifests"
> $updatemanifestv2
> $updatemanifestv3
: > "$updatemanifestv2"
: > "$updatemanifestv3"
notice " type partial"
echo "type \"partial\"" >> $updatemanifestv2
echo "type \"partial\"" >> $updatemanifestv3
echo 'type "partial"' >> "$updatemanifestv2"
echo 'type "partial"' >> "$updatemanifestv3"
notice ""
notice "Adding file patch and add instructions to update manifests"
num_oldfiles=${#oldfiles[*]}
remove_array=
declare -a remove_array
num_removes=0
for ((i=0; $i<$num_oldfiles; i=$i+1)); do
f="${oldfiles[$i]}"
for f in "${oldfiles[@]}"
do
# If this file exists in the new directory as well, then check if it differs.
if [ -f "$newdir/$f" ]; then
if check_for_add_if_not_update "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
mkdir -p "$(dirname "$workdir/$f")"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
@ -182,7 +167,7 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
if check_for_forced_update "$requested_forced_updates" "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
mkdir -p "$(dirname "$workdir/$f")"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
@ -212,51 +197,41 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
#
# Note: patches are bzipped or xz stashed in funsize to gain more speed
# if service is not enabled then default to old behavior
if [ -z "$MBSDIFF_HOOK" ]; then
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -z9 "$workdir/$f.patch"
if [[ -n $MAR_OLD_FORMAT ]]
then
COMPRESS="$BZIP2 -z9"
EXT=".bz2"
STDOUT="-c"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
COMPRESS="$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force"
EXT=".xz"
STDOUT="--stdout"
fi
if [ -z "$MBSDIFF_HOOK" ]
then
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$COMPRESS "$workdir/$f.patch"
else
# if service enabled then check patch existence for retrieval
if [[ -n $MAR_OLD_FORMAT ]]; then
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch${EXT}"
then
notice "file \"$f\" found in funsize, diffing skipped"
else
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
fi
else
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
notice "file \"$f\" found in funsize, diffing skipped"
else
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
$COMPRESS "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch${EXT}"
fi
fi
fi
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
$COMPRESS $STDOUT "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
if [[ -n $MAR_OLD_FORMAT ]]; then
patchfile="$workdir/$f.patch.bz2"
else
patchfile="$workdir/$f.patch.xz"
fi
patchfile="$workdir/$f.patch${EXT}"
patchsize=$(get_file_size "$patchfile")
fullsize=$(get_file_size "$workdir/$f")
if [ $patchsize -lt $fullsize ]; then
if [ "$patchsize" -lt "$fullsize" ]; then
make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
mv -f "$patchfile" "$workdir/$f.patch"
rm -f "$workdir/$f"
@ -278,18 +253,16 @@ done
# Newly added files
notice ""
notice "Adding file add instructions to update manifests"
num_newfiles=${#newfiles[*]}
for ((i=0; $i<$num_newfiles; i=$i+1)); do
f="${newfiles[$i]}"
# If we've already tested this file, then skip it
for ((j=0; $j<$num_oldfiles; j=$j+1)); do
if [ "$f" = "${oldfiles[j]}" ]; then
for f in "${newfiles[@]}"
do
for j in "${oldfiles[@]}"
do
if [ "$f" = "$j" ]
then
continue 2
fi
done
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
@ -312,11 +285,11 @@ done
notice ""
notice "Adding file remove instructions to update manifests"
for ((i=0; $i<$num_removes; i=$i+1)); do
f="${remove_array[$i]}"
for f in "${remove_array[@]}"
do
notice " remove \"$f\""
echo "remove \"$f\"" >> $updatemanifestv2
echo "remove \"$f\"" >> $updatemanifestv3
echo "remove \"$f\"" >> "$updatemanifestv2"
echo "remove \"$f\"" >> "$updatemanifestv3"
done
# Add remove instructions for any dead files.
@ -326,15 +299,14 @@ append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
notice ""
notice "Adding directory remove instructions for directories that no longer exist"
num_olddirs=${#olddirs[*]}
for ((i=0; $i<$num_olddirs; i=$i+1)); do
f="${olddirs[$i]}"
for f in "${olddirs[@]}"
do
# If this dir doesn't exist in the new directory remove it.
if [ ! -d "$newdir/$f" ]; then
notice " rmdir $f/"
echo "rmdir \"$f/\"" >> $updatemanifestv2
echo "rmdir \"$f/\"" >> $updatemanifestv3
echo "rmdir \"$f/\"" >> "$updatemanifestv2"
echo "rmdir \"$f/\"" >> "$updatemanifestv3"
fi
done

View File

@ -8,8 +8,8 @@ if [ -f "ref-mac.mar" ]; then
rm "ref-mac.mar"
fi
../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to
../make_incremental_update.sh ref-mac.mar `pwd`/from-mac `pwd`/to-mac
../make_incremental_update.sh ref.mar "$(pwd)/from" "$(pwd)/to"
../make_incremental_update.sh ref-mac.mar "$(pwd)/from-mac" "$(pwd)/to-mac"
if [ -f "product-1.0.lang.platform.complete.mar" ]; then
rm "product-1.0.lang.platform.complete.mar"
@ -21,7 +21,7 @@ if [ -f "product-2.0.lang.mac.complete.mar" ]; then
rm "product-2.0.lang.mac.complete.mar"
fi
./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from"
./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to"
./make_full_update.sh product-1.0.lang.mac.complete.mar "`pwd`/from-mac"
./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac"
./make_full_update.sh product-1.0.lang.platform.complete.mar "$(pwd)/from"
./make_full_update.sh product-2.0.lang.platform.complete.mar "$(pwd)/to"
./make_full_update.sh product-1.0.lang.mac.complete.mar "$(pwd)/from-mac"
./make_full_update.sh product-2.0.lang.mac.complete.mar "$(pwd)/to-mac"

View File

@ -6,9 +6,9 @@ workdir="/tmp/catmanifest"
rm -rf "$workdir"
mkdir -p "$workdir"
cp "$1" "$workdir"
cd "$workdir"
mar -x "$1"
cp "$mar" "$workdir"
cd "$workdir" || exit 1
mar -x "$mar"
mv updatev2.manifest updatev2.manifest.xz
xz -d updatev2.manifest.xz
cat updatev2.manifest

View File

@ -7,13 +7,34 @@
# Code shared by update packaging scripts.
# Author: Darin Fisher
#
# In here to use the local common.sh to allow the full mars to have unfiltered files
# -----------------------------------------------------------------------------
# By default just assume that these tools exist on our path
MAR=${MAR:-mar}
XZ=${XZ:-xz}
MBSDIFF=${MBSDIFF:-mbsdiff}
if [[ -z "${MAR_OLD_FORMAT}" ]]; then
while read -r XZ
do
if ${XZ} --version > /dev/null 2>&1
then
break
fi
done << EOM
${XZ:-xz}
$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe
$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe
EOM
if [ -z "${XZ}" ]
then
echo "xz was not found on this system!"
echo "exiting"
exit 1
fi
else
MAR_OLD_FORMAT=1
BZIP2=${BZIP2:-bzip2}
fi
# -----------------------------------------------------------------------------
# Helper routines
@ -23,8 +44,7 @@ notice() {
}
get_file_size() {
info=($(ls -ln "$1"))
echo ${info[4]}
stat -f"%z" "$1"
}
copy_perm() {
@ -49,27 +69,27 @@ make_add_instruction() {
filev3="$3"
# Used to log to the console
if [ $4 ]; then
if [ "$4" ]; then
forced=" (forced)"
else
forced=
fi
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
if [[ "$f" =~ distribution/extensions/.*/ ]]
then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " add-if \"$testdir\" \"$f\""
echo "add-if \"$testdir\" \"$f\"" >> $filev2
if [ ! $filev3 = "" ]; then
echo "add-if \"$testdir\" \"$f\"" >> $filev3
printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev2}"
if [ -n "${filev3}" ]; then
printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev3}"
fi
else
notice " add \"$f\"$forced"
echo "add \"$f\"" >> $filev2
if [ ! $filev3 = "" ]; then
echo "add \"$f\"" >> $filev3
printf 'add "%s"\n' "${f}" >> "${filev2}"
if [ ! "$filev3" = "" ]; then
printf 'add "%s"\n' "${f}" >> "${filev3}"
fi
fi
}
@ -77,23 +97,21 @@ make_add_instruction() {
check_for_add_if_not_update() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
`basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
if [ "$(basename "$add_if_not_file_chk")" = "channel-prefs.js" ] || \
[ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]
then
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
check_for_add_to_manifestv2() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
if [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]
then
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
@ -102,7 +120,7 @@ make_add_if_not_instruction() {
filev3="$2"
notice " add-if-not \"$f\" \"$f\""
echo "add-if-not \"$f\" \"$f\"" >> $filev3
printf 'add-if-not "%s" "%s"\n' "${f}" "${f}" >> "${filev3}"
}
make_patch_instruction() {
@ -110,18 +128,18 @@ make_patch_instruction() {
filev2="$2"
filev3="$3"
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
if [[ "$f" =~ distribution/extensions/.*/ ]]
then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev2}"
printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev3}"
else
notice " patch \"$f.patch\" \"$f\""
echo "patch \"$f.patch\" \"$f\"" >> $filev2
echo "patch \"$f.patch\" \"$f\"" >> $filev3
printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev2}"
printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev3}"
fi
}
@ -130,73 +148,83 @@ append_remove_instructions() {
filev2="$2"
filev3="$3"
if [ -f "$dir/removed-files" ]; then
if [ -f "$dir/removed-files" ]
then
listfile="$dir/removed-files"
elif [ -f "$dir/Contents/Resources/removed-files" ]; then
elif [ -f "$dir/Contents/Resources/removed-files" ]
then
listfile="$dir/Contents/Resources/removed-files"
fi
if [ -n "$listfile" ]; then
# Map spaces to pipes so that we correctly handle filenames with spaces.
files=($(cat "$listfile" | tr " " "|" | sort -r))
num_files=${#files[*]}
for ((i=0; $i<$num_files; i=$i+1)); do
# Map pipes back to whitespace and remove carriage returns
f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
# Trim whitespace
f=$(echo $f)
# Exclude blank lines.
if [ -n "$f" ]; then
# Exclude comments
if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
if [ -z "$listfile" ]; then
return
fi
while read -r f
do
# skip blank lines
if [ -z "$f" ]
then
continue
fi
# skip comments
if [[ "$f" =~ ^\#.* ]]
then
continue
fi
if [[ "$f" =~ .*/$ ]]
then
notice " rmdir \"$f\""
echo "rmdir \"$f\"" >> $filev2
echo "rmdir \"$f\"" >> $filev3
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
printf 'rmdir "%s"\n' "${f}" >> "${filev2}"
printf 'rmdir "%s"\n' "${f}" >> "${filev3}"
elif [[ "$f" =~ .*/\*$ ]]
then
# Remove the *
f=$(echo "$f" | sed -e 's:\*$::')
f=${f%\*}
notice " rmrfdir \"$f\""
echo "rmrfdir \"$f\"" >> $filev2
echo "rmrfdir \"$f\"" >> $filev3
printf 'rmrfdir "%s"\n' "${f}" >> "${filev2}"
printf 'rmrfdir "%s"\n' "${f}" >> "${filev3}"
else
notice " remove \"$f\""
echo "remove \"$f\"" >> $filev2
echo "remove \"$f\"" >> $filev3
fi
fi
fi
done
printf 'remove "%s"\n' "${f}" >> "${filev2}"
printf 'remove "%s"\n' "${f}" >> "${filev3}"
fi
done <"$listfile"
}
# List all files in the current directory, stripping leading "./"
# Pass a variable name and it will be filled as an array.
list_files() {
count=0
# Removed the exclusion cases here to allow for generation of testing mars
tmpfile="$(mktemp)"
find . -type f \
! -name "update.manifest" \
! -name "updatev2.manifest" \
! -name "updatev3.manifest" \
! -name "temp-dirlist" \
! -name "temp-filelist" \
| sed 's/\.\/\(.*\)/\1/' \
| sort -r > "$workdir/temp-filelist"
while read file; do
| sort -r > "${tmpfile}"
while read -r file; do
eval "${1}[$count]=\"$file\""
(( count++ ))
done < "$workdir/temp-filelist"
rm "$workdir/temp-filelist"
done <"${tmpfile}"
rm -f "${tmpfile}"
}
# List all directories in the current directory, stripping leading "./"
list_dirs() {
count=0
tmpfile="$(mktemp)"
find . -type d \
! -name "." \
! -name ".." \
| sed 's/\.\/\(.*\)/\1/' \
| sort -r > "$workdir/temp-dirlist"
while read dir; do
| sort -r > "${tmpfile}"
while read -r dir; do
eval "${1}[$count]=\"$dir\""
(( count++ ))
done < "$workdir/temp-dirlist"
rm "$workdir/temp-dirlist"
done <"${tmpfile}"
rm -f "${tmpfile}"
}

View File

@ -12,10 +12,10 @@ todir="$workdir/1"
# the fromdir and todir due to them being extracted synchronously so use
# time-style and exclude seconds from the creation time.
lsargs="-algR"
unamestr=`uname`
unamestr=$(uname)
if [ ! "$unamestr" = 'Darwin' ]; then
unamestr=`uname -o`
if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then
unamestr=$(uname -o)
if [ "$unamestr" = 'Msys' ] || [ "$unamestr" = "Cygwin" ]; then
lsargs="-algR --time-style=+%Y-%m-%d-%H:%M"
fi
fi
@ -27,23 +27,23 @@ mkdir -p "$todir"
cp "$1" "$fromdir"
cp "$2" "$todir"
cd "$fromdir"
mar -x "$1"
rm "$1"
cd "$fromdir" || exit 1
mar -x "$marA"
rm "$marA"
mv updatev2.manifest updatev2.manifest.xz
xz -d updatev2.manifest.xz
mv updatev3.manifest updatev3.manifest.xz
xz -d updatev3.manifest.xz
ls $lsargs > files.txt
ls "$lsargs" > files.txt
cd "$todir"
mar -x "$2"
rm "$2"
cd "$todir" || exit 1
mar -x "$marB"
rm "$marB"
mv updatev2.manifest updatev2.manifest.xz
xz -d updatev2.manifest.xz
mv updatev3.manifest updatev3.manifest.xz
xz -d updatev3.manifest.xz
ls $lsargs > files.txt
ls "$lsargs" > files.txt
echo "diffing $fromdir and $todir"
echo "on linux shell sort and python sort return different results"

View File

@ -9,12 +9,13 @@
#
# In here to use the local common.sh to allow the full mars to have unfiltered files
. $(dirname "$0")/common.sh
# shellcheck disable=SC1090
. "$(dirname "$0")/common.sh"
# -----------------------------------------------------------------------------
print_usage() {
notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY"
notice "Usage: $(basename "$0") [OPTIONS] ARCHIVE DIRECTORY"
}
if [ $# = 0 ]; then
@ -22,7 +23,7 @@ if [ $# = 0 ]; then
exit 1
fi
if [ $1 = -h ]; then
if [ "$1" = -h ]; then
print_usage
notice ""
notice "The contents of DIRECTORY will be stored in ARCHIVE."
@ -39,9 +40,9 @@ archive="$1"
targetdir="$2"
# Prevent the workdir from being inside the targetdir so it isn't included in
# the update mar.
if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
if [ "$(echo "$targetdir" | grep -c '\/$')" = 1 ]; then
# Remove the /
targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
targetdir=${targetdir%/}
fi
workdir="$targetdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
@ -51,7 +52,7 @@ targetfiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
# Generate a list of all files in the target directory.
pushd "$targetdir"
pushd "$targetdir" || exit 1
if test $? -ne 0 ; then
exit 1
fi
@ -63,26 +64,25 @@ if [ ! -f "precomplete" ]; then
fi
fi
declare -a files
list_files files
popd
popd || exit 1
# Add the type of update to the beginning of the update manifests.
> $updatemanifestv2
> $updatemanifestv3
: > "$updatemanifestv2"
: > "$updatemanifestv3"
notice ""
notice "Adding type instruction to update manifests"
notice " type complete"
echo "type \"complete\"" >> $updatemanifestv2
echo "type \"complete\"" >> $updatemanifestv3
echo "type \"complete\"" >> "$updatemanifestv2"
echo "type \"complete\"" >> "$updatemanifestv3"
notice ""
notice "Adding file add instructions to update manifests"
num_files=${#files[*]}
for ((i=0; $i<$num_files; i=$i+1)); do
f="${files[$i]}"
for f in "${files[@]}"
do
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
if check_for_add_to_manifestv2 "$f"; then