Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-08 02:14:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 900251 - Add support for add-if-not instruction added by bug 759469 to the mar generation scripts. r=nthomas

Browse Source
This commit is contained in:
Robert Strong 2014-03-05 11:42:56 -08:00
parent 376e960b5a
commit 40e187af87
73 changed files with 588 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
config/createprecomplete.py
View File

@ -20,14 +20,17 @@ def get_build_entries(root_path):
parent_dir_rel_path = root[len(root_path)+1:]
rel_path_file = os.path.join(parent_dir_rel_path, file_name)
rel_path_file = rel_path_file.replace("\\", "/")
if not (rel_path_file.endswith("channel-prefs.js")):
if not (rel_path_file.endswith("channel-prefs.js") or
rel_path_file.endswith("update-settings.ini") or
rel_path_file.find("distribution/") != -1):
rel_file_path_set.add(rel_path_file)
for dir_name in dirs:
parent_dir_rel_path = root[len(root_path)+1:]
rel_path_dir = os.path.join(parent_dir_rel_path, dir_name)
rel_path_dir = rel_path_dir.replace("\\", "/")+"/"
rel_dir_path_set.add(rel_path_dir)
if rel_path_dir.find("distribution/") == -1:
rel_dir_path_set.add(rel_path_dir)
rel_file_path_list = list(rel_file_path_set)
rel_file_path_list.sort(reverse=True)

84
tools/update-packaging/common.sh
View File

@ -39,9 +39,16 @@ copy_perm() {
make_add_instruction() {
f="$1"
filev2="$2"
# The third param will be an empty string when a file add instruction is only
# needed in the version 2 manifest. This only happens when the file has an
# add-if-not instruction in the version 3 manifest. This is due to the
# precomplete file prior to the version 3 manifest having a remove instruction
# for this file so the file is removed before applying a complete update.
filev3="$3"
# Used to log to the console
if [ $2 ]; then
if [ $4 ]; then
forced=" (forced)"
else
forced=
@ -52,32 +59,76 @@ make_add_instruction() {
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " add-if: $f$forced"
echo "add-if \"$testdir\" \"$f\""
notice " add-if \"$testdir\" \"$f\""
echo "add-if \"$testdir\" \"$f\"" >> $filev2
if [ ! $filev3 = "" ]; then
echo "add-if \"$testdir\" \"$f\"" >> $filev3
fi
else
notice " add: $f$forced"
echo "add \"$f\""
notice " add \"$f\"$forced"
echo "add \"$f\"" >> $filev2
if [ ! $filev3 = "" ]; then
echo "add \"$f\"" >> $filev3
fi
fi
}
check_for_add_if_not_update() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
`basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
check_for_add_to_manifestv2() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
make_add_if_not_instruction() {
f="$1"
filev3="$2"
notice " add-if-not \"$f\" \"$f\""
echo "add-if-not \"$f\" \"$f\"" >> $filev3
}
make_patch_instruction() {
f="$1"
filev2="$2"
filev3="$3"
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " patch-if: $f"
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
else
notice " patch: $f"
echo "patch \"$f.patch\" \"$f\""
notice " patch \"$f.patch\" \"$f\""
echo "patch \"$f.patch\" \"$f\"" >> $filev2
echo "patch \"$f.patch\" \"$f\"" >> $filev3
fi
}
append_remove_instructions() {
dir="$1"
filev2="$2"
filev3="$3"
if [ -f "$dir/removed-files" ]; then
prefix=
listfile="$dir/removed-files"
@ -112,16 +163,19 @@ append_remove_instructions() {
fi
fi
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
notice " rmdir: $fixedprefix$f"
notice " rmdir \"$fixedprefix$f\""
echo "rmdir \"$fixedprefix$f\"" >> $filev2
echo "rmdir \"$fixedprefix$f\"" >> $filev3
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
# Remove the *
f=$(echo "$f" | sed -e 's:\*$::')
notice " rmrfdir: $fixedprefix$f"
notice " rmrfdir \"$fixedprefix$f\""
echo "rmrfdir \"$fixedprefix$f\"" >> $filev2
echo "rmrfdir \"$fixedprefix$f\"" >> $filev3
else
notice " remove: $fixedprefix$f"
notice " remove \"$fixedprefix$f\""
echo "remove \"$fixedprefix$f\"" >> $filev2
echo "remove \"$fixedprefix$f\"" >> $filev3
fi
fi
fi
@ -130,16 +184,14 @@ append_remove_instructions() {
}
# List all files in the current directory, stripping leading "./"
# Skip the channel-prefs.js file as it should not be included in any
# generated MAR files (see bug 306077). Pass a variable name and it will be
# filled as an array.
# Pass a variable name and it will be filled as an array.
list_files() {
count=0
find . -type f \
! -name "channel-prefs.js" \
! -name "update.manifest" \
! -name "updatev2.manifest" \
! -name "updatev3.manifest" \
! -name "temp-dirlist" \
! -name "temp-filelist" \
| sed 's/\.\/\(.*\)/\1/' \

26
tools/update-packaging/make_full_update.sh
View File

@ -44,7 +44,8 @@ if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
fi
workdir="$targetdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
targetfiles="updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest"
targetfiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
@ -66,21 +67,30 @@ list_files files
popd
# Add the type of update to the beginning of version 2 update manifest.
# Add the type of update to the beginning of the update manifests.
> $updatemanifestv2
> $updatemanifestv3
notice ""
notice "Adding type instruction to file 'updatev2.manifest'"
notice " type: complete"
notice "Adding type instruction to update manifests"
notice " type complete"
echo "type \"complete\"" >> $updatemanifestv2
echo "type \"complete\"" >> $updatemanifestv3
notice ""
notice "Adding file add instructions to file 'updatev2.manifest'"
notice "Adding file add instructions to update manifests"
num_files=${#files[*]}
for ((i=0; $i<$num_files; i=$i+1)); do
f="${files[$i]}"
make_add_instruction "$f" >> $updatemanifestv2
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
if check_for_add_to_manifestv2 "$f"; then
make_add_instruction "$f" "$updatemanifestv2" "" 1
fi
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
dir=$(dirname "$f")
mkdir -p "$workdir/$dir"
@ -93,9 +103,10 @@ done
# Append remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
mv -f "$workdir/output.mar" "$archive"
@ -105,3 +116,4 @@ rm -fr "$workdir"
notice ""
notice "Finished"
notice ""

63
tools/update-packaging/make_incremental_update.sh
View File

@ -87,7 +87,8 @@ if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
fi
workdir="$newdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
archivefiles="updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest"
archivefiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
@ -120,15 +121,17 @@ list_files newfiles
popd
# Add the type of update to the beginning of version 2 update manifest.
# Add the type of update to the beginning of the update manifests.
notice ""
notice "Adding type instruction to file 'updatev2.manifest'"
notice "Adding type instruction to update manifests"
> $updatemanifestv2
notice " type: partial"
> $updatemanifestv3
notice " type partial"
echo "type \"partial\"" >> $updatemanifestv2
echo "type \"partial\"" >> $updatemanifestv3
notice ""
notice "Adding file patch and add instructions to file 'updatev2.manifest'"
notice "Adding file patch and add instructions to update manifests"
num_oldfiles=${#oldfiles[*]}
remove_array=
@ -151,12 +154,22 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
# If this file exists in the new directory as well, then check if it differs.
if [ -f "$newdir/$f" ]; then
if check_for_add_if_not_update "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
make_add_if_not_instruction "$f" "$updatemanifestv3"
archivefiles="$archivefiles \"$f\""
continue 1
fi
if check_for_forced_update "$requested_forced_updates" "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
make_add_instruction "$f" 1 >> $updatemanifestv2
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
archivefiles="$archivefiles \"$f\""
continue 1
fi
@ -166,6 +179,7 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
# compare the sizes. Then choose the smaller of the two to package.
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
notice "diffing \"$f\""
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch"
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
@ -175,12 +189,12 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
fullsize=$(get_file_size "$workdir/$f")
if [ $patchsize -lt $fullsize ]; then
make_patch_instruction "$f" >> $updatemanifestv2
make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
mv -f "$patchfile" "$workdir/$f.patch"
rm -f "$workdir/$f"
archivefiles="$archivefiles \"$f.patch\""
else
make_add_instruction "$f" >> $updatemanifestv2
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
rm -f "$patchfile"
archivefiles="$archivefiles \"$f\""
fi
@ -195,7 +209,7 @@ done
# Newly added files
notice ""
notice "Adding file add instructions to file 'updatev2.manifest'"
notice "Adding file add instructions to update manifests"
num_newfiles=${#newfiles[*]}
for ((i=0; $i<$num_newfiles; i=$i+1)); do
@ -220,22 +234,40 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); do
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
make_add_instruction "$f" >> "$updatemanifestv2"
if check_for_add_if_not_update "$f"; then
# Raise an exception if a new channel-prefs.js or update-settings.ini file
# is found to prevent it from being to a new location like happened in
# bug 756325.
if [ `basename $f` = "channel-prefs.js" ]; then
notice "new channel-prefs.js file found: $f"
exit 1
fi
if [ `basename $f` = "update-settings.ini" ]; then
notice "new update-settings.ini file found: $f"
exit 1
fi
make_add_if_not_instruction "$f" "$updatemanifestv3"
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
archivefiles="$archivefiles \"$f\""
done
notice ""
notice "Adding file remove instructions to file 'updatev2.manifest'"
notice "Adding file remove instructions to update manifests"
for ((i=0; $i<$num_removes; i=$i+1)); do
f="${remove_array[$i]}"
notice " remove: $f"
notice " remove \"$f\""
echo "remove \"$f\"" >> $updatemanifestv2
echo "remove \"$f\"" >> $updatemanifestv3
done
# Add remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$newdir" "$updatemanifestv2"
append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
notice ""
notice "Adding directory remove instructions for directories that no longer exist"
@ -245,12 +277,14 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
f="${olddirs[$i]}"
# If this dir doesn't exist in the new directory remove it.
if [ ! -d "$newdir/$f" ]; then
notice " rmdir: $f/"
notice " rmdir $f/"
echo "rmdir \"$f/\"" >> $updatemanifestv2
echo "rmdir \"$f/\"" >> $updatemanifestv3
fi
done
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
eval "$MAR -C \"$workdir\" -c output.mar $archivefiles"
mv -f "$workdir/output.mar" "$archive"
@ -260,3 +294,4 @@ rm -fr "$workdir"
notice ""
notice "Finished"
notice ""

177
tools/update-packaging/make_incremental_updates.py
View File

@ -21,66 +21,90 @@ class PatchInfo:
work_dir = working dir where files are stored for this patch
archive_files = list of files to include in this patch
manifestv2 = set of manifest version 2 patch instructions
manifestv3 = set of manifest version 3 patch instructions
file_exclusion_list =
files to exclude from this patch. names without slashes will be
excluded anywhere in the directory hiearchy. names with slashes
will only be excluded at that exact path
excluded anywhere in the directory hiearchy. names with slashes
will only be excluded at that exact path
"""
def __init__(self, work_dir, file_exclusion_list, path_exclusion_list):
self.work_dir=work_dir
self.archive_files=[]
self.manifestv2=[]
self.manifestv3=[]
self.file_exclusion_list=file_exclusion_list
self.path_exclusion_list=path_exclusion_list
def append_add_instruction(self, filename):
""" Appends an add instruction for this patch.
if the filename starts with distribution/extensions/ adds an add-if
instruction to test the existence of the subdirectory. This was
ported from mozilla/tools/update-packaging/common.sh's
make_add_instruction.
""" Appends an add instruction for this patch.
if filename starts with distribution/extensions/.*/ this will add an
add-if instruction that will add the file if the parent directory
of the file exists. This was ported from
mozilla/tools/update-packaging/common.sh's make_add_instruction.
"""
m = re.match("((?:|.*/)distribution/extensions)/", filename)
m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
if m:
# Directory immediately following extensions is used for the test
testdir = m.group(1)
print ' add-if "'+testdir+'" "'+filename+'"'
self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
self.manifestv3.append('add-if "'+testdir+'" "'+filename+'"')
else:
print ' add "'+filename+'"'
self.manifestv2.append('add "'+filename+'"')
self.manifestv3.append('add "'+filename+'"')
def append_add_if_not_instruction(self, filename):
""" Appends an add-if-not instruction to the version 3 manifest for this patch.
This was ported from mozilla/tools/update-packaging/common.sh's
make_add_if_not_instruction.
"""
print ' add-if-not "'+filename+'" "'+filename+'"'
self.manifestv3.append('add-if-not "'+filename+'" "'+filename+'"')
def append_patch_instruction(self, filename, patchname):
""" Appends an patch instruction for this patch.
""" Appends a patch instruction for this patch.
filename = file to patch
patchname = patchfile to apply to file
if the filename starts with distribution/extensions/ adds a
patch-if instruction to test the existence of the subdirectory.
This was ported from
if filename starts with distribution/extensions/.*/ this will add a
patch-if instruction that will patch the file if the parent
directory of the file exists. This was ported from
mozilla/tools/update-packaging/common.sh's make_patch_instruction.
"""
m = re.match("((?:|.*/)distribution/extensions)/", filename)
m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
if m:
testdir = m.group(1)
print ' patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"'
self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
self.manifestv3.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
else:
print ' patch "'+patchname+'" "'+filename+'"'
self.manifestv2.append('patch "'+patchname+'" "'+filename+'"')
self.manifestv3.append('patch "'+patchname+'" "'+filename+'"')
def append_remove_instruction(self, filename):
""" Appends an remove instruction for this patch.
""" Appends an remove instruction for this patch.
This was ported from
mozilla/tools/update-packaging/common.sh/make_remove_instruction
"""
if filename.endswith("/"):
print ' rmdir "'+filename+'"'
self.manifestv2.append('rmdir "'+filename+'"')
self.manifestv3.append('rmdir "'+filename+'"')
elif filename.endswith("/*"):
filename = filename[:-1]
print ' rmrfdir "'+filename+'"'
self.manifestv2.append('rmrfdir "'+filename+'"')
self.manifestv3.append('rmrfdir "'+filename+'"')
else:
print ' remove "'+filename+'"'
self.manifestv2.append('remove "'+filename+'"')
self.manifestv3.append('remove "'+filename+'"')
def create_manifest_files(self):
""" Createst the v2 manifest file in the root of the work_dir """
""" Create the v2 manifest file in the root of the work_dir """
manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
manifest_file = open(manifest_file_path, "wb")
manifest_file.writelines("type \"partial\"\n")
@ -91,6 +115,17 @@ class PatchInfo:
bzip_file(manifest_file_path)
self.archive_files.append('"updatev2.manifest"')
""" Create the v3 manifest file in the root of the work_dir """
manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest")
manifest_file = open(manifest_file_path, "wb")
manifest_file.writelines("type \"partial\"\n")
manifest_file.writelines(string.join(self.manifestv3, '\n'))
manifest_file.writelines("\n")
manifest_file.close()
bzip_file(manifest_file_path)
self.archive_files.append('"updatev3.manifest"')
def build_marfile_entry_hash(self, root_path):
""" Iterates through the root_path, creating a MarFileEntry for each file
and directory in that path. Excludes any filenames in the file_exclusion_list
@ -120,7 +155,7 @@ class PatchInfo:
return mar_entry_hash, filename_set, dirname_set
class MarFileEntry:
"""Represents a file inside a Mozilla Archive Format (MAR)
abs_path = abspath to the the file
@ -139,7 +174,7 @@ class MarFileEntry:
def __str__(self):
return 'Name: %s FullPath: %s' %(self.name,self.abs_path)
def calc_file_sha_digest(self, filename):
def calc_file_sha_digest(self, filename):
""" Returns sha digest of given filename"""
file_content = open(filename, 'r').read()
return sha.new(file_content).digest()
@ -169,7 +204,7 @@ def bzip_file(filename):
assumes the path is absolute"""
exec_shell_cmd('bzip2 -z9 "' + filename+'"')
os.rename(filename+".bz2",filename)
def bunzip_file(filename):
""" Bzip's the file in palce. The original file is replaced with a bunzip'd version of itself.
doesn't matter if the filename ends in .bz2 or not"""
@ -186,7 +221,7 @@ def extract_mar(filename, work_dir):
saved_path = os.getcwd()
try:
os.chdir(work_dir)
exec_shell_cmd("mar -x "+filename)
exec_shell_cmd("mar -x "+filename)
finally:
os.chdir(saved_path)
@ -194,8 +229,7 @@ def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, pa
""" Creates the partial patch file and manifest entry for the pair of files passed in
"""
if not (from_marfile_entry.sha(),to_marfile_entry.sha()) in shas:
print "diffing: " + from_marfile_entry.name
print 'diffing "'+from_marfile_entry.name+'\"'
#bunzip to/from
bunzip_file(from_marfile_entry.abs_path)
bunzip_file(to_marfile_entry.abs_path)
@ -212,48 +246,52 @@ def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, pa
bzip_file(patch_file_abs_path)
# Create bzip's full file
full_file_abs_path = os.path.join(patch_info.work_dir, to_marfile_entry.name)
full_file_abs_path = os.path.join(patch_info.work_dir, to_marfile_entry.name)
shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
bzip_file(full_file_abs_path)
## TOODO NEED TO ADD HANDLING FOR FORCED UPDATES
if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
# Patch is smaller than file. Remove the file and add patch to manifest
os.remove(full_file_abs_path)
file_in_manifest_name = from_marfile_entry.name+".patch"
file_in_manifest_abspath = patch_file_abs_path
patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
else:
else:
# File is smaller than patch. Remove the patch and add file to manifest
os.remove(patch_file_abs_path)
file_in_manifest_name = from_marfile_entry.name
file_in_manifest_abspath = full_file_abs_path
patch_info.append_add_instruction(file_in_manifest_name)
shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath)
patch_info.archive_files.append('"'+file_in_manifest_name+'"')
patch_info.archive_files.append('"'+file_in_manifest_name+'"')
else:
filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()]
# We've already calculated the patch for this pair of files.
# We've already calculated the patch for this pair of files.
if (filename.endswith(".patch")):
print "skipping diff: " + from_marfile_entry.name
# print "skipping diff: "+from_marfile_entry.name
# Patch was smaller than file - add patch instruction to manifest
file_in_manifest_name = to_marfile_entry.name+'.patch';
patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
else:
# File was smaller than file - add file to manifest
file_in_manifest_name = to_marfile_entry.name
patch_info.append_add_instruction(file_in_manifest_name)
patch_info.append_add_instruction(file_in_manifest_name)
# Copy the pre-calculated file into our new patch work aread
copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name))
patch_info.archive_files.append('"'+file_in_manifest_name+'"')
def create_add_patch_for_file(to_marfile_entry, patch_info):
def create_add_patch_for_file(to_marfile_entry, patch_info):
""" Copy the file to the working dir, add the add instruction, and add it to the list of archive files """
print "Adding New File " + to_marfile_entry.name
copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
patch_info.append_add_instruction(to_marfile_entry.name)
patch_info.archive_files.append('"'+to_marfile_entry.name+'"')
patch_info.archive_files.append('"'+to_marfile_entry.name+'"')
def create_add_if_not_patch_for_file(to_marfile_entry, patch_info):
""" Copy the file to the working dir, add the add-if-not instruction, and add it to the list of archive files """
copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
patch_info.append_add_if_not_instruction(to_marfile_entry.name)
patch_info.archive_files.append('"'+to_marfile_entry.name+'"')
def process_explicit_remove_files(dir_path, patch_info):
""" Looks for a 'removed-files' file in the dir_path. If the removed-files does not exist
@ -290,7 +328,7 @@ def process_explicit_remove_files(dir_path, patch_info):
line = line.replace("\\", "/")
patch_info.append_remove_instruction(line)
def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates):
def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates, add_if_not_list):
""" Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path"""
# Cannocolize the paths for safey
from_dir_path = os.path.abspath(from_dir_path)
@ -298,7 +336,7 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch
# Create a hashtable of the from and to directories
from_dir_hash,from_file_set,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
to_dir_hash,to_file_set,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
# Require that the precomplete file is included in the to complete update
# Require that the precomplete file is included in the complete update
if "precomplete" not in to_file_set:
raise Exception, "missing precomplete file in: "+to_dir_path
# Create a list of the forced updates
@ -311,8 +349,11 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch
for filename in patch_filenames:
from_marfile_entry = from_dir_hash[filename]
to_marfile_entry = to_dir_hash[filename]
if filename in forced_list:
print "Forcing "+ filename
if os.path.basename(filename) in add_if_not_list:
# This filename is in the add if not list, explicitly add-if-not
create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info)
elif filename in forced_list:
print 'Forcing "'+filename+'"'
# This filename is in the forced list, explicitly add
create_add_patch_for_file(to_dir_hash[filename], patch_info)
else:
@ -324,7 +365,19 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch
add_filenames = list(to_file_set - from_file_set)
add_filenames.sort(reverse=True)
for filename in add_filenames:
create_add_patch_for_file(to_dir_hash[filename], patch_info)
if os.path.basename(filename) in add_if_not_list:
# Raise an exception if a new channel-prefs.js or
# update-settings.ini file is found to prevent it from being
# to a new location like happened in bug 756325.
if os.path.basename(filename) == 'channel-prefs.js':
raise Exception, "new channel-prefs.js file found: "+filename
if os.path.basename(filename) == 'update-settings.ini':
raise Exception, "new update-settings.ini file found: "+filename
create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info)
else:
create_add_patch_for_file(to_dir_hash[filename], patch_info)
# files in from_dir not in to_dir need to be removed
remove_filenames = list(from_file_set - to_file_set)
@ -333,7 +386,7 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch
patch_info.append_remove_instruction(from_dir_hash[filename].name)
process_explicit_remove_files(to_dir_path, patch_info)
# directories in from_dir not in to_dir need to be removed
remove_dirnames = list(from_dir_set - to_dir_set)
remove_dirnames.sort(reverse=True)
@ -342,9 +395,9 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch
# Construct the Manifest files
patch_info.create_manifest_files()
# And construct the mar
mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
exec_shell_cmd(mar_cmd)
# Copy mar to final destination
@ -353,7 +406,7 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch
os.makedirs(patch_file_dir)
shutil.copy2(os.path.join(patch_info.work_dir,"output.mar"), patch_filename)
return patch_filename
def usage():
print "-h for help"
print "-f for patchlist_file"
@ -426,7 +479,7 @@ def create_partial_patches(patches):
from_buildid = get_buildid(work_dir_from, from_decoded['platform'])
from_shasum = sha.sha(open(from_filename).read()).hexdigest()
from_size = str(os.path.getsize(to_filename))
# Extract to mar into to dir
work_dir_to = os.path.join(work_dir,"to")
os.mkdir(work_dir_to)
@ -438,7 +491,7 @@ def create_partial_patches(patches):
mar_extract_time = time.time()
partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt']),forced_updates)
partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest','removed-files'],['/readme.txt']),forced_updates,['channel-prefs.js','update-settings.ini'])
partial_buildid = to_buildid
partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
partial_size = str(os.path.getsize(partial_filename))
@ -466,26 +519,26 @@ def create_partial_patches(patches):
finally:
# If we fail or get a ctrl-c during run be sure to clean up temp dir
if (work_dir_root and os.path.exists(work_dir_root)):
shutil.rmtree(work_dir_root)
shutil.rmtree(work_dir_root)
def main(argv):
def main(argv):
patchlist_file = None
try:
try:
opts, args = getopt.getopt(argv, "hf:", ["help", "patchlist_file="])
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
sys.exit()
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
sys.exit()
elif opt in ("-f", "--patchlist_file"):
patchlist_file = arg
except getopt.GetoptError:
usage()
sys.exit(2)
patchlist_file = arg
except getopt.GetoptError:
usage()
sys.exit(2)
if not patchlist_file:
usage()
sys.exit(2)
patches = []
f = open(patchlist_file, 'r')
for line in f.readlines():

3
tools/update-packaging/test/buildrefmars.sh
View File

@ -9,7 +9,7 @@ if [ -f "ref-mac.mar" ]; then
fi
../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to
../make_incremental_update.sh ref-mac.mar `pwd`/from `pwd`/to-mac
../make_incremental_update.sh ref-mac.mar `pwd`/from-mac `pwd`/to-mac
if [ -f "product-1.0.lang.platform.complete.mar" ]; then
rm "product-1.0.lang.platform.complete.mar"
@ -23,4 +23,5 @@ fi
./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from"
./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to"
./make_full_update.sh product-1.0.lang.mac.complete.mar "`pwd`/from-mac"
./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac"

82
tools/update-packaging/test/common.sh
View File

@ -40,9 +40,16 @@ copy_perm() {
make_add_instruction() {
f="$1"
filev2="$2"
# The third param will be an empty string when a file add instruction is only
# needed in the version 2 manifest. This only happens when the file has an
# add-if-not instruction in the version 3 manifest. This is due to the
# precomplete file prior to the version 3 manifest having a remove instruction
# for this file so the file is removed before applying a complete update.
filev3="$3"
# Used to log to the console
if [ $2 ]; then
if [ $4 ]; then
forced=" (forced)"
else
forced=
@ -53,32 +60,76 @@ make_add_instruction() {
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " add-if: $f$forced"
echo "add-if \"$testdir\" \"$f\""
notice " add-if \"$testdir\" \"$f\""
echo "add-if \"$testdir\" \"$f\"" >> $filev2
if [ ! $filev3 = "" ]; then
echo "add-if \"$testdir\" \"$f\"" >> $filev3
fi
else
notice " add: $f$forced"
echo "add \"$f\""
notice " add \"$f\"$forced"
echo "add \"$f\"" >> $filev2
if [ ! $filev3 = "" ]; then
echo "add \"$f\"" >> $filev3
fi
fi
}
check_for_add_if_not_update() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
`basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
check_for_add_to_manifestv2() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
## "true" *giggle*
return 0;
fi
## 'false'... because this is bash. Oh yay!
return 1;
}
make_add_if_not_instruction() {
f="$1"
filev3="$2"
notice " add-if-not \"$f\" \"$f\""
echo "add-if-not \"$f\" \"$f\"" >> $filev3
}
make_patch_instruction() {
f="$1"
filev2="$2"
filev3="$3"
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " patch-if: $f"
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
else
notice " patch: $f"
echo "patch \"$f.patch\" \"$f\""
notice " patch \"$f.patch\" \"$f\""
echo "patch \"$f.patch\" \"$f\"" >> $filev2
echo "patch \"$f.patch\" \"$f\"" >> $filev3
fi
}
append_remove_instructions() {
dir="$1"
filev2="$2"
filev3="$3"
if [ -f "$dir/removed-files" ]; then
prefix=
listfile="$dir/removed-files"
@ -113,16 +164,19 @@ append_remove_instructions() {
fi
fi
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
notice " rmdir: $fixedprefix$f"
notice " rmdir \"$fixedprefix$f\""
echo "rmdir \"$fixedprefix$f\"" >> $filev2
echo "rmdir \"$fixedprefix$f\"" >> $filev3
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
# Remove the *
f=$(echo "$f" | sed -e 's:\*$::')
notice " rmrfdir: $fixedprefix$f"
notice " rmrfdir \"$fixedprefix$f\""
echo "rmrfdir \"$fixedprefix$f\"" >> $filev2
echo "rmrfdir \"$fixedprefix$f\"" >> $filev3
else
notice " remove: $fixedprefix$f"
notice " remove \"$fixedprefix$f\""
echo "remove \"$fixedprefix$f\"" >> $filev2
echo "remove \"$fixedprefix$f\"" >> $filev3
fi
fi
fi
@ -131,9 +185,7 @@ append_remove_instructions() {
}
# List all files in the current directory, stripping leading "./"
# Skip the channel-prefs.js file as it should not be included in any
# generated MAR files (see bug 306077). Pass a variable name and it will be
# filled as an array.
# Pass a variable name and it will be filled as an array.
list_files() {
count=0

9
tools/update-packaging/test/diffmar.sh
View File

@ -3,7 +3,8 @@
marA="$1"
marB="$2"
workdir="/tmp/diffmar"
testDir="$3"
workdir="/tmp/diffmar/$testDir"
fromdir="$workdir/0"
todir="$workdir/1"
@ -31,6 +32,8 @@ mar -x "$1"
rm "$1"
mv updatev2.manifest updatev2.manifest.bz2
bzip2 -d updatev2.manifest.bz2
mv updatev3.manifest updatev3.manifest.bz2
bzip2 -d updatev3.manifest.bz2
ls $lsargs > files.txt
cd "$todir"
@ -38,9 +41,11 @@ mar -x "$2"
rm "$2"
mv updatev2.manifest updatev2.manifest.bz2
bzip2 -d updatev2.manifest.bz2
mv updatev3.manifest updatev3.manifest.bz2
bzip2 -d updatev3.manifest.bz2
ls $lsargs > files.txt
echo "diffing $fromdir and $todir"
echo "on linux shell sort and python sort return different results"
echo "which cause differences in the manifest files"
echo "which can cause differences in the manifest files"
diff -ru "$fromdir" "$todir"

5
tools/update-packaging/test/from-mac/Contents/MacOS/application.ini Normal file
View File

@ -0,0 +1,5 @@
[App]
Vendor=Mozilla
Name=MarTest
Version=1
BuildID=20120101010101

1
tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
from file

1
tools/update-packaging/test/from-mac/Contents/MacOS/distribution/extensions/diff/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
from file

1
tools/update-packaging/test/from-mac/Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
from file

1
tools/update-packaging/test/from-mac/Contents/MacOS/force.txt Normal file
View File

@ -0,0 +1 @@
file is same

2
tools/update-packaging/test/from-mac/Contents/MacOS/readme.txt Normal file
View File

@ -0,0 +1,2 @@
This from file should be ignored

1
tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt Normal file
View File

@ -0,0 +1 @@
removed

BIN
tools/update-packaging/test/from-mac/Contents/MacOS/same.bin Normal file
View File

Binary file not shown.

1
tools/update-packaging/test/from-mac/Contents/MacOS/same.txt Normal file
View File

@ -0,0 +1 @@
file is same

1
tools/update-packaging/test/from-mac/Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
from file

1
tools/update-packaging/test/from-mac/Contents/MacOS/update-settings.ini Normal file
View File

@ -0,0 +1 @@
add-if-not from complete file

0
tools/update-packaging/test/from/channel-prefs.js → tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest
View File

1
tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/channel-prefs.js Normal file
View File

@ -0,0 +1 @@
add-if-not from complete file

1
tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
from file

1
tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/force.txt Normal file
View File

@ -0,0 +1 @@
file is same

1
tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt Normal file
View File

@ -0,0 +1 @@
This from file should be ignored

1
tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt Normal file
View File

@ -0,0 +1 @@
removed

BIN
tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin Normal file
View File

Binary file not shown.

1
tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt Normal file
View File

@ -0,0 +1 @@
file is same

1
tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest Normal file
View File

@ -0,0 +1 @@
from file shouldn't go in update

25
tools/update-packaging/test/from-mac/precomplete Normal file
View File

@ -0,0 +1,25 @@
remove "precomplete"
remove "Contents/MacOS/{foodir/update.manifest"
remove "Contents/MacOS/{foodir/same.txt"
remove "Contents/MacOS/{foodir/same.bin"
remove "Contents/MacOS/{foodir/removed.txt"
remove "Contents/MacOS/{foodir/readme.txt"
remove "Contents/MacOS/{foodir/force.txt"
remove "Contents/MacOS/{foodir/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/update.manifest"
remove "Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/same.txt"
remove "Contents/MacOS/same.bin"
remove "Contents/MacOS/removed.txt"
remove "Contents/MacOS/readme.txt"
remove "Contents/MacOS/force.txt"
remove "Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/application.ini"
rmdir "Contents/MacOS/{foodir/"
rmdir "Contents/MacOS/searchplugins/diff/"
rmdir "Contents/MacOS/searchplugins/"
rmdir "Contents/MacOS/extensions/diff/"
rmdir "Contents/MacOS/extensions/"
rmdir "Contents/MacOS/"
rmdir "Contents/"

5
tools/update-packaging/test/from/application.ini Normal file
View File

@ -0,0 +1,5 @@
[App]
Vendor=Mozilla
Name=MarTest
Version=1
BuildID=20120101010101

1
tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
from file

44
tools/update-packaging/test/from/precomplete
View File

@ -1,21 +1,23 @@
remove {foodir/update.manifest
remove {foodir/same.txt
remove {foodir/same.bin
remove {foodir/removed.txt
remove {foodir/readme.txt
remove {foodir/force.txt
remove {foodir/diff-patch-larger-than-file.txt
remove-cc {foodir/channel-prefs.js
remove update.manifest
remove searchplugins/diff/diff-patch-larger-than-file.txt
remove same.txt
remove same.bin
remove removed.txt
remove readme.txt
remove precomplete
remove force.txt
remove diff-patch-larger-than-file.txt
remove-cc channel-prefs.js
rmdir {foodir/
rmdir searchplugins/diff/
rmdir searchplugins/
remove "{foodir/update.manifest"
remove "{foodir/same.txt"
remove "{foodir/same.bin"
remove "{foodir/removed.txt"
remove "{foodir/readme.txt"
remove "{foodir/force.txt"
remove "{foodir/diff-patch-larger-than-file.txt"
remove "update.manifest"
remove "searchplugins/diff/diff-patch-larger-than-file.txt"
remove "same.txt"
remove "same.bin"
remove "removed.txt"
remove "readme.txt"
remove "precomplete"
remove "force.txt"
remove "extensions/diff/diff-patch-larger-than-file.txt"
remove "diff-patch-larger-than-file.txt"
remove "application.ini"
rmdir "{foodir/"
rmdir "searchplugins/diff/"
rmdir "searchplugins/"
rmdir "extensions/diff/"
rmdir "extensions/"

1
tools/update-packaging/test/from/update-settings.ini Normal file
View File

@ -0,0 +1 @@
add-if-not from complete file

2
tools/update-packaging/test/from/{foodir/channel-prefs.js
View File

@ -1 +1 @@
from file shouldn't go in update
add-if-not from complete file

26
tools/update-packaging/test/make_full_update.sh
View File

@ -45,7 +45,8 @@ if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
fi
workdir="$targetdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
targetfiles="updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest"
targetfiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
@ -67,21 +68,30 @@ list_files files
popd
# Add the type of update to the beginning of version 2 update manifest.
# Add the type of update to the beginning of the update manifests.
> $updatemanifestv2
> $updatemanifestv3
notice ""
notice "Adding type instruction to file 'updatev2.manifest'"
notice " type: complete"
notice "Adding type instruction to update manifests"
notice " type complete"
echo "type \"complete\"" >> $updatemanifestv2
echo "type \"complete\"" >> $updatemanifestv3
notice ""
notice "Adding file add instructions to file 'updatev2.manifest'"
notice "Adding file add instructions to update manifests"
num_files=${#files[*]}
for ((i=0; $i<$num_files; i=$i+1)); do
f="${files[$i]}"
make_add_instruction "$f" >> $updatemanifestv2
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
if check_for_add_to_manifestv2 "$f"; then
make_add_instruction "$f" "$updatemanifestv2" "" 1
fi
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
dir=$(dirname "$f")
mkdir -p "$workdir/$dir"
@ -94,9 +104,10 @@ done
# Append remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
mv -f "$workdir/output.mar" "$archive"
@ -106,3 +117,4 @@ rm -fr "$workdir"
notice ""
notice "Finished"
notice ""

7
tools/update-packaging/test/runtests.sh
View File

@ -3,7 +3,10 @@
echo "testing make_incremental_updates.py"
python ../make_incremental_updates.py -f testpatchfile.txt
echo ""
echo "diffing ref.mar and test.mar"
./diffmar.sh ref.mar test.mar
./diffmar.sh ref.mar test.mar test
echo ""
echo "diffing ref-mac.mar and test-mac.mar"
./diffmar.sh ref-mac.mar test-mac.mar
./diffmar.sh ref-mac.mar test-mac.mar test-mac

2
tools/update-packaging/test/testpatchfile.txt
View File

@ -1,2 +1,2 @@
product-1.0.lang.platform.complete.mar,product-2.0.lang.platform.complete.mar,test.mar,""
product-1.0.lang.platform.complete.mar,product-2.0.lang.mac.complete.mar,test-mac.mar,""
product-1.0.lang.mac.complete.mar,product-2.0.lang.mac.complete.mar,test-mac.mar,""

1
tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js Normal file
View File

@ -0,0 +1 @@
this is a new file

1
tools/update-packaging/test/to-mac/Contents/MacOS/added.txt Normal file
View File

@ -0,0 +1 @@
added file

5
tools/update-packaging/test/to-mac/Contents/MacOS/application.ini Normal file
View File

@ -0,0 +1,5 @@
[App]
Vendor=Mozilla
Name=MarTest
Version=2
BuildID=20130101010101

BIN
tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin Normal file
View File

Binary file not shown.

1
tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
file to

1
tools/update-packaging/test/to-mac/Contents/MacOS/distribution/extensions/added/file.txt Normal file
View File

@ -0,0 +1 @@
extfile

1
tools/update-packaging/test/to-mac/Contents/MacOS/distribution/extensions/diff/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
to file

1
tools/update-packaging/test/to-mac/Contents/MacOS/extensions/added/file.txt Normal file
View File

@ -0,0 +1 @@
extfile

1
tools/update-packaging/test/to-mac/Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
to file

1
tools/update-packaging/test/to-mac/Contents/MacOS/force.txt Normal file
View File

@ -0,0 +1 @@
file is same

19
tools/update-packaging/test/to-mac/Contents/MacOS/precomplete Normal file
View File

@ -0,0 +1,19 @@
remove "Contents/MacOS/{foodir/update.manifest"
remove "Contents/MacOS/{foodir/same.txt"
remove "Contents/MacOS/{foodir/same.bin"
remove "Contents/MacOS/{foodir/readme.txt"
remove "Contents/MacOS/{foodir/force.txt"
remove "Contents/MacOS/{foodir/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/update.manifest"
remove "Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/same.txt"
remove "Contents/MacOS/same.bin"
remove "Contents/MacOS/removed-files"
remove "Contents/MacOS/readme.txt"
remove "Contents/MacOS/precomplete"
remove "Contents/MacOS/force.txt"
remove "Contents/MacOS/diff-patch-larger-than-file.txt"
rmdir "Contents/MacOS/{foodir/"
rmdir "Contents/MacOS/searchplugins/diff/"
rmdir "Contents/MacOS/searchplugins/added/"
rmdir "Contents/MacOS/searchplugins/"

1
tools/update-packaging/test/to-mac/Contents/MacOS/readme.txt Normal file
View File

@ -0,0 +1 @@
This to file should be ignored

1
tools/update-packaging/test/to-mac/Contents/MacOS/removed-files
View File

@ -1,5 +1,6 @@
removed1.txt
removed2.bin
recursivedir/meh/*
removed3-foo.txt
dir/
this file has spaces

BIN
tools/update-packaging/test/to-mac/Contents/MacOS/same.bin Normal file
View File

Binary file not shown.

1
tools/update-packaging/test/to-mac/Contents/MacOS/same.txt Normal file
View File

@ -0,0 +1 @@
file is same

1
tools/update-packaging/test/to-mac/Contents/MacOS/searchplugins/added/file.txt Normal file
View File

@ -0,0 +1 @@
extfile

1
tools/update-packaging/test/to-mac/Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
to file

1
tools/update-packaging/test/to-mac/Contents/MacOS/update-settings.ini Normal file
View File

@ -0,0 +1 @@
add-if-not from complete file

0
tools/update-packaging/test/to/channel-prefs.js → tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest
View File

1
tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt Normal file
View File

@ -0,0 +1 @@
added file

1
tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/channel-prefs.js Normal file
View File

@ -0,0 +1 @@
add-if-not from partial file

1
tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
file to

1
tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/force.txt Normal file
View File

@ -0,0 +1 @@
file is same

1
tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt Normal file
View File

@ -0,0 +1 @@
This to file should be ignored

BIN
tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin Normal file
View File

Binary file not shown.

1
tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt Normal file
View File

@ -0,0 +1 @@
file is same

1
tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest Normal file
View File

@ -0,0 +1 @@
to file shouldn't go in update

37
tools/update-packaging/test/to-mac/precomplete
View File

@ -1,4 +1,33 @@
remove precomplete
remove Contents/MacOS/removed-files
rmdir Contents/MacOS/
rmdir Contents/
remove "precomplete"
remove "Contents/MacOS/{foodir/update.manifest"
remove "Contents/MacOS/{foodir/same.txt"
remove "Contents/MacOS/{foodir/same.bin"
remove "Contents/MacOS/{foodir/readme.txt"
remove "Contents/MacOS/{foodir/force.txt"
remove "Contents/MacOS/{foodir/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/{foodir/added.txt"
remove "Contents/MacOS/update.manifest"
remove "Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/searchplugins/added/file.txt"
remove "Contents/MacOS/same.txt"
remove "Contents/MacOS/same.bin"
remove "Contents/MacOS/removed-files"
remove "Contents/MacOS/readme.txt"
remove "Contents/MacOS/precomplete"
remove "Contents/MacOS/force.txt"
remove "Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/extensions/added/file.txt"
remove "Contents/MacOS/diff-patch-larger-than-file.txt"
remove "Contents/MacOS/diff-patch-larger-than-file.bin"
remove "Contents/MacOS/application.ini"
remove "Contents/MacOS/added.txt"
remove "Contents/MacOS/addFeedPrefs.js"
rmdir "Contents/MacOS/{foodir/"
rmdir "Contents/MacOS/searchplugins/diff/"
rmdir "Contents/MacOS/searchplugins/added/"
rmdir "Contents/MacOS/searchplugins/"
rmdir "Contents/MacOS/extensions/diff/"
rmdir "Contents/MacOS/extensions/added/"
rmdir "Contents/MacOS/extensions/"
rmdir "Contents/MacOS/"
rmdir "Contents/"

5
tools/update-packaging/test/to/application.ini Normal file
View File

@ -0,0 +1,5 @@
[App]
Vendor=Mozilla
Name=MarTest
Version=2
BuildID=20130101010101

1
tools/update-packaging/test/to/distribution/extensions/added/file.txt Normal file
View File

@ -0,0 +1 @@
extfile

1
tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt Normal file
View File

@ -0,0 +1 @@
to file

51
tools/update-packaging/test/to/precomplete
View File

@ -1,21 +1,30 @@
remove {foodir/update.manifest
remove {foodir/same.txt
remove {foodir/same.bin
remove {foodir/readme.txt
remove {foodir/force.txt
remove {foodir/diff-patch-larger-than-file.txt
remove-cc {foodir/channel-prefs.js
remove update.manifest
remove searchplugins/diff/diff-patch-larger-than-file.txt
remove same.txt
remove same.bin
remove removed-files
remove readme.txt
remove precomplete
remove force.txt
remove diff-patch-larger-than-file.txt
remove-cc channel-prefs.js
rmdir {foodir/
rmdir searchplugins/diff/
rmdir searchplugins/added/
rmdir searchplugins/
remove "{foodir/update.manifest"
remove "{foodir/same.txt"
remove "{foodir/same.bin"
remove "{foodir/readme.txt"
remove "{foodir/force.txt"
remove "{foodir/diff-patch-larger-than-file.txt"
remove "{foodir/added.txt"
remove "update.manifest"
remove "searchplugins/diff/diff-patch-larger-than-file.txt"
remove "searchplugins/added/file.txt"
remove "same.txt"
remove "same.bin"
remove "removed-files"
remove "readme.txt"
remove "precomplete"
remove "force.txt"
remove "extensions/diff/diff-patch-larger-than-file.txt"
remove "extensions/added/file.txt"
remove "diff-patch-larger-than-file.txt"
remove "diff-patch-larger-than-file.bin"
remove "application.ini"
remove "added.txt"
remove "addFeedPrefs.js"
rmdir "{foodir/"
rmdir "searchplugins/diff/"
rmdir "searchplugins/added/"
rmdir "searchplugins/"
rmdir "extensions/diff/"
rmdir "extensions/added/"
rmdir "extensions/"

1
tools/update-packaging/test/to/removed-files
View File

@ -1,5 +1,6 @@
removed1.txt
removed2.bin
recursivedir/meh/*
removed3-foo.txt
dir/
this file has spaces

1
tools/update-packaging/test/to/update-settings.ini Normal file
View File

@ -0,0 +1 @@
add-if-not from complete file

2
tools/update-packaging/test/to/{foodir/channel-prefs.js
View File

@ -1 +1 @@
to file shouldn't go in update
add-if-not from complete file

41
tools/update-packaging/test_make_incremental_updates.py
View File

@ -11,7 +11,7 @@ from make_incremental_updates import PatchInfo, MarFileEntry
class TestPatchInfo(unittest.TestCase):
def setUp(self):
self.work_dir = 'work_dir'
self.file_exclusion_list = ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files']
self.file_exclusion_list = ['update.manifest','updatev2.manifest','updatev3.manifest','removed-files']
self.path_exclusion_list = ['/readme.txt']
self.patch_info = PatchInfo(self.work_dir, self.file_exclusion_list, self.path_exclusion_list)
@ -19,22 +19,51 @@ class TestPatchInfo(unittest.TestCase):
self.assertEquals(self.work_dir, self.patch_info.work_dir)
self.assertEquals([], self.patch_info.archive_files)
self.assertEquals([], self.patch_info.manifestv2)
self.assertEquals([], self.patch_info.manifestv3)
self.assertEquals(self.file_exclusion_list, self.patch_info.file_exclusion_list)
self.assertEquals(self.path_exclusion_list, self.patch_info.path_exclusion_list)
def test_append_add_instruction(self):
self.patch_info.append_add_instruction('file.test')
self.assertEquals(['add "file.test"'], self.patch_info.manifestv2)
self.assertEquals(['add "file.test"'], self.patch_info.manifestv3)
def test_append_add_if_instruction(self):
self.patch_info.append_add_instruction('distribution/extensions/extension/file.test')
self.assertEquals(['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2)
self.assertEquals(['add-if "distribution/extensions/extension" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3)
def test_append_add_if_not_instruction(self):
self.patch_info.append_add_if_not_instruction('file.test')
self.assertEquals([], self.patch_info.manifestv2)
self.assertEquals(['add-if-not "file.test" "file.test"'], self.patch_info.manifestv3)
def test_append_patch_instruction(self):
self.patch_info.append_patch_instruction('file.test', 'patchname')
self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv2)
self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv3)
def test_append_patch_if_instruction(self):
self.patch_info.append_patch_instruction('distribution/extensions/extension/file.test', 'patchname')
self.assertEquals(['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv2)
self.assertEquals(['patch-if "distribution/extensions/extension" "patchname" "distribution/extensions/extension/file.test"'], self.patch_info.manifestv3)
""" FIXME touches the filesystem, need refactoring
def test_append_remove_instruction(self):
self.patch_info.append_remove_instruction('file.test')
self.assertEquals(['remove "file.test"'], self.patch_info.manifestv2)
self.assertEquals(['remove "file.test"'], self.patch_info.manifestv3)
def test_append_rmdir_instruction(self):
self.patch_info.append_remove_instruction('dirtest/')
self.assertEquals(['rmdir "dirtest/"'], self.patch_info.manifestv2)
self.assertEquals(['rmdir "dirtest/"'], self.patch_info.manifestv3)
def test_append_rmrfdir_instruction(self):
self.patch_info.append_remove_instruction('dirtest/*')
self.assertEquals(['rmrfdir "dirtest/"'], self.patch_info.manifestv2)
self.assertEquals(['rmrfdir "dirtest/"'], self.patch_info.manifestv3)
""" FIXME touches the filesystem, need refactoring
def test_create_manifest_file(self):
self.patch_info.create_manifest_file()
"""
@ -70,7 +99,7 @@ class TestMarFileEntry(unittest.TestCase):
class TestMakeIncrementalUpdates(unittest.TestCase):
def setUp(self):
work_dir = '.'
self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt'])
self.patch_info = PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest','removed-files'],['/readme.txt'])
root_path = '/'
filename = 'test.file'
self.mar_file_entry = MarFileEntry(root_path, filename)
@ -96,10 +125,10 @@ class TestMakeIncrementalUpdates(unittest.TestCase):
def test_create_add_patch_for_file(self):
mkup.create_add_patch_for_file('to_marfile_entry', self.patch_info)
def test_process_explicit_remove_files(self):
mkup.process_explicit_remove_files('dir_path', self.patch_info)
def test_create_partial_patch(self):
mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas', self.patch_info, 'forced_updates')
@ -117,6 +146,6 @@ class TestMakeIncrementalUpdates(unittest.TestCase):
expected = {'locale': 'lang', 'platform': 'platform', 'product': 'product', 'version': '1.0', 'type': 'complete'}
self.assertEquals(expected, mkup.decode_filename('product-1.0.lang.platform.complete.mar'))
self.assertRaises(Exception, mkup.decode_filename, 'fail')
if __name__ == '__main__':
unittest.main()