Bug 305039 - (Win32 only) Make Windows installer more 'Unicode-aware'. r=rstrong

browser/installer/windows/Makefile.in

@@ -21,6 +21,7 @@
 # Contributor(s):
 #  Brian Ryner <bryner@brianryner.com>
 #  Chase Phillips <cmp@mozilla.org>
+#  Ehsan Akhgari <ehsan.akhgari@gmail.com>
 #
 # Alternatively, the contents of this file may be used under the terms of
 # either the GNU General Public License Version 2 or later (the "GPL"), or
@@ -52,15 +53,25 @@ PP_LOCALIZED_FILES = \
 	packages-static \
 	$(NULL)
 
-INSTALLER_FILES = \
-	app.tag \
+# All script and locale files used by the Unicode version of NSIS need to be
+# converted from UTF-8 to UTF-16LE
+INSTALLER_FILES_CONV = \
 	nsis/installer.nsi \
 	nsis/uninstaller.nsi \
 	nsis/shared.nsh \
 	$(NULL)
 
-BRANDING_FILES = \
+INSTALLER_FILES = \
+	app.tag \
+	$(NULL)
+
+# All script and locale files used by the Unicode version of NSIS need to be
+# converted from UTF-8 to UTF-16LE
+BRANDING_FILES_CONV = \
 	branding.nsi \
+	$(NULL)
+
+BRANDING_FILES = \
 	wizHeader.bmp \
 	wizHeaderRTL.bmp \
 	wizWatermark.bmp \
@@ -77,7 +88,6 @@ DEFINES += \
 	$(NULL)
 
 include $(topsrcdir)/config/config.mk
-include $(call EXPAND_LOCALE_SRCDIR,toolkit/locales)/installer/windows/charset.mk
 
 installer::
 	$(MAKE) -C .. installer-stage
@@ -87,21 +97,43 @@ installer::
 # included for mar file generation.
 uninstaller::
 	$(RM) -rf $(CONFIG_DIR) && mkdir $(CONFIG_DIR)
+	for i in $(INSTALLER_FILES_CONV); do \
+	  iconv -f UTF-8 -t UTF-16LE $(srcdir)/$$i | \
+	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
+	    $(CONFIG_DIR)/`basename $$i`; \
+	done
 	$(INSTALL) $(addprefix $(srcdir)/,$(INSTALLER_FILES)) $(CONFIG_DIR)
+	for i in $(BRANDING_FILES_CONV); do \
+	  iconv -f UTF-8 -t UTF-16LE $(DIST)/branding/$$i | \
+	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
+	    $(CONFIG_DIR)/$$i; \
+	done
 	$(INSTALL) $(addprefix $(DIST)/branding/,$(BRANDING_FILES)) $(CONFIG_DIR)
 	$(EXIT_ON_ERROR) \
 	for i in $(PP_LOCALIZED_FILES); do \
 	  $(PERL) $(topsrcdir)/config/preprocessor.pl $(DEFINES) $(ACDEFINES) $(srcdir)/$$i > $(CONFIG_DIR)/$$i; \
 	done
 	$(PERL) $(topsrcdir)/config/preprocessor.pl -Fsubstitution $(DEFINES) $(ACDEFINES) \
-	  $(srcdir)/nsis/defines.nsi.in > $(CONFIG_DIR)/defines.nsi
+	  $(srcdir)/nsis/defines.nsi.in | iconv -f UTF-8 -t UTF-16LE | \
+	  cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
+	  $(CONFIG_DIR)/defines.nsi
 	$(PERL) $(topsrcdir)/toolkit/mozapps/installer/windows/nsis/preprocess-locale.pl \
 	  $(topsrcdir) $(call EXPAND_LOCALE_SRCDIR,browser/locales)/installer $(AB_CD) \
-	  $(WIN_INSTALLER_CHARSET) $(CONFIG_DIR)
+	  $(CONFIG_DIR)
 
 $(CONFIG_DIR)/setup.exe::
 	$(RM) -rf $(CONFIG_DIR) && mkdir $(CONFIG_DIR)
+	for i in $(INSTALLER_FILES_CONV); do \
+	  iconv -f UTF-8 -t UTF-16LE $(srcdir)/$$i | \
+	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
+	    $(CONFIG_DIR)/`basename $$i`; \
+	done
 	$(INSTALL) $(addprefix $(srcdir)/,$(INSTALLER_FILES)) $(CONFIG_DIR)
+	for i in $(BRANDING_FILES_CONV); do \
+	  iconv -f UTF-8 -t UTF-16LE $(DIST)/branding/$$i | \
+	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
+	    $(CONFIG_DIR)/$$i; \
+	done
 	$(INSTALL) $(addprefix $(DIST)/branding/,$(BRANDING_FILES)) $(CONFIG_DIR)
 	$(EXIT_ON_ERROR) \
 	for i in $(PP_LOCALIZED_FILES); do \
@@ -110,10 +142,12 @@ $(CONFIG_DIR)/setup.exe::
 	$(PERL) $(topsrcdir)/toolkit/mozapps/installer/windows/nsis/make-installremoves.pl \
 	  ../removed-files > $(CONFIG_DIR)/removed-files.log
 	$(PERL) $(topsrcdir)/config/preprocessor.pl -Fsubstitution $(DEFINES) $(ACDEFINES) \
-	  $(srcdir)/nsis/defines.nsi.in > $(CONFIG_DIR)/defines.nsi
+	  $(srcdir)/nsis/defines.nsi.in | iconv -f UTF-8 -t UTF-16LE | \
+	  cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > \
+	  $(CONFIG_DIR)/defines.nsi
 	$(PERL) $(topsrcdir)/toolkit/mozapps/installer/windows/nsis/preprocess-locale.pl \
 	  $(topsrcdir) $(call EXPAND_LOCALE_SRCDIR,browser/locales)/installer $(AB_CD) \
-	  $(WIN_INSTALLER_CHARSET) $(CONFIG_DIR)
+	  $(CONFIG_DIR)
 
 include $(topsrcdir)/config/rules.mk
 include $(topsrcdir)/toolkit/mozapps/installer/windows/nsis/makensis.mk

toolkit/locales/en-US/installer/windows/charset.mk

@@ -1,21 +0,0 @@
-WIN_INSTALLER_CHARSET = CP1252
-
-# possibilities are:
-# ANSI_CHARSET                    CP1252
-# BALTIC_CHARSET                  CP1257
-# CHINESEBIG5_CHARSET             CP950
-# EASTEUROPE_CHARSET              CP1250
-# GB2312_CHARSET                  CP936
-# GREEK_CHARSET                   CP1253
-# HANGUL_CHARSET                  CP949
-# RUSSIAN_CHARSET                 CP1251
-# SHIFTJIS_CHARSET                CP932
-# TURKISH_CHARSET                 CP1254
-# VIETNAMESE_CHARSET              CP1258
-# Korean language edition of Windows:
-# JOHAB_CHARSET                   CP1361
-# Middle East language edition of Windows:
-# ARABIC_CHARSET                  CP1256
-# HEBREW_CHARSET                  CP1255
-# Thai language edition of Windows:
-# THAI_CHARSET                    CP874

toolkit/mozapps/installer/windows/nsis/makensis.mk

@@ -21,6 +21,7 @@
 # Contributor(s):
 #   Robert Strong <rstrong@mozilla.com> - Initial perl scripts (install_sub.pl)
 #   Benjamin Smedberg <benjamin@smedbergs.us> - Makefile-izing
+#   Ehsan Akhgari <ehsan.akhgari@gmail.com> - Unicode installer support
 #
 # Alternatively, the contents of this file may be used under the terms of
 # either the GNU General Public License Version 2 or later (the "GPL"), or
@@ -44,24 +45,32 @@ ABS_CONFIG_DIR := $(shell pwd)/$(CONFIG_DIR)
 
 SFX_MODULE ?= $(error SFX_MODULE is not defined)
 
-TOOLKIT_NSIS_FILES = \
-	AppAssocReg.dll \
+# UTF-8 encoded files that need to be converted to UTF-16LE for NSIS
+TOOLKIT_NSIS_FILES_CONV = \
 	common.nsh \
 	locales.nsi \
-	nsProcess.dll \
 	overrides.nsh \
-	ShellLink.dll \
-	UAC.dll \
 	version.nsh \
 	$(NULL)
 
+TOOLKIT_NSIS_FILES = \
+	AppAssocReg.dll \
+	nsProcess.dll \
+	ShellLink.dll \
+	UAC.dll \
+	$(NULL)
+
 $(CONFIG_DIR)/setup.exe::
+	for i in $(TOOLKIT_NSIS_FILES_CONV); do \
+	  iconv -f UTF-8 -t UTF-16LE $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/$$i | \
+	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > $(CONFIG_DIR)/$$i; \
+	done
 	$(INSTALL) $(addprefix $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/,$(TOOLKIT_NSIS_FILES)) $(CONFIG_DIR)
 	$(INSTALL) $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/setup.ico $(CONFIG_DIR)
-	cd $(CONFIG_DIR) && makensis.exe installer.nsi
+	cd $(CONFIG_DIR) && makensisu.exe installer.nsi
 # Support for building the uninstaller when repackaging locales
 ifeq ($(CONFIG_DIR),l10ngen)
-	cd $(CONFIG_DIR) && makensis.exe uninstaller.nsi
+	cd $(CONFIG_DIR) && makensisu.exe uninstaller.nsi
 endif
 
 $(CONFIG_DIR)/7zSD.sfx:
@@ -78,8 +87,12 @@ installer::
 # For building the uninstaller during the application build so it can be
 # included for mar file generation.
 uninstaller::
+	for i in $(TOOLKIT_NSIS_FILES_CONV); do \
+	  iconv -f UTF-8 -t UTF-16LE $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/$$i | \
+	    cat $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin - > $(CONFIG_DIR)/$$i; \
+	done
 	$(INSTALL) $(addprefix $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/,$(TOOLKIT_NSIS_FILES)) $(CONFIG_DIR)
 	$(INSTALL) $(MOZILLA_DIR)/toolkit/mozapps/installer/windows/nsis/setup.ico $(CONFIG_DIR)
-	cd $(CONFIG_DIR) && makensis.exe uninstaller.nsi
+	cd $(CONFIG_DIR) && makensisu.exe uninstaller.nsi
 	$(NSINSTALL) -D $(DIST)/bin/uninstall
 	cp $(CONFIG_DIR)/helper.exe $(DIST)/bin/uninstall

toolkit/mozapps/installer/windows/nsis/preprocess-locale.pl

@@ -19,6 +19,7 @@
 #
 # Contributor(s):
 #  Robert Strong <robert.bugzilla@gmail.com>
+#  Ehsan Akhgari <ehsan.akhgari@gmail.com>
 #
 # Alternatively, the contents of this file may be used under the terms of
 # either the GNU General Public License Version 2 or later (the "GPL"), or
@@ -34,11 +35,10 @@
 #
 # ***** END LICENSE BLOCK *****
 
-my $topsrcdir = "$ARGV[0]";
+my $mozsrcdir = "$ARGV[0]";    # dir that contains the toolkit source
 my $appLocaleDir = "$ARGV[1]";
 my $AB_CD = "$ARGV[2]";
-my $langCP = "$ARGV[3]";
-my $configDir = "$ARGV[4]";
+my $configDir = "$ARGV[3]";
 my $nsisVer = "v6";
 # Set the language ID to 0 to make this locale the default locale. An actual
 # ID will need to be used to create a multi-language installer (e.g. for CD
@@ -53,8 +53,11 @@ my $lnum;
 
 # Read the codepage for the locale and the optional font name, font size, and
 # whether the locale is right to left from locales.nsi.
-my $inFile = "$topsrcdir/toolkit/mozapps/installer/windows/nsis/locales.nsi";
+my $inFile = "$mozsrcdir/toolkit/mozapps/installer/windows/nsis/locales.nsi";
 open(locales, "<$inFile");
+if (!-e $inFile) {
+  die "Error $inFile does not exist!";
+}
 
 $lnum = 1;
 while( $line = <locales> ) {
@@ -66,13 +69,6 @@ while( $line = <locales> ) {
 }
 close locales;
 
-# In NSIS codepage CP1252 is specified with a '-'. For all other locales
-# specify the number for the locales codepage.
-if ($langCP ne "CP1252") {
-  $nsisCP = $langCP;
-  $nsisCP =~ s/^CP(.*)$/$1/g;
-}
-
 # Create the main NSIS language file with just the codepage, font, and
 # RTL information
 open(outfile, ">$configDir/nlf.in");
@@ -82,7 +78,7 @@ print outfile "# Font and size - dash (-) means default\r\n$fontName\r\n$fontSiz
 print outfile "# Codepage - dash (-) means ANSI code page\r\n$nsisCP\r\n";
 print outfile "# RTL - anything else than RTL means LTR\r\n$RTL\r\n";
 close outfile;
-&cpConvert("nlf.in", "baseLocale.nlf", $langCP);
+&cpConvert("nlf.in", "baseLocale.nlf");
 
 
 # Create the main NSIS language file
@@ -103,13 +99,13 @@ while( $line = <infile> ) {
   $value =~ s/\s+$//; # trim whitespace from the end of the string
   $value =~ s/^"(.*)"$/$1/g; # remove " at the beginning and end of the value
   $value =~ s/(")/\$\\$1/g;  # prefix " with $\
-  $value =~ s/…/.../g;     # replace … (unicode ellipsis) with ...
+  $value =~ s/(\\[rnt])/\$$1/g; # prefix all occurences of \r, \n and \t with $
   print outfile "LangString  ^@values[0] $langID \"$value\"\r\n";
   $lnum++;
 }
 close infile;
 close outfile;
-&cpConvert("override.properties", "overrideLocale.nsh", $langCP);
+&cpConvert("override.properties", "overrideLocale.nsh");
 
 
 # Create the main Modern User Interface language file
@@ -133,16 +129,16 @@ while( $line = <infile> ) {
   next if (@values[0] eq undef) || (@values[1] eq undef);
   my $value = @values[1];
   $value =~ s/(")/\$\\$1/g;  # prefix " with $\
-  $value =~ s/(\\n)/\\r$1/g; # insert \\r before each occurence of \\n
-  $value =~ s/(\\r)\\r/$1/g; # replace all ocurrences of \\r\\r with \\r
-  $value =~ s/…/.../g;     # replace … (unicode ellipsis) with ...
+  $value =~ s/(\\n)/\\r$1/g; # insert \r before each occurence of \n
+  $value =~ s/(\\r)\\r/$1/g; # replace all ocurrences of \r\r with \r
+  $value =~ s/(\\[rnt])/\$$1/g; # prefix all occurences of \r, \n and \t with $
   print outfile "!define @values[0] \"$value\"\r\n";
   $lnum++;
 }
 print outfile "!insertmacro MOZ_MUI_LANGUAGEFILE_END\r\n";
 close infile;
 close outfile;
-&cpConvert("mui.properties", "baseLocale.nsh", $langCP);
+&cpConvert("mui.properties", "baseLocale.nsh");
 
 
 # Create the custom language file for our custom strings
@@ -161,16 +157,15 @@ while( $line = <infile> ) {
   next if (@values[0] eq undef) || (@values[1] eq undef);
   my $string = @values[1];
   $string =~ s/"/\$\\"/g;       # replace " with $\"
-  $string =~ s/(\\n)/\\r$1/g;   # insert \\r before each occurence of \\n
-  $string =~ s/(\\r)\\r/$1/g;   # replace all ocurrences of \\r\\r with \\r
-  $string =~ s/(\\[rn])/\$$1/g; # prefix all occurences of \\r and \\n with $
-  $string =~ s/…/.../g;       # replace … (unicode ellipsis) with ...
+  $string =~ s/(\\n)/\\r$1/g;   # insert \r before each occurence of \n
+  $string =~ s/(\\r)\\r/$1/g;   # replace all ocurrences of \r\r with \r
+  $string =~ s/(\\[rnt])/\$$1/g; # prefix all occurences of \r, \n and \t with $
   print outfile "LangString @values[0] $langID \"$string\"\r\n";
   $lnum++;
 }
 close infile;
 close outfile;
-&cpConvert("custom.properties", "customLocale.nsh", $langCP);
+&cpConvert("custom.properties", "customLocale.nsh");
 
 
 # Converts a file's codepage
@@ -178,9 +173,9 @@ sub cpConvert
 {
   my $srcFile = $_[0];
   my $targetFile = $_[1];
-  my $targetCodepage = $_[2];
-  print "iconv -f UTF-8 -t $targetCodepage $configDir/$srcFile > $configDir/$targetFile\n";
-  system("iconv -f UTF-8 -t $targetCodepage $configDir/$srcFile > $configDir/$targetFile") &&
-    die "Error converting codepage to $targetCodepage for $configDir/$srcFile";
+  my $prependBOM = "cat $mozsrcdir/toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin -";
+  print "iconv -f UTF-8 -t UTF-16LE $configDir/$srcFile | $prependBOM > $configDir/$targetFile\n";
+  system("iconv -f UTF-8 -t UTF-16LE $configDir/$srcFile | $prependBOM > $configDir/$targetFile") &&
+    die "Error converting codepage to UTF-16LE for $configDir/$srcFile";
   unlink <$configDir/$srcFile>;
 }

toolkit/mozapps/installer/windows/nsis/utf16-le-bom.bin

@@ -0,0 +1 @@
+