bug 409956, #filter emptyLines is brittle on newlines, r=ted.mielczarek, a=mossop

config/Preprocessor.py

@@ -147,7 +147,9 @@ class Preprocessor:
         self.writtenLines = ln
     for f in self.filters:
       aLine = f[1](aLine)
-    aLine = aLine.rstrip('\r\n') + self.LE
+    # ensure our line ending. Only need to handle \n, as we're reading
+    # with universal line ending support, at least for files.
+    aLine = re.sub('\n', self.LE, aLine)
     self.out.write(aLine)
   
   def handleCommandLine(self, args, defaultToStdin = False):
@@ -347,7 +349,7 @@ class Preprocessor:
     lst.append('\n') # add back the newline
     self.write(reduce(lambda x, y: x+y, lst, ''))
   def do_literal(self, args):
-    self.write(args)
+    self.write(args + self.LE)
   def do_filter(self, args):
     filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
     if len(filters) == 0:
@@ -417,7 +419,7 @@ class Preprocessor:
         args = str(args)
         if not os.path.isabs(args):
           args = os.path.join(self.context['DIRECTORY'], args)
-        args = open(args)
+        args = open(args, 'rU')
       except:
         raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
     self.checkLineNumbers = bool(re.search('\.js(?:\.in)?$', args.name))

config/tests/unit-LineEndings.py

@@ -0,0 +1,46 @@
+import unittest
+
+from StringIO import StringIO
+import os
+import sys
+import os.path
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+from Preprocessor import Preprocessor
+
+class TestLineEndings(unittest.TestCase):
+  """
+  Unit tests for the Context class
+  """
+
+  def setUp(self):
+    self.pp = Preprocessor()
+    self.pp.out = StringIO()
+    self.tempnam = os.tempnam('.')
+
+  def tearDown(self):
+    os.remove(self.tempnam)
+
+  def createFile(self, lineendings):
+    f = open(self.tempnam, 'wb')
+    for line, ending in zip(['a', '#literal b', 'c'], lineendings):
+      f.write(line+ending)
+    f.close()
+
+  def testMac(self):
+    self.createFile(['\x0D']*3)
+    self.pp.do_include(self.tempnam)
+    self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
+
+  def testUnix(self):
+    self.createFile(['\x0A']*3)
+    self.pp.do_include(self.tempnam)
+    self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
+
+  def testWindows(self):
+    self.createFile(['\x0D\x0A']*3)
+    self.pp.do_include(self.tempnam)
+    self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
+
+if __name__ == '__main__':
+  unittest.main()

config/tests/unit-Preprocessor.py

@@ -177,6 +177,30 @@ P@VAR@ASS
     self.pp.do_include(f)
     self.assertEqual(self.pp.out.getvalue(), "PASS\n")
   
+  def test_filter_emptyLines(self):
+    f = NamedIO('filter_emptyLines.in', '''lines with a
+
+blank line
+#filter emptyLines
+lines with
+
+no blank lines
+#unfilter emptyLines
+yet more lines with
+
+blank lines
+''')
+    self.pp.do_include(f)
+    self.assertEqual(self.pp.out.getvalue(), '''lines with a
+
+blank line
+lines with
+no blank lines
+yet more lines with
+
+blank lines
+''')
+  
   def test_filter_slashslash(self):
     f = NamedIO('filter_slashslash.in', '''#filter slashslash
 PASS//FAIL  // FAIL
@@ -372,5 +396,13 @@ FAIL
     self.pp.do_include(f)
     self.assertEqual(self.pp.out.getvalue(), "PASS\n")
 
+  def test_lineEndings(self):
+    f = NamedIO('lineEndings.in', '''first
+#literal second
+''')
+    self.pp.setLineEndings('cr')
+    self.pp.do_include(f)
+    self.assertEqual(self.pp.out.getvalue(), "first\rsecond\r")
+
 if __name__ == '__main__':
   unittest.main()