PP, nonfunctional: Remove crufty bit-twiddling of tokens.

Test/baseResults/badChars.frag.out

@@ -4,7 +4,7 @@ ERROR: 0:1: '#if' : unexpected tokens following directive
 ERROR: 0:3: '#error' : A <bad token> B  
 ERROR: 0:4: 'preprocessor evaluation' : bad expression 
 ERROR: 0:4: '#if' : unexpected tokens following directive 
-ERROR: 0:6: 'ÿ' : unexpected token 
+ERROR: 0:6: '€' : unexpected token 
 ERROR: 0:7: '' :  syntax error
 ERROR: 7 compilation errors.  No code generated.
 

glslang/Include/revision.h

@@ -2,5 +2,5 @@
 // For the version, it uses the latest git tag followed by the number of commits.
 // For the date, it uses the current date (when then script is run).
 
-#define GLSLANG_REVISION "Overload400-PrecQual.1820"
+#define GLSLANG_REVISION "Overload400-PrecQual.1825"
 #define GLSLANG_DATE "10-Feb-2017"

glslang/MachineIndependent/preprocessor/Pp.cpp

@@ -1076,16 +1076,16 @@ bool TPpContext::tMacroInput::peekMacPasting()
     size_t savePos = mac->body.current;
 
     // skip white-space
-    int ltoken;
+    int subtoken;
     do {
-        ltoken = pp->lReadByte(mac->body);
+        subtoken = pp->getSubtoken(mac->body);
-    } while (ltoken == ' ');
+    } while (subtoken == ' ');
 
     // check for ##
     bool pasting = false;
-    if (ltoken == '#') {
+    if (subtoken == '#') {
-        ltoken = pp->lReadByte(mac->body);
+        subtoken = pp->getSubtoken(mac->body);
-        if (ltoken == '#')
+        if (subtoken == '#')
             pasting = true;
     }
 

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -375,9 +375,9 @@ protected:
     //
     // From PpTokens.cpp
     //
-    void lAddByte(TokenStream&, unsigned char fVal);
+    void putSubtoken(TokenStream&, int fVal);
-    int lReadByte(TokenStream&);
+    int getSubtoken(TokenStream&);
-    void lUnreadByte(TokenStream&);
+    void ungetSubtoken(TokenStream&);
     void RecordToken(TokenStream&, int token, TPpToken* ppToken);
     void RewindTokenStream(TokenStream&);
     int ReadToken(TokenStream&, TPpToken*);

glslang/MachineIndependent/preprocessor/PpScanner.cpp

@@ -232,6 +232,8 @@ int TPpContext::tStringInput::scan(TPpToken* ppToken)
         switch (ch) {
         default:
             // Single character token, including EndOfInput, '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token)
+            if (ch > PpAtomMaxSingle)
+                ch = PpAtomBadToken;
             return ch;
 
         case 'A': case 'B': case 'C': case 'D': case 'E':

glslang/MachineIndependent/preprocessor/PpTokens.cpp

@@ -95,26 +95,27 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace glslang {
 
-void TPpContext::lAddByte(TokenStream& fTok, unsigned char fVal)
+// push onto back of stream
+void TPpContext::putSubtoken(TokenStream& stream, int subtoken)
 {
-    fTok.data.push_back(fVal);
+    assert((subtoken & ~0xff) == 0);
+    stream.data.push_back(static_cast<unsigned char>(subtoken));
 }
 
-/*
+// get the next token in stream
-* Get the next byte from a stream.
+int TPpContext::getSubtoken(TokenStream& stream)
-*/
-int TPpContext::lReadByte(TokenStream& pTok)
 {
-    if (pTok.current < pTok.data.size())
+    if (stream.current < stream.data.size())
-        return pTok.data[pTok.current++];
+        return stream.data[stream.current++];
     else
         return EndOfInput;
 }
 
-void TPpContext::lUnreadByte(TokenStream& pTok)
+// back up one position in the stream
+void TPpContext::ungetSubtoken(TokenStream& stream)
 {
-    if (pTok.current > 0)
+    if (stream.current > 0)
-        --pTok.current;
+        --stream.current;
 }
 
 /*
@@ -125,18 +126,15 @@ void TPpContext::RecordToken(TokenStream& pTok, int token, TPpToken* ppToken)
     const char* s;
     char* str = NULL;
 
-    if (token > PpAtomMaxSingle)
+    putSubtoken(pTok, token);
-        lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80));
-    else
-        lAddByte(pTok, (unsigned char)(token & 0x7f));
 
     switch (token) {
     case PpAtomIdentifier:
     case PpAtomConstString:
         s = ppToken->name;
         while (*s)
-            lAddByte(pTok, (unsigned char) *s++);
+            putSubtoken(pTok, *s++);
-        lAddByte(pTok, 0);
+        putSubtoken(pTok, 0);
         break;
     case PpAtomConstInt:
     case PpAtomConstUint:
@@ -149,10 +147,10 @@ void TPpContext::RecordToken(TokenStream& pTok, int token, TPpToken* ppToken)
 #endif
         str = ppToken->name;
         while (*str) {
-            lAddByte(pTok, (unsigned char) *str);
+            putSubtoken(pTok, *str);
             str++;
         }
-        lAddByte(pTok, 0);
+        putSubtoken(pTok, 0);
         break;
     default:
         break;
@@ -172,23 +170,21 @@ void TPpContext::RewindTokenStream(TokenStream& pTok)
 */
 int TPpContext::ReadToken(TokenStream& pTok, TPpToken *ppToken)
 {
-    int ltoken, len;
+    int len;
     int ch;
 
-    ltoken = lReadByte(pTok);
+    int subtoken = getSubtoken(pTok);
     ppToken->loc = parseContext.getCurrentLoc();
-    if (ltoken > 127)
+    switch (subtoken) {
-        ltoken += 128;
-    switch (ltoken) {
     case '#':
         // Check for ##, unless the current # is the last character
         if (pTok.current < pTok.data.size()) {
-            if (lReadByte(pTok) == '#') {
+            if (getSubtoken(pTok) == '#') {
                 parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
                 parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
-                ltoken = PpAtomPaste;
+                subtoken = PpAtomPaste;
             } else
-                lUnreadByte(pTok);
+                ungetSubtoken(pTok);
         }
         break;
     case PpAtomConstString:
@@ -203,12 +199,12 @@ int TPpContext::ReadToken(TokenStream& pTok, TPpToken *ppToken)
     case PpAtomConstInt64:
     case PpAtomConstUint64:
         len = 0;
-        ch = lReadByte(pTok);
+        ch = getSubtoken(pTok);
         while (ch != 0 && ch != EndOfInput) {
             if (len < MaxTokenLength) {
                 ppToken->name[len] = (char)ch;
                 len++;
-                ch = lReadByte(pTok);
+                ch = getSubtoken(pTok);
             } else {
                 parseContext.error(ppToken->loc, "token too long", "", "");
                 break;
@@ -216,7 +212,7 @@ int TPpContext::ReadToken(TokenStream& pTok, TPpToken *ppToken)
         }
         ppToken->name[len] = 0;
 
-        switch (ltoken) {
+        switch (subtoken) {
         case PpAtomIdentifier:
             break;
         case PpAtomConstString:
@@ -267,7 +263,7 @@ int TPpContext::ReadToken(TokenStream& pTok, TPpToken *ppToken)
         }
     }
 
-    return ltoken;
+    return subtoken;
 }
 
 int TPpContext::tTokenInput::scan(TPpToken* ppToken)
@@ -285,14 +281,13 @@ bool TPpContext::tTokenInput::peekPasting()
     // 1. preceding ##?
 
     size_t savePos = tokens->current;
-    int byte;
+    int subtoken;
     // skip white space
     do {
-        byte = pp->lReadByte(*tokens);
+        subtoken = pp->getSubtoken(*tokens);
-    } while (byte == ' ');
+    } while (subtoken == ' ');
-    bool pasting = (byte == ((PpAtomPaste & 0x7f) + 0x80));
     tokens->current = savePos;
-    if (pasting)
+    if (subtoken == PpAtomPaste)
         return true;
 
     // 2. last token and we've been told after this there will be a ##
@@ -305,10 +300,10 @@ bool TPpContext::tTokenInput::peekPasting()
     savePos = tokens->current;
     bool moreTokens = false;
     do {
-        byte = pp->lReadByte(*tokens);
+        subtoken = pp->getSubtoken(*tokens);
-        if (byte == EndOfInput)
+        if (subtoken == EndOfInput)
             break;
-        if (byte != ' ') {
+        if (subtoken != ' ') {
             moreTokens = true;
             break;
         }

glslang/MachineIndependent/preprocessor/PpTokens.h

@@ -82,7 +82,11 @@ namespace glslang {
 
 // Multi-character tokens
 enum EFixedAtoms {
-    PpAtomMaxSingle = 256, // single character tokens get their own char value as their token, skip them
+    // single character tokens get their own char value as their token; start here for multi-character tokens
+    PpAtomMaxSingle = 127,
+
+    // replace bad character tokens with this, to avoid accidental aliasing with the below
+    PpAtomBadToken,
 
     // Operators