Change the token representation to take a Start and Length instead of a

Start/End pointer.

llvm-svn: 38548

clang/Driver/clang.cpp

@@ -626,7 +626,7 @@ void DoPrintPreprocessedInput(Preprocessor &PP) {
     }
     isFirstToken = false;    
     
-    if (Tok.getEnd()-Tok.getStart() < 256) {
+    if (Tok.getLength() < 256) {
       unsigned Len = Lexer::getSpelling(Tok, Buffer, PP.getLangOptions());
       Buffer[Len] = 0;
       std::cout << Buffer;

clang/Lex/Lexer.cpp

@@ -86,10 +86,10 @@ void LexerToken::dump(bool DumpFlags) const {
       // #define TWELVE 1\    <whitespace only>
       // 2
       // TWELVE
-      std::cerr << "*unspelled*" << std::string(Start, End);
+      std::cerr << "*unspelled*" << std::string(getStart(), getEnd());
     }
   } else
-    std::cerr << std::string(Start, End);
+    std::cerr << std::string(getStart(), getEnd());
   std::cerr << "'";
   
   if (DumpFlags) {
@@ -99,7 +99,7 @@ void LexerToken::dump(bool DumpFlags) const {
     if (hasLeadingSpace())
       std::cerr << " [LeadingSpace]";
     if (needsCleaning())
-      std::cerr << " [Spelling='" << std::string(Start, End) << "']";
+      std::cerr << " [Spelling='" << std::string(getStart(), getEnd()) << "']";
   }
 }
 
@@ -387,14 +387,14 @@ std::string Lexer::getSpelling(const LexerToken &Tok,
   
   // Otherwise, hard case, relex the characters into the string.
   std::string Result;
-  Result.reserve(Tok.getEnd()-Tok.getStart());
+  Result.reserve(Tok.getLength());
   
   for (const char *Ptr = Tok.getStart(), *End = Tok.getEnd(); Ptr != End; ) {
     unsigned CharSize;
     Result.push_back(getCharAndSizeNoWarn(Ptr, CharSize, Features));
     Ptr += CharSize;
   }
-  assert(Result.size() != unsigned(Tok.getEnd()-Tok.getStart()) &&
+  assert(Result.size() != unsigned(Tok.getLength()) &&
          "NeedsCleaning flag set on something that didn't need cleaning!");
   return Result;
 }
@@ -409,13 +409,13 @@ unsigned Lexer::getSpelling(const LexerToken &Tok, char *Buffer,
 
   // If this token contains nothing interesting, return it directly.
   if (!Tok.needsCleaning()) {
-    unsigned Size = Tok.getEnd()-Tok.getStart();
+    unsigned Size = Tok.getLength();
     memcpy(Buffer, Tok.getStart(), Size);
     return Size;
   }
   // Otherwise, hard case, relex the characters into the string.
   std::string Result;
-  Result.reserve(Tok.getEnd()-Tok.getStart());
+  Result.reserve(Tok.getLength());
   
   char *OutBuf = Buffer;
   for (const char *Ptr = Tok.getStart(), *End = Tok.getEnd(); Ptr != End; ) {
@@ -423,7 +423,7 @@ unsigned Lexer::getSpelling(const LexerToken &Tok, char *Buffer,
     *OutBuf++ = getCharAndSizeNoWarn(Ptr, CharSize, Features);
     Ptr += CharSize;
   }
-  assert(OutBuf-Buffer != Tok.getEnd()-Tok.getStart() &&
+  assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
          "NeedsCleaning flag set on something that didn't need cleaning!");
   
   return OutBuf-Buffer;
@@ -459,7 +459,7 @@ FinishIdentifier:
       SpelledTokEnd   = Result.getEnd();
     } else {
       // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
-      char *TmpBuf = (char*)alloca(Result.getEnd()-Result.getStart());
+      char *TmpBuf = (char*)alloca(Result.getLength());
       unsigned Size = getSpelling(Result, TmpBuf);
       SpelledTokStart = TmpBuf;
       SpelledTokEnd = TmpBuf+Size;

clang/include/clang/Lex/Lexer.h

@@ -54,7 +54,8 @@ struct LangOptions {
 /// compressed into a smaller form if memory footprint is important.
 class LexerToken {
   /// The start and end of the token text itself.
-  const char *Start, *End;
+  const char *Start;
+  unsigned Length;
   
   /// TheLexer - The lexer object this token came from.
   const Lexer *TheLexer;
@@ -84,9 +85,13 @@ public:
   void SetKind(tok::TokenKind K) { Kind = K; }
 
   const char *getStart() const { return Start; }
-  const char *getEnd() const { return End; }
+  const char *getEnd() const { return Start+Length; }
+  unsigned getLength() const { return Length; }
   void SetStart(const char *S) { Start = S; }
-  void SetEnd  (const char *E) { End = E; }
+  
+  /// SetEnd - Specify the length of the token as lexed.  This relies on the
+  /// start of the token having already been set.
+  void SetEnd(const char *End) { Length = End-Start; }
   
   const Lexer *getLexer() const { return TheLexer; }