fix: improve token types and add missing type guards (#1497)

.eslintignore

@@ -4,5 +4,6 @@ jest.config.js
 fixtures
 shared-fixtures
 coverage
+__snapshots__
 
 packages/eslint-plugin-tslint/tests

packages/eslint-plugin/src/rules/comma-spacing.ts

@@ -54,7 +54,7 @@ export default createRule<Options, MessageIds>({
   create(context, [{ before: spaceBefore, after: spaceAfter }]) {
     const sourceCode = context.getSourceCode();
     const tokensAndComments = sourceCode.tokensAndComments;
-    const ignoredTokens = new Set<TSESTree.Token>();
+    const ignoredTokens = new Set<TSESTree.PunctuatorToken>();
 
     /**
      * Adds null elements of the ArrayExpression or ArrayPattern node to the ignore list
@@ -67,7 +67,7 @@ export default createRule<Options, MessageIds>({
       for (const element of node.elements) {
         let token: TSESTree.Token | null;
         if (element === null) {
-          token = sourceCode.getTokenAfter(previousToken as TSESTree.Token);
+          token = sourceCode.getTokenAfter(previousToken!);
           if (token && isCommaToken(token)) {
             ignoredTokens.add(token);
           }
@@ -100,9 +100,9 @@ export default createRule<Options, MessageIds>({
      * @param nextToken The first token after the comma
      */
     function validateCommaSpacing(
-      commaToken: TSESTree.Token,
-      prevToken: TSESTree.Token | null,
-      nextToken: TSESTree.Token | null,
+      commaToken: TSESTree.PunctuatorToken,
+      prevToken: TSESTree.Token | TSESTree.Comment | null,
+      nextToken: TSESTree.Token | TSESTree.Comment | null,
     ): void {
       if (
         prevToken &&
@@ -166,20 +166,15 @@ export default createRule<Options, MessageIds>({
             return;
           }
 
-          if (token.type === AST_TOKEN_TYPES.JSXText) {
-            return;
-          }
-
-          const commaToken = token as TSESTree.Token;
-          const prevToken = tokensAndComments[i - 1] as TSESTree.Token;
-          const nextToken = tokensAndComments[i + 1] as TSESTree.Token;
+          const prevToken = tokensAndComments[i - 1];
+          const nextToken = tokensAndComments[i + 1];
 
           validateCommaSpacing(
-            commaToken,
-            isCommaToken(prevToken) || ignoredTokens.has(commaToken)
+            token,
+            isCommaToken(prevToken) || ignoredTokens.has(token)
               ? null
               : prevToken,
-            isCommaToken(nextToken) || ignoredTokens.has(commaToken)
+            isCommaToken(nextToken) || ignoredTokens.has(token)
               ? null
               : nextToken,
           );

packages/eslint-plugin/src/util/astUtils.ts

@@ -8,7 +8,7 @@ const LINEBREAK_MATCHER = /\r\n|[\r\n\u2028\u2029]/;
 
 function isOptionalChainPunctuator(
   token: TSESTree.Token | TSESTree.Comment,
-): boolean {
+): token is TSESTree.PunctuatorToken & { value: '?.' } {
   return token.type === AST_TOKEN_TYPES.Punctuator && token.value === '?.';
 }
 function isNotOptionalChainPunctuator(
@@ -19,7 +19,7 @@ function isNotOptionalChainPunctuator(
 
 function isNonNullAssertionPunctuator(
   token: TSESTree.Token | TSESTree.Comment,
-): boolean {
+): token is TSESTree.PunctuatorToken & { value: '!' } {
   return token.type === AST_TOKEN_TYPES.Punctuator && token.value === '!';
 }
 function isNotNonNullAssertionPunctuator(
@@ -33,7 +33,7 @@ function isNotNonNullAssertionPunctuator(
  */
 function isOptionalOptionalChain(
   node: TSESTree.Node,
-): node is TSESTree.OptionalCallExpression {
+): node is TSESTree.OptionalCallExpression & { optional: true } {
   return (
     node.type === AST_NODE_TYPES.OptionalCallExpression &&
     // this flag means the call expression itself is option
@@ -45,7 +45,9 @@ function isOptionalOptionalChain(
 /**
  * Returns true if and only if the node represents logical OR
  */
-function isLogicalOrOperator(node: TSESTree.Node): boolean {
+function isLogicalOrOperator(
+  node: TSESTree.Node,
+): node is TSESTree.LogicalExpression & { operator: '||' } {
   return (
     node.type === AST_NODE_TYPES.LogicalExpression && node.operator === '||'
   );

packages/eslint-plugin/typings/eslint-utils.d.ts

@@ -113,67 +113,67 @@ declare module 'eslint-utils' {
 
   export function isArrowToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: '=>' };
   export function isNotArrowToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;
   export function isClosingBraceToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: '}' };
   export function isNotClosingBraceToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;
   export function isClosingBracketToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: ']' };
   export function isNotClosingBracketToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;
   export function isClosingParenToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: ')' };
   export function isNotClosingParenToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;
   export function isColonToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: ':' };
   export function isNotColonToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;
   export function isCommaToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: ',' };
   export function isNotCommaToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;
   export function isCommentToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
-  export function isNotCommentToken(
-    token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.Comment;
+  export function isNotCommentToken<
+    T extends TSESTree.Token | TSESTree.Comment
+  >(token: T): token is Exclude<T, TSESTree.Comment>;
   export function isOpeningBraceToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: '{' };
   export function isNotOpeningBraceToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;
   export function isOpeningBracketToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: '[' };
   export function isNotOpeningBracketToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;
   export function isOpeningParenToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: '(' };
   export function isNotOpeningParenToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;
   export function isSemicolonToken(
     token: TSESTree.Token | TSESTree.Comment,
-  ): boolean;
+  ): token is TSESTree.PunctuatorToken & { value: ';' };
   export function isNotSemicolonToken(
     token: TSESTree.Token | TSESTree.Comment,
   ): boolean;

packages/experimental-utils/src/ts-eslint/SourceCode.ts

@@ -33,8 +33,8 @@ declare interface SourceCode {
   getNodeByRangeIndex(index: number): TSESTree.Node | null;
 
   isSpaceBetween(
-    first: TSESTree.Token | TSESTree.Node,
-    second: TSESTree.Token | TSESTree.Node,
+    first: TSESTree.Token | TSESTree.Comment | TSESTree.Node,
+    second: TSESTree.Token | TSESTree.Comment | TSESTree.Node,
   ): boolean;
 
   /**
@@ -49,93 +49,90 @@ declare interface SourceCode {
   // Inherited methods from TokenStore
   // ---------------------------------
 
-  getTokenByRangeStart(
+  getTokenByRangeStart<T extends { includeComments?: boolean }>(
     offset: number,
-    options?: { includeComments?: boolean },
-  ): TSESTree.Token | null;
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T> | null;
 
-  getFirstToken(
+  getFirstToken<T extends SourceCode.CursorWithSkipOptions>(
     node: TSESTree.Node,
-    options?: SourceCode.CursorWithSkipOptions,
-  ): TSESTree.Token | null;
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T> | null;
 
-  getFirstTokens(
+  getFirstTokens<T extends SourceCode.CursorWithCountOptions>(
     node: TSESTree.Node,
-    options?: SourceCode.CursorWithCountOptions,
-  ): TSESTree.Token[];
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T>[];
 
-  getLastToken(
+  getLastToken<T extends SourceCode.CursorWithSkipOptions>(
     node: TSESTree.Node,
-    options?: SourceCode.CursorWithSkipOptions,
-  ): TSESTree.Token | null;
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T> | null;
 
-  getLastTokens(
+  getLastTokens<T extends SourceCode.CursorWithCountOptions>(
     node: TSESTree.Node,
-    options?: SourceCode.CursorWithCountOptions,
-  ): TSESTree.Token[];
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T>[];
 
-  getTokenBefore(
+  getTokenBefore<T extends SourceCode.CursorWithSkipOptions>(
     node: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
-    options?: SourceCode.CursorWithSkipOptions,
-  ): TSESTree.Token | null;
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T> | null;
 
-  getTokensBefore(
+  getTokensBefore<T extends SourceCode.CursorWithCountOptions>(
     node: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
-    options?: SourceCode.CursorWithCountOptions,
-  ): TSESTree.Token[];
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T>[];
 
-  getTokenAfter(
+  getTokenAfter<T extends SourceCode.CursorWithSkipOptions>(
     node: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
-    options?: SourceCode.CursorWithSkipOptions,
-  ): TSESTree.Token | null;
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T> | null;
 
-  getTokensAfter(
+  getTokensAfter<T extends SourceCode.CursorWithCountOptions>(
     node: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
-    options?: SourceCode.CursorWithCountOptions,
-  ): TSESTree.Token[];
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T>[];
 
-  getFirstTokenBetween(
+  getFirstTokenBetween<T extends SourceCode.CursorWithSkipOptions>(
     left: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
     right: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
-    options?: SourceCode.CursorWithSkipOptions,
-  ): TSESTree.Token | null;
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T> | null;
 
-  getFirstTokensBetween(
+  getFirstTokensBetween<T extends SourceCode.CursorWithCountOptions>(
     left: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
     right: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
-    options?: SourceCode.CursorWithCountOptions,
-  ): TSESTree.Token[];
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T>[];
 
-  getLastTokenBetween(
+  getLastTokenBetween<T extends SourceCode.CursorWithSkipOptions>(
     left: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
     right: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
-    options?: SourceCode.CursorWithSkipOptions,
-  ): TSESTree.Token | null;
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T> | null;
 
-  getLastTokensBetween(
+  getLastTokensBetween<T extends SourceCode.CursorWithCountOptions>(
     left: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
     right: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
-    options?: SourceCode.CursorWithCountOptions,
-  ): TSESTree.Token[];
+    options?: T,
+  ): SourceCode.ReturnTypeFromOptions<T>[];
 
-  getTokensBetween(
+  getTokensBetween<T extends SourceCode.CursorWithCountOptions>(
     left: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
     right: TSESTree.Node | TSESTree.Token | TSESTree.Comment,
-    padding?:
-      | number
-      | SourceCode.FilterPredicate
-      | SourceCode.CursorWithCountOptions,
-  ): TSESTree.Token[];
+    padding?: T,
+  ): SourceCode.ReturnTypeFromOptions<T>[];
 
   getTokens(
     node: TSESTree.Node,
     beforeCount?: number,
     afterCount?: number,
   ): TSESTree.Token[];
-  getTokens(
+  getTokens<T extends SourceCode.CursorWithCountOptions>(
     node: TSESTree.Node,
-    options: SourceCode.FilterPredicate | SourceCode.CursorWithCountOptions,
-  ): TSESTree.Token[];
+    options: T,
+  ): SourceCode.ReturnTypeFromOptions<T>[];
 
   commentsExistBetween(
     left: TSESTree.Node | TSESTree.Token,
@@ -175,6 +172,10 @@ namespace SourceCode {
     tokenOrComment: TSESTree.Token | TSESTree.Comment,
   ) => boolean;
 
+  export type ReturnTypeFromOptions<T> = T extends { includeComments: true }
+    ? TSESTree.Token | TSESTree.Comment
+    : TSESTree.Token;
+
   export type CursorWithSkipOptions =
     | number
     | FilterPredicate

packages/typescript-estree/src/convert-comments.ts

@@ -1,7 +1,7 @@
 import * as ts from 'typescript';
 import { forEachComment } from 'tsutils/util/util';
 import { getLocFor } from './node-utils';
-import { TSESTree } from './ts-estree';
+import { AST_TOKEN_TYPES, TSESTree } from './ts-estree';
 
 /**
  * Convert all comments for the given AST.
@@ -21,8 +21,8 @@ export function convertComments(
     (_, comment) => {
       const type =
         comment.kind == ts.SyntaxKind.SingleLineCommentTrivia
-          ? 'Line'
-          : 'Block';
+          ? AST_TOKEN_TYPES.Line
+          : AST_TOKEN_TYPES.Block;
       const range: TSESTree.Range = [comment.pos, comment.end];
       const loc = getLocFor(range[0], range[1], ast);
 

packages/typescript-estree/src/node-utils.ts

@@ -455,7 +455,7 @@ export function isOptional(node: {
  */
 export function getTokenType(
   token: ts.Identifier | ts.Token<ts.SyntaxKind>,
-): AST_TOKEN_TYPES {
+): Exclude<AST_TOKEN_TYPES, AST_TOKEN_TYPES.Line | AST_TOKEN_TYPES.Block> {
   if ('originalKeywordKind' in token && token.originalKeywordKind) {
     if (token.originalKeywordKind === SyntaxKind.NullKeyword) {
       return AST_TOKEN_TYPES.Null;
@@ -561,21 +561,27 @@ export function convertToken(
       : token.getStart(ast);
   const end = token.getEnd();
   const value = ast.text.slice(start, end);
-  const newToken: TSESTree.Token = {
-    type: getTokenType(token),
-    value,
-    range: [start, end],
-    loc: getLocFor(start, end, ast),
-  };
+  const tokenType = getTokenType(token);
 
-  if (newToken.type === AST_TOKEN_TYPES.RegularExpression) {
-    newToken.regex = {
-      pattern: value.slice(1, value.lastIndexOf('/')),
-      flags: value.slice(value.lastIndexOf('/') + 1),
+  if (tokenType === AST_TOKEN_TYPES.RegularExpression) {
+    return {
+      type: tokenType,
+      value,
+      range: [start, end],
+      loc: getLocFor(start, end, ast),
+      regex: {
+        pattern: value.slice(1, value.lastIndexOf('/')),
+        flags: value.slice(value.lastIndexOf('/') + 1),
+      },
+    };
+  } else {
+    return {
+      type: tokenType,
+      value,
+      range: [start, end],
+      loc: getLocFor(start, end, ast),
     };
   }
-
-  return newToken;
 }
 
 /**

packages/typescript-estree/src/ts-estree/ts-estree.ts

@@ -52,20 +52,80 @@ export interface BaseNode {
  * They are not included in the `Node` union below on purpose because they
  * are not ever included as part of the standard AST tree.
  */
-
-export interface Token extends BaseNode {
-  type: AST_TOKEN_TYPES;
+interface BaseToken extends BaseNode {
   value: string;
-  regex?: {
+}
+
+export interface BooleanToken extends BaseToken {
+  type: AST_TOKEN_TYPES.Boolean;
+}
+
+export interface IdentifierToken extends BaseToken {
+  type: AST_TOKEN_TYPES.Identifier;
+}
+
+export interface JSXIdentifierToken extends BaseToken {
+  type: AST_TOKEN_TYPES.JSXIdentifier;
+}
+
+export interface JSXTextToken extends BaseToken {
+  type: AST_TOKEN_TYPES.JSXText;
+}
+
+export interface KeywordToken extends BaseToken {
+  type: AST_TOKEN_TYPES.Keyword;
+}
+
+export interface NullToken extends BaseToken {
+  type: AST_TOKEN_TYPES.Null;
+}
+
+export interface NumericToken extends BaseToken {
+  type: AST_TOKEN_TYPES.Numeric;
+}
+
+export interface PunctuatorToken extends BaseToken {
+  type: AST_TOKEN_TYPES.Punctuator;
+}
+
+export interface RegularExpressionToken extends BaseToken {
+  type: AST_TOKEN_TYPES.RegularExpression;
+  regex: {
     pattern: string;
     flags: string;
   };
 }
-export interface Comment extends BaseNode {
-  type: 'Line' | 'Block';
-  value: string;
+
+export interface StringToken extends BaseToken {
+  type: AST_TOKEN_TYPES.String;
 }
 
+export interface TemplateToken extends BaseToken {
+  type: AST_TOKEN_TYPES.Template;
+}
+
+export interface BlockComment extends BaseToken {
+  type: AST_TOKEN_TYPES.Block;
+}
+
+export interface LineComment extends BaseToken {
+  type: AST_TOKEN_TYPES.Line;
+}
+
+export type Comment = BlockComment | LineComment;
+export type Token =
+  | BooleanToken
+  | IdentifierToken
+  | JSXIdentifierToken
+  | JSXTextToken
+  | KeywordToken
+  | NullToken
+  | NumericToken
+  | PunctuatorToken
+  | RegularExpressionToken
+  | StringToken
+  | TemplateToken;
+
 export type OptionalRangeAndLoc<T> = Pick<
   T,
   Exclude<keyof T, 'range' | 'loc'>