// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;

namespace Microsoft.AspNetCore.Razor.Language.Legacy
{
    internal class CSharpTokenizer : Tokenizer
    {
        private readonly Dictionary<char, Func<SyntaxKind>> _operatorHandlers;

        private static readonly Dictionary<string, CSharpKeyword> _keywords = new Dictionary<string, CSharpKeyword>(StringComparer.Ordinal)
        {
            { "await", CSharpKeyword.Await },
            { "abstract", CSharpKeyword.Abstract },
            { "byte", CSharpKeyword.Byte },
            { "class", CSharpKeyword.Class },
            { "delegate", CSharpKeyword.Delegate },
            { "event", CSharpKeyword.Event },
            { "fixed", CSharpKeyword.Fixed },
            { "if", CSharpKeyword.If },
            { "internal", CSharpKeyword.Internal },
            { "new", CSharpKeyword.New },
            { "override", CSharpKeyword.Override },
            { "readonly", CSharpKeyword.Readonly },
            { "short", CSharpKeyword.Short },
            { "struct", CSharpKeyword.Struct },
            { "try", CSharpKeyword.Try },
            { "unsafe", CSharpKeyword.Unsafe },
            { "volatile", CSharpKeyword.Volatile },
            { "as", CSharpKeyword.As },
            { "do", CSharpKeyword.Do },
            { "is", CSharpKeyword.Is },
            { "params", CSharpKeyword.Params },
            { "ref", CSharpKeyword.Ref },
            { "switch", CSharpKeyword.Switch },
            { "ushort", CSharpKeyword.Ushort },
            { "while", CSharpKeyword.While },
            { "case", CSharpKeyword.Case },
            { "const", CSharpKeyword.Const },
            { "explicit", CSharpKeyword.Explicit },
            { "float", CSharpKeyword.Float },
            { "null", CSharpKeyword.Null },
            { "sizeof", CSharpKeyword.Sizeof },
            { "typeof", CSharpKeyword.Typeof },
            { "implicit", CSharpKeyword.Implicit },
            { "private", CSharpKeyword.Private },
            { "this", CSharpKeyword.This },
            { "using", CSharpKeyword.Using },
            { "extern", CSharpKeyword.Extern },
            { "return", CSharpKeyword.Return },
            { "stackalloc", CSharpKeyword.Stackalloc },
            { "uint", CSharpKeyword.Uint },
            { "base", CSharpKeyword.Base },
            { "catch", CSharpKeyword.Catch },
            { "continue", CSharpKeyword.Continue },
            { "double", CSharpKeyword.Double },
            { "for", CSharpKeyword.For },
            { "in", CSharpKeyword.In },
            { "lock", CSharpKeyword.Lock },
            { "object", CSharpKeyword.Object },
            { "protected", CSharpKeyword.Protected },
            { "static", CSharpKeyword.Static },
            { "false", CSharpKeyword.False },
            { "public", CSharpKeyword.Public },
            { "sbyte", CSharpKeyword.Sbyte },
            { "throw", CSharpKeyword.Throw },
            { "virtual", CSharpKeyword.Virtual },
            { "decimal", CSharpKeyword.Decimal },
            { "else", CSharpKeyword.Else },
            { "operator", CSharpKeyword.Operator },
            { "string", CSharpKeyword.String },
            { "ulong", CSharpKeyword.Ulong },
            { "bool", CSharpKeyword.Bool },
            { "char", CSharpKeyword.Char },
            { "default", CSharpKeyword.Default },
            { "foreach", CSharpKeyword.Foreach },
            { "long", CSharpKeyword.Long },
            { "void", CSharpKeyword.Void },
            { "enum", CSharpKeyword.Enum },
            { "finally", CSharpKeyword.Finally },
            { "int", CSharpKeyword.Int },
            { "out", CSharpKeyword.Out },
            { "sealed", CSharpKeyword.Sealed },
            { "true", CSharpKeyword.True },
            { "goto", CSharpKeyword.Goto },
            { "unchecked", CSharpKeyword.Unchecked },
            { "interface", CSharpKeyword.Interface },
            { "break", CSharpKeyword.Break },
            { "checked", CSharpKeyword.Checked },
            { "namespace", CSharpKeyword.Namespace },
            { "when", CSharpKeyword.When },
            { "where", CSharpKeyword.Where }
        };

        public CSharpTokenizer(ITextDocument source)
            : base(source)
        {
            base.CurrentState = StartState;

            _operatorHandlers = new Dictionary<char, Func<SyntaxKind>>()
            {
                { '-', MinusOperator },
                { '<', LessThanOperator },
                { '>', GreaterThanOperator },
                { '&', CreateTwoCharOperatorHandler(SyntaxKind.And, '=', SyntaxKind.AndAssign, '&', SyntaxKind.DoubleAnd) },
                { '|', CreateTwoCharOperatorHandler(SyntaxKind.Or, '=', SyntaxKind.OrAssign, '|', SyntaxKind.DoubleOr) },
                { '+', CreateTwoCharOperatorHandler(SyntaxKind.Plus, '=', SyntaxKind.PlusAssign, '+', SyntaxKind.Increment) },
                { '=', CreateTwoCharOperatorHandler(SyntaxKind.Assign, '=', SyntaxKind.Equals, '>', SyntaxKind.GreaterThanEqual) },
                { '!', CreateTwoCharOperatorHandler(SyntaxKind.Not, '=', SyntaxKind.NotEqual) },
                { '%', CreateTwoCharOperatorHandler(SyntaxKind.Modulo, '=', SyntaxKind.ModuloAssign) },
                { '*', CreateTwoCharOperatorHandler(SyntaxKind.Star, '=', SyntaxKind.MultiplyAssign) },
                { ':', CreateTwoCharOperatorHandler(SyntaxKind.Colon, ':', SyntaxKind.DoubleColon) },
                { '?', CreateTwoCharOperatorHandler(SyntaxKind.QuestionMark, '?', SyntaxKind.NullCoalesce) },
                { '^', CreateTwoCharOperatorHandler(SyntaxKind.Xor, '=', SyntaxKind.XorAssign) },
                { '(', () => SyntaxKind.LeftParenthesis },
                { ')', () => SyntaxKind.RightParenthesis },
                { '{', () => SyntaxKind.LeftBrace },
                { '}', () => SyntaxKind.RightBrace },
                { '[', () => SyntaxKind.LeftBracket },
                { ']', () => SyntaxKind.RightBracket },
                { ',', () => SyntaxKind.Comma },
                { ';', () => SyntaxKind.Semicolon },
                { '~', () => SyntaxKind.Tilde },
                { '#', () => SyntaxKind.Hash }
            };
        }

        protected override int StartState => (int)CSharpTokenizerState.Data;

        private new CSharpTokenizerState? CurrentState => (CSharpTokenizerState?)base.CurrentState;

        public override SyntaxKind RazorCommentKind => SyntaxKind.RazorCommentLiteral;

        public override SyntaxKind RazorCommentTransitionKind => SyntaxKind.RazorCommentTransition;

        public override SyntaxKind RazorCommentStarKind => SyntaxKind.RazorCommentStar;

        protected override StateResult Dispatch()
        {
            switch (CurrentState)
            {
                case CSharpTokenizerState.Data:
                    return Data();
                case CSharpTokenizerState.BlockComment:
                    return BlockComment();
                case CSharpTokenizerState.QuotedCharacterLiteral:
                    return QuotedCharacterLiteral();
                case CSharpTokenizerState.QuotedStringLiteral:
                    return QuotedStringLiteral();
                case CSharpTokenizerState.VerbatimStringLiteral:
                    return VerbatimStringLiteral();
                case CSharpTokenizerState.AfterRazorCommentTransition:
                    return AfterRazorCommentTransition();
                case CSharpTokenizerState.EscapedRazorCommentTransition:
                    return EscapedRazorCommentTransition();
                case CSharpTokenizerState.RazorCommentBody:
                    return RazorCommentBody();
                case CSharpTokenizerState.StarAfterRazorCommentBody:
                    return StarAfterRazorCommentBody();
                case CSharpTokenizerState.AtTokenAfterRazorCommentBody:
                    return AtTokenAfterRazorCommentBody();
                default:
                    Debug.Fail("Invalid TokenizerState");
                    return default(StateResult);
            }
        }

        // Optimize memory allocation by returning constants for the most frequent cases
        protected override string GetTokenContent(SyntaxKind type)
        {
            var tokenLength = Buffer.Length;

            if (tokenLength == 1)
            {
                switch (type)
                {
                    case SyntaxKind.IntegerLiteral:
                        switch (Buffer[0])
                        {
                            case '0':
                                return "0";
                            case '1':
                                return "1";
                            case '2':
                                return "2";
                            case '3':
                                return "3";
                            case '4':
                                return "4";
                            case '5':
                                return "5";
                            case '6':
                                return "6";
                            case '7':
                                return "7";
                            case '8':
                                return "8";
                            case '9':
                                return "9";
                        }
                        break;
                    case SyntaxKind.NewLine:
                        if (Buffer[0] == '\n')
                        {
                            return "\n";
                        }
                        break;
                    case SyntaxKind.Whitespace:
                        if (Buffer[0] == ' ')
                        {
                            return " ";
                        }
                        if (Buffer[0] == '\t')
                        {
                            return "\t";
                        }
                        break;
                    case SyntaxKind.Minus:
                        return "-";
                    case SyntaxKind.Not:
                        return "!";
                    case SyntaxKind.Modulo:
                        return "%";
                    case SyntaxKind.And:
                        return "&";
                    case SyntaxKind.LeftParenthesis:
                        return "(";
                    case SyntaxKind.RightParenthesis:
                        return ")";
                    case SyntaxKind.Star:
                        return "*";
                    case SyntaxKind.Comma:
                        return ",";
                    case SyntaxKind.Dot:
                        return ".";
                    case SyntaxKind.Slash:
                        return "/";
                    case SyntaxKind.Colon:
                        return ":";
                    case SyntaxKind.Semicolon:
                        return ";";
                    case SyntaxKind.QuestionMark:
                        return "?";
                    case SyntaxKind.RightBracket:
                        return "]";
                    case SyntaxKind.LeftBracket:
                        return "[";
                    case SyntaxKind.Xor:
                        return "^";
                    case SyntaxKind.LeftBrace:
                        return "{";
                    case SyntaxKind.Or:
                        return "|";
                    case SyntaxKind.RightBrace:
                        return "}";
                    case SyntaxKind.Tilde:
                        return "~";
                    case SyntaxKind.Plus:
                        return "+";
                    case SyntaxKind.LessThan:
                        return "<";
                    case SyntaxKind.Assign:
                        return "=";
                    case SyntaxKind.GreaterThan:
                        return ">";
                    case SyntaxKind.Hash:
                        return "#";
                    case SyntaxKind.Transition:
                        return "@";

                }
            }
            else if (tokenLength == 2)
            {
                switch (type)
                {
                    case SyntaxKind.NewLine:
                        return "\r\n";
                    case SyntaxKind.Arrow:
                        return "->";
                    case SyntaxKind.Decrement:
                        return "--";
                    case SyntaxKind.MinusAssign:
                        return "-=";
                    case SyntaxKind.NotEqual:
                        return "!=";
                    case SyntaxKind.ModuloAssign:
                        return "%=";
                    case SyntaxKind.AndAssign:
                        return "&=";
                    case SyntaxKind.DoubleAnd:
                        return "&&";
                    case SyntaxKind.MultiplyAssign:
                        return "*=";
                    case SyntaxKind.DivideAssign:
                        return "/=";
                    case SyntaxKind.DoubleColon:
                        return "::";
                    case SyntaxKind.NullCoalesce:
                        return "??";
                    case SyntaxKind.XorAssign:
                        return "^=";
                    case SyntaxKind.OrAssign:
                        return "|=";
                    case SyntaxKind.DoubleOr:
                        return "||";
                    case SyntaxKind.PlusAssign:
                        return "+=";
                    case SyntaxKind.Increment:
                        return "++";
                    case SyntaxKind.LessThanEqual:
                        return "<=";
                    case SyntaxKind.LeftShift:
                        return "<<";
                    case SyntaxKind.Equals:
                        return "==";
                    case SyntaxKind.GreaterThanEqual:
                        if (Buffer[0] == '=')
                        {
                            return "=>";
                        }
                        return ">=";
                    case SyntaxKind.RightShift:
                        return ">>";


                }
            }
            else if (tokenLength == 3)
            {
                switch (type)
                {
                    case SyntaxKind.LeftShiftAssign:
                        return "<<=";
                    case SyntaxKind.RightShiftAssign:
                        return ">>=";
                }
            }

            return base.GetTokenContent(type);
        }

        protected override SyntaxToken CreateToken(string content, SyntaxKind kind, RazorDiagnostic [] errors)
        {
            return SyntaxFactory.Token(kind, content, errors);
        }

        private StateResult Data()
        {
            if (ParserHelpers.IsNewLine(CurrentCharacter))
            {
                // CSharp Spec §2.3.1
                var checkTwoCharNewline = CurrentCharacter == '\r';
                TakeCurrent();
                if (checkTwoCharNewline && CurrentCharacter == '\n')
                {
                    TakeCurrent();
                }
                return Stay(EndToken(SyntaxKind.NewLine));
            }
            else if (ParserHelpers.IsWhitespace(CurrentCharacter))
            {
                // CSharp Spec §2.3.3
                TakeUntil(c => !ParserHelpers.IsWhitespace(c));
                return Stay(EndToken(SyntaxKind.Whitespace));
            }
            else if (IsIdentifierStart(CurrentCharacter))
            {
                return Identifier();
            }
            else if (char.IsDigit(CurrentCharacter))
            {
                return NumericLiteral();
            }
            switch (CurrentCharacter)
            {
                case '@':
                    return AtToken();
                case '\'':
                    TakeCurrent();
                    return Transition(CSharpTokenizerState.QuotedCharacterLiteral);
                case '"':
                    TakeCurrent();
                    return Transition(CSharpTokenizerState.QuotedStringLiteral);
                case '.':
                    if (char.IsDigit(Peek()))
                    {
                        return RealLiteral();
                    }
                    return Stay(Single(SyntaxKind.Dot));
                case '/':
                    TakeCurrent();
                    if (CurrentCharacter == '/')
                    {
                        TakeCurrent();
                        return SingleLineComment();
                    }
                    else if (CurrentCharacter == '*')
                    {
                        TakeCurrent();
                        return Transition(CSharpTokenizerState.BlockComment);
                    }
                    else if (CurrentCharacter == '=')
                    {
                        TakeCurrent();
                        return Stay(EndToken(SyntaxKind.DivideAssign));
                    }
                    else
                    {
                        return Stay(EndToken(SyntaxKind.Slash));
                    }
                default:
                    return Stay(EndToken(Operator()));
            }
        }

        private StateResult AtToken()
        {
            TakeCurrent();
            if (CurrentCharacter == '"')
            {
                TakeCurrent();
                return Transition(CSharpTokenizerState.VerbatimStringLiteral);
            }
            else if (CurrentCharacter == '*')
            {
                return Transition(
                    CSharpTokenizerState.AfterRazorCommentTransition,
                    EndToken(SyntaxKind.RazorCommentTransition));
            }
            else if (CurrentCharacter == '@')
            {
                // Could be escaped comment transition
                return Transition(
                    CSharpTokenizerState.EscapedRazorCommentTransition,
                    EndToken(SyntaxKind.Transition));
            }

            return Stay(EndToken(SyntaxKind.Transition));
        }

        private StateResult EscapedRazorCommentTransition()
        {
            TakeCurrent();
            return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.Transition));
        }

        private SyntaxKind Operator()
        {
            var first = CurrentCharacter;
            TakeCurrent();
            Func<SyntaxKind> handler;
            if (_operatorHandlers.TryGetValue(first, out handler))
            {
                return handler();
            }
            return SyntaxKind.Marker;
        }

        private SyntaxKind LessThanOperator()
        {
            if (CurrentCharacter == '=')
            {
                TakeCurrent();
                return SyntaxKind.LessThanEqual;
            }
            return SyntaxKind.LessThan;
        }

        private SyntaxKind GreaterThanOperator()
        {
            if (CurrentCharacter == '=')
            {
                TakeCurrent();
                return SyntaxKind.GreaterThanEqual;
            }
            return SyntaxKind.GreaterThan;
        }

        private SyntaxKind MinusOperator()
        {
            if (CurrentCharacter == '>')
            {
                TakeCurrent();
                return SyntaxKind.Arrow;
            }
            else if (CurrentCharacter == '-')
            {
                TakeCurrent();
                return SyntaxKind.Decrement;
            }
            else if (CurrentCharacter == '=')
            {
                TakeCurrent();
                return SyntaxKind.MinusAssign;
            }
            return SyntaxKind.Minus;
        }

        private Func<SyntaxKind> CreateTwoCharOperatorHandler(SyntaxKind typeIfOnlyFirst, char second, SyntaxKind typeIfBoth)
        {
            return () =>
            {
                if (CurrentCharacter == second)
                {
                    TakeCurrent();
                    return typeIfBoth;
                }
                return typeIfOnlyFirst;
            };
        }

        private Func<SyntaxKind> CreateTwoCharOperatorHandler(SyntaxKind typeIfOnlyFirst, char option1, SyntaxKind typeIfOption1, char option2, SyntaxKind typeIfOption2)
        {
            return () =>
            {
                if (CurrentCharacter == option1)
                {
                    TakeCurrent();
                    return typeIfOption1;
                }
                else if (CurrentCharacter == option2)
                {
                    TakeCurrent();
                    return typeIfOption2;
                }
                return typeIfOnlyFirst;
            };
        }

        private StateResult VerbatimStringLiteral()
        {
            TakeUntil(c => c == '"');
            if (CurrentCharacter == '"')
            {
                TakeCurrent();
                if (CurrentCharacter == '"')
                {
                    TakeCurrent();
                    // Stay in the literal, this is an escaped "
                    return Stay();
                }
            }
            else if (EndOfFile)
            {
                CurrentErrors.Add(
                    RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral(
                        new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
            }
            return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.StringLiteral));
        }

        private StateResult QuotedCharacterLiteral() => QuotedLiteral('\'', IsEndQuotedCharacterLiteral, SyntaxKind.CharacterLiteral);

        private StateResult QuotedStringLiteral() => QuotedLiteral('\"', IsEndQuotedStringLiteral, SyntaxKind.StringLiteral);

        private readonly Func<char, bool> IsEndQuotedCharacterLiteral = (c) => c == '\\' || c == '\'' || ParserHelpers.IsNewLine(c);
        private readonly Func<char, bool> IsEndQuotedStringLiteral = (c) => c == '\\' || c == '\"' || ParserHelpers.IsNewLine(c);

        private StateResult QuotedLiteral(char quote, Func<char, bool> isEndQuotedLiteral, SyntaxKind literalType)
        {
            TakeUntil(isEndQuotedLiteral);
            if (CurrentCharacter == '\\')
            {
                TakeCurrent(); // Take the '\'

                // If the next char is the same quote that started this
                if (CurrentCharacter == quote || CurrentCharacter == '\\')
                {
                    TakeCurrent(); // Take it so that we don't prematurely end the literal.
                }
                return Stay();
            }
            else if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter))
            {
                CurrentErrors.Add(
                    RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral(
                        new SourceSpan(CurrentStart, contentLength: 1 /* " */)));
            }
            else
            {
                TakeCurrent(); // No-op if at EOF
            }
            return Transition(CSharpTokenizerState.Data, EndToken(literalType));
        }

        // CSharp Spec §2.3.2
        private StateResult BlockComment()
        {
            TakeUntil(c => c == '*');
            if (EndOfFile)
            {
                CurrentErrors.Add(
                    RazorDiagnosticFactory.CreateParsing_BlockCommentNotTerminated(
                        new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));

                return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.CSharpComment));
            }
            if (CurrentCharacter == '*')
            {
                TakeCurrent();
                if (CurrentCharacter == '/')
                {
                    TakeCurrent();
                    return Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.CSharpComment));
                }
            }
            return Stay();
        }

        // CSharp Spec §2.3.2
        private StateResult SingleLineComment()
        {
            TakeUntil(c => ParserHelpers.IsNewLine(c));
            return Stay(EndToken(SyntaxKind.CSharpComment));
        }

        // CSharp Spec §2.4.4
        private StateResult NumericLiteral()
        {
            if (TakeAll("0x", caseSensitive: true))
            {
                return HexLiteral();
            }
            else
            {
                return DecimalLiteral();
            }
        }

        private StateResult HexLiteral()
        {
            TakeUntil(c => !IsHexDigit(c));
            TakeIntegerSuffix();
            return Stay(EndToken(SyntaxKind.IntegerLiteral));
        }

        private StateResult DecimalLiteral()
        {
            TakeUntil(c => !Char.IsDigit(c));
            if (CurrentCharacter == '.' && Char.IsDigit(Peek()))
            {
                return RealLiteral();
            }
            else if (IsRealLiteralSuffix(CurrentCharacter) ||
                     CurrentCharacter == 'E' || CurrentCharacter == 'e')
            {
                return RealLiteralExponentPart();
            }
            else
            {
                TakeIntegerSuffix();
                return Stay(EndToken(SyntaxKind.IntegerLiteral));
            }
        }

        private StateResult RealLiteralExponentPart()
        {
            if (CurrentCharacter == 'E' || CurrentCharacter == 'e')
            {
                TakeCurrent();
                if (CurrentCharacter == '+' || CurrentCharacter == '-')
                {
                    TakeCurrent();
                }
                TakeUntil(c => !Char.IsDigit(c));
            }
            if (IsRealLiteralSuffix(CurrentCharacter))
            {
                TakeCurrent();
            }
            return Stay(EndToken(SyntaxKind.RealLiteral));
        }

        // CSharp Spec §2.4.4.3
        private StateResult RealLiteral()
        {
            AssertCurrent('.');
            TakeCurrent();
            Debug.Assert(Char.IsDigit(CurrentCharacter));
            TakeUntil(c => !Char.IsDigit(c));
            return RealLiteralExponentPart();
        }

        private void TakeIntegerSuffix()
        {
            if (Char.ToLowerInvariant(CurrentCharacter) == 'u')
            {
                TakeCurrent();
                if (Char.ToLowerInvariant(CurrentCharacter) == 'l')
                {
                    TakeCurrent();
                }
            }
            else if (Char.ToLowerInvariant(CurrentCharacter) == 'l')
            {
                TakeCurrent();
                if (Char.ToLowerInvariant(CurrentCharacter) == 'u')
                {
                    TakeCurrent();
                }
            }
        }

        // CSharp Spec §2.4.2
        private StateResult Identifier()
        {
            Debug.Assert(IsIdentifierStart(CurrentCharacter));
            TakeCurrent();
            TakeUntil(c => !IsIdentifierPart(c));
            SyntaxToken token = null;
            if (HaveContent)
            {
                CSharpKeyword keyword;
                var type = SyntaxKind.Identifier;
                var tokenContent = Buffer.ToString();
                if (_keywords.TryGetValue(tokenContent, out keyword))
                {
                    type = SyntaxKind.Keyword;
                }

                token = SyntaxFactory.Token(type, tokenContent);

                Buffer.Clear();
                CurrentErrors.Clear();
            }

            return Stay(token);
        }

        private StateResult Transition(CSharpTokenizerState state)
        {
            return Transition((int)state, result: null);
        }

        private StateResult Transition(CSharpTokenizerState state, SyntaxToken result)
        {
            return Transition((int)state, result);
        }

        private static bool IsIdentifierStart(char character)
        {
            return char.IsLetter(character) ||
                   character == '_' ||
                   CharUnicodeInfo.GetUnicodeCategory(character) == UnicodeCategory.LetterNumber;
        }

        private static bool IsIdentifierPart(char character)
        {
            return char.IsDigit(character) ||
                   IsIdentifierStart(character) ||
                   IsIdentifierPartByUnicodeCategory(character);
        }

        private static bool IsRealLiteralSuffix(char character)
        {
            return character == 'F' ||
                   character == 'f' ||
                   character == 'D' ||
                   character == 'd' ||
                   character == 'M' ||
                   character == 'm';
        }

        private static bool IsIdentifierPartByUnicodeCategory(char character)
        {
            var category = CharUnicodeInfo.GetUnicodeCategory(character);

            return category == UnicodeCategory.NonSpacingMark || // Mn
                   category == UnicodeCategory.SpacingCombiningMark || // Mc
                   category == UnicodeCategory.ConnectorPunctuation || // Pc
                   category == UnicodeCategory.Format; // Cf
        }

        private static bool IsHexDigit(char value)
        {
            return (value >= '0' && value <= '9') || (value >= 'A' && value <= 'F') || (value >= 'a' && value <= 'f');
        }

        internal static CSharpKeyword? GetTokenKeyword(SyntaxToken token)
        {
            if (token != null && _keywords.TryGetValue(token.Content, out var keyword))
            {
                return keyword;
            }

            return null;
        }

        private enum CSharpTokenizerState
        {
            Data,
            BlockComment,
            QuotedCharacterLiteral,
            QuotedStringLiteral,
            VerbatimStringLiteral,

            // Razor Comments - need to be the same for HTML and CSharp
            AfterRazorCommentTransition = RazorCommentTokenizerState.AfterRazorCommentTransition,
            EscapedRazorCommentTransition = RazorCommentTokenizerState.EscapedRazorCommentTransition,
            RazorCommentBody = RazorCommentTokenizerState.RazorCommentBody,
            StarAfterRazorCommentBody = RazorCommentTokenizerState.StarAfterRazorCommentBody,
            AtTokenAfterRazorCommentBody = RazorCommentTokenizerState.AtTokenAfterRazorCommentBody,
        }
    }
}
