tokenize.py File Reference

Go to the source code of this file.

Classes
class	cpp.tokenize.Token

Namespaces
namespace	cpp.tokenize

Functions
def	cpp.tokenize._GetString
def	cpp.tokenize._GetChar
def	cpp.tokenize.GetTokens
def	cpp.tokenize.main

Variables
string	cpp.tokenize.__author__ 'nnorwitz@google.com (Neal Norwitz)'
string	cpp.tokenize._letters 'abcdefghijklmnopqrstuvwxyz'
tuple	cpp.tokenize.VALID_IDENTIFIER_CHARS set(_letters + _letters.upper() + '_0123456789$')
tuple	cpp.tokenize.HEX_DIGITS set('0123456789abcdefABCDEF')
tuple	cpp.tokenize.INT_OR_FLOAT_DIGITS set('01234567890eE-+')
tuple	cpp.tokenize._STR_PREFIXES set(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR'))
string	cpp.tokenize.UNKNOWN 'UNKNOWN'
string	cpp.tokenize.SYNTAX 'SYNTAX'
string	cpp.tokenize.CONSTANT 'CONSTANT'
string	cpp.tokenize.NAME 'NAME'
string	cpp.tokenize.PREPROCESSOR 'PREPROCESSOR'