token.h 2.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192
  1. /* Token types */
  2. #ifndef Py_LIMITED_API
  3. #ifndef Py_TOKEN_H
  4. #define Py_TOKEN_H
  5. #ifdef __cplusplus
  6. extern "C" {
  7. #endif
  8. #undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
  9. #define ENDMARKER 0
  10. #define NAME 1
  11. #define NUMBER 2
  12. #define STRING 3
  13. #define NEWLINE 4
  14. #define INDENT 5
  15. #define DEDENT 6
  16. #define LPAR 7
  17. #define RPAR 8
  18. #define LSQB 9
  19. #define RSQB 10
  20. #define COLON 11
  21. #define COMMA 12
  22. #define SEMI 13
  23. #define PLUS 14
  24. #define MINUS 15
  25. #define STAR 16
  26. #define SLASH 17
  27. #define VBAR 18
  28. #define AMPER 19
  29. #define LESS 20
  30. #define GREATER 21
  31. #define EQUAL 22
  32. #define DOT 23
  33. #define PERCENT 24
  34. #define LBRACE 25
  35. #define RBRACE 26
  36. #define EQEQUAL 27
  37. #define NOTEQUAL 28
  38. #define LESSEQUAL 29
  39. #define GREATEREQUAL 30
  40. #define TILDE 31
  41. #define CIRCUMFLEX 32
  42. #define LEFTSHIFT 33
  43. #define RIGHTSHIFT 34
  44. #define DOUBLESTAR 35
  45. #define PLUSEQUAL 36
  46. #define MINEQUAL 37
  47. #define STAREQUAL 38
  48. #define SLASHEQUAL 39
  49. #define PERCENTEQUAL 40
  50. #define AMPEREQUAL 41
  51. #define VBAREQUAL 42
  52. #define CIRCUMFLEXEQUAL 43
  53. #define LEFTSHIFTEQUAL 44
  54. #define RIGHTSHIFTEQUAL 45
  55. #define DOUBLESTAREQUAL 46
  56. #define DOUBLESLASH 47
  57. #define DOUBLESLASHEQUAL 48
  58. #define AT 49
  59. #define ATEQUAL 50
  60. #define RARROW 51
  61. #define ELLIPSIS 52
  62. /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
  63. #define OP 53
  64. #define ERRORTOKEN 54
  65. /* These aren't used by the C tokenizer but are needed for tokenize.py */
  66. #define COMMENT 55
  67. #define NL 56
  68. #define ENCODING 57
  69. #define N_TOKENS 58
  70. /* Special definitions for cooperation with parser */
  71. #define NT_OFFSET 256
  72. #define ISTERMINAL(x) ((x) < NT_OFFSET)
  73. #define ISNONTERMINAL(x) ((x) >= NT_OFFSET)
  74. #define ISEOF(x) ((x) == ENDMARKER)
  75. PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */
  76. PyAPI_FUNC(int) PyToken_OneChar(int);
  77. PyAPI_FUNC(int) PyToken_TwoChars(int, int);
  78. PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
  79. #ifdef __cplusplus
  80. }
  81. #endif
  82. #endif /* !Py_TOKEN_H */
  83. #endif /* Py_LIMITED_API */