PKG-INFO

 Metadata-Version: 1.1
 Name: typed-ast
-Version: 1.2.0
+Version: 1.3.0
 Summary: a fork of Python 2 and 3 ast modules with type comment support
 Home-page: https://github.com/python/typed_ast
 Author: David Fisher

ast27/Python/Python-ast.c

@@ -477,7 +477,7 @@ static PyMethodDef ast_type_methods[] = {
 
 static PyTypeObject AST_type = {
     PyVarObject_HEAD_INIT(NULL, 0)
-    "_ast27.AST",
+    "typed_ast._ast27.AST",
     sizeof(PyObject),
     0,
     0,                       /* tp_dealloc */
@@ -533,7 +533,7 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
         PyTuple_SET_ITEM(fnames, i, field);
     }
     result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}",
-                    type, base, "_fields", fnames, "__module__", "_ast27");
+                    type, base, "_fields", fnames, "__module__", "typed_ast._ast27");
     Py_DECREF(fnames);
     return (PyTypeObject*)result;
 }

ast27/Python/ast.c

@@ -1500,7 +1500,6 @@ ast_for_atom(struct compiling *c, const node *n)
     case STRING: {
         PyObject *kind, *str = parsestrplus(c, n);
         const char *raw, *s = STR(CHILD(n, 0));
-        int quote = Py_CHARMASK(*s);
         /* currently Python allows up to 2 string modifiers */
         char *ch, s_kind[3] = {0, 0, 0};
         ch = s_kind;
@@ -1519,7 +1518,7 @@ ast_for_atom(struct compiling *c, const node *n)
                 PyErr_Fetch(&type, &value, &tback);
                 errstr = PyObject_Str(value);
                 if (errstr) {
-                    char *s = "";
+                    const char *s = "";
                     char buf[128];
                     s = _PyUnicode_AsString(errstr);
                     PyOS_snprintf(buf, sizeof(buf), "(unicode error) %s", s);
@@ -2190,7 +2189,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
                 keyword_ty kw;
                 identifier key;
                 int k;
-                char *tmp;
+                const char *tmp;
 
                 /* CHILD(ch, 0) is test, but must be an identifier? */
                 e = ast_for_expr(c, CHILD(ch, 0));

ast3/Custom/typed_ast.c

 #include "Python.h"
 #include "Python-ast.h"
-#include "compile.h"
+#include "compile-ast3.h"
 #include "node.h"
 #include "grammar.h"
 #include "token.h"
@@ -211,9 +211,33 @@ err_free(perrdetail *err)
     Py_CLEAR(err->filename);
 }
 
+// from Python/pythonrun.c
+node *
+Ta3Parser_SimpleParseStringFlagsFilename(const char *str, const char *filename,
+                                         int start, int flags)
+{
+    perrdetail err;
+    node *n = Ta3Parser_ParseStringFlagsFilename(str, filename,
+                            &_Ta3Parser_Grammar, start, &err, flags);
+    if (n == NULL)
+        err_input(&err);
+    err_free(&err);
+    return n;
+}
+
+/* update compiler and parser flags based on feature version */
+void
+_Ta3Parser_UpdateFlags(PyCompilerFlags *flags, int *iflags, int feature_version)
+{
+    *iflags = PARSER_FLAGS(flags);
+    if (feature_version >= 7)
+        *iflags |= PyPARSE_ASYNC_ALWAYS;
+    flags->cf_flags |= *iflags & PyCF_MASK;
+}
+
 // copy of PyParser_ASTFromStringObject in Python/pythonrun.c
 /* Preferred access to parser is through AST. */
-mod_ty
+static mod_ty
 string_object_to_c_ast(const char *s, PyObject *filename, int start,
                              PyCompilerFlags *flags, int feature_version,
                              PyArena *arena)
@@ -221,15 +245,17 @@ string_object_to_c_ast(const char *s, PyObject *filename, int start,
     mod_ty mod;
     PyCompilerFlags localflags;
     perrdetail err;
-    int iflags = PARSER_FLAGS(flags);
+    node *n;
+    int iflags;
 
-    node *n = Ta3Parser_ParseStringObject(s, filename,
-                                         &_Ta3Parser_Grammar, start, &err,
-                                         &iflags);
     if (flags == NULL) {
         localflags.cf_flags = 0;
         flags = &localflags;
     }
+    _Ta3Parser_UpdateFlags(flags, &iflags, feature_version);
+    n = Ta3Parser_ParseStringObject(s, filename,
+                                    &_Ta3Parser_Grammar, start, &err,
+                                    &iflags);
     if (n) {
         flags->cf_flags |= iflags & PyCF_MASK;
         mod = Ta3AST_FromNodeObject(n, flags, filename, feature_version, arena);

ast3/Include/Python-ast.h

@@ -50,24 +50,24 @@ struct _mod {
             asdl_seq *body;
             asdl_seq *type_ignores;
         } Module;
-        
+
         struct {
             asdl_seq *body;
         } Interactive;
-        
+
         struct {
             expr_ty body;
         } Expression;
-        
+
         struct {
             asdl_seq *argtypes;
             expr_ty returns;
         } FunctionType;
-        
+
         struct {
             asdl_seq *body;
         } Suite;
-        
+
     } v;
 };
 
@@ -90,7 +90,7 @@ struct _stmt {
             expr_ty returns;
             string type_comment;
         } FunctionDef;
-        
+
         struct {
             identifier name;
             arguments_ty args;
@@ -99,7 +99,7 @@ struct _stmt {
             expr_ty returns;
             string type_comment;
         } AsyncFunctionDef;
-        
+
         struct {
             identifier name;
             asdl_seq *bases;
@@ -107,34 +107,34 @@ struct _stmt {
             asdl_seq *body;
             asdl_seq *decorator_list;
         } ClassDef;
-        
+
         struct {
             expr_ty value;
         } Return;
-        
+
         struct {
             asdl_seq *targets;
         } Delete;
-        
+
         struct {
             asdl_seq *targets;
             expr_ty value;
             string type_comment;
         } Assign;
-        
+
         struct {
             expr_ty target;
             operator_ty op;
             expr_ty value;
         } AugAssign;
-        
+
         struct {
             expr_ty target;
             expr_ty annotation;
             expr_ty value;
             int simple;
         } AnnAssign;
-        
+
         struct {
             expr_ty target;
             expr_ty iter;
@@ -142,7 +142,7 @@ struct _stmt {
             asdl_seq *orelse;
             string type_comment;
         } For;
-        
+
         struct {
             expr_ty target;
             expr_ty iter;
@@ -150,70 +150,70 @@ struct _stmt {
             asdl_seq *orelse;
             string type_comment;
         } AsyncFor;
-        
+
         struct {
             expr_ty test;
             asdl_seq *body;
             asdl_seq *orelse;
         } While;
-        
+
         struct {
             expr_ty test;
             asdl_seq *body;
             asdl_seq *orelse;
         } If;
-        
+
         struct {
             asdl_seq *items;
             asdl_seq *body;
             string type_comment;
         } With;
-        
+
         struct {
             asdl_seq *items;
             asdl_seq *body;
             string type_comment;
         } AsyncWith;
-        
+
         struct {
             expr_ty exc;
             expr_ty cause;
         } Raise;
-        
+
         struct {
             asdl_seq *body;
             asdl_seq *handlers;
             asdl_seq *orelse;
             asdl_seq *finalbody;
         } Try;
-        
+
         struct {
             expr_ty test;
             expr_ty msg;
         } Assert;
-        
+
         struct {
             asdl_seq *names;
         } Import;
-        
+
         struct {
             identifier module;
             asdl_seq *names;
             int level;
         } ImportFrom;
-        
+
         struct {
             asdl_seq *names;
         } Global;
-        
+
         struct {
             asdl_seq *names;
         } Nonlocal;
-        
+
         struct {
             expr_ty value;
         } Expr;
-        
+
     } v;
     int lineno;
     int col_offset;
@@ -235,146 +235,147 @@ struct _expr {
             boolop_ty op;
             asdl_seq *values;
         } BoolOp;
-        
+
         struct {
             expr_ty left;
             operator_ty op;
             expr_ty right;
         } BinOp;
-        
+
         struct {
             unaryop_ty op;
             expr_ty operand;
         } UnaryOp;
-        
+
         struct {
             arguments_ty args;
             expr_ty body;
         } Lambda;
-        
+
         struct {
             expr_ty test;
             expr_ty body;
             expr_ty orelse;
         } IfExp;
-        
+
         struct {
             asdl_seq *keys;
             asdl_seq *values;
         } Dict;
-        
+
         struct {
             asdl_seq *elts;
         } Set;
-        
+
         struct {
             expr_ty elt;
             asdl_seq *generators;
         } ListComp;
-        
+
         struct {
             expr_ty elt;
             asdl_seq *generators;
         } SetComp;
-        
+
         struct {
             expr_ty key;
             expr_ty value;
             asdl_seq *generators;
         } DictComp;
-        
+
         struct {
             expr_ty elt;
             asdl_seq *generators;
         } GeneratorExp;
-        
+
         struct {
             expr_ty value;
         } Await;
-        
+
         struct {
             expr_ty value;
         } Yield;
-        
+
         struct {
             expr_ty value;
         } YieldFrom;
-        
+
         struct {
             expr_ty left;
             asdl_int_seq *ops;
             asdl_seq *comparators;
         } Compare;
-        
+
         struct {
             expr_ty func;
             asdl_seq *args;
             asdl_seq *keywords;
         } Call;
-        
+
         struct {
             object n;
         } Num;
-        
+
         struct {
             string s;
             string kind;
         } Str;
-        
+
         struct {
             expr_ty value;
             int conversion;
             expr_ty format_spec;
         } FormattedValue;
-        
+
         struct {
             asdl_seq *values;
         } JoinedStr;
-        
+
         struct {
             bytes s;
+            string kind;
         } Bytes;
-        
+
         struct {
             singleton value;
         } NameConstant;
-        
+
         struct {
             constant value;
         } Constant;
-        
+
         struct {
             expr_ty value;
             identifier attr;
             expr_context_ty ctx;
         } Attribute;
-        
+
         struct {
             expr_ty value;
             slice_ty slice;
             expr_context_ty ctx;
         } Subscript;
-        
+
         struct {
             expr_ty value;
             expr_context_ty ctx;
         } Starred;
-        
+
         struct {
             identifier id;
             expr_context_ty ctx;
         } Name;
-        
+
         struct {
             asdl_seq *elts;
             expr_context_ty ctx;
         } List;
-        
+
         struct {
             asdl_seq *elts;
             expr_context_ty ctx;
         } Tuple;
-        
+
     } v;
     int lineno;
     int col_offset;
@@ -389,15 +390,15 @@ struct _slice {
             expr_ty upper;
             expr_ty step;
         } Slice;
-        
+
         struct {
             asdl_seq *dims;
         } ExtSlice;
-        
+
         struct {
             expr_ty value;
         } Index;
-        
+
     } v;
 };
 
@@ -417,7 +418,7 @@ struct _excepthandler {
             identifier name;
             asdl_seq *body;
         } ExceptHandler;
-        
+
     } v;
     int lineno;
     int col_offset;
@@ -462,7 +463,7 @@ struct _type_ignore {
         struct {
             int lineno;
         } TypeIgnore;
-        
+
     } v;
 };
 
@@ -611,8 +612,9 @@ expr_ty _Ta3_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
 #define JoinedStr(a0, a1, a2, a3) _Ta3_JoinedStr(a0, a1, a2, a3)
 expr_ty _Ta3_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena
                        *arena);
-#define Bytes(a0, a1, a2, a3) _Ta3_Bytes(a0, a1, a2, a3)
-expr_ty _Ta3_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
+#define Bytes(a0, a1, a2, a3, a4) _Ta3_Bytes(a0, a1, a2, a3, a4)
+expr_ty _Ta3_Bytes(bytes s, string kind, int lineno, int col_offset, PyArena
+                   *arena);
 #define NameConstant(a0, a1, a2, a3) _Ta3_NameConstant(a0, a1, a2, a3)
 expr_ty _Ta3_NameConstant(singleton value, int lineno, int col_offset, PyArena
                           *arena);

ast3/Include/ast.h

@@ -18,6 +18,13 @@ extern mod_ty Ta3AST_FromNodeObject(
     int feature_version,
     PyArena *arena);
 
+#ifndef Py_LIMITED_API
+
+/* _PyAST_ExprAsUnicode is defined in ast_unparse.c */
+extern PyObject * _PyAST_ExprAsUnicode(expr_ty);
+
+#endif /* !Py_LIMITED_API */
+
 #ifdef __cplusplus
 }
 #endif

ast3/Include/bitset.h

@@ -7,7 +7,7 @@ extern "C" {
 
 /* Bitset interface */
 
-#define BYTE		char
+#define BYTE            char
 
 typedef BYTE *bitset;
 
@@ -18,13 +18,13 @@ int addbit(bitset bs, int ibit); /* Returns 0 if already set */
 int samebitset(bitset bs1, bitset bs2, int nbits);
 void mergebitset(bitset bs1, bitset bs2, int nbits);
 
-#define BITSPERBYTE	(8*sizeof(BYTE))
-#define NBYTES(nbits)	(((nbits) + BITSPERBYTE - 1) / BITSPERBYTE)
+#define BITSPERBYTE     (8*sizeof(BYTE))
+#define NBYTES(nbits)   (((nbits) + BITSPERBYTE - 1) / BITSPERBYTE)
 
-#define BIT2BYTE(ibit)	((ibit) / BITSPERBYTE)
-#define BIT2SHIFT(ibit)	((ibit) % BITSPERBYTE)
-#define BIT2MASK(ibit)	(1 << BIT2SHIFT(ibit))
-#define BYTE2BIT(ibyte)	((ibyte) * BITSPERBYTE)
+#define BIT2BYTE(ibit)  ((ibit) / BITSPERBYTE)
+#define BIT2SHIFT(ibit) ((ibit) % BITSPERBYTE)
+#define BIT2MASK(ibit)  (1 << BIT2SHIFT(ibit))
+#define BYTE2BIT(ibyte) ((ibyte) * BITSPERBYTE)
 
 #ifdef __cplusplus
 }

ast3/Include/compile.h → ast3/Include/compile-ast3.h

-#ifndef Ta3_COMPILE_H
-#define Ta3_COMPILE_H
-
 /* These definitions must match corresponding definitions in graminit.h.
    There's code in compile.c that checks that they are the same. */
 #define Py_single_input 256
 #define Py_file_input 257
 #define Py_eval_input 258
-#define Py_func_type_input 342
-
-#endif /* !Ta3_COMPILE_H */
+#define Py_func_type_input 343

ast3/Include/errcode.h

@@ -13,24 +13,24 @@ extern "C" {
    the parser only returns E_EOF when it hits EOF immediately, and it
    never returns E_OK. */
 
-#define E_OK		10	/* No error */
-#define E_EOF		11	/* End Of File */
-#define E_INTR		12	/* Interrupted */
-#define E_TOKEN		13	/* Bad token */
-#define E_SYNTAX	14	/* Syntax error */
-#define E_NOMEM		15	/* Ran out of memory */
-#define E_DONE		16	/* Parsing complete */
-#define E_ERROR		17	/* Execution error */
-#define E_TABSPACE	18	/* Inconsistent mixing of tabs and spaces */
-#define E_OVERFLOW      19	/* Node had too many children */
-#define E_TOODEEP	20	/* Too many indentation levels */
-#define E_DEDENT	21	/* No matching outer block for dedent */
-#define E_DECODE	22	/* Error in decoding into Unicode */
-#define E_EOFS		23	/* EOF in triple-quoted string */
-#define E_EOLS		24	/* EOL in single-quoted string */
-#define E_LINECONT	25	/* Unexpected characters after a line continuation */
+#define E_OK            10      /* No error */
+#define E_EOF           11      /* End Of File */
+#define E_INTR          12      /* Interrupted */
+#define E_TOKEN         13      /* Bad token */
+#define E_SYNTAX        14      /* Syntax error */
+#define E_NOMEM         15      /* Ran out of memory */
+#define E_DONE          16      /* Parsing complete */
+#define E_ERROR         17      /* Execution error */
+#define E_TABSPACE      18      /* Inconsistent mixing of tabs and spaces */
+#define E_OVERFLOW      19      /* Node had too many children */
+#define E_TOODEEP       20      /* Too many indentation levels */
+#define E_DEDENT        21      /* No matching outer block for dedent */
+#define E_DECODE        22      /* Error in decoding into Unicode */
+#define E_EOFS          23      /* EOF in triple-quoted string */
+#define E_EOLS          24      /* EOL in single-quoted string */
+#define E_LINECONT      25      /* Unexpected characters after a line continuation */
 #define E_IDENTIFIER    26      /* Invalid characters in identifier */
-#define E_BADSINGLE	27	/* Ill-formed single statement input */
+#define E_BADSINGLE     27      /* Ill-formed single statement input */
 
 #ifdef __cplusplus
 }

ast3/Include/graminit.h

@@ -81,11 +81,12 @@
 #define arglist 334
 #define argument 335
 #define comp_iter 336
-#define comp_for 337
-#define comp_if 338
-#define encoding_decl 339
-#define yield_expr 340
-#define yield_arg 341
-#define func_type_input 342
-#define func_type 343
-#define typelist 344
+#define sync_comp_for 337
+#define comp_for 338
+#define comp_if 339
+#define encoding_decl 340
+#define yield_expr 341
+#define yield_arg 342
+#define func_type_input 343
+#define func_type 344
+#define typelist 345

ast3/Include/grammar.h

@@ -12,58 +12,58 @@ extern "C" {
 /* A label of an arc */
 
 typedef struct {
-    int		 lb_type;
-    char	*lb_str;
+    int          lb_type;
+    char        *lb_str;
 } label;
 
-#define EMPTY 0		/* Label number 0 is by definition the empty label */
+#define EMPTY 0         /* Label number 0 is by definition the empty label */
 
 /* A list of labels */
 
 typedef struct {
-    int		 ll_nlabels;
-    label	*ll_label;
+    int          ll_nlabels;
+    label       *ll_label;
 } labellist;
 
 /* An arc from one state to another */
 
 typedef struct {
-    short	a_lbl;		/* Label of this arc */
-    short	a_arrow;	/* State where this arc goes to */
+    short       a_lbl;          /* Label of this arc */
+    short       a_arrow;        /* State where this arc goes to */
 } arc;
 
 /* A state in a DFA */
 
 typedef struct {
-    int		 s_narcs;
-    arc		*s_arc;		/* Array of arcs */
+    int          s_narcs;
+    arc         *s_arc;         /* Array of arcs */
 
     /* Optional accelerators */
-    int		 s_lower;	/* Lowest label index */
-    int		 s_upper;	/* Highest label index */
-    int		*s_accel;	/* Accelerator */
-    int		 s_accept;	/* Nonzero for accepting state */
+    int          s_lower;       /* Lowest label index */
+    int          s_upper;       /* Highest label index */
+    int         *s_accel;       /* Accelerator */
+    int          s_accept;      /* Nonzero for accepting state */
 } state;
 
 /* A DFA */
 
 typedef struct {
-    int		 d_type;	/* Non-terminal this represents */
-    char	*d_name;	/* For printing */
-    int		 d_initial;	/* Initial state */
-    int		 d_nstates;
-    state	*d_state;	/* Array of states */
-    bitset	 d_first;
+    int          d_type;        /* Non-terminal this represents */
+    char        *d_name;        /* For printing */
+    int          d_initial;     /* Initial state */
+    int          d_nstates;
+    state       *d_state;       /* Array of states */
+    bitset       d_first;
 } dfa;
 
 /* A grammar */
 
 typedef struct {
-    int		 g_ndfas;
-    dfa		*g_dfa;		/* Array of DFAs */
-    labellist	 g_ll;
-    int		 g_start;	/* Start symbol of the grammar */
-    int		 g_accel;	/* Set if accelerators present */
+    int          g_ndfas;
+    dfa         *g_dfa;         /* Array of DFAs */
+    labellist    g_ll;
+    int          g_start;       /* Start symbol of the grammar */
+    int          g_accel;       /* Set if accelerators present */
 } grammar;
 
 /* FUNCTIONS */

ast3/Include/node.h

@@ -8,15 +8,15 @@ extern "C" {
 #endif
 
 typedef struct _node {
-    short		n_type;
-    char		*n_str;
-    int			n_lineno;
-    int			n_col_offset;
-    int			n_nchildren;
-    struct _node	*n_child;
+    short               n_type;
+    char                *n_str;
+    int                 n_lineno;
+    int                 n_col_offset;
+    int                 n_nchildren;
+    struct _node        *n_child;
 } node;
 
-extern node *Ta3Node_New(int type);
+extern node * Ta3Node_New(int type);
 extern int Ta3Node_AddChild(node *n, int type,
                                       char *str, int lineno, int col_offset);
 extern void Ta3Node_Free(node *n);
@@ -25,12 +25,12 @@ extern Py_ssize_t _Ta3Node_SizeOf(node *n);
 #endif
 
 /* Node access functions */
-#define NCH(n)		((n)->n_nchildren)
+#define NCH(n)          ((n)->n_nchildren)
 
-#define CHILD(n, i)	(&(n)->n_child[i])
-#define RCHILD(n, i)	(CHILD(n, NCH(n) + i))
-#define TYPE(n)		((n)->n_type)
-#define STR(n)		((n)->n_str)
+#define CHILD(n, i)     (&(n)->n_child[i])
+#define RCHILD(n, i)    (CHILD(n, NCH(n) + i))
+#define TYPE(n)         ((n)->n_type)
+#define STR(n)          ((n)->n_str)
 #define LINENO(n)       ((n)->n_lineno)
 
 /* Assert that the type of a node is what we expect */

ast3/Include/parsetok.h

@@ -21,19 +21,20 @@ typedef struct {
 } perrdetail;
 
 #if 0
-#define PyPARSE_YIELD_IS_KEYWORD	0x0001
+#define PyPARSE_YIELD_IS_KEYWORD        0x0001
 #endif
 
-#define PyPARSE_DONT_IMPLY_DEDENT	0x0002
+#define PyPARSE_DONT_IMPLY_DEDENT       0x0002
 
 #if 0
-#define PyPARSE_WITH_IS_KEYWORD		0x0003
+#define PyPARSE_WITH_IS_KEYWORD         0x0003
 #define PyPARSE_PRINT_IS_FUNCTION       0x0004
 #define PyPARSE_UNICODE_LITERALS        0x0008
 #endif
 
 #define PyPARSE_IGNORE_COOKIE 0x0010
 #define PyPARSE_BARRY_AS_BDFL 0x0020
+#define PyPARSE_ASYNC_ALWAYS  0x8000
 
 extern node * Ta3Parser_ParseString(const char *, grammar *, int,
                                               perrdetail *);
@@ -98,8 +99,8 @@ extern node * Ta3Parser_ParseStringObject(
 
 /* Note that the following functions are defined in pythonrun.c,
    not in parsetok.c */
-PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
-PyAPI_FUNC(void) PyParser_ClearError(perrdetail *);
+extern void PyParser_SetError(perrdetail *);
+extern void PyParser_ClearError(perrdetail *);
 
 #ifdef __cplusplus
 }

ast3/Include/token.h

@@ -9,78 +9,82 @@ extern "C" {
 
 #undef TILDE   /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
 
-#define ENDMARKER	0
-#define NAME		1
-#define NUMBER		2
-#define STRING		3
-#define NEWLINE		4
-#define INDENT		5
-#define DEDENT		6
-#define LPAR		7
-#define RPAR		8
-#define LSQB		9
-#define RSQB		10
-#define COLON		11
-#define COMMA		12
-#define SEMI		13
-#define PLUS		14
-#define MINUS		15
-#define STAR		16
-#define SLASH		17
-#define VBAR		18
-#define AMPER		19
-#define LESS		20
-#define GREATER		21
-#define EQUAL		22
-#define DOT		23
-#define PERCENT		24
-#define LBRACE		25
-#define RBRACE		26
-#define EQEQUAL		27
-#define NOTEQUAL	28
-#define LESSEQUAL	29
-#define GREATEREQUAL	30
-#define TILDE		31
-#define CIRCUMFLEX	32
-#define LEFTSHIFT	33
-#define RIGHTSHIFT	34
-#define DOUBLESTAR	35
-#define PLUSEQUAL	36
-#define MINEQUAL	37
-#define STAREQUAL	38
-#define SLASHEQUAL	39
-#define PERCENTEQUAL	40
-#define AMPEREQUAL	41
-#define VBAREQUAL	42
-#define CIRCUMFLEXEQUAL	43
-#define LEFTSHIFTEQUAL	44
-#define RIGHTSHIFTEQUAL	45
-#define DOUBLESTAREQUAL	46
-#define DOUBLESLASH	47
+#define ENDMARKER       0
+#define NAME            1
+#define NUMBER          2
+#define STRING          3
+#define NEWLINE         4
+#define INDENT          5
+#define DEDENT          6
+#define LPAR            7
+#define RPAR            8
+#define LSQB            9
+#define RSQB            10
+#define COLON           11
+#define COMMA           12
+#define SEMI            13
+#define PLUS            14
+#define MINUS           15
+#define STAR            16
+#define SLASH           17
+#define VBAR            18
+#define AMPER           19
+#define LESS            20
+#define GREATER         21
+#define EQUAL           22
+#define DOT             23
+#define PERCENT         24
+#define LBRACE          25
+#define RBRACE          26
+#define EQEQUAL         27
+#define NOTEQUAL        28
+#define LESSEQUAL       29
+#define GREATEREQUAL    30
+#define TILDE           31
+#define CIRCUMFLEX      32
+#define LEFTSHIFT       33
+#define RIGHTSHIFT      34
+#define DOUBLESTAR      35
+#define PLUSEQUAL       36
+#define MINEQUAL        37
+#define STAREQUAL       38
+#define SLASHEQUAL      39
+#define PERCENTEQUAL    40
+#define AMPEREQUAL      41
+#define VBAREQUAL       42
+#define CIRCUMFLEXEQUAL 43
+#define LEFTSHIFTEQUAL  44
+#define RIGHTSHIFTEQUAL 45
+#define DOUBLESTAREQUAL 46
+#define DOUBLESLASH     47
 #define DOUBLESLASHEQUAL 48
 #define AT              49
-#define ATEQUAL		50
+#define ATEQUAL         50
 #define RARROW          51
 #define ELLIPSIS        52
 /* Don't forget to update the table _Ta3Parser_TokenNames in tokenizer.c! */
-#define OP		53
-#define AWAIT		54
-#define ASYNC		55
+#define OP              53
+#define AWAIT           54
+#define ASYNC           55
 #define TYPE_IGNORE	56
 #define TYPE_COMMENT	57
-#define ERRORTOKEN	58
-#define N_TOKENS	59
+#define ERRORTOKEN      58
+/* These aren't used by the C tokenizer but are needed for tokenize.py */
+#define COMMENT         59
+#define NL              60
+#define ENCODING                61
+#define N_TOKENS        62
 
 /* Special definitions for cooperation with parser */
 
-#define NT_OFFSET		256
+#define NT_OFFSET               256
 
-#define ISTERMINAL(x)		((x) < NT_OFFSET)
-#define ISNONTERMINAL(x)	((x) >= NT_OFFSET)
-#define ISEOF(x)		((x) == ENDMARKER)
+#define ISTERMINAL(x)           ((x) < NT_OFFSET)
+#define ISNONTERMINAL(x)        ((x) >= NT_OFFSET)
+#define ISEOF(x)                ((x) == ENDMARKER)
 
 
-extern const char *_Ta3Parser_TokenNames[]; /* Token names */
+extern const char * _Ta3Parser_TokenNames[]; /* Token names */
 extern int Ta3Token_OneChar(int);
 extern int Ta3Token_TwoChars(int, int);
 extern int Ta3Token_ThreeChars(int, int, int);

ast3/Parser/grammar1.c

@@ -25,8 +25,7 @@ Ta3Grammar_FindDFA(grammar *g, int type)
         if (d->d_type == type)
             return d;
     }
-    assert(0);
-    /* NOTREACHED */
+    abort();
 #endif
 }
 

ast3/Parser/parser.h

@@ -10,23 +10,23 @@ extern "C" {
 #define MAXSTACK 1500
 
 typedef struct {
-	int		 s_state;	/* State in current DFA */
-	dfa		*s_dfa;		/* Current DFA */
-	struct _node	*s_parent;	/* Where to add next node */
+    int              s_state;       /* State in current DFA */
+    dfa             *s_dfa;         /* Current DFA */
+    struct _node    *s_parent;      /* Where to add next node */
 } stackentry;
 
 typedef struct {
-	stackentry	*s_top;		/* Top entry */
-	stackentry	 s_base[MAXSTACK];/* Array of stack entries */
-					/* NB The stack grows down */
+    stackentry      *s_top;         /* Top entry */
+    stackentry       s_base[MAXSTACK];/* Array of stack entries */
+                                    /* NB The stack grows down */
 } stack;
 
 typedef struct {
-	stack	 	p_stack;	/* Stack of parser states */
-	grammar		*p_grammar;	/* Grammar to use */
-	node		*p_tree;	/* Top of parse tree */
+    stack           p_stack;        /* Stack of parser states */
+    grammar         *p_grammar;     /* Grammar to use */
+    node            *p_tree;        /* Top of parse tree */
 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
-	unsigned long	p_flags;	/* see co_flags in Include/code.h */
+    unsigned long   p_flags;        /* see co_flags in Include/code.h */
 #endif
 } parser_state;
 

ast3/Parser/parsetok.c

@@ -64,6 +64,8 @@ Ta3Parser_ParseStringObject(const char *s, PyObject *filename,
     Py_INCREF(err_ret->filename);
     tok->filename = err_ret->filename;
 #endif
+    if (*flags & PyPARSE_ASYNC_ALWAYS)
+        tok->async_always = 1;
     return parsetok(tok, g, start, err_ret, flags);
 }
 
@@ -264,7 +266,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
         }
         else
             started = 1;
-        len = b - a; /* XXX this may compute NULL - NULL */
+        len = (a != NULL && b != NULL) ? b - a : 0;
         str = (char *) PyObject_MALLOC(len + 1);
         if (str == NULL) {
             err_ret->error = E_NOMEM;
@@ -285,18 +287,19 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
                             strcmp(str, "<>")) {
                 PyObject_FREE(str);
-                err_ret->text = "with Barry as BDFL, use '<>' "
-                                "instead of '!='";
+                err_ret->expected = NOTEQUAL;
                 err_ret->error = E_SYNTAX;
                 break;
             }
         }
 #endif
-        if (a >= tok->line_start)
+        if (a != NULL && a >= tok->line_start) {
             col_offset = Py_SAFE_DOWNCAST(a - tok->line_start,
                                           intptr_t, int);
-        else
+        }
+        else {
             col_offset = -1;
+        }
 
         if (type == TYPE_IGNORE) {
             if (!growable_int_array_add(&type_ignores, tok->lineno)) {

ast3/Parser/tokenizer.c

@@ -27,6 +27,13 @@
     } while (0)
 #endif /* Py_XSETREF */
 
+#ifndef _PyObject_CallNoArg
+#define _PyObject_CallNoArg(func) PyObject_CallObject(func, NULL)
+#endif
+
+/* Alternate tab spacing */
+#define ALTTABSIZE 1
+
 #define is_potential_identifier_start(c) (\
               (c >= 'a' && c <= 'z')\
                || (c >= 'A' && c <= 'Z')\
@@ -40,12 +47,7 @@
                || c == '_'\
                || (c >= 128))
 
-#if PY_MINOR_VERSION >= 4
 PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *);
-#else
-// Python 3.3 doesn't have PyAPI_FUNC, but it's not supported on Windows anyway.
-char *PyOS_Readline(FILE *, FILE *, char *);
-#endif
 /* Return malloc'ed string including trailing \n;
    empty malloc'ed string for EOF;
    NULL if interrupted */
@@ -122,6 +124,9 @@ const char *_Ta3Parser_TokenNames[] = {
     "TYPE_IGNORE",
     "TYPE_COMMENT",
     "<ERRORTOKEN>",
+    "COMMENT",
+    "NL",
+    "ENCODING",
     "<N_TOKENS>"
 };
 
@@ -152,9 +157,6 @@ tok_new(void)
     tok->prompt = tok->nextprompt = NULL;
     tok->lineno = 0;
     tok->level = 0;
-    tok->altwarning = 1;
-    tok->alterror = 1;
-    tok->alttabsize = 1;
     tok->altindstack[0] = 0;
     tok->decoding_state = STATE_INIT;
     tok->decoding_erred = 0;
@@ -171,6 +173,7 @@ tok_new(void)
     tok->async_def = 0;
     tok->async_def_indent = 0;
     tok->async_def_nl = 0;
+    tok->async_always = 0;
 
     return tok;
 }
@@ -460,7 +463,7 @@ fp_readl(char *s, int size, struct tok_state *tok)
     }
     else
     {
-        bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
+        bufobj = _PyObject_CallNoArg(tok->decoding_readline);
         if (bufobj == NULL)
             goto error;
     }
@@ -553,7 +556,7 @@ fp_setreadl(struct tok_state *tok, const char* enc)
     Py_XSETREF(tok->decoding_readline, readline);
 
     if (pos > 0) {
-        PyObject *bufobj = PyObject_CallObject(readline, NULL);
+        PyObject *bufobj = _PyObject_CallNoArg(readline);
         if (bufobj == NULL)
             return 0;
         Py_DECREF(bufobj);
@@ -670,7 +673,7 @@ decoding_feof(struct tok_state *tok)
     } else {
         PyObject* buf = tok->decoding_buffer;
         if (buf == NULL) {
-            buf = PyObject_CallObject(tok->decoding_readline, NULL);
+            buf = _PyObject_CallNoArg(tok->decoding_readline);
             if (buf == NULL) {
                 error_ret(tok);
                 return 1;
@@ -976,6 +979,11 @@ tok_nextc(struct tok_state *tok)
                 buflen = PyBytes_GET_SIZE(u);
                 buf = PyBytes_AS_STRING(u);
                 newtok = PyMem_MALLOC(buflen+1);
+                if (newtok == NULL) {
+                    Py_DECREF(u);
+                    tok->done = E_NOMEM;
+                    return EOF;
+                }
                 strcpy(newtok, buf);
                 Py_DECREF(u);
             }
@@ -1306,22 +1314,9 @@ Ta3Token_ThreeChars(int c1, int c2, int c3)
 static int
 indenterror(struct tok_state *tok)
 {
-    if (tok->alterror) {
-        tok->done = E_TABSPACE;
-        tok->cur = tok->inp;
-        return 1;
-    }
-    if (tok->altwarning) {
-#ifdef PGEN
-        PySys_WriteStderr("inconsistent use of tabs and spaces "
-                          "in indentation\n");
-#else
-        PySys_FormatStderr("%U: inconsistent use of tabs and spaces "
-                          "in indentation\n", tok->filename);
-#endif
-        tok->altwarning = 0;
-    }
-    return 0;
+    tok->done = E_TABSPACE;
+    tok->cur = tok->inp;
+    return ERRORTOKEN;
 }
 
 #ifdef PGEN
@@ -1401,9 +1396,8 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                 col++, altcol++;
             }
             else if (c == '\t') {
-                col = (col/tok->tabsize + 1) * tok->tabsize;
-                altcol = (altcol/tok->alttabsize + 1)
-                    * tok->alttabsize;
+                col = (col / tok->tabsize + 1) * tok->tabsize;
+                altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
             }
             else if (c == '\014')  {/* Control-L (formfeed) */
                 col = altcol = 0; /* For Emacs users */
@@ -1432,9 +1426,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             if (col == tok->indstack[tok->indent]) {
                 /* No change */
                 if (altcol != tok->altindstack[tok->indent]) {
-                    if (indenterror(tok)) {
-                        return ERRORTOKEN;
-                    }
+                    return indenterror(tok);
                 }
             }
             else if (col > tok->indstack[tok->indent]) {
@@ -1445,9 +1437,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                     return ERRORTOKEN;
                 }
                 if (altcol <= tok->altindstack[tok->indent]) {
-                    if (indenterror(tok)) {
-                        return ERRORTOKEN;
-                    }
+                    return indenterror(tok);
                 }
                 tok->pendin++;
                 tok->indstack[++tok->indent] = col;
@@ -1466,9 +1456,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                     return ERRORTOKEN;
                 }
                 if (altcol != tok->altindstack[tok->indent]) {
-                    if (indenterror(tok)) {
-                        return ERRORTOKEN;
-                    }
+                    return indenterror(tok);
                 }
             }
         }
@@ -1488,8 +1476,18 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
         }
     }
 
+    /* Peek ahead at the next character */
+    c = tok_nextc(tok);
+    tok_backup(tok, c);
+    /* Check if we are closing an async function */
     if (tok->async_def
         && !blankline
+        /* Due to some implementation artifacts of type comments,
+         * a TYPE_COMMENT at the start of a function won't set an
+         * indentation level and it will produce a NEWLINE after it.
+         * To avoid spuriously ending an async function due to this,
+         * wait until we have some non-newline char in front of us. */
+        && c != '\n'
         && tok->level == 0
         /* There was a NEWLINE after ASYNC DEF,
            so we're past the signature. */
@@ -1574,7 +1572,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     /* Identifier (most frequent token!) */
     nonascii = 0;
     if (is_potential_identifier_start(c)) {
-        /* Process b"", r"", u"", br"" and rb"" */
+        /* Process the various legal combinations of b"", r"", u"", and f"". */
         int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0;
         while (1) {
             if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B'))
@@ -1616,7 +1614,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
         /* async/await parsing block. */
         if (tok->cur - tok->start == 5) {
             /* Current token length is 5. */
-            if (tok->async_def) {
+            if (tok->async_always || tok->async_def) {
                 /* We're inside an 'async def' function. */
                 if (memcmp(tok->start, "async", 5) == 0) {
                     return ASYNC;
@@ -1988,9 +1986,7 @@ Ta3Tokenizer_FindEncodingFilename(int fd, PyObject *filename)
     char *p_start =NULL , *p_end =NULL , *encoding = NULL;
 
 #ifndef PGEN
-#if PY_MINOR_VERSION >= 4
     fd = _Py_dup(fd);
-#endif
 #else
     fd = dup(fd);
 #endif

ast3/Parser/tokenizer.h

@@ -47,9 +47,6 @@ struct tok_state {
        (Grammar/Grammar). */
     PyObject *filename;
 #endif
-    int altwarning;     /* Issue warning if alternate tabs don't match */
-    int alterror;       /* Issue error if alternate tabs don't match */
-    int alttabsize;     /* Alternate tab spacing */
     int altindstack[MAXINDENT];         /* Stack of alternate indents */
     /* Stuff for PEP 0263 */
     enum decoding_state decoding_state;
@@ -72,6 +69,7 @@ struct tok_state {
     int async_def_indent; /* Indentation level of the outermost 'async def'. */
     int async_def_nl;     /* =1 if the outermost 'async def' had at least one
                              NEWLINE token after it. */
+    int async_always;     /* =1 if async/await are always keywords */
 };
 
 extern struct tok_state *Ta3Tokenizer_FromString(const char *, int);
@@ -80,8 +78,6 @@ extern struct tok_state *Ta3Tokenizer_FromFile(FILE *, const char*,
                                               const char *, const char *);
 extern void Ta3Tokenizer_Free(struct tok_state *);
 extern int Ta3Tokenizer_Get(struct tok_state *, char **, char **);
-extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
-                                          int len, int *offset);
 
 #ifdef __cplusplus
 }