MANIFEST.in

 recursive-include ast27 *.h
 recursive-include ast3 *.h
+recursive-include ast3/tests *.py
 include LICENSE

PKG-INFO

 Metadata-Version: 1.1
-Name: typed-ast
-Version: 1.3.1
+Name: typed_ast
+Version: 1.4.0
 Summary: a fork of Python 2 and 3 ast modules with type comment support
 Home-page: https://github.com/python/typed_ast
 Author: David Fisher
-Author-email: ddfisher@dropbox.com
+Author-email: UNKNOWN
 License: Apache License 2.0
 Description: `typed_ast` is a Python 3 package that provides a Python 2.7 and Python 3
         parser similar to the standard `ast` library.  Unlike `ast`, the parsers in
@@ -19,9 +19,8 @@ Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: POSIX
 Classifier: Operating System :: Microsoft
-Classifier: Programming Language :: Python :: 3.3
-Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
 Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
 Classifier: Topic :: Software Development

README.md

@@ -8,8 +8,8 @@ parser similar to the standard `ast` library.  Unlike `ast`, the parsers in
 `typed_ast` include [PEP 484](https://www.python.org/dev/peps/pep-0484/) type
 comments and are independent of the version of Python under which they are run.
 The `typed_ast` parsers produce the standard Python AST (plus type comments),
-and are both fast and correct, as they are based on the CPython 2.7 and 3.6
-parsers.  `typed_ast` runs on Python 3.3-3.7 on Linux, OS X and Windows.
+and are both fast and correct, as they are based on the CPython 2.7 and 3.7
+parsers.  `typed_ast` runs on CPython 3.5-3.8 on Linux, OS X and Windows.
 
 ## Development Philosophy
 
@@ -22,18 +22,19 @@ instead.  To avoid feature bloat, any new features for `typed_ast` should have
 the potential to be broadly useful and not be built just for one niche usecase
 or in a manner such that only one project can use them.
 
-### Incompatabilities
+### Incompatibilities
 
 For the purposes of *consuming* syntax trees, this should be a drop-in replacement.
 It is not a drop-in replacement for users that wish to create or transform ASTs,
 as a number of syntax tree classes have additional fields that must be populated
 when constructing them.
 
-### Python 3.7
+### Python 3.8
 
-`typed_ast` has not yet been updated to be based on the Python 3.7
-parser. The main consequence of this that `await` and `async` are
-not treated as keywords.
+`typed_ast` will not be updated to support parsing Python 3.8 and
+newer.  Instead, it is recommended to use the stdlib `ast` module
+there, which has been augmented to support extracting type comments
+and has limited support for parsing older versions of Python 3.
 
 ## Submodules
 ### ast3

ast27/Custom/typed_ast.c

@@ -7,6 +7,7 @@
 #include "ast.h"
 #include "parsetok.h"
 #include "errcode.h"
+#include "graminit.h"
 
 extern grammar _Ta27Parser_Grammar; /* from graminit.c */
 
@@ -264,7 +265,7 @@ ast27_parse_impl(PyObject *source,
     const char *str;
     int compile_mode = -1;
     PyCompilerFlags cf;
-    int start[] = {Py_file_input, Py_eval_input, Py_single_input /*, Py_func_type_input */};
+    int start[] = {file_input, eval_input, single_input, func_type_input };
     PyObject *result;
 
     cf.cf_flags = PyCF_ONLY_AST | PyCF_SOURCE_IS_UTF8;

ast27/Include/Python-ast.h

@@ -387,6 +387,7 @@ struct _type_ignore {
         union {
                 struct {
                         int lineno;
+                        string tag;
                 } TypeIgnore;
                 
         } v;
@@ -540,8 +541,8 @@ arguments_ty _Ta27_arguments(asdl_seq * args, identifier vararg, identifier kwar
 keyword_ty _Ta27_keyword(identifier arg, expr_ty value, PyArena *arena);
 #define alias(a0, a1, a2) _Ta27_alias(a0, a1, a2)
 alias_ty _Ta27_alias(identifier name, identifier asname, PyArena *arena);
-#define TypeIgnore(a0, a1) _Ta27_TypeIgnore(a0, a1)
-type_ignore_ty _Ta27_TypeIgnore(int lineno, PyArena *arena);
+#define TypeIgnore(a0, a1, a2) _Ta27_TypeIgnore(a0, a1, a2)
+type_ignore_ty _Ta27_TypeIgnore(int lineno, string tag, PyArena *arena);
 
 PyObject* Ta27AST_mod2obj(mod_ty t);
 mod_ty Ta27AST_obj2mod(PyObject* ast, PyArena* arena, int mode);

ast27/Include/pgenheaders.h

+#ifndef DUMMY_Py_PGENHEADERS_H
+#define DUMMY_Py_PGENHEADERS_H
+
+/* pgenheaders.h is included by a bunch of files but nothing in it is
+ * used except for the Python.h import, and it was removed in Python
+ * 3.8. Since some of those files are generated we provide a dummy
+ * pgenheaders.h. */
+#include "Python.h"
+
+#endif /* !DUMMY_Py_PGENHEADERS_H */

ast27/Parser/parsetok.c

@@ -152,12 +152,16 @@ warn(const char *msg, const char *filename, int lineno)
 
 
 typedef struct {
-    int *items;
+    struct {
+        int lineno;
+        char *comment;
+    } *items;
     size_t size;
     size_t num_items;
-} growable_int_array;
+} growable_comment_array;
 
-int growable_int_array_init(growable_int_array *arr, size_t initial_size) {
+static int
+growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
     assert(initial_size > 0);
     arr->items = malloc(initial_size * sizeof(*arr->items));
     arr->size = initial_size;
@@ -166,20 +170,28 @@ int growable_int_array_init(growable_int_array *arr, size_t initial_size) {
     return arr->items != NULL;
 }
 
-int growable_int_array_add(growable_int_array *arr, int item) {
+static int
+growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
     if (arr->num_items >= arr->size) {
         arr->size *= 2;
         arr->items = realloc(arr->items, arr->size * sizeof(*arr->items));
-        if (!arr->items)
+        if (!arr->items) {
             return 0;
         }
+    }
 
-    arr->items[arr->num_items] = item;
+    arr->items[arr->num_items].lineno = lineno;
+    arr->items[arr->num_items].comment = comment;
     arr->num_items++;
     return 1;
 }
 
-void growable_int_array_deallocate(growable_int_array *arr) {
+static void
+growable_comment_array_deallocate(growable_comment_array *arr) {
+    unsigned i;
+    for (i = 0; i < arr->num_items; i++) {
+        PyObject_FREE(arr->items[i].comment);
+    }
     free(arr->items);
 }
 
@@ -195,8 +207,8 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
     node *n;
     int started = 0;
 
-    growable_int_array type_ignores;
-    if (!growable_int_array_init(&type_ignores, 10)) {
+    growable_comment_array type_ignores;
+    if (!growable_comment_array_init(&type_ignores, 10)) {
         err_ret->error = E_NOMEM;
         Ta27Tokenizer_Free(tok);
         return NULL;
@@ -264,7 +276,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             col_offset = -1;
 
         if (type == TYPE_IGNORE) {
-            if (!growable_int_array_add(&type_ignores, tok->lineno)) {
+            if (!growable_comment_array_add(&type_ignores, tok->lineno, str)) {
                 err_ret->error = E_NOMEM;
                 break;
             }
@@ -297,15 +309,23 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             REQ(ch, ENDMARKER);
 
             for (i = 0; i < type_ignores.num_items; i++) {
-                Ta27Node_AddChild(ch, TYPE_IGNORE, NULL, type_ignores.items[i], 0);
+                int res = Ta27Node_AddChild(ch, TYPE_IGNORE, type_ignores.items[i].comment,
+                                            type_ignores.items[i].lineno, 0);
+                if (res != 0) {
+                    err_ret->error = res;
+                    Ta27Node_Free(n);
+                    n = NULL;
+                    break;
+                }
+                type_ignores.items[i].comment = NULL;
             }
         }
-        growable_int_array_deallocate(&type_ignores);
-
     }
     else
         n = NULL;
 
+    growable_comment_array_deallocate(&type_ignores);
+
 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
     *flags = ps->p_flags;
 #endif

ast27/Parser/tokenizer.c

@@ -1400,20 +1400,22 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
         /* This is a type comment if we matched all of type_comment_prefix. */
         if (!*prefix) {
             int is_type_ignore = 1;
+            const char *ignore_end = p + 6;
             tok_backup(tok, c);  /* don't eat the newline or EOF */
 
             type_start = p;
 
-            is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0;
-            p += 6;
-            while (is_type_ignore && p < tok->cur) {
-              if (*p == '#')
-                  break;
-              is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t');
-              p++;
-            }
+            /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
+             * or anything ASCII and non-alphanumeric. */
+            is_type_ignore = (
+                tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
+                && !(tok->cur > ignore_end
+                     && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
 
             if (is_type_ignore) {
+                *p_start = (char *) ignore_end;
+                *p_end = tok->cur;
+
                 /* If this type ignore is the only thing on the line, consume the newline also. */
                 if (blankline) {
                     tok_nextc(tok);

ast27/Python/Python-ast.c

@@ -396,6 +396,7 @@ static PyObject* ast2obj_type_ignore(void*);
 static PyTypeObject *TypeIgnore_type;
 static char *TypeIgnore_fields[]={
         "lineno",
+        "tag",
 };
 
 
@@ -974,7 +975,7 @@ static int init_types(void)
         type_ignore_type = make_type("type_ignore", &AST_type, NULL, 0);
         if (!type_ignore_type) return 0;
         if (!add_attributes(type_ignore_type, NULL, 0)) return 0;
-        TypeIgnore_type = make_type("TypeIgnore", type_ignore_type, TypeIgnore_fields, 1);
+        TypeIgnore_type = make_type("TypeIgnore", type_ignore_type, TypeIgnore_fields, 2);
         if (!TypeIgnore_type) return 0;
         initialized = 1;
         return 1;
@@ -2153,14 +2154,20 @@ alias(identifier name, identifier asname, PyArena *arena)
 }
 
 type_ignore_ty
-TypeIgnore(int lineno, PyArena *arena)
+TypeIgnore(int lineno, string tag, PyArena *arena)
 {
         type_ignore_ty p;
+        if (!tag) {
+                PyErr_SetString(PyExc_ValueError,
+                                "field tag is required for TypeIgnore");
+                return NULL;
+        }
         p = (type_ignore_ty)PyArena_Malloc(arena, sizeof(*p));
         if (!p)
                 return NULL;
         p->kind = TypeIgnore_kind;
         p->v.TypeIgnore.lineno = lineno;
+        p->v.TypeIgnore.tag = tag;
         return p;
 }
 
@@ -3408,6 +3415,11 @@ ast2obj_type_ignore(void* _o)
                 if (PyObject_SetAttrString(result, "lineno", value) == -1)
                         goto failed;
                 Py_DECREF(value);
+                value = ast2obj_string(o->v.TypeIgnore.tag);
+                if (!value) goto failed;
+                if (PyObject_SetAttrString(result, "tag", value) == -1)
+                        goto failed;
+                Py_DECREF(value);
                 break;
         }
         return result;
@@ -6848,6 +6860,7 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
         }
         if (isinstance) {
                 int lineno;
+                string tag;
 
                 if (PyObject_HasAttrString(obj, "lineno")) {
                         int res;
@@ -6861,7 +6874,19 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
                         PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from TypeIgnore");
                         return 1;
                 }
-                *out = TypeIgnore(lineno, arena);
+                if (PyObject_HasAttrString(obj, "tag")) {
+                        int res;
+                        tmp = PyObject_GetAttrString(obj, "tag");
+                        if (tmp == NULL) goto failed;
+                        res = obj2ast_string(tmp, &tag, arena);
+                        if (res != 0) goto failed;
+                        Py_XDECREF(tmp);
+                        tmp = NULL;
+                } else {
+                        PyErr_SetString(PyExc_TypeError, "required field \"tag\" missing from TypeIgnore");
+                        return 1;
+                }
+                *out = TypeIgnore(lineno, tag, arena);
                 if (*out == NULL) goto failed;
                 return 0;
         }

ast27/Python/ast.c

@@ -297,7 +297,10 @@ Ta27AST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
                 goto error;
 
             for (i = 0; i < num; i++) {
-                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), arena);
+                string type_comment = new_type_comment(STR(CHILD(ch, i)), &c);
+                if (!type_comment)
+                    goto error;
+                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), type_comment, arena);
                 if (!ti)
                     goto error;
                 asdl_seq_SET(type_ignores, i, ti);

ast3/Custom/typed_ast.c

 #include "Python.h"
 #include "Python-ast.h"
-#include "compile-ast3.h"
 #include "node.h"
 #include "grammar.h"
 #include "token.h"
 #include "ast.h"
 #include "parsetok.h"
 #include "errcode.h"
+#include "graminit.h"
 
 extern grammar _Ta3Parser_Grammar; /* from graminit.c */
 
@@ -302,7 +302,7 @@ ast3_parse_impl(PyObject *source,
     const char *str;
     int compile_mode = -1;
     PyCompilerFlags cf;
-    int start[] = {Py_file_input, Py_eval_input, Py_single_input, Py_func_type_input};
+    int start[] = {file_input, eval_input, single_input, func_type_input};
     PyObject *result;
 
     cf.cf_flags = PyCF_ONLY_AST | PyCF_SOURCE_IS_UTF8;

ast3/Include/Python-ast.h

@@ -462,6 +462,7 @@ struct _type_ignore {
     union {
         struct {
             int lineno;
+            string tag;
         } TypeIgnore;
 
     } v;
@@ -668,8 +669,8 @@ alias_ty _Ta3_alias(identifier name, identifier asname, PyArena *arena);
 #define withitem(a0, a1, a2) _Ta3_withitem(a0, a1, a2)
 withitem_ty _Ta3_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena
                           *arena);
-#define TypeIgnore(a0, a1) _Ta3_TypeIgnore(a0, a1)
-type_ignore_ty _Ta3_TypeIgnore(int lineno, PyArena *arena);
+#define TypeIgnore(a0, a1, a2) _Ta3_TypeIgnore(a0, a1, a2)
+type_ignore_ty _Ta3_TypeIgnore(int lineno, string tag, PyArena *arena);
 
 PyObject* Ta3AST_mod2obj(mod_ty t);
 mod_ty Ta3AST_obj2mod(PyObject* ast, PyArena* arena, int mode);

ast3/Include/compile-ast3.h

-/* These definitions must match corresponding definitions in graminit.h.
-   There's code in compile.c that checks that they are the same. */
-#define Py_single_input 256
-#define Py_file_input 257
-#define Py_eval_input 258
-#define Py_func_type_input 343

ast3/Include/pgenheaders.h

+#ifndef DUMMY_Py_PGENHEADERS_H
+#define DUMMY_Py_PGENHEADERS_H
+
+/* pgenheaders.h is included by a bunch of files but nothing in it is
+ * used except for the Python.h import, and it was removed in Python
+ * 3.8. Since some of those files are generated we provide a dummy
+ * pgenheaders.h. */
+#include "Python.h"
+
+#endif /* !DUMMY_Py_PGENHEADERS_H */

ast3/Parser/parsetok.c

@@ -180,12 +180,16 @@ warn(const char *msg, const char *filename, int lineno)
 #endif
 
 typedef struct {
-    int *items;
+    struct {
+        int lineno;
+        char *comment;
+    } *items;
     size_t size;
     size_t num_items;
-} growable_int_array;
+} growable_comment_array;
 
-int growable_int_array_init(growable_int_array *arr, size_t initial_size) {
+static int
+growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
     assert(initial_size > 0);
     arr->items = malloc(initial_size * sizeof(*arr->items));
     arr->size = initial_size;
@@ -194,20 +198,28 @@ int growable_int_array_init(growable_int_array *arr, size_t initial_size) {
     return arr->items != NULL;
 }
 
-int growable_int_array_add(growable_int_array *arr, int item) {
+static int
+growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
     if (arr->num_items >= arr->size) {
         arr->size *= 2;
         arr->items = realloc(arr->items, arr->size * sizeof(*arr->items));
-        if (!arr->items)
+        if (!arr->items) {
             return 0;
         }
+    }
 
-    arr->items[arr->num_items] = item;
+    arr->items[arr->num_items].lineno = lineno;
+    arr->items[arr->num_items].comment = comment;
     arr->num_items++;
     return 1;
 }
 
-void growable_int_array_deallocate(growable_int_array *arr) {
+static void
+growable_comment_array_deallocate(growable_comment_array *arr) {
+    unsigned i;
+    for (i = 0; i < arr->num_items; i++) {
+        PyObject_FREE(arr->items[i].comment);
+    }
     free(arr->items);
 }
 
@@ -222,8 +234,8 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
     node *n;
     int started = 0;
 
-    growable_int_array type_ignores;
-    if (!growable_int_array_init(&type_ignores, 10)) {
+    growable_comment_array type_ignores;
+    if (!growable_comment_array_init(&type_ignores, 10)) {
         err_ret->error = E_NOMEM;
         Ta3Tokenizer_Free(tok);
         return NULL;
@@ -302,8 +314,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
         }
 
         if (type == TYPE_IGNORE) {
-            PyObject_FREE(str);
-            if (!growable_int_array_add(&type_ignores, tok->lineno)) {
+            if (!growable_comment_array_add(&type_ignores, tok->lineno, str)) {
                 err_ret->error = E_NOMEM;
                 break;
             }
@@ -337,17 +348,24 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             REQ(ch, ENDMARKER);
 
             for (i = 0; i < type_ignores.num_items; i++) {
-                Ta3Node_AddChild(ch, TYPE_IGNORE, NULL, type_ignores.items[i], 0);
+                int res = Ta3Node_AddChild(ch, TYPE_IGNORE, type_ignores.items[i].comment,
+                                           type_ignores.items[i].lineno, 0);
+                if (res != 0) {
+                    err_ret->error = res;
+                    Ta3Node_Free(n);
+                    n = NULL;
+                    break;
+                }
+                type_ignores.items[i].comment = NULL;
             }
         }
-        growable_int_array_deallocate(&type_ignores);
 
 #ifndef PGEN
         /* Check that the source for a single input statement really
            is a single statement by looking at what is left in the
            buffer after parsing.  Trailing whitespace and comments
            are OK.  */
-        if (start == single_input) {
+        if (err_ret->error == E_DONE && start == single_input) {
             char *cur = tok->cur;
             char c = *tok->cur;
 
@@ -375,6 +393,8 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
     else
         n = NULL;
 
+    growable_comment_array_deallocate(&type_ignores);
+
 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
     *flags = ps->p_flags;
 #endif

ast3/Parser/tokenizer.c

@@ -1536,20 +1536,22 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
         /* This is a type comment if we matched all of type_comment_prefix. */
         if (!*prefix) {
             int is_type_ignore = 1;
+            const char *ignore_end = p + 6;
             tok_backup(tok, c);  /* don't eat the newline or EOF */
 
             type_start = p;
 
-            is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0;
-            p += 6;
-            while (is_type_ignore && p < tok->cur) {
-              if (*p == '#')
-                  break;
-              is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t');
-              p++;
-            }
+            /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
+             * or anything ASCII and non-alphanumeric. */
+            is_type_ignore = (
+                tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
+                && !(tok->cur > ignore_end
+                     && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
 
             if (is_type_ignore) {
+                *p_start = (char *) ignore_end;
+                *p_end = tok->cur;
+
                 /* If this type ignore is the only thing on the line, consume the newline also. */
                 if (blankline) {
                     tok_nextc(tok);

ast3/Python/Python-ast.c

@@ -525,8 +525,10 @@ static char *withitem_fields[]={
 static PyTypeObject *type_ignore_type;
 static PyObject* ast2obj_type_ignore(void*);
 static PyTypeObject *TypeIgnore_type;
+_Py_IDENTIFIER(tag);
 static char *TypeIgnore_fields[]={
     "lineno",
+    "tag",
 };
 
 
@@ -1213,7 +1215,7 @@ static int init_types(void)
     if (!type_ignore_type) return 0;
     if (!add_attributes(type_ignore_type, NULL, 0)) return 0;
     TypeIgnore_type = make_type("TypeIgnore", type_ignore_type,
-                                TypeIgnore_fields, 1);
+                                TypeIgnore_fields, 2);
     if (!TypeIgnore_type) return 0;
     initialized = 1;
     return 1;
@@ -2658,14 +2660,20 @@ withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena)
 }
 
 type_ignore_ty
-TypeIgnore(int lineno, PyArena *arena)
+TypeIgnore(int lineno, string tag, PyArena *arena)
 {
     type_ignore_ty p;
+    if (!tag) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field tag is required for TypeIgnore");
+        return NULL;
+    }
     p = (type_ignore_ty)PyArena_Malloc(arena, sizeof(*p));
     if (!p)
         return NULL;
     p->kind = TypeIgnore_kind;
     p->v.TypeIgnore.lineno = lineno;
+    p->v.TypeIgnore.tag = tag;
     return p;
 }
 
@@ -4130,6 +4138,11 @@ ast2obj_type_ignore(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_lineno, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_string(o->v.TypeIgnore.tag);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_tag, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     }
     return result;
@@ -8591,6 +8604,7 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
     }
     if (isinstance) {
         int lineno;
+        string tag;
 
         if (lookup_attr_id(obj, &PyId_lineno, &tmp) < 0) {
             return 1;
@@ -8605,7 +8619,20 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = TypeIgnore(lineno, arena);
+        if (lookup_attr_id(obj, &PyId_tag, &tmp) < 0) {
+            return 1;
+        }
+        if (tmp == NULL) {
+            PyErr_SetString(PyExc_TypeError, "required field \"tag\" missing from TypeIgnore");
+            return 1;
+        }
+        else {
+            int res;
+            res = obj2ast_string(tmp, &tag, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        }
+        *out = TypeIgnore(lineno, tag, arena);
         if (*out == NULL) goto failed;
         return 0;
     }

ast3/Python/ast.c

@@ -17,7 +17,7 @@ typedef int bool;
 #define false 0
 #define true 1
 
-#ifndef _PyObject_FastCall
+#if PY_MINOR_VERSION < 6
 static PyObject *
 _PyObject_FastCall(PyObject *func, PyObject *const *args, int nargs)
 {
@@ -29,6 +29,7 @@ _PyObject_FastCall(PyObject *func, PyObject *const *args, int nargs)
         return NULL;
     }
     for (i = 0; i < nargs; i++) {
+        Py_INCREF(args[i]);
         if (PyTuple_SetItem(t, i, args[i]) < 0) {
             Py_DECREF(t);
             return NULL;
@@ -909,7 +910,10 @@ Ta3AST_FromNodeObject(const node *n, PyCompilerFlags *flags,
                 goto out;
 
             for (i = 0; i < num; i++) {
-                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), arena);
+                string type_comment = new_type_comment(STR(CHILD(ch, i)), &c);
+                if (!type_comment)
+                    goto out;
+                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), type_comment, arena);
                 if (!ti)
                    goto out;
                asdl_seq_SET(type_ignores, i, ti);
@@ -1445,7 +1449,7 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start,
                     goto error;
                 asdl_seq_SET(kwonlyargs, j++, arg);
                 i += 1; /* the name */
-                if (TYPE(CHILD(n, i)) == COMMA)
+                if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA)
                     i += 1; /* the comma, if present */
                 break;
             case TYPE_COMMENT:
@@ -1644,7 +1648,7 @@ ast_for_arguments(struct compiling *c, const node *n)
                 if (!kwarg)
                     return NULL;
                 i += 2; /* the double star and the name */
-                if (TYPE(CHILD(n, i)) == COMMA)
+                if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA)
                     i += 1; /* the comma, if present */
                 break;
             case TYPE_COMMENT:

ast3/tests/test_basics.py

+import os
+
+import pytest
+
+from typed_ast import _ast3
+from typed_ast import _ast27
+import typed_ast.conversions
+
+# Lowest and highest supported Python 3 minor version (inclusive)
+MIN_VER = 4
+MAX_VER = 7
+NEXT_VER = MAX_VER + 1
+
+
+basics = """\
+def foo():
+    # type: () -> int
+    pass
+
+def bar():  # type: () -> None
+    pass
+"""
+def test_basics():
+    for version in range(MIN_VER, NEXT_VER):
+        tree = _ast3._parse(basics, "<basics>", "exec", version)
+        assert tree.body[0].type_comment == "() -> int"
+        assert tree.body[1].type_comment == "() -> None"
+
+
+redundantdef = """\
+def foo():  # type: () -> int
+    # type: () -> str
+    return ''
+"""
+def test_redundantdef():
+    for version in range(MIN_VER, NEXT_VER):
+        with pytest.raises(SyntaxError):
+            t = _ast3._parse(redundantdef, "<redundantdef>", "exec", version)
+
+
+vardecl = """\
+a = 0  # type: int
+a  # type: int
+"""
+def test_vardecl():
+    for version in range(MIN_VER, NEXT_VER):
+        tree = _ast3._parse(vardecl, "<vardecl>", "exec", version)
+        assert tree.body[0].type_comment == "int"
+        # Curious fact: an expression can have a type comment
+        # but it is lost in the AST.
+
+
+forstmt = """\
+for a in []:  # type: int
+    pass
+"""
+def test_forstmt():
+    for version in range(MIN_VER, NEXT_VER):
+        tree = _ast3._parse(forstmt, "<forstmt>", "exec", version)
+        assert tree.body[0].type_comment == "int"
+
+
+withstmt = """\
+with context():  # type: int
+    pass
+"""
+def test_withstmt():
+    for version in range(MIN_VER, NEXT_VER):
+        tree = _ast3._parse(withstmt, "<withstmt>", "exec", version)
+        assert tree.body[0].type_comment == "int"
+
+
+# A test function named 'fabvk' would have two positional args, a and b,
+# plus a var-arg *v, plus a kw-arg **k.  It is verified in test_longargs()
+# that it has exactly these arguments, no more, no fewer.
+longargs = """\
+def fa(
+    a = 1,  # type: A
+):
+    pass
+
+def fa(
+    a = 1  # type: A
+):
+    pass
+
+def fab(
+    a,  # type: A
+    b,  # type: B
+):
+    pass
+
+def fab(
+    a,  # type: A
+    b  # type: B
+):
+    pass
+
+def fv(
+    *v,  # type: V
+):
+    pass
+
+def fv(
+    *v  # type: V
+):
+    pass
+
+def fk(
+    **k,  # type: K
+):
+    pass
+
+def fk(
+    **k  # type: K
+):
+    pass
+
+def fvk(
+    *v,  # type: V
+    **k,  # type: K
+):
+    pass
+
+def fvk(
+    *v,  # type: V
+    **k  # type: K
+):
+    pass
+
+def fav(
+    a,  # type: A
+    *v,  # type: V
+):
+    pass
+
+def fav(
+    a,  # type: A
+    *v  # type: V
+):
+    pass
+
+def fak(
+    a,  # type: A
+    **k,  # type: K
+):
+    pass
+
+def fak(
+    a,  # type: A
+    **k  # type: K
+):
+    pass
+
+def favk(
+    a,  # type: A
+    *v,  # type: V
+    **k,  # type: K
+):
+    pass
+
+def favk(
+    a,  # type: A
+    *v,  # type: V
+    **k  # type: K
+):
+    pass
+
+"""
+def test_longargs():
+    for version in range(MIN_VER, NEXT_VER):
+        tree = _ast3._parse(longargs, "<longargs>", "exec", version)
+        for t in tree.body:
+            # The expected args are encoded in the function name
+            todo = set(t.name[1:])
+            assert len(t.args.args) == len(todo) - bool(t.args.vararg) - bool(t.args.kwarg)
+            assert t.name.startswith('f')
+            for c in t.name[1:]:
+                todo.remove(c)
+                if c == 'v':
+                    arg = t.args.vararg
+                elif c == 'k':
+                    arg = t.args.kwarg
+                else:
+                    assert 0 <= ord(c) - ord('a') < len(t.args.args)
+                    arg = t.args.args[ord(c) - ord('a')]
+                assert arg.arg == c  # That's the argument name
+                assert arg.type_comment == arg.arg.upper()
+            assert not todo
+
+
+ignores = """\
+def foo():
+    pass  # type: ignore
+
+def bar():
+    x = 1  # type: ignore
+
+def baz():
+    pass  # type: ignore[excuse]
+    pass  # type: ignore=excuse
+    pass  # type: ignore [excuse]
+    x = 1  # type: ignore whatever
+"""
+def test_ignores():
+    expected = [
+        (2, ''),
+        (5, ''),
+        (8, '[excuse]'),
+        (9, '=excuse'),
+        (10, ' [excuse]'),
+        (11, ' whatever'),
+    ]
+
+    for version in range(MIN_VER, NEXT_VER):
+        tree = _ast3._parse(ignores, "<ignores>", "exec", version)
+        assert [(ti.lineno, ti.tag) for ti in tree.type_ignores] == expected
+        with pytest.raises(SyntaxError):
+            _ast3._parse("pass  # type: ignoreé\n", "<ignores>", "exec", version)
+
+
+    tree = _ast27.parse(ignores, "<ignores>", "exec")
+    assert [(ti.lineno, ti.tag) for ti in tree.type_ignores] == expected
+    with pytest.raises(SyntaxError):
+        _ast27.parse("pass  # type: ignoreé\n", "<ignores>", "exec")
+
+
+
+asyncfunc = """\
+async def foo():
+    # type: () -> int
+    return await bar()
+"""
+def test_asyncfunc():
+    for version in range(3, 5):
+        with pytest.raises(SyntaxError):
+            _ast3._parse(asyncfunc, "<asyncfunc>", "exec", version)
+    for version in range(5, NEXT_VER):
+        tree = _ast3._parse(asyncfunc, "<asyncfunc>", "exec", version)
+        assert tree.body[0].type_comment == "() -> int"
+
+
+asyncvar = """\
+async = 12
+await = 13
+"""
+def test_asyncvar():
+    for version in range(3, 7):
+        tree = _ast3._parse(asyncvar, "<asyncvar>", "exec", version)
+    for version in range(7, NEXT_VER):
+        with pytest.raises(SyntaxError):
+            _ast3._parse(asyncvar, "<asyncvar>", "exec", version)
+
+
+asynccomp = """\
+async def foo(xs):
+    [x async for x in xs]
+"""
+def test_asynccomp():
+    for version in range(3, 6):
+        with pytest.raises(SyntaxError):
+            tree = _ast3._parse(asynccomp, "<asynccomp>", "exec", version)
+    for version in range(6, NEXT_VER):
+            _ast3._parse(asynccomp, "<asynccomp>", "exec", version)
+
+
+matmul = """\
+a = b @ c
+"""
+def test_matmul():
+    for version in range(3, 5):
+        with pytest.raises(SyntaxError):
+            tree = _ast3._parse(matmul, "<matmul>", "exec", version)
+    for version in range(5, NEXT_VER):
+        tree = _ast3._parse(matmul, "<matmul>", "exec", version)
+
+
+strkind = """\
+plain = 'abc'
+raw = r'abc'
+plain_bytes = b'abc'
+raw_bytes = br'abc'
+"""
+def test_strkind():
+    # Test that Str() objects have a kind argument/attribute.
+    node = _ast3.Str("foo", "r")
+    assert node.s == "foo"
+    assert node.kind == "r"
+    for version in range(MIN_VER, NEXT_VER):
+        tree = _ast3._parse(strkind, "<strkind>", "exec", version)
+        assert tree.body[0].value.kind == ""
+        assert tree.body[1].value.kind == "r"
+        assert tree.body[2].value.kind == "b"
+        assert tree.body[3].value.kind == "br"
+
+
+basic_py2 = """\
+a = 'hello'
+b = u'hello'
+c = b'hello'
+"""
+def test_convert_strs():
+    ast = _ast27.parse(basic_py2, "<basic_py2>", "exec")
+    tree = typed_ast.conversions.py2to3(ast)
+    assert tree.body[0].value.kind == ""
+    assert tree.body[1].value.kind == "u"
+    assert tree.body[2].value.kind == "b"
+
+simple_fstring = """\
+f'{5}'
+"""
+def test_simple_fstring():
+    for version in range(6, NEXT_VER):
+        tree = _ast3._parse(simple_fstring, "<fstring>", "exec", version)
+        assert isinstance(tree.body[0].value, _ast3.JoinedStr)
+        assert isinstance(tree.body[0].value.values[0].value, _ast3.Num)
+
+# Test the interaction between versions and f strings
+await_fstring = """\
+f'1 + {f"{await}"}'
+"""
+def test_await_fstring():
+    # Should work on 6 but fail on 7
+    _ast3._parse(await_fstring, "<bad-f-string>", "exec", 6)
+    with pytest.raises(SyntaxError):
+        _ast3._parse(await_fstring, "<bad-f-string>", "exec", 7)

debian/changelog

+python3-typed-ast (1.4.0-1) unstable; urgency=medium
+
+  * New upstream release.
+
+ -- Michael R. Crusoe <michael.crusoe@gmail.com>  Sat, 13 Jul 2019 14:29:34 +0200
+
 python3-typed-ast (1.3.1-1) unstable; urgency=medium
 
   * New upstream version