PKG-INFO

 Metadata-Version: 1.1
 Name: typed-ast
-Version: 1.2.0
+Version: 1.3.0
 Summary: a fork of Python 2 and 3 ast modules with type comment support
 Home-page: https://github.com/python/typed_ast
 Author: David Fisher

ast27/Python/Python-ast.c

@@ -477,7 +477,7 @@ static PyMethodDef ast_type_methods[] = {
 
 static PyTypeObject AST_type = {
     PyVarObject_HEAD_INIT(NULL, 0)
-    "_ast27.AST",
+    "typed_ast._ast27.AST",
     sizeof(PyObject),
     0,
     0,                       /* tp_dealloc */
@@ -533,7 +533,7 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
         PyTuple_SET_ITEM(fnames, i, field);
     }
     result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}",
-                    type, base, "_fields", fnames, "__module__", "_ast27");
+                    type, base, "_fields", fnames, "__module__", "typed_ast._ast27");
     Py_DECREF(fnames);
     return (PyTypeObject*)result;
 }

ast27/Python/ast.c

@@ -1500,7 +1500,6 @@ ast_for_atom(struct compiling *c, const node *n)
     case STRING: {
         PyObject *kind, *str = parsestrplus(c, n);
         const char *raw, *s = STR(CHILD(n, 0));
-        int quote = Py_CHARMASK(*s);
         /* currently Python allows up to 2 string modifiers */
         char *ch, s_kind[3] = {0, 0, 0};
         ch = s_kind;
@@ -1519,7 +1518,7 @@ ast_for_atom(struct compiling *c, const node *n)
                 PyErr_Fetch(&type, &value, &tback);
                 errstr = PyObject_Str(value);
                 if (errstr) {
-                    char *s = "";
+                    const char *s = "";
                     char buf[128];
                     s = _PyUnicode_AsString(errstr);
                     PyOS_snprintf(buf, sizeof(buf), "(unicode error) %s", s);
@@ -2190,7 +2189,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
                 keyword_ty kw;
                 identifier key;
                 int k;
-                char *tmp;
+                const char *tmp;
 
                 /* CHILD(ch, 0) is test, but must be an identifier? */
                 e = ast_for_expr(c, CHILD(ch, 0));

ast3/Custom/typed_ast.c

 #include "Python.h"
 #include "Python-ast.h"
-#include "compile.h"
+#include "compile-ast3.h"
 #include "node.h"
 #include "grammar.h"
 #include "token.h"
@@ -211,9 +211,33 @@ err_free(perrdetail *err)
     Py_CLEAR(err->filename);
 }
 
+// from Python/pythonrun.c
+node *
+Ta3Parser_SimpleParseStringFlagsFilename(const char *str, const char *filename,
+                                         int start, int flags)
+{
+    perrdetail err;
+    node *n = Ta3Parser_ParseStringFlagsFilename(str, filename,
+                            &_Ta3Parser_Grammar, start, &err, flags);
+    if (n == NULL)
+        err_input(&err);
+    err_free(&err);
+    return n;
+}
+
+/* update compiler and parser flags based on feature version */
+void
+_Ta3Parser_UpdateFlags(PyCompilerFlags *flags, int *iflags, int feature_version)
+{
+    *iflags = PARSER_FLAGS(flags);
+    if (feature_version >= 7)
+        *iflags |= PyPARSE_ASYNC_ALWAYS;
+    flags->cf_flags |= *iflags & PyCF_MASK;
+}
+
 // copy of PyParser_ASTFromStringObject in Python/pythonrun.c
 /* Preferred access to parser is through AST. */
-mod_ty
+static mod_ty
 string_object_to_c_ast(const char *s, PyObject *filename, int start,
                              PyCompilerFlags *flags, int feature_version,
                              PyArena *arena)
@@ -221,15 +245,17 @@ string_object_to_c_ast(const char *s, PyObject *filename, int start,
     mod_ty mod;
     PyCompilerFlags localflags;
     perrdetail err;
-    int iflags = PARSER_FLAGS(flags);
+    node *n;
+    int iflags;
 
-    node *n = Ta3Parser_ParseStringObject(s, filename,
-                                         &_Ta3Parser_Grammar, start, &err,
-                                         &iflags);
     if (flags == NULL) {
         localflags.cf_flags = 0;
         flags = &localflags;
     }
+    _Ta3Parser_UpdateFlags(flags, &iflags, feature_version);
+    n = Ta3Parser_ParseStringObject(s, filename,
+                                    &_Ta3Parser_Grammar, start, &err,
+                                    &iflags);
     if (n) {
         flags->cf_flags |= iflags & PyCF_MASK;
         mod = Ta3AST_FromNodeObject(n, flags, filename, feature_version, arena);

ast3/Include/Python-ast.h

@@ -333,6 +333,7 @@ struct _expr {
 
         struct {
             bytes s;
+            string kind;
         } Bytes;
 
         struct {
@@ -611,8 +612,9 @@ expr_ty _Ta3_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
 #define JoinedStr(a0, a1, a2, a3) _Ta3_JoinedStr(a0, a1, a2, a3)
 expr_ty _Ta3_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena
                        *arena);
-#define Bytes(a0, a1, a2, a3) _Ta3_Bytes(a0, a1, a2, a3)
-expr_ty _Ta3_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
+#define Bytes(a0, a1, a2, a3, a4) _Ta3_Bytes(a0, a1, a2, a3, a4)
+expr_ty _Ta3_Bytes(bytes s, string kind, int lineno, int col_offset, PyArena
+                   *arena);
 #define NameConstant(a0, a1, a2, a3) _Ta3_NameConstant(a0, a1, a2, a3)
 expr_ty _Ta3_NameConstant(singleton value, int lineno, int col_offset, PyArena
                           *arena);

ast3/Include/ast.h

@@ -18,6 +18,13 @@ extern mod_ty Ta3AST_FromNodeObject(
     int feature_version,
     PyArena *arena);
 
+#ifndef Py_LIMITED_API
+
+/* _PyAST_ExprAsUnicode is defined in ast_unparse.c */
+extern PyObject * _PyAST_ExprAsUnicode(expr_ty);
+
+#endif /* !Py_LIMITED_API */
+
 #ifdef __cplusplus
 }
 #endif

ast3/Include/compile.h → ast3/Include/compile-ast3.h

-#ifndef Ta3_COMPILE_H
-#define Ta3_COMPILE_H
-
 /* These definitions must match corresponding definitions in graminit.h.
    There's code in compile.c that checks that they are the same. */
 #define Py_single_input 256
 #define Py_file_input 257
 #define Py_eval_input 258
-#define Py_func_type_input 342
-
-#endif /* !Ta3_COMPILE_H */
+#define Py_func_type_input 343

ast3/Include/graminit.h

@@ -81,11 +81,12 @@
 #define arglist 334
 #define argument 335
 #define comp_iter 336
-#define comp_for 337
-#define comp_if 338
-#define encoding_decl 339
-#define yield_expr 340
-#define yield_arg 341
-#define func_type_input 342
-#define func_type 343
-#define typelist 344
+#define sync_comp_for 337
+#define comp_for 338
+#define comp_if 339
+#define encoding_decl 340
+#define yield_expr 341
+#define yield_arg 342
+#define func_type_input 343
+#define func_type 344
+#define typelist 345

ast3/Include/parsetok.h

@@ -34,6 +34,7 @@ typedef struct {
 
 #define PyPARSE_IGNORE_COOKIE 0x0010
 #define PyPARSE_BARRY_AS_BDFL 0x0020
+#define PyPARSE_ASYNC_ALWAYS  0x8000
 
 extern node * Ta3Parser_ParseString(const char *, grammar *, int,
                                               perrdetail *);
@@ -98,8 +99,8 @@ extern node * Ta3Parser_ParseStringObject(
 
 /* Note that the following functions are defined in pythonrun.c,
    not in parsetok.c */
-PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
-PyAPI_FUNC(void) PyParser_ClearError(perrdetail *);
+extern void PyParser_SetError(perrdetail *);
+extern void PyParser_ClearError(perrdetail *);
 
 #ifdef __cplusplus
 }

ast3/Include/token.h

@@ -69,7 +69,11 @@ extern "C" {
 #define TYPE_IGNORE	56
 #define TYPE_COMMENT	57
 #define ERRORTOKEN      58
-#define N_TOKENS	59
+/* These aren't used by the C tokenizer but are needed for tokenize.py */
+#define COMMENT         59
+#define NL              60
+#define ENCODING                61
+#define N_TOKENS        62
 
 /* Special definitions for cooperation with parser */
 

ast3/Parser/grammar1.c

@@ -25,8 +25,7 @@ Ta3Grammar_FindDFA(grammar *g, int type)
         if (d->d_type == type)
             return d;
     }
-    assert(0);
-    /* NOTREACHED */
+    abort();
 #endif
 }
 

ast3/Parser/parsetok.c

@@ -64,6 +64,8 @@ Ta3Parser_ParseStringObject(const char *s, PyObject *filename,
     Py_INCREF(err_ret->filename);
     tok->filename = err_ret->filename;
 #endif
+    if (*flags & PyPARSE_ASYNC_ALWAYS)
+        tok->async_always = 1;
     return parsetok(tok, g, start, err_ret, flags);
 }
 
@@ -264,7 +266,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
         }
         else
             started = 1;
-        len = b - a; /* XXX this may compute NULL - NULL */
+        len = (a != NULL && b != NULL) ? b - a : 0;
         str = (char *) PyObject_MALLOC(len + 1);
         if (str == NULL) {
             err_ret->error = E_NOMEM;
@@ -285,18 +287,19 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
                             strcmp(str, "<>")) {
                 PyObject_FREE(str);
-                err_ret->text = "with Barry as BDFL, use '<>' "
-                                "instead of '!='";
+                err_ret->expected = NOTEQUAL;
                 err_ret->error = E_SYNTAX;
                 break;
             }
         }
 #endif
-        if (a >= tok->line_start)
+        if (a != NULL && a >= tok->line_start) {
             col_offset = Py_SAFE_DOWNCAST(a - tok->line_start,
                                           intptr_t, int);
-        else
+        }
+        else {
             col_offset = -1;
+        }
 
         if (type == TYPE_IGNORE) {
             if (!growable_int_array_add(&type_ignores, tok->lineno)) {

ast3/Parser/tokenizer.c

@@ -27,6 +27,13 @@
     } while (0)
 #endif /* Py_XSETREF */
 
+#ifndef _PyObject_CallNoArg
+#define _PyObject_CallNoArg(func) PyObject_CallObject(func, NULL)
+#endif
+
+/* Alternate tab spacing */
+#define ALTTABSIZE 1
+
 #define is_potential_identifier_start(c) (\
               (c >= 'a' && c <= 'z')\
                || (c >= 'A' && c <= 'Z')\
@@ -40,12 +47,7 @@
                || c == '_'\
                || (c >= 128))
 
-#if PY_MINOR_VERSION >= 4
 PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *);
-#else
-// Python 3.3 doesn't have PyAPI_FUNC, but it's not supported on Windows anyway.
-char *PyOS_Readline(FILE *, FILE *, char *);
-#endif
 /* Return malloc'ed string including trailing \n;
    empty malloc'ed string for EOF;
    NULL if interrupted */
@@ -122,6 +124,9 @@ const char *_Ta3Parser_TokenNames[] = {
     "TYPE_IGNORE",
     "TYPE_COMMENT",
     "<ERRORTOKEN>",
+    "COMMENT",
+    "NL",
+    "ENCODING",
     "<N_TOKENS>"
 };
 
@@ -152,9 +157,6 @@ tok_new(void)
     tok->prompt = tok->nextprompt = NULL;
     tok->lineno = 0;
     tok->level = 0;
-    tok->altwarning = 1;
-    tok->alterror = 1;
-    tok->alttabsize = 1;
     tok->altindstack[0] = 0;
     tok->decoding_state = STATE_INIT;
     tok->decoding_erred = 0;
@@ -171,6 +173,7 @@ tok_new(void)
     tok->async_def = 0;
     tok->async_def_indent = 0;
     tok->async_def_nl = 0;
+    tok->async_always = 0;
 
     return tok;
 }
@@ -460,7 +463,7 @@ fp_readl(char *s, int size, struct tok_state *tok)
     }
     else
     {
-        bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
+        bufobj = _PyObject_CallNoArg(tok->decoding_readline);
         if (bufobj == NULL)
             goto error;
     }
@@ -553,7 +556,7 @@ fp_setreadl(struct tok_state *tok, const char* enc)
     Py_XSETREF(tok->decoding_readline, readline);
 
     if (pos > 0) {
-        PyObject *bufobj = PyObject_CallObject(readline, NULL);
+        PyObject *bufobj = _PyObject_CallNoArg(readline);
         if (bufobj == NULL)
             return 0;
         Py_DECREF(bufobj);
@@ -670,7 +673,7 @@ decoding_feof(struct tok_state *tok)
     } else {
         PyObject* buf = tok->decoding_buffer;
         if (buf == NULL) {
-            buf = PyObject_CallObject(tok->decoding_readline, NULL);
+            buf = _PyObject_CallNoArg(tok->decoding_readline);
             if (buf == NULL) {
                 error_ret(tok);
                 return 1;
@@ -976,6 +979,11 @@ tok_nextc(struct tok_state *tok)
                 buflen = PyBytes_GET_SIZE(u);
                 buf = PyBytes_AS_STRING(u);
                 newtok = PyMem_MALLOC(buflen+1);
+                if (newtok == NULL) {
+                    Py_DECREF(u);
+                    tok->done = E_NOMEM;
+                    return EOF;
+                }
                 strcpy(newtok, buf);
                 Py_DECREF(u);
             }
@@ -1306,22 +1314,9 @@ Ta3Token_ThreeChars(int c1, int c2, int c3)
 static int
 indenterror(struct tok_state *tok)
 {
-    if (tok->alterror) {
     tok->done = E_TABSPACE;
     tok->cur = tok->inp;
-        return 1;
-    }
-    if (tok->altwarning) {
-#ifdef PGEN
-        PySys_WriteStderr("inconsistent use of tabs and spaces "
-                          "in indentation\n");
-#else
-        PySys_FormatStderr("%U: inconsistent use of tabs and spaces "
-                          "in indentation\n", tok->filename);
-#endif
-        tok->altwarning = 0;
-    }
-    return 0;
+    return ERRORTOKEN;
 }
 
 #ifdef PGEN
@@ -1402,8 +1397,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             }
             else if (c == '\t') {
                 col = (col / tok->tabsize + 1) * tok->tabsize;
-                altcol = (altcol/tok->alttabsize + 1)
-                    * tok->alttabsize;
+                altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
             }
             else if (c == '\014')  {/* Control-L (formfeed) */
                 col = altcol = 0; /* For Emacs users */
@@ -1432,9 +1426,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             if (col == tok->indstack[tok->indent]) {
                 /* No change */
                 if (altcol != tok->altindstack[tok->indent]) {
-                    if (indenterror(tok)) {
-                        return ERRORTOKEN;
-                    }
+                    return indenterror(tok);
                 }
             }
             else if (col > tok->indstack[tok->indent]) {
@@ -1445,9 +1437,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                     return ERRORTOKEN;
                 }
                 if (altcol <= tok->altindstack[tok->indent]) {
-                    if (indenterror(tok)) {
-                        return ERRORTOKEN;
-                    }
+                    return indenterror(tok);
                 }
                 tok->pendin++;
                 tok->indstack[++tok->indent] = col;
@@ -1466,9 +1456,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                     return ERRORTOKEN;
                 }
                 if (altcol != tok->altindstack[tok->indent]) {
-                    if (indenterror(tok)) {
-                        return ERRORTOKEN;
-                    }
+                    return indenterror(tok);
                 }
             }
         }
@@ -1488,8 +1476,18 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
         }
     }
 
+    /* Peek ahead at the next character */
+    c = tok_nextc(tok);
+    tok_backup(tok, c);
+    /* Check if we are closing an async function */
     if (tok->async_def
         && !blankline
+        /* Due to some implementation artifacts of type comments,
+         * a TYPE_COMMENT at the start of a function won't set an
+         * indentation level and it will produce a NEWLINE after it.
+         * To avoid spuriously ending an async function due to this,
+         * wait until we have some non-newline char in front of us. */
+        && c != '\n'
         && tok->level == 0
         /* There was a NEWLINE after ASYNC DEF,
            so we're past the signature. */
@@ -1574,7 +1572,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     /* Identifier (most frequent token!) */
     nonascii = 0;
     if (is_potential_identifier_start(c)) {
-        /* Process b"", r"", u"", br"" and rb"" */
+        /* Process the various legal combinations of b"", r"", u"", and f"". */
         int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0;
         while (1) {
             if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B'))
@@ -1616,7 +1614,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
         /* async/await parsing block. */
         if (tok->cur - tok->start == 5) {
             /* Current token length is 5. */
-            if (tok->async_def) {
+            if (tok->async_always || tok->async_def) {
                 /* We're inside an 'async def' function. */
                 if (memcmp(tok->start, "async", 5) == 0) {
                     return ASYNC;
@@ -1988,9 +1986,7 @@ Ta3Tokenizer_FindEncodingFilename(int fd, PyObject *filename)
     char *p_start =NULL , *p_end =NULL , *encoding = NULL;
 
 #ifndef PGEN
-#if PY_MINOR_VERSION >= 4
     fd = _Py_dup(fd);
-#endif
 #else
     fd = dup(fd);
 #endif

ast3/Parser/tokenizer.h

@@ -47,9 +47,6 @@ struct tok_state {
        (Grammar/Grammar). */
     PyObject *filename;
 #endif
-    int altwarning;     /* Issue warning if alternate tabs don't match */
-    int alterror;       /* Issue error if alternate tabs don't match */
-    int alttabsize;     /* Alternate tab spacing */
     int altindstack[MAXINDENT];         /* Stack of alternate indents */
     /* Stuff for PEP 0263 */
     enum decoding_state decoding_state;
@@ -72,6 +69,7 @@ struct tok_state {
     int async_def_indent; /* Indentation level of the outermost 'async def'. */
     int async_def_nl;     /* =1 if the outermost 'async def' had at least one
                              NEWLINE token after it. */
+    int async_always;     /* =1 if async/await are always keywords */
 };
 
 extern struct tok_state *Ta3Tokenizer_FromString(const char *, int);
@@ -80,8 +78,6 @@ extern struct tok_state *Ta3Tokenizer_FromFile(FILE *, const char*,
                                               const char *, const char *);
 extern void Ta3Tokenizer_Free(struct tok_state *);
 extern int Ta3Tokenizer_Get(struct tok_state *, char **, char **);
-extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
-                                          int len, int *offset);
 
 #ifdef __cplusplus
 }