Distinguish 'x' and b'x' in Python 2 (#17)

Fixes #10 This is a naive fix. It only checks the first string in concatenation at CST level, for ``b`` or ``B`` prefix and sets the ``has_b`` flag in ``Str`` AST node accordingly.
python · Sep 20, 2016 · f4495f4 · f4495f4
1 parent 8ae1b7f
commit f4495f4
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 7 deletions.
diff --git a/ast27/Include/Python-ast.h b/ast27/Include/Python-ast.h
@@ -286,6 +286,7 @@ struct _expr {
 
                 struct {
                         string s;
+                        int has_b;
                 } Str;
 
                 struct {
@@ -503,8 +504,8 @@ expr_ty _Ta27_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty s
 expr_ty _Ta27_Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Num(a0, a1, a2, a3) _Ta27_Num(a0, a1, a2, a3)
 expr_ty _Ta27_Num(object n, int lineno, int col_offset, PyArena *arena);
-#define Str(a0, a1, a2, a3) _Ta27_Str(a0, a1, a2, a3)
-expr_ty _Ta27_Str(string s, int lineno, int col_offset, PyArena *arena);
+#define Str(a0, a1, a2, a3, a4) _Ta27_Str(a0, a1, a2, a3, a4)
+expr_ty _Ta27_Str(string s, int has_b, int lineno, int col_offset, PyArena *arena);
 #define Attribute(a0, a1, a2, a3, a4, a5) _Ta27_Attribute(a0, a1, a2, a3, a4, a5)
 expr_ty _Ta27_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
                         col_offset, PyArena *arena);

diff --git a/ast27/Parser/Python.asdl b/ast27/Parser/Python.asdl
@@ -71,7 +71,7 @@ module Python version "$Revision$"
 			 expr? starargs, expr? kwargs)
 	     | Repr(expr value)
 	     | Num(object n) -- a number as a PyObject.
-	     | Str(string s) -- need to specify raw, unicode, etc?
+	     | Str(string s, int? has_b) -- need to specify raw, unicode, etc?
 	     -- other literals? bools?
 
 	     -- the following expression can appear in assignment context

diff --git a/ast27/Python/Python-ast.c b/ast27/Python/Python-ast.c
@@ -252,6 +252,7 @@ static char *Num_fields[]={
 static PyTypeObject *Str_type;
 static char *Str_fields[]={
         "s",
+        "has_b",
 };
 static PyTypeObject *Attribute_type;
 static char *Attribute_fields[]={
@@ -782,7 +783,7 @@ static int init_types(void)
         if (!Repr_type) return 0;
         Num_type = make_type("Num", expr_type, Num_fields, 1);
         if (!Num_type) return 0;
-        Str_type = make_type("Str", expr_type, Str_fields, 1);
+        Str_type = make_type("Str", expr_type, Str_fields, 2);
         if (!Str_type) return 0;
         Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
         if (!Attribute_type) return 0;
@@ -1849,7 +1850,7 @@ Num(object n, int lineno, int col_offset, PyArena *arena)
 }
 
 expr_ty
-Str(string s, int lineno, int col_offset, PyArena *arena)
+Str(string s, int has_b, int lineno, int col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!s) {
@@ -1862,6 +1863,7 @@ Str(string s, int lineno, int col_offset, PyArena *arena)
                 return NULL;
         p->kind = Str_kind;
         p->v.Str.s = s;
+        p->v.Str.has_b = has_b;
         p->lineno = lineno;
         p->col_offset = col_offset;
         return p;
@@ -2887,6 +2889,11 @@ ast2obj_expr(void* _o)
                 if (PyObject_SetAttrString(result, "s", value) == -1)
                         goto failed;
                 Py_DECREF(value);
+                value = ast2obj_int(o->v.Str.has_b);
+                if (!value) goto failed;
+                if (PyObject_SetAttrString(result, "has_b", value) == -1)
+                        goto failed;
+                Py_DECREF(value);
                 break;
         case Attribute_kind:
                 result = PyType_GenericNew(Attribute_type, NULL, NULL);
@@ -5707,6 +5714,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         }
         if (isinstance) {
                 string s;
+                int has_b;
 
                 if (PyObject_HasAttrString(obj, "s")) {
                         int res;
@@ -5720,7 +5728,18 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
                         PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str");
                         return 1;
                 }
-                *out = Str(s, lineno, col_offset, arena);
+                if (PyObject_HasAttrString(obj, "has_b")) {
+                        int res;
+                        tmp = PyObject_GetAttrString(obj, "has_b");
+                        if (tmp == NULL) goto failed;
+                        res = obj2ast_int(tmp, &has_b, arena);
+                        if (res != 0) goto failed;
+                        Py_XDECREF(tmp);
+                        tmp = NULL;
+                } else {
+                        has_b = 0;
+                }
+                *out = Str(s, has_b, lineno, col_offset, arena);
                 if (*out == NULL) goto failed;
                 return 0;
         }

diff --git a/ast27/Python/ast.c b/ast27/Python/ast.c
@@ -1499,6 +1499,9 @@ ast_for_atom(struct compiling *c, const node *n)
     }
     case STRING: {
         PyObject *str = parsestrplus(c, n);
+        const char *s = STR(CHILD(n, 0));
+        int quote = Py_CHARMASK(*s);
+        int has_b = 0;
         if (!str) {
 #ifdef Py_USING_UNICODE
             if (PyErr_ExceptionMatches(PyExc_UnicodeError)){
@@ -1523,7 +1526,10 @@ ast_for_atom(struct compiling *c, const node *n)
             return NULL;
         }
         PyArena_AddPyObject(c->c_arena, str);
-        return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
+        if (quote == 'b' || quote == 'B') {
+            has_b = 1;
+        }
+        return Str(str, has_b, LINENO(n), n->n_col_offset, c->c_arena);
     }
     case NUMBER: {
         PyObject *pynum = parsenumber(c, STR(ch));