bpo-32911: Add new AST node for docstring

Doc/library/ast.rst

@@ -142,8 +142,9 @@ and classes for traversing abstract syntax trees:
       :class:`AsyncFunctionDef` is now supported.
 
    .. versionchanged:: 3.7
-      The docstring is now exported from the node docstring field, instead of
-      the first body statement.
+      ``Str`` and ``Constant`` are not treated as docstring anymore.
+      The new AST node named ``DocString`` is added and only it is
+      considered as docstring.
 
 
 .. function:: fix_missing_locations(node)

Doc/whatsnew/3.7.rst

@@ -1116,12 +1116,9 @@ Changes in the Python API
   Use the :meth:`~http.cookies.Morsel.set` method for setting them.
   (Contributed by Serhiy Storchaka in :issue:`29192`.)
 
-* ``Module``, ``FunctionDef``, ``AsyncFunctionDef``, and
-  ``ClassDef`` AST nodes now have a new ``docstring`` field.
-  The first statement in their body is not considered as a docstring
-  anymore.  ``co_firstlineno`` and ``co_lnotab`` of code object for class
-  and module are affected by this change. (Contributed by INADA Naoki and
-  Eugene Toder in :issue:`29463`.)
+* Add new AST node ``DocString``.  Only it is considered as docstring.
+  ``Str`` and ``Const`` are not considered as docstring anymore.
+  (Contributed by INADA Naoki and Eugene Toder in :issue:`29463` and :issue:`32911`.)
 
 * The *mode* argument of :func:`os.makedirs` no longer affects the file
   permission bits of newly-created intermediate-level directories.

Include/Python-ast.h

@@ -46,7 +46,6 @@ struct _mod {
     union {
         struct {
             asdl_seq *body;
-            string docstring;
         } Module;
 
         struct {
@@ -71,7 +70,7 @@ enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3,
                   AsyncWith_kind=14, Raise_kind=15, Try_kind=16,
                   Assert_kind=17, Import_kind=18, ImportFrom_kind=19,
                   Global_kind=20, Nonlocal_kind=21, Expr_kind=22, Pass_kind=23,
-                  Break_kind=24, Continue_kind=25};
+                  Break_kind=24, Continue_kind=25, DocString_kind=26};
 struct _stmt {
     enum _stmt_kind kind;
     union {
@@ -81,7 +80,6 @@ struct _stmt {
             asdl_seq *body;
             asdl_seq *decorator_list;
             expr_ty returns;
-            string docstring;
         } FunctionDef;
 
         struct {
@@ -90,7 +88,6 @@ struct _stmt {
             asdl_seq *body;
             asdl_seq *decorator_list;
             expr_ty returns;
-            string docstring;
         } AsyncFunctionDef;
 
         struct {
@@ -99,7 +96,6 @@ struct _stmt {
             asdl_seq *keywords;
             asdl_seq *body;
             asdl_seq *decorator_list;
-            string docstring;
         } ClassDef;
 
         struct {
@@ -203,6 +199,10 @@ struct _stmt {
             expr_ty value;
         } Expr;
 
+        struct {
+            string s;
+        } DocString;
+
     } v;
     int lineno;
     int col_offset;
@@ -443,27 +443,26 @@ struct _withitem {
 };
 
 
-#define Module(a0, a1, a2) _Py_Module(a0, a1, a2)
-mod_ty _Py_Module(asdl_seq * body, string docstring, PyArena *arena);
+#define Module(a0, a1) _Py_Module(a0, a1)
+mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
 #define Interactive(a0, a1) _Py_Interactive(a0, a1)
 mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
 #define Expression(a0, a1) _Py_Expression(a0, a1)
 mod_ty _Py_Expression(expr_ty body, PyArena *arena);
 #define Suite(a0, a1) _Py_Suite(a0, a1)
 mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
-#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
+#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
-                        asdl_seq * decorator_list, expr_ty returns, string
-                        docstring, int lineno, int col_offset, PyArena *arena);
-#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
+                        asdl_seq * decorator_list, expr_ty returns, int lineno,
+                        int col_offset, PyArena *arena);
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
                              body, asdl_seq * decorator_list, expr_ty returns,
-                             string docstring, int lineno, int col_offset,
-                             PyArena *arena);
-#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
+                             int lineno, int col_offset, PyArena *arena);
+#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
-                     asdl_seq * body, asdl_seq * decorator_list, string
-                     docstring, int lineno, int col_offset, PyArena *arena);
+                     asdl_seq * body, asdl_seq * decorator_list, int lineno,
+                     int col_offset, PyArena *arena);
 #define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
 stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)
@@ -526,6 +525,8 @@ stmt_ty _Py_Pass(int lineno, int col_offset, PyArena *arena);
 stmt_ty _Py_Break(int lineno, int col_offset, PyArena *arena);
 #define Continue(a0, a1, a2) _Py_Continue(a0, a1, a2)
 stmt_ty _Py_Continue(int lineno, int col_offset, PyArena *arena);
+#define DocString(a0, a1, a2, a3) _Py_DocString(a0, a1, a2, a3)
+stmt_ty _Py_DocString(string s, int lineno, int col_offset, PyArena *arena);
 #define BoolOp(a0, a1, a2, a3, a4) _Py_BoolOp(a0, a1, a2, a3, a4)
 expr_ty _Py_BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
                    PyArena *arena);

Lib/ast.py

@@ -206,7 +206,9 @@ def get_docstring(node, clean=True):
     """
     if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
         raise TypeError("%r can't have docstrings" % node.__class__.__name__)
-    text = node.docstring
+    if not(node.body and isinstance(node.body[0], DocString)):
+        return
+    text = node.body[0].s
     if clean and text:
         import inspect
         text = inspect.cleandoc(text)