added PonyORM, fixed several bugs

Author: xnuinside

.flake8

@@ -1,4 +1,4 @@
 [flake8]
-exclude = .github,.git,__pycache__,docs/source/conf.py,old,build,dist
+exclude = .github,.git,__pycache__,docs/source/conf.py,old,build,dist,test.py
 max-complexity = 10
 max-line-length = 120

CHANGELOG.txt

@@ -1,3 +1,8 @@
+**v0.4.0**
+1. return tuples (multiple values) is parsed correctly now
+2. symbols like `*&^%$#!±~`§<>` now does not cause any errors
+3. classes without any args does not cause an error anymore
+
 **v0.3.0**
 1. Added cli - `pmp` command with args -d, --dump  
 2. Added support for simple Django ORM models

README.md

@@ -13,6 +13,7 @@ Py-Models-Parser can parse & extract information from models:
 - Django ORM Model,
 - Pydantic,
 - Python Enum,
+- Pony ORM,
 - Python Dataclasses
 - pure Python Classes
 
@@ -176,6 +177,11 @@ For model from point 1 (above) library will produce the result:
 4. Add support for Piccolo ORM models
 
 ## Changelog
+**v0.4.0**
+1. return tuples (multiple values) is parsed correctly now
+2. symbols like `*&^%$#!±~`§<>` now does not cause any errors
+3. classes without any args does not cause an error anymore
+
 **v0.3.0**
 1. Added cli - `pmp` command with args -d, --dump  
 2. Added support for simple Django ORM models

docs/README.rst

@@ -31,6 +31,7 @@ Py-Models-Parser can parse & extract information from models:
 * Django ORM Model,
 * Pydantic,
 * Python Enum,
+* Pony ORM,
 * Python Dataclasses
 * pure Python Classes
 
@@ -200,6 +201,13 @@ TODO: in next Release
 Changelog
 ---------
 
+**v0.4.0**
+
+
+#. return tuples (multiple values) is parsed correctly now
+#. symbols like ``*&^%$#!±~``\ §<>` now does not cause any errors
+#. classes without any args does not cause an error anymore
+
 **v0.3.0**
 
 

py_models_parser/grammar.py

@@ -2,8 +2,12 @@
 
 grammar = Grammar(
     r"""
-    expr = (class / call_result / attr_def / emptyline / funct_def)*
-    class = class_def attr_def* funct_def* ws?
+    expr = (class / if_else/ call_result / return / attr_def / emptyline / funct_def)*
+    return = "return" (id* ","*)*
+    if_else= ("if" (compare/ id / attr_def) ":")/("elif" (id/attr_def) ":")/("else" ":")
+    compare = (call_result / id / args /args_in_brackets  ) operator (call_result/id/args_in_brackets/args)
+    operator = "==" / "!=" / ">" / "<" / ">=" / "<="
+    class = class_def attr_def* funct_def*
     class_def   = intend? class_name args? ":"* ws?
     attr_def  = intend? id type? ("=" (right_part))* ws?
     right_part =  (id args_in_brackets) / string / args  / call_result / args_in_brackets / id / text
@@ -13,14 +17,14 @@ class = class_def attr_def* funct_def* ws?
     double_quotes_str = ~'"[^\"]+"'i
     funct_def = intend? "def" id args? ":"* ws?
     args_in_brackets = "[" ((id/string)* ","* )* "]"
-    args     = "(" (( call_result / args / attr_def / id )* ","* )* ")"
+    args        = "(" (( call_result / args / attr_def / id  )* ","* )* ")"
     call_result = id args ws?
     class_name  = "class" id
     id          = (((dot_id / text)+ ) *  / dot_id / text) ws?
     dot_id      = (text".")*text
     intend      = "    " / "\t" / "\n"
-    text        = !class ~"['\_A-Z 0-9\{\}_\"\-\/\$<%>\+\-\w]*"i
-    ws          = ~"\s*"
+    text        = !"class" ~"['_A-Z 0-9{}_\"\-\/\$<%>\+\-\w*&^%$#!±~`§]*"i
+    ws          = ~"\\s*"
     emptyline   = ws+
 """
 )

py_models_parser/visitor.py

@@ -48,6 +48,8 @@ def extract_orm_attr(self, text: str):
         not_orm = True
         properties = {}
         orm_columns = ["Column", "Field", "relationship", "ForeignKey"]
+        pony_orm_fields = ["Required", "Set", "Optional", "PrimaryKey"]
+        orm_columns.extend(pony_orm_fields)
         for i in orm_columns:
             if i in text:
                 not_orm = False
@@ -57,34 +59,44 @@ def extract_orm_attr(self, text: str):
                 text = text[index + 1 : -1]  # noqa E203
                 text = text.split(",")
                 text = self.clean_up_cases_with_inner_pars(text)
+                prop_index = 1
                 if i == "Field":
                     _type, properties = get_django_info(text, base_text, properties)
                     prop_index = 0
                 elif i == "ForeignKey":
                     # mean it is a Django model.ForeignKey
                     _type = "serial"
                     properties["foreign_key"] = text[0]
-                    prop_index = 1
+                elif i in pony_orm_fields:
+                    # mean it is a Pony ORM
+                    _type, properties = get_pony_orm_info(
+                        text, i, base_text, properties
+                    )
                 else:
-                    prop_index = 1
                     _type = text[0]
                 if i == "relationship":
                     properties["relationship"] = True
-                for i in text[prop_index:]:
-                    if "=" in i:
-                        # can be backref=db.backref('pages', lazy=True)
-                        index = i.find("=")
-                        left = i[:index].strip()
-                        right = i[index + 1 :].strip()  # noqa: E203
-                        if left == "default":
-                            default = right
-                        else:
-                            properties[left] = right
-                    elif "foreign" in i.lower():
-                        properties["foreign_key"] = i.split("(")[1].split(")")[0]
+                for item in text[prop_index:]:
+                    properties, default = self.add_property(item, properties)
                 break
         return default, _type, properties, not_orm
 
+    @staticmethod
+    def add_property(item: str, properties: Dict) -> Tuple[Dict, str]:
+        default = None
+        if "=" in item:
+            # can be backref=db.backref('pages', lazy=True)
+            index = item.find("=")
+            left = item[:index].strip()
+            right = item[index + 1 :].strip()  # noqa: E203
+            if left == "default":
+                default = right
+            else:
+                properties[left] = right
+        elif "foreign" in item.lower():
+            properties["foreign_key"] = item.split("(")[1].split(")")[0]
+        return properties, default
+
     def extractor(self, text: str) -> Dict:
         _type = None
         default = None
@@ -102,12 +114,11 @@ def visit_right_part(self, node, visited_children):
 
     def visit_attr_def(self, node, visited_children):
         """Makes a dict of the section (as key) and the key/value pairs."""
-        left = node.children[1].children[0].text.strip()
+        left = node.children[1].children[0].children[0].text.strip()
         default = None
         _type = None
         if "def " in left:
             attr = {"attr": {"name": None, "type": _type, "default": default}}
-
             return attr
         if ":" in left:
             _type = left.split(":")[-1].strip()
@@ -183,7 +194,6 @@ def visit_expr(self, node, visited_children):
                     children_values[n]["properties"]["init"] = final_child[
                         "properties"
                     ]["init"]
-
         return children_values
 
     def visit_type(self, node, visited_children):
@@ -197,17 +207,36 @@ def generic_visit(self, node, visited_children):
 
 
 def process_no_name_attrs(final_child: Dict, child: Dict) -> None:
-    if child["attr"]["default"]:
-        final_child["attrs"][-1]["default"] = child["attr"]["default"]
-        if not final_child["attrs"][-1].get("properties"):
-            final_child["attrs"][-1]["properties"] = {}
-    elif child["attr"]["type"]:
-        final_child["attrs"][-1]["default"] += f':{child["attr"]["type"]}'
+    if final_child["attrs"]:
+        if child["attr"]["default"]:
+            final_child["attrs"][-1]["default"] = child["attr"]["default"]
+            if not final_child["attrs"][-1].get("properties"):
+                final_child["attrs"][-1]["properties"] = {}
+        elif child["attr"]["type"] and final_child["attrs"][-1]["default"]:
+            final_child["attrs"][-1]["default"] += f':{child["attr"]["type"]}'
     return final_child
 
 
+def get_pony_orm_info(
+    text: list, field: str, base_text: str, properties: Dict
+) -> Tuple:
+    if field == "Required":
+        properties["nullable"] = False
+    elif field == "PrimaryKey":
+        properties["primary_key"] = True
+    elif field == "Optional":
+        properties["nullable"] = True
+    elif field == "Set":
+        # relationship
+        properties["relationship"] = True
+        properties["foreign_key"] = text[0]
+    _type = text[0]
+
+    return _type, properties
+
+
 def get_django_info(text: list, base_text: str, properties: Dict) -> Tuple:
-    #    for tortoise orm & django orm
+    # for tortoise orm & django orm
     split_by_field = base_text.split("Field")[0].split(".")
     if len(split_by_field) == 2:
         _type = split_by_field[1]

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "py-models-parser"
-version = "0.3.0"
-description = "Parser for Different Python Models (Pydantic, Enums, ORMs: Tortoise, SqlAlchemy, GinoORM) to extract information about columns(attrs), model, table args,etc in one format."
+version = "0.4.0"
+description = "Parser for Different Python Models (Pydantic, Enums, ORMs: Tortoise, SqlAlchemy, GinoORM, PonyORM) to extract information about columns(attrs), model, table args,etc in one format."
 authors = ["Iuliia Volkova <[email protected]>"]
 license = "MIT"
 readme = "docs/README.rst"

tests/test_pony.py

@@ -0,0 +1,128 @@
+from py_models_parser import parse
+
+
+def test_simple_pony_model():
+    model = """
+class Person(db.Entity):
+     name = Required(str)
+     age = Required(int)
+     cars = Set('Car')
+
+class Car(db.Entity):
+     make = Required(str)
+     model = Required(str)
+     owner = Required(Person)
+
+"""
+
+    result = parse(model)
+    expected = [
+        {
+            "attrs": [
+                {
+                    "default": None,
+                    "name": "name",
+                    "properties": {"nullable": False},
+                    "type": "str",
+                },
+                {
+                    "default": None,
+                    "name": "age",
+                    "properties": {"nullable": False},
+                    "type": "int",
+                },
+                {
+                    "default": None,
+                    "name": "cars",
+                    "properties": {"foreign_key": "'Car'", "relationship": True},
+                    "type": "'Car'",
+                },
+            ],
+            "name": "Person",
+            "parents": ["db.Entity"],
+            "properties": {},
+        },
+        {
+            "attrs": [
+                {
+                    "default": None,
+                    "name": "make",
+                    "properties": {"nullable": False},
+                    "type": "str",
+                },
+                {
+                    "default": None,
+                    "name": "model",
+                    "properties": {"nullable": False},
+                    "type": "str",
+                },
+                {
+                    "default": None,
+                    "name": "owner",
+                    "properties": {"nullable": False},
+                    "type": "Person",
+                },
+            ],
+            "name": "Car",
+            "parents": ["db.Entity"],
+            "properties": {},
+        },
+    ]
+    assert result == expected
+
+
+def test_primary_and_optional():
+
+    model = """
+    from pony.orm import *
+
+    db = Database()
+
+
+    class Product(db.Entity):
+        id = PrimaryKey(int, auto=True)
+        name = Required(str)
+        info = Required(Json)
+        tags = Optional(Json)
+
+
+    db.bind('sqlite', ':memory:', create_db=True)
+    db.generate_mapping(create_tables=True)
+    """
+
+    result = parse(model)
+    expected = [
+        {
+            "attrs": [
+                {
+                    "default": None,
+                    "name": "id",
+                    "properties": {"auto": "True", "primary_key": True},
+                    "type": "int",
+                },
+                {
+                    "default": None,
+                    "name": "name",
+                    "properties": {"nullable": False},
+                    "type": "str",
+                },
+                {
+                    "default": None,
+                    "name": "info",
+                    "properties": {"nullable": False},
+                    "type": "Json",
+                },
+                {
+                    "default": None,
+                    "name": "tags",
+                    "properties": {"nullable": True},
+                    "type": "Json",
+                },
+                {"default": None, "name": "db.bind", "type": None},
+            ],
+            "name": "Product",
+            "parents": ["db.Entity"],
+            "properties": {},
+        }
+    ]
+    assert result == expected