Merge pull request #17 from ispras/customization

Boilerplate customization

Make QDT more friendly for boilerplate customization and add an example.

Side changes:
- support v4.1.0

### v2
- fix errors
- prettify example of boilerplate customization

Author: BDanAnd

examples/customizing_device.py

@@ -0,0 +1,117 @@
+from sys import (
+    path as PYTHONPATH,
+)
+from os import (
+    environ,
+)
+from os.path import (
+    abspath,
+    dirname,
+)
+
+# Imports below are from QDT.
+PYTHONPATH.insert(0, dirname(dirname(abspath(__file__))))
+
+from qemu import (
+    qvd_load_with_cache,
+    describable,
+    SysBusDeviceType,
+    QProject,
+)
+from source import (
+    Function,
+    Type,
+    Header,
+    Call,
+)
+
+
+@describable
+class CustomSBDType(SysBusDeviceType):
+    """ This is an example of system bus device boilerplate extension with
+custom functionality.
+    """
+
+    def fill_source(self):
+        """ Adds a hook callback in MMIO read handler.
+        """
+
+        super(CustomSBDType, self).fill_source()
+
+        # Now, code of device's module is generated. It's time to patch it!
+
+        # Assuming, your code uses functions and types from your header.
+        custom_h = Header("custom-api.h", is_global = True)
+
+        # If your header is inside Qemu source tree and under Git, then you
+        # likely should write above statement as follows:
+        #
+        # custom_h = Header["custom/custom-api.h"]
+        #
+        # Because the header is already registered.
+
+        # To use custom functions (and other things) you must declare them.
+        # Note that, macros are added automatically if header is inside Qemu
+        # source tree.
+        # Only name is sufficient because no code will be generated for them.
+        custom_h.add_types([
+            Function("custom_callback")
+        ])
+
+        read_func = self.find_mmio_read_helper(0)
+
+        # We are going to pass second and following argument values of the
+        # read helper to the custom callback.
+        # Of course, you can made different choice in your code.
+        args2callback = read_func.args[1:]
+
+        # Insert function call statement before 3-nd statement.
+        # Note, first statements are likely variable declarations.
+        # This is for prettiness only.
+        read_func.body.children.insert(2,
+            Call("custom_callback", *args2callback)
+        )
+
+    def find_mmio_read_helper(self, mmioN):
+        # Evaluate name of the read helper.
+        # This code came from superclass `fill_source`.
+        component = self.get_Ith_mmio_id_component(mmioN)
+        name = self.qtn.for_id_name + "_" + component + "_read"
+
+        # Currently all functions (including `static`) are in global name
+        # space (it's an implementation limitation yet).
+        # And they are "types"... Here it's more generic term then in C.
+        return Type[name]
+
+
+def main():
+    p = QProject(
+        descriptions = [
+            CustomSBDDescription(# This class is defined by `@describable`
+                name = "custom_device",
+                directory = "misc",
+                mmio_num = 1
+            ),
+        ]
+    )
+
+    # We need information about target Qemu source tree.
+    # QDT gets it starting from build directory (where `configure` has work).
+    qemu_build = environ["QEMU_BUILD_DIR"]
+
+    qemu_version_description = qvd_load_with_cache(qemu_build,
+        version = "v4.1.0"
+    )
+    # First time the loading may take few minutes because Qemu sources
+    # are analyzed.
+    # Then result is cached in a file to be reused.
+
+    # Apply Qemu's source code environment.
+    qemu_version_description.use()
+
+    # And finally, generate the boilerplate.
+    p.gen_all(qemu_src = qemu_version_description.src_path)
+
+
+if __name__ == "__main__":
+    exit(main() or 0)

qemu/project.py

@@ -132,13 +132,6 @@ def register_in_build_system(self, folder, known_targets):
         if not isfile(Makefile_obj):
             open(Makefile_obj, "w").close()
 
-    def make_src_dirs(self, full_path, known_targets):
-        if not isdir(full_path):
-            # Provide required directory.
-            makedirs(full_path)
-
-        self.register_in_build_system(full_path, known_targets)
-
     def gen(self, *args, **kw):
         "Backward compatibility wrapper for co_gen"
         callco(self.co_gen(*args, **kw))

qemu/qom_desc.py

@@ -60,7 +60,10 @@ def describable(QOMTemplate):
     module.__dict__[desc_name] = desc_class
 
     # export the description class
-    module.__all__.append(desc_name)
+    try:
+        module.__all__.append(desc_name)
+    except AttributeError:
+        pass # The module does not define `__all__`
 
     # The template is not actually changed.
     return QOMTemplate

qemu/version.py

@@ -186,11 +186,11 @@ def define_only_qemu_2_6_0_types():
         Structure("I2CBus") # the structure is defined in .c file
     ]).add_reference(osdep_fake_type)
 
-    Header["disas/bfd.h"].add_types([
+    Header[get_vp("disas header")].add_types([
         Type("disassemble_info", False)
     ]).add_reference(osdep_fake_type)
 
-    Header["qemu/fprintf-fn.h"].add_types([
+    Header[get_vp("fprintf_function definer")].add_types([
         Type("fprintf_function", False)
     ]).add_reference(osdep_fake_type)
 
@@ -552,13 +552,13 @@ def define_only_qemu_2_6_0_types():
         osdep_fake_type
     ])
 
-    Header["disas/bfd.h"].add_types([
+    Header[get_vp("disas header")].add_types([
         Type("bfd_vma", False),
         Type("bfd_byte", False),
         Type("const bfd_byte", False)
     ])
 
-    Header["disas/bfd.h"].add_types([
+    Header[get_vp("disas header")].add_types([
         Function(
             name = name,
             ret_type = Type["bfd_vma"],
@@ -777,6 +777,19 @@ def machine_register_2_6(mach):
     )
 
 qemu_heuristic_db = {
+    # Next two commits precede v4.1.0-rc0. They are in one branch.
+    u'ede9a8a656c992deecce45f8175985dd81cc6be9' : [
+        QEMUVersionParameterDescription("fprintf_function definer",
+            new_value = "disas/dis-asm.h",
+            old_value = "qemu/fprintf-fn.h"
+        )
+    ],
+    u'3979fca4b69fc31c372687cd0bb6950592f248bd' : [
+        QEMUVersionParameterDescription("disas header",
+            new_value = "disas/dis-asm.h",
+            old_value = "disas/bfd.h"
+        )
+    ],
     u'b59821a95bd1d7cb4697fd7748725c910582e0e7' : [
         QEMUVersionParameterDescription("explicit global memory registration",
             new_value = False,

source/model.py

@@ -427,6 +427,13 @@ def add_type(self, _type):
         _type.definer = self
         self.types[_type.name] = _type
 
+        # Some types (like `Enumeration`) contains types without definer or
+        # may reference to other just created types.
+        # They must be added right now (by `TypeFixerVisitor`) to prevent
+        # capturing by other sources at generation begin.
+        # Note, replacement of foreign types with `TypeReference` is actually
+        # not required right now (see `gen_chunks`).
+
         fixer = TypeFixerVisitor(self, _type).visit()
 
         # Auto add references to types this one does depend on
@@ -448,6 +455,8 @@ def gen_chunks(self, inherit_references = False):
         if inherit_references:
             assert(isinstance(self, Header))
 
+        # This header includes other headers to provide types for includers
+        # of self. This is list of references to those types.
         ref_list = []
 
         if isinstance(self, Header):
@@ -456,7 +465,11 @@ def gen_chunks(self, inherit_references = False):
                     if ref.definer not in user.inclusions:
                         ref_list.append(TypeReference(ref))
 
-        TypeFixerVisitor(self, self.global_variables).visit()
+        # Finally, we must fix types just before generation because user can
+        # change already added types.
+        fixer = TypeFixerVisitor(self, self).visit()
+
+        self.add_references(tr.type for tr in fixer.new_type_references)
 
         # fix up types for headers with references
         # list of types must be copied because it is changed during each
@@ -576,6 +589,8 @@ def generate(self, inherit_references = False):
 
         return file
 
+    __type_references__ = ("types", "global_variables")
+
 
 class CPP(object):
     "This class used as definer for CPPMacro"
@@ -624,6 +639,24 @@ def __getitem__(self, path):
 class Header(Source):
     reg = {}
 
+    def __init__(self, path, is_global = False, protection = True):
+        """
+:param path: it is used in #include statements, as unique identifier and
+    somehow during code generation.
+:param is_global: inclusions will use <path> instead of "path".
+:param protection: wrap content in multiple inclusion protection macro logic.
+        """
+        super(Header, self).__init__(path)
+        self.is_global = is_global
+        self.includers = []
+        self.protection = protection
+
+        tpath = path2tuple(path)
+        if tpath in Header.reg:
+            raise RuntimeError("Header %s is already registered" % path)
+
+        Header.reg[tpath] = self
+
     @staticmethod
     def _on_include(includer, inclusion, is_global):
         if path2tuple(inclusion) not in Header.reg:
@@ -771,18 +804,6 @@ def lookup(path):
             raise RuntimeError("Header with path %s is not registered" % path)
         return Header.reg[tpath]
 
-    def __init__(self, path, is_global = False, protection = True):
-        super(Header, self).__init__(path)
-        self.is_global = is_global
-        self.includers = []
-        self.protection = protection
-
-        tpath = path2tuple(path)
-        if tpath in Header.reg:
-            raise RuntimeError("Header %s is already registered" % path)
-
-        Header.reg[tpath] = self
-
     def __str__(self):
         if self.is_global:
             return "<%s>" % self.path