move common stuff to utils, remove views, remove typedefs of primitive from cid["types"]

gen/gen_cid/generate_cid.py

@@ -1,22 +1,8 @@
 # -*- coding: utf-8 -*-
 import os
-import json
 
 from . import ast_parser
-
+from . import utils
-
-def join_dir_and_filenames(dirname, filenames):
-    paths = []
-    for basename in filenames:
-        path = dirname + basename
-        paths.append(path)
-    return paths
-
-
-def read_file(path):
-    with open(path) as f:
-        file_content = f.read()
-    return file_content
 
 
 # out-dir is in-dir if not specified
@@ -35,57 +21,10 @@ def create_header(path, out_dir=None):
     if out_dir:
         header["out_dir"] = out_dir
 
-    header["sourcecode"] = read_file(path)
+    header["sourcecode"] = utils.read_file(path)
     return header
 
 
-def write_json(content, outpath):
-    # create path if not existing
-    out_dir = os.path.dirname(outpath)
-    if not os.path.isdir(out_dir):
-        os.makedirs(out_dir)
-    # write
-    with open(outpath, "w+") as f:
-        json.dump(content, f, indent=4)
-
-
-# works on valid json like structure lists/dict
-def recursive_query(data, filter, transform=lambda x: x):
-    # all datatypes accepted
-    # print(type(data))
-    # assert (type(data) in [list, dict, int, str, bool])
-    resultset = []
-
-    # decorator just handling exceptions
-    def filter_decorator(data):
-        try:
-            return filter(data)
-        except KeyError:
-            pass
-
-    # filter current data
-    # only dict types are filtered
-    if type(data) in [dict]:
-        if filter_decorator(data):
-            # transform result
-            xformed = transform(data)
-            if xformed:
-                resultset.append(xformed)
-
-    if isinstance(data, dict):
-        for item in data.values():
-            childres = recursive_query(item, filter, transform)
-            if childres:
-                resultset += childres
-    elif isinstance(data, list):
-        for item in data:
-            childres = recursive_query(item, filter, transform)
-            if childres:
-                resultset += childres
-
-    return resultset
-
-
 def extract_forward_declarations(data):
     def filter(item):
         if (item["is_definition"] == False
@@ -96,51 +35,73 @@ def extract_forward_declarations(data):
     def xform(item):
         return item
 
-    return recursive_query(data, filter, xform)
+    return utils.recursive_query(data, filter, xform)
 
 
-def extract_functions(data, function_names):
+def extract_vars_named(data, var_names):
     def filter(item):
-        if (item["kind"] == "CursorKind.FUNCTION_DECL"
+        if (item["kind"] == "CursorKind.VAR_DECL"
-            and item["name"] in function_names
+            and item["name"] in var_names
         ):
             return True
 
     def xform(item):
         return item
 
-    return recursive_query(data, filter, xform)
+    found = utils.recursive_query(data, filter, xform)
 
+    found_names = []
+    for i in found:
+        found_names.append(i["name"])
 
-# just the typenames of all typerefs no dups
+    notfound_names = []
-def collect_typerefs(data):
+    for name in var_names:
+        if not name in found_names:
+            notfound_names.append(name)
+
+    return (found, notfound_names)
+
+
+def extract_functions_named(data, function_names):
     def filter(item):
-        if (item["kind"] == "CursorKind.TYPE_REF"
+        if (item["kind"] == "CursorKind.FUNCTION_DECL"
+            and item["name"] in function_names
         ):
             return True
 
     def xform(item):
-        return item["type"]
+        return item
 
-    results = recursive_query(data, filter, xform)
+    found = utils.recursive_query(data, filter, xform)
-    no_dups = list(set(results))
-    return no_dups
 
+    found_names = []
+    for i in found:
+        found_names.append(i["name"])
 
-def extract_vars(data, var_names):
+    notfound_names = []
+    for name in function_names:
+        if not name in found_names:
+            notfound_names.append(name)
+
+    return (found, notfound_names)
+
+
+# just the typenames of all typerefs no dups
+def collect_all_typerefs(data):
     def filter(item):
-        if (item["kind"] == "CursorKind.VAR_DECL"
+        if (item["kind"] == "CursorKind.TYPE_REF"
-            and item["name"] in var_names
         ):
             return True
 
     def xform(item):
-        return item
+        return item["type"]
 
-    return recursive_query(data, filter, xform)
+    results = utils.recursive_query(data, filter, xform)
+    no_dups = list(set(results))
+    return no_dups
 
 
-def extract_types(data):
+def extract_type_names(data):
     def filter(item):
         if (item["kind"] == "CursorKind.TYPEDEF_DECL"
             or item["kind"] == "CursorKind.ENUM_DECL"
@@ -149,15 +110,10 @@ def extract_types(data):
             return True
 
     def xform(item):
-        ret = item["name"]
+        ret = item["type"]
         return ret
 
-    return recursive_query(data, filter, xform)
+    return utils.recursive_query(data, filter, xform)
-
-
-def remove_dup_dicts(arr_of_dicts):
-    arr_no_dups = [dict(i) for i in {tuple(d.items()) for d in arr_of_dicts}]
-    return arr_no_dups
 
 
 def resolve_typerefs(ast, typeref_names):
@@ -197,7 +153,7 @@ def extract_enum_decl(ast, name):
         ):
             return True
 
-    res = recursive_query(ast, filter)
+    res = utils.recursive_query(ast, filter)
     if len(res) > 1:
         assert False, "duplicate definition"
     if len(res) == 1:
@@ -215,7 +171,7 @@ def extract_struct_decl(ast, name):
         ):
             return True
 
-    res = recursive_query(ast, filter)
+    res = utils.recursive_query(ast, filter)
     if len(res) > 1:
         assert False, "duplicate definition"
     if len(res) == 1:
@@ -233,14 +189,19 @@ def extract_typedef_decl(ast, name):
         ):
             return True
 
-    res = recursive_query(ast, filter)
+    res = utils.recursive_query(ast, filter)
     if res:
         ret = res.pop()
 
         if ret["utypekind"] == "Typedef":
             ret = extract_typedef_decl(ast, ret["utype"])
         elif ret["utypekind"] == "Elaborated":
-            ret = extract_enum_decl(ast, ret["utype"]) or extract_struct_decl(ast, ret["utype"])
+            enum_decl = extract_enum_decl(ast, ret["utype"])
+            struct_decl = extract_struct_decl(ast, ret["utype"])
+            if enum_decl and struct_decl:
+                assert False, "duplicate types"
+            else:
+                ret = enum_decl or struct_decl
 
     return ret
 
@@ -254,7 +215,7 @@ def find_dup_types(data):
     def xform(item):
         return item["type"]
 
-    all_types = recursive_query(data, filter, xform)
+    all_types = utils.recursive_query(data, filter, xform)
     dups = set()
     for type in all_types:
         if all_types.count(type) > 1:
@@ -263,47 +224,55 @@ def find_dup_types(data):
     return (all_types, list(dups))
 
 
-def parse(libclang_path, function_names, header_filename, var_names):
+# TODO: Check for primitive type
+def remove_typedefs_of_primitive(types):
+    types_struct_enum = []
+    for type in types:
+        if (type["kind"] == "CursorKind.TYPEDEF_DECL"):
+            pass
+        else:
+            types_struct_enum.append(type)
+
+    return types_struct_enum
+
+
+def parse(libclang_path, header_filename, function_names, var_names):
     astp = ast_parser.ASTParser(libclang_path)
     # header = create_header("/Users/heck/local-default/include/pEp/pEpEngine.h", out_dir="./")
     header = create_header(header_filename)
     header["ast"] = astp.parse(header["path"], follow_includes=True)
-    # VIEWS
-    views = {}
-    views["forward_declarations"] = extract_forward_declarations(header["ast"])
-    views["types"] = extract_types(header["ast"])
-    views["types_count"] = len(views["types"])
     # CDL
     cid = {"functions": "",
            "vars": "",
-           "structs": "",
+           "types": ""}
-           "enums": ""}
+
     # stage 1: extract functions and vars
-    cid["functions"] = extract_functions(header["ast"], function_names)
+    (cid["functions"], cid["functions_notfound"]) = extract_functions_named(header["ast"], function_names)
-    cid["vars"] = extract_vars(header["ast"], var_names)
+    (cid["vars"], cid["vars_notfound"]) = extract_vars_named(header["ast"], var_names)
+
     # stage 2: collect type refs
-    cid["types_resolved"] = []
+    cid["types"] = []
-    cid["typerefs_unresolved"] = []
+    typerefs_unresolved = []
     cid["typerefs_resolved"] = []
     cid["typerefs_notfound"] = []
+    
     while True:
-        cid["typerefs_unresolved"] = collect_typerefs(cid)  # only list of typenames
+        typerefs_unresolved = collect_all_typerefs(cid)  # only list of typenames
-        cid["typerefs_unresolved"] = list(set(cid["typerefs_unresolved"]) - (set(cid["typerefs_resolved"]).union(set(cid["typerefs_notfound"]))))
+        typerefs_unresolved = list(set(typerefs_unresolved) - (set(cid["typerefs_resolved"]).union(set(cid["typerefs_notfound"]))))
-        if (len(cid["typerefs_unresolved"]) <= 0):
+        if (len(typerefs_unresolved) <= 0):
             break
 
-        (types_resolved, typerefs_resolved, notfound) = resolve_typerefs(header["ast"], cid["typerefs_unresolved"])
+        (types, typerefs_resolved, notfound) = resolve_typerefs(header["ast"], typerefs_unresolved)
-        cid["types_resolved"] += types_resolved
+        cid["types"] += types
         cid["typerefs_resolved"] += typerefs_resolved
         cid["typerefs_notfound"] += notfound
 
+
+    cid["types"] = remove_typedefs_of_primitive(cid["types"])
+    cid["type_names"] = extract_type_names(cid["types"])
     header["cid"] = cid
-    header["views"] = views
-    (views["all_types"], views["dup_types"]) = find_dup_types(header["cid"])
-    (views["ast_all_types"], views["ast_dup_types"]) = find_dup_types(header["ast"])
     return header
 
-
 # generates simple-ast for each header specified in spec out dir.
 # def main_old():
 #     parser = ASTParser("/opt/local/libexec/llvm-9.0/lib/libclang.dylib")

gen/gen_cid/utils.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+#
+import os
+import json
+
+def join_dir_and_filenames(dirname, filenames):
+    paths = []
+    for basename in filenames:
+        path = dirname + basename
+        paths.append(path)
+    return paths
+
+def read_file(path):
+    with open(path) as f:
+        file_content = f.read()
+    return file_content
+
+def write_json(content, outpath):
+    # create path if not existing
+    out_dir = os.path.dirname(outpath)
+    if not os.path.isdir(out_dir):
+        os.makedirs(out_dir)
+    # write
+    with open(outpath, "w+") as f:
+        json.dump(content, f, indent=4)
+
+# works on valid json like structure lists/dict
+def recursive_query(data, filter, transform=lambda x: x):
+    # all datatypes accepted
+    # print(type(data))
+    # assert (type(data) in [list, dict, int, str, bool])
+    resultset = []
+
+    # decorator just handling exceptions
+    def filter_decorator(data):
+        try:
+            return filter(data)
+        except KeyError:
+            pass
+
+    # filter current data
+    # only dict types are filtered
+    if type(data) in [dict]:
+        if filter_decorator(data):
+            # transform result
+            xformed = transform(data)
+            if xformed:
+                resultset.append(xformed)
+
+    if isinstance(data, dict):
+        for item in data.values():
+            childres = recursive_query(item, filter, transform)
+            if childres:
+                resultset += childres
+    elif isinstance(data, list):
+        for item in data:
+            childres = recursive_query(item, filter, transform)
+            if childres:
+                resultset += childres
+
+    return resultset
+
+
+def remove_dup_dicts(arr_of_dicts):
+    arr_no_dups = [dict(i) for i in {tuple(d.items()) for d in arr_of_dicts}]
+    return arr_no_dups