diff --git a/gen/gen_cid/generate_cid.py b/gen/gen_cid/generate_cid.py index 8641a2f..e1004bf 100755 --- a/gen/gen_cid/generate_cid.py +++ b/gen/gen_cid/generate_cid.py @@ -1,22 +1,8 @@ # -*- coding: utf-8 -*- import os -import json from . import ast_parser - - -def join_dir_and_filenames(dirname, filenames): - paths = [] - for basename in filenames: - path = dirname + basename - paths.append(path) - return paths - - -def read_file(path): - with open(path) as f: - file_content = f.read() - return file_content +from . import utils # out-dir is in-dir if not specified @@ -35,57 +21,10 @@ def create_header(path, out_dir=None): if out_dir: header["out_dir"] = out_dir - header["sourcecode"] = read_file(path) + header["sourcecode"] = utils.read_file(path) return header -def write_json(content, outpath): - # create path if not existing - out_dir = os.path.dirname(outpath) - if not os.path.isdir(out_dir): - os.makedirs(out_dir) - # write - with open(outpath, "w+") as f: - json.dump(content, f, indent=4) - - -# works on valid json like structure lists/dict -def recursive_query(data, filter, transform=lambda x: x): - # all datatypes accepted - # print(type(data)) - # assert (type(data) in [list, dict, int, str, bool]) - resultset = [] - - # decorator just handling exceptions - def filter_decorator(data): - try: - return filter(data) - except KeyError: - pass - - # filter current data - # only dict types are filtered - if type(data) in [dict]: - if filter_decorator(data): - # transform result - xformed = transform(data) - if xformed: - resultset.append(xformed) - - if isinstance(data, dict): - for item in data.values(): - childres = recursive_query(item, filter, transform) - if childres: - resultset += childres - elif isinstance(data, list): - for item in data: - childres = recursive_query(item, filter, transform) - if childres: - resultset += childres - - return resultset - - def extract_forward_declarations(data): def filter(item): if (item["is_definition"] == False @@ -96,51 +35,73 @@ def extract_forward_declarations(data): def xform(item): return item - return recursive_query(data, filter, xform) + return utils.recursive_query(data, filter, xform) -def extract_functions(data, function_names): +def extract_vars_named(data, var_names): def filter(item): - if (item["kind"] == "CursorKind.FUNCTION_DECL" - and item["name"] in function_names + if (item["kind"] == "CursorKind.VAR_DECL" + and item["name"] in var_names ): return True def xform(item): return item - return recursive_query(data, filter, xform) + found = utils.recursive_query(data, filter, xform) + found_names = [] + for i in found: + found_names.append(i["name"]) -# just the typenames of all typerefs no dups -def collect_typerefs(data): + notfound_names = [] + for name in var_names: + if not name in found_names: + notfound_names.append(name) + + return (found, notfound_names) + + +def extract_functions_named(data, function_names): def filter(item): - if (item["kind"] == "CursorKind.TYPE_REF" + if (item["kind"] == "CursorKind.FUNCTION_DECL" + and item["name"] in function_names ): return True def xform(item): - return item["type"] + return item - results = recursive_query(data, filter, xform) - no_dups = list(set(results)) - return no_dups + found = utils.recursive_query(data, filter, xform) + found_names = [] + for i in found: + found_names.append(i["name"]) -def extract_vars(data, var_names): + notfound_names = [] + for name in function_names: + if not name in found_names: + notfound_names.append(name) + + return (found, notfound_names) + + +# just the typenames of all typerefs no dups +def collect_all_typerefs(data): def filter(item): - if (item["kind"] == "CursorKind.VAR_DECL" - and item["name"] in var_names + if (item["kind"] == "CursorKind.TYPE_REF" ): return True def xform(item): - return item + return item["type"] - return recursive_query(data, filter, xform) + results = utils.recursive_query(data, filter, xform) + no_dups = list(set(results)) + return no_dups -def extract_types(data): +def extract_type_names(data): def filter(item): if (item["kind"] == "CursorKind.TYPEDEF_DECL" or item["kind"] == "CursorKind.ENUM_DECL" @@ -149,15 +110,10 @@ def extract_types(data): return True def xform(item): - ret = item["name"] + ret = item["type"] return ret - return recursive_query(data, filter, xform) - - -def remove_dup_dicts(arr_of_dicts): - arr_no_dups = [dict(i) for i in {tuple(d.items()) for d in arr_of_dicts}] - return arr_no_dups + return utils.recursive_query(data, filter, xform) def resolve_typerefs(ast, typeref_names): @@ -197,7 +153,7 @@ def extract_enum_decl(ast, name): ): return True - res = recursive_query(ast, filter) + res = utils.recursive_query(ast, filter) if len(res) > 1: assert False, "duplicate definition" if len(res) == 1: @@ -215,7 +171,7 @@ def extract_struct_decl(ast, name): ): return True - res = recursive_query(ast, filter) + res = utils.recursive_query(ast, filter) if len(res) > 1: assert False, "duplicate definition" if len(res) == 1: @@ -233,14 +189,19 @@ def extract_typedef_decl(ast, name): ): return True - res = recursive_query(ast, filter) + res = utils.recursive_query(ast, filter) if res: ret = res.pop() if ret["utypekind"] == "Typedef": ret = extract_typedef_decl(ast, ret["utype"]) elif ret["utypekind"] == "Elaborated": - ret = extract_enum_decl(ast, ret["utype"]) or extract_struct_decl(ast, ret["utype"]) + enum_decl = extract_enum_decl(ast, ret["utype"]) + struct_decl = extract_struct_decl(ast, ret["utype"]) + if enum_decl and struct_decl: + assert False, "duplicate types" + else: + ret = enum_decl or struct_decl return ret @@ -254,7 +215,7 @@ def find_dup_types(data): def xform(item): return item["type"] - all_types = recursive_query(data, filter, xform) + all_types = utils.recursive_query(data, filter, xform) dups = set() for type in all_types: if all_types.count(type) > 1: @@ -263,47 +224,55 @@ def find_dup_types(data): return (all_types, list(dups)) -def parse(libclang_path, function_names, header_filename, var_names): +# TODO: Check for primitive type +def remove_typedefs_of_primitive(types): + types_struct_enum = [] + for type in types: + if (type["kind"] == "CursorKind.TYPEDEF_DECL"): + pass + else: + types_struct_enum.append(type) + + return types_struct_enum + + +def parse(libclang_path, header_filename, function_names, var_names): astp = ast_parser.ASTParser(libclang_path) # header = create_header("/Users/heck/local-default/include/pEp/pEpEngine.h", out_dir="./") header = create_header(header_filename) header["ast"] = astp.parse(header["path"], follow_includes=True) - # VIEWS - views = {} - views["forward_declarations"] = extract_forward_declarations(header["ast"]) - views["types"] = extract_types(header["ast"]) - views["types_count"] = len(views["types"]) # CDL cid = {"functions": "", "vars": "", - "structs": "", - "enums": ""} + "types": ""} + # stage 1: extract functions and vars - cid["functions"] = extract_functions(header["ast"], function_names) - cid["vars"] = extract_vars(header["ast"], var_names) + (cid["functions"], cid["functions_notfound"]) = extract_functions_named(header["ast"], function_names) + (cid["vars"], cid["vars_notfound"]) = extract_vars_named(header["ast"], var_names) + # stage 2: collect type refs - cid["types_resolved"] = [] - cid["typerefs_unresolved"] = [] + cid["types"] = [] + typerefs_unresolved = [] cid["typerefs_resolved"] = [] cid["typerefs_notfound"] = [] + while True: - cid["typerefs_unresolved"] = collect_typerefs(cid) # only list of typenames - cid["typerefs_unresolved"] = list(set(cid["typerefs_unresolved"]) - (set(cid["typerefs_resolved"]).union(set(cid["typerefs_notfound"])))) - if (len(cid["typerefs_unresolved"]) <= 0): + typerefs_unresolved = collect_all_typerefs(cid) # only list of typenames + typerefs_unresolved = list(set(typerefs_unresolved) - (set(cid["typerefs_resolved"]).union(set(cid["typerefs_notfound"])))) + if (len(typerefs_unresolved) <= 0): break - (types_resolved, typerefs_resolved, notfound) = resolve_typerefs(header["ast"], cid["typerefs_unresolved"]) - cid["types_resolved"] += types_resolved + (types, typerefs_resolved, notfound) = resolve_typerefs(header["ast"], typerefs_unresolved) + cid["types"] += types cid["typerefs_resolved"] += typerefs_resolved cid["typerefs_notfound"] += notfound + + cid["types"] = remove_typedefs_of_primitive(cid["types"]) + cid["type_names"] = extract_type_names(cid["types"]) header["cid"] = cid - header["views"] = views - (views["all_types"], views["dup_types"]) = find_dup_types(header["cid"]) - (views["ast_all_types"], views["ast_dup_types"]) = find_dup_types(header["ast"]) return header - # generates simple-ast for each header specified in spec out dir. # def main_old(): # parser = ASTParser("/opt/local/libexec/llvm-9.0/lib/libclang.dylib") diff --git a/gen/gen_cid/utils.py b/gen/gen_cid/utils.py new file mode 100644 index 0000000..aa93505 --- /dev/null +++ b/gen/gen_cid/utils.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# +import os +import json + +def join_dir_and_filenames(dirname, filenames): + paths = [] + for basename in filenames: + path = dirname + basename + paths.append(path) + return paths + +def read_file(path): + with open(path) as f: + file_content = f.read() + return file_content + +def write_json(content, outpath): + # create path if not existing + out_dir = os.path.dirname(outpath) + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + # write + with open(outpath, "w+") as f: + json.dump(content, f, indent=4) + +# works on valid json like structure lists/dict +def recursive_query(data, filter, transform=lambda x: x): + # all datatypes accepted + # print(type(data)) + # assert (type(data) in [list, dict, int, str, bool]) + resultset = [] + + # decorator just handling exceptions + def filter_decorator(data): + try: + return filter(data) + except KeyError: + pass + + # filter current data + # only dict types are filtered + if type(data) in [dict]: + if filter_decorator(data): + # transform result + xformed = transform(data) + if xformed: + resultset.append(xformed) + + if isinstance(data, dict): + for item in data.values(): + childres = recursive_query(item, filter, transform) + if childres: + resultset += childres + elif isinstance(data, list): + for item in data: + childres = recursive_query(item, filter, transform) + if childres: + resultset += childres + + return resultset + + +def remove_dup_dicts(arr_of_dicts): + arr_no_dups = [dict(i) for i in {tuple(d.items()) for d in arr_of_dicts}] + return arr_no_dups