pEp.foundation
/
pEpACIDgen
2
0
Fork 0
Code Pull Requests Releases Activity
Browse Source

move common stuff to utils, remove views, remove typedefs of primitive from cid["types"]

master
heck 5 years ago
parent
e96be841e3
commit
a617466578
2 changed files with 149 additions and 114 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 197
      gen/gen_cid/generate_cid.py
  2. 66
      gen/gen_cid/utils.py

197
gen/gen_cid/generate_cid.py
View File

@ -1,22 +1,8 @@
# -*- coding: utf-8 -*-
import os
import json
from . import ast_parser
def join_dir_and_filenames(dirname, filenames):
paths = []
for basename in filenames:
path = dirname + basename
paths.append(path)
return paths
def read_file(path):
with open(path) as f:
file_content = f.read()
return file_content
from . import utils
# out-dir is in-dir if not specified
@ -35,57 +21,10 @@ def create_header(path, out_dir=None):
if out_dir:
header["out_dir"] = out_dir
header["sourcecode"] = read_file(path)
header["sourcecode"] = utils.read_file(path)
return header
def write_json(content, outpath):
# create path if not existing
out_dir = os.path.dirname(outpath)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
# write
with open(outpath, "w+") as f:
json.dump(content, f, indent=4)
# works on valid json like structure lists/dict
def recursive_query(data, filter, transform=lambda x: x):
# all datatypes accepted
# print(type(data))
# assert (type(data) in [list, dict, int, str, bool])
resultset = []
# decorator just handling exceptions
def filter_decorator(data):
try:
return filter(data)
except KeyError:
pass
# filter current data
# only dict types are filtered
if type(data) in [dict]:
if filter_decorator(data):
# transform result
xformed = transform(data)
if xformed:
resultset.append(xformed)
if isinstance(data, dict):
for item in data.values():
childres = recursive_query(item, filter, transform)
if childres:
resultset += childres
elif isinstance(data, list):
for item in data:
childres = recursive_query(item, filter, transform)
if childres:
resultset += childres
return resultset
def extract_forward_declarations(data):
def filter(item):
if (item["is_definition"] == False
@ -96,51 +35,73 @@ def extract_forward_declarations(data):
def xform(item):
return item
return recursive_query(data, filter, xform)
return utils.recursive_query(data, filter, xform)
def extract_functions(data, function_names):
def extract_vars_named(data, var_names):
def filter(item):
if (item["kind"] == "CursorKind.FUNCTION_DECL"
and item["name"] in function_names
if (item["kind"] == "CursorKind.VAR_DECL"
and item["name"] in var_names
):
return True
def xform(item):
return item
return recursive_query(data, filter, xform)
found = utils.recursive_query(data, filter, xform)
found_names = []
for i in found:
found_names.append(i["name"])
# just the typenames of all typerefs no dups
def collect_typerefs(data):
notfound_names = []
for name in var_names:
if not name in found_names:
notfound_names.append(name)
return (found, notfound_names)
def extract_functions_named(data, function_names):
def filter(item):
if (item["kind"] == "CursorKind.TYPE_REF"
if (item["kind"] == "CursorKind.FUNCTION_DECL"
and item["name"] in function_names
):
return True
def xform(item):
return item["type"]
return item
results = recursive_query(data, filter, xform)
no_dups = list(set(results))
return no_dups
found = utils.recursive_query(data, filter, xform)
found_names = []
for i in found:
found_names.append(i["name"])
def extract_vars(data, var_names):
notfound_names = []
for name in function_names:
if not name in found_names:
notfound_names.append(name)
return (found, notfound_names)
# just the typenames of all typerefs no dups
def collect_all_typerefs(data):
def filter(item):
if (item["kind"] == "CursorKind.VAR_DECL"
and item["name"] in var_names
if (item["kind"] == "CursorKind.TYPE_REF"
):
return True
def xform(item):
return item
return item["type"]
return recursive_query(data, filter, xform)
results = utils.recursive_query(data, filter, xform)
no_dups = list(set(results))
return no_dups
def extract_types(data):
def extract_type_names(data):
def filter(item):
if (item["kind"] == "CursorKind.TYPEDEF_DECL"
or item["kind"] == "CursorKind.ENUM_DECL"
@ -149,15 +110,10 @@ def extract_types(data):
return True
def xform(item):
ret = item["name"]
ret = item["type"]
return ret
return recursive_query(data, filter, xform)
def remove_dup_dicts(arr_of_dicts):
arr_no_dups = [dict(i) for i in {tuple(d.items()) for d in arr_of_dicts}]
return arr_no_dups
return utils.recursive_query(data, filter, xform)
def resolve_typerefs(ast, typeref_names):
@ -197,7 +153,7 @@ def extract_enum_decl(ast, name):
):
return True
res = recursive_query(ast, filter)
res = utils.recursive_query(ast, filter)
if len(res) > 1:
assert False, "duplicate definition"
if len(res) == 1:
@ -215,7 +171,7 @@ def extract_struct_decl(ast, name):
):
return True
res = recursive_query(ast, filter)
res = utils.recursive_query(ast, filter)
if len(res) > 1:
assert False, "duplicate definition"
if len(res) == 1:
@ -233,14 +189,19 @@ def extract_typedef_decl(ast, name):
):
return True
res = recursive_query(ast, filter)
res = utils.recursive_query(ast, filter)
if res:
ret = res.pop()
if ret["utypekind"] == "Typedef":
ret = extract_typedef_decl(ast, ret["utype"])
elif ret["utypekind"] == "Elaborated":
ret = extract_enum_decl(ast, ret["utype"]) or extract_struct_decl(ast, ret["utype"])
enum_decl = extract_enum_decl(ast, ret["utype"])
struct_decl = extract_struct_decl(ast, ret["utype"])
if enum_decl and struct_decl:
assert False, "duplicate types"
else:
ret = enum_decl or struct_decl
return ret
@ -254,7 +215,7 @@ def find_dup_types(data):
def xform(item):
return item["type"]
all_types = recursive_query(data, filter, xform)
all_types = utils.recursive_query(data, filter, xform)
dups = set()
for type in all_types:
if all_types.count(type) > 1:
@ -263,47 +224,55 @@ def find_dup_types(data):
return (all_types, list(dups))
def parse(libclang_path, function_names, header_filename, var_names):
# TODO: Check for primitive type
def remove_typedefs_of_primitive(types):
types_struct_enum = []
for type in types:
if (type["kind"] == "CursorKind.TYPEDEF_DECL"):
pass
else:
types_struct_enum.append(type)
return types_struct_enum
def parse(libclang_path, header_filename, function_names, var_names):
astp = ast_parser.ASTParser(libclang_path)
# header = create_header("/Users/heck/local-default/include/pEp/pEpEngine.h", out_dir="./")
header = create_header(header_filename)
header["ast"] = astp.parse(header["path"], follow_includes=True)
# VIEWS
views = {}
views["forward_declarations"] = extract_forward_declarations(header["ast"])
views["types"] = extract_types(header["ast"])
views["types_count"] = len(views["types"])
# CDL
cid = {"functions": "",
"vars": "",
"structs": "",
"enums": ""}
"types": ""}
# stage 1: extract functions and vars
cid["functions"] = extract_functions(header["ast"], function_names)
cid["vars"] = extract_vars(header["ast"], var_names)
(cid["functions"], cid["functions_notfound"]) = extract_functions_named(header["ast"], function_names)
(cid["vars"], cid["vars_notfound"]) = extract_vars_named(header["ast"], var_names)
# stage 2: collect type refs
cid["types_resolved"] = []
cid["typerefs_unresolved"] = []
cid["types"] = []
typerefs_unresolved = []
cid["typerefs_resolved"] = []
cid["typerefs_notfound"] = []
while True:
cid["typerefs_unresolved"] = collect_typerefs(cid) # only list of typenames
cid["typerefs_unresolved"] = list(set(cid["typerefs_unresolved"]) - (set(cid["typerefs_resolved"]).union(set(cid["typerefs_notfound"]))))
if (len(cid["typerefs_unresolved"]) <= 0):
typerefs_unresolved = collect_all_typerefs(cid) # only list of typenames
typerefs_unresolved = list(set(typerefs_unresolved) - (set(cid["typerefs_resolved"]).union(set(cid["typerefs_notfound"]))))
if (len(typerefs_unresolved) <= 0):
break
(types_resolved, typerefs_resolved, notfound) = resolve_typerefs(header["ast"], cid["typerefs_unresolved"])
cid["types_resolved"] += types_resolved
(types, typerefs_resolved, notfound) = resolve_typerefs(header["ast"], typerefs_unresolved)
cid["types"] += types
cid["typerefs_resolved"] += typerefs_resolved
cid["typerefs_notfound"] += notfound
cid["types"] = remove_typedefs_of_primitive(cid["types"])
cid["type_names"] = extract_type_names(cid["types"])
header["cid"] = cid
header["views"] = views
(views["all_types"], views["dup_types"]) = find_dup_types(header["cid"])
(views["ast_all_types"], views["ast_dup_types"]) = find_dup_types(header["ast"])
return header
# generates simple-ast for each header specified in spec out dir.
# def main_old():
# parser = ASTParser("/opt/local/libexec/llvm-9.0/lib/libclang.dylib")

66
gen/gen_cid/utils.py
View File

@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
#
import os
import json
def join_dir_and_filenames(dirname, filenames):
paths = []
for basename in filenames:
path = dirname + basename
paths.append(path)
return paths
def read_file(path):
with open(path) as f:
file_content = f.read()
return file_content
def write_json(content, outpath):
# create path if not existing
out_dir = os.path.dirname(outpath)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
# write
with open(outpath, "w+") as f:
json.dump(content, f, indent=4)
# works on valid json like structure lists/dict
def recursive_query(data, filter, transform=lambda x: x):
# all datatypes accepted
# print(type(data))
# assert (type(data) in [list, dict, int, str, bool])
resultset = []
# decorator just handling exceptions
def filter_decorator(data):
try:
return filter(data)
except KeyError:
pass
# filter current data
# only dict types are filtered
if type(data) in [dict]:
if filter_decorator(data):
# transform result
xformed = transform(data)
if xformed:
resultset.append(xformed)
if isinstance(data, dict):
for item in data.values():
childres = recursive_query(item, filter, transform)
if childres:
resultset += childres
elif isinstance(data, list):
for item in data:
childres = recursive_query(item, filter, transform)
if childres:
resultset += childres
return resultset
def remove_dup_dicts(arr_of_dicts):
arr_no_dups = [dict(i) for i in {tuple(d.items()) for d in arr_of_dicts}]
return arr_no_dups
Write Preview
Loading…
Cancel
Save