From aa322f7277910c667a3404099a236b0eeb19f73e Mon Sep 17 00:00:00 2001 From: heck Date: Fri, 8 Jan 2021 22:33:20 +0100 Subject: [PATCH] repo and project structure --- gen/Makefile | 4 +- gen/README.md | 26 +++ gen/{gen_cid/gen_yml.py => bin/pEp_acid_gen} | 23 ++- gen/examples/ext/synth_shed/Makefile | 6 - gen/examples/ext/synth_shed/Makefile.conf | 8 +- gen/examples/ext/synth_shed/gen/Makefile | 4 +- gen/examples/ext/synth_shed/gen/config.json | 2 +- gen/examples/lib/Makefile | 10 -- .../{synth_shed => lib_synth_shed}/Makefile | 5 - .../lib/{synth_shed => lib_synth_shed}/main.c | 0 .../synth_shed.c | 0 .../synth_shed.h | 0 .../lib/{test_cid => lib_test}/Makefile | 4 +- gen/examples/lib/lib_test/README.md | 161 ++++++++++++++++++ .../lib/{test_cid => lib_test}/enums.h | 0 .../lib/{test_cid => lib_test}/functions.h | 0 .../test_lib.c => lib_test/lib_test.c} | 2 +- gen/examples/lib/lib_test/lib_test.h | 10 ++ .../lib/{test_cid => lib_test}/main.c | 4 +- .../lib/{test_cid => lib_test}/structs.h | 0 .../lib/{test_cid => lib_test}/typedefs.h | 2 +- .../lib/{test_cid => lib_test}/vars.h | 0 gen/examples/lib/test_cid/test_lib.h | 110 ------------ gen/gen_cid/__init__.py | 59 ------- gen/pEpACIDgen/__init__.py | 58 +++++++ .../cid_2_yml.py => pEpACIDgen/acid_yml.py} | 20 +-- .../ast_2_cid.py => pEpACIDgen/ast_2_acid.py} | 74 ++++---- gen/{gen_cid => pEpACIDgen}/c_2_ast.py | 0 .../gen_backend}/gen_cc.ysl2 | 0 gen/{gen_cid => pEpACIDgen}/utils.py | 0 gen/setup.cfg | 8 +- gen/tests/{test_cid.py => test_pEpACIDgen.py} | 18 +- 32 files changed, 347 insertions(+), 271 deletions(-) create mode 100644 gen/README.md rename gen/{gen_cid/gen_yml.py => bin/pEp_acid_gen} (68%) delete mode 100644 gen/examples/lib/Makefile rename gen/examples/lib/{synth_shed => lib_synth_shed}/Makefile (96%) rename gen/examples/lib/{synth_shed => lib_synth_shed}/main.c (100%) rename gen/examples/lib/{synth_shed => lib_synth_shed}/synth_shed.c (100%) rename gen/examples/lib/{synth_shed => lib_synth_shed}/synth_shed.h (100%) rename gen/examples/lib/{test_cid => lib_test}/Makefile (87%) create mode 100644 gen/examples/lib/lib_test/README.md rename gen/examples/lib/{test_cid => lib_test}/enums.h (100%) rename gen/examples/lib/{test_cid => lib_test}/functions.h (100%) rename gen/examples/lib/{test_cid/test_lib.c => lib_test/lib_test.c} (82%) create mode 100644 gen/examples/lib/lib_test/lib_test.h rename gen/examples/lib/{test_cid => lib_test}/main.c (63%) rename gen/examples/lib/{test_cid => lib_test}/structs.h (100%) rename gen/examples/lib/{test_cid => lib_test}/typedefs.h (99%) rename gen/examples/lib/{test_cid => lib_test}/vars.h (100%) delete mode 100644 gen/examples/lib/test_cid/test_lib.h delete mode 100644 gen/gen_cid/__init__.py create mode 100644 gen/pEpACIDgen/__init__.py rename gen/{gen_cid/cid_2_yml.py => pEpACIDgen/acid_yml.py} (90%) rename gen/{gen_cid/ast_2_cid.py => pEpACIDgen/ast_2_acid.py} (76%) rename gen/{gen_cid => pEpACIDgen}/c_2_ast.py (100%) rename gen/{gen_cid => pEpACIDgen/gen_backend}/gen_cc.ysl2 (100%) rename gen/{gen_cid => pEpACIDgen}/utils.py (100%) rename gen/tests/{test_cid.py => test_pEpACIDgen.py} (94%) diff --git a/gen/Makefile b/gen/Makefile index 0a20284..bfb3cda 100644 --- a/gen/Makefile +++ b/gen/Makefile @@ -2,7 +2,7 @@ VENV_DIR = ./venv BUILD_DIR = ./build DIST_DIR = ./dist PYTHON_ARTIFACTS += ./.eggs -PYTHON_ARTIFACTS += ./gen_cid.egg-info +PYTHON_ARTIFACTS += ./pEpACIDgen.egg-info PYTHON_ARTIFACTS += ./.pytest_cache @@ -50,7 +50,7 @@ venv: # Tests if the current environment is able to load the pEp module envtest: - python3 -c 'import gen_cid' + python3 -c 'import pEpACIDgen' # Test # ==== diff --git a/gen/README.md b/gen/README.md new file mode 100644 index 0000000..fa27ca3 --- /dev/null +++ b/gen/README.md @@ -0,0 +1,26 @@ +pEpACIDgen +========== + +pEpACIDgen stands for "pEp Abstract C Interface Description Generator" + +pEpACIDgen generates an Abstract C Interface Defintion file (json or yml2) given the inputs: +* headerfile +* list of function names needed +* list of var names needed + +The generator searches each function/var in the headerfile (recursively) and collects all the +types needed (var type, return type, parm types). +As structs can contain further types, these dependent types need to be collected +recursively. +Finally, all the collected types will be resolved to their final underlying type. +If a type is primitive, nothing needs to be done, cause its already defined. +But types of typekind struct or enum need to be defined. +Their definition will be searched for in the headerfile and included in the interface definition +The ACID file (yml2) contains all the information needed to represent: +* functions +* vars +* structs +* enums + +well enough so they can be expressed in pyBind11. +use "pyBind11-CID" to generate the pyBind11 code out of the ACID file. diff --git a/gen/gen_cid/gen_yml.py b/gen/bin/pEp_acid_gen similarity index 68% rename from gen/gen_cid/gen_yml.py rename to gen/bin/pEp_acid_gen index a71c1aa..8a635b3 100755 --- a/gen/gen_cid/gen_yml.py +++ b/gen/bin/pEp_acid_gen @@ -5,22 +5,31 @@ import sys import json -import gen_cid -from gen_cid import utils +import pEpACIDgen +from pEpACIDgen import utils +def usage(): + print("Usage:",) + print("pEp_acid_gen config-file",) + def print_indent(*args, indent=0): print('\t' * indent, *args) def main(): + if len(sys.argv) <= 1: + usage() + exit(-1) + cfg_file = sys.argv[1] + print("reading cfg file:", cfg_file) data = [] try: with open(cfg_file) as json_data_file: data = json.load(json_data_file) except IOError: - print("cfg file not readble:", cfg_file) + print("cfg file not readable:", cfg_file) exit(-1) @@ -48,12 +57,12 @@ def main(): print_indent(func, indent=2) - cidtools = gen_cid.CIDTools(libclang_path, header_filename, pymodule_name) - header = cidtools.extract(vars, funcs, debug_ast=False, debug_cid=False, debug_yml=False) + acidgen = pEpACIDgen.pEpACIDGen(libclang_path, header_filename, pymodule_name) + header = acidgen.extract(vars, funcs, debug_ast=False, debug_acid=False, debug_yml=False) print_indent("OUTPUT:") - print_indent(header["yml"]) - utils.write_string(header["yml"], "./" + "py_module" + ".yml2", ) + print_indent(header["acid_yml"]) + utils.write_string(header["acid_yml"], "./" + "py_module" + ".yml2", ) if __name__ == "__main__": diff --git a/gen/examples/ext/synth_shed/Makefile b/gen/examples/ext/synth_shed/Makefile index 88c143e..7c9f0ed 100644 --- a/gen/examples/ext/synth_shed/Makefile +++ b/gen/examples/ext/synth_shed/Makefile @@ -1,21 +1,15 @@ include Makefile.conf TARGET_MODULE=synth_shed.so -# C++ CXX=clang CXXFLAGS+=-std=c++11 -g -# Sources SRCS+=$(wildcard *.cc) OBJS+=$(SRCS:.cc=.o) -#Compile CXXFLAGS+=$(INCLUDES) -I$(PREFIX)/include - -#Link LDFLAGS+=-undefined dynamic_lookup $(LIBS_PATH) $(LIBS) - $(info -----BUILD INFO----) $(info SRCS $(SRCS)) $(info OBJS $(OBJS)) diff --git a/gen/examples/ext/synth_shed/Makefile.conf b/gen/examples/ext/synth_shed/Makefile.conf index 04a970a..803c484 100644 --- a/gen/examples/ext/synth_shed/Makefile.conf +++ b/gen/examples/ext/synth_shed/Makefile.conf @@ -6,13 +6,13 @@ YML2_PATH=$(HOME)/src/pepbase/default/yml2 YML2_PROC=$(YML2_PATH)/yml2proc $(YML2_OPTS) YML2_OPTS=--encoding=utf8 -# GEN-CID -GEN_CID_DIR=../../../../gen_cid/ +# pEpACIDgen +pEpACIDgen_ROOT=../../../../ # C Python headers INCLUDES+=-I/opt/local/Library/Frameworks/Python.framework/Versions/3.8/include/python3.8 # example lib -INCLUDES+=-I../../lib/synth_shed +INCLUDES+=-I../../lib/lib_synth_shed LIBS+=-lsynth_shed -LIBS_PATH+=-L../../lib/synth_shed/ +LIBS_PATH+=-L../../lib/lib_synth_shed/ diff --git a/gen/examples/ext/synth_shed/gen/Makefile b/gen/examples/ext/synth_shed/gen/Makefile index f0bce30..7d67702 100644 --- a/gen/examples/ext/synth_shed/gen/Makefile +++ b/gen/examples/ext/synth_shed/gen/Makefile @@ -8,10 +8,10 @@ CC_FILE=py_module.cc all: yml cc yml: - $(GEN_CID_DIR)/gen_yml.py $(abspath config.json) + $(pEpACIDgen_ROOT)/bin/pEp_acid_gen.py $(abspath config.json) cc : $(YML2_FILE) - $(YML2_PROC) -y $(GEN_CID_DIR)/gen_cc.ysl2 $(YML2_FILE) + $(YML2_PROC) -y $(pEpACIDgen_ROOT)/pEpACIDgen/gen_backend/gen_cc.ysl2 $(YML2_FILE) clean: rm -f $(YML2_FILE) diff --git a/gen/examples/ext/synth_shed/gen/config.json b/gen/examples/ext/synth_shed/gen/config.json index eb53bb5..022b5d1 100644 --- a/gen/examples/ext/synth_shed/gen/config.json +++ b/gen/examples/ext/synth_shed/gen/config.json @@ -1,6 +1,6 @@ { "module_name": "synth_shed", - "header_filename": "../../../../examples/lib/synth_shed/synth_shed.h", + "header_filename": "../../../../examples/lib/lib_synth_shed/synth_shed.h", "libclang_path": "/opt/local/libexec/llvm-9.0/lib/libclang.dylib", "variables": [ ], diff --git a/gen/examples/lib/Makefile b/gen/examples/lib/Makefile deleted file mode 100644 index 43151d1..0000000 --- a/gen/examples/lib/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -.PHONY: all clean - -all: - $(MAKE) -C synth_shed - $(MAKE) -C test_cid - -clean: - $(MAKE) -C synth_shed clean - $(MAKE) -C test_cid clean - diff --git a/gen/examples/lib/synth_shed/Makefile b/gen/examples/lib/lib_synth_shed/Makefile similarity index 96% rename from gen/examples/lib/synth_shed/Makefile rename to gen/examples/lib/lib_synth_shed/Makefile index dfb0160..420dfc2 100644 --- a/gen/examples/lib/synth_shed/Makefile +++ b/gen/examples/lib/lib_synth_shed/Makefile @@ -2,16 +2,11 @@ TARGET_EXE=synth_shed LIB_STATIC=libsynth_shed.a LIB_DYN=libsynth_shed.so -# C CFLAGS+=-std=c99 -g -fPIC -# Sources SRCS=$(wildcard *.c) - -#Link OBJS=$(SRCS:.c=.o) - $(info -----BUILD INFO----) $(info SRCS $(SRCS)) $(info OBJS $(OBJS)) diff --git a/gen/examples/lib/synth_shed/main.c b/gen/examples/lib/lib_synth_shed/main.c similarity index 100% rename from gen/examples/lib/synth_shed/main.c rename to gen/examples/lib/lib_synth_shed/main.c diff --git a/gen/examples/lib/synth_shed/synth_shed.c b/gen/examples/lib/lib_synth_shed/synth_shed.c similarity index 100% rename from gen/examples/lib/synth_shed/synth_shed.c rename to gen/examples/lib/lib_synth_shed/synth_shed.c diff --git a/gen/examples/lib/synth_shed/synth_shed.h b/gen/examples/lib/lib_synth_shed/synth_shed.h similarity index 100% rename from gen/examples/lib/synth_shed/synth_shed.h rename to gen/examples/lib/lib_synth_shed/synth_shed.h diff --git a/gen/examples/lib/test_cid/Makefile b/gen/examples/lib/lib_test/Makefile similarity index 87% rename from gen/examples/lib/test_cid/Makefile rename to gen/examples/lib/lib_test/Makefile index 827f905..e1d0b14 100644 --- a/gen/examples/lib/test_cid/Makefile +++ b/gen/examples/lib/lib_test/Makefile @@ -1,4 +1,4 @@ -TARGET=test_lib +TARGET=lib_test SRCS+=$(wildcard *.c) CFLAGS+=-std=c99 -g @@ -9,7 +9,7 @@ LDFLAGS+= CXXFLAGS+=$(INCLUDES) -LDFLAGS+=$(LIB_DIRS+) +LDFLAGS+=$(LIB_DIRS) LDFLAGS+=$(LIBS) OBJS+=$(SRCS:.c=.o) diff --git a/gen/examples/lib/lib_test/README.md b/gen/examples/lib/lib_test/README.md new file mode 100644 index 0000000..2a3cef2 --- /dev/null +++ b/gen/examples/lib/lib_test/README.md @@ -0,0 +1,161 @@ +pEpACIDgen test library +----------------------- +This here is the test library for pEpACIDgen. +The C language is broken down into its components and common pattern. +It is a humble attempt to cover all possible combinations that can be created in the +C language. (like, struct containing` primitive, struct containing struct, typedef chains, +etc..). +Please note, that pointer types have not been addressed yet (at all) + + +The intended use of this test lib is: If you encounter a scenario using pEpACIDgen that poses problem +it is most likely that the scenario can be reproduced using this test lib pretty easily. +lets say you pEpACIDgen a function with sig: + +TT_E(*)(ACCS), where: + TT_E = typedef of typedef of an enum without an alias. + ACCS = Alias to a complex struct containing another complex struct + +These types exist already along with a systematic permutation of possible types that amount +to over 100 of types. +just write your func/var using these types and debug the generation. + + +The ACID format deals with items. Everything is an item. +There are qualifiers that can be used and combined to define an item. (list below) +e.g PS = primitive struct that possibly has a typedef + +Of all the possible combinations, not all are valid (C lang). Its a futile attempt +trying to cover all the valid ones. But an effort has been made to cover the most +of the valid combinations. +Combination of structs, enums and typedefs are systematically explored in the files: +* structs.h +* enums.h +* typedefs.h + + +For all the types defined the file: +* vars.h + +contains an instance of each. + +for all the possible function signature, well with a 100 types, +we can create 10000 different sigantures using 1 ret and 1 arg. +Therefore, no functions have been defined. +define them when you are interested in the combination, and pEpACIDgen the defintion. + + +Item Qualifier +=============== +V = Void already defined +P = Primitive Type already defined +S = Struct -> struct.h +E = Enum -> enums.h +A = Alias -> structs.h/enums.h +N = Nested (Structs only, cant be aliased) -> structs.h +H = Hosting (Structs only -> structs.h + containing the nested struct, cant be primitive) +T = typedef -> typedefs.h +I = Incomplete / (forward decl) -> struct.h/enums.h + +func_ = function -> functions.h +var_ = variable -> vars.h + +prefixed underline (e.g _E) means no typedef/alias exists for this item (enum in this case) + +Alias vs. Typedef +----------------- +aliased means typedef'd directly, like: + +typedef struct _X { + int x; +} X; + +(structs and enums only) + + +Typedef means, not typedef'd directly, but anywhere, like + +typedef _X X; + + + + + + +Structs +======= +For structs, a combination is needed to define the type of struct +IS = Struct (Empty or incomplete, neither primitive nor complex) +PS = Primitive struct (struct containing only primitive types) +CS = Complex struct (struct containing other structs and primitive) +NPS = Nested primitive struct + +Covered combinations +==================== +_IS = incomplete struct without a typedef (forward decl) +IS = incomplete struct +AIS = alias of an incomplete struct +(aliased) struct [ primitive | complex ] +---------------------------------------- +_PS = primitive struct without a an alias +PS = primitive struct +APS = Alias of primitive struct +_CS = complex struct without a an alias +CS = complex struct +ACS = alias of a complex struct +_CCS = complex complex struct without an alias +CCS = complex complex struct +ACCS = alias of a complex complex struct + +Nested structs +-------------- +there is always a hosting struct and a nesting struct +in case of double nesting it can be both + +Simply nested struct [ primitive | complex ] +-------------------------------------------- +(aliased) hosting struct (complex always) +_HS = hosting struct without an alias +HS = hosting struct +AHS = alias of a hosting struct +_NPS = nested primitive struct without an alias +_NCS = nested complex struct without an alias + +Doubly nested struct (complex always) +------------------------------------- +_HHS = hosting hosting struct without an alias +HHS = hosting hosting struct +AHHS = alias of a hosting hosting struct +_NHS = nested hosting struct +_NNPS = nested nested struct +_NNCS = nested nested complex struct + +enums in structs +_NEHS = nested enum in hosting struct +_NENHS = nested enum in nested hosting struct + + + +Enums +===== +Enums only have the item qualifier I and A, +so their combinations are simple. + +Covered combinations +==================== +_IE = incomplete enum without an alias +IE = incomplete enum +AIE = alias of an incomplete enum +_E = enum without an alias +E = enum +AE = alias of an enum + + +Typedefs +======== +Typedefs are simply applied to all defined enums and structs, and the +their respective "alias". +Then a typedef os applied to every typedef (TT*) + + diff --git a/gen/examples/lib/test_cid/enums.h b/gen/examples/lib/lib_test/enums.h similarity index 100% rename from gen/examples/lib/test_cid/enums.h rename to gen/examples/lib/lib_test/enums.h diff --git a/gen/examples/lib/test_cid/functions.h b/gen/examples/lib/lib_test/functions.h similarity index 100% rename from gen/examples/lib/test_cid/functions.h rename to gen/examples/lib/lib_test/functions.h diff --git a/gen/examples/lib/test_cid/test_lib.c b/gen/examples/lib/lib_test/lib_test.c similarity index 82% rename from gen/examples/lib/test_cid/test_lib.c rename to gen/examples/lib/lib_test/lib_test.c index 7458858..3b5b1d6 100644 --- a/gen/examples/lib/test_cid/test_lib.c +++ b/gen/examples/lib/lib_test/lib_test.c @@ -1,4 +1,4 @@ -#include "test_lib.h" +#include "lib_test.h" struct _PS func__PS_V() { return var__PS; diff --git a/gen/examples/lib/lib_test/lib_test.h b/gen/examples/lib/lib_test/lib_test.h new file mode 100644 index 0000000..5939a9e --- /dev/null +++ b/gen/examples/lib/lib_test/lib_test.h @@ -0,0 +1,10 @@ +#ifndef MAIN_INCLUDE_H +#define MAIN_INCLUDE_H + +#include "typedefs.h" +#include "enums.h" +#include "structs.h" +#include "functions.h" +#include "vars.h" + +#endif //MAIN_INCLUDE_H diff --git a/gen/examples/lib/test_cid/main.c b/gen/examples/lib/lib_test/main.c similarity index 63% rename from gen/examples/lib/test_cid/main.c rename to gen/examples/lib/lib_test/main.c index 4a03cb8..ef2b4c5 100644 --- a/gen/examples/lib/test_cid/main.c +++ b/gen/examples/lib/lib_test/main.c @@ -1,9 +1,9 @@ -#include "test_lib.h" +#include "lib_test.h" #include int main() { - printf("test_lib starting...\n"); + printf("lib_test starting...\n"); TT_HHS a; diff --git a/gen/examples/lib/test_cid/structs.h b/gen/examples/lib/lib_test/structs.h similarity index 100% rename from gen/examples/lib/test_cid/structs.h rename to gen/examples/lib/lib_test/structs.h diff --git a/gen/examples/lib/test_cid/typedefs.h b/gen/examples/lib/lib_test/typedefs.h similarity index 99% rename from gen/examples/lib/test_cid/typedefs.h rename to gen/examples/lib/lib_test/typedefs.h index 68f4be8..6d35e08 100644 --- a/gen/examples/lib/test_cid/typedefs.h +++ b/gen/examples/lib/lib_test/typedefs.h @@ -7,7 +7,7 @@ // Typedefs // ======== // Typedefs are simply applied to all defined enums and structs, and the -// their respecitve "alias". +// their respective "alias". // Then a typedef os applied to every typedef (TT*) typedef void T_V; diff --git a/gen/examples/lib/test_cid/vars.h b/gen/examples/lib/lib_test/vars.h similarity index 100% rename from gen/examples/lib/test_cid/vars.h rename to gen/examples/lib/lib_test/vars.h diff --git a/gen/examples/lib/test_cid/test_lib.h b/gen/examples/lib/test_cid/test_lib.h deleted file mode 100644 index 87fded2..0000000 --- a/gen/examples/lib/test_cid/test_lib.h +++ /dev/null @@ -1,110 +0,0 @@ -#ifndef MAIN_INCLUDE_H -#define MAIN_INCLUDE_H - -#include "typedefs.h" -#include "enums.h" -#include "structs.h" -#include "functions.h" -#include "vars.h" - -// Gen-CID -// ======= -// Gen-CID generates a C Interface Defintion file (json) given the inputs: -// * headerfile -// * list of function names needed -// * list of var names needed -// The generator searches each function/var in the headerfile (recursively) and collects all the -// types needed (var type, return type, parm types). -// As structs can contain further types, these dependent types need to be collected -// recursively. -// Finally, all the collected types will be resolved to their final underlying type. -// If a type is primitive, nothing needs to be done, cause its already defined. -// But types of typekind struct or enum need to be defined. -// Their definition will be searched for in the headerfile and included in the interface definition -// The CID file contains all the information needed to represent: -// * functions -// * vars -// * structs -// * enums -// Well enough so they can be expressed in pyBind11 -// use "pyBind11-CID" to generate the pyBind11 code out of the CID file. -// -// Gen-CID test library -// -------------------- -// This here is the test main test library for Gen-CID. -// The C language is broken down into its components and common pattern. -// It is a humble attempt to cover all possible combinations that can be created in the -// C language. (like, struct containing` primitive, struct containing struct, typedef chains, -// etc..). -// Please note, that pointer types have not been addressed yet (at all) -// -// The intended use of this test lib is: If you encounter a scenario using Gen-CID that poses problem -// it is most likely that the scenario can be reproduced using this test lib pretty easily. -// lets say you Gen-CID a function with sig: -// TT_E(*)(ACCS), where: -// TT_E = typedef of typedef of an enum without an alias. -// ACCS = Alias to a complex struct containing another complex struct -// these types exist already along with a systematic permutation of possible types that amount -// to over 100 of types. -// just write your func/var using these types and debug the generation. - - -// The CID format deals with items. Everything is an item. -// There are qualifiers that can be used and combined to define an item. (list below) -// e.g PS = primitive struct that possibly has a typedef -// -// Of all the possible combinations, not all are valid (C lang). Its a futile attempt -// trying to cover all the valid ones. But an effort has been made to cover the most -// of the valid combinations. -// Combination of structs, enums and typedefs are systematically explored in the files: -// * structs.h -// * enums.h -// * typedefs.h -// -// For all the types defined the file: -// * vars.h -// contains an instance of each. -// -// for all the possible function signature, well with a 100 types, -// we can create 10000 different sigantures using 1 ret and 1 arg. -// Therefore, no functions have been defined. -// define them when you are interested in the combination, and Gen-CID the defintion. - - -// Item Qualitfier -// =============== -// V = Void already defined -// P = Primitive Type already defined -// S = Struct -> struct.h -// E = Enum -> enums.h -// A = Alias -> structs.h/enums.h -// N = Nested (Structs only, cant be aliased) -> structs.h -// H = Hosting (Structs only -> structs.h -// containing the nested struct, cant be primitive) -// T = typedef -> typedefs.h -// I = Incomplete / (forward decl) -> struct.h/enums.h -// -// func_ = function -> functions.h -// var_ = variable -> vars.h -// -// prefixed underline (e.g _E) means no typedef/alias exists for this item (enum in this case) -// -// Alias vs. Typedef -// ----------------- -// aliased means direct typedef directly, like -// structs and enums only -/* - * typedef struct _X { - int x; - * } X; - * - */ - -// Typedef means, not typdeffed directly, but anywhere, like -/* - * typedef _X X; - * - */ - - -#endif //MAIN_INCLUDE_H diff --git a/gen/gen_cid/__init__.py b/gen/gen_cid/__init__.py deleted file mode 100644 index 3ca27e7..0000000 --- a/gen/gen_cid/__init__.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is under GNU Affero General Public License 3.0 -# see LICENSE.txt - -from . import c_2_ast -from . import ast_2_cid -from . import cid_2_yml -from . import utils - -import os - - -class CIDTools: - - def __init__(self, libclang_path, header_filename, pymodule_name, out_dir=None): - self.header = self._create_header(header_filename, out_dir) - self.c2ast = c_2_ast.C2AST(libclang_path) - self.pymodule_name = pymodule_name - - # out-dir is in-dir if not specified - def _create_header(self, path, out_dir=None): - header = {"path": "", - "dir": "", - "filename": "", - "out_dir": "", - "sourcecode": "", - "ast": "", - "cid": "", - "yml": ""} - - header["path"] = path - header["dir"] = os.path.dirname(path) - header["filename"] = os.path.basename(path) - - header["out_dir"] = header["dir"] - if out_dir: - header["out_dir"] = out_dir - - header["sourcecode"] = utils.read_file(path) - return header - - - def extract(self, var_names, func_names, debug_ast=False, debug_cid=False, debug_yml=False): - # ast - self.header["ast"] = self.c2ast.parse(self.header["path"], follow_includes=True) - if debug_ast: - utils.write_json(self.header["ast"], self.header["out_dir"] + "/" + self.header["filename"] + ".ast.json") - - # cid - self.header["cid"] = ast_2_cid.extract_cid_selectively(self.header["ast"], var_names, func_names) - if debug_cid: - utils.write_json(self.header["cid"], self.header["out_dir"] + "/" + self.header["filename"] + ".cid.json") - - # yml - self.header["yml"] = cid_2_yml.generate_yml(self.header["cid"], self.pymodule_name) - if debug_yml: - utils.write_string(self.header["yml"], self.header["out_dir"] + "/" + self.header["filename"] + ".cid.yml") - - return self.header diff --git a/gen/pEpACIDgen/__init__.py b/gen/pEpACIDgen/__init__.py new file mode 100644 index 0000000..fd3125d --- /dev/null +++ b/gen/pEpACIDgen/__init__.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# This file is under GNU Affero General Public License 3.0 +# see LICENSE.txt + +from . import c_2_ast +from . import ast_2_acid +from . import acid_yml +from . import utils + +import os + + +class pEpACIDGen: + + def __init__(self, libclang_path, header_filename, pymodule_name, out_dir=None): + self.model = self._acid_model_create(header_filename, out_dir) + self.c2ast = c_2_ast.C2AST(libclang_path) + self.pymodule_name = pymodule_name + + # out-dir is in-dir if not specified + def _acid_model_create(self, path, out_dir=None): + model = {"path": "", + "dir": "", + "filename": "", + "out_dir": "", + "sourcecode": "", + "ast": "", + "acid": "", + "acid_yml": ""} + + model["path"] = path + model["dir"] = os.path.dirname(path) + model["filename"] = os.path.basename(path) + + model["out_dir"] = model["dir"] + if out_dir: + model["out_dir"] = out_dir + + model["sourcecode"] = utils.read_file(path) + return model + + def extract(self, var_names, func_names, debug_ast=False, debug_acid=False, debug_yml=False): + # ast + self.model["ast"] = self.c2ast.parse(self.model["path"], follow_includes=True) + if debug_ast: + utils.write_json(self.model["ast"], self.model["out_dir"] + "/" + self.model["filename"] + ".ast.json") + + # acid + self.model["acid"] = ast_2_acid.extract_acid_selectively(self.model["ast"], var_names, func_names) + if debug_acid: + utils.write_json(self.model["acid"], self.model["out_dir"] + "/" + self.model["filename"] + ".acid.json") + + # yml + self.model["acid_yml"] = acid_yml.generate_acid_yml(self.model["acid"], self.pymodule_name) + if debug_yml: + utils.write_string(self.model["acid_yml"], self.model["out_dir"] + "/" + self.model["filename"] + ".acid.yml") + + return self.model diff --git a/gen/gen_cid/cid_2_yml.py b/gen/pEpACIDgen/acid_yml.py similarity index 90% rename from gen/gen_cid/cid_2_yml.py rename to gen/pEpACIDgen/acid_yml.py index 3d92bd5..3c135b5 100644 --- a/gen/gen_cid/cid_2_yml.py +++ b/gen/pEpACIDgen/acid_yml.py @@ -5,13 +5,13 @@ from . import utils -def generate_yml(cid, module_name): +def generate_acid_yml(acid, module_name): ret = "" ret += _grammar_header() ret += "module " + module_name + " {\n" - ret += _generate_functions(cid) - ret += _generate_structs(cid) - ret += _generate_enums(cid) + ret += _generate_functions(acid) + ret += _generate_structs(acid) + ret += _generate_enums(acid) ret += "}" return ret @@ -31,8 +31,8 @@ decl item @name; return grammar -def _generate_functions(cid): - data_root = cid["functions"] +def _generate_functions(acid): + data_root = acid["functions"] # Main tmpl_main = "" @@ -65,8 +65,8 @@ def _generate_functions(cid): return ret -def _generate_structs(cid): - data_root = cid["types"] +def _generate_structs(acid): + data_root = acid["types"] # Main tmpl_main = "" @@ -96,8 +96,8 @@ def _generate_structs(cid): return ret -def _generate_enums(cid): - data_root = cid["types"] +def _generate_enums(acid): + data_root = acid["types"] # Main tmpl_main = "" diff --git a/gen/gen_cid/ast_2_cid.py b/gen/pEpACIDgen/ast_2_acid.py similarity index 76% rename from gen/gen_cid/ast_2_cid.py rename to gen/pEpACIDgen/ast_2_acid.py index fb0383f..645e7ab 100755 --- a/gen/gen_cid/ast_2_cid.py +++ b/gen/pEpACIDgen/ast_2_acid.py @@ -5,7 +5,7 @@ from . import utils -def extract_forward_declarations(data): +def _extract_forward_declarations(data): def filter(item): if (item["is_definition"] == False and item["kind"] == "CursorKind.STRUCT_DECL" @@ -18,7 +18,7 @@ def extract_forward_declarations(data): return utils.recursive_query(data, filter, xform) -def extract_vars_named(data, var_names): +def _extract_vars_named(data, var_names): def filter(item): if (item["kind"] == "CursorKind.VAR_DECL" and item["name"] in var_names @@ -42,7 +42,7 @@ def extract_vars_named(data, var_names): return (found, notfound_names) -def extract_functions_named(data, function_names): +def _extract_functions_named(data, function_names): def filter(item): if (item["kind"] == "CursorKind.FUNCTION_DECL" and item["name"] in function_names @@ -67,7 +67,7 @@ def extract_functions_named(data, function_names): # just the typenames of all typerefs no dups -def collect_all_typerefs(data): +def _collect_all_typerefs(data): def filter(item): if (item["kind"] == "CursorKind.TYPE_REF" ): @@ -81,7 +81,7 @@ def collect_all_typerefs(data): return no_dups -def extract_type_names(data): +def _extract_type_names(data): def filter(item): if (item["kind"] == "CursorKind.TYPEDEF_DECL" or item["kind"] == "CursorKind.ENUM_DECL" @@ -96,13 +96,13 @@ def extract_type_names(data): return utils.recursive_query(data, filter, xform) -def resolve_typerefs(ast, typeref_names): +def _resolve_typerefs(ast, typeref_names): types = [] typeref_resolved = [] typeref_notfound = [] for typeref_name in typeref_names: - res = resolve_typeref(ast, typeref_name) + res = _resolve_typeref(ast, typeref_name) if not res: typeref_notfound.append(typeref_name) else: @@ -112,19 +112,19 @@ def resolve_typerefs(ast, typeref_names): return (types, typeref_resolved, typeref_notfound) -def resolve_typeref(ast, typeref_name): +def _resolve_typeref(ast, typeref_name): ret = None - ret = extract_enum_decl(ast, typeref_name) + ret = _extract_enum_decl(ast, typeref_name) if not ret: - ret = extract_struct_decl(ast, typeref_name) + ret = _extract_struct_decl(ast, typeref_name) if not ret: - ret = extract_typedef_decl(ast, typeref_name) + ret = _extract_typedef_decl(ast, typeref_name) return ret -def extract_enum_decl(ast, name): +def _extract_enum_decl(ast, name): ret = None def filter(data): @@ -143,7 +143,7 @@ def extract_enum_decl(ast, name): return ret -def extract_struct_decl(ast, name): +def _extract_struct_decl(ast, name): ret = None def filter(data): @@ -162,7 +162,7 @@ def extract_struct_decl(ast, name): return ret -def extract_typedef_decl(ast, name): +def _extract_typedef_decl(ast, name): ret = None def filter(data): @@ -176,10 +176,10 @@ def extract_typedef_decl(ast, name): ret = res.pop() if ret["utypekind"] == "Typedef": - ret = extract_typedef_decl(ast, ret["utype"]) + ret = _extract_typedef_decl(ast, ret["utype"]) elif ret["utypekind"] == "Elaborated": - enum_decl = extract_enum_decl(ast, ret["utype"]) - struct_decl = extract_struct_decl(ast, ret["utype"]) + enum_decl = _extract_enum_decl(ast, ret["utype"]) + struct_decl = _extract_struct_decl(ast, ret["utype"]) if enum_decl and struct_decl: assert False, "duplicate types" else: @@ -188,7 +188,7 @@ def extract_typedef_decl(ast, name): return ret # TODO: move to tests -def find_dup_types(data): +def _find_dup_types(data): def filter(item): if (item["kind"] == "CursorKind.STRUCT_DECL" or item["kind"] == "CursorKind.ENUM_DECL"): @@ -207,7 +207,7 @@ def find_dup_types(data): # TODO: Check for primitive type -def remove_typedefs_of_primitive(types): +def _remove_typedefs_of_primitive(types): types_struct_enum = [] for type in types: if (type["kind"] == "CursorKind.TYPEDEF_DECL"): @@ -218,41 +218,41 @@ def remove_typedefs_of_primitive(types): return types_struct_enum -def extract_cid_selectively(ast, var_names, function_names): - # CDL - cid = {"functions": "", +def extract_acid_selectively(ast, var_names, function_names): + # ACID + acid = {"functions": "", "vars": "", "types": ""} # stage 1: extract functions and vars - (cid["functions"], cid["functions_notfound"]) = extract_functions_named(ast, function_names) - (cid["vars"], cid["vars_notfound"]) = extract_vars_named(ast, var_names) + (acid["functions"], acid["functions_notfound"]) = _extract_functions_named(ast, function_names) + (acid["vars"], acid["vars_notfound"]) = _extract_vars_named(ast, var_names) # stage 2: collect type refs - cid["types"] = [] + acid["types"] = [] typerefs_unresolved = [] - cid["typerefs_resolved"] = [] - cid["typerefs_notfound"] = [] + acid["typerefs_resolved"] = [] + acid["typerefs_notfound"] = [] while True: - typerefs_unresolved = collect_all_typerefs(cid) # only list of typenames - typerefs_unresolved = list(set(typerefs_unresolved) - (set(cid["typerefs_resolved"]).union(set(cid["typerefs_notfound"])))) + typerefs_unresolved = _collect_all_typerefs(acid) # only list of typenames + typerefs_unresolved = list(set(typerefs_unresolved) - (set(acid["typerefs_resolved"]).union(set(acid["typerefs_notfound"])))) if (len(typerefs_unresolved) <= 0): break - (types, typerefs_resolved, notfound) = resolve_typerefs(ast, typerefs_unresolved) - cid["types"] += types - cid["typerefs_resolved"] += typerefs_resolved - cid["typerefs_notfound"] += notfound + (types, typerefs_resolved, notfound) = _resolve_typerefs(ast, typerefs_unresolved) + acid["types"] += types + acid["typerefs_resolved"] += typerefs_resolved + acid["typerefs_notfound"] += notfound - cid["types"] = remove_typedefs_of_primitive(cid["types"]) - cid["type_names"] = extract_type_names(cid["types"]) + acid["types"] = _remove_typedefs_of_primitive(acid["types"]) + acid["type_names"] = _extract_type_names(acid["types"]) - return cid + return acid -def extract_cid_all(ast, function_names, var_names): +def extract_acid_all(ast, function_names, var_names): pass diff --git a/gen/gen_cid/c_2_ast.py b/gen/pEpACIDgen/c_2_ast.py similarity index 100% rename from gen/gen_cid/c_2_ast.py rename to gen/pEpACIDgen/c_2_ast.py diff --git a/gen/gen_cid/gen_cc.ysl2 b/gen/pEpACIDgen/gen_backend/gen_cc.ysl2 similarity index 100% rename from gen/gen_cid/gen_cc.ysl2 rename to gen/pEpACIDgen/gen_backend/gen_cc.ysl2 diff --git a/gen/gen_cid/utils.py b/gen/pEpACIDgen/utils.py similarity index 100% rename from gen/gen_cid/utils.py rename to gen/pEpACIDgen/utils.py diff --git a/gen/setup.cfg b/gen/setup.cfg index 5e6416c..5d0b35e 100644 --- a/gen/setup.cfg +++ b/gen/setup.cfg @@ -1,12 +1,12 @@ [metadata] -name = gen_cid +name = pEpACIDgen version = 1.0 url = author = Heck author_email = heck@pep.foundation -description = C Interface definition generator +description = pEp Abstract C Interface Description Generator long_description = file: README.rst -keywords = pyBind11, C, C99, code generator +keywords = pyBind11, c, c99, code generator, bindings generator, extension, ffi, cffi, ctypes, cpython, yml2, libclang, license = GNU Affero General Public License license_files = LICENSE.txt classifiers = @@ -22,6 +22,8 @@ classifiers = [options] +scripts = bin/pEp_acid_gen + packages = find: python_requires = >= 3.6 diff --git a/gen/tests/test_cid.py b/gen/tests/test_pEpACIDgen.py similarity index 94% rename from gen/tests/test_cid.py rename to gen/tests/test_pEpACIDgen.py index d4a640e..8c94ad0 100755 --- a/gen/tests/test_cid.py +++ b/gen/tests/test_pEpACIDgen.py @@ -4,8 +4,8 @@ # see LICENSE.txt import pytest -from gen_cid import utils -import gen_cid + +import pEpACIDgen types_with_alias = [] types_no_alias = [] @@ -155,18 +155,18 @@ types_no_alias.append({"var": ["_NEHS"], ]}) -def resolve_vars_and_funcs_to_cid(vars, funcs): - header_filename = "examples/lib/test_cid/test_lib.h" +def resolve_vars_and_funcs_to_acid(vars, funcs): + header_filename = "examples/lib/lib_test/lib_test.h" libclang_path = "/opt/local/libexec/llvm-9.0/lib/libclang.dylib" - cidtools = gen_cid.CIDTools(libclang_path, header_filename, "test_cid") - header = cidtools.extract(vars,funcs) + acidgen = pEpACIDgen.pEpACIDGen(libclang_path, header_filename, "test_pEpACIDgen") + header = acidgen.extract(vars,funcs) - return header["cid"] + return header["acid"] def check_expected_types_resolved(vars, funcs, type_names_expected): print("RESOLVING: ", vars, funcs) - cid = resolve_vars_and_funcs_to_cid(vars, funcs) + cid = resolve_vars_and_funcs_to_acid(vars, funcs) # Check on unresolved vars assert len(cid["vars_notfound"]) == 0, "vars not found" @@ -195,7 +195,7 @@ def check_expected_types_resolved(vars, funcs, type_names_expected): @pytest.mark.parametrize('vars', [["var_TP"]]) @pytest.mark.parametrize('funcs', [""]) def test_manual(vars, funcs): - resolve_vars_and_funcs_to_cid(vars, funcs) + resolve_vars_and_funcs_to_acid(vars, funcs) # @pytest.mark.skip